mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-06-14 10:29:52 +08:00
🐛 fix(ci/driver): 修复驱动发布错配与 dev 驱动下载命中旧资产
- 发布链路新增 driver release 资产自检,校验已发布二进制与 manifest revision/sha256 一致 - dev-build 与 release 工作流在复用已发布驱动前先校验实际资产,发现旧二进制混入时强制全量重建 - driver manifest 新增 sha256 字段,避免仅凭 source commit 与 manifest 一致就误判发布有效 - 驱动变更检测与全量重建判定纳入 release asset 校验脚本变更,确保链路修复提交会触发自愈重建 - 驱动下载优先使用 GitHub release asset API 地址,并对资产接口按 octet-stream 下载,降低 dev-latest 同名资产命中旧缓存的风险 - 保持 DuckDB 无主键编辑修复不回退,并通过 internal/db 与前端相关回归测试
This commit is contained in:
9
.github/workflows/dev-build.yml
vendored
9
.github/workflows/dev-build.yml
vendored
@@ -99,8 +99,13 @@ jobs:
|
||||
fi
|
||||
if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then
|
||||
if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then
|
||||
echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 Published dev driver release manifest is consistent with its source commit"
|
||||
if python3 tools/validate-driver-release-assets.py --repo Syngnat/GoNavi-DriverAgents --tag dev-latest; then
|
||||
echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 Published dev driver release manifest and actual assets are consistent with its source commit"
|
||||
else
|
||||
echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
|
||||
echo "⚠️ Published dev driver release assets do not match manifest; forcing full rebuild"
|
||||
fi
|
||||
else
|
||||
echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
|
||||
echo "⚠️ Published dev driver release manifest is stale; forcing full rebuild"
|
||||
|
||||
9
.github/workflows/release.yml
vendored
9
.github/workflows/release.yml
vendored
@@ -104,8 +104,13 @@ jobs:
|
||||
fi
|
||||
if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then
|
||||
if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then
|
||||
echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 Published driver release manifest is consistent with its source commit"
|
||||
if python3 tools/validate-driver-release-assets.py --repo Syngnat/GoNavi-DriverAgents --tag "$PREV_TAG"; then
|
||||
echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 Published driver release manifest and actual assets are consistent with its source commit"
|
||||
else
|
||||
echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
|
||||
echo "⚠️ Published driver release assets do not match manifest; forcing full rebuild"
|
||||
fi
|
||||
else
|
||||
echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
|
||||
echo "⚠️ Published driver release manifest is stale; forcing full rebuild"
|
||||
|
||||
@@ -3573,8 +3573,14 @@ func isOptionalDriverDownloadZipURL(urlText string) bool {
|
||||
if trimmedURL == "" {
|
||||
return false
|
||||
}
|
||||
if parsed, err := url.Parse(trimmedURL); err == nil && strings.TrimSpace(parsed.Path) != "" {
|
||||
return strings.EqualFold(path.Ext(parsed.Path), ".zip")
|
||||
if parsed, err := url.Parse(trimmedURL); err == nil {
|
||||
if strings.TrimSpace(parsed.Path) != "" && strings.EqualFold(path.Ext(parsed.Path), ".zip") {
|
||||
return true
|
||||
}
|
||||
if strings.TrimSpace(parsed.Fragment) != "" && strings.EqualFold(path.Ext(parsed.Fragment), ".zip") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(filepath.Ext(trimmedURL), ".zip")
|
||||
}
|
||||
@@ -4492,6 +4498,45 @@ func driverReleaseLatestDownloadURL(assetName string) string {
|
||||
return fmt.Sprintf("https://github.com/%s/releases/latest/download/%s", driverReleaseRepo, url.PathEscape(asset))
|
||||
}
|
||||
|
||||
func findReleaseAssetByName(release *githubRelease, assetNames []string) (githubAsset, bool) {
|
||||
if release == nil || len(release.Assets) == 0 || len(assetNames) == 0 {
|
||||
return githubAsset{}, false
|
||||
}
|
||||
for _, expected := range assetNames {
|
||||
trimmed := strings.TrimSpace(expected)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
for _, asset := range release.Assets {
|
||||
if strings.EqualFold(strings.TrimSpace(asset.Name), trimmed) {
|
||||
return asset, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return githubAsset{}, false
|
||||
}
|
||||
|
||||
func driverReleaseAssetAPIURL(asset githubAsset) string {
|
||||
urlText := strings.TrimSpace(asset.URL)
|
||||
if urlText != "" {
|
||||
name := strings.TrimSpace(asset.Name)
|
||||
if name == "" {
|
||||
return urlText
|
||||
}
|
||||
parsed, err := url.Parse(urlText)
|
||||
if err != nil {
|
||||
return urlText
|
||||
}
|
||||
parsed.Fragment = name
|
||||
return parsed.String()
|
||||
}
|
||||
urlText = strings.TrimSpace(asset.BrowserDownloadURL)
|
||||
if urlText == "" {
|
||||
return ""
|
||||
}
|
||||
return urlText
|
||||
}
|
||||
|
||||
func optionalDriverBundlePlatformDir(goos string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(goos)) {
|
||||
case "windows":
|
||||
@@ -4579,8 +4624,18 @@ func resolveOptionalDriverBundleDownloadURLs() []string {
|
||||
}
|
||||
|
||||
if tag := currentDriverReleaseTag(); tag != "" {
|
||||
if release, err := fetchReleaseByTag(tag); err == nil {
|
||||
if asset, ok := findReleaseAssetByName(release, []string{optionalDriverBundleAssetName}); ok {
|
||||
appendURL(driverReleaseAssetAPIURL(asset))
|
||||
}
|
||||
}
|
||||
appendURL(driverReleaseDownloadURL(tag, optionalDriverBundleAssetName))
|
||||
}
|
||||
if release, err := fetchLatestReleaseForDriverAssets(); err == nil {
|
||||
if asset, ok := findReleaseAssetByName(release, []string{optionalDriverBundleAssetName}); ok {
|
||||
appendURL(driverReleaseAssetAPIURL(asset))
|
||||
}
|
||||
}
|
||||
appendURL(driverReleaseLatestDownloadURL(optionalDriverBundleAssetName))
|
||||
return candidates
|
||||
}
|
||||
@@ -5428,6 +5483,11 @@ func resolveLatestPublishedDriverDownloadURL(definition driverDefinition) (strin
|
||||
if shouldUseDuckDBWindowsDynamicLibrary(driverType) {
|
||||
if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache("latest"); ok {
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
if release, err := fetchLatestReleaseForDriverAssets(); err == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName), true
|
||||
}
|
||||
return "", false
|
||||
@@ -5438,6 +5498,11 @@ func resolveLatestPublishedDriverDownloadURL(definition driverDefinition) (strin
|
||||
return "", false
|
||||
}
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
if release, relErr := fetchLatestReleaseForDriverAssets(); relErr == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName), true
|
||||
}
|
||||
return "", false
|
||||
@@ -5451,6 +5516,11 @@ func resolveLatestPublishedDriverDownloadURL(definition driverDefinition) (strin
|
||||
if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache("latest"); ok {
|
||||
for _, assetName := range assetNames {
|
||||
if publishedAssets[assetName] && sizeByAsset[assetName] > 0 {
|
||||
if release, err := fetchLatestReleaseForDriverAssets(); err == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{assetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURL(assetName), true
|
||||
}
|
||||
}
|
||||
@@ -5463,6 +5533,11 @@ func resolveLatestPublishedDriverDownloadURL(definition driverDefinition) (strin
|
||||
}
|
||||
for _, assetName := range assetNames {
|
||||
if publishedAssets[assetName] && sizeByAsset[assetName] > 0 {
|
||||
if release, relErr := fetchLatestReleaseForDriverAssets(); relErr == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{assetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURL(assetName), true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,11 +92,40 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing
|
||||
urls := resolveOptionalDriverBundleDownloadURLs()
|
||||
wantTagged := driverReleaseDownloadURL("v0.7.4", optionalDriverBundleAssetName)
|
||||
wantLatest := driverReleaseLatestDownloadURL(optionalDriverBundleAssetName)
|
||||
if len(urls) != 2 {
|
||||
t.Fatalf("expected tagged and latest bundle URLs, got %v", urls)
|
||||
if len(urls) < 2 {
|
||||
t.Fatalf("expected at least tagged and latest bundle URLs, got %v", urls)
|
||||
}
|
||||
if urls[0] != wantTagged || urls[1] != wantLatest {
|
||||
t.Fatalf("unexpected driver bundle URLs: got %v want [%q %q]", urls, wantTagged, wantLatest)
|
||||
foundTagged := false
|
||||
foundLatest := false
|
||||
for _, candidate := range urls {
|
||||
if candidate == wantTagged {
|
||||
foundTagged = true
|
||||
}
|
||||
if candidate == wantLatest {
|
||||
foundLatest = true
|
||||
}
|
||||
}
|
||||
if !foundTagged || !foundLatest {
|
||||
t.Fatalf("expected bundle URLs to include tagged=%q and latest=%q, got %v", wantTagged, wantLatest, urls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDriverReleaseAssetAPIURLUsesReleaseAssetEndpoint(t *testing.T) {
|
||||
asset := githubAsset{
|
||||
Name: "kingbase-driver-agent-darwin-arm64",
|
||||
BrowserDownloadURL: "https://github.com/Syngnat/GoNavi-DriverAgents/releases/download/dev-latest/kingbase-driver-agent-darwin-arm64",
|
||||
URL: "https://api.github.com/repos/Syngnat/GoNavi-DriverAgents/releases/assets/123456",
|
||||
Size: 18 << 20,
|
||||
}
|
||||
if got := driverReleaseAssetAPIURL(asset); got != "https://api.github.com/repos/Syngnat/GoNavi-DriverAgents/releases/assets/123456#kingbase-driver-agent-darwin-arm64" {
|
||||
t.Fatalf("expected release asset API URL, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionalDriverDownloadZipURLAcceptsAssetAPIFragment(t *testing.T) {
|
||||
urlText := "https://api.github.com/repos/Syngnat/GoNavi-DriverAgents/releases/assets/123456#duckdb-driver.zip"
|
||||
if !isOptionalDriverDownloadZipURL(urlText) {
|
||||
t.Fatalf("expected asset API URL with zip fragment to be treated as zip download: %q", urlText)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
urlpkg "net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -102,6 +103,8 @@ type githubRelease struct {
|
||||
type githubAsset struct {
|
||||
Name string `json:"name"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
URL string `json:"url"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
@@ -655,6 +658,9 @@ func downloadFileWithHashWithTimeout(url, filePath string, onProgress func(downl
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoNavi-Updater")
|
||||
if isGitHubReleaseAssetAPIURL(url) {
|
||||
req.Header.Set("Accept", "application/octet-stream")
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -713,6 +719,17 @@ func downloadFileWithHashWithTimeout(url, filePath string, onProgress func(downl
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func isGitHubReleaseAssetAPIURL(urlText string) bool {
|
||||
parsed, err := urlpkg.Parse(strings.TrimSpace(urlText))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(parsed.Host, "api.github.com") {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(strings.TrimSpace(parsed.Path)), "/releases/assets/")
|
||||
}
|
||||
|
||||
func buildUpdateDownloadResult(info UpdateInfo, staged *stagedUpdate) updateDownloadResult {
|
||||
result := updateDownloadResult{
|
||||
Info: info,
|
||||
|
||||
@@ -517,6 +517,7 @@ for file in "${!changed_file_set[@]}"; do
|
||||
tools/compress-driver-artifact.sh|\
|
||||
tools/package-driver-release-assets.py|\
|
||||
tools/generate-driver-release-manifest.py|\
|
||||
tools/validate-driver-release-assets.py|\
|
||||
tools/complete-driver-release-assets.py|\
|
||||
tools/resolve-driver-release-source.py|\
|
||||
tools/validate-driver-release-manifest.sh|\
|
||||
|
||||
@@ -164,4 +164,29 @@ PYEOF
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_release_validation="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-release-validation-change.XXXXXX")"
|
||||
git init -q "$tmpdir_release_validation"
|
||||
mkdir -p "$tmpdir_release_validation/tools"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_release_validation/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_release_validation/tools/validate-driver-release-assets.py" <<'PYEOF'
|
||||
print("validate")
|
||||
PYEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_release_validation"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
printf '\nprint("changed")\n' >> tools/validate-driver-release-assets.py
|
||||
git add tools/validate-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update release validation script'
|
||||
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD 2>/dev/null)"
|
||||
if [[ "$actual" != *"mariadb"* || "$actual" != *"clickhouse"* || "$actual" != *"duckdb"* || "$actual" != *"elasticsearch"* ]]; then
|
||||
echo "expected release validation script change to trigger all driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
echo "detect-changed-driver-agents revision test passed"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -151,6 +152,7 @@ def main():
|
||||
"platform": platform,
|
||||
"revision": revision,
|
||||
"size": child.stat().st_size,
|
||||
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -70,6 +70,7 @@ while IFS= read -r file; do
|
||||
tools/diff-driver-agent-revisions.sh|\
|
||||
tools/package-driver-release-assets.py|\
|
||||
tools/generate-driver-release-manifest.py|\
|
||||
tools/validate-driver-release-assets.py|\
|
||||
tools/complete-driver-release-assets.py|\
|
||||
tools/resolve-driver-release-source.py|\
|
||||
tools/validate-driver-release-manifest.sh|\
|
||||
|
||||
@@ -20,6 +20,9 @@ YAMLEOF
|
||||
cat >"$tmpdir/tools/package-driver-release-assets.py" <<'PYEOF'
|
||||
print("package")
|
||||
PYEOF
|
||||
cat >"$tmpdir/tools/validate-driver-release-assets.py" <<'PYEOF'
|
||||
print("validate")
|
||||
PYEOF
|
||||
cat >"$tmpdir/internal/db/duckdb_impl.go" <<'GOEOF'
|
||||
package db
|
||||
GOEOF
|
||||
@@ -55,6 +58,19 @@ base_ref="$(git rev-parse HEAD)"
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git reset --hard -q "$base_ref"
|
||||
printf '\nprint("changed")\n' >> tools/validate-driver-release-assets.py
|
||||
git add tools/validate-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'release asset validation change'
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base "$base_ref" --head HEAD)"
|
||||
if [[ "$actual" != "true" ]]; then
|
||||
echo "expected release asset validation change to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git reset --hard -q "$base_ref"
|
||||
|
||||
184
tools/validate-driver-release-assets.py
Normal file
184
tools/validate-driver-release-assets.py
Normal file
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"
|
||||
|
||||
|
||||
def github_headers(binary: bool = False):
|
||||
headers = {
|
||||
"Accept": "application/octet-stream" if binary else "application/vnd.github+json",
|
||||
"User-Agent": "GoNavi-CI",
|
||||
}
|
||||
token = os.environ.get("DRIVER_RELEASE_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def fetch_json(url: str):
|
||||
req = urllib.request.Request(url, headers=github_headers(False))
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def download_url(url: str, destination: Path):
|
||||
req = urllib.request.Request(url, headers=github_headers(False))
|
||||
with urllib.request.urlopen(req, timeout=120) as resp, open(destination, "wb") as out:
|
||||
shutil.copyfileobj(resp, out)
|
||||
|
||||
|
||||
def load_release(repo: str, tag: str):
|
||||
owner_repo = repo.strip()
|
||||
if not owner_repo:
|
||||
raise ValueError("repo is required")
|
||||
if tag == "latest":
|
||||
url = f"https://api.github.com/repos/{owner_repo}/releases/latest"
|
||||
else:
|
||||
url = f"https://api.github.com/repos/{owner_repo}/releases/tags/{urllib.parse.quote(tag, safe='')}"
|
||||
return fetch_json(url)
|
||||
|
||||
|
||||
def asset_map(release: dict):
|
||||
result = {}
|
||||
for asset in release.get("assets", []):
|
||||
name = str(asset.get("name") or "").strip()
|
||||
if name:
|
||||
result[name] = asset
|
||||
return result
|
||||
|
||||
|
||||
def infer_asset_path(name: str):
|
||||
trimmed = str(name or "").strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
if trimmed == "duckdb.dll":
|
||||
return "Windows/duckdb.dll"
|
||||
if trimmed == "duckdb-driver.zip":
|
||||
return None
|
||||
if trimmed.endswith("-driver-agent-windows-amd64.exe") or trimmed.endswith("-driver-agent-windows-arm64.exe"):
|
||||
return f"Windows/{trimmed}"
|
||||
if trimmed.endswith("-driver-agent-darwin-amd64") or trimmed.endswith("-driver-agent-darwin-arm64"):
|
||||
return f"MacOS/{trimmed}"
|
||||
if trimmed.endswith("-driver-agent-linux-amd64"):
|
||||
return f"Linux/{trimmed}"
|
||||
return None
|
||||
|
||||
|
||||
def sha256_file(path: Path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def probe_metadata_revision(path: Path):
|
||||
current_mode = path.stat().st_mode
|
||||
path.chmod(current_mode | stat.S_IXUSR)
|
||||
proc = subprocess.run(
|
||||
[str(path)],
|
||||
input=b'{"id":1,"method":"metadata"}\n',
|
||||
capture_output=True,
|
||||
timeout=20,
|
||||
check=True,
|
||||
)
|
||||
if not proc.stdout:
|
||||
raise RuntimeError(f"{path.name}: metadata output is empty")
|
||||
payload = json.loads(proc.stdout.decode("utf-8"))
|
||||
return str(((payload.get("data") or {}).get("agentRevision") or "")).strip()
|
||||
|
||||
|
||||
def validate_release_assets(release: dict, manifest: dict):
|
||||
assets = asset_map(release)
|
||||
manifest_assets = manifest.get("assets") or {}
|
||||
if not isinstance(manifest_assets, dict) or not manifest_assets:
|
||||
raise RuntimeError("manifest assets is empty")
|
||||
|
||||
mismatches = []
|
||||
skipped = []
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-assets-") as tmp:
|
||||
tmp_root = Path(tmp)
|
||||
for name, meta in sorted(manifest_assets.items()):
|
||||
if name == MANIFEST_ASSET_NAME:
|
||||
continue
|
||||
asset = assets.get(name)
|
||||
if asset is None:
|
||||
mismatches.append((name, "missing_release_asset", "", "present in manifest"))
|
||||
continue
|
||||
|
||||
expected_sha = str(meta.get("sha256") or "").strip().lower()
|
||||
expected_revision = str(meta.get("revision") or "").strip()
|
||||
path_hint = infer_asset_path(name)
|
||||
if path_hint is None:
|
||||
skipped.append(name)
|
||||
continue
|
||||
|
||||
local_path = tmp_root / name
|
||||
download_url(str(asset.get("browser_download_url") or "").strip(), local_path)
|
||||
|
||||
actual_sha = sha256_file(local_path).lower()
|
||||
if expected_sha and actual_sha != expected_sha:
|
||||
mismatches.append((name, "sha256", actual_sha, expected_sha))
|
||||
continue
|
||||
|
||||
if expected_revision:
|
||||
actual_revision = probe_metadata_revision(local_path)
|
||||
if actual_revision != expected_revision:
|
||||
mismatches.append((name, "revision", actual_revision, expected_revision))
|
||||
|
||||
return mismatches, skipped
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
|
||||
parser.add_argument("--tag", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
release = load_release(args.repo, args.tag)
|
||||
assets = asset_map(release)
|
||||
manifest_asset = assets.get(MANIFEST_ASSET_NAME)
|
||||
if manifest_asset is None:
|
||||
raise SystemExit(f"release {args.repo}@{args.tag} missing {MANIFEST_ASSET_NAME}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-") as tmp:
|
||||
manifest_path = Path(tmp) / MANIFEST_ASSET_NAME
|
||||
download_url(str(manifest_asset.get("browser_download_url") or "").strip(), manifest_path)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
mismatches, skipped = validate_release_assets(release, manifest)
|
||||
if mismatches:
|
||||
print("published driver release assets mismatch manifest:", file=sys.stderr)
|
||||
for name, field, actual, expected in mismatches:
|
||||
print(f" - {name} [{field}] actual={actual or '<empty>'} expected={expected or '<empty>'}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
checked = len((manifest.get("assets") or {})) - len(skipped)
|
||||
print(f"driver release assets validation passed: checked={checked} skipped={len(skipped)}")
|
||||
if skipped:
|
||||
print("skipped assets: " + ", ".join(sorted(skipped)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"http error: {exc.code}", file=sys.stderr)
|
||||
raise
|
||||
114
tools/validate-driver-release-assets.test.py
Normal file
114
tools/validate-driver-release-assets.test.py
Normal file
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
MODULE_PATH = pathlib.Path(__file__).with_name("validate-driver-release-assets.py")
|
||||
SPEC = importlib.util.spec_from_file_location("validate_driver_release_assets", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class ValidateDriverReleaseAssetsTests(unittest.TestCase):
|
||||
def test_infer_asset_path(self):
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("clickhouse-driver-agent-darwin-arm64"),
|
||||
"MacOS/clickhouse-driver-agent-darwin-arm64",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("clickhouse-driver-agent-windows-arm64.exe"),
|
||||
"Windows/clickhouse-driver-agent-windows-arm64.exe",
|
||||
)
|
||||
self.assertEqual(MODULE.infer_asset_path("duckdb.dll"), "Windows/duckdb.dll")
|
||||
self.assertIsNone(MODULE.infer_asset_path("duckdb-driver.zip"))
|
||||
|
||||
def test_validate_release_assets_reports_sha_mismatch(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-arm64",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-arm64",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-arm64": {
|
||||
"revision": "src-expected",
|
||||
"sha256": "deadbeef",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
payload_path = pathlib.Path(tmp) / "clickhouse-driver-agent-darwin-arm64"
|
||||
payload_path.write_bytes(b"test-binary")
|
||||
|
||||
def fake_download(url, destination):
|
||||
destination.write_bytes(payload_path.read_bytes())
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-expected"
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(len(mismatches), 1)
|
||||
name, field, actual, expected = mismatches[0]
|
||||
self.assertEqual(name, "clickhouse-driver-agent-darwin-arm64")
|
||||
self.assertEqual(field, "sha256")
|
||||
self.assertEqual(actual, hashlib.sha256(b"test-binary").hexdigest())
|
||||
self.assertEqual(expected, "deadbeef")
|
||||
|
||||
def test_validate_release_assets_reports_revision_mismatch(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-arm64",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-arm64",
|
||||
}
|
||||
]
|
||||
}
|
||||
payload = b"test-binary"
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-arm64": {
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
payload_path = pathlib.Path(tmp) / "clickhouse-driver-agent-darwin-arm64"
|
||||
payload_path.write_bytes(payload)
|
||||
|
||||
def fake_download(url, destination):
|
||||
destination.write_bytes(payload_path.read_bytes())
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-actual"
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(mismatches, [("clickhouse-driver-agent-darwin-arm64", "revision", "src-actual", "src-expected")])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user