🐛 fix(ci): 修复脏 driver release 资产导致 revision 错配

tools/generate-driver-release-manifest.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets")
+    parser.add_argument("--output", required=True, help="manifest json output path")
+    return parser.parse_args()
+
+
+def infer_driver_and_platform(file_name: str):
+    suffixes = [
+        "-driver-agent-darwin-amd64",
+        "-driver-agent-darwin-arm64",
+        "-driver-agent-linux-amd64",
+        "-driver-agent-windows-amd64.exe",
+        "-driver-agent-windows-arm64.exe",
+    ]
+    for suffix in suffixes:
+        if file_name.endswith(suffix):
+            driver = file_name[: -len(suffix)]
+            if suffix.endswith(".exe"):
+                platform = suffix.replace("-driver-agent-", "").removesuffix(".exe")
+            else:
+                platform = suffix.replace("-driver-agent-", "")
+            return driver, platform
+    return None, None
+
+
+def probe_revision(path: Path):
+    request = b'{"id":1,"method":"metadata"}\n'
+    proc = subprocess.run(
+        [str(path)],
+        input=request,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        timeout=10,
+        check=True,
+    )
+    line = proc.stdout.decode("utf-8", errors="replace").strip().splitlines()
+    if not line:
+        raise RuntimeError(f"{path.name}: metadata response is empty")
+    payload = json.loads(line[0])
+    data = payload.get("data") or {}
+    revision = str(data.get("agentRevision") or "").strip()
+    driver_type = str(data.get("driverType") or "").strip()
+    if not revision:
+        raise RuntimeError(f"{path.name}: metadata agentRevision is empty")
+    return driver_type, revision
+
+
+def main():
+    args = parse_args()
+    assets_dir = Path(args.assets_dir).resolve()
+    output_path = Path(args.output).resolve()
+
+    manifest = {
+        "schemaVersion": 1,
+        "generatedFrom": os.environ.get("GITHUB_SHA", "").strip(),
+        "assets": {},
+    }
+
+    for child in sorted(assets_dir.rglob("*")):
+        if not child.is_file():
+            continue
+        driver, platform = infer_driver_and_platform(child.name)
+        if not driver or not platform:
+            continue
+        if child.stat().st_size == 0:
+            raise RuntimeError(f"{child.name}: asset is empty")
+        driver_type, revision = probe_revision(child)
+        manifest["assets"][child.name] = {
+            "driver": driver,
+            "driverType": driver_type or driver,
+            "platform": platform,
+            "revision": revision,
+            "size": child.stat().st_size,
+        }
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(f"wrote manifest: {output_path}")
+    print(f"asset count: {len(manifest['assets'])}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except subprocess.TimeoutExpired as exc:
+        print(f"error: probe timed out for {exc.cmd}", file=sys.stderr)
+        raise

tools/resolve-driver-release-source.py

@@ -12,6 +12,7 @@ import urllib.request
 
 COMMIT_LINK_RE = re.compile(r"/commit/([0-9a-f]{40})(?:\b|/)")
 FULL_SHA_RE = re.compile(r"\b([0-9a-f]{40})\b")
+MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"
 
 
 def github_headers():
@@ -31,6 +32,15 @@ def fetch_json(url):
         return json.loads(response.read().decode("utf-8"))
 
 
+def download_asset(asset, destination):
+    headers = github_headers()
+    headers["Accept"] = "application/octet-stream"
+    request = urllib.request.Request(asset["url"], headers=headers)
+    with urllib.request.urlopen(request, timeout=120) as response:
+        with open(destination, "wb") as output:
+            output.write(response.read())
+
+
 def load_release(repo, tag):
     owner_repo = repo.strip()
     if not owner_repo:
@@ -77,16 +87,41 @@ def extract_source_commit(release):
     return None
 
 
+def find_manifest_asset(release):
+    if not isinstance(release, dict):
+        return None
+
+    for asset in release.get("assets", []):
+        if str(asset.get("name") or "").strip() == MANIFEST_ASSET_NAME:
+            return asset
+    return None
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
     parser.add_argument("--tag", required=True, help="release tag name such as dev-latest or v1.0.0")
+    parser.add_argument("--manifest-output", help="optional path to download the published revision manifest asset")
     args = parser.parse_args()
 
     release = load_release(args.repo, args.tag)
     if release is None:
         return 0
 
+    if args.manifest_output:
+        manifest_path = os.path.abspath(args.manifest_output)
+        manifest_asset = find_manifest_asset(release)
+        if manifest_asset is None:
+            if os.path.exists(manifest_path):
+                os.remove(manifest_path)
+            print(
+                f"warning: release {args.repo}@{args.tag} does not expose {MANIFEST_ASSET_NAME}",
+                file=sys.stderr,
+            )
+        else:
+            os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
+            download_asset(manifest_asset, manifest_path)
+
     source_commit = extract_source_commit(release)
     if not source_commit:
         print(

tools/resolve-driver-release-source.test.py

@@ -37,6 +37,21 @@ class ResolveDriverReleaseSourceTests(unittest.TestCase):
         release = {"body": "", "target_commitish": "main"}
         self.assertIsNone(MODULE.extract_source_commit(release))
 
+    def test_finds_manifest_asset(self):
+        release = {
+            "assets": [
+                {"name": "foo.txt"},
+                {"name": "GoNavi-DriverAgents-Manifest.json", "url": "https://example.test/manifest"},
+            ]
+        }
+        asset = MODULE.find_manifest_asset(release)
+        self.assertIsNotNone(asset)
+        self.assertEqual(asset["url"], "https://example.test/manifest")
+
+    def test_returns_none_when_manifest_asset_missing(self):
+        release = {"assets": [{"name": "GoNavi-DriverAgents.zip"}]}
+        self.assertIsNone(MODULE.find_manifest_asset(release))
+
 
 if __name__ == "__main__":
     unittest.main()

tools/validate-driver-release-manifest.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$SCRIPT_DIR"
+
+usage() {
+  cat <<'EOF'
+用法：
+  ./tools/validate-driver-release-manifest.sh --commit <ref> --manifest <path>
+
+说明：
+  校验已发布 driver release manifest 中记录的每个 driver revision，
+  是否与指定源码提交在对应平台上重新生成出的 revision 完全一致。
+EOF
+}
+
+source_commit=""
+manifest_path=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --commit)
+      source_commit="${2:-}"
+      shift 2
+      ;;
+    --manifest)
+      manifest_path="${2:-}"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "未知参数：$1" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "$source_commit" || -z "$manifest_path" ]]; then
+  usage >&2
+  exit 1
+fi
+
+if [[ ! -f "$manifest_path" ]]; then
+  echo "manifest 不存在：$manifest_path" >&2
+  exit 1
+fi
+
+if ! git rev-parse --verify "${source_commit}^{commit}" >/dev/null 2>&1; then
+  echo "无法解析源码提交：$source_commit" >&2
+  exit 1
+fi
+
+extract_revision() {
+  local file="$1"
+  local driver="$2"
+  awk -v target="$driver" '
+    $0 ~ "\"" target "\"" {
+      if (match($0, /"src-[^"]+"/)) {
+        print substr($0, RSTART + 1, RLENGTH - 2)
+        exit
+      }
+    }
+  ' "$file"
+}
+
+normalize_driver() {
+  local value
+  value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
+  case "$value" in
+    doris|diros) echo "diros" ;;
+    *) echo "$value" ;;
+  esac
+}
+
+worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-manifest.XXXXXX")"
+cleanup() {
+  git worktree remove --force "$worktree" >/dev/null 2>&1 || true
+  rm -rf "$worktree"
+}
+trap cleanup EXIT
+
+git worktree add --detach "$worktree" "${source_commit}^{commit}" >/dev/null
+
+python3 - "$manifest_path" "$worktree" <<'PY'
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+manifest_path = Path(sys.argv[1]).resolve()
+worktree = Path(sys.argv[2]).resolve()
+
+with manifest_path.open("r", encoding="utf-8") as fh:
+    manifest = json.load(fh)
+
+assets = manifest.get("assets") or {}
+if not isinstance(assets, dict) or not assets:
+    raise SystemExit("manifest assets 为空")
+
+platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.values() if str(meta.get("platform") or "").strip()})
+if not platforms:
+    raise SystemExit("manifest 未包含平台信息")
+
+def extract_revision(file_path: Path, driver: str) -> str:
+    import re
+    text = file_path.read_text(encoding="utf-8")
+    pattern = re.compile(rf'"{re.escape(driver)}"\s*:\s*"([^"]+)"')
+    match = pattern.search(text)
+    return match.group(1) if match else ""
+
+revision_files = {}
+for platform in platforms:
+    subprocess.run(
+        ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform],
+        cwd=worktree,
+        check=True,
+        stdout=subprocess.DEVNULL,
+    )
+    revision_files[platform] = worktree / "internal/db/driver_agent_revisions_gen.go"
+
+mismatches = []
+for asset_name, meta in sorted(assets.items()):
+    driver = str(meta.get("driver") or meta.get("driverType") or "").strip().lower()
+    if driver == "doris":
+        driver = "diros"
+    platform = str(meta.get("platform") or "").strip()
+    published_revision = str(meta.get("revision") or "").strip()
+    expected_revision = extract_revision(revision_files[platform], driver)
+    if not expected_revision:
+        mismatches.append((asset_name, platform, driver, published_revision, "<missing>"))
+        continue
+    if published_revision != expected_revision:
+        mismatches.append((asset_name, platform, driver, published_revision, expected_revision))
+
+if mismatches:
+    print("published driver release manifest 与源码重算 revision 不一致：", file=sys.stderr)
+    for asset_name, platform, driver, published_revision, expected_revision in mismatches:
+        print(
+            f"  - {asset_name} [{platform}/{driver}] published={published_revision} expected={expected_revision}",
+            file=sys.stderr,
+        )
+    raise SystemExit(1)
+
+print(f"manifest validation passed: {len(assets)} assets")
+PY

tools/validate-driver-release-manifest.test.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$SCRIPT_DIR"
+
+tmpdir="$(mktemp -d)"
+cleanup() {
+  rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+manifest_path="$tmpdir/manifest.json"
+worktree="$tmpdir/worktree"
+git worktree add --detach "$worktree" HEAD >/dev/null
+trap 'git worktree remove --force "$worktree" >/dev/null 2>&1 || true; rm -rf "$tmpdir"' EXIT
+
+(
+  cd "$worktree"
+  bash ./tools/generate-driver-agent-revisions.sh --platform darwin/arm64 >/dev/null
+)
+
+cat >"$manifest_path" <<'EOF'
+{
+  "schemaVersion": 1,
+  "generatedFrom": "test",
+  "assets": {
+    "clickhouse-driver-agent-darwin-arm64": {
+      "driver": "clickhouse",
+      "driverType": "clickhouse",
+      "platform": "darwin/arm64",
+      "revision": "__CLICKHOUSE__",
+      "size": 1
+    },
+    "mariadb-driver-agent-darwin-arm64": {
+      "driver": "mariadb",
+      "driverType": "mariadb",
+      "platform": "darwin/arm64",
+      "revision": "__MARIADB__",
+      "size": 1
+    }
+  }
+}
+EOF
+
+revision_file="$worktree/internal/db/driver_agent_revisions_gen.go"
+clickhouse_revision="$(awk '/"clickhouse"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")"
+mariadb_revision="$(awk '/"mariadb"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")"
+
+python3 - "$manifest_path" "$clickhouse_revision" "$mariadb_revision" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+clickhouse = sys.argv[2]
+mariadb = sys.argv[3]
+data = json.loads(path.read_text(encoding="utf-8"))
+data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse
+data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb
+path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+PY
+
+bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"
+echo "validate-driver-release-manifest test passed"