🐛 fix(ci): 修复脏 driver release 资产导致 revision 错配

2026-06-14 10:29:52 +08:00 · 2026-06-05 16:10:05 +08:00
parent f7dd90a5d1
commit be26970761
7 changed files with 460 additions and 1 deletions
--- a/.github/workflows/dev-build.yml
+++ b/.github/workflows/dev-build.yml
@@ -71,6 +71,7 @@ jobs:
      has_changes: ${{ steps.detect.outputs.has_changes }}
      release_source: ${{ steps.detect.outputs.release_source }}
      source_commit: ${{ steps.published_source.outputs.source_commit }}
+      has_manifest: ${{ steps.published_source.outputs.has_manifest }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
@@ -84,8 +85,27 @@ jobs:
        shell: bash
        run: |
          set -euo pipefail
-          SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag dev-latest)"
+          manifest_path="$RUNNER_TEMP/published-driver-manifest.json"
+          SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag dev-latest --manifest-output "$manifest_path")"
          echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT"
+          if [[ -s "$manifest_path" ]]; then
+            echo "has_manifest=true" >> "$GITHUB_OUTPUT"
+            echo "🧭 Published dev driver release exposes revision manifest"
+          else
+            echo "has_manifest=false" >> "$GITHUB_OUTPUT"
+            echo "🧭 Published dev driver release has no revision manifest"
+          fi
+          if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then
+            if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then
+              echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
+              echo "🧭 Published dev driver release manifest is consistent with its source commit"
+            else
+              echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
+              echo "⚠️ Published dev driver release manifest is stale; forcing full rebuild"
+            fi
+          else
+            echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
+          fi
          if [[ -n "$SOURCE_COMMIT" ]]; then
            echo "🧭 Last published dev driver release source commit: $SOURCE_COMMIT"
          else
@@ -123,6 +143,15 @@ jobs:
          }

          BASE_REF="${{ steps.published_source.outputs.source_commit }}"
+          HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}"
+          MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}"
+          if [[ "$HAS_MANIFEST" != "true" ]]; then
+            echo "⚠️ Published driver release lacks revision manifest; forcing full rebuild to self-heal old assets"
+            BASE_REF="all"
+          elif [[ "$MANIFEST_VALID" != "true" ]]; then
+            echo "⚠️ Published driver release manifest is stale; forcing full rebuild to self-heal old assets"
+            BASE_REF="all"
+          fi
          if [[ -n "$BASE_REF" ]]; then
            if git rev-parse --verify "${BASE_REF}^{commit}" >/dev/null 2>&1 && git merge-base --is-ancestor "$BASE_REF" "$GITHUB_SHA"; then
              echo "🧭 Using last published driver release source commit as detection base: $BASE_REF"
@@ -807,6 +836,10 @@ jobs:
            exit 0
          fi

+          python3 tools/generate-driver-release-manifest.py \
+            --assets-dir . \
+            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
+
          echo "📦 打包驱动总包：GoNavi-DriverAgents.zip"
          python3 - <<'PY'
          import json
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -67,12 +67,57 @@ jobs:
      has_changes: ${{ steps.detect.outputs.has_changes }}
      release_source: ${{ steps.detect.outputs.release_source }}
      compare_base: ${{ steps.detect.outputs.compare_base }}
+      published_source_commit: ${{ steps.published_source.outputs.source_commit }}
+      has_manifest: ${{ steps.published_source.outputs.has_manifest }}
+      manifest_valid: ${{ steps.published_source.outputs.manifest_valid }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

+      - name: Resolve published driver release source
+        id: published_source
+        env:
+          DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          PREV_TAG="$(git describe --tags --match 'v*' --abbrev=0 "${GITHUB_SHA}^" 2>/dev/null || true)"
+          manifest_path="$RUNNER_TEMP/published-driver-manifest.json"
+          if [[ -z "$PREV_TAG" ]]; then
+            echo "source_commit=" >> "$GITHUB_OUTPUT"
+            echo "has_manifest=false" >> "$GITHUB_OUTPUT"
+            echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
+            echo "🧭 No previous stable tag found; full rebuild will be used"
+            exit 0
+          fi
+          SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag "$PREV_TAG" --manifest-output "$manifest_path")"
+          echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT"
+          if [[ -s "$manifest_path" ]]; then
+            echo "has_manifest=true" >> "$GITHUB_OUTPUT"
+            echo "🧭 Published driver release exposes revision manifest"
+          else
+            echo "has_manifest=false" >> "$GITHUB_OUTPUT"
+            echo "🧭 Published driver release has no revision manifest"
+          fi
+          if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then
+            if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then
+              echo "manifest_valid=true" >> "$GITHUB_OUTPUT"
+              echo "🧭 Published driver release manifest is consistent with its source commit"
+            else
+              echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
+              echo "⚠️ Published driver release manifest is stale; forcing full rebuild"
+            fi
+          else
+            echo "manifest_valid=false" >> "$GITHUB_OUTPUT"
+          fi
+          if [[ -n "$SOURCE_COMMIT" ]]; then
+            echo "🧭 Last published release source commit (${PREV_TAG}): $SOURCE_COMMIT"
+          else
+            echo "🧭 Unable to resolve published release source commit for ${PREV_TAG}; fallback to previous tag diff"
+          fi
+
      - name: Detect changed driver agents
        id: detect
        shell: bash
@@ -112,6 +157,17 @@ jobs:
            BASE_REF="all"
            RELEASE_SOURCE="all"
          fi
+          HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}"
+          MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}"
+          if [[ "$RELEASE_SOURCE" != "all" && "$HAS_MANIFEST" != "true" ]]; then
+            echo "⚠️ Published driver release lacks revision manifest; forcing full rebuild to self-heal old assets"
+            BASE_REF="all"
+            RELEASE_SOURCE="all"
+          elif [[ "$RELEASE_SOURCE" != "all" && "$MANIFEST_VALID" != "true" ]]; then
+            echo "⚠️ Published driver release manifest is stale; forcing full rebuild to self-heal old assets"
+            BASE_REF="all"
+            RELEASE_SOURCE="all"
+          fi
          DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
          if [[ "$BASE_REF" != "all" ]]; then
            REVISION_DRIVERS=""
@@ -837,6 +893,10 @@ jobs:
            exit 0
          fi

+          python3 tools/generate-driver-release-manifest.py \
+            --assets-dir . \
+            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
+
          echo "📦 打包驱动总包：GoNavi-DriverAgents.zip"
          python3 - <<'PY'
          import json
--- a/tools/generate-driver-release-manifest.py
+++ b/tools/generate-driver-release-manifest.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets")
+    parser.add_argument("--output", required=True, help="manifest json output path")
+    return parser.parse_args()
+
+
+def infer_driver_and_platform(file_name: str):
+    suffixes = [
+        "-driver-agent-darwin-amd64",
+        "-driver-agent-darwin-arm64",
+        "-driver-agent-linux-amd64",
+        "-driver-agent-windows-amd64.exe",
+        "-driver-agent-windows-arm64.exe",
+    ]
+    for suffix in suffixes:
+        if file_name.endswith(suffix):
+            driver = file_name[: -len(suffix)]
+            if suffix.endswith(".exe"):
+                platform = suffix.replace("-driver-agent-", "").removesuffix(".exe")
+            else:
+                platform = suffix.replace("-driver-agent-", "")
+            return driver, platform
+    return None, None
+
+
+def probe_revision(path: Path):
+    request = b'{"id":1,"method":"metadata"}\n'
+    proc = subprocess.run(
+        [str(path)],
+        input=request,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        timeout=10,
+        check=True,
+    )
+    line = proc.stdout.decode("utf-8", errors="replace").strip().splitlines()
+    if not line:
+        raise RuntimeError(f"{path.name}: metadata response is empty")
+    payload = json.loads(line[0])
+    data = payload.get("data") or {}
+    revision = str(data.get("agentRevision") or "").strip()
+    driver_type = str(data.get("driverType") or "").strip()
+    if not revision:
+        raise RuntimeError(f"{path.name}: metadata agentRevision is empty")
+    return driver_type, revision
+
+
+def main():
+    args = parse_args()
+    assets_dir = Path(args.assets_dir).resolve()
+    output_path = Path(args.output).resolve()
+
+    manifest = {
+        "schemaVersion": 1,
+        "generatedFrom": os.environ.get("GITHUB_SHA", "").strip(),
+        "assets": {},
+    }
+
+    for child in sorted(assets_dir.rglob("*")):
+        if not child.is_file():
+            continue
+        driver, platform = infer_driver_and_platform(child.name)
+        if not driver or not platform:
+            continue
+        if child.stat().st_size == 0:
+            raise RuntimeError(f"{child.name}: asset is empty")
+        driver_type, revision = probe_revision(child)
+        manifest["assets"][child.name] = {
+            "driver": driver,
+            "driverType": driver_type or driver,
+            "platform": platform,
+            "revision": revision,
+            "size": child.stat().st_size,
+        }
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(f"wrote manifest: {output_path}")
+    print(f"asset count: {len(manifest['assets'])}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except subprocess.TimeoutExpired as exc:
+        print(f"error: probe timed out for {exc.cmd}", file=sys.stderr)
+        raise
--- a/tools/resolve-driver-release-source.py
+++ b/tools/resolve-driver-release-source.py
@@ -12,6 +12,7 @@ import urllib.request

 COMMIT_LINK_RE = re.compile(r"/commit/([0-9a-f]{40})(?:\b|/)")
 FULL_SHA_RE = re.compile(r"\b([0-9a-f]{40})\b")
+MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"


 def github_headers():
@@ -31,6 +32,15 @@ def fetch_json(url):
        return json.loads(response.read().decode("utf-8"))


+def download_asset(asset, destination):
+    headers = github_headers()
+    headers["Accept"] = "application/octet-stream"
+    request = urllib.request.Request(asset["url"], headers=headers)
+    with urllib.request.urlopen(request, timeout=120) as response:
+        with open(destination, "wb") as output:
+            output.write(response.read())
+
+
 def load_release(repo, tag):
    owner_repo = repo.strip()
    if not owner_repo:
@@ -77,16 +87,41 @@ def extract_source_commit(release):
    return None


+def find_manifest_asset(release):
+    if not isinstance(release, dict):
+        return None
+
+    for asset in release.get("assets", []):
+        if str(asset.get("name") or "").strip() == MANIFEST_ASSET_NAME:
+            return asset
+    return None
+
+
 def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
    parser.add_argument("--tag", required=True, help="release tag name such as dev-latest or v1.0.0")
+    parser.add_argument("--manifest-output", help="optional path to download the published revision manifest asset")
    args = parser.parse_args()

    release = load_release(args.repo, args.tag)
    if release is None:
        return 0

+    if args.manifest_output:
+        manifest_path = os.path.abspath(args.manifest_output)
+        manifest_asset = find_manifest_asset(release)
+        if manifest_asset is None:
+            if os.path.exists(manifest_path):
+                os.remove(manifest_path)
+            print(
+                f"warning: release {args.repo}@{args.tag} does not expose {MANIFEST_ASSET_NAME}",
+                file=sys.stderr,
+            )
+        else:
+            os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
+            download_asset(manifest_asset, manifest_path)
+
    source_commit = extract_source_commit(release)
    if not source_commit:
        print(
--- a/tools/resolve-driver-release-source.test.py
+++ b/tools/resolve-driver-release-source.test.py
@@ -37,6 +37,21 @@ class ResolveDriverReleaseSourceTests(unittest.TestCase):
        release = {"body": "", "target_commitish": "main"}
        self.assertIsNone(MODULE.extract_source_commit(release))

+    def test_finds_manifest_asset(self):
+        release = {
+            "assets": [
+                {"name": "foo.txt"},
+                {"name": "GoNavi-DriverAgents-Manifest.json", "url": "https://example.test/manifest"},
+            ]
+        }
+        asset = MODULE.find_manifest_asset(release)
+        self.assertIsNotNone(asset)
+        self.assertEqual(asset["url"], "https://example.test/manifest")
+
+    def test_returns_none_when_manifest_asset_missing(self):
+        release = {"assets": [{"name": "GoNavi-DriverAgents.zip"}]}
+        self.assertIsNone(MODULE.find_manifest_asset(release))
+

 if __name__ == "__main__":
    unittest.main()
--- a/tools/validate-driver-release-manifest.sh
+++ b/tools/validate-driver-release-manifest.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$SCRIPT_DIR"
+
+usage() {
+  cat <<'EOF'
+用法：
+  ./tools/validate-driver-release-manifest.sh --commit <ref> --manifest <path>
+
+说明：
+  校验已发布 driver release manifest 中记录的每个 driver revision，
+  是否与指定源码提交在对应平台上重新生成出的 revision 完全一致。
+EOF
+}
+
+source_commit=""
+manifest_path=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --commit)
+      source_commit="${2:-}"
+      shift 2
+      ;;
+    --manifest)
+      manifest_path="${2:-}"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "未知参数：$1" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "$source_commit" || -z "$manifest_path" ]]; then
+  usage >&2
+  exit 1
+fi
+
+if [[ ! -f "$manifest_path" ]]; then
+  echo "manifest 不存在：$manifest_path" >&2
+  exit 1
+fi
+
+if ! git rev-parse --verify "${source_commit}^{commit}" >/dev/null 2>&1; then
+  echo "无法解析源码提交：$source_commit" >&2
+  exit 1
+fi
+
+extract_revision() {
+  local file="$1"
+  local driver="$2"
+  awk -v target="$driver" '
+    $0 ~ "\"" target "\"" {
+      if (match($0, /"src-[^"]+"/)) {
+        print substr($0, RSTART + 1, RLENGTH - 2)
+        exit
+      }
+    }
+  ' "$file"
+}
+
+normalize_driver() {
+  local value
+  value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
+  case "$value" in
+    doris|diros) echo "diros" ;;
+    *) echo "$value" ;;
+  esac
+}
+
+worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-manifest.XXXXXX")"
+cleanup() {
+  git worktree remove --force "$worktree" >/dev/null 2>&1 || true
+  rm -rf "$worktree"
+}
+trap cleanup EXIT
+
+git worktree add --detach "$worktree" "${source_commit}^{commit}" >/dev/null
+
+python3 - "$manifest_path" "$worktree" <<'PY'
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+manifest_path = Path(sys.argv[1]).resolve()
+worktree = Path(sys.argv[2]).resolve()
+
+with manifest_path.open("r", encoding="utf-8") as fh:
+    manifest = json.load(fh)
+
+assets = manifest.get("assets") or {}
+if not isinstance(assets, dict) or not assets:
+    raise SystemExit("manifest assets 为空")
+
+platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.values() if str(meta.get("platform") or "").strip()})
+if not platforms:
+    raise SystemExit("manifest 未包含平台信息")
+
+def extract_revision(file_path: Path, driver: str) -> str:
+    import re
+    text = file_path.read_text(encoding="utf-8")
+    pattern = re.compile(rf'"{re.escape(driver)}"\s*:\s*"([^"]+)"')
+    match = pattern.search(text)
+    return match.group(1) if match else ""
+
+revision_files = {}
+for platform in platforms:
+    subprocess.run(
+        ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform],
+        cwd=worktree,
+        check=True,
+        stdout=subprocess.DEVNULL,
+    )
+    revision_files[platform] = worktree / "internal/db/driver_agent_revisions_gen.go"
+
+mismatches = []
+for asset_name, meta in sorted(assets.items()):
+    driver = str(meta.get("driver") or meta.get("driverType") or "").strip().lower()
+    if driver == "doris":
+        driver = "diros"
+    platform = str(meta.get("platform") or "").strip()
+    published_revision = str(meta.get("revision") or "").strip()
+    expected_revision = extract_revision(revision_files[platform], driver)
+    if not expected_revision:
+        mismatches.append((asset_name, platform, driver, published_revision, "<missing>"))
+        continue
+    if published_revision != expected_revision:
+        mismatches.append((asset_name, platform, driver, published_revision, expected_revision))
+
+if mismatches:
+    print("published driver release manifest 与源码重算 revision 不一致：", file=sys.stderr)
+    for asset_name, platform, driver, published_revision, expected_revision in mismatches:
+        print(
+            f"  - {asset_name} [{platform}/{driver}] published={published_revision} expected={expected_revision}",
+            file=sys.stderr,
+        )
+    raise SystemExit(1)
+
+print(f"manifest validation passed: {len(assets)} assets")
+PY
--- a/tools/validate-driver-release-manifest.test.sh
+++ b/tools/validate-driver-release-manifest.test.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$SCRIPT_DIR"
+
+tmpdir="$(mktemp -d)"
+cleanup() {
+  rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+manifest_path="$tmpdir/manifest.json"
+worktree="$tmpdir/worktree"
+git worktree add --detach "$worktree" HEAD >/dev/null
+trap 'git worktree remove --force "$worktree" >/dev/null 2>&1 || true; rm -rf "$tmpdir"' EXIT
+
+(
+  cd "$worktree"
+  bash ./tools/generate-driver-agent-revisions.sh --platform darwin/arm64 >/dev/null
+)
+
+cat >"$manifest_path" <<'EOF'
+{
+  "schemaVersion": 1,
+  "generatedFrom": "test",
+  "assets": {
+    "clickhouse-driver-agent-darwin-arm64": {
+      "driver": "clickhouse",
+      "driverType": "clickhouse",
+      "platform": "darwin/arm64",
+      "revision": "__CLICKHOUSE__",
+      "size": 1
+    },
+    "mariadb-driver-agent-darwin-arm64": {
+      "driver": "mariadb",
+      "driverType": "mariadb",
+      "platform": "darwin/arm64",
+      "revision": "__MARIADB__",
+      "size": 1
+    }
+  }
+}
+EOF
+
+revision_file="$worktree/internal/db/driver_agent_revisions_gen.go"
+clickhouse_revision="$(awk '/"clickhouse"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")"
+mariadb_revision="$(awk '/"mariadb"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")"
+
+python3 - "$manifest_path" "$clickhouse_revision" "$mariadb_revision" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+clickhouse = sys.argv[2]
+mariadb = sys.argv[3]
+data = json.loads(path.read_text(encoding="utf-8"))
+data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse
+data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb
+path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+PY
+
+bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"
+echo "validate-driver-release-manifest test passed"