🐛 fix(ci): 修复驱动总包打包与 manifest 生成失败

- 抽离 driver release 打包脚本，避免内联脚本删掉已生成 manifest
- 修复 release-assets 场景下 tools 相对路径错误，统一通过仓库根脚本打包
- manifest 改为按源码和目标平台重算 revision，不再执行跨平台 driver 二进制
- 补充 driver release 打包与 manifest 生成回归测试

.github/workflows/dev-build.yml

@@ -836,58 +836,14 @@ jobs:
             exit 0
           fi
 
-          python3 tools/generate-driver-release-manifest.py \
-            --assets-dir . \
-            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
-
           echo "📦 打包驱动总包：GoNavi-DriverAgents.zip"
-          python3 - <<'PY'
-          import json
-          import os
-          import shutil
-          import zipfile
-          from pathlib import Path
+          python3 ../tools/package-driver-release-assets.py \
+            drivers \
+            ../driver-release-assets
 
-          out_name = "GoNavi-DriverAgents.zip"
-          index_name = "GoNavi-DriverAgents-Index.json"
-          base = Path("drivers")
-          driver_release_dir = Path("../driver-release-assets")
-          if driver_release_dir.exists():
-              shutil.rmtree(driver_release_dir)
-          driver_release_dir.mkdir(parents=True, exist_ok=True)
-          out_path = driver_release_dir / out_name
-          index_path = driver_release_dir / index_name
-          if out_path.exists():
-              out_path.unlink()
-          if index_path.exists():
-              index_path.unlink()
-
-          size_index = {}
-          standalone_assets = []
-          with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
-              for p in sorted(base.rglob("*")):
-                  if not p.is_file():
-                      continue
-                  arcname = p.relative_to(base).as_posix()
-                  if p.name in size_index:
-                      raise RuntimeError(f"driver asset name conflict: {p.name}")
-                  zf.write(p, arcname)
-                  size_index[p.name] = p.stat().st_size
-                  standalone_path = driver_release_dir / p.name
-                  if standalone_path.exists():
-                      raise RuntimeError(f"release asset already exists: {standalone_path}")
-                  shutil.copy2(p, standalone_path)
-                  standalone_assets.append(standalone_path.name)
-
-          index_path.write_text(
-              json.dumps({"assets": size_index}, ensure_ascii=False, indent=2),
-              encoding="utf-8",
-          )
-
-          print(f"created {out_name} size={out_path.stat().st_size} bytes")
-          print(f"created {index_name} entries={len(size_index)}")
-          print(f"published standalone driver assets={len(standalone_assets)}")
-          PY
+          python3 ../tools/generate-driver-release-manifest.py \
+            --assets-dir drivers \
+            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
 
           rm -rf drivers
           echo "has_driver_assets=true" >> "$GITHUB_OUTPUT"

.github/workflows/release.yml

@@ -893,58 +893,14 @@ jobs:
             exit 0
           fi
 
-          python3 tools/generate-driver-release-manifest.py \
-            --assets-dir . \
-            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
-
           echo "📦 打包驱动总包：GoNavi-DriverAgents.zip"
-          python3 - <<'PY'
-          import json
-          import os
-          import shutil
-          import zipfile
-          from pathlib import Path
+          python3 ../tools/package-driver-release-assets.py \
+            drivers \
+            ../driver-release-assets
 
-          out_name = "GoNavi-DriverAgents.zip"
-          index_name = "GoNavi-DriverAgents-Index.json"
-          base = Path("drivers")
-          driver_release_dir = Path("../driver-release-assets")
-          if driver_release_dir.exists():
-              shutil.rmtree(driver_release_dir)
-          driver_release_dir.mkdir(parents=True, exist_ok=True)
-          out_path = driver_release_dir / out_name
-          index_path = driver_release_dir / index_name
-          if out_path.exists():
-              out_path.unlink()
-          if index_path.exists():
-              index_path.unlink()
-
-          size_index = {}
-          standalone_assets = []
-          with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
-              for p in sorted(base.rglob("*")):
-                  if not p.is_file():
-                      continue
-                  arcname = p.relative_to(base).as_posix()
-                  if p.name in size_index:
-                      raise RuntimeError(f"driver asset name conflict: {p.name}")
-                  zf.write(p, arcname)
-                  size_index[p.name] = p.stat().st_size
-                  standalone_path = driver_release_dir / p.name
-                  if standalone_path.exists():
-                      raise RuntimeError(f"release asset already exists: {standalone_path}")
-                  shutil.copy2(p, standalone_path)
-                  standalone_assets.append(standalone_path.name)
-
-          index_path.write_text(
-              json.dumps({"assets": size_index}, ensure_ascii=False, indent=2),
-              encoding="utf-8",
-          )
-
-          print(f"created {out_name} size={out_path.stat().st_size} bytes")
-          print(f"created {index_name} entries={len(size_index)}")
-          print(f"published standalone driver assets={len(standalone_assets)}")
-          PY
+          python3 ../tools/generate-driver-release-manifest.py \
+            --assets-dir drivers \
+            --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json
 
           # GoNavi 主仓库只保留主程序包；驱动资产发布到独立仓库。
           rm -rf drivers

tools/generate-driver-release-manifest.py

@@ -5,6 +5,7 @@ import json
 import os
 import subprocess
 import sys
+import tempfile
 from pathlib import Path
 
 
@@ -26,47 +27,100 @@ def infer_driver_and_platform(file_name: str):
     for suffix in suffixes:
         if file_name.endswith(suffix):
             driver = file_name[: -len(suffix)]
-            if suffix.endswith(".exe"):
-                platform = suffix.replace("-driver-agent-", "").removesuffix(".exe")
-            else:
-                platform = suffix.replace("-driver-agent-", "")
+            platform_name = suffix.replace("-driver-agent-", "")
+            if platform_name.endswith(".exe"):
+                platform_name = platform_name.removesuffix(".exe")
+            platform = platform_name.replace("-", "/", 1)
             return driver, platform
     return None, None
 
 
-def probe_revision(path: Path):
-    request = b'{"id":1,"method":"metadata"}\n'
+def normalize_driver(driver: str):
+    value = str(driver or "").strip().lower()
+    if value == "doris":
+        return "diros"
+    return value
+
+
+def repo_root():
+    return Path(__file__).resolve().parent.parent
+
+
+def resolve_head_commit(root: Path):
     proc = subprocess.run(
-        [str(path)],
-        input=request,
+        ["git", "rev-parse", "HEAD"],
+        cwd=root,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        timeout=10,
+        text=True,
         check=True,
     )
-    line = proc.stdout.decode("utf-8", errors="replace").strip().splitlines()
-    if not line:
-        raise RuntimeError(f"{path.name}: metadata response is empty")
-    payload = json.loads(line[0])
-    data = payload.get("data") or {}
-    revision = str(data.get("agentRevision") or "").strip()
-    driver_type = str(data.get("driverType") or "").strip()
-    if not revision:
-        raise RuntimeError(f"{path.name}: metadata agentRevision is empty")
-    return driver_type, revision
+    return proc.stdout.strip()
+
+
+def parse_revision_file(path: Path):
+    revisions = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if not stripped.startswith('"'):
+            continue
+        try:
+            driver, revision = stripped.rstrip(",").split(":", 1)
+        except ValueError:
+            continue
+        revisions[driver.strip().strip('"')] = revision.strip().strip('"')
+    return revisions
+
+
+def generate_platform_revisions(root: Path, drivers_by_platform):
+    if not drivers_by_platform:
+        return {}
+
+    with tempfile.TemporaryDirectory(prefix="gonavi-driver-release-manifest-") as tmp:
+        worktree = Path(tmp) / "worktree"
+        subprocess.run(
+            ["git", "worktree", "add", "--detach", str(worktree), "HEAD"],
+            cwd=root,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=True,
+        )
+        try:
+            revision_file = worktree / "internal/db/driver_agent_revisions_gen.go"
+            result = {}
+            for platform in sorted(drivers_by_platform):
+                drivers = sorted({normalize_driver(driver) for driver in drivers_by_platform[platform] if normalize_driver(driver)})
+                command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform]
+                if drivers:
+                    command.extend(["--drivers", ",".join(drivers)])
+                subprocess.run(
+                    command,
+                    cwd=worktree,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    check=True,
+                )
+                result[platform] = parse_revision_file(revision_file)
+            return result
+        finally:
+            subprocess.run(
+                ["git", "worktree", "remove", "--force", str(worktree)],
+                cwd=root,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=False,
+            )
 
 
 def main():
     args = parse_args()
     assets_dir = Path(args.assets_dir).resolve()
     output_path = Path(args.output).resolve()
+    root = repo_root()
 
-    manifest = {
-        "schemaVersion": 1,
-        "generatedFrom": os.environ.get("GITHUB_SHA", "").strip(),
-        "assets": {},
-    }
-
+    asset_entries = []
+    drivers_by_platform = {}
     for child in sorted(assets_dir.rglob("*")):
         if not child.is_file():
             continue
@@ -75,10 +129,25 @@ def main():
             continue
         if child.stat().st_size == 0:
             raise RuntimeError(f"{child.name}: asset is empty")
-        driver_type, revision = probe_revision(child)
+        asset_entries.append((child, driver, platform))
+        drivers_by_platform.setdefault(platform, set()).add(driver)
+
+    revisions_by_platform = generate_platform_revisions(root, drivers_by_platform)
+
+    manifest = {
+        "schemaVersion": 1,
+        "generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root),
+        "assets": {},
+    }
+
+    for child, driver, platform in asset_entries:
+        normalized_driver = normalize_driver(driver)
+        revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
+        if not revision:
+            raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
         manifest["assets"][child.name] = {
             "driver": driver,
-            "driverType": driver_type or driver,
+            "driverType": driver,
             "platform": platform,
             "revision": revision,
             "size": child.stat().st_size,
@@ -94,6 +163,10 @@ def main():
 if __name__ == "__main__":
     try:
         raise SystemExit(main())
-    except subprocess.TimeoutExpired as exc:
-        print(f"error: probe timed out for {exc.cmd}", file=sys.stderr)
+    except subprocess.CalledProcessError as exc:
+        command = exc.cmd if isinstance(exc.cmd, str) else " ".join(exc.cmd)
+        stderr = (exc.stderr or "").strip()
+        if stderr:
+            print(stderr, file=sys.stderr)
+        print(f"error: command failed: {command}", file=sys.stderr)
         raise

tools/generate-driver-release-manifest.test.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import stat
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent.parent
+SCRIPT = ROOT / "tools" / "generate-driver-release-manifest.py"
+
+
+def expected_revision(revision_file: Path, driver: str):
+    for line in revision_file.read_text(encoding="utf-8").splitlines():
+        if f'"{driver}"' not in line:
+            continue
+        left, right = line.strip().rstrip(",").split(":", 1)
+        if left.strip().strip('"') == driver:
+            return right.strip().strip('"')
+    raise AssertionError(f"missing revision for {driver}")
+
+
+class GenerateDriverReleaseManifestTest(unittest.TestCase):
+    def _generate_revision_file(self, platform: str):
+        worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-"))
+        subprocess.run(
+            ["git", "worktree", "add", "--detach", str(worktree), "HEAD"],
+            cwd=ROOT,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=True,
+        )
+        self.addCleanup(
+            lambda: subprocess.run(
+                ["git", "worktree", "remove", "--force", str(worktree)],
+                cwd=ROOT,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=False,
+            )
+        )
+        subprocess.run(
+            ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform, "--drivers", "clickhouse"],
+            cwd=worktree,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=True,
+        )
+        return worktree / "internal" / "db" / "driver_agent_revisions_gen.go"
+
+    def test_generates_manifest_without_executing_cross_platform_assets(self):
+        with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp:
+            tmpdir = Path(tmp)
+            assets_dir = tmpdir / "drivers"
+            (assets_dir / "MacOS").mkdir(parents=True)
+            (assets_dir / "Linux").mkdir(parents=True)
+            (assets_dir / "Windows").mkdir(parents=True)
+
+            fixtures = {
+                assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary",
+                assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary",
+                assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary",
+            }
+            for path, content in fixtures.items():
+                path.write_bytes(content)
+                os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+
+            output = tmpdir / "manifest.json"
+            proc = subprocess.run(
+                ["python3", str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
+                cwd=ROOT,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
+
+            self.assertIn("asset count: 3", proc.stdout)
+            manifest = json.loads(output.read_text(encoding="utf-8"))
+            assets = manifest["assets"]
+            darwin_revision_file = self._generate_revision_file("darwin/arm64")
+            linux_revision_file = self._generate_revision_file("linux/amd64")
+            windows_revision_file = self._generate_revision_file("windows/amd64")
+            self.assertEqual(
+                assets["clickhouse-driver-agent-darwin-arm64"]["revision"],
+                expected_revision(darwin_revision_file, "clickhouse"),
+            )
+            self.assertEqual(
+                assets["clickhouse-driver-agent-linux-amd64"]["revision"],
+                expected_revision(linux_revision_file, "clickhouse"),
+            )
+            self.assertEqual(
+                assets["clickhouse-driver-agent-windows-amd64.exe"]["revision"],
+                expected_revision(windows_revision_file, "clickhouse"),
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/package-driver-release-assets.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import json
+import shutil
+import sys
+import zipfile
+from pathlib import Path
+
+
+def main():
+    if len(sys.argv) != 3:
+        raise SystemExit("usage: package-driver-release-assets.py <drivers-dir> <output-dir>")
+
+    drivers_dir = Path(sys.argv[1]).resolve()
+    output_dir = Path(sys.argv[2]).resolve()
+
+    if not drivers_dir.is_dir():
+        raise SystemExit(f"drivers dir not found: {drivers_dir}")
+
+    out_name = "GoNavi-DriverAgents.zip"
+    index_name = "GoNavi-DriverAgents-Index.json"
+    manifest_name = "GoNavi-DriverAgents-Manifest.json"
+
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    out_path = output_dir / out_name
+    index_path = output_dir / index_name
+    manifest_path = output_dir / manifest_name
+
+    size_index = {}
+    standalone_assets = []
+    with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for asset in sorted(drivers_dir.rglob("*")):
+            if not asset.is_file():
+                continue
+            arcname = asset.relative_to(drivers_dir).as_posix()
+            if asset.name in size_index:
+                raise RuntimeError(f"driver asset name conflict: {asset.name}")
+            zf.write(asset, arcname)
+            size_index[asset.name] = asset.stat().st_size
+            standalone_path = output_dir / asset.name
+            if standalone_path.exists():
+                raise RuntimeError(f"release asset already exists: {standalone_path}")
+            shutil.copy2(asset, standalone_path)
+            standalone_assets.append(standalone_path.name)
+
+    index_path.write_text(
+        json.dumps({"assets": size_index}, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    print(f"created {out_name} size={out_path.stat().st_size} bytes")
+    print(f"created {index_name} entries={len(size_index)}")
+    print(f"published standalone driver assets={len(standalone_assets)}")
+    print(f"reserved manifest output path: {manifest_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/package-driver-release-assets.test.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent.parent
+SCRIPT = ROOT / "tools" / "package-driver-release-assets.py"
+
+
+class PackageDriverReleaseAssetsTest(unittest.TestCase):
+    def test_packages_bundle_and_standalone_assets(self):
+        with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp:
+            tmpdir = Path(tmp)
+            drivers_dir = tmpdir / "drivers"
+            output_dir = tmpdir / "driver-release-assets"
+            (drivers_dir / "Windows").mkdir(parents=True)
+            (drivers_dir / "MacOS").mkdir(parents=True)
+
+            windows_asset = drivers_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe"
+            darwin_asset = drivers_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64"
+            windows_asset.write_bytes(b"windows-asset")
+            darwin_asset.write_bytes(b"darwin-asset")
+
+            proc = subprocess.run(
+                ["python3", str(SCRIPT), str(drivers_dir), str(output_dir)],
+                cwd=ROOT,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
+
+            self.assertIn("created GoNavi-DriverAgents.zip", proc.stdout)
+            self.assertTrue((output_dir / "GoNavi-DriverAgents.zip").is_file())
+            self.assertTrue((output_dir / windows_asset.name).is_file())
+            self.assertTrue((output_dir / darwin_asset.name).is_file())
+
+            index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8"))
+            self.assertEqual(index["assets"][windows_asset.name], len(b"windows-asset"))
+            self.assertEqual(index["assets"][darwin_asset.name], len(b"darwin-asset"))
+
+            with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf:
+                self.assertEqual(
+                    sorted(zf.namelist()),
+                    [
+                        "MacOS/clickhouse-driver-agent-darwin-arm64",
+                        "Windows/clickhouse-driver-agent-windows-amd64.exe",
+                    ],
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()