From ea53430d70872651c13940ff1e2a5583eab8628f Mon Sep 17 00:00:00 2001 From: Syngnat Date: Fri, 5 Jun 2026 16:53:01 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E6=80=BB=E5=8C=85=E6=89=93=E5=8C=85=E4=B8=8E?= =?UTF-8?q?=20manifest=20=E7=94=9F=E6=88=90=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 抽离 driver release 打包脚本，避免内联脚本删掉已生成 manifest - 修复 release-assets 场景下 tools 相对路径错误，统一通过仓库根脚本打包 - manifest 改为按源码和目标平台重算 revision，不再执行跨平台 driver 二进制 - 补充 driver release 打包与 manifest 生成回归测试 --- .github/workflows/dev-build.yml | 56 +------- .github/workflows/release.yml | 56 +------- tools/generate-driver-release-manifest.py | 131 ++++++++++++++---- .../generate-driver-release-manifest.test.py | 102 ++++++++++++++ tools/package-driver-release-assets.py | 62 +++++++++ tools/package-driver-release-assets.test.py | 58 ++++++++ 6 files changed, 336 insertions(+), 129 deletions(-) create mode 100644 tools/generate-driver-release-manifest.test.py create mode 100644 tools/package-driver-release-assets.py create mode 100644 tools/package-driver-release-assets.test.py diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 9963e96..1f42e14 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -836,58 +836,14 @@ jobs: exit 0 fi - python3 tools/generate-driver-release-manifest.py \ - --assets-dir . \ - --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json - echo "📦 打包驱动总包：GoNavi-DriverAgents.zip" - python3 - <<'PY' - import json - import os - import shutil - import zipfile - from pathlib import Path + python3 ../tools/package-driver-release-assets.py \ + drivers \ + ../driver-release-assets - out_name = "GoNavi-DriverAgents.zip" - index_name = "GoNavi-DriverAgents-Index.json" - base = Path("drivers") - driver_release_dir = Path("../driver-release-assets") - if driver_release_dir.exists(): - shutil.rmtree(driver_release_dir) - driver_release_dir.mkdir(parents=True, exist_ok=True) - out_path = driver_release_dir / out_name - index_path = driver_release_dir / index_name - if out_path.exists(): - out_path.unlink() - if index_path.exists(): - index_path.unlink() - - size_index = {} - standalone_assets = [] - with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for p in sorted(base.rglob("*")): - if not p.is_file(): - continue - arcname = p.relative_to(base).as_posix() - if p.name in size_index: - raise RuntimeError(f"driver asset name conflict: {p.name}") - zf.write(p, arcname) - size_index[p.name] = p.stat().st_size - standalone_path = driver_release_dir / p.name - if standalone_path.exists(): - raise RuntimeError(f"release asset already exists: {standalone_path}") - shutil.copy2(p, standalone_path) - standalone_assets.append(standalone_path.name) - - index_path.write_text( - json.dumps({"assets": size_index}, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - print(f"created {out_name} size={out_path.stat().st_size} bytes") - print(f"created {index_name} entries={len(size_index)}") - print(f"published standalone driver assets={len(standalone_assets)}") - PY + python3 ../tools/generate-driver-release-manifest.py \ + --assets-dir drivers \ + --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json rm -rf drivers echo "has_driver_assets=true" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec68e25..21f1605 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -893,58 +893,14 @@ jobs: exit 0 fi - python3 tools/generate-driver-release-manifest.py \ - --assets-dir . \ - --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json - echo "📦 打包驱动总包：GoNavi-DriverAgents.zip" - python3 - <<'PY' - import json - import os - import shutil - import zipfile - from pathlib import Path + python3 ../tools/package-driver-release-assets.py \ + drivers \ + ../driver-release-assets - out_name = "GoNavi-DriverAgents.zip" - index_name = "GoNavi-DriverAgents-Index.json" - base = Path("drivers") - driver_release_dir = Path("../driver-release-assets") - if driver_release_dir.exists(): - shutil.rmtree(driver_release_dir) - driver_release_dir.mkdir(parents=True, exist_ok=True) - out_path = driver_release_dir / out_name - index_path = driver_release_dir / index_name - if out_path.exists(): - out_path.unlink() - if index_path.exists(): - index_path.unlink() - - size_index = {} - standalone_assets = [] - with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for p in sorted(base.rglob("*")): - if not p.is_file(): - continue - arcname = p.relative_to(base).as_posix() - if p.name in size_index: - raise RuntimeError(f"driver asset name conflict: {p.name}") - zf.write(p, arcname) - size_index[p.name] = p.stat().st_size - standalone_path = driver_release_dir / p.name - if standalone_path.exists(): - raise RuntimeError(f"release asset already exists: {standalone_path}") - shutil.copy2(p, standalone_path) - standalone_assets.append(standalone_path.name) - - index_path.write_text( - json.dumps({"assets": size_index}, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - print(f"created {out_name} size={out_path.stat().st_size} bytes") - print(f"created {index_name} entries={len(size_index)}") - print(f"published standalone driver assets={len(standalone_assets)}") - PY + python3 ../tools/generate-driver-release-manifest.py \ + --assets-dir drivers \ + --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json # GoNavi 主仓库只保留主程序包；驱动资产发布到独立仓库。 rm -rf drivers diff --git a/tools/generate-driver-release-manifest.py b/tools/generate-driver-release-manifest.py index c3ca55e..69f3bfd 100644 --- a/tools/generate-driver-release-manifest.py +++ b/tools/generate-driver-release-manifest.py @@ -5,6 +5,7 @@ import json import os import subprocess import sys +import tempfile from pathlib import Path @@ -26,47 +27,100 @@ def infer_driver_and_platform(file_name: str): for suffix in suffixes: if file_name.endswith(suffix): driver = file_name[: -len(suffix)] - if suffix.endswith(".exe"): - platform = suffix.replace("-driver-agent-", "").removesuffix(".exe") - else: - platform = suffix.replace("-driver-agent-", "") + platform_name = suffix.replace("-driver-agent-", "") + if platform_name.endswith(".exe"): + platform_name = platform_name.removesuffix(".exe") + platform = platform_name.replace("-", "/", 1) return driver, platform return None, None -def probe_revision(path: Path): - request = b'{"id":1,"method":"metadata"}\n' +def normalize_driver(driver: str): + value = str(driver or "").strip().lower() + if value == "doris": + return "diros" + return value + + +def repo_root(): + return Path(__file__).resolve().parent.parent + + +def resolve_head_commit(root: Path): proc = subprocess.run( - [str(path)], - input=request, + ["git", "rev-parse", "HEAD"], + cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=10, + text=True, check=True, ) - line = proc.stdout.decode("utf-8", errors="replace").strip().splitlines() - if not line: - raise RuntimeError(f"{path.name}: metadata response is empty") - payload = json.loads(line[0]) - data = payload.get("data") or {} - revision = str(data.get("agentRevision") or "").strip() - driver_type = str(data.get("driverType") or "").strip() - if not revision: - raise RuntimeError(f"{path.name}: metadata agentRevision is empty") - return driver_type, revision + return proc.stdout.strip() + + +def parse_revision_file(path: Path): + revisions = {} + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped.startswith('"'): + continue + try: + driver, revision = stripped.rstrip(",").split(":", 1) + except ValueError: + continue + revisions[driver.strip().strip('"')] = revision.strip().strip('"') + return revisions + + +def generate_platform_revisions(root: Path, drivers_by_platform): + if not drivers_by_platform: + return {} + + with tempfile.TemporaryDirectory(prefix="gonavi-driver-release-manifest-") as tmp: + worktree = Path(tmp) / "worktree" + subprocess.run( + ["git", "worktree", "add", "--detach", str(worktree), "HEAD"], + cwd=root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + try: + revision_file = worktree / "internal/db/driver_agent_revisions_gen.go" + result = {} + for platform in sorted(drivers_by_platform): + drivers = sorted({normalize_driver(driver) for driver in drivers_by_platform[platform] if normalize_driver(driver)}) + command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform] + if drivers: + command.extend(["--drivers", ",".join(drivers)]) + subprocess.run( + command, + cwd=worktree, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + result[platform] = parse_revision_file(revision_file) + return result + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) def main(): args = parse_args() assets_dir = Path(args.assets_dir).resolve() output_path = Path(args.output).resolve() + root = repo_root() - manifest = { - "schemaVersion": 1, - "generatedFrom": os.environ.get("GITHUB_SHA", "").strip(), - "assets": {}, - } - + asset_entries = [] + drivers_by_platform = {} for child in sorted(assets_dir.rglob("*")): if not child.is_file(): continue @@ -75,10 +129,25 @@ def main(): continue if child.stat().st_size == 0: raise RuntimeError(f"{child.name}: asset is empty") - driver_type, revision = probe_revision(child) + asset_entries.append((child, driver, platform)) + drivers_by_platform.setdefault(platform, set()).add(driver) + + revisions_by_platform = generate_platform_revisions(root, drivers_by_platform) + + manifest = { + "schemaVersion": 1, + "generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root), + "assets": {}, + } + + for child, driver, platform in asset_entries: + normalized_driver = normalize_driver(driver) + revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip() + if not revision: + raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}") manifest["assets"][child.name] = { "driver": driver, - "driverType": driver_type or driver, + "driverType": driver, "platform": platform, "revision": revision, "size": child.stat().st_size, @@ -94,6 +163,10 @@ def main(): if __name__ == "__main__": try: raise SystemExit(main()) - except subprocess.TimeoutExpired as exc: - print(f"error: probe timed out for {exc.cmd}", file=sys.stderr) + except subprocess.CalledProcessError as exc: + command = exc.cmd if isinstance(exc.cmd, str) else " ".join(exc.cmd) + stderr = (exc.stderr or "").strip() + if stderr: + print(stderr, file=sys.stderr) + print(f"error: command failed: {command}", file=sys.stderr) raise diff --git a/tools/generate-driver-release-manifest.test.py b/tools/generate-driver-release-manifest.test.py new file mode 100644 index 0000000..3b1fe6d --- /dev/null +++ b/tools/generate-driver-release-manifest.test.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import json +import os +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "tools" / "generate-driver-release-manifest.py" + + +def expected_revision(revision_file: Path, driver: str): + for line in revision_file.read_text(encoding="utf-8").splitlines(): + if f'"{driver}"' not in line: + continue + left, right = line.strip().rstrip(",").split(":", 1) + if left.strip().strip('"') == driver: + return right.strip().strip('"') + raise AssertionError(f"missing revision for {driver}") + + +class GenerateDriverReleaseManifestTest(unittest.TestCase): + def _generate_revision_file(self, platform: str): + worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-")) + subprocess.run( + ["git", "worktree", "add", "--detach", str(worktree), "HEAD"], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + self.addCleanup( + lambda: subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + ) + subprocess.run( + ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform, "--drivers", "clickhouse"], + cwd=worktree, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return worktree / "internal" / "db" / "driver_agent_revisions_gen.go" + + def test_generates_manifest_without_executing_cross_platform_assets(self): + with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + (assets_dir / "MacOS").mkdir(parents=True) + (assets_dir / "Linux").mkdir(parents=True) + (assets_dir / "Windows").mkdir(parents=True) + + fixtures = { + assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary", + assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary", + assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary", + } + for path, content in fixtures.items(): + path.write_bytes(content) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + + output = tmpdir / "manifest.json" + proc = subprocess.run( + ["python3", str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + + self.assertIn("asset count: 3", proc.stdout) + manifest = json.loads(output.read_text(encoding="utf-8")) + assets = manifest["assets"] + darwin_revision_file = self._generate_revision_file("darwin/arm64") + linux_revision_file = self._generate_revision_file("linux/amd64") + windows_revision_file = self._generate_revision_file("windows/amd64") + self.assertEqual( + assets["clickhouse-driver-agent-darwin-arm64"]["revision"], + expected_revision(darwin_revision_file, "clickhouse"), + ) + self.assertEqual( + assets["clickhouse-driver-agent-linux-amd64"]["revision"], + expected_revision(linux_revision_file, "clickhouse"), + ) + self.assertEqual( + assets["clickhouse-driver-agent-windows-amd64.exe"]["revision"], + expected_revision(windows_revision_file, "clickhouse"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/package-driver-release-assets.py b/tools/package-driver-release-assets.py new file mode 100644 index 0000000..7afb4e4 --- /dev/null +++ b/tools/package-driver-release-assets.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import json +import shutil +import sys +import zipfile +from pathlib import Path + + +def main(): + if len(sys.argv) != 3: + raise SystemExit("usage: package-driver-release-assets.py ") + + drivers_dir = Path(sys.argv[1]).resolve() + output_dir = Path(sys.argv[2]).resolve() + + if not drivers_dir.is_dir(): + raise SystemExit(f"drivers dir not found: {drivers_dir}") + + out_name = "GoNavi-DriverAgents.zip" + index_name = "GoNavi-DriverAgents-Index.json" + manifest_name = "GoNavi-DriverAgents-Manifest.json" + + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + out_path = output_dir / out_name + index_path = output_dir / index_name + manifest_path = output_dir / manifest_name + + size_index = {} + standalone_assets = [] + with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for asset in sorted(drivers_dir.rglob("*")): + if not asset.is_file(): + continue + arcname = asset.relative_to(drivers_dir).as_posix() + if asset.name in size_index: + raise RuntimeError(f"driver asset name conflict: {asset.name}") + zf.write(asset, arcname) + size_index[asset.name] = asset.stat().st_size + standalone_path = output_dir / asset.name + if standalone_path.exists(): + raise RuntimeError(f"release asset already exists: {standalone_path}") + shutil.copy2(asset, standalone_path) + standalone_assets.append(standalone_path.name) + + index_path.write_text( + json.dumps({"assets": size_index}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + print(f"created {out_name} size={out_path.stat().st_size} bytes") + print(f"created {index_name} entries={len(size_index)}") + print(f"published standalone driver assets={len(standalone_assets)}") + print(f"reserved manifest output path: {manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/package-driver-release-assets.test.py b/tools/package-driver-release-assets.test.py new file mode 100644 index 0000000..1f7a939 --- /dev/null +++ b/tools/package-driver-release-assets.test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import tempfile +import unittest +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "tools" / "package-driver-release-assets.py" + + +class PackageDriverReleaseAssetsTest(unittest.TestCase): + def test_packages_bundle_and_standalone_assets(self): + with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp: + tmpdir = Path(tmp) + drivers_dir = tmpdir / "drivers" + output_dir = tmpdir / "driver-release-assets" + (drivers_dir / "Windows").mkdir(parents=True) + (drivers_dir / "MacOS").mkdir(parents=True) + + windows_asset = drivers_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe" + darwin_asset = drivers_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64" + windows_asset.write_bytes(b"windows-asset") + darwin_asset.write_bytes(b"darwin-asset") + + proc = subprocess.run( + ["python3", str(SCRIPT), str(drivers_dir), str(output_dir)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + + self.assertIn("created GoNavi-DriverAgents.zip", proc.stdout) + self.assertTrue((output_dir / "GoNavi-DriverAgents.zip").is_file()) + self.assertTrue((output_dir / windows_asset.name).is_file()) + self.assertTrue((output_dir / darwin_asset.name).is_file()) + + index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8")) + self.assertEqual(index["assets"][windows_asset.name], len(b"windows-asset")) + self.assertEqual(index["assets"][darwin_asset.name], len(b"darwin-asset")) + + with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf: + self.assertEqual( + sorted(zf.namelist()), + [ + "MacOS/clickhouse-driver-agent-darwin-arm64", + "Windows/clickhouse-driver-agent-windows-amd64.exe", + ], + ) + + +if __name__ == "__main__": + unittest.main()