diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 6cba5bdb..611e00c8 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -832,6 +832,11 @@ jobs: --platform "$TARGET_PLATFORM" \ --drivers "$NORMALIZED_CHANGED_DRIVER_AGENTS" + mkdir -p driver-provenance + python3 ./tools/generate-driver-release-manifest.py \ + --assets-dir drivers \ + --provenance-output "driver-provenance/${GOOS}-${GOARCH}.json" + # macOS Packaging - name: Package macOS DMG if: contains(matrix.platform, 'darwin') @@ -1094,6 +1099,7 @@ jobs: GoNavi-*.tar.gz GoNavi-*.AppImage drivers/** + driver-provenance/** retention-days: 7 # 汇总所有产物并发布为 Pre-release @@ -1121,6 +1127,22 @@ jobs: - name: List Assets run: ls -R release-assets + - name: Download Previous Driver Manifest + if: needs.driver_agents.outputs.has_changes == 'true' && needs.driver_agents.outputs.release_source != 'all' + env: + DRIVER_RELEASE_SOURCE: ${{ needs.driver_agents.outputs.release_source }} + shell: bash + run: | + set -euo pipefail + manifest_path="$RUNNER_TEMP/previous-driver-manifest.json" + manifest_url="https://github.com/Syngnat/GoNavi-DriverAgents/releases/download/${DRIVER_RELEASE_SOURCE}/GoNavi-DriverAgents-Manifest.json" + if curl --retry 3 --retry-delay 2 --retry-all-errors -fsSL "$manifest_url" -o "$manifest_path"; then + echo "🧭 已下载上一版 driver manifest 作为复用资产 provenance" + else + rm -f "$manifest_path" + echo "⚠️ 上一版 driver manifest 不可用；未重建资产将由 manifest 生成阶段拒绝" + fi + - name: Complete Driver Agent Assets if: needs.driver_agents.outputs.has_changes == 'true' env: @@ -1154,11 +1176,19 @@ jobs: drivers \ ../driver-release-assets - python3 ../tools/generate-driver-release-manifest.py \ - --assets-dir drivers \ + MANIFEST_ARGS=( + --assets-dir drivers --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json + ) + if [ -d driver-provenance ]; then + MANIFEST_ARGS+=(--provenance driver-provenance) + fi + if [ -s "$RUNNER_TEMP/previous-driver-manifest.json" ]; then + MANIFEST_ARGS+=(--provenance "$RUNNER_TEMP/previous-driver-manifest.json") + fi + python3 ../tools/generate-driver-release-manifest.py "${MANIFEST_ARGS[@]}" - rm -rf drivers + rm -rf drivers driver-provenance echo "has_driver_assets=true" >> "$GITHUB_OUTPUT" - name: Generate SHA256SUMS diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b2fe97da..e9118017 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -774,6 +774,11 @@ jobs: --platform "$TARGET_PLATFORM" \ --drivers "$NORMALIZED_CHANGED_DRIVER_AGENTS" + mkdir -p driver-provenance + python3 ./tools/generate-driver-release-manifest.py \ + --assets-dir drivers \ + --provenance-output "driver-provenance/${GOOS}-${GOARCH}.json" + # macOS Packaging - name: Package macOS DMG if: contains(matrix.platform, 'darwin') @@ -1059,6 +1064,7 @@ jobs: GoNavi-*.tar.gz GoNavi-*.AppImage drivers/** + driver-provenance/** retention-days: 1 # Phase 2: Collect all artifacts and Publish Release (Single Job) @@ -1086,6 +1092,22 @@ jobs: - name: List Assets run: ls -R release-assets + - name: Download Previous Driver Manifest + if: needs.driver_agents.outputs.has_changes == 'true' && needs.driver_agents.outputs.release_source != 'all' + env: + DRIVER_RELEASE_SOURCE: ${{ needs.driver_agents.outputs.release_source }} + shell: bash + run: | + set -euo pipefail + manifest_path="$RUNNER_TEMP/previous-driver-manifest.json" + manifest_url="https://github.com/Syngnat/GoNavi-DriverAgents/releases/download/${DRIVER_RELEASE_SOURCE}/GoNavi-DriverAgents-Manifest.json" + if curl --retry 3 --retry-delay 2 --retry-all-errors -fsSL "$manifest_url" -o "$manifest_path"; then + echo "🧭 已下载上一版 driver manifest 作为复用资产 provenance" + else + rm -f "$manifest_path" + echo "⚠️ 上一版 driver manifest 不可用；未重建资产将由 manifest 生成阶段拒绝" + fi + - name: Complete Driver Agent Assets if: needs.driver_agents.outputs.has_changes == 'true' && needs.driver_agents.outputs.release_source != 'all' env: @@ -1159,12 +1181,20 @@ jobs: drivers \ ../driver-release-assets - python3 ../tools/generate-driver-release-manifest.py \ - --assets-dir drivers \ + MANIFEST_ARGS=( + --assets-dir drivers --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json + ) + if [ -d driver-provenance ]; then + MANIFEST_ARGS+=(--provenance driver-provenance) + fi + if [ -s "$RUNNER_TEMP/previous-driver-manifest.json" ]; then + MANIFEST_ARGS+=(--provenance "$RUNNER_TEMP/previous-driver-manifest.json") + fi + python3 ../tools/generate-driver-release-manifest.py "${MANIFEST_ARGS[@]}" # GoNavi 主仓库只保留主程序包；驱动资产发布到独立仓库。 - rm -rf drivers + rm -rf drivers driver-provenance echo "has_driver_assets=true" >> "$GITHUB_OUTPUT" - name: Generate SHA256SUMS diff --git a/tools/generate-driver-release-manifest.py b/tools/generate-driver-release-manifest.py index cb9873ae..3833b30f 100644 --- a/tools/generate-driver-release-manifest.py +++ b/tools/generate-driver-release-manifest.py @@ -15,8 +15,19 @@ from pathlib import Path def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets") - parser.add_argument("--output", required=True, help="manifest json output path") - return parser.parse_args() + parser.add_argument("--output", help="release manifest json output path") + parser.add_argument( + "--provenance", + action="append", + default=[], + help="manifest/provenance JSON file or directory; may be passed more than once", + ) + parser.add_argument("--provenance-output", help="write SHA-bound build provenance and exit when --output is omitted") + parser.add_argument("--revision-file", help="generated driver revision file used by --provenance-output") + args = parser.parse_args() + if not args.output and not args.provenance_output: + parser.error("one of --output or --provenance-output is required") + return args def infer_driver_and_platform(file_name: str): @@ -127,6 +138,174 @@ def parse_revision_file(path: Path): return revisions +def resolve_host_platform(): + goos = subprocess.run( + ["go", "env", "GOOS"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip() + goarch = subprocess.run( + ["go", "env", "GOARCH"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip() + return f"{goos}/{goarch}" + + +def probe_agent_metadata(asset_path: Path): + if os.name != "nt": + os.chmod(asset_path, asset_path.stat().st_mode | 0o111) + proc = subprocess.run( + [str(asset_path)], + input='{"id":1,"method":"metadata"}\n', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=30, + ) + if proc.returncode != 0: + detail = proc.stderr.strip() or f"exit code {proc.returncode}" + raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}") + lines = [line.strip() for line in proc.stdout.splitlines() if line.strip()] + if not lines: + raise RuntimeError(f"{asset_path.name}: metadata probe returned no response") + try: + response = json.loads(lines[0]) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{asset_path.name}: metadata probe returned invalid JSON: {exc}") from exc + if not response.get("success"): + detail = str(response.get("error") or "metadata request failed").strip() + raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}") + data = response.get("data") or {} + driver_type = normalize_driver(data.get("driverType")) + revision = str(data.get("agentRevision") or "").strip() + if not driver_type or not revision: + raise RuntimeError(f"{asset_path.name}: metadata response is missing driverType or agentRevision") + return driver_type, revision + + +def load_asset_provenance(paths): + entries = {} + for raw_path in paths: + source = Path(raw_path).resolve() + if source.is_dir(): + files = sorted(source.rglob("*.json")) + elif source.is_file(): + files = [source] + else: + raise RuntimeError(f"binary revision provenance path does not exist: {source}") + for file_path in files: + try: + payload = json.loads(file_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError(f"failed to read binary revision provenance {file_path}: {exc}") from exc + assets = payload.get("assets") or {} + if not isinstance(assets, dict): + raise RuntimeError(f"binary revision provenance {file_path} has invalid assets") + for asset_name, metadata in assets.items(): + if not isinstance(metadata, dict): + continue + name = str(asset_name or "").strip() + if not name: + continue + entries.setdefault(name, []).append((file_path, metadata)) + return entries + + +def resolve_asset_provenance(entries, asset_name, driver, platform, sha256, size): + candidates = entries.get(asset_name) or [] + matching = [] + for source, metadata in candidates: + recorded_sha = str(metadata.get("sha256") or "").strip().lower() + if recorded_sha == sha256.lower(): + matching.append((source, metadata)) + if not matching: + if candidates: + recorded = sorted( + { + str(metadata.get("sha256") or "").strip() or "" + for _, metadata in candidates + } + ) + raise RuntimeError( + f"{asset_name}: binary revision provenance SHA256 mismatch: " + f"asset={sha256} recorded={','.join(recorded)}" + ) + raise RuntimeError( + f"{asset_name}: binary revision provenance is required for " + f"cross-platform asset {platform}" + ) + + revisions = set() + for source, metadata in matching: + recorded_driver = normalize_driver(metadata.get("driver") or metadata.get("driverType")) + recorded_platform = str(metadata.get("platform") or "").strip() + recorded_revision = str(metadata.get("revision") or "").strip() + recorded_size = metadata.get("size") + if recorded_driver != driver or recorded_platform != platform: + raise RuntimeError( + f"{asset_name}: binary revision provenance identity mismatch in {source}: " + f"recorded={recorded_platform}/{recorded_driver} expected={platform}/{driver}" + ) + if recorded_size is not None and int(recorded_size) != size: + raise RuntimeError( + f"{asset_name}: binary revision provenance size mismatch in {source}: " + f"recorded={recorded_size} expected={size}" + ) + if not recorded_revision: + raise RuntimeError(f"{asset_name}: binary revision provenance is missing revision in {source}") + revisions.add(recorded_revision) + if len(revisions) != 1: + raise RuntimeError( + f"{asset_name}: conflicting binary revision provenance: {','.join(sorted(revisions))}" + ) + return revisions.pop() + + +def write_build_provenance(asset_entries, revision_file: Path, output_path: Path, generated_from: str): + revisions = parse_revision_file(revision_file) + host_platform = resolve_host_platform() + assets = {} + for child, driver, platform in asset_entries: + normalized_driver = normalize_driver(driver) + revision = str(revisions.get(normalized_driver) or "").strip() + if not revision: + raise RuntimeError( + f"{child.name}: missing build revision for {platform}/{normalized_driver} in {revision_file}" + ) + if platform == host_platform: + binary_driver, binary_revision = probe_agent_metadata(child) + if binary_driver != normalized_driver or binary_revision != revision: + raise RuntimeError( + f"{child.name}: build provenance metadata mismatch: " + f"binary={binary_driver}/{binary_revision} " + f"expected={normalized_driver}/{revision}" + ) + revision = binary_revision + size = child.stat().st_size + assets[child.name] = { + "driver": driver, + "driverType": driver, + "platform": platform, + "revision": revision, + "size": size, + "sha256": hashlib.sha256(child.read_bytes()).hexdigest(), + } + payload = { + "schemaVersion": 1, + "generatedFrom": generated_from, + "assets": assets, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"wrote build provenance: {output_path}") + print(f"asset count: {len(assets)}") + + def generate_platform_revisions(root: Path, drivers_by_platform): if not drivers_by_platform: return {} @@ -170,8 +349,8 @@ def generate_platform_revisions(root: Path, drivers_by_platform): def main(): args = parse_args() assets_dir = Path(args.assets_dir).resolve() - output_path = Path(args.output).resolve() root = repo_root() + generated_from = os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root) asset_entries = [] drivers_by_platform = {} @@ -186,26 +365,70 @@ def main(): asset_entries.append((child, driver, platform)) drivers_by_platform.setdefault(platform, set()).add(driver) + if args.provenance_output: + revision_file = Path(args.revision_file).resolve() if args.revision_file else root / "internal" / "db" / "driver_agent_revisions_gen.go" + write_build_provenance( + asset_entries, + revision_file, + Path(args.provenance_output).resolve(), + generated_from, + ) + if not args.output: + return 0 + + output_path = Path(args.output).resolve() + provenance_entries = load_asset_provenance(args.provenance) revisions_by_platform = generate_platform_revisions(root, drivers_by_platform) + host_platform = resolve_host_platform() manifest = { "schemaVersion": 1, - "generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root), + "generatedFrom": generated_from, "assets": {}, } for child, driver, platform in asset_entries: normalized_driver = normalize_driver(driver) + size = child.stat().st_size + sha256 = hashlib.sha256(child.read_bytes()).hexdigest() revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip() if not revision: raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}") + if platform == host_platform: + binary_driver, binary_revision = probe_agent_metadata(child) + if binary_driver != normalized_driver: + raise RuntimeError( + f"{child.name}: embedded driver type mismatch: " + f"binary={binary_driver} expected={normalized_driver}" + ) + if binary_revision != revision: + raise RuntimeError( + f"{child.name}: embedded revision mismatch: " + f"binary={binary_revision} expected={revision}" + ) + revision = binary_revision + else: + binary_revision = resolve_asset_provenance( + provenance_entries, + child.name, + normalized_driver, + platform, + sha256, + size, + ) + if binary_revision != revision: + raise RuntimeError( + f"{child.name}: provenance revision mismatch: " + f"binary={binary_revision} expected={revision}" + ) + revision = binary_revision manifest["assets"][child.name] = { "driver": driver, "driverType": driver, "platform": platform, "revision": revision, - "size": child.stat().st_size, - "sha256": hashlib.sha256(child.read_bytes()).hexdigest(), + "size": size, + "sha256": sha256, } output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tools/generate-driver-release-manifest.test.py b/tools/generate-driver-release-manifest.test.py index 35e39bd9..07ce13d5 100644 --- a/tools/generate-driver-release-manifest.test.py +++ b/tools/generate-driver-release-manifest.test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json +import hashlib import os import shlex import shutil @@ -70,6 +71,255 @@ def expected_revision(revision_file: Path, driver: str): class GenerateDriverReleaseManifestTest(unittest.TestCase): + def _host_platform(self): + goos = subprocess.run( + ["go", "env", "GOOS"], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip() + goarch = subprocess.run( + ["go", "env", "GOARCH"], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ).stdout.strip() + return goos, goarch + + def _build_metadata_agent(self, output: Path, revision: str): + source = output.parent / "metadata-agent.go" + source.write_text( + "package main\n" + "import (\"bufio\"; \"fmt\"; \"os\")\n" + "func main() {\n" + " scanner := bufio.NewScanner(os.Stdin)\n" + " for scanner.Scan() {\n" + f" fmt.Println(`{{\"id\":1,\"success\":true,\"data\":{{\"driverType\":\"clickhouse\",\"agentRevision\":\"{revision}\",\"protocolSchema\":\"json-lines-v1\"}}}}`)\n" + " }\n" + "}\n", + encoding="utf-8", + ) + subprocess.run( + ["go", "build", "-o", str(output), str(source)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + + def _cross_platform(self): + host = "/".join(self._host_platform()) + for platform in ("linux/amd64", "darwin/arm64", "windows/amd64"): + if platform != host: + return platform + raise AssertionError(f"unable to select cross platform for {host}") + + def test_rejects_native_asset_with_stale_embedded_revision(self): + goos, goarch = self._host_platform() + extension = ".exe" if goos == "windows" else "" + with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-agent-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + assets_dir.mkdir(parents=True) + asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}" + self._build_metadata_agent(asset, "src-stale-agent") + + output = tmpdir / "manifest.json" + proc = subprocess.run( + [sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(proc.returncode, 0, proc.stdout) + self.assertIn("src-stale-agent", proc.stderr) + self.assertFalse(output.exists()) + + def test_rejects_cross_platform_asset_without_binary_revision_provenance(self): + goos, goarch = self._cross_platform().split("/", 1) + extension = ".exe" if goos == "windows" else "" + with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-cross-agent-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + assets_dir.mkdir(parents=True) + asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}" + asset.write_bytes(b"cross-platform-driver-agent") + + output = tmpdir / "manifest.json" + proc = subprocess.run( + [sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(proc.returncode, 0, proc.stdout) + self.assertIn("binary revision provenance", proc.stderr) + self.assertFalse(output.exists()) + + def test_accepts_cross_platform_asset_with_matching_sha_bound_provenance(self): + platform = self._cross_platform() + goos, goarch = platform.split("/", 1) + extension = ".exe" if goos == "windows" else "" + revision_file = self._generate_revision_file(platform) + revision = expected_revision(revision_file, "clickhouse") + + with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-provenance-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + assets_dir.mkdir(parents=True) + asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}" + content = b"cross-platform-driver-agent-with-provenance" + asset.write_bytes(content) + provenance = tmpdir / "provenance.json" + provenance.write_text( + json.dumps( + { + "schemaVersion": 1, + "assets": { + asset.name: { + "driver": "clickhouse", + "driverType": "clickhouse", + "platform": platform, + "revision": revision, + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + }, + } + ), + encoding="utf-8", + ) + + output = tmpdir / "manifest.json" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--assets-dir", + str(assets_dir), + "--output", + str(output), + "--provenance", + str(provenance), + ], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(proc.returncode, 0, proc.stderr) + manifest = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(manifest["assets"][asset.name]["revision"], revision) + + def test_rejects_sha_matching_provenance_with_stale_revision(self): + platform = self._cross_platform() + goos, goarch = platform.split("/", 1) + extension = ".exe" if goos == "windows" else "" + with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-provenance-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + assets_dir.mkdir(parents=True) + asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}" + content = b"stale-cross-platform-driver-agent" + asset.write_bytes(content) + provenance = tmpdir / "previous-manifest.json" + provenance.write_text( + json.dumps( + { + "schemaVersion": 1, + "assets": { + asset.name: { + "driver": "clickhouse", + "driverType": "clickhouse", + "platform": platform, + "revision": "src-stale-agent", + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + }, + } + ), + encoding="utf-8", + ) + + output = tmpdir / "manifest.json" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--assets-dir", + str(assets_dir), + "--output", + str(output), + "--provenance", + str(provenance), + ], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(proc.returncode, 0, proc.stdout) + self.assertIn("provenance revision mismatch", proc.stderr) + self.assertIn("src-stale-agent", proc.stderr) + self.assertFalse(output.exists()) + + def test_writes_sha_bound_build_provenance_from_revision_file(self): + platform = self._cross_platform() + goos, goarch = platform.split("/", 1) + extension = ".exe" if goos == "windows" else "" + with tempfile.TemporaryDirectory(prefix="gonavi-release-build-provenance-") as tmp: + tmpdir = Path(tmp) + assets_dir = tmpdir / "drivers" + assets_dir.mkdir(parents=True) + asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}" + content = b"freshly-built-cross-platform-agent" + asset.write_bytes(content) + revision_file = tmpdir / "driver_agent_revisions_gen.go" + revision_file.write_text( + "package db\n" + "var revisions = map[string]string{\n" + ' "clickhouse": "src-build-revision",\n' + "}\n", + encoding="utf-8", + ) + provenance = tmpdir / "build-provenance.json" + + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--assets-dir", + str(assets_dir), + "--provenance-output", + str(provenance), + "--revision-file", + str(revision_file), + ], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(proc.returncode, 0, proc.stderr) + payload = json.loads(provenance.read_text(encoding="utf-8")) + metadata = payload["assets"][asset.name] + self.assertEqual(metadata["revision"], "src-build-revision") + self.assertEqual(metadata["sha256"], hashlib.sha256(content).hexdigest()) + self.assertEqual(metadata["platform"], platform) + def _generate_revision_file(self, platform: str, drivers: str = "clickhouse"): bash_executable = resolve_bash_executable() worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-")) @@ -99,7 +349,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase): ) return worktree / "internal" / "db" / "driver_agent_revisions_gen.go" - def test_generates_manifest_without_executing_cross_platform_assets(self): + def test_generates_manifest_from_verified_native_and_provenance_backed_cross_platform_assets(self): with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp: tmpdir = Path(tmp) assets_dir = tmpdir / "drivers" @@ -107,21 +357,61 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase): (assets_dir / "Linux").mkdir(parents=True) (assets_dir / "Windows").mkdir(parents=True) - fixtures = { - assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary", - assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary", - assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": b"linux-arm64-binary", - assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary", - assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": b"MZfake-mongodb-v1", - assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": b"MZfake-mongodb-v2", + darwin_revision_file = self._generate_revision_file("darwin/arm64") + linux_revision_file = self._generate_revision_file("linux/amd64") + linux_arm64_revision_file = self._generate_revision_file("linux/arm64") + windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb") + revision_by_platform_driver = { + ("darwin/arm64", "clickhouse"): expected_revision(darwin_revision_file, "clickhouse"), + ("linux/amd64", "clickhouse"): expected_revision(linux_revision_file, "clickhouse"), + ("linux/arm64", "clickhouse"): expected_revision(linux_arm64_revision_file, "clickhouse"), + ("windows/amd64", "clickhouse"): expected_revision(windows_revision_file, "clickhouse"), + ("windows/amd64", "mongodb"): expected_revision(windows_revision_file, "mongodb"), } - for path, content in fixtures.items(): + fixtures = { + assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": ("clickhouse", "darwin/arm64", b"darwin-binary"), + assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": ("clickhouse", "linux/amd64", b"linux-binary"), + assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": ("clickhouse", "linux/arm64", b"linux-arm64-binary"), + assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": ("clickhouse", "windows/amd64", b"MZfake-binary"), + assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v1"), + assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v2"), + } + host_platform = "/".join(self._host_platform()) + for path, (driver, platform, content) in fixtures.items(): path.write_bytes(content) os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + if platform == host_platform and driver == "clickhouse": + self._build_metadata_agent(path, revision_by_platform_driver[(platform, driver)]) + + provenance_assets = {} + for path, (driver, platform, _) in fixtures.items(): + content = path.read_bytes() + provenance_assets[path.name] = { + "driver": driver, + "driverType": driver, + "platform": platform, + "revision": revision_by_platform_driver[(platform, driver)], + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + provenance = tmpdir / "provenance.json" + provenance.write_text( + json.dumps({"schemaVersion": 1, "assets": provenance_assets}), + encoding="utf-8", + ) output = tmpdir / "manifest.json" proc = subprocess.run( - [sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)], + [ + sys.executable, + str(SCRIPT), + "--assets-dir", + str(assets_dir), + "--output", + str(output), + "--provenance", + str(provenance), + ], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -132,10 +422,6 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase): self.assertIn("asset count: 6", proc.stdout) manifest = json.loads(output.read_text(encoding="utf-8")) assets = manifest["assets"] - darwin_revision_file = self._generate_revision_file("darwin/arm64") - linux_revision_file = self._generate_revision_file("linux/amd64") - linux_arm64_revision_file = self._generate_revision_file("linux/arm64") - windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb") self.assertEqual( assets["clickhouse-driver-agent-darwin-arm64"]["revision"], expected_revision(darwin_revision_file, "clickhouse"),