🐛 fix(ci): 校验驱动代理发布产物版本来源

- 原生平台执行 metadata 校验真实 driver type 与 revision
- 跨平台构建生成与二进制 SHA256 绑定的 provenance
- 复用旧资产时要求清单 SHA 与当前源码 revision 同时匹配
- 将 provenance 接入开发版和正式版发布聚合流程
- 发布链变更触发全平台 driver-agent 重建
This commit is contained in:
Syngnat
2026-07-17 17:07:37 +08:00
parent 54f18cdd2f
commit ca8be18531
4 changed files with 595 additions and 26 deletions

View File

@@ -15,8 +15,19 @@ from pathlib import Path
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets")
parser.add_argument("--output", required=True, help="manifest json output path")
return parser.parse_args()
parser.add_argument("--output", help="release manifest json output path")
parser.add_argument(
"--provenance",
action="append",
default=[],
help="manifest/provenance JSON file or directory; may be passed more than once",
)
parser.add_argument("--provenance-output", help="write SHA-bound build provenance and exit when --output is omitted")
parser.add_argument("--revision-file", help="generated driver revision file used by --provenance-output")
args = parser.parse_args()
if not args.output and not args.provenance_output:
parser.error("one of --output or --provenance-output is required")
return args
def infer_driver_and_platform(file_name: str):
@@ -127,6 +138,174 @@ def parse_revision_file(path: Path):
return revisions
def resolve_host_platform():
goos = subprocess.run(
["go", "env", "GOOS"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
).stdout.strip()
goarch = subprocess.run(
["go", "env", "GOARCH"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
).stdout.strip()
return f"{goos}/{goarch}"
def probe_agent_metadata(asset_path: Path):
if os.name != "nt":
os.chmod(asset_path, asset_path.stat().st_mode | 0o111)
proc = subprocess.run(
[str(asset_path)],
input='{"id":1,"method":"metadata"}\n',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=30,
)
if proc.returncode != 0:
detail = proc.stderr.strip() or f"exit code {proc.returncode}"
raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}")
lines = [line.strip() for line in proc.stdout.splitlines() if line.strip()]
if not lines:
raise RuntimeError(f"{asset_path.name}: metadata probe returned no response")
try:
response = json.loads(lines[0])
except json.JSONDecodeError as exc:
raise RuntimeError(f"{asset_path.name}: metadata probe returned invalid JSON: {exc}") from exc
if not response.get("success"):
detail = str(response.get("error") or "metadata request failed").strip()
raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}")
data = response.get("data") or {}
driver_type = normalize_driver(data.get("driverType"))
revision = str(data.get("agentRevision") or "").strip()
if not driver_type or not revision:
raise RuntimeError(f"{asset_path.name}: metadata response is missing driverType or agentRevision")
return driver_type, revision
def load_asset_provenance(paths):
entries = {}
for raw_path in paths:
source = Path(raw_path).resolve()
if source.is_dir():
files = sorted(source.rglob("*.json"))
elif source.is_file():
files = [source]
else:
raise RuntimeError(f"binary revision provenance path does not exist: {source}")
for file_path in files:
try:
payload = json.loads(file_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError(f"failed to read binary revision provenance {file_path}: {exc}") from exc
assets = payload.get("assets") or {}
if not isinstance(assets, dict):
raise RuntimeError(f"binary revision provenance {file_path} has invalid assets")
for asset_name, metadata in assets.items():
if not isinstance(metadata, dict):
continue
name = str(asset_name or "").strip()
if not name:
continue
entries.setdefault(name, []).append((file_path, metadata))
return entries
def resolve_asset_provenance(entries, asset_name, driver, platform, sha256, size):
candidates = entries.get(asset_name) or []
matching = []
for source, metadata in candidates:
recorded_sha = str(metadata.get("sha256") or "").strip().lower()
if recorded_sha == sha256.lower():
matching.append((source, metadata))
if not matching:
if candidates:
recorded = sorted(
{
str(metadata.get("sha256") or "<missing>").strip() or "<missing>"
for _, metadata in candidates
}
)
raise RuntimeError(
f"{asset_name}: binary revision provenance SHA256 mismatch: "
f"asset={sha256} recorded={','.join(recorded)}"
)
raise RuntimeError(
f"{asset_name}: binary revision provenance is required for "
f"cross-platform asset {platform}"
)
revisions = set()
for source, metadata in matching:
recorded_driver = normalize_driver(metadata.get("driver") or metadata.get("driverType"))
recorded_platform = str(metadata.get("platform") or "").strip()
recorded_revision = str(metadata.get("revision") or "").strip()
recorded_size = metadata.get("size")
if recorded_driver != driver or recorded_platform != platform:
raise RuntimeError(
f"{asset_name}: binary revision provenance identity mismatch in {source}: "
f"recorded={recorded_platform}/{recorded_driver} expected={platform}/{driver}"
)
if recorded_size is not None and int(recorded_size) != size:
raise RuntimeError(
f"{asset_name}: binary revision provenance size mismatch in {source}: "
f"recorded={recorded_size} expected={size}"
)
if not recorded_revision:
raise RuntimeError(f"{asset_name}: binary revision provenance is missing revision in {source}")
revisions.add(recorded_revision)
if len(revisions) != 1:
raise RuntimeError(
f"{asset_name}: conflicting binary revision provenance: {','.join(sorted(revisions))}"
)
return revisions.pop()
def write_build_provenance(asset_entries, revision_file: Path, output_path: Path, generated_from: str):
revisions = parse_revision_file(revision_file)
host_platform = resolve_host_platform()
assets = {}
for child, driver, platform in asset_entries:
normalized_driver = normalize_driver(driver)
revision = str(revisions.get(normalized_driver) or "").strip()
if not revision:
raise RuntimeError(
f"{child.name}: missing build revision for {platform}/{normalized_driver} in {revision_file}"
)
if platform == host_platform:
binary_driver, binary_revision = probe_agent_metadata(child)
if binary_driver != normalized_driver or binary_revision != revision:
raise RuntimeError(
f"{child.name}: build provenance metadata mismatch: "
f"binary={binary_driver}/{binary_revision} "
f"expected={normalized_driver}/{revision}"
)
revision = binary_revision
size = child.stat().st_size
assets[child.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": revision,
"size": size,
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
}
payload = {
"schemaVersion": 1,
"generatedFrom": generated_from,
"assets": assets,
}
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"wrote build provenance: {output_path}")
print(f"asset count: {len(assets)}")
def generate_platform_revisions(root: Path, drivers_by_platform):
if not drivers_by_platform:
return {}
@@ -170,8 +349,8 @@ def generate_platform_revisions(root: Path, drivers_by_platform):
def main():
args = parse_args()
assets_dir = Path(args.assets_dir).resolve()
output_path = Path(args.output).resolve()
root = repo_root()
generated_from = os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root)
asset_entries = []
drivers_by_platform = {}
@@ -186,26 +365,70 @@ def main():
asset_entries.append((child, driver, platform))
drivers_by_platform.setdefault(platform, set()).add(driver)
if args.provenance_output:
revision_file = Path(args.revision_file).resolve() if args.revision_file else root / "internal" / "db" / "driver_agent_revisions_gen.go"
write_build_provenance(
asset_entries,
revision_file,
Path(args.provenance_output).resolve(),
generated_from,
)
if not args.output:
return 0
output_path = Path(args.output).resolve()
provenance_entries = load_asset_provenance(args.provenance)
revisions_by_platform = generate_platform_revisions(root, drivers_by_platform)
host_platform = resolve_host_platform()
manifest = {
"schemaVersion": 1,
"generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root),
"generatedFrom": generated_from,
"assets": {},
}
for child, driver, platform in asset_entries:
normalized_driver = normalize_driver(driver)
size = child.stat().st_size
sha256 = hashlib.sha256(child.read_bytes()).hexdigest()
revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
if not revision:
raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
if platform == host_platform:
binary_driver, binary_revision = probe_agent_metadata(child)
if binary_driver != normalized_driver:
raise RuntimeError(
f"{child.name}: embedded driver type mismatch: "
f"binary={binary_driver} expected={normalized_driver}"
)
if binary_revision != revision:
raise RuntimeError(
f"{child.name}: embedded revision mismatch: "
f"binary={binary_revision} expected={revision}"
)
revision = binary_revision
else:
binary_revision = resolve_asset_provenance(
provenance_entries,
child.name,
normalized_driver,
platform,
sha256,
size,
)
if binary_revision != revision:
raise RuntimeError(
f"{child.name}: provenance revision mismatch: "
f"binary={binary_revision} expected={revision}"
)
revision = binary_revision
manifest["assets"][child.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": revision,
"size": child.stat().st_size,
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
"size": size,
"sha256": sha256,
}
output_path.parent.mkdir(parents=True, exist_ok=True)

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import json
import hashlib
import os
import shlex
import shutil
@@ -70,6 +71,255 @@ def expected_revision(revision_file: Path, driver: str):
class GenerateDriverReleaseManifestTest(unittest.TestCase):
def _host_platform(self):
goos = subprocess.run(
["go", "env", "GOOS"],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
).stdout.strip()
goarch = subprocess.run(
["go", "env", "GOARCH"],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
).stdout.strip()
return goos, goarch
def _build_metadata_agent(self, output: Path, revision: str):
source = output.parent / "metadata-agent.go"
source.write_text(
"package main\n"
"import (\"bufio\"; \"fmt\"; \"os\")\n"
"func main() {\n"
" scanner := bufio.NewScanner(os.Stdin)\n"
" for scanner.Scan() {\n"
f" fmt.Println(`{{\"id\":1,\"success\":true,\"data\":{{\"driverType\":\"clickhouse\",\"agentRevision\":\"{revision}\",\"protocolSchema\":\"json-lines-v1\"}}}}`)\n"
" }\n"
"}\n",
encoding="utf-8",
)
subprocess.run(
["go", "build", "-o", str(output), str(source)],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
)
def _cross_platform(self):
host = "/".join(self._host_platform())
for platform in ("linux/amd64", "darwin/arm64", "windows/amd64"):
if platform != host:
return platform
raise AssertionError(f"unable to select cross platform for {host}")
def test_rejects_native_asset_with_stale_embedded_revision(self):
goos, goarch = self._host_platform()
extension = ".exe" if goos == "windows" else ""
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-agent-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
self._build_metadata_agent(asset, "src-stale-agent")
output = tmpdir / "manifest.json"
proc = subprocess.run(
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout)
self.assertIn("src-stale-agent", proc.stderr)
self.assertFalse(output.exists())
def test_rejects_cross_platform_asset_without_binary_revision_provenance(self):
goos, goarch = self._cross_platform().split("/", 1)
extension = ".exe" if goos == "windows" else ""
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-cross-agent-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
asset.write_bytes(b"cross-platform-driver-agent")
output = tmpdir / "manifest.json"
proc = subprocess.run(
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout)
self.assertIn("binary revision provenance", proc.stderr)
self.assertFalse(output.exists())
def test_accepts_cross_platform_asset_with_matching_sha_bound_provenance(self):
platform = self._cross_platform()
goos, goarch = platform.split("/", 1)
extension = ".exe" if goos == "windows" else ""
revision_file = self._generate_revision_file(platform)
revision = expected_revision(revision_file, "clickhouse")
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-provenance-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
content = b"cross-platform-driver-agent-with-provenance"
asset.write_bytes(content)
provenance = tmpdir / "provenance.json"
provenance.write_text(
json.dumps(
{
"schemaVersion": 1,
"assets": {
asset.name: {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": platform,
"revision": revision,
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
}
},
}
),
encoding="utf-8",
)
output = tmpdir / "manifest.json"
proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--assets-dir",
str(assets_dir),
"--output",
str(output),
"--provenance",
str(provenance),
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertEqual(proc.returncode, 0, proc.stderr)
manifest = json.loads(output.read_text(encoding="utf-8"))
self.assertEqual(manifest["assets"][asset.name]["revision"], revision)
def test_rejects_sha_matching_provenance_with_stale_revision(self):
platform = self._cross_platform()
goos, goarch = platform.split("/", 1)
extension = ".exe" if goos == "windows" else ""
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-provenance-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
content = b"stale-cross-platform-driver-agent"
asset.write_bytes(content)
provenance = tmpdir / "previous-manifest.json"
provenance.write_text(
json.dumps(
{
"schemaVersion": 1,
"assets": {
asset.name: {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": platform,
"revision": "src-stale-agent",
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
}
},
}
),
encoding="utf-8",
)
output = tmpdir / "manifest.json"
proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--assets-dir",
str(assets_dir),
"--output",
str(output),
"--provenance",
str(provenance),
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout)
self.assertIn("provenance revision mismatch", proc.stderr)
self.assertIn("src-stale-agent", proc.stderr)
self.assertFalse(output.exists())
def test_writes_sha_bound_build_provenance_from_revision_file(self):
platform = self._cross_platform()
goos, goarch = platform.split("/", 1)
extension = ".exe" if goos == "windows" else ""
with tempfile.TemporaryDirectory(prefix="gonavi-release-build-provenance-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
content = b"freshly-built-cross-platform-agent"
asset.write_bytes(content)
revision_file = tmpdir / "driver_agent_revisions_gen.go"
revision_file.write_text(
"package db\n"
"var revisions = map[string]string{\n"
' "clickhouse": "src-build-revision",\n'
"}\n",
encoding="utf-8",
)
provenance = tmpdir / "build-provenance.json"
proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--assets-dir",
str(assets_dir),
"--provenance-output",
str(provenance),
"--revision-file",
str(revision_file),
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertEqual(proc.returncode, 0, proc.stderr)
payload = json.loads(provenance.read_text(encoding="utf-8"))
metadata = payload["assets"][asset.name]
self.assertEqual(metadata["revision"], "src-build-revision")
self.assertEqual(metadata["sha256"], hashlib.sha256(content).hexdigest())
self.assertEqual(metadata["platform"], platform)
def _generate_revision_file(self, platform: str, drivers: str = "clickhouse"):
bash_executable = resolve_bash_executable()
worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-"))
@@ -99,7 +349,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
)
return worktree / "internal" / "db" / "driver_agent_revisions_gen.go"
def test_generates_manifest_without_executing_cross_platform_assets(self):
def test_generates_manifest_from_verified_native_and_provenance_backed_cross_platform_assets(self):
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
@@ -107,21 +357,61 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
(assets_dir / "Linux").mkdir(parents=True)
(assets_dir / "Windows").mkdir(parents=True)
fixtures = {
assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary",
assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary",
assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": b"linux-arm64-binary",
assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary",
assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": b"MZfake-mongodb-v1",
assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": b"MZfake-mongodb-v2",
darwin_revision_file = self._generate_revision_file("darwin/arm64")
linux_revision_file = self._generate_revision_file("linux/amd64")
linux_arm64_revision_file = self._generate_revision_file("linux/arm64")
windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb")
revision_by_platform_driver = {
("darwin/arm64", "clickhouse"): expected_revision(darwin_revision_file, "clickhouse"),
("linux/amd64", "clickhouse"): expected_revision(linux_revision_file, "clickhouse"),
("linux/arm64", "clickhouse"): expected_revision(linux_arm64_revision_file, "clickhouse"),
("windows/amd64", "clickhouse"): expected_revision(windows_revision_file, "clickhouse"),
("windows/amd64", "mongodb"): expected_revision(windows_revision_file, "mongodb"),
}
for path, content in fixtures.items():
fixtures = {
assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": ("clickhouse", "darwin/arm64", b"darwin-binary"),
assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": ("clickhouse", "linux/amd64", b"linux-binary"),
assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": ("clickhouse", "linux/arm64", b"linux-arm64-binary"),
assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": ("clickhouse", "windows/amd64", b"MZfake-binary"),
assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v1"),
assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v2"),
}
host_platform = "/".join(self._host_platform())
for path, (driver, platform, content) in fixtures.items():
path.write_bytes(content)
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
if platform == host_platform and driver == "clickhouse":
self._build_metadata_agent(path, revision_by_platform_driver[(platform, driver)])
provenance_assets = {}
for path, (driver, platform, _) in fixtures.items():
content = path.read_bytes()
provenance_assets[path.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": revision_by_platform_driver[(platform, driver)],
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
}
provenance = tmpdir / "provenance.json"
provenance.write_text(
json.dumps({"schemaVersion": 1, "assets": provenance_assets}),
encoding="utf-8",
)
output = tmpdir / "manifest.json"
proc = subprocess.run(
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
[
sys.executable,
str(SCRIPT),
"--assets-dir",
str(assets_dir),
"--output",
str(output),
"--provenance",
str(provenance),
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
@@ -132,10 +422,6 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
self.assertIn("asset count: 6", proc.stdout)
manifest = json.loads(output.read_text(encoding="utf-8"))
assets = manifest["assets"]
darwin_revision_file = self._generate_revision_file("darwin/arm64")
linux_revision_file = self._generate_revision_file("linux/amd64")
linux_arm64_revision_file = self._generate_revision_file("linux/arm64")
windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb")
self.assertEqual(
assets["clickhouse-driver-agent-darwin-arm64"]["revision"],
expected_revision(darwin_revision_file, "clickhouse"),