mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-10 08:43:34 +08:00
🐛 fix(ci): 校验驱动代理发布产物版本来源
- 原生平台执行 metadata 校验真实 driver type 与 revision - 跨平台构建生成与二进制 SHA256 绑定的 provenance - 复用旧资产时要求清单 SHA 与当前源码 revision 同时匹配 - 将 provenance 接入开发版和正式版发布聚合流程 - 发布链变更触发全平台 driver-agent 重建
This commit is contained in:
@@ -15,8 +15,19 @@ from pathlib import Path
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets")
|
||||
parser.add_argument("--output", required=True, help="manifest json output path")
|
||||
return parser.parse_args()
|
||||
parser.add_argument("--output", help="release manifest json output path")
|
||||
parser.add_argument(
|
||||
"--provenance",
|
||||
action="append",
|
||||
default=[],
|
||||
help="manifest/provenance JSON file or directory; may be passed more than once",
|
||||
)
|
||||
parser.add_argument("--provenance-output", help="write SHA-bound build provenance and exit when --output is omitted")
|
||||
parser.add_argument("--revision-file", help="generated driver revision file used by --provenance-output")
|
||||
args = parser.parse_args()
|
||||
if not args.output and not args.provenance_output:
|
||||
parser.error("one of --output or --provenance-output is required")
|
||||
return args
|
||||
|
||||
|
||||
def infer_driver_and_platform(file_name: str):
|
||||
@@ -127,6 +138,174 @@ def parse_revision_file(path: Path):
|
||||
return revisions
|
||||
|
||||
|
||||
def resolve_host_platform():
|
||||
goos = subprocess.run(
|
||||
["go", "env", "GOOS"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
goarch = subprocess.run(
|
||||
["go", "env", "GOARCH"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return f"{goos}/{goarch}"
|
||||
|
||||
|
||||
def probe_agent_metadata(asset_path: Path):
|
||||
if os.name != "nt":
|
||||
os.chmod(asset_path, asset_path.stat().st_mode | 0o111)
|
||||
proc = subprocess.run(
|
||||
[str(asset_path)],
|
||||
input='{"id":1,"method":"metadata"}\n',
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
detail = proc.stderr.strip() or f"exit code {proc.returncode}"
|
||||
raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}")
|
||||
lines = [line.strip() for line in proc.stdout.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
raise RuntimeError(f"{asset_path.name}: metadata probe returned no response")
|
||||
try:
|
||||
response = json.loads(lines[0])
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"{asset_path.name}: metadata probe returned invalid JSON: {exc}") from exc
|
||||
if not response.get("success"):
|
||||
detail = str(response.get("error") or "metadata request failed").strip()
|
||||
raise RuntimeError(f"{asset_path.name}: metadata probe failed: {detail}")
|
||||
data = response.get("data") or {}
|
||||
driver_type = normalize_driver(data.get("driverType"))
|
||||
revision = str(data.get("agentRevision") or "").strip()
|
||||
if not driver_type or not revision:
|
||||
raise RuntimeError(f"{asset_path.name}: metadata response is missing driverType or agentRevision")
|
||||
return driver_type, revision
|
||||
|
||||
|
||||
def load_asset_provenance(paths):
|
||||
entries = {}
|
||||
for raw_path in paths:
|
||||
source = Path(raw_path).resolve()
|
||||
if source.is_dir():
|
||||
files = sorted(source.rglob("*.json"))
|
||||
elif source.is_file():
|
||||
files = [source]
|
||||
else:
|
||||
raise RuntimeError(f"binary revision provenance path does not exist: {source}")
|
||||
for file_path in files:
|
||||
try:
|
||||
payload = json.loads(file_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"failed to read binary revision provenance {file_path}: {exc}") from exc
|
||||
assets = payload.get("assets") or {}
|
||||
if not isinstance(assets, dict):
|
||||
raise RuntimeError(f"binary revision provenance {file_path} has invalid assets")
|
||||
for asset_name, metadata in assets.items():
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
name = str(asset_name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
entries.setdefault(name, []).append((file_path, metadata))
|
||||
return entries
|
||||
|
||||
|
||||
def resolve_asset_provenance(entries, asset_name, driver, platform, sha256, size):
|
||||
candidates = entries.get(asset_name) or []
|
||||
matching = []
|
||||
for source, metadata in candidates:
|
||||
recorded_sha = str(metadata.get("sha256") or "").strip().lower()
|
||||
if recorded_sha == sha256.lower():
|
||||
matching.append((source, metadata))
|
||||
if not matching:
|
||||
if candidates:
|
||||
recorded = sorted(
|
||||
{
|
||||
str(metadata.get("sha256") or "<missing>").strip() or "<missing>"
|
||||
for _, metadata in candidates
|
||||
}
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"{asset_name}: binary revision provenance SHA256 mismatch: "
|
||||
f"asset={sha256} recorded={','.join(recorded)}"
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"{asset_name}: binary revision provenance is required for "
|
||||
f"cross-platform asset {platform}"
|
||||
)
|
||||
|
||||
revisions = set()
|
||||
for source, metadata in matching:
|
||||
recorded_driver = normalize_driver(metadata.get("driver") or metadata.get("driverType"))
|
||||
recorded_platform = str(metadata.get("platform") or "").strip()
|
||||
recorded_revision = str(metadata.get("revision") or "").strip()
|
||||
recorded_size = metadata.get("size")
|
||||
if recorded_driver != driver or recorded_platform != platform:
|
||||
raise RuntimeError(
|
||||
f"{asset_name}: binary revision provenance identity mismatch in {source}: "
|
||||
f"recorded={recorded_platform}/{recorded_driver} expected={platform}/{driver}"
|
||||
)
|
||||
if recorded_size is not None and int(recorded_size) != size:
|
||||
raise RuntimeError(
|
||||
f"{asset_name}: binary revision provenance size mismatch in {source}: "
|
||||
f"recorded={recorded_size} expected={size}"
|
||||
)
|
||||
if not recorded_revision:
|
||||
raise RuntimeError(f"{asset_name}: binary revision provenance is missing revision in {source}")
|
||||
revisions.add(recorded_revision)
|
||||
if len(revisions) != 1:
|
||||
raise RuntimeError(
|
||||
f"{asset_name}: conflicting binary revision provenance: {','.join(sorted(revisions))}"
|
||||
)
|
||||
return revisions.pop()
|
||||
|
||||
|
||||
def write_build_provenance(asset_entries, revision_file: Path, output_path: Path, generated_from: str):
|
||||
revisions = parse_revision_file(revision_file)
|
||||
host_platform = resolve_host_platform()
|
||||
assets = {}
|
||||
for child, driver, platform in asset_entries:
|
||||
normalized_driver = normalize_driver(driver)
|
||||
revision = str(revisions.get(normalized_driver) or "").strip()
|
||||
if not revision:
|
||||
raise RuntimeError(
|
||||
f"{child.name}: missing build revision for {platform}/{normalized_driver} in {revision_file}"
|
||||
)
|
||||
if platform == host_platform:
|
||||
binary_driver, binary_revision = probe_agent_metadata(child)
|
||||
if binary_driver != normalized_driver or binary_revision != revision:
|
||||
raise RuntimeError(
|
||||
f"{child.name}: build provenance metadata mismatch: "
|
||||
f"binary={binary_driver}/{binary_revision} "
|
||||
f"expected={normalized_driver}/{revision}"
|
||||
)
|
||||
revision = binary_revision
|
||||
size = child.stat().st_size
|
||||
assets[child.name] = {
|
||||
"driver": driver,
|
||||
"driverType": driver,
|
||||
"platform": platform,
|
||||
"revision": revision,
|
||||
"size": size,
|
||||
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
|
||||
}
|
||||
payload = {
|
||||
"schemaVersion": 1,
|
||||
"generatedFrom": generated_from,
|
||||
"assets": assets,
|
||||
}
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(f"wrote build provenance: {output_path}")
|
||||
print(f"asset count: {len(assets)}")
|
||||
|
||||
|
||||
def generate_platform_revisions(root: Path, drivers_by_platform):
|
||||
if not drivers_by_platform:
|
||||
return {}
|
||||
@@ -170,8 +349,8 @@ def generate_platform_revisions(root: Path, drivers_by_platform):
|
||||
def main():
|
||||
args = parse_args()
|
||||
assets_dir = Path(args.assets_dir).resolve()
|
||||
output_path = Path(args.output).resolve()
|
||||
root = repo_root()
|
||||
generated_from = os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root)
|
||||
|
||||
asset_entries = []
|
||||
drivers_by_platform = {}
|
||||
@@ -186,26 +365,70 @@ def main():
|
||||
asset_entries.append((child, driver, platform))
|
||||
drivers_by_platform.setdefault(platform, set()).add(driver)
|
||||
|
||||
if args.provenance_output:
|
||||
revision_file = Path(args.revision_file).resolve() if args.revision_file else root / "internal" / "db" / "driver_agent_revisions_gen.go"
|
||||
write_build_provenance(
|
||||
asset_entries,
|
||||
revision_file,
|
||||
Path(args.provenance_output).resolve(),
|
||||
generated_from,
|
||||
)
|
||||
if not args.output:
|
||||
return 0
|
||||
|
||||
output_path = Path(args.output).resolve()
|
||||
provenance_entries = load_asset_provenance(args.provenance)
|
||||
revisions_by_platform = generate_platform_revisions(root, drivers_by_platform)
|
||||
host_platform = resolve_host_platform()
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root),
|
||||
"generatedFrom": generated_from,
|
||||
"assets": {},
|
||||
}
|
||||
|
||||
for child, driver, platform in asset_entries:
|
||||
normalized_driver = normalize_driver(driver)
|
||||
size = child.stat().st_size
|
||||
sha256 = hashlib.sha256(child.read_bytes()).hexdigest()
|
||||
revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
|
||||
if not revision:
|
||||
raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
|
||||
if platform == host_platform:
|
||||
binary_driver, binary_revision = probe_agent_metadata(child)
|
||||
if binary_driver != normalized_driver:
|
||||
raise RuntimeError(
|
||||
f"{child.name}: embedded driver type mismatch: "
|
||||
f"binary={binary_driver} expected={normalized_driver}"
|
||||
)
|
||||
if binary_revision != revision:
|
||||
raise RuntimeError(
|
||||
f"{child.name}: embedded revision mismatch: "
|
||||
f"binary={binary_revision} expected={revision}"
|
||||
)
|
||||
revision = binary_revision
|
||||
else:
|
||||
binary_revision = resolve_asset_provenance(
|
||||
provenance_entries,
|
||||
child.name,
|
||||
normalized_driver,
|
||||
platform,
|
||||
sha256,
|
||||
size,
|
||||
)
|
||||
if binary_revision != revision:
|
||||
raise RuntimeError(
|
||||
f"{child.name}: provenance revision mismatch: "
|
||||
f"binary={binary_revision} expected={revision}"
|
||||
)
|
||||
revision = binary_revision
|
||||
manifest["assets"][child.name] = {
|
||||
"driver": driver,
|
||||
"driverType": driver,
|
||||
"platform": platform,
|
||||
"revision": revision,
|
||||
"size": child.stat().st_size,
|
||||
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
|
||||
"size": size,
|
||||
"sha256": sha256,
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
@@ -70,6 +71,255 @@ def expected_revision(revision_file: Path, driver: str):
|
||||
|
||||
|
||||
class GenerateDriverReleaseManifestTest(unittest.TestCase):
|
||||
def _host_platform(self):
|
||||
goos = subprocess.run(
|
||||
["go", "env", "GOOS"],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
goarch = subprocess.run(
|
||||
["go", "env", "GOARCH"],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return goos, goarch
|
||||
|
||||
def _build_metadata_agent(self, output: Path, revision: str):
|
||||
source = output.parent / "metadata-agent.go"
|
||||
source.write_text(
|
||||
"package main\n"
|
||||
"import (\"bufio\"; \"fmt\"; \"os\")\n"
|
||||
"func main() {\n"
|
||||
" scanner := bufio.NewScanner(os.Stdin)\n"
|
||||
" for scanner.Scan() {\n"
|
||||
f" fmt.Println(`{{\"id\":1,\"success\":true,\"data\":{{\"driverType\":\"clickhouse\",\"agentRevision\":\"{revision}\",\"protocolSchema\":\"json-lines-v1\"}}}}`)\n"
|
||||
" }\n"
|
||||
"}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
subprocess.run(
|
||||
["go", "build", "-o", str(output), str(source)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
def _cross_platform(self):
|
||||
host = "/".join(self._host_platform())
|
||||
for platform in ("linux/amd64", "darwin/arm64", "windows/amd64"):
|
||||
if platform != host:
|
||||
return platform
|
||||
raise AssertionError(f"unable to select cross platform for {host}")
|
||||
|
||||
def test_rejects_native_asset_with_stale_embedded_revision(self):
|
||||
goos, goarch = self._host_platform()
|
||||
extension = ".exe" if goos == "windows" else ""
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-agent-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
assets_dir.mkdir(parents=True)
|
||||
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
|
||||
self._build_metadata_agent(asset, "src-stale-agent")
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout)
|
||||
self.assertIn("src-stale-agent", proc.stderr)
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_rejects_cross_platform_asset_without_binary_revision_provenance(self):
|
||||
goos, goarch = self._cross_platform().split("/", 1)
|
||||
extension = ".exe" if goos == "windows" else ""
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-cross-agent-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
assets_dir.mkdir(parents=True)
|
||||
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
|
||||
asset.write_bytes(b"cross-platform-driver-agent")
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout)
|
||||
self.assertIn("binary revision provenance", proc.stderr)
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_accepts_cross_platform_asset_with_matching_sha_bound_provenance(self):
|
||||
platform = self._cross_platform()
|
||||
goos, goarch = platform.split("/", 1)
|
||||
extension = ".exe" if goos == "windows" else ""
|
||||
revision_file = self._generate_revision_file(platform)
|
||||
revision = expected_revision(revision_file, "clickhouse")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-provenance-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
assets_dir.mkdir(parents=True)
|
||||
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
|
||||
content = b"cross-platform-driver-agent-with-provenance"
|
||||
asset.write_bytes(content)
|
||||
provenance = tmpdir / "provenance.json"
|
||||
provenance.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"assets": {
|
||||
asset.name: {
|
||||
"driver": "clickhouse",
|
||||
"driverType": "clickhouse",
|
||||
"platform": platform,
|
||||
"revision": revision,
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"size": len(content),
|
||||
}
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--assets-dir",
|
||||
str(assets_dir),
|
||||
"--output",
|
||||
str(output),
|
||||
"--provenance",
|
||||
str(provenance),
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
manifest = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["assets"][asset.name]["revision"], revision)
|
||||
|
||||
def test_rejects_sha_matching_provenance_with_stale_revision(self):
|
||||
platform = self._cross_platform()
|
||||
goos, goarch = platform.split("/", 1)
|
||||
extension = ".exe" if goos == "windows" else ""
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-stale-provenance-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
assets_dir.mkdir(parents=True)
|
||||
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
|
||||
content = b"stale-cross-platform-driver-agent"
|
||||
asset.write_bytes(content)
|
||||
provenance = tmpdir / "previous-manifest.json"
|
||||
provenance.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"assets": {
|
||||
asset.name: {
|
||||
"driver": "clickhouse",
|
||||
"driverType": "clickhouse",
|
||||
"platform": platform,
|
||||
"revision": "src-stale-agent",
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"size": len(content),
|
||||
}
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--assets-dir",
|
||||
str(assets_dir),
|
||||
"--output",
|
||||
str(output),
|
||||
"--provenance",
|
||||
str(provenance),
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout)
|
||||
self.assertIn("provenance revision mismatch", proc.stderr)
|
||||
self.assertIn("src-stale-agent", proc.stderr)
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_writes_sha_bound_build_provenance_from_revision_file(self):
|
||||
platform = self._cross_platform()
|
||||
goos, goarch = platform.split("/", 1)
|
||||
extension = ".exe" if goos == "windows" else ""
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-build-provenance-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
assets_dir.mkdir(parents=True)
|
||||
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
|
||||
content = b"freshly-built-cross-platform-agent"
|
||||
asset.write_bytes(content)
|
||||
revision_file = tmpdir / "driver_agent_revisions_gen.go"
|
||||
revision_file.write_text(
|
||||
"package db\n"
|
||||
"var revisions = map[string]string{\n"
|
||||
' "clickhouse": "src-build-revision",\n'
|
||||
"}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
provenance = tmpdir / "build-provenance.json"
|
||||
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--assets-dir",
|
||||
str(assets_dir),
|
||||
"--provenance-output",
|
||||
str(provenance),
|
||||
"--revision-file",
|
||||
str(revision_file),
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
payload = json.loads(provenance.read_text(encoding="utf-8"))
|
||||
metadata = payload["assets"][asset.name]
|
||||
self.assertEqual(metadata["revision"], "src-build-revision")
|
||||
self.assertEqual(metadata["sha256"], hashlib.sha256(content).hexdigest())
|
||||
self.assertEqual(metadata["platform"], platform)
|
||||
|
||||
def _generate_revision_file(self, platform: str, drivers: str = "clickhouse"):
|
||||
bash_executable = resolve_bash_executable()
|
||||
worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-"))
|
||||
@@ -99,7 +349,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
|
||||
)
|
||||
return worktree / "internal" / "db" / "driver_agent_revisions_gen.go"
|
||||
|
||||
def test_generates_manifest_without_executing_cross_platform_assets(self):
|
||||
def test_generates_manifest_from_verified_native_and_provenance_backed_cross_platform_assets(self):
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
@@ -107,21 +357,61 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
|
||||
(assets_dir / "Linux").mkdir(parents=True)
|
||||
(assets_dir / "Windows").mkdir(parents=True)
|
||||
|
||||
fixtures = {
|
||||
assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary",
|
||||
assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary",
|
||||
assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": b"linux-arm64-binary",
|
||||
assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary",
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": b"MZfake-mongodb-v1",
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": b"MZfake-mongodb-v2",
|
||||
darwin_revision_file = self._generate_revision_file("darwin/arm64")
|
||||
linux_revision_file = self._generate_revision_file("linux/amd64")
|
||||
linux_arm64_revision_file = self._generate_revision_file("linux/arm64")
|
||||
windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb")
|
||||
revision_by_platform_driver = {
|
||||
("darwin/arm64", "clickhouse"): expected_revision(darwin_revision_file, "clickhouse"),
|
||||
("linux/amd64", "clickhouse"): expected_revision(linux_revision_file, "clickhouse"),
|
||||
("linux/arm64", "clickhouse"): expected_revision(linux_arm64_revision_file, "clickhouse"),
|
||||
("windows/amd64", "clickhouse"): expected_revision(windows_revision_file, "clickhouse"),
|
||||
("windows/amd64", "mongodb"): expected_revision(windows_revision_file, "mongodb"),
|
||||
}
|
||||
for path, content in fixtures.items():
|
||||
fixtures = {
|
||||
assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": ("clickhouse", "darwin/arm64", b"darwin-binary"),
|
||||
assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": ("clickhouse", "linux/amd64", b"linux-binary"),
|
||||
assets_dir / "Linux" / "clickhouse-driver-agent-linux-arm64": ("clickhouse", "linux/arm64", b"linux-arm64-binary"),
|
||||
assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": ("clickhouse", "windows/amd64", b"MZfake-binary"),
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v1"),
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": ("mongodb", "windows/amd64", b"MZfake-mongodb-v2"),
|
||||
}
|
||||
host_platform = "/".join(self._host_platform())
|
||||
for path, (driver, platform, content) in fixtures.items():
|
||||
path.write_bytes(content)
|
||||
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
||||
if platform == host_platform and driver == "clickhouse":
|
||||
self._build_metadata_agent(path, revision_by_platform_driver[(platform, driver)])
|
||||
|
||||
provenance_assets = {}
|
||||
for path, (driver, platform, _) in fixtures.items():
|
||||
content = path.read_bytes()
|
||||
provenance_assets[path.name] = {
|
||||
"driver": driver,
|
||||
"driverType": driver,
|
||||
"platform": platform,
|
||||
"revision": revision_by_platform_driver[(platform, driver)],
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"size": len(content),
|
||||
}
|
||||
provenance = tmpdir / "provenance.json"
|
||||
provenance.write_text(
|
||||
json.dumps({"schemaVersion": 1, "assets": provenance_assets}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--assets-dir",
|
||||
str(assets_dir),
|
||||
"--output",
|
||||
str(output),
|
||||
"--provenance",
|
||||
str(provenance),
|
||||
],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
@@ -132,10 +422,6 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
|
||||
self.assertIn("asset count: 6", proc.stdout)
|
||||
manifest = json.loads(output.read_text(encoding="utf-8"))
|
||||
assets = manifest["assets"]
|
||||
darwin_revision_file = self._generate_revision_file("darwin/arm64")
|
||||
linux_revision_file = self._generate_revision_file("linux/amd64")
|
||||
linux_arm64_revision_file = self._generate_revision_file("linux/arm64")
|
||||
windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb")
|
||||
self.assertEqual(
|
||||
assets["clickhouse-driver-agent-darwin-arm64"]["revision"],
|
||||
expected_revision(darwin_revision_file, "clickhouse"),
|
||||
|
||||
Reference in New Issue
Block a user