GithubBackup/MyGoNavi
1
0
Fork 0
mirror of https://github.com/Syngnat/GoNavi.git synced 2026-08-21 16:34:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

🐛 fix(ci): 修复跨 Runner 驱动版本误判

Browse Source 
使用构建阶段 SHA 绑定 provenance 校验驱动发布资产。

保留原生二进制元数据和 legacy fallback 校验，并固定发布任务 Go 环境。

Fixes #622
This commit is contained in:
Syngnat
2026-07-17 21:50:06 +08:00
parent 8f52cbc319
commit e7857bb2e4
6 changed files with 205 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
tools/generate-driver-release-manifest.py
View File

@@ -353,7 +353,6 @@ def main():
generated_from = os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root)
asset_entries = []
drivers_by_platform = {}
for child in sorted(assets_dir.rglob("*")):
if not child.is_file():
continue
@@ -363,7 +362,6 @@ def main():
if child.stat().st_size == 0:
raise RuntimeError(f"{child.name}: asset is empty")
asset_entries.append((child, driver, platform))
drivers_by_platform.setdefault(platform, set()).add(driver)
if args.provenance_output:
revision_file = Path(args.revision_file).resolve() if args.revision_file else root / "internal" / "db" / "driver_agent_revisions_gen.go"
@@ -378,7 +376,6 @@ def main():
output_path = Path(args.output).resolve()
provenance_entries = load_asset_provenance(args.provenance)
revisions_by_platform = generate_platform_revisions(root, drivers_by_platform)
host_platform = resolve_host_platform()
manifest = {
@@ -386,16 +383,64 @@ def main():
"generatedFrom": generated_from,
"assets": {},
}
fallback_entries = []
fallback_drivers_by_platform = {}
for child, driver, platform in asset_entries:
normalized_driver = normalize_driver(driver)
size = child.stat().st_size
sha256 = hashlib.sha256(child.read_bytes()).hexdigest()
revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
if not revision:
raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
if child.name not in provenance_entries:
if platform != host_platform:
resolve_asset_provenance(
provenance_entries,
child.name,
normalized_driver,
platform,
sha256,
size,
)
fallback_entries.append((child, driver, platform, normalized_driver, size, sha256))
fallback_drivers_by_platform.setdefault(platform, set()).add(normalized_driver)
continue
revision = resolve_asset_provenance(
provenance_entries,
child.name,
normalized_driver,
platform,
sha256,
size,
)
if platform == host_platform:
binary_driver, binary_revision = probe_agent_metadata(child)
if binary_driver != normalized_driver:
raise RuntimeError(
f"{child.name}: embedded driver type mismatch: "
f"binary={binary_driver} expected={normalized_driver}"
)
if binary_revision != revision:
raise RuntimeError(
f"{child.name}: provenance revision mismatch: "
f"binary={binary_revision} expected={revision}"
)
manifest["assets"][child.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": revision,
"size": size,
"sha256": sha256,
}
if fallback_entries:
fallback_revisions_by_platform = generate_platform_revisions(root, fallback_drivers_by_platform)
for child, driver, platform, normalized_driver, size, sha256 in fallback_entries:
revision = str((fallback_revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
if not revision:
raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
binary_driver, binary_revision = probe_agent_metadata(child)
if binary_driver != normalized_driver:
raise RuntimeError(
f"{child.name}: embedded driver type mismatch: "
@@ -406,30 +451,14 @@ def main():
f"{child.name}: embedded revision mismatch: "
f"binary={binary_revision} expected={revision}"
)
revision = binary_revision
else:
binary_revision = resolve_asset_provenance(
provenance_entries,
child.name,
normalized_driver,
platform,
sha256,
size,
)
if binary_revision != revision:
raise RuntimeError(
f"{child.name}: provenance revision mismatch: "
f"binary={binary_revision} expected={revision}"
)
revision = binary_revision
manifest["assets"][child.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": revision,
"size": size,
"sha256": sha256,
}
manifest["assets"][child.name] = {
"driver": driver,
"driverType": driver,
"platform": platform,
"revision": binary_revision,
"size": size,
"sha256": sha256,
}
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")

84
tools/generate-driver-release-manifest.test.py
View File

@@ -71,6 +71,21 @@ def expected_revision(revision_file: Path, driver: str):
class GenerateDriverReleaseManifestTest(unittest.TestCase):
def test_release_jobs_pin_go_toolchain_before_generating_driver_manifest(self):
setup_go_step = (
" - name: Setup Go\n"
" uses: actions/setup-go@v6\n"
" with:\n"
" go-version-file: 'go.mod'"
)
manifest_command = "python3 ../tools/generate-driver-release-manifest.py"
for workflow_name in ("dev-build.yml", "release.yml"):
workflow = (ROOT / ".github" / "workflows" / workflow_name).read_text(encoding="utf-8")
release_job = workflow.split("\n release:\n", 1)[1]
self.assertIn(setup_go_step, release_job)
self.assertLess(release_job.index(setup_go_step), release_job.index(manifest_command))
def _host_platform(self):
goos = subprocess.run(
["go", "env", "GOOS"],
@@ -142,6 +157,61 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
self.assertIn("src-stale-agent", proc.stderr)
self.assertFalse(output.exists())
def test_rejects_native_asset_when_sha_bound_provenance_disagrees_with_binary(self):
goos, goarch = self._host_platform()
platform = f"{goos}/{goarch}"
extension = ".exe" if goos == "windows" else ""
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-native-provenance-") as tmp:
tmpdir = Path(tmp)
assets_dir = tmpdir / "drivers"
assets_dir.mkdir(parents=True)
asset = assets_dir / f"clickhouse-driver-agent-{goos}-{goarch}{extension}"
self._build_metadata_agent(asset, "src-binary-revision")
content = asset.read_bytes()
provenance = tmpdir / "provenance.json"
provenance.write_text(
json.dumps(
{
"schemaVersion": 1,
"assets": {
asset.name: {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": platform,
"revision": "src-provenance-revision",
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
}
},
}
),
encoding="utf-8",
)
output = tmpdir / "manifest.json"
proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--assets-dir",
str(assets_dir),
"--output",
str(output),
"--provenance",
str(provenance),
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout)
self.assertIn("provenance revision mismatch", proc.stderr)
self.assertIn("src-binary-revision", proc.stderr)
self.assertIn("src-provenance-revision", proc.stderr)
self.assertFalse(output.exists())
def test_rejects_cross_platform_asset_without_binary_revision_provenance(self):
goos, goarch = self._cross_platform().split("/", 1)
extension = ".exe" if goos == "windows" else ""
@@ -169,8 +239,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
platform = self._cross_platform()
goos, goarch = platform.split("/", 1)
extension = ".exe" if goos == "windows" else ""
revision_file = self._generate_revision_file(platform)
revision = expected_revision(revision_file, "clickhouse")
revision = "src-build-provenance"
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-provenance-") as tmp:
tmpdir = Path(tmp)
@@ -221,7 +290,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
manifest = json.loads(output.read_text(encoding="utf-8"))
self.assertEqual(manifest["assets"][asset.name]["revision"], revision)
def test_rejects_sha_matching_provenance_with_stale_revision(self):
def test_accepts_sha_matching_provenance_from_the_build_runner(self):
platform = self._cross_platform()
goos, goarch = platform.split("/", 1)
extension = ".exe" if goos == "windows" else ""
@@ -242,7 +311,7 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": platform,
"revision": "src-stale-agent",
"revision": "src-build-runner",
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
}
@@ -270,10 +339,9 @@ class GenerateDriverReleaseManifestTest(unittest.TestCase):
text=True,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout)
self.assertIn("provenance revision mismatch", proc.stderr)
self.assertIn("src-stale-agent", proc.stderr)
self.assertFalse(output.exists())
self.assertEqual(proc.returncode, 0, proc.stderr)
manifest = json.loads(output.read_text(encoding="utf-8"))
self.assertEqual(manifest["assets"][asset.name]["revision"], "src-build-runner")
def test_writes_sha_bound_build_provenance_from_revision_file(self):
platform = self._cross_platform()

130
tools/validate-driver-release-manifest.sh
View File

@@ -11,8 +11,8 @@ usage() {
./tools/validate-driver-release-manifest.sh --commit <ref> --manifest <path>
说明：
校验已发布 driver release manifest 中记录的每个 driver revision，
是否与指定源码提交在对应平台上重新生成出的 revision 完全一致。
校验已发布 driver release manifest 是否与指定源码提交绑定，
且每个资产均包含完整的 SHA 绑定 provenance 元数据。
EOF
}
@@ -72,108 +72,56 @@ if ! git rev-parse --verify "${source_commit}^{commit}" >/dev/null 2>&1; then
echo "无法解析源码提交：$source_commit" >&2
exit 1
fi
source_commit="$(git rev-parse "${source_commit}^{commit}")"
extract_revision() {
local file="$1"
local driver="$2"
awk -v target="$driver" '
$0 ~ "\"" target "\"" {
if (match($0, /"src-[^"]+"/)) {
print substr($0, RSTART + 1, RLENGTH - 2)
exit
}
}
' "$file"
}
normalize_driver() {
local value
value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
case "$value" in
doris|diros) echo "diros" ;;
*) echo "$value" ;;
esac
}
worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-manifest.XXXXXX")"
cleanup() {
git worktree remove --force "$worktree" >/dev/null 2>&1 || true
rm -rf "$worktree"
}
trap cleanup EXIT
git worktree add --detach "$worktree" "${source_commit}^{commit}" >/dev/null
"$PYTHON_BIN" - "$manifest_path" "$worktree" <<'PY'
"$PYTHON_BIN" - "$manifest_path" "$source_commit" <<'PY'
import json
import subprocess
import re
import sys
from pathlib import Path
manifest_path = Path(sys.argv[1]).resolve()
worktree = Path(sys.argv[2]).resolve()
source_commit = sys.argv[2]
with manifest_path.open("r", encoding="utf-8") as fh:
manifest = json.load(fh)
assets = manifest.get("assets") or {}
errors = []
if manifest.get("schemaVersion") != 1:
errors.append(f"schemaVersion={manifest.get('schemaVersion')!r}，期望 1")
if str(manifest.get("generatedFrom") or "").strip() != source_commit:
errors.append(
f"generatedFrom={manifest.get('generatedFrom')!r}，期望源码提交 {source_commit}"
)
if not isinstance(assets, dict) or not assets:
raise SystemExit("manifest assets 为空")
errors.append("assets 为空或格式无效")
else:
for asset_name, meta in sorted(assets.items()):
if not isinstance(meta, dict):
errors.append(f"{asset_name}: 元数据格式无效")
continue
driver = str(meta.get("driver") or meta.get("driverType") or "").strip()
platform = str(meta.get("platform") or "").strip()
revision = str(meta.get("revision") or "").strip()
sha256 = str(meta.get("sha256") or "").strip().lower()
size = meta.get("size")
if not driver:
errors.append(f"{asset_name}: 缺少 driver")
if not re.fullmatch(r"[^/\\]+/[^/\\]+", platform):
errors.append(f"{asset_name}: platform 无效: {platform!r}")
if not revision.startswith("src-"):
errors.append(f"{asset_name}: revision 无效: {revision!r}")
if not re.fullmatch(r"[0-9a-f]{64}", sha256):
errors.append(f"{asset_name}: sha256 无效")
if not isinstance(size, int) or isinstance(size, bool) or size <= 0:
errors.append(f"{asset_name}: size 无效: {size!r}")
platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.values() if str(meta.get("platform") or "").strip()})
if not platforms:
raise SystemExit("manifest 未包含平台信息")
def normalize_driver(driver: str) -> str:
value = str(driver or "").strip().lower()
if value == "doris":
return "diros"
return value
def parse_revision_file(file_path: Path):
import re
text = file_path.read_text(encoding="utf-8")
revisions = {}
for match in re.finditer(r'"([^"]+)"\s*:\s*"([^"]+)"', text):
revisions[match.group(1)] = match.group(2)
return revisions
drivers_by_platform = {}
for meta in assets.values():
platform = str(meta.get("platform") or "").strip()
driver = normalize_driver(meta.get("driver") or meta.get("driverType"))
if platform and driver:
drivers_by_platform.setdefault(platform, set()).add(driver)
revision_maps = {}
for platform in platforms:
command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform]
platform_drivers = sorted(drivers_by_platform.get(platform) or [])
if platform_drivers:
command.extend(["--drivers", ",".join(platform_drivers)])
subprocess.run(command, cwd=worktree, check=True, stdout=subprocess.DEVNULL)
revision_maps[platform] = parse_revision_file(worktree / "internal/db/driver_agent_revisions_gen.go")
mismatches = []
for asset_name, meta in sorted(assets.items()):
driver = normalize_driver(meta.get("driver") or meta.get("driverType"))
platform = str(meta.get("platform") or "").strip()
published_revision = str(meta.get("revision") or "").strip()
expected_revision = (revision_maps.get(platform) or {}).get(driver, "")
if not expected_revision:
mismatches.append((asset_name, platform, driver, published_revision, "<missing>"))
continue
if published_revision != expected_revision:
mismatches.append((asset_name, platform, driver, published_revision, expected_revision))
if mismatches:
print("published driver release manifest 与源码重算 revision 不一致：", file=sys.stderr)
for asset_name, platform, driver, published_revision, expected_revision in mismatches:
print(
f" - {asset_name} [{platform}/{driver}] published={published_revision} expected={expected_revision}",
file=sys.stderr,
)
if errors:
print("published driver release manifest 元数据无效：", file=sys.stderr)
for error in errors:
print(f" - {error}", file=sys.stderr)
raise SystemExit(1)
print(f"manifest validation passed: {len(assets)} assets")

96
tools/validate-driver-release-manifest.test.sh
View File

@@ -6,17 +6,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$SCRIPT_DIR"
tmpdir="$(mktemp -d)"
worktrees=()
cleanup() {
local worktree
for worktree in "${worktrees[@]+"${worktrees[@]}"}"; do
git worktree remove --force "$worktree" >/dev/null 2>&1 || true
done
rm -rf "$tmpdir"
}
trap cleanup EXIT
manifest_path="$tmpdir/manifest.json"
resolve_python_bin() {
local candidate
for candidate in python3 python; do
@@ -34,97 +28,49 @@ if [[ -z "$PYTHON_BIN" ]]; then
exit 0
fi
generate_revision() {
local platform="$1"
local driver="$2"
local worktree revision_file
worktree="$tmpdir/worktree-${platform//\//-}-${driver}"
git worktree add --detach "$worktree" HEAD >/dev/null
worktrees+=("$worktree")
(
cd "$worktree"
bash ./tools/generate-driver-agent-revisions.sh --platform "$platform" --drivers "$driver" >/dev/null
)
revision_file="$worktree/internal/db/driver_agent_revisions_gen.go"
awk -v target="$driver" '
$0 ~ "\"" target "\"" {
if (match($0, /"src-[^"]+"/)) {
print substr($0, RSTART + 1, RLENGTH - 2)
exit
}
}
' "$revision_file"
}
cat >"$manifest_path" <<'EOF'
source_commit="$(git rev-parse HEAD)"
manifest_path="$tmpdir/manifest.json"
cat >"$manifest_path" <<EOF
{
"schemaVersion": 1,
"generatedFrom": "test",
"generatedFrom": "$source_commit",
"assets": {
"clickhouse-driver-agent-darwin-arm64": {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": "darwin/arm64",
"revision": "__CLICKHOUSE_DARWIN_ARM64__",
"revision": "src-build-runner",
"sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"size": 1
},
"clickhouse-driver-agent-linux-amd64": {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": "linux/amd64",
"revision": "__CLICKHOUSE_LINUX_AMD64__",
"size": 1
},
"clickhouse-driver-agent-linux-arm64": {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": "linux/arm64",
"revision": "__CLICKHOUSE_LINUX_ARM64__",
"size": 1
},
"clickhouse-driver-agent-windows-amd64.exe": {
"driver": "clickhouse",
"driverType": "clickhouse",
"platform": "windows/amd64",
"revision": "__CLICKHOUSE_WINDOWS_AMD64__",
"size": 1
},
"mariadb-driver-agent-darwin-arm64": {
"mariadb-driver-agent-linux-amd64": {
"driver": "mariadb",
"driverType": "mariadb",
"platform": "darwin/arm64",
"revision": "__MARIADB__",
"size": 1
"platform": "linux/amd64",
"revision": "src-build-runner",
"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"size": 2
}
}
}
EOF
clickhouse_darwin_revision="$(generate_revision darwin/arm64 clickhouse)"
clickhouse_linux_revision="$(generate_revision linux/amd64 clickhouse)"
clickhouse_linux_arm64_revision="$(generate_revision linux/arm64 clickhouse)"
clickhouse_windows_revision="$(generate_revision windows/amd64 clickhouse)"
mariadb_darwin_revision="$(generate_revision darwin/arm64 mariadb)"
bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"
"$PYTHON_BIN" - "$manifest_path" "$clickhouse_darwin_revision" "$clickhouse_linux_revision" "$clickhouse_linux_arm64_revision" "$clickhouse_windows_revision" "$mariadb_darwin_revision" <<'PY'
"$PYTHON_BIN" - "$manifest_path" <<'PY'
import json
import sys
from pathlib import Path
path = Path(sys.argv[1])
clickhouse_darwin = sys.argv[2]
clickhouse_linux = sys.argv[3]
clickhouse_linux_arm64 = sys.argv[4]
clickhouse_windows = sys.argv[5]
mariadb = sys.argv[6]
data = json.loads(path.read_text(encoding="utf-8"))
data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse_darwin
data["assets"]["clickhouse-driver-agent-linux-amd64"]["revision"] = clickhouse_linux
data["assets"]["clickhouse-driver-agent-linux-arm64"]["revision"] = clickhouse_linux_arm64
data["assets"]["clickhouse-driver-agent-windows-amd64.exe"]["revision"] = clickhouse_windows
data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
payload = json.loads(path.read_text(encoding="utf-8"))
payload["generatedFrom"] = "0" * 40
path.write_text(json.dumps(payload), encoding="utf-8")
PY
bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"
if bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"; then
echo "expected manifest source-commit mismatch to fail" >&2
exit 1
fi
echo "validate-driver-release-manifest test passed"