From be269707612e60d2f3bcb4bc667931e2d2dc6af2 Mon Sep 17 00:00:00 2001 From: Syngnat Date: Fri, 5 Jun 2026 16:10:05 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=84=8F=20driver=20release=20=E8=B5=84=E4=BA=A7=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=20revision=20=E9=94=99=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dev-build.yml | 35 +++- .github/workflows/release.yml | 60 +++++++ tools/generate-driver-release-manifest.py | 99 ++++++++++++ tools/resolve-driver-release-source.py | 35 ++++ tools/resolve-driver-release-source.test.py | 15 ++ tools/validate-driver-release-manifest.sh | 151 ++++++++++++++++++ .../validate-driver-release-manifest.test.sh | 66 ++++++++ 7 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 tools/generate-driver-release-manifest.py create mode 100644 tools/validate-driver-release-manifest.sh create mode 100644 tools/validate-driver-release-manifest.test.sh diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index d2a4a19..9963e96 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -71,6 +71,7 @@ jobs: has_changes: ${{ steps.detect.outputs.has_changes }} release_source: ${{ steps.detect.outputs.release_source }} source_commit: ${{ steps.published_source.outputs.source_commit }} + has_manifest: ${{ steps.published_source.outputs.has_manifest }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -84,8 +85,27 @@ jobs: shell: bash run: | set -euo pipefail - SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag dev-latest)" + manifest_path="$RUNNER_TEMP/published-driver-manifest.json" + SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag dev-latest --manifest-output "$manifest_path")" echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT" + if [[ -s "$manifest_path" ]]; then + echo "has_manifest=true" >> "$GITHUB_OUTPUT" + echo "🧭 Published dev driver release exposes revision manifest" + else + echo "has_manifest=false" >> "$GITHUB_OUTPUT" + echo "🧭 Published dev driver release has no revision manifest" + fi + if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then + if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then + echo "manifest_valid=true" >> "$GITHUB_OUTPUT" + echo "🧭 Published dev driver release manifest is consistent with its source commit" + else + echo "manifest_valid=false" >> "$GITHUB_OUTPUT" + echo "⚠️ Published dev driver release manifest is stale; forcing full rebuild" + fi + else + echo "manifest_valid=false" >> "$GITHUB_OUTPUT" + fi if [[ -n "$SOURCE_COMMIT" ]]; then echo "🧭 Last published dev driver release source commit: $SOURCE_COMMIT" else @@ -123,6 +143,15 @@ jobs: } BASE_REF="${{ steps.published_source.outputs.source_commit }}" + HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}" + MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}" + if [[ "$HAS_MANIFEST" != "true" ]]; then + echo "⚠️ Published driver release lacks revision manifest; forcing full rebuild to self-heal old assets" + BASE_REF="all" + elif [[ "$MANIFEST_VALID" != "true" ]]; then + echo "⚠️ Published driver release manifest is stale; forcing full rebuild to self-heal old assets" + BASE_REF="all" + fi if [[ -n "$BASE_REF" ]]; then if git rev-parse --verify "${BASE_REF}^{commit}" >/dev/null 2>&1 && git merge-base --is-ancestor "$BASE_REF" "$GITHUB_SHA"; then echo "🧭 Using last published driver release source commit as detection base: $BASE_REF" @@ -807,6 +836,10 @@ jobs: exit 0 fi + python3 tools/generate-driver-release-manifest.py \ + --assets-dir . \ + --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json + echo "📦 打包驱动总包：GoNavi-DriverAgents.zip" python3 - <<'PY' import json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 914234a..ec68e25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,12 +67,57 @@ jobs: has_changes: ${{ steps.detect.outputs.has_changes }} release_source: ${{ steps.detect.outputs.release_source }} compare_base: ${{ steps.detect.outputs.compare_base }} + published_source_commit: ${{ steps.published_source.outputs.source_commit }} + has_manifest: ${{ steps.published_source.outputs.has_manifest }} + manifest_valid: ${{ steps.published_source.outputs.manifest_valid }} steps: - name: Checkout code uses: actions/checkout@v5 with: fetch-depth: 0 + - name: Resolve published driver release source + id: published_source + env: + DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }} + shell: bash + run: | + set -euo pipefail + PREV_TAG="$(git describe --tags --match 'v*' --abbrev=0 "${GITHUB_SHA}^" 2>/dev/null || true)" + manifest_path="$RUNNER_TEMP/published-driver-manifest.json" + if [[ -z "$PREV_TAG" ]]; then + echo "source_commit=" >> "$GITHUB_OUTPUT" + echo "has_manifest=false" >> "$GITHUB_OUTPUT" + echo "manifest_valid=false" >> "$GITHUB_OUTPUT" + echo "🧭 No previous stable tag found; full rebuild will be used" + exit 0 + fi + SOURCE_COMMIT="$(python3 tools/resolve-driver-release-source.py --repo Syngnat/GoNavi-DriverAgents --tag "$PREV_TAG" --manifest-output "$manifest_path")" + echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT" + if [[ -s "$manifest_path" ]]; then + echo "has_manifest=true" >> "$GITHUB_OUTPUT" + echo "🧭 Published driver release exposes revision manifest" + else + echo "has_manifest=false" >> "$GITHUB_OUTPUT" + echo "🧭 Published driver release has no revision manifest" + fi + if [[ -n "$SOURCE_COMMIT" && -s "$manifest_path" ]]; then + if bash ./tools/validate-driver-release-manifest.sh --commit "$SOURCE_COMMIT" --manifest "$manifest_path"; then + echo "manifest_valid=true" >> "$GITHUB_OUTPUT" + echo "🧭 Published driver release manifest is consistent with its source commit" + else + echo "manifest_valid=false" >> "$GITHUB_OUTPUT" + echo "⚠️ Published driver release manifest is stale; forcing full rebuild" + fi + else + echo "manifest_valid=false" >> "$GITHUB_OUTPUT" + fi + if [[ -n "$SOURCE_COMMIT" ]]; then + echo "🧭 Last published release source commit (${PREV_TAG}): $SOURCE_COMMIT" + else + echo "🧭 Unable to resolve published release source commit for ${PREV_TAG}; fallback to previous tag diff" + fi + - name: Detect changed driver agents id: detect shell: bash @@ -112,6 +157,17 @@ jobs: BASE_REF="all" RELEASE_SOURCE="all" fi + HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}" + MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}" + if [[ "$RELEASE_SOURCE" != "all" && "$HAS_MANIFEST" != "true" ]]; then + echo "⚠️ Published driver release lacks revision manifest; forcing full rebuild to self-heal old assets" + BASE_REF="all" + RELEASE_SOURCE="all" + elif [[ "$RELEASE_SOURCE" != "all" && "$MANIFEST_VALID" != "true" ]]; then + echo "⚠️ Published driver release manifest is stale; forcing full rebuild to self-heal old assets" + BASE_REF="all" + RELEASE_SOURCE="all" + fi DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")" if [[ "$BASE_REF" != "all" ]]; then REVISION_DRIVERS="" @@ -837,6 +893,10 @@ jobs: exit 0 fi + python3 tools/generate-driver-release-manifest.py \ + --assets-dir . \ + --output ../driver-release-assets/GoNavi-DriverAgents-Manifest.json + echo "📦 打包驱动总包：GoNavi-DriverAgents.zip" python3 - <<'PY' import json diff --git a/tools/generate-driver-release-manifest.py b/tools/generate-driver-release-manifest.py new file mode 100644 index 0000000..c3ca55e --- /dev/null +++ b/tools/generate-driver-release-manifest.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets") + parser.add_argument("--output", required=True, help="manifest json output path") + return parser.parse_args() + + +def infer_driver_and_platform(file_name: str): + suffixes = [ + "-driver-agent-darwin-amd64", + "-driver-agent-darwin-arm64", + "-driver-agent-linux-amd64", + "-driver-agent-windows-amd64.exe", + "-driver-agent-windows-arm64.exe", + ] + for suffix in suffixes: + if file_name.endswith(suffix): + driver = file_name[: -len(suffix)] + if suffix.endswith(".exe"): + platform = suffix.replace("-driver-agent-", "").removesuffix(".exe") + else: + platform = suffix.replace("-driver-agent-", "") + return driver, platform + return None, None + + +def probe_revision(path: Path): + request = b'{"id":1,"method":"metadata"}\n' + proc = subprocess.run( + [str(path)], + input=request, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=10, + check=True, + ) + line = proc.stdout.decode("utf-8", errors="replace").strip().splitlines() + if not line: + raise RuntimeError(f"{path.name}: metadata response is empty") + payload = json.loads(line[0]) + data = payload.get("data") or {} + revision = str(data.get("agentRevision") or "").strip() + driver_type = str(data.get("driverType") or "").strip() + if not revision: + raise RuntimeError(f"{path.name}: metadata agentRevision is empty") + return driver_type, revision + + +def main(): + args = parse_args() + assets_dir = Path(args.assets_dir).resolve() + output_path = Path(args.output).resolve() + + manifest = { + "schemaVersion": 1, + "generatedFrom": os.environ.get("GITHUB_SHA", "").strip(), + "assets": {}, + } + + for child in sorted(assets_dir.rglob("*")): + if not child.is_file(): + continue + driver, platform = infer_driver_and_platform(child.name) + if not driver or not platform: + continue + if child.stat().st_size == 0: + raise RuntimeError(f"{child.name}: asset is empty") + driver_type, revision = probe_revision(child) + manifest["assets"][child.name] = { + "driver": driver, + "driverType": driver_type or driver, + "platform": platform, + "revision": revision, + "size": child.stat().st_size, + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"wrote manifest: {output_path}") + print(f"asset count: {len(manifest['assets'])}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except subprocess.TimeoutExpired as exc: + print(f"error: probe timed out for {exc.cmd}", file=sys.stderr) + raise diff --git a/tools/resolve-driver-release-source.py b/tools/resolve-driver-release-source.py index 876f5c4..cce846e 100644 --- a/tools/resolve-driver-release-source.py +++ b/tools/resolve-driver-release-source.py @@ -12,6 +12,7 @@ import urllib.request COMMIT_LINK_RE = re.compile(r"/commit/([0-9a-f]{40})(?:\b|/)") FULL_SHA_RE = re.compile(r"\b([0-9a-f]{40})\b") +MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json" def github_headers(): @@ -31,6 +32,15 @@ def fetch_json(url): return json.loads(response.read().decode("utf-8")) +def download_asset(asset, destination): + headers = github_headers() + headers["Accept"] = "application/octet-stream" + request = urllib.request.Request(asset["url"], headers=headers) + with urllib.request.urlopen(request, timeout=120) as response: + with open(destination, "wb") as output: + output.write(response.read()) + + def load_release(repo, tag): owner_repo = repo.strip() if not owner_repo: @@ -77,16 +87,41 @@ def extract_source_commit(release): return None +def find_manifest_asset(release): + if not isinstance(release, dict): + return None + + for asset in release.get("assets", []): + if str(asset.get("name") or "").strip() == MANIFEST_ASSET_NAME: + return asset + return None + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents") parser.add_argument("--tag", required=True, help="release tag name such as dev-latest or v1.0.0") + parser.add_argument("--manifest-output", help="optional path to download the published revision manifest asset") args = parser.parse_args() release = load_release(args.repo, args.tag) if release is None: return 0 + if args.manifest_output: + manifest_path = os.path.abspath(args.manifest_output) + manifest_asset = find_manifest_asset(release) + if manifest_asset is None: + if os.path.exists(manifest_path): + os.remove(manifest_path) + print( + f"warning: release {args.repo}@{args.tag} does not expose {MANIFEST_ASSET_NAME}", + file=sys.stderr, + ) + else: + os.makedirs(os.path.dirname(manifest_path), exist_ok=True) + download_asset(manifest_asset, manifest_path) + source_commit = extract_source_commit(release) if not source_commit: print( diff --git a/tools/resolve-driver-release-source.test.py b/tools/resolve-driver-release-source.test.py index 856c5a0..72fc56f 100644 --- a/tools/resolve-driver-release-source.test.py +++ b/tools/resolve-driver-release-source.test.py @@ -37,6 +37,21 @@ class ResolveDriverReleaseSourceTests(unittest.TestCase): release = {"body": "", "target_commitish": "main"} self.assertIsNone(MODULE.extract_source_commit(release)) + def test_finds_manifest_asset(self): + release = { + "assets": [ + {"name": "foo.txt"}, + {"name": "GoNavi-DriverAgents-Manifest.json", "url": "https://example.test/manifest"}, + ] + } + asset = MODULE.find_manifest_asset(release) + self.assertIsNotNone(asset) + self.assertEqual(asset["url"], "https://example.test/manifest") + + def test_returns_none_when_manifest_asset_missing(self): + release = {"assets": [{"name": "GoNavi-DriverAgents.zip"}]} + self.assertIsNone(MODULE.find_manifest_asset(release)) + if __name__ == "__main__": unittest.main() diff --git a/tools/validate-driver-release-manifest.sh b/tools/validate-driver-release-manifest.sh new file mode 100644 index 0000000..f2b41ae --- /dev/null +++ b/tools/validate-driver-release-manifest.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SCRIPT_DIR" + +usage() { + cat <<'EOF' +用法： + ./tools/validate-driver-release-manifest.sh --commit --manifest + +说明： + 校验已发布 driver release manifest 中记录的每个 driver revision， + 是否与指定源码提交在对应平台上重新生成出的 revision 完全一致。 +EOF +} + +source_commit="" +manifest_path="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --commit) + source_commit="${2:-}" + shift 2 + ;; + --manifest) + manifest_path="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "未知参数：$1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$source_commit" || -z "$manifest_path" ]]; then + usage >&2 + exit 1 +fi + +if [[ ! -f "$manifest_path" ]]; then + echo "manifest 不存在：$manifest_path" >&2 + exit 1 +fi + +if ! git rev-parse --verify "${source_commit}^{commit}" >/dev/null 2>&1; then + echo "无法解析源码提交：$source_commit" >&2 + exit 1 +fi + +extract_revision() { + local file="$1" + local driver="$2" + awk -v target="$driver" ' + $0 ~ "\"" target "\"" { + if (match($0, /"src-[^"]+"/)) { + print substr($0, RSTART + 1, RLENGTH - 2) + exit + } + } + ' "$file" +} + +normalize_driver() { + local value + value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + case "$value" in + doris|diros) echo "diros" ;; + *) echo "$value" ;; + esac +} + +worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-manifest.XXXXXX")" +cleanup() { + git worktree remove --force "$worktree" >/dev/null 2>&1 || true + rm -rf "$worktree" +} +trap cleanup EXIT + +git worktree add --detach "$worktree" "${source_commit}^{commit}" >/dev/null + +python3 - "$manifest_path" "$worktree" <<'PY' +import json +import subprocess +import sys +from pathlib import Path + +manifest_path = Path(sys.argv[1]).resolve() +worktree = Path(sys.argv[2]).resolve() + +with manifest_path.open("r", encoding="utf-8") as fh: + manifest = json.load(fh) + +assets = manifest.get("assets") or {} +if not isinstance(assets, dict) or not assets: + raise SystemExit("manifest assets 为空") + +platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.values() if str(meta.get("platform") or "").strip()}) +if not platforms: + raise SystemExit("manifest 未包含平台信息") + +def extract_revision(file_path: Path, driver: str) -> str: + import re + text = file_path.read_text(encoding="utf-8") + pattern = re.compile(rf'"{re.escape(driver)}"\s*:\s*"([^"]+)"') + match = pattern.search(text) + return match.group(1) if match else "" + +revision_files = {} +for platform in platforms: + subprocess.run( + ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform], + cwd=worktree, + check=True, + stdout=subprocess.DEVNULL, + ) + revision_files[platform] = worktree / "internal/db/driver_agent_revisions_gen.go" + +mismatches = [] +for asset_name, meta in sorted(assets.items()): + driver = str(meta.get("driver") or meta.get("driverType") or "").strip().lower() + if driver == "doris": + driver = "diros" + platform = str(meta.get("platform") or "").strip() + published_revision = str(meta.get("revision") or "").strip() + expected_revision = extract_revision(revision_files[platform], driver) + if not expected_revision: + mismatches.append((asset_name, platform, driver, published_revision, "")) + continue + if published_revision != expected_revision: + mismatches.append((asset_name, platform, driver, published_revision, expected_revision)) + +if mismatches: + print("published driver release manifest 与源码重算 revision 不一致：", file=sys.stderr) + for asset_name, platform, driver, published_revision, expected_revision in mismatches: + print( + f" - {asset_name} [{platform}/{driver}] published={published_revision} expected={expected_revision}", + file=sys.stderr, + ) + raise SystemExit(1) + +print(f"manifest validation passed: {len(assets)} assets") +PY diff --git a/tools/validate-driver-release-manifest.test.sh b/tools/validate-driver-release-manifest.test.sh new file mode 100644 index 0000000..af2b50e --- /dev/null +++ b/tools/validate-driver-release-manifest.test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SCRIPT_DIR" + +tmpdir="$(mktemp -d)" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +manifest_path="$tmpdir/manifest.json" +worktree="$tmpdir/worktree" +git worktree add --detach "$worktree" HEAD >/dev/null +trap 'git worktree remove --force "$worktree" >/dev/null 2>&1 || true; rm -rf "$tmpdir"' EXIT + +( + cd "$worktree" + bash ./tools/generate-driver-agent-revisions.sh --platform darwin/arm64 >/dev/null +) + +cat >"$manifest_path" <<'EOF' +{ + "schemaVersion": 1, + "generatedFrom": "test", + "assets": { + "clickhouse-driver-agent-darwin-arm64": { + "driver": "clickhouse", + "driverType": "clickhouse", + "platform": "darwin/arm64", + "revision": "__CLICKHOUSE__", + "size": 1 + }, + "mariadb-driver-agent-darwin-arm64": { + "driver": "mariadb", + "driverType": "mariadb", + "platform": "darwin/arm64", + "revision": "__MARIADB__", + "size": 1 + } + } +} +EOF + +revision_file="$worktree/internal/db/driver_agent_revisions_gen.go" +clickhouse_revision="$(awk '/"clickhouse"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")" +mariadb_revision="$(awk '/"mariadb"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")" + +python3 - "$manifest_path" "$clickhouse_revision" "$mariadb_revision" <<'PY' +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +clickhouse = sys.argv[2] +mariadb = sys.argv[3] +data = json.loads(path.read_text(encoding="utf-8")) +data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse +data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb +path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY + +bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path" +echo "validate-driver-release-manifest test passed"