From f5ee61f589f083210c01caf36b9fb1f752b525db Mon Sep 17 00:00:00 2001 From: Syngnat Date: Fri, 12 Jun 2026 15:57:04 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(ci/driver):=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E9=A9=B1=E5=8A=A8=20manifest=20=E8=B7=A8=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E6=A0=A1=E9=AA=8C=E8=AF=AF=E5=88=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修复 validate-driver-release-manifest 复用同一 revision 文件路径的问题 - 按平台生成后立即解析 revision 快照，避免全部平台读到最后一次生成结果 - 补充 darwin/linux/windows 跨平台 manifest 校验回归测试 - 验证线上 dev manifest 可通过 89 个资产 revision 校验 --- tools/validate-driver-release-manifest.sh | 44 +++++++----- .../validate-driver-release-manifest.test.sh | 68 +++++++++++++++---- 2 files changed, 81 insertions(+), 31 deletions(-) diff --git a/tools/validate-driver-release-manifest.sh b/tools/validate-driver-release-manifest.sh index f2b41ae..5126475 100644 --- a/tools/validate-driver-release-manifest.sh +++ b/tools/validate-driver-release-manifest.sh @@ -107,31 +107,43 @@ platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.v if not platforms: raise SystemExit("manifest 未包含平台信息") -def extract_revision(file_path: Path, driver: str) -> str: +def normalize_driver(driver: str) -> str: + value = str(driver or "").strip().lower() + if value == "doris": + return "diros" + return value + + +def parse_revision_file(file_path: Path): import re text = file_path.read_text(encoding="utf-8") - pattern = re.compile(rf'"{re.escape(driver)}"\s*:\s*"([^"]+)"') - match = pattern.search(text) - return match.group(1) if match else "" + revisions = {} + for match in re.finditer(r'"([^"]+)"\s*:\s*"([^"]+)"', text): + revisions[match.group(1)] = match.group(2) + return revisions -revision_files = {} +drivers_by_platform = {} +for meta in assets.values(): + platform = str(meta.get("platform") or "").strip() + driver = normalize_driver(meta.get("driver") or meta.get("driverType")) + if platform and driver: + drivers_by_platform.setdefault(platform, set()).add(driver) + +revision_maps = {} for platform in platforms: - subprocess.run( - ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform], - cwd=worktree, - check=True, - stdout=subprocess.DEVNULL, - ) - revision_files[platform] = worktree / "internal/db/driver_agent_revisions_gen.go" + command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform] + platform_drivers = sorted(drivers_by_platform.get(platform) or []) + if platform_drivers: + command.extend(["--drivers", ",".join(platform_drivers)]) + subprocess.run(command, cwd=worktree, check=True, stdout=subprocess.DEVNULL) + revision_maps[platform] = parse_revision_file(worktree / "internal/db/driver_agent_revisions_gen.go") mismatches = [] for asset_name, meta in sorted(assets.items()): - driver = str(meta.get("driver") or meta.get("driverType") or "").strip().lower() - if driver == "doris": - driver = "diros" + driver = normalize_driver(meta.get("driver") or meta.get("driverType")) platform = str(meta.get("platform") or "").strip() published_revision = str(meta.get("revision") or "").strip() - expected_revision = extract_revision(revision_files[platform], driver) + expected_revision = (revision_maps.get(platform) or {}).get(driver, "") if not expected_revision: mismatches.append((asset_name, platform, driver, published_revision, "")) continue diff --git a/tools/validate-driver-release-manifest.test.sh b/tools/validate-driver-release-manifest.test.sh index af2b50e..6e3e2e6 100644 --- a/tools/validate-driver-release-manifest.test.sh +++ b/tools/validate-driver-release-manifest.test.sh @@ -6,20 +6,39 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$SCRIPT_DIR" tmpdir="$(mktemp -d)" +worktrees=() cleanup() { + local worktree + for worktree in "${worktrees[@]+"${worktrees[@]}"}"; do + git worktree remove --force "$worktree" >/dev/null 2>&1 || true + done rm -rf "$tmpdir" } trap cleanup EXIT manifest_path="$tmpdir/manifest.json" -worktree="$tmpdir/worktree" -git worktree add --detach "$worktree" HEAD >/dev/null -trap 'git worktree remove --force "$worktree" >/dev/null 2>&1 || true; rm -rf "$tmpdir"' EXIT -( - cd "$worktree" - bash ./tools/generate-driver-agent-revisions.sh --platform darwin/arm64 >/dev/null -) +generate_revision() { + local platform="$1" + local driver="$2" + local worktree revision_file + worktree="$tmpdir/worktree-${platform//\//-}-${driver}" + git worktree add --detach "$worktree" HEAD >/dev/null + worktrees+=("$worktree") + ( + cd "$worktree" + bash ./tools/generate-driver-agent-revisions.sh --platform "$platform" --drivers "$driver" >/dev/null + ) + revision_file="$worktree/internal/db/driver_agent_revisions_gen.go" + awk -v target="$driver" ' + $0 ~ "\"" target "\"" { + if (match($0, /"src-[^"]+"/)) { + print substr($0, RSTART + 1, RLENGTH - 2) + exit + } + } + ' "$revision_file" +} cat >"$manifest_path" <<'EOF' { @@ -30,7 +49,21 @@ cat >"$manifest_path" <<'EOF' "driver": "clickhouse", "driverType": "clickhouse", "platform": "darwin/arm64", - "revision": "__CLICKHOUSE__", + "revision": "__CLICKHOUSE_DARWIN_ARM64__", + "size": 1 + }, + "clickhouse-driver-agent-linux-amd64": { + "driver": "clickhouse", + "driverType": "clickhouse", + "platform": "linux/amd64", + "revision": "__CLICKHOUSE_LINUX_AMD64__", + "size": 1 + }, + "clickhouse-driver-agent-windows-amd64.exe": { + "driver": "clickhouse", + "driverType": "clickhouse", + "platform": "windows/amd64", + "revision": "__CLICKHOUSE_WINDOWS_AMD64__", "size": 1 }, "mariadb-driver-agent-darwin-arm64": { @@ -44,20 +77,25 @@ cat >"$manifest_path" <<'EOF' } EOF -revision_file="$worktree/internal/db/driver_agent_revisions_gen.go" -clickhouse_revision="$(awk '/"clickhouse"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")" -mariadb_revision="$(awk '/"mariadb"/ { if (match($0, /"src-[^"]+"/)) { print substr($0, RSTART + 1, RLENGTH - 2); exit } }' "$revision_file")" +clickhouse_darwin_revision="$(generate_revision darwin/arm64 clickhouse)" +clickhouse_linux_revision="$(generate_revision linux/amd64 clickhouse)" +clickhouse_windows_revision="$(generate_revision windows/amd64 clickhouse)" +mariadb_darwin_revision="$(generate_revision darwin/arm64 mariadb)" -python3 - "$manifest_path" "$clickhouse_revision" "$mariadb_revision" <<'PY' +python3 - "$manifest_path" "$clickhouse_darwin_revision" "$clickhouse_linux_revision" "$clickhouse_windows_revision" "$mariadb_darwin_revision" <<'PY' import json import sys from pathlib import Path path = Path(sys.argv[1]) -clickhouse = sys.argv[2] -mariadb = sys.argv[3] +clickhouse_darwin = sys.argv[2] +clickhouse_linux = sys.argv[3] +clickhouse_windows = sys.argv[4] +mariadb = sys.argv[5] data = json.loads(path.read_text(encoding="utf-8")) -data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse +data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse_darwin +data["assets"]["clickhouse-driver-agent-linux-amd64"]["revision"] = clickhouse_linux +data["assets"]["clickhouse-driver-agent-windows-amd64.exe"]["revision"] = clickhouse_windows data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") PY