mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-07-06 10:31:30 +08:00
Merge branch 'dev' into feature/20260602_connection_driver_i18n
# Conflicts: # frontend/package.json.md5 # frontend/src/App.tsx # frontend/src/components/AIChatPanel.message-boundary.test.tsx # frontend/src/components/AIChatPanel.tsx # frontend/src/components/AISettingsModal.tsx # frontend/src/components/ConnectionModal.tsx # frontend/src/components/DataGrid.ddl.test.tsx # frontend/src/components/DataGrid.layout.test.tsx # frontend/src/components/DataGrid.tsx # frontend/src/components/DataGridColumnTitle.test.tsx # frontend/src/components/DataGridLegacyCellContextMenu.tsx # frontend/src/components/DataGridSecondaryActions.tsx # frontend/src/components/DataGridToolbarFrame.tsx # frontend/src/components/DataSyncModal.tsx # frontend/src/components/DataViewer.tsx # frontend/src/components/DefinitionViewer.tsx # frontend/src/components/DriverManagerModal.tsx # frontend/src/components/QueryEditor.external-sql-save.test.tsx # frontend/src/components/QueryEditor.tsx # frontend/src/components/RedisViewer.tsx # frontend/src/components/Sidebar.locate-toolbar.test.tsx # frontend/src/components/Sidebar.tsx # frontend/src/components/TabManager.hover.test.tsx # frontend/src/components/TabManager.tsx # frontend/src/components/TableDesigner.tsx # frontend/src/components/V2TableContextMenu.tsx # frontend/src/components/ai/AIChatHeader.tsx # frontend/src/components/ai/AIHistoryDrawer.tsx # frontend/src/main.tsx # frontend/src/store.ts # frontend/src/utils/aiComposerNotice.test.ts # frontend/src/utils/aiComposerNotice.ts # frontend/src/utils/connectionModalPresentation.ts # frontend/src/utils/driverImportGuidance.ts # frontend/src/utils/externalSqlTree.test.ts # frontend/src/utils/externalSqlTree.ts # frontend/src/utils/sqlDialect.ts # internal/ai/service/service.go
This commit is contained in:
tianqijiuyun-latiao
2
tools/claude-gonavi-mcp.cmd
Normal file
2
tools/claude-gonavi-mcp.cmd
Normal file
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0claude-gonavi-mcp.ps1" %*
|
||||
44
tools/claude-gonavi-mcp.ps1
Normal file
44
tools/claude-gonavi-mcp.ps1
Normal file
@@ -0,0 +1,44 @@
|
||||
param(
|
||||
[switch]$SkipBuild
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ClaudeArgs = $args
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
$binDir = Join-Path $repoRoot 'bin'
|
||||
$serverExe = Join-Path $binDir 'gonavi-mcp-server.exe'
|
||||
|
||||
if (-not $SkipBuild) {
|
||||
if (-not (Test-Path $binDir)) {
|
||||
New-Item -ItemType Directory -Path $binDir | Out-Null
|
||||
}
|
||||
|
||||
& go build -o $serverExe .\cmd\gonavi-mcp-server
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "构建 gonavi-mcp-server 失败"
|
||||
}
|
||||
} elseif (-not (Test-Path $serverExe)) {
|
||||
throw "未找到已编译的 gonavi-mcp-server.exe，请去掉 -SkipBuild 或先手动构建"
|
||||
}
|
||||
|
||||
$mcpConfig = @{
|
||||
mcpServers = @{
|
||||
gonavi = @{
|
||||
type = 'stdio'
|
||||
command = $serverExe
|
||||
args = @()
|
||||
env = @{}
|
||||
}
|
||||
}
|
||||
} | ConvertTo-Json -Compress -Depth 6
|
||||
|
||||
$tempConfig = Join-Path ([System.IO.Path]::GetTempPath()) ("gonavi-claude-mcp-" + [System.Guid]::NewGuid().ToString("N") + ".json")
|
||||
|
||||
try {
|
||||
Set-Content -LiteralPath $tempConfig -Value $mcpConfig -Encoding UTF8
|
||||
& claude @ClaudeArgs --mcp-config $tempConfig --strict-mcp-config
|
||||
exit $LASTEXITCODE
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $tempConfig -ErrorAction SilentlyContinue
|
||||
}
|
||||
@@ -27,9 +27,13 @@ DRIVERS = [
|
||||
"highgo",
|
||||
"vastbase",
|
||||
"opengauss",
|
||||
"gaussdb",
|
||||
"iris",
|
||||
"mongodb",
|
||||
"tdengine",
|
||||
"iotdb",
|
||||
"clickhouse",
|
||||
"elasticsearch",
|
||||
]
|
||||
|
||||
BUNDLE_NAME = "GoNavi-DriverAgents.zip"
|
||||
|
||||
@@ -59,7 +59,11 @@ goos="${platform%%/*}"
|
||||
goarch="${platform##*/}"
|
||||
|
||||
case "$goos/$goarch" in
|
||||
linux/amd64|linux/arm64|windows/amd64)
|
||||
windows/amd64)
|
||||
echo "ℹ️ Windows amd64 驱动产物不执行 UPX 压缩：$label"
|
||||
exit 0
|
||||
;;
|
||||
linux/amd64|linux/arm64)
|
||||
;;
|
||||
*)
|
||||
echo "ℹ️ UPX 跳过不支持的平台：$label ($platform)"
|
||||
@@ -67,6 +71,56 @@ case "$goos/$goarch" in
|
||||
;;
|
||||
esac
|
||||
|
||||
host_platform="$(go env GOOS)/$(go env GOARCH)"
|
||||
|
||||
is_driver_agent_binary() {
|
||||
local artifact_name
|
||||
artifact_name="$(basename "$artifact_path")"
|
||||
[[ "$artifact_name" == *"-driver-agent-"* ]]
|
||||
}
|
||||
|
||||
can_smoke_test_driver_agent() {
|
||||
is_driver_agent_binary && [[ "$host_platform" == "$platform" ]]
|
||||
}
|
||||
|
||||
smoke_test_driver_agent_metadata() {
|
||||
local stdout_file stderr_file
|
||||
stdout_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-driver-agent-stdout.XXXXXX")"
|
||||
stderr_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-driver-agent-stderr.XXXXXX")"
|
||||
|
||||
if ! printf '%s\n' '{"id":1,"method":"metadata"}' | "$artifact_path" >"$stdout_file" 2>"$stderr_file"; then
|
||||
[[ -s "$stderr_file" ]] && cat "$stderr_file" >&2
|
||||
rm -f "$stdout_file" "$stderr_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! python3 - "$stdout_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
lines = [line.strip() for line in Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()]
|
||||
if not lines:
|
||||
raise SystemExit(1)
|
||||
|
||||
payload = json.loads(lines[0])
|
||||
if not payload.get("success"):
|
||||
raise SystemExit(1)
|
||||
|
||||
data = payload.get("data") or {}
|
||||
if not str(data.get("agentRevision") or "").strip():
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
then
|
||||
[[ -s "$stderr_file" ]] && cat "$stderr_file" >&2
|
||||
rm -f "$stdout_file" "$stderr_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -f "$stdout_file" "$stderr_file"
|
||||
return 0
|
||||
}
|
||||
|
||||
if ! command -v upx >/dev/null 2>&1; then
|
||||
if [[ "$mode" == "required" ]]; then
|
||||
echo "❌ 未找到 upx，无法压缩：$label" >&2
|
||||
@@ -124,6 +178,18 @@ if ! upx -t "$artifact_path" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if can_smoke_test_driver_agent; then
|
||||
if ! smoke_test_driver_agent_metadata; then
|
||||
cp "$backup_path" "$artifact_path"
|
||||
if [[ "$mode" == "required" ]]; then
|
||||
echo "❌ UPX 压缩后 driver-agent metadata 自检失败：$label" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "⚠️ UPX 压缩后 driver-agent metadata 自检失败，已恢复原文件：$label"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
after_bytes="$(file_size_bytes "$artifact_path")"
|
||||
if [[ "$after_bytes" -lt "$before_bytes" ]]; then
|
||||
saved_bytes=$((before_bytes - after_bytes))
|
||||
|
||||
118
tools/compress-driver-artifact.test.sh
Executable file
118
tools/compress-driver-artifact.test.sh
Executable file
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-compress-driver-artifact.XXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
windows_amd64_bin="$tmpdir/windows-driver-agent-windows-amd64.exe"
|
||||
printf 'fake windows amd64 driver' >"$windows_amd64_bin"
|
||||
windows_amd64_before="$(cat "$windows_amd64_bin")"
|
||||
windows_skip_output="$(bash "$repo_root/tools/compress-driver-artifact.sh" "$windows_amd64_bin" "windows/amd64" "windows-amd64-driver" 2>&1)"
|
||||
if [[ "$windows_skip_output" != *"Windows amd64 驱动产物不执行 UPX 压缩"* ]]; then
|
||||
echo "expected Windows amd64 driver artifact UPX compression to be skipped" >&2
|
||||
echo "$windows_skip_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$(cat "$windows_amd64_bin")" != "$windows_amd64_before" ]]; then
|
||||
echo "expected Windows amd64 driver artifact to remain unchanged when UPX is skipped" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
host_platform="$(go env GOOS)/$(go env GOARCH)"
|
||||
case "$host_platform" in
|
||||
linux/amd64|linux/arm64)
|
||||
;;
|
||||
*)
|
||||
echo "skip compress-driver-artifact smoke test on unsupported host platform: $host_platform"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
suffix=""
|
||||
if [[ "$host_platform" == windows/* ]]; then
|
||||
suffix=".exe"
|
||||
fi
|
||||
|
||||
good_src="$tmpdir/good.go"
|
||||
bad_src="$tmpdir/bad.go"
|
||||
good_bin="$tmpdir/good-driver-agent-${host_platform/\//-}${suffix}"
|
||||
bad_bin="$tmpdir/bad-driver-agent-${host_platform/\//-}${suffix}"
|
||||
recover_bin="$tmpdir/recover-driver-agent-${host_platform/\//-}${suffix}"
|
||||
fakebin="$tmpdir/bin"
|
||||
mkdir -p "$fakebin"
|
||||
|
||||
cat >"$good_src" <<'GOEOF'
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
if strings.TrimSpace(scanner.Text()) == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Println(`{"id":1,"success":true,"data":{"agentRevision":"src-test"}}`)
|
||||
return
|
||||
}
|
||||
}
|
||||
GOEOF
|
||||
|
||||
cat >"$bad_src" <<'GOEOF'
|
||||
package main
|
||||
|
||||
func main() {}
|
||||
GOEOF
|
||||
|
||||
go build -o "$good_bin" "$good_src"
|
||||
go build -o "$bad_bin" "$bad_src"
|
||||
cp "$good_bin" "$recover_bin"
|
||||
|
||||
cat >"$fakebin/upx" <<'SHEOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${1:-}" == "-t" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
target="${*: -1}"
|
||||
if [[ -n "${FAKE_UPX_REPLACE:-}" && "$target" == *"recover-driver-agent-"* ]]; then
|
||||
cp "$FAKE_UPX_REPLACE" "$target"
|
||||
fi
|
||||
SHEOF
|
||||
chmod +x "$fakebin/upx"
|
||||
|
||||
PATH="$fakebin:$PATH" bash "$repo_root/tools/compress-driver-artifact.sh" "$good_bin" "$host_platform" "good"
|
||||
metadata_output="$(printf '%s\n' '{"id":1,"method":"metadata"}' | "$good_bin")"
|
||||
if [[ "$metadata_output" != *'"agentRevision":"src-test"'* ]]; then
|
||||
echo "expected metadata smoke test to keep good driver-agent executable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
warning_output="$(
|
||||
FAKE_UPX_REPLACE="$bad_bin" PATH="$fakebin:$PATH" bash "$repo_root/tools/compress-driver-artifact.sh" "$recover_bin" "$host_platform" "recover" 2>&1
|
||||
)"
|
||||
if [[ "$warning_output" != *"metadata 自检失败"* ]]; then
|
||||
echo "expected metadata smoke-test warning when fake UPX replacement breaks the executable" >&2
|
||||
echo "$warning_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
recovered_output="$(printf '%s\n' '{"id":1,"method":"metadata"}' | "$recover_bin")"
|
||||
if [[ "$recovered_output" != *'"agentRevision":"src-test"'* ]]; then
|
||||
echo "expected metadata smoke-test failure to restore original driver-agent binary" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "compress-driver-artifact smoke test passed"
|
||||
@@ -7,7 +7,7 @@ cd "$SCRIPT_DIR"
|
||||
SCRIPT_DIR_WINDOWS="$(pwd -W 2>/dev/null || true)"
|
||||
SCRIPT_DIR_WINDOWS="${SCRIPT_DIR_WINDOWS//\\//}"
|
||||
|
||||
DEFAULT_DRIVERS=(mariadb oceanbase doris starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss iris mongodb tdengine clickhouse)
|
||||
DEFAULT_DRIVERS=(mariadb oceanbase doris starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss gaussdb iris mongodb tdengine iotdb clickhouse elasticsearch)
|
||||
TARGET_PLATFORMS=(darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64)
|
||||
|
||||
usage() {
|
||||
@@ -21,6 +21,8 @@ usage() {
|
||||
说明：
|
||||
通过 go list -deps 计算每个 driver-agent 的真实源码依赖，再与 git diff 文件求交集。
|
||||
如果无法解析基准或依赖分析失败，会保守输出全部 driver。
|
||||
如果 driver 构建 / 发布工作流本身发生变化，也会保守输出全部 driver，
|
||||
避免新应用 revision 与旧 driver-assets 再次错配。
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -50,7 +52,9 @@ normalize_driver() {
|
||||
case "$value" in
|
||||
doris|diros) echo "doris" ;;
|
||||
open_gauss|open-gauss) echo "opengauss" ;;
|
||||
mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|opengauss|iris|mongodb|tdengine|clickhouse)
|
||||
gaussdb|gauss_db|gauss-db) echo "gaussdb" ;;
|
||||
elastic|elasticsearch) echo "elasticsearch" ;;
|
||||
mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|opengauss|gaussdb|iris|mongodb|tdengine|iotdb|clickhouse)
|
||||
echo "$value"
|
||||
;;
|
||||
*)
|
||||
@@ -154,10 +158,13 @@ driver_tokens_from_text() {
|
||||
case "$text" in *highgo*) emit_driver_token highgo ;; esac
|
||||
case "$text" in *vastbase*) emit_driver_token vastbase ;; esac
|
||||
case "$text" in *opengauss*) emit_driver_token opengauss ;; esac
|
||||
case "$text" in *gaussdb*|*gauss_db*|*gauss-db*) emit_driver_token gaussdb ;; esac
|
||||
case "$text" in *iris*) emit_driver_token iris ;; esac
|
||||
case "$text" in *mongodb*) emit_driver_token mongodb ;; esac
|
||||
case "$text" in *tdengine*) emit_driver_token tdengine ;; esac
|
||||
case "$text" in *iotdb*|*apache-iotdb*|*apache_iotdb*) emit_driver_token iotdb ;; esac
|
||||
case "$text" in *clickhouse*) emit_driver_token clickhouse ;; esac
|
||||
case "$text" in *elasticsearch*) emit_driver_token elasticsearch ;; esac
|
||||
|
||||
case "$text" in
|
||||
*github.com/go-sql-driver/mysql*)
|
||||
@@ -180,10 +187,13 @@ driver_tokens_from_text() {
|
||||
emit_driver_token opengauss
|
||||
;;
|
||||
esac
|
||||
case "$text" in *github.com/!huawei!cloud!developer/gaussdb-go*|*github.com/HuaweiCloudDeveloper/gaussdb-go*) emit_driver_token gaussdb ;; esac
|
||||
case "$text" in *github.com/caretdev/go-irisnative*|*third_party/go-irisnative*) emit_driver_token iris ;; esac
|
||||
case "$text" in *go.mongodb.org/mongo-driver*|*go.mongodb.org/mongo-driver/v2*) emit_driver_token mongodb ;; esac
|
||||
case "$text" in *github.com/taosdata/driver-go/v3*) emit_driver_token tdengine ;; esac
|
||||
case "$text" in *github.com/apache/iotdb-client-go*) emit_driver_token iotdb ;; esac
|
||||
case "$text" in *github.com/clickhouse/clickhouse-go/v2*|*github.com/clickhouse/ch-go*) emit_driver_token clickhouse ;; esac
|
||||
case "$text" in *github.com/elastic/go-elasticsearch/v8*) emit_driver_token elasticsearch ;; esac
|
||||
}
|
||||
|
||||
emit_driver_token() {
|
||||
@@ -317,9 +327,31 @@ add_all_forced_drivers() {
|
||||
done
|
||||
}
|
||||
|
||||
revision_file_changed_drivers() {
|
||||
local line driver emitted_seen
|
||||
emitted_seen="|"
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
+++*|---*|@@*)
|
||||
continue
|
||||
;;
|
||||
+*|-*)
|
||||
if [[ "$line" =~ \"([^\"]+)\"[[:space:]]*:[[:space:]]*\"src-[^\"]+\" ]]; then
|
||||
driver="$(normalize_driver "${BASH_REMATCH[1]}")" || continue
|
||||
if [[ "$emitted_seen" == *"|$driver|"* ]]; then
|
||||
continue
|
||||
fi
|
||||
printf '%s\n' "$driver"
|
||||
emitted_seen="${emitted_seen}${driver}|"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done < <(git diff --unified=0 "$base_commit" "$head_commit" -- internal/db/driver_agent_revisions_gen.go)
|
||||
}
|
||||
|
||||
is_ignored_driver_agent_source_file() {
|
||||
case "$1" in
|
||||
*_test.go|frontend/*|internal/app/*|internal/db/driver_agent_revisions_gen.go)
|
||||
*_test.go|frontend/*|internal/app/*|internal/appdata/*|internal/connection/*|internal/logger/*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -462,10 +494,35 @@ if [[ ${#changed_file_set[@]} -eq 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
has_revision_file_change=false
|
||||
only_workflow_changes=true
|
||||
for file in "${!changed_file_set[@]}"; do
|
||||
case "$file" in
|
||||
internal/db/driver_agent_revisions_gen.go)
|
||||
has_revision_file_change=true
|
||||
only_workflow_changes=false
|
||||
;;
|
||||
.github/workflows/dev-build.yml|.github/workflows/release.yml)
|
||||
;;
|
||||
*)
|
||||
only_workflow_changes=false
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
declare -a forced_changed_drivers=()
|
||||
forced_driver_seen="|"
|
||||
for file in "${!changed_file_set[@]}"; do
|
||||
case "$file" in
|
||||
internal/db/driver_agent_revisions_gen.go)
|
||||
revision_delta="$(revision_file_changed_drivers)"
|
||||
if [[ -z "$revision_delta" ]]; then
|
||||
echo "检测到 driver-agent revision 文件存在无法归因的变更；保守构建全部 driver-agent：$file" >&2
|
||||
all_drivers_csv
|
||||
exit 0
|
||||
fi
|
||||
add_forced_drivers_from_tokens "$revision_delta"
|
||||
;;
|
||||
go.mod|go.sum|build-driver-agents.sh|tools/generate-driver-agent-revisions.sh)
|
||||
set +e
|
||||
shared_delta="$(shared_file_driver_delta "$file")"
|
||||
@@ -478,13 +535,33 @@ for file in "${!changed_file_set[@]}"; do
|
||||
fi
|
||||
add_forced_drivers_from_tokens "$shared_delta"
|
||||
;;
|
||||
tools/compress-driver-artifact.sh)
|
||||
echo "检测到 driver-agent 压缩脚本变更；保守构建全部 driver-agent：$file" >&2
|
||||
tools/compress-driver-artifact.sh|\
|
||||
tools/package-driver-release-assets.py|\
|
||||
tools/generate-driver-release-manifest.py|\
|
||||
tools/validate-driver-release-assets.py|\
|
||||
tools/complete-driver-release-assets.py|\
|
||||
tools/resolve-driver-release-source.py|\
|
||||
tools/validate-driver-release-manifest.sh|\
|
||||
tools/should-force-global-driver-builds.sh)
|
||||
echo "检测到 driver-agent 构建/发布链路脚本变更；保守构建全部 driver-agent：$file" >&2
|
||||
all_drivers_csv
|
||||
exit 0
|
||||
;;
|
||||
.github/workflows/dev-build.yml|.github/workflows/release.yml)
|
||||
if [[ "$has_revision_file_change" == "true" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ "$only_workflow_changes" == "true" && -f internal/db/driver_agent_revisions_gen.go ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "检测到 driver-agent 构建/发布工作流变更；保守构建全部 driver-agent：$file" >&2
|
||||
all_drivers_csv
|
||||
exit 0
|
||||
;;
|
||||
tools/detect-changed-driver-agents.sh)
|
||||
# This script only selects CI work; it is not embedded in driver-agent binaries.
|
||||
echo "检测到 driver-agent 变更检测脚本更新；保守构建全部 driver-agent：$file" >&2
|
||||
all_drivers_csv
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
223
tools/detect-changed-driver-agents.test.sh
Executable file
223
tools/detect-changed-driver-agents.test.sh
Executable file
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_VERSINFO[0]:-0}" -lt 4 ]]; then
|
||||
echo "skip: detect-changed-driver-agents.sh requires Bash 4+ for associative arrays; current bash is ${BASH_VERSION:-unknown}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-driver-revisions.XXXXXX")"
|
||||
tmpdir_connection=""
|
||||
tmpdir_script=""
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
if [[ -n "$tmpdir_connection" ]]; then
|
||||
rm -rf "$tmpdir_connection"
|
||||
fi
|
||||
if [[ -n "$tmpdir_script" ]]; then
|
||||
rm -rf "$tmpdir_script"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git init -q "$tmpdir"
|
||||
mkdir -p "$tmpdir/tools"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir/tools/detect-changed-driver-agents.sh"
|
||||
mkdir -p "$tmpdir/internal/db"
|
||||
cat >"$tmpdir/internal/db/driver_agent_revisions_gen.go" <<'GOEOF'
|
||||
package db
|
||||
|
||||
func init() {
|
||||
optionalDriverAgentRevisions = map[string]string{
|
||||
"mariadb": "src-old-mariadb",
|
||||
"clickhouse": "src-old-clickhouse",
|
||||
}
|
||||
}
|
||||
GOEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
perl -0pi -e 's/src-old-clickhouse/src-new-clickhouse/' internal/db/driver_agent_revisions_gen.go
|
||||
git add internal/db/driver_agent_revisions_gen.go
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update clickhouse revision'
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD)"
|
||||
if [[ "$actual" != "clickhouse" ]]; then
|
||||
echo "expected clickhouse revision-only change to trigger clickhouse build, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_connection="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-connection-change.XXXXXX")"
|
||||
git init -q "$tmpdir_connection"
|
||||
mkdir -p "$tmpdir_connection/tools" "$tmpdir_connection/internal/connection"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_connection/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_connection/internal/connection/types.go" <<'GOEOF'
|
||||
package connection
|
||||
|
||||
type ConnectionConfig struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
GOEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_connection"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
perl -0pi -e 's/Type string/RedisSentinelLabel string `json:"redisSentinelLabel,omitempty"`\n\tType string/' internal/connection/types.go
|
||||
git add internal/connection/types.go
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'add redis-only connection field'
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD)"
|
||||
if [[ -n "$actual" ]]; then
|
||||
echo "expected connection-only field change to keep driver-agent detection empty, got: ${actual}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_script="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-script-change.XXXXXX")"
|
||||
git init -q "$tmpdir_script"
|
||||
mkdir -p "$tmpdir_script/tools"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_script/tools/detect-changed-driver-agents.sh"
|
||||
(
|
||||
cd "$tmpdir_script"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
printf '\n# test change\n' >> tools/detect-changed-driver-agents.sh
|
||||
git add tools/detect-changed-driver-agents.sh
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update detection script'
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD 2>/dev/null)"
|
||||
if [[ "$actual" != *"mariadb"* || "$actual" != *"clickhouse"* || "$actual" != *"elasticsearch"* ]]; then
|
||||
echo "expected detection script change to trigger all driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_compensation="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-compensation.XXXXXX")"
|
||||
git init -q "$tmpdir_compensation"
|
||||
mkdir -p "$tmpdir_compensation/tools" "$tmpdir_compensation/internal/db" "$tmpdir_compensation/.github/workflows"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_compensation/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_compensation/internal/db/driver_agent_revisions_gen.go" <<'GOEOF'
|
||||
package db
|
||||
|
||||
func init() {
|
||||
optionalDriverAgentRevisions = map[string]string{
|
||||
"clickhouse": "src-old-clickhouse",
|
||||
}
|
||||
}
|
||||
GOEOF
|
||||
cat >"$tmpdir_compensation/.github/workflows/dev-build.yml" <<'YAMLEOF'
|
||||
name: Dev Build
|
||||
YAMLEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_compensation"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
published_base="$(git rev-parse HEAD)"
|
||||
|
||||
perl -0pi -e 's/src-old-clickhouse/src-new-clickhouse/' internal/db/driver_agent_revisions_gen.go
|
||||
git add internal/db/driver_agent_revisions_gen.go
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update clickhouse revision'
|
||||
push_base="$(git rev-parse HEAD)"
|
||||
|
||||
printf '\n# workflow fix\n' >> .github/workflows/dev-build.yml
|
||||
git add .github/workflows/dev-build.yml
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'fix workflow'
|
||||
|
||||
actual_push_base="$(bash ./tools/detect-changed-driver-agents.sh --base "$push_base" --head HEAD)"
|
||||
if [[ -n "$actual_push_base" ]]; then
|
||||
echo "expected workflow-only fix commit to be empty when using push diff base, got: ${actual_push_base:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
actual_published_base="$(bash ./tools/detect-changed-driver-agents.sh --base "$published_base" --head HEAD)"
|
||||
if [[ "$actual_published_base" != "clickhouse" ]]; then
|
||||
echo "expected published release base to recover clickhouse rebuild, got: ${actual_published_base:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_workflow="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-workflow-change.XXXXXX")"
|
||||
git init -q "$tmpdir_workflow"
|
||||
mkdir -p "$tmpdir_workflow/tools" "$tmpdir_workflow/.github/workflows"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_workflow/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_workflow/.github/workflows/dev-build.yml" <<'YAMLEOF'
|
||||
name: Dev Build
|
||||
YAMLEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_workflow"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
printf '\n# workflow logic changed\n' >> .github/workflows/dev-build.yml
|
||||
git add .github/workflows/dev-build.yml
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update workflow'
|
||||
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD 2>/dev/null)"
|
||||
if [[ "$actual" != *"mariadb"* || "$actual" != *"clickhouse"* || "$actual" != *"duckdb"* || "$actual" != *"elasticsearch"* ]]; then
|
||||
echo "expected workflow change to trigger all driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_packaging="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-packaging-change.XXXXXX")"
|
||||
git init -q "$tmpdir_packaging"
|
||||
mkdir -p "$tmpdir_packaging/tools"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_packaging/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_packaging/tools/package-driver-release-assets.py" <<'PYEOF'
|
||||
print("package")
|
||||
PYEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_packaging"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
printf '\nprint("changed")\n' >> tools/package-driver-release-assets.py
|
||||
git add tools/package-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update packaging script'
|
||||
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD 2>/dev/null)"
|
||||
if [[ "$actual" != *"mariadb"* || "$actual" != *"clickhouse"* || "$actual" != *"duckdb"* || "$actual" != *"elasticsearch"* ]]; then
|
||||
echo "expected packaging script change to trigger all driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_release_validation="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-detect-release-validation-change.XXXXXX")"
|
||||
git init -q "$tmpdir_release_validation"
|
||||
mkdir -p "$tmpdir_release_validation/tools"
|
||||
cp tools/detect-changed-driver-agents.sh "$tmpdir_release_validation/tools/detect-changed-driver-agents.sh"
|
||||
cat >"$tmpdir_release_validation/tools/validate-driver-release-assets.py" <<'PYEOF'
|
||||
print("validate")
|
||||
PYEOF
|
||||
|
||||
(
|
||||
cd "$tmpdir_release_validation"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
printf '\nprint("changed")\n' >> tools/validate-driver-release-assets.py
|
||||
git add tools/validate-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'update release validation script'
|
||||
|
||||
actual="$(bash ./tools/detect-changed-driver-agents.sh --base "$base" --head HEAD 2>/dev/null)"
|
||||
if [[ "$actual" != *"mariadb"* || "$actual" != *"clickhouse"* || "$actual" != *"duckdb"* || "$actual" != *"elasticsearch"* ]]; then
|
||||
echo "expected release validation script change to trigger all driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
echo "detect-changed-driver-agents revision test passed"
|
||||
146
tools/diff-driver-agent-revisions.sh
Normal file
146
tools/diff-driver-agent-revisions.sh
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
DEFAULT_DRIVERS=(mariadb oceanbase diros starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss gaussdb iris mongodb tdengine iotdb clickhouse elasticsearch)
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
./tools/diff-driver-agent-revisions.sh --base <ref> --head <ref> --platform <GOOS/GOARCH>
|
||||
|
||||
输出：
|
||||
逗号分隔的 driver-agent 列表；当 base/head 在当前 runner + 指定平台上生成出的 revision 完全一致时输出空行。
|
||||
|
||||
说明：
|
||||
该脚本会分别在 base/head 对应源码上重算指定平台的 driver-agent revision，
|
||||
并按实际 revision 差异判定哪些驱动必须重建。
|
||||
EOF
|
||||
}
|
||||
|
||||
join_drivers() {
|
||||
local IFS=,
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
public_driver_name() {
|
||||
case "$1" in
|
||||
diros) echo "doris" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
extract_revision() {
|
||||
local file="$1"
|
||||
local driver="$2"
|
||||
awk -v target="$driver" '
|
||||
$0 ~ "\"" target "\"" {
|
||||
if (match($0, /"src-[^"]+"/)) {
|
||||
print substr($0, RSTART + 1, RLENGTH - 2)
|
||||
exit
|
||||
}
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
base_ref=""
|
||||
head_ref=""
|
||||
target_platform=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--base)
|
||||
base_ref="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--head)
|
||||
head_ref="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--platform)
|
||||
target_platform="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "未知参数：$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$base_ref" || -z "$head_ref" || -z "$target_platform" ]]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$target_platform" != */* ]]; then
|
||||
echo "--platform 参数格式错误，应为 GOOS/GOARCH，例如 darwin/arm64" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git rev-parse --verify "${base_ref}^{commit}" >/dev/null 2>&1; then
|
||||
echo "无法解析 base ref：$base_ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! git rev-parse --verify "${head_ref}^{commit}" >/dev/null 2>&1; then
|
||||
echo "无法解析 head ref：$head_ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
base_commit="$(git rev-parse "${base_ref}^{commit}")"
|
||||
head_commit="$(git rev-parse "${head_ref}^{commit}")"
|
||||
|
||||
if [[ "$base_commit" == "$head_commit" ]]; then
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
base_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-base.XXXXXX")"
|
||||
head_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-head.XXXXXX")"
|
||||
|
||||
cleanup() {
|
||||
git worktree remove --force "$base_worktree" >/dev/null 2>&1 || true
|
||||
git worktree remove --force "$head_worktree" >/dev/null 2>&1 || true
|
||||
rm -rf "$base_worktree" "$head_worktree"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git worktree add --detach "$base_worktree" "$base_commit" >/dev/null
|
||||
git worktree add --detach "$head_worktree" "$head_commit" >/dev/null
|
||||
|
||||
generate_revisions() {
|
||||
local worktree="$1"
|
||||
(
|
||||
cd "$worktree"
|
||||
GONAVI_DRIVER_REVISION_JOBS="${GONAVI_DRIVER_REVISION_JOBS:-1}" \
|
||||
bash ./tools/generate-driver-agent-revisions.sh --platform "$target_platform" >/dev/null
|
||||
)
|
||||
}
|
||||
|
||||
generate_revisions "$base_worktree"
|
||||
generate_revisions "$head_worktree"
|
||||
|
||||
base_file="$base_worktree/internal/db/driver_agent_revisions_gen.go"
|
||||
head_file="$head_worktree/internal/db/driver_agent_revisions_gen.go"
|
||||
|
||||
declare -a changed_drivers=()
|
||||
for driver in "${DEFAULT_DRIVERS[@]}"; do
|
||||
base_revision="$(extract_revision "$base_file" "$driver")"
|
||||
head_revision="$(extract_revision "$head_file" "$driver")"
|
||||
if [[ "$base_revision" != "$head_revision" ]]; then
|
||||
changed_drivers+=("$(public_driver_name "$driver")")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#changed_drivers[@]} -eq 0 ]]; then
|
||||
echo ""
|
||||
else
|
||||
join_drivers "${changed_drivers[@]}"
|
||||
fi
|
||||
66
tools/diff-driver-agent-revisions.test.sh
Normal file
66
tools/diff-driver-agent-revisions.test.sh
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
same_commit_result="$(bash ./tools/diff-driver-agent-revisions.sh --base HEAD --head HEAD --platform darwin/arm64)"
|
||||
if [[ -n "$same_commit_result" ]]; then
|
||||
echo "expected same commit revision diff to be empty, got: ${same_commit_result}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-diff-driver-revisions.XXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
rsync -a --exclude .git ./ "$tmpdir/" >/dev/null
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git init -q
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
perl -0pi -e 's/type DuckDB struct \{/\/\/ test revision change\n&type DuckDB struct {/' internal/db/duckdb_impl.go
|
||||
git add internal/db/duckdb_impl.go
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'touch duckdb impl'
|
||||
|
||||
actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64)"
|
||||
if [[ "$actual" != *"duckdb"* ]]; then
|
||||
echo "expected duckdb-specific source change to include duckdb revision rebuild, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_frontend="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-diff-driver-revisions-frontend.XXXXXX")"
|
||||
cleanup_frontend() {
|
||||
rm -rf "$tmpdir_frontend"
|
||||
}
|
||||
trap cleanup_frontend EXIT
|
||||
|
||||
rsync -a --exclude .git ./ "$tmpdir_frontend/" >/dev/null
|
||||
|
||||
(
|
||||
cd "$tmpdir_frontend"
|
||||
git init -q
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base="$(git rev-parse HEAD)"
|
||||
|
||||
perl -0pi -e 's/isAddingPrimaryKey:/isAddingPrimaryKeyFlag:/' frontend/src/components/tableDesignerDuckDbPrimaryKey.ts
|
||||
git add frontend/src/components/tableDesignerDuckDbPrimaryKey.ts
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'touch frontend only'
|
||||
|
||||
actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64)"
|
||||
if [[ -n "$actual" ]]; then
|
||||
echo "expected frontend-only change to keep driver revision diff empty, got: ${actual}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
echo "diff-driver-agent-revisions test passed"
|
||||
@@ -4,8 +4,10 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
SCRIPT_DIR_WINDOWS="$(pwd -W 2>/dev/null || true)"
|
||||
SCRIPT_DIR_WINDOWS="${SCRIPT_DIR_WINDOWS//\\//}"
|
||||
|
||||
DEFAULT_DRIVERS=(mariadb oceanbase diros starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss iris mongodb tdengine clickhouse)
|
||||
DEFAULT_DRIVERS=(mariadb oceanbase diros starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss gaussdb iris mongodb tdengine iotdb clickhouse elasticsearch)
|
||||
OUTPUT_FILE="internal/db/driver_agent_revisions_gen.go"
|
||||
|
||||
usage() {
|
||||
@@ -27,7 +29,9 @@ normalize_driver() {
|
||||
doris|diros) echo "diros" ;;
|
||||
oceanbase) echo "oceanbase" ;;
|
||||
opengauss|open_gauss|open-gauss) echo "opengauss" ;;
|
||||
mariadb|diros|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|iris|mongodb|tdengine|clickhouse)
|
||||
gaussdb|gauss_db|gauss-db) echo "gaussdb" ;;
|
||||
elasticsearch|elastic) echo "elasticsearch" ;;
|
||||
mariadb|diros|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|gaussdb|iris|mongodb|tdengine|iotdb|clickhouse)
|
||||
echo "$value"
|
||||
;;
|
||||
*)
|
||||
@@ -115,6 +119,7 @@ sphinx:internal/db/mysql_impl.go|\
|
||||
sqlserver:internal/db/sqlserver_impl.go|\
|
||||
sqlite:internal/db/sqlite_impl.go|\
|
||||
duckdb:internal/db/duckdb_impl.go|\
|
||||
duckdb:internal/db/duckdb_metadata.go|\
|
||||
duckdb:internal/db/duckdb_driver_import.go|\
|
||||
duckdb:internal/db/duckdb_platform_supported.go|\
|
||||
duckdb:internal/db/duckdb_platform_unsupported.go|\
|
||||
@@ -126,11 +131,15 @@ highgo:internal/db/highgo_impl.go|\
|
||||
vastbase:internal/db/vastbase_impl.go|\
|
||||
opengauss:internal/db/opengauss_impl.go|\
|
||||
opengauss:internal/db/postgres_impl.go|\
|
||||
gaussdb:internal/db/gaussdb_impl.go|\
|
||||
iris:internal/db/iris_impl.go|\
|
||||
mongodb:internal/db/mongodb_impl.go|\
|
||||
mongodb:internal/db/mongodb_impl_v1.go|\
|
||||
tdengine:internal/db/tdengine_impl.go|\
|
||||
clickhouse:internal/db/clickhouse_impl.go)
|
||||
iotdb:internal/db/iotdb_impl.go|\
|
||||
clickhouse:internal/db/clickhouse_impl.go|\
|
||||
elasticsearch:internal/db/elasticsearch_impl.go|\
|
||||
elasticsearch:internal/db/elasticsearch_helpers.go)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -141,6 +150,11 @@ clickhouse:internal/db/clickhouse_impl.go)
|
||||
should_include_source_file() {
|
||||
local driver="$1"
|
||||
local identity="$2"
|
||||
case "$identity" in
|
||||
internal/appdata/*|internal/connection/*|internal/logger/*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
if [[ "$identity" == internal/db/* ]]; then
|
||||
should_include_internal_db_file "$driver" "$identity"
|
||||
return
|
||||
@@ -189,6 +203,7 @@ fi
|
||||
goos="${target_platform%%/*}"
|
||||
goarch="${target_platform##*/}"
|
||||
gomodcache="$(go env GOMODCACHE)"
|
||||
gomodcache="${gomodcache//\\//}"
|
||||
|
||||
declare -a drivers=()
|
||||
if [[ -n "$driver_csv" ]]; then
|
||||
@@ -273,18 +288,23 @@ fingerprint_driver() {
|
||||
} >"$tmp"
|
||||
|
||||
while IFS= read -r file; do
|
||||
file="${file//\\//}"
|
||||
[[ -n "$file" && -f "$file" ]] || continue
|
||||
case "$file" in
|
||||
"$SCRIPT_DIR"/*)
|
||||
identity="${file#$SCRIPT_DIR/}"
|
||||
;;
|
||||
"$gomodcache"/*)
|
||||
identity="gomod/${file#$gomodcache/}"
|
||||
;;
|
||||
*)
|
||||
identity="$file"
|
||||
;;
|
||||
esac
|
||||
if [[ -n "$SCRIPT_DIR_WINDOWS" && "$file" == "$SCRIPT_DIR_WINDOWS"/* ]]; then
|
||||
identity="${file#$SCRIPT_DIR_WINDOWS/}"
|
||||
else
|
||||
case "$file" in
|
||||
"$SCRIPT_DIR"/*)
|
||||
identity="${file#$SCRIPT_DIR/}"
|
||||
;;
|
||||
"$gomodcache"/*)
|
||||
identity="gomod/${file#$gomodcache/}"
|
||||
;;
|
||||
*)
|
||||
identity="$file"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
if [[ "$identity" == "$OUTPUT_FILE" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
79
tools/generate-driver-agent-revisions.test.sh
Normal file
79
tools/generate-driver-agent-revisions.test.sh
Normal file
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
extract_revision() {
|
||||
local file="$1"
|
||||
local driver="$2"
|
||||
sed -n "s/.*\"${driver}\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p" "$file" | head -n 1
|
||||
}
|
||||
|
||||
copy_repo_to_tmp() {
|
||||
local target="$1"
|
||||
git ls-files -z | tar --null -T - -cf - | (cd "$target" && tar -xf -)
|
||||
}
|
||||
|
||||
tmpdir_platform="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-generate-driver-revisions-platform.XXXXXX")"
|
||||
tmpdir_connection="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-generate-driver-revisions-connection.XXXXXX")"
|
||||
darwin_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-darwin-revisions.XXXXXX")"
|
||||
windows_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-windows-revisions.XXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir_platform" "$tmpdir_connection"
|
||||
rm -f "$darwin_file" "$windows_file"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
copy_repo_to_tmp "$tmpdir_platform"
|
||||
|
||||
(
|
||||
cd "$tmpdir_platform"
|
||||
GONAVI_DRIVER_REVISION_JOBS=1 bash ./tools/generate-driver-agent-revisions.sh --platform darwin/arm64 --drivers duckdb >/dev/null
|
||||
cp internal/db/driver_agent_revisions_gen.go "$darwin_file"
|
||||
GONAVI_DRIVER_REVISION_JOBS=1 bash ./tools/generate-driver-agent-revisions.sh --platform windows/amd64 --drivers duckdb >/dev/null
|
||||
cp internal/db/driver_agent_revisions_gen.go "$windows_file"
|
||||
)
|
||||
|
||||
darwin_duckdb="$(extract_revision "$darwin_file" duckdb)"
|
||||
windows_duckdb="$(extract_revision "$windows_file" duckdb)"
|
||||
if [[ -z "$darwin_duckdb" || -z "$windows_duckdb" ]]; then
|
||||
echo "expected duckdb revision to be generated for both platforms" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$darwin_duckdb" == "$windows_duckdb" ]]; then
|
||||
echo "expected duckdb revision to differ between darwin/arm64 and windows/amd64, got identical value: $darwin_duckdb" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
copy_repo_to_tmp "$tmpdir_connection"
|
||||
|
||||
(
|
||||
cd "$tmpdir_connection"
|
||||
GONAVI_DRIVER_REVISION_JOBS=1 bash ./tools/generate-driver-agent-revisions.sh --platform windows/amd64 --drivers sqlserver >/dev/null
|
||||
before_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-sqlserver-revision-before.XXXXXX")"
|
||||
after_file="$(mktemp "${TMPDIR:-/tmp}/gonavi-sqlserver-revision-after.XXXXXX")"
|
||||
cleanup_sqlserver_revision_files() {
|
||||
rm -f "$before_file" "$after_file"
|
||||
}
|
||||
trap cleanup_sqlserver_revision_files EXIT
|
||||
|
||||
cp internal/db/driver_agent_revisions_gen.go "$before_file"
|
||||
perl -0pi -e 's/RedisSentinelMaster string/RedisSentinelLabel string `json:"redisSentinelLabel,omitempty"`\n\tRedisSentinelMaster string/' internal/connection/types.go
|
||||
GONAVI_DRIVER_REVISION_JOBS=1 bash ./tools/generate-driver-agent-revisions.sh --platform windows/amd64 --drivers sqlserver >/dev/null
|
||||
cp internal/db/driver_agent_revisions_gen.go "$after_file"
|
||||
|
||||
before_sqlserver="$(extract_revision "$before_file" sqlserver)"
|
||||
after_sqlserver="$(extract_revision "$after_file" sqlserver)"
|
||||
if [[ -z "$before_sqlserver" || -z "$after_sqlserver" ]]; then
|
||||
echo "expected sqlserver revision to be generated before and after connection-only change" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$before_sqlserver" != "$after_sqlserver" ]]; then
|
||||
echo "expected Redis-only connection field change to keep sqlserver revision stable, before=$before_sqlserver after=$after_sqlserver" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
echo "generate-driver-agent-revisions platform test passed"
|
||||
180
tools/generate-driver-release-manifest.py
Normal file
180
tools/generate-driver-release-manifest.py
Normal file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--assets-dir", required=True, help="driver release staging dir that contains standalone driver assets")
|
||||
parser.add_argument("--output", required=True, help="manifest json output path")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def infer_driver_and_platform(file_name: str):
|
||||
suffixes = [
|
||||
("-driver-agent-v1-darwin-amd64", "darwin/amd64"),
|
||||
("-driver-agent-v1-darwin-arm64", "darwin/arm64"),
|
||||
("-driver-agent-v1-linux-amd64", "linux/amd64"),
|
||||
("-driver-agent-v1-windows-amd64.exe", "windows/amd64"),
|
||||
("-driver-agent-v1-windows-arm64.exe", "windows/arm64"),
|
||||
("-driver-agent-v2-darwin-amd64", "darwin/amd64"),
|
||||
("-driver-agent-v2-darwin-arm64", "darwin/arm64"),
|
||||
("-driver-agent-v2-linux-amd64", "linux/amd64"),
|
||||
("-driver-agent-v2-windows-amd64.exe", "windows/amd64"),
|
||||
("-driver-agent-v2-windows-arm64.exe", "windows/arm64"),
|
||||
("-driver-agent-darwin-amd64", "darwin/amd64"),
|
||||
("-driver-agent-darwin-arm64", "darwin/arm64"),
|
||||
("-driver-agent-linux-amd64", "linux/amd64"),
|
||||
("-driver-agent-windows-amd64.exe", "windows/amd64"),
|
||||
("-driver-agent-windows-arm64.exe", "windows/arm64"),
|
||||
]
|
||||
for suffix, platform in suffixes:
|
||||
if file_name.endswith(suffix):
|
||||
driver = file_name[: -len(suffix)]
|
||||
return driver, platform
|
||||
return None, None
|
||||
|
||||
|
||||
def normalize_driver(driver: str):
|
||||
value = str(driver or "").strip().lower()
|
||||
if value == "doris":
|
||||
return "diros"
|
||||
return value
|
||||
|
||||
|
||||
def repo_root():
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def resolve_head_commit(root: Path):
|
||||
proc = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def parse_revision_file(path: Path):
|
||||
revisions = {}
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith('"'):
|
||||
continue
|
||||
try:
|
||||
driver, revision = stripped.rstrip(",").split(":", 1)
|
||||
except ValueError:
|
||||
continue
|
||||
revisions[driver.strip().strip('"')] = revision.strip().strip('"')
|
||||
return revisions
|
||||
|
||||
|
||||
def generate_platform_revisions(root: Path, drivers_by_platform):
|
||||
if not drivers_by_platform:
|
||||
return {}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-driver-release-manifest-") as tmp:
|
||||
worktree = Path(tmp) / "worktree"
|
||||
subprocess.run(
|
||||
["git", "worktree", "add", "--detach", str(worktree), "HEAD"],
|
||||
cwd=root,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
try:
|
||||
revision_file = worktree / "internal/db/driver_agent_revisions_gen.go"
|
||||
result = {}
|
||||
for platform in sorted(drivers_by_platform):
|
||||
drivers = sorted({normalize_driver(driver) for driver in drivers_by_platform[platform] if normalize_driver(driver)})
|
||||
command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform]
|
||||
if drivers:
|
||||
command.extend(["--drivers", ",".join(drivers)])
|
||||
subprocess.run(
|
||||
command,
|
||||
cwd=worktree,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
result[platform] = parse_revision_file(revision_file)
|
||||
return result
|
||||
finally:
|
||||
subprocess.run(
|
||||
["git", "worktree", "remove", "--force", str(worktree)],
|
||||
cwd=root,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
assets_dir = Path(args.assets_dir).resolve()
|
||||
output_path = Path(args.output).resolve()
|
||||
root = repo_root()
|
||||
|
||||
asset_entries = []
|
||||
drivers_by_platform = {}
|
||||
for child in sorted(assets_dir.rglob("*")):
|
||||
if not child.is_file():
|
||||
continue
|
||||
driver, platform = infer_driver_and_platform(child.name)
|
||||
if not driver or not platform:
|
||||
continue
|
||||
if child.stat().st_size == 0:
|
||||
raise RuntimeError(f"{child.name}: asset is empty")
|
||||
asset_entries.append((child, driver, platform))
|
||||
drivers_by_platform.setdefault(platform, set()).add(driver)
|
||||
|
||||
revisions_by_platform = generate_platform_revisions(root, drivers_by_platform)
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"generatedFrom": os.environ.get("GITHUB_SHA", "").strip() or resolve_head_commit(root),
|
||||
"assets": {},
|
||||
}
|
||||
|
||||
for child, driver, platform in asset_entries:
|
||||
normalized_driver = normalize_driver(driver)
|
||||
revision = str((revisions_by_platform.get(platform) or {}).get(normalized_driver) or "").strip()
|
||||
if not revision:
|
||||
raise RuntimeError(f"{child.name}: missing revision for {platform}/{normalized_driver}")
|
||||
manifest["assets"][child.name] = {
|
||||
"driver": driver,
|
||||
"driverType": driver,
|
||||
"platform": platform,
|
||||
"revision": revision,
|
||||
"size": child.stat().st_size,
|
||||
"sha256": hashlib.sha256(child.read_bytes()).hexdigest(),
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(f"wrote manifest: {output_path}")
|
||||
print(f"asset count: {len(manifest['assets'])}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except subprocess.CalledProcessError as exc:
|
||||
command = exc.cmd if isinstance(exc.cmd, str) else " ".join(exc.cmd)
|
||||
stderr = (exc.stderr or "").strip()
|
||||
if stderr:
|
||||
print(stderr, file=sys.stderr)
|
||||
print(f"error: command failed: {command}", file=sys.stderr)
|
||||
raise
|
||||
120
tools/generate-driver-release-manifest.test.py
Normal file
120
tools/generate-driver-release-manifest.test.py
Normal file
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "tools" / "generate-driver-release-manifest.py"
|
||||
|
||||
|
||||
def expected_revision(revision_file: Path, driver: str):
|
||||
for line in revision_file.read_text(encoding="utf-8").splitlines():
|
||||
if f'"{driver}"' not in line:
|
||||
continue
|
||||
left, right = line.strip().rstrip(",").split(":", 1)
|
||||
if left.strip().strip('"') == driver:
|
||||
return right.strip().strip('"')
|
||||
raise AssertionError(f"missing revision for {driver}")
|
||||
|
||||
|
||||
class GenerateDriverReleaseManifestTest(unittest.TestCase):
|
||||
def _generate_revision_file(self, platform: str, drivers: str = "clickhouse"):
|
||||
worktree = Path(tempfile.mkdtemp(prefix="gonavi-release-manifest-worktree-"))
|
||||
subprocess.run(
|
||||
["git", "worktree", "add", "--detach", str(worktree), "HEAD"],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
self.addCleanup(
|
||||
lambda: subprocess.run(
|
||||
["git", "worktree", "remove", "--force", str(worktree)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
)
|
||||
subprocess.run(
|
||||
["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform, "--drivers", drivers],
|
||||
cwd=worktree,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=True,
|
||||
)
|
||||
return worktree / "internal" / "db" / "driver_agent_revisions_gen.go"
|
||||
|
||||
def test_generates_manifest_without_executing_cross_platform_assets(self):
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-test-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
assets_dir = tmpdir / "drivers"
|
||||
(assets_dir / "MacOS").mkdir(parents=True)
|
||||
(assets_dir / "Linux").mkdir(parents=True)
|
||||
(assets_dir / "Windows").mkdir(parents=True)
|
||||
|
||||
fixtures = {
|
||||
assets_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64": b"darwin-binary",
|
||||
assets_dir / "Linux" / "clickhouse-driver-agent-linux-amd64": b"linux-binary",
|
||||
assets_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe": b"MZfake-binary",
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v1-windows-amd64.exe": b"MZfake-mongodb-v1",
|
||||
assets_dir / "Windows" / "mongodb-driver-agent-v2-windows-amd64.exe": b"MZfake-mongodb-v2",
|
||||
}
|
||||
for path, content in fixtures.items():
|
||||
path.write_bytes(content)
|
||||
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
||||
|
||||
output = tmpdir / "manifest.json"
|
||||
proc = subprocess.run(
|
||||
["python3", str(SCRIPT), "--assets-dir", str(assets_dir), "--output", str(output)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
self.assertIn("asset count: 5", proc.stdout)
|
||||
manifest = json.loads(output.read_text(encoding="utf-8"))
|
||||
assets = manifest["assets"]
|
||||
darwin_revision_file = self._generate_revision_file("darwin/arm64")
|
||||
linux_revision_file = self._generate_revision_file("linux/amd64")
|
||||
windows_revision_file = self._generate_revision_file("windows/amd64", "clickhouse,mongodb")
|
||||
self.assertEqual(
|
||||
assets["clickhouse-driver-agent-darwin-arm64"]["revision"],
|
||||
expected_revision(darwin_revision_file, "clickhouse"),
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["clickhouse-driver-agent-linux-amd64"]["revision"],
|
||||
expected_revision(linux_revision_file, "clickhouse"),
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["clickhouse-driver-agent-windows-amd64.exe"]["revision"],
|
||||
expected_revision(windows_revision_file, "clickhouse"),
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["mongodb-driver-agent-v1-windows-amd64.exe"]["driver"],
|
||||
"mongodb",
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["mongodb-driver-agent-v1-windows-amd64.exe"]["platform"],
|
||||
"windows/amd64",
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["mongodb-driver-agent-v1-windows-amd64.exe"]["revision"],
|
||||
expected_revision(windows_revision_file, "mongodb"),
|
||||
)
|
||||
self.assertEqual(
|
||||
assets["mongodb-driver-agent-v2-windows-amd64.exe"]["driver"],
|
||||
"mongodb",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
62
tools/package-driver-release-assets.py
Normal file
62
tools/package-driver-release-assets.py
Normal file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit("usage: package-driver-release-assets.py <drivers-dir> <output-dir>")
|
||||
|
||||
drivers_dir = Path(sys.argv[1]).resolve()
|
||||
output_dir = Path(sys.argv[2]).resolve()
|
||||
|
||||
if not drivers_dir.is_dir():
|
||||
raise SystemExit(f"drivers dir not found: {drivers_dir}")
|
||||
|
||||
out_name = "GoNavi-DriverAgents.zip"
|
||||
index_name = "GoNavi-DriverAgents-Index.json"
|
||||
manifest_name = "GoNavi-DriverAgents-Manifest.json"
|
||||
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
out_path = output_dir / out_name
|
||||
index_path = output_dir / index_name
|
||||
manifest_path = output_dir / manifest_name
|
||||
|
||||
size_index = {}
|
||||
standalone_assets = []
|
||||
with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for asset in sorted(drivers_dir.rglob("*")):
|
||||
if not asset.is_file():
|
||||
continue
|
||||
arcname = asset.relative_to(drivers_dir).as_posix()
|
||||
if asset.name in size_index:
|
||||
raise RuntimeError(f"driver asset name conflict: {asset.name}")
|
||||
zf.write(asset, arcname)
|
||||
size_index[asset.name] = asset.stat().st_size
|
||||
standalone_path = output_dir / asset.name
|
||||
if standalone_path.exists():
|
||||
raise RuntimeError(f"release asset already exists: {standalone_path}")
|
||||
shutil.copy2(asset, standalone_path)
|
||||
standalone_assets.append(standalone_path.name)
|
||||
|
||||
index_path.write_text(
|
||||
json.dumps({"assets": size_index}, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"created {out_name} size={out_path.stat().st_size} bytes")
|
||||
print(f"created {index_name} entries={len(size_index)}")
|
||||
print(f"published standalone driver assets={len(standalone_assets)}")
|
||||
print(f"reserved manifest output path: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
58
tools/package-driver-release-assets.test.py
Normal file
58
tools/package-driver-release-assets.test.py
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "tools" / "package-driver-release-assets.py"
|
||||
|
||||
|
||||
class PackageDriverReleaseAssetsTest(unittest.TestCase):
|
||||
def test_packages_bundle_and_standalone_assets(self):
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
drivers_dir = tmpdir / "drivers"
|
||||
output_dir = tmpdir / "driver-release-assets"
|
||||
(drivers_dir / "Windows").mkdir(parents=True)
|
||||
(drivers_dir / "MacOS").mkdir(parents=True)
|
||||
|
||||
windows_asset = drivers_dir / "Windows" / "clickhouse-driver-agent-windows-amd64.exe"
|
||||
darwin_asset = drivers_dir / "MacOS" / "clickhouse-driver-agent-darwin-arm64"
|
||||
windows_asset.write_bytes(b"windows-asset")
|
||||
darwin_asset.write_bytes(b"darwin-asset")
|
||||
|
||||
proc = subprocess.run(
|
||||
["python3", str(SCRIPT), str(drivers_dir), str(output_dir)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
self.assertIn("created GoNavi-DriverAgents.zip", proc.stdout)
|
||||
self.assertTrue((output_dir / "GoNavi-DriverAgents.zip").is_file())
|
||||
self.assertTrue((output_dir / windows_asset.name).is_file())
|
||||
self.assertTrue((output_dir / darwin_asset.name).is_file())
|
||||
|
||||
index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(index["assets"][windows_asset.name], len(b"windows-asset"))
|
||||
self.assertEqual(index["assets"][darwin_asset.name], len(b"darwin-asset"))
|
||||
|
||||
with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf:
|
||||
self.assertEqual(
|
||||
sorted(zf.namelist()),
|
||||
[
|
||||
"MacOS/clickhouse-driver-agent-darwin-arm64",
|
||||
"Windows/clickhouse-driver-agent-windows-amd64.exe",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
138
tools/resolve-driver-release-source.py
Normal file
138
tools/resolve-driver-release-source.py
Normal file
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
COMMIT_LINK_RE = re.compile(r"/commit/([0-9a-f]{40})(?:\b|/)")
|
||||
FULL_SHA_RE = re.compile(r"\b([0-9a-f]{40})\b")
|
||||
MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"
|
||||
|
||||
|
||||
def github_headers():
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "GoNavi-CI",
|
||||
}
|
||||
token = os.environ.get("DRIVER_RELEASE_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
request = urllib.request.Request(url, headers=github_headers())
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def download_asset(asset, destination):
|
||||
headers = github_headers()
|
||||
headers["Accept"] = "application/octet-stream"
|
||||
request = urllib.request.Request(asset["url"], headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
with open(destination, "wb") as output:
|
||||
output.write(response.read())
|
||||
|
||||
|
||||
def load_release(repo, tag):
|
||||
owner_repo = repo.strip()
|
||||
if not owner_repo:
|
||||
raise ValueError("repo is required")
|
||||
|
||||
if tag == "latest":
|
||||
url = f"https://api.github.com/repos/{owner_repo}/releases/latest"
|
||||
else:
|
||||
url = (
|
||||
f"https://api.github.com/repos/{owner_repo}/releases/tags/"
|
||||
f"{urllib.parse.quote(tag, safe='')}"
|
||||
)
|
||||
|
||||
try:
|
||||
return fetch_json(url)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
print(f"warning: release {owner_repo}@{tag} not found", file=sys.stderr)
|
||||
return None
|
||||
print(
|
||||
f"warning: failed to load release {owner_repo}@{tag}: HTTP {exc.code}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return None
|
||||
except Exception as exc: # pragma: no cover - defensive logging path
|
||||
print(f"warning: failed to load release {owner_repo}@{tag}: {exc}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def extract_source_commit(release):
|
||||
if not isinstance(release, dict):
|
||||
return None
|
||||
|
||||
body = str(release.get("body") or "")
|
||||
for pattern in (COMMIT_LINK_RE, FULL_SHA_RE):
|
||||
match = pattern.search(body)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
target_commitish = str(release.get("target_commitish") or "").strip()
|
||||
if FULL_SHA_RE.fullmatch(target_commitish):
|
||||
return target_commitish
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_manifest_asset(release):
|
||||
if not isinstance(release, dict):
|
||||
return None
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
if str(asset.get("name") or "").strip() == MANIFEST_ASSET_NAME:
|
||||
return asset
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
|
||||
parser.add_argument("--tag", required=True, help="release tag name such as dev-latest or v1.0.0")
|
||||
parser.add_argument("--manifest-output", help="optional path to download the published revision manifest asset")
|
||||
args = parser.parse_args()
|
||||
|
||||
release = load_release(args.repo, args.tag)
|
||||
if release is None:
|
||||
return 0
|
||||
|
||||
if args.manifest_output:
|
||||
manifest_path = os.path.abspath(args.manifest_output)
|
||||
manifest_asset = find_manifest_asset(release)
|
||||
if manifest_asset is None:
|
||||
if os.path.exists(manifest_path):
|
||||
os.remove(manifest_path)
|
||||
print(
|
||||
f"warning: release {args.repo}@{args.tag} does not expose {MANIFEST_ASSET_NAME}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
os.makedirs(os.path.dirname(manifest_path), exist_ok=True)
|
||||
download_asset(manifest_asset, manifest_path)
|
||||
|
||||
source_commit = extract_source_commit(release)
|
||||
if not source_commit:
|
||||
print(
|
||||
f"warning: release {args.repo}@{args.tag} does not expose source commit",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
print(source_commit)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
57
tools/resolve-driver-release-source.test.py
Normal file
57
tools/resolve-driver-release-source.test.py
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import importlib.util
|
||||
import pathlib
|
||||
import unittest
|
||||
|
||||
|
||||
MODULE_PATH = pathlib.Path(__file__).with_name("resolve-driver-release-source.py")
|
||||
SPEC = importlib.util.spec_from_file_location("resolve_driver_release_source", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class ResolveDriverReleaseSourceTests(unittest.TestCase):
|
||||
def test_extracts_commit_from_release_body_link(self):
|
||||
commit = "a" * 40
|
||||
release = {
|
||||
"body": (
|
||||
f"GoNavi dev driver-agent assets.\n\n"
|
||||
f"**提交**: [`{commit}`](https://github.com/Syngnat/GoNavi/commit/{commit})"
|
||||
)
|
||||
}
|
||||
self.assertEqual(MODULE.extract_source_commit(release), commit)
|
||||
|
||||
def test_extracts_commit_from_plain_body_sha(self):
|
||||
commit = "b" * 40
|
||||
release = {"body": f"source commit: {commit}"}
|
||||
self.assertEqual(MODULE.extract_source_commit(release), commit)
|
||||
|
||||
def test_falls_back_to_full_sha_target_commitish(self):
|
||||
commit = "c" * 40
|
||||
release = {"target_commitish": commit}
|
||||
self.assertEqual(MODULE.extract_source_commit(release), commit)
|
||||
|
||||
def test_ignores_branch_name_target_commitish(self):
|
||||
release = {"body": "", "target_commitish": "main"}
|
||||
self.assertIsNone(MODULE.extract_source_commit(release))
|
||||
|
||||
def test_finds_manifest_asset(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{"name": "foo.txt"},
|
||||
{"name": "GoNavi-DriverAgents-Manifest.json", "url": "https://example.test/manifest"},
|
||||
]
|
||||
}
|
||||
asset = MODULE.find_manifest_asset(release)
|
||||
self.assertIsNotNone(asset)
|
||||
self.assertEqual(asset["url"], "https://example.test/manifest")
|
||||
|
||||
def test_returns_none_when_manifest_asset_missing(self):
|
||||
release = {"assets": [{"name": "GoNavi-DriverAgents.zip"}]}
|
||||
self.assertIsNone(MODULE.find_manifest_asset(release))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
84
tools/should-force-global-driver-builds.sh
Executable file
84
tools/should-force-global-driver-builds.sh
Executable file
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
./tools/should-force-global-driver-builds.sh --base <ref> --head <ref>
|
||||
|
||||
输出：
|
||||
true 表示本次变更涉及 driver 构建/发布链路，平台构建阶段必须保留全局驱动重建结果
|
||||
false 表示可继续按平台 revision diff 缩小重建范围
|
||||
EOF
|
||||
}
|
||||
|
||||
base_ref=""
|
||||
head_ref=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--base)
|
||||
base_ref="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--head)
|
||||
head_ref="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "未知参数：$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$base_ref" || -z "$head_ref" ]]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$base_ref" == "all" ]]; then
|
||||
echo "true"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! git rev-parse --verify "${base_ref}^{commit}" >/dev/null 2>&1; then
|
||||
echo "无法解析 base ref：$base_ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! git rev-parse --verify "${head_ref}^{commit}" >/dev/null 2>&1; then
|
||||
echo "无法解析 head ref：$head_ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
case "$file" in
|
||||
.github/workflows/dev-build.yml|\
|
||||
.github/workflows/release.yml|\
|
||||
build-driver-agents.sh|\
|
||||
tools/compress-driver-artifact.sh|\
|
||||
tools/detect-changed-driver-agents.sh|\
|
||||
tools/diff-driver-agent-revisions.sh|\
|
||||
tools/package-driver-release-assets.py|\
|
||||
tools/generate-driver-release-manifest.py|\
|
||||
tools/validate-driver-release-assets.py|\
|
||||
tools/complete-driver-release-assets.py|\
|
||||
tools/resolve-driver-release-source.py|\
|
||||
tools/validate-driver-release-manifest.sh|\
|
||||
tools/should-force-global-driver-builds.sh)
|
||||
echo "true"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done < <(git diff --name-only "$base_ref" "$head_ref")
|
||||
|
||||
echo "false"
|
||||
96
tools/should-force-global-driver-builds.test.sh
Executable file
96
tools/should-force-global-driver-builds.test.sh
Executable file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-force-global-driver-builds.XXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git init -q "$tmpdir"
|
||||
mkdir -p "$tmpdir/tools" "$tmpdir/.github/workflows" "$tmpdir/internal/db"
|
||||
cp tools/should-force-global-driver-builds.sh "$tmpdir/tools/should-force-global-driver-builds.sh"
|
||||
cat >"$tmpdir/.github/workflows/dev-build.yml" <<'YAMLEOF'
|
||||
name: Dev Build
|
||||
YAMLEOF
|
||||
cat >"$tmpdir/tools/package-driver-release-assets.py" <<'PYEOF'
|
||||
print("package")
|
||||
PYEOF
|
||||
cat >"$tmpdir/tools/validate-driver-release-assets.py" <<'PYEOF'
|
||||
print("validate")
|
||||
PYEOF
|
||||
cat >"$tmpdir/internal/db/duckdb_impl.go" <<'GOEOF'
|
||||
package db
|
||||
GOEOF
|
||||
|
||||
base_ref=""
|
||||
cd "$tmpdir"
|
||||
git add .
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m initial
|
||||
base_ref="$(git rev-parse HEAD)"
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
printf '\n# workflow change\n' >> .github/workflows/dev-build.yml
|
||||
git add .github/workflows/dev-build.yml
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'workflow change'
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base "$base_ref" --head HEAD)"
|
||||
if [[ "$actual" != "true" ]]; then
|
||||
echo "expected workflow change to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git reset --hard -q "$base_ref"
|
||||
printf '\nprint("changed")\n' >> tools/package-driver-release-assets.py
|
||||
git add tools/package-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'packaging change'
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base "$base_ref" --head HEAD)"
|
||||
if [[ "$actual" != "true" ]]; then
|
||||
echo "expected packaging change to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git reset --hard -q "$base_ref"
|
||||
printf '\nprint("changed")\n' >> tools/validate-driver-release-assets.py
|
||||
git add tools/validate-driver-release-assets.py
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'release asset validation change'
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base "$base_ref" --head HEAD)"
|
||||
if [[ "$actual" != "true" ]]; then
|
||||
echo "expected release asset validation change to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
git reset --hard -q "$base_ref"
|
||||
printf '\n// source-only change\n' >> internal/db/duckdb_impl.go
|
||||
git add internal/db/duckdb_impl.go
|
||||
git -c user.name=GoNavi -c user.email=gonavi@example.test commit -q -m 'duckdb source change'
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base "$base_ref" --head HEAD)"
|
||||
if [[ "$actual" != "false" ]]; then
|
||||
echo "expected source-only driver change not to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
(
|
||||
cd "$tmpdir"
|
||||
actual="$(bash ./tools/should-force-global-driver-builds.sh --base all --head HEAD)"
|
||||
if [[ "$actual" != "true" ]]; then
|
||||
echo "expected base=all to force global driver builds, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
echo "should-force-global-driver-builds test passed"
|
||||
249
tools/validate-driver-release-assets.py
Normal file
249
tools/validate-driver-release-assets.py
Normal file
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"
|
||||
|
||||
|
||||
def github_headers(binary: bool = False):
|
||||
headers = {
|
||||
"Accept": "application/octet-stream" if binary else "application/vnd.github+json",
|
||||
"User-Agent": "GoNavi-CI",
|
||||
}
|
||||
token = os.environ.get("DRIVER_RELEASE_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def fetch_json(url: str):
|
||||
req = urllib.request.Request(url, headers=github_headers(False))
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def download_url(url: str, destination: Path):
|
||||
req = urllib.request.Request(url, headers=github_headers(False))
|
||||
with urllib.request.urlopen(req, timeout=120) as resp, open(destination, "wb") as out:
|
||||
shutil.copyfileobj(resp, out)
|
||||
|
||||
|
||||
def load_release(repo: str, tag: str):
|
||||
owner_repo = repo.strip()
|
||||
if not owner_repo:
|
||||
raise ValueError("repo is required")
|
||||
if tag == "latest":
|
||||
url = f"https://api.github.com/repos/{owner_repo}/releases/latest"
|
||||
else:
|
||||
url = f"https://api.github.com/repos/{owner_repo}/releases/tags/{urllib.parse.quote(tag, safe='')}"
|
||||
return fetch_json(url)
|
||||
|
||||
|
||||
def asset_map(release: dict):
|
||||
result = {}
|
||||
for asset in release.get("assets", []):
|
||||
name = str(asset.get("name") or "").strip()
|
||||
if name:
|
||||
result[name] = asset
|
||||
return result
|
||||
|
||||
|
||||
def asset_sha256_digest(asset: dict):
|
||||
digest = str(asset.get("digest") or "").strip().lower()
|
||||
prefix = "sha256:"
|
||||
if digest.startswith(prefix):
|
||||
value = digest[len(prefix) :].strip()
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def infer_asset_path(name: str):
|
||||
trimmed = str(name or "").strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
if trimmed == "duckdb.dll":
|
||||
return "Windows/duckdb.dll"
|
||||
if trimmed == "duckdb-driver.zip":
|
||||
return None
|
||||
if (
|
||||
trimmed.endswith("-driver-agent-windows-amd64.exe")
|
||||
or trimmed.endswith("-driver-agent-windows-arm64.exe")
|
||||
or trimmed.endswith("-driver-agent-v1-windows-amd64.exe")
|
||||
or trimmed.endswith("-driver-agent-v1-windows-arm64.exe")
|
||||
or trimmed.endswith("-driver-agent-v2-windows-amd64.exe")
|
||||
or trimmed.endswith("-driver-agent-v2-windows-arm64.exe")
|
||||
):
|
||||
return f"Windows/{trimmed}"
|
||||
if (
|
||||
trimmed.endswith("-driver-agent-darwin-amd64")
|
||||
or trimmed.endswith("-driver-agent-darwin-arm64")
|
||||
or trimmed.endswith("-driver-agent-v1-darwin-amd64")
|
||||
or trimmed.endswith("-driver-agent-v1-darwin-arm64")
|
||||
or trimmed.endswith("-driver-agent-v2-darwin-amd64")
|
||||
or trimmed.endswith("-driver-agent-v2-darwin-arm64")
|
||||
):
|
||||
return f"MacOS/{trimmed}"
|
||||
if (
|
||||
trimmed.endswith("-driver-agent-linux-amd64")
|
||||
or trimmed.endswith("-driver-agent-v1-linux-amd64")
|
||||
or trimmed.endswith("-driver-agent-v2-linux-amd64")
|
||||
):
|
||||
return f"Linux/{trimmed}"
|
||||
return None
|
||||
|
||||
|
||||
def normalize_machine(value: str):
|
||||
machine = str(value or "").strip().lower()
|
||||
if machine in {"x86_64", "amd64"}:
|
||||
return "amd64"
|
||||
if machine in {"aarch64", "arm64"}:
|
||||
return "arm64"
|
||||
return machine
|
||||
|
||||
|
||||
def current_runtime_platform():
|
||||
system = platform.system().lower()
|
||||
if system == "darwin":
|
||||
goos = "darwin"
|
||||
elif system == "windows":
|
||||
goos = "windows"
|
||||
elif system == "linux":
|
||||
goos = "linux"
|
||||
else:
|
||||
return ""
|
||||
goarch = normalize_machine(platform.machine())
|
||||
if not goarch:
|
||||
return ""
|
||||
return f"{goos}/{goarch}"
|
||||
|
||||
|
||||
def sha256_file(path: Path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def probe_metadata_revision(path: Path):
|
||||
current_mode = path.stat().st_mode
|
||||
path.chmod(current_mode | stat.S_IXUSR)
|
||||
proc = subprocess.run(
|
||||
[str(path)],
|
||||
input=b'{"id":1,"method":"metadata"}\n',
|
||||
capture_output=True,
|
||||
timeout=20,
|
||||
check=True,
|
||||
)
|
||||
if not proc.stdout:
|
||||
raise RuntimeError(f"{path.name}: metadata output is empty")
|
||||
payload = json.loads(proc.stdout.decode("utf-8"))
|
||||
return str(((payload.get("data") or {}).get("agentRevision") or "")).strip()
|
||||
|
||||
|
||||
def validate_release_assets(release: dict, manifest: dict, runtime_platform=None):
|
||||
assets = asset_map(release)
|
||||
manifest_assets = manifest.get("assets") or {}
|
||||
if not isinstance(manifest_assets, dict) or not manifest_assets:
|
||||
raise RuntimeError("manifest assets is empty")
|
||||
|
||||
if runtime_platform is None:
|
||||
runtime_platform = current_runtime_platform()
|
||||
runtime_platform = str(runtime_platform or "").strip().lower()
|
||||
|
||||
mismatches = []
|
||||
skipped = []
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-assets-") as tmp:
|
||||
tmp_root = Path(tmp)
|
||||
for name, meta in sorted(manifest_assets.items()):
|
||||
if name == MANIFEST_ASSET_NAME:
|
||||
continue
|
||||
asset = assets.get(name)
|
||||
if asset is None:
|
||||
mismatches.append((name, "missing_release_asset", "", "present in manifest"))
|
||||
continue
|
||||
|
||||
expected_sha = str(meta.get("sha256") or "").strip().lower()
|
||||
expected_revision = str(meta.get("revision") or "").strip()
|
||||
asset_platform = str(meta.get("platform") or "").strip().lower()
|
||||
|
||||
local_path = None
|
||||
actual_sha = asset_sha256_digest(asset)
|
||||
if expected_sha and not actual_sha:
|
||||
local_path = tmp_root / name
|
||||
download_url(str(asset.get("browser_download_url") or "").strip(), local_path)
|
||||
actual_sha = sha256_file(local_path).lower()
|
||||
if expected_sha and actual_sha != expected_sha:
|
||||
mismatches.append((name, "sha256", actual_sha, expected_sha))
|
||||
continue
|
||||
|
||||
path_hint = infer_asset_path(name)
|
||||
if path_hint is None:
|
||||
skipped.append(name)
|
||||
continue
|
||||
|
||||
if expected_revision and asset_platform == runtime_platform:
|
||||
if local_path is None:
|
||||
local_path = tmp_root / name
|
||||
download_url(str(asset.get("browser_download_url") or "").strip(), local_path)
|
||||
actual_revision = probe_metadata_revision(local_path)
|
||||
if actual_revision != expected_revision:
|
||||
mismatches.append((name, "revision", actual_revision, expected_revision))
|
||||
|
||||
return mismatches, skipped
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
|
||||
parser.add_argument("--tag", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
release = load_release(args.repo, args.tag)
|
||||
assets = asset_map(release)
|
||||
manifest_asset = assets.get(MANIFEST_ASSET_NAME)
|
||||
if manifest_asset is None:
|
||||
raise SystemExit(f"release {args.repo}@{args.tag} missing {MANIFEST_ASSET_NAME}")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-") as tmp:
|
||||
manifest_path = Path(tmp) / MANIFEST_ASSET_NAME
|
||||
download_url(str(manifest_asset.get("browser_download_url") or "").strip(), manifest_path)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
mismatches, skipped = validate_release_assets(release, manifest)
|
||||
if mismatches:
|
||||
print("published driver release assets mismatch manifest:", file=sys.stderr)
|
||||
for name, field, actual, expected in mismatches:
|
||||
print(f" - {name} [{field}] actual={actual or '<empty>'} expected={expected or '<empty>'}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
checked = len((manifest.get("assets") or {})) - len(skipped)
|
||||
print(f"driver release assets validation passed: checked={checked} skipped={len(skipped)}")
|
||||
if skipped:
|
||||
print("skipped assets: " + ", ".join(sorted(skipped)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"http error: {exc.code}", file=sys.stderr)
|
||||
raise
|
||||
201
tools/validate-driver-release-assets.test.py
Normal file
201
tools/validate-driver-release-assets.test.py
Normal file
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
MODULE_PATH = pathlib.Path(__file__).with_name("validate-driver-release-assets.py")
|
||||
SPEC = importlib.util.spec_from_file_location("validate_driver_release_assets", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class ValidateDriverReleaseAssetsTests(unittest.TestCase):
|
||||
def test_infer_asset_path(self):
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("clickhouse-driver-agent-darwin-arm64"),
|
||||
"MacOS/clickhouse-driver-agent-darwin-arm64",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("clickhouse-driver-agent-windows-arm64.exe"),
|
||||
"Windows/clickhouse-driver-agent-windows-arm64.exe",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("mongodb-driver-agent-v1-windows-amd64.exe"),
|
||||
"Windows/mongodb-driver-agent-v1-windows-amd64.exe",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_asset_path("mongodb-driver-agent-v2-darwin-arm64"),
|
||||
"MacOS/mongodb-driver-agent-v2-darwin-arm64",
|
||||
)
|
||||
self.assertEqual(MODULE.infer_asset_path("duckdb.dll"), "Windows/duckdb.dll")
|
||||
self.assertIsNone(MODULE.infer_asset_path("duckdb-driver.zip"))
|
||||
|
||||
def test_validate_release_assets_reports_sha_mismatch(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-arm64",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-arm64",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-arm64": {
|
||||
"revision": "src-expected",
|
||||
"sha256": "deadbeef",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
payload_path = pathlib.Path(tmp) / "clickhouse-driver-agent-darwin-arm64"
|
||||
payload_path.write_bytes(b"test-binary")
|
||||
|
||||
def fake_download(url, destination):
|
||||
destination.write_bytes(payload_path.read_bytes())
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-expected"
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(len(mismatches), 1)
|
||||
name, field, actual, expected = mismatches[0]
|
||||
self.assertEqual(name, "clickhouse-driver-agent-darwin-arm64")
|
||||
self.assertEqual(field, "sha256")
|
||||
self.assertEqual(actual, hashlib.sha256(b"test-binary").hexdigest())
|
||||
self.assertEqual(expected, "deadbeef")
|
||||
|
||||
def test_validate_release_assets_reports_revision_mismatch(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-arm64",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-arm64",
|
||||
}
|
||||
]
|
||||
}
|
||||
payload = b"test-binary"
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-arm64": {
|
||||
"platform": "darwin/arm64",
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
payload_path = pathlib.Path(tmp) / "clickhouse-driver-agent-darwin-arm64"
|
||||
payload_path.write_bytes(payload)
|
||||
|
||||
def fake_download(url, destination):
|
||||
destination.write_bytes(payload_path.read_bytes())
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-actual"
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest, runtime_platform="darwin/arm64")
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(mismatches, [("clickhouse-driver-agent-darwin-arm64", "revision", "src-actual", "src-expected")])
|
||||
|
||||
def test_validate_release_assets_skips_cross_platform_revision_probe(self):
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-amd64",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-amd64",
|
||||
}
|
||||
]
|
||||
}
|
||||
payload = b"darwin-binary"
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-amd64": {
|
||||
"platform": "darwin/amd64",
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
payload_path = pathlib.Path(tmp) / "clickhouse-driver-agent-darwin-amd64"
|
||||
payload_path.write_bytes(payload)
|
||||
|
||||
def fake_download(url, destination):
|
||||
destination.write_bytes(payload_path.read_bytes())
|
||||
|
||||
def fail_probe(_path):
|
||||
raise AssertionError("cross-platform asset should not be executed")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = fail_probe
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest, runtime_platform="linux/amd64")
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(mismatches, [])
|
||||
self.assertEqual(skipped, [])
|
||||
|
||||
def test_validate_release_assets_uses_release_digest_without_download(self):
|
||||
payload = b"darwin-binary"
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": "clickhouse-driver-agent-darwin-amd64",
|
||||
"digest": f"sha256:{digest}",
|
||||
"browser_download_url": "https://example.test/clickhouse-driver-agent-darwin-amd64",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-amd64": {
|
||||
"platform": "darwin/amd64",
|
||||
"revision": "src-expected",
|
||||
"sha256": digest,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("release digest should avoid downloading cross-platform assets")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(release, manifest, runtime_platform="linux/amd64")
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(mismatches, [])
|
||||
self.assertEqual(skipped, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
163
tools/validate-driver-release-manifest.sh
Normal file
163
tools/validate-driver-release-manifest.sh
Normal file
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
./tools/validate-driver-release-manifest.sh --commit <ref> --manifest <path>
|
||||
|
||||
说明：
|
||||
校验已发布 driver release manifest 中记录的每个 driver revision，
|
||||
是否与指定源码提交在对应平台上重新生成出的 revision 完全一致。
|
||||
EOF
|
||||
}
|
||||
|
||||
source_commit=""
|
||||
manifest_path=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--commit)
|
||||
source_commit="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--manifest)
|
||||
manifest_path="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "未知参数：$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$source_commit" || -z "$manifest_path" ]]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$manifest_path" ]]; then
|
||||
echo "manifest 不存在：$manifest_path" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git rev-parse --verify "${source_commit}^{commit}" >/dev/null 2>&1; then
|
||||
echo "无法解析源码提交：$source_commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
extract_revision() {
|
||||
local file="$1"
|
||||
local driver="$2"
|
||||
awk -v target="$driver" '
|
||||
$0 ~ "\"" target "\"" {
|
||||
if (match($0, /"src-[^"]+"/)) {
|
||||
print substr($0, RSTART + 1, RLENGTH - 2)
|
||||
exit
|
||||
}
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
normalize_driver() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
|
||||
case "$value" in
|
||||
doris|diros) echo "diros" ;;
|
||||
*) echo "$value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-manifest.XXXXXX")"
|
||||
cleanup() {
|
||||
git worktree remove --force "$worktree" >/dev/null 2>&1 || true
|
||||
rm -rf "$worktree"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git worktree add --detach "$worktree" "${source_commit}^{commit}" >/dev/null
|
||||
|
||||
python3 - "$manifest_path" "$worktree" <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
manifest_path = Path(sys.argv[1]).resolve()
|
||||
worktree = Path(sys.argv[2]).resolve()
|
||||
|
||||
with manifest_path.open("r", encoding="utf-8") as fh:
|
||||
manifest = json.load(fh)
|
||||
|
||||
assets = manifest.get("assets") or {}
|
||||
if not isinstance(assets, dict) or not assets:
|
||||
raise SystemExit("manifest assets 为空")
|
||||
|
||||
platforms = sorted({str(meta.get("platform") or "").strip() for meta in assets.values() if str(meta.get("platform") or "").strip()})
|
||||
if not platforms:
|
||||
raise SystemExit("manifest 未包含平台信息")
|
||||
|
||||
def normalize_driver(driver: str) -> str:
|
||||
value = str(driver or "").strip().lower()
|
||||
if value == "doris":
|
||||
return "diros"
|
||||
return value
|
||||
|
||||
|
||||
def parse_revision_file(file_path: Path):
|
||||
import re
|
||||
text = file_path.read_text(encoding="utf-8")
|
||||
revisions = {}
|
||||
for match in re.finditer(r'"([^"]+)"\s*:\s*"([^"]+)"', text):
|
||||
revisions[match.group(1)] = match.group(2)
|
||||
return revisions
|
||||
|
||||
drivers_by_platform = {}
|
||||
for meta in assets.values():
|
||||
platform = str(meta.get("platform") or "").strip()
|
||||
driver = normalize_driver(meta.get("driver") or meta.get("driverType"))
|
||||
if platform and driver:
|
||||
drivers_by_platform.setdefault(platform, set()).add(driver)
|
||||
|
||||
revision_maps = {}
|
||||
for platform in platforms:
|
||||
command = ["bash", "./tools/generate-driver-agent-revisions.sh", "--platform", platform]
|
||||
platform_drivers = sorted(drivers_by_platform.get(platform) or [])
|
||||
if platform_drivers:
|
||||
command.extend(["--drivers", ",".join(platform_drivers)])
|
||||
subprocess.run(command, cwd=worktree, check=True, stdout=subprocess.DEVNULL)
|
||||
revision_maps[platform] = parse_revision_file(worktree / "internal/db/driver_agent_revisions_gen.go")
|
||||
|
||||
mismatches = []
|
||||
for asset_name, meta in sorted(assets.items()):
|
||||
driver = normalize_driver(meta.get("driver") or meta.get("driverType"))
|
||||
platform = str(meta.get("platform") or "").strip()
|
||||
published_revision = str(meta.get("revision") or "").strip()
|
||||
expected_revision = (revision_maps.get(platform) or {}).get(driver, "")
|
||||
if not expected_revision:
|
||||
mismatches.append((asset_name, platform, driver, published_revision, "<missing>"))
|
||||
continue
|
||||
if published_revision != expected_revision:
|
||||
mismatches.append((asset_name, platform, driver, published_revision, expected_revision))
|
||||
|
||||
if mismatches:
|
||||
print("published driver release manifest 与源码重算 revision 不一致：", file=sys.stderr)
|
||||
for asset_name, platform, driver, published_revision, expected_revision in mismatches:
|
||||
print(
|
||||
f" - {asset_name} [{platform}/{driver}] published={published_revision} expected={expected_revision}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"manifest validation passed: {len(assets)} assets")
|
||||
PY
|
||||
104
tools/validate-driver-release-manifest.test.sh
Normal file
104
tools/validate-driver-release-manifest.test.sh
Normal file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
worktrees=()
|
||||
cleanup() {
|
||||
local worktree
|
||||
for worktree in "${worktrees[@]+"${worktrees[@]}"}"; do
|
||||
git worktree remove --force "$worktree" >/dev/null 2>&1 || true
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
manifest_path="$tmpdir/manifest.json"
|
||||
|
||||
generate_revision() {
|
||||
local platform="$1"
|
||||
local driver="$2"
|
||||
local worktree revision_file
|
||||
worktree="$tmpdir/worktree-${platform//\//-}-${driver}"
|
||||
git worktree add --detach "$worktree" HEAD >/dev/null
|
||||
worktrees+=("$worktree")
|
||||
(
|
||||
cd "$worktree"
|
||||
bash ./tools/generate-driver-agent-revisions.sh --platform "$platform" --drivers "$driver" >/dev/null
|
||||
)
|
||||
revision_file="$worktree/internal/db/driver_agent_revisions_gen.go"
|
||||
awk -v target="$driver" '
|
||||
$0 ~ "\"" target "\"" {
|
||||
if (match($0, /"src-[^"]+"/)) {
|
||||
print substr($0, RSTART + 1, RLENGTH - 2)
|
||||
exit
|
||||
}
|
||||
}
|
||||
' "$revision_file"
|
||||
}
|
||||
|
||||
cat >"$manifest_path" <<'EOF'
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generatedFrom": "test",
|
||||
"assets": {
|
||||
"clickhouse-driver-agent-darwin-arm64": {
|
||||
"driver": "clickhouse",
|
||||
"driverType": "clickhouse",
|
||||
"platform": "darwin/arm64",
|
||||
"revision": "__CLICKHOUSE_DARWIN_ARM64__",
|
||||
"size": 1
|
||||
},
|
||||
"clickhouse-driver-agent-linux-amd64": {
|
||||
"driver": "clickhouse",
|
||||
"driverType": "clickhouse",
|
||||
"platform": "linux/amd64",
|
||||
"revision": "__CLICKHOUSE_LINUX_AMD64__",
|
||||
"size": 1
|
||||
},
|
||||
"clickhouse-driver-agent-windows-amd64.exe": {
|
||||
"driver": "clickhouse",
|
||||
"driverType": "clickhouse",
|
||||
"platform": "windows/amd64",
|
||||
"revision": "__CLICKHOUSE_WINDOWS_AMD64__",
|
||||
"size": 1
|
||||
},
|
||||
"mariadb-driver-agent-darwin-arm64": {
|
||||
"driver": "mariadb",
|
||||
"driverType": "mariadb",
|
||||
"platform": "darwin/arm64",
|
||||
"revision": "__MARIADB__",
|
||||
"size": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
clickhouse_darwin_revision="$(generate_revision darwin/arm64 clickhouse)"
|
||||
clickhouse_linux_revision="$(generate_revision linux/amd64 clickhouse)"
|
||||
clickhouse_windows_revision="$(generate_revision windows/amd64 clickhouse)"
|
||||
mariadb_darwin_revision="$(generate_revision darwin/arm64 mariadb)"
|
||||
|
||||
python3 - "$manifest_path" "$clickhouse_darwin_revision" "$clickhouse_linux_revision" "$clickhouse_windows_revision" "$mariadb_darwin_revision" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
clickhouse_darwin = sys.argv[2]
|
||||
clickhouse_linux = sys.argv[3]
|
||||
clickhouse_windows = sys.argv[4]
|
||||
mariadb = sys.argv[5]
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
data["assets"]["clickhouse-driver-agent-darwin-arm64"]["revision"] = clickhouse_darwin
|
||||
data["assets"]["clickhouse-driver-agent-linux-amd64"]["revision"] = clickhouse_linux
|
||||
data["assets"]["clickhouse-driver-agent-windows-amd64.exe"]["revision"] = clickhouse_windows
|
||||
data["assets"]["mariadb-driver-agent-darwin-arm64"]["revision"] = mariadb
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
bash ./tools/validate-driver-release-manifest.sh --commit HEAD --manifest "$manifest_path"
|
||||
echo "validate-driver-release-manifest test passed"
|
||||
302
tools/verify-driver-agent-revisions.sh
Executable file
302
tools/verify-driver-agent-revisions.sh
Executable file
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
./tools/verify-driver-agent-revisions.sh --assets-dir <目录> --platform <GOOS/GOARCH> --drivers <列表>
|
||||
|
||||
说明：
|
||||
校验已构建 driver-agent 资产返回的 agentRevision 是否等于当前源码生成的 revision。
|
||||
EOF
|
||||
}
|
||||
|
||||
assets_dir=""
|
||||
target_platform=""
|
||||
driver_csv=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--assets-dir)
|
||||
assets_dir="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--platform)
|
||||
target_platform="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--drivers)
|
||||
driver_csv="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "未知参数：$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$assets_dir" || -z "$target_platform" || -z "$driver_csv" ]]; then
|
||||
usage >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$target_platform" != */* ]]; then
|
||||
echo "--platform 参数格式错误，应为 GOOS/GOARCH，例如 darwin/arm64" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
goos="${target_platform%%/*}"
|
||||
goarch="${target_platform##*/}"
|
||||
platform_dir="Unknown"
|
||||
case "$goos" in
|
||||
windows) platform_dir="Windows" ;;
|
||||
darwin) platform_dir="MacOS" ;;
|
||||
linux) platform_dir="Linux" ;;
|
||||
esac
|
||||
|
||||
normalize_driver() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
|
||||
case "$value" in
|
||||
doris|diros) echo "diros" ;;
|
||||
opengauss|open_gauss|open-gauss) echo "opengauss" ;;
|
||||
gaussdb|gauss_db|gauss-db) echo "gaussdb" ;;
|
||||
elasticsearch|elastic) echo "elasticsearch" ;;
|
||||
mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|gaussdb|iris|mongodb|tdengine|iotdb|clickhouse)
|
||||
echo "$value"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
public_driver_name() {
|
||||
case "$1" in
|
||||
diros) echo "doris" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
expected_revision_for() {
|
||||
local target="$1"
|
||||
awk -v target="$target" '
|
||||
$0 ~ "\"" target "\"" {
|
||||
if (match($0, /"src-[^"]+"/)) {
|
||||
value=substr($0, RSTART + 1, RLENGTH - 2)
|
||||
print value
|
||||
exit
|
||||
}
|
||||
}
|
||||
' internal/db/driver_agent_revisions_gen.go
|
||||
}
|
||||
|
||||
build_tags_for_driver() {
|
||||
local driver="$1"
|
||||
local variant="${2:-}"
|
||||
local tags="gonavi_${driver}_driver"
|
||||
if [[ "$driver" == "mongodb" && "$variant" == "v1" ]]; then
|
||||
tags="gonavi_mongodb_driver_v1"
|
||||
fi
|
||||
if [[ "$driver" == "duckdb" && "$host_goos" == "windows" && "$host_goarch" == "amd64" ]]; then
|
||||
tags="${tags} duckdb_use_lib"
|
||||
fi
|
||||
printf '%s\n' "$tags"
|
||||
}
|
||||
|
||||
agent_path_for() {
|
||||
local driver="$1"
|
||||
local variant="${2:-}"
|
||||
local public_name asset
|
||||
public_name="$(public_driver_name "$driver")"
|
||||
if [[ "$driver" == "mongodb" && -n "$variant" ]]; then
|
||||
asset="${public_name}-driver-agent-${variant}-${goos}-${goarch}"
|
||||
else
|
||||
asset="${public_name}-driver-agent-${goos}-${goarch}"
|
||||
fi
|
||||
if [[ "$goos" == "windows" ]]; then
|
||||
asset="${asset}.exe"
|
||||
fi
|
||||
printf '%s\n' "${assets_dir%/}/${platform_dir}/${asset}"
|
||||
}
|
||||
|
||||
agent_variants_for() {
|
||||
local driver="$1"
|
||||
if [[ "$driver" == "mongodb" ]]; then
|
||||
printf '%s\n' "v1" "v2" ""
|
||||
return
|
||||
fi
|
||||
printf '%s\n' ""
|
||||
}
|
||||
|
||||
probe_agent_revision() {
|
||||
local agent_path="$1"
|
||||
local request
|
||||
request='{"id":1,"method":"metadata"}'
|
||||
printf '%s\n' "$request" | "$agent_path" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
line = sys.stdin.readline()
|
||||
payload = json.loads(line)
|
||||
data = payload.get("data") or {}
|
||||
print(data.get("agentRevision", ""))
|
||||
'
|
||||
}
|
||||
|
||||
probe_host_agent_revision() {
|
||||
local driver="$1"
|
||||
local variant="${2:-}"
|
||||
local build_tags probe_dir probe_path revision
|
||||
build_tags="$(build_tags_for_driver "$driver" "$variant")"
|
||||
probe_dir="$(mktemp -d)"
|
||||
probe_path="${probe_dir}/probe-agent"
|
||||
if [[ "$host_goos" == "windows" ]]; then
|
||||
probe_path="${probe_path}.exe"
|
||||
fi
|
||||
|
||||
CGO_ENABLED=0 go build \
|
||||
-tags "${build_tags}" \
|
||||
-trimpath \
|
||||
-ldflags "-s -w" \
|
||||
-o "${probe_path}" \
|
||||
./cmd/optional-driver-agent >/dev/null
|
||||
|
||||
chmod +x "$probe_path" 2>/dev/null || true
|
||||
revision="$(probe_agent_revision "$probe_path")"
|
||||
rm -rf "$probe_dir"
|
||||
printf '%s\n' "$revision"
|
||||
}
|
||||
|
||||
can_execute_target_binary() {
|
||||
[[ "$target_platform" == "$host_platform" ]]
|
||||
}
|
||||
|
||||
validate_windows_pe_machine() {
|
||||
local agent_path="$1"
|
||||
local expected_goarch="$2"
|
||||
python3 - "$agent_path" "$expected_goarch" <<'PY'
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
goarch = sys.argv[2].strip().lower()
|
||||
expected = {
|
||||
"386": 0x014C,
|
||||
"amd64": 0x8664,
|
||||
"arm64": 0xAA64,
|
||||
}
|
||||
labels = {
|
||||
0x014C: "windows-386",
|
||||
0x8664: "windows-amd64",
|
||||
0xAA64: "windows-arm64",
|
||||
}
|
||||
|
||||
if goarch not in expected:
|
||||
sys.exit(0)
|
||||
|
||||
with open(path, "rb") as fh:
|
||||
fh.seek(0, os.SEEK_END)
|
||||
size = fh.tell()
|
||||
if size < 0x40:
|
||||
raise SystemExit("文件头不完整")
|
||||
|
||||
fh.seek(0)
|
||||
if fh.read(2) != b"MZ":
|
||||
raise SystemExit("缺少 MZ 头")
|
||||
|
||||
fh.seek(0x3C)
|
||||
pe_offset_raw = fh.read(4)
|
||||
if len(pe_offset_raw) != 4:
|
||||
raise SystemExit("读取 PE 头偏移失败")
|
||||
pe_offset = struct.unpack("<I", pe_offset_raw)[0]
|
||||
if pe_offset < 0x40 or pe_offset + 24 > size:
|
||||
raise SystemExit("PE 头不完整")
|
||||
|
||||
fh.seek(pe_offset)
|
||||
if fh.read(4) != b"PE\0\0":
|
||||
raise SystemExit("缺少 PE 签名")
|
||||
|
||||
machine_raw = fh.read(2)
|
||||
if len(machine_raw) != 2:
|
||||
raise SystemExit("读取 PE 架构失败")
|
||||
machine = struct.unpack("<H", machine_raw)[0]
|
||||
|
||||
expected_machine = expected[goarch]
|
||||
if machine != expected_machine:
|
||||
raise SystemExit(f"可执行文件架构不兼容（文件={labels.get(machine, hex(machine))}，期望={labels[expected_machine]})")
|
||||
PY
|
||||
}
|
||||
|
||||
declare -a raw_drivers=()
|
||||
IFS=',' read -r -a raw_drivers <<<"$driver_csv"
|
||||
|
||||
host_goos="$(go env GOOS)"
|
||||
host_goarch="$(go env GOARCH)"
|
||||
host_platform="${host_goos}/${host_goarch}"
|
||||
|
||||
failed=0
|
||||
for raw_driver in "${raw_drivers[@]}"; do
|
||||
[[ -n "$raw_driver" ]] || continue
|
||||
driver="$(normalize_driver "$raw_driver")"
|
||||
if [[ "$driver" == "duckdb" && "$goos" == "windows" && "$goarch" != "amd64" ]]; then
|
||||
echo "⚠️ 跳过 duckdb revision 校验（$target_platform 不构建 agent）"
|
||||
continue
|
||||
fi
|
||||
|
||||
expected="$(expected_revision_for "$driver")"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "❌ $driver 缺少期望 revision"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
|
||||
while IFS= read -r variant; do
|
||||
agent_path="$(agent_path_for "$driver" "$variant")"
|
||||
variant_label="$driver"
|
||||
if [[ -n "$variant" ]]; then
|
||||
variant_label="${driver}-${variant}"
|
||||
fi
|
||||
if [[ ! -f "$agent_path" ]]; then
|
||||
echo "❌ $variant_label 缺少 driver-agent 资产：$agent_path"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
chmod +x "$agent_path" 2>/dev/null || true
|
||||
|
||||
if [[ "$goos" == "windows" ]]; then
|
||||
if ! validate_windows_pe_machine "$agent_path" "$goarch"; then
|
||||
echo "❌ $variant_label Windows driver-agent 架构校验失败：asset=$agent_path target=$target_platform"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
actual=""
|
||||
if can_execute_target_binary; then
|
||||
actual="$(probe_agent_revision "$agent_path" || true)"
|
||||
else
|
||||
echo "ℹ️ runner 平台 ${host_platform} 无法直接执行目标二进制 ${target_platform}，已先完成目标资产架构校验，再用 host-native probe 校验相同 build tags 的 revision"
|
||||
actual="$(probe_host_agent_revision "$driver" "$variant" || true)"
|
||||
fi
|
||||
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "❌ $variant_label driver-agent revision 不匹配：asset=$agent_path actual=${actual:-空} expected=$expected"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
echo "✅ $variant_label driver-agent revision 校验通过：$actual"
|
||||
done < <(agent_variants_for "$driver")
|
||||
done
|
||||
|
||||
exit "$failed"
|
||||
Reference in New Issue
Block a user