️ perf(ci): 优化 DriverAgents 按需构建流程

- 增加 driver-agent 变更检测任务
- 跳过未变更 driver 的构建与 revision 生成
- 复用前端构建产物,减少矩阵任务重复耗时
This commit is contained in:
Syngnat
2026-05-16 13:38:00 +08:00
parent 0ff3f99c18
commit d791303967
5 changed files with 1012 additions and 70 deletions

View File

@@ -16,8 +16,96 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
frontend:
name: Build frontend
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.24'
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Cache frontend node_modules
uses: actions/cache@v4
with:
path: frontend/node_modules
key: ${{ runner.os }}-node20-frontend-${{ hashFiles('frontend/package-lock.json') }}
- name: Install Wails
run: go install -v github.com/wailsapp/wails/v2/cmd/wails@v2.11.0
- name: Build frontend dist
shell: bash
run: |
set -euo pipefail
wails generate module
node frontend/scripts/wails-frontend-install.mjs
npm --prefix frontend run build
- name: Pack frontend dist
shell: bash
run: tar -cf frontend-dist.tar -C frontend/dist .
- name: Upload frontend dist
uses: actions/upload-artifact@v4
with:
name: frontend-dist
path: frontend-dist.tar
if-no-files-found: error
retention-days: 1
driver_agents:
name: Detect changed driver agents
runs-on: ubuntu-latest
outputs:
drivers: ${{ steps.detect.outputs.drivers }}
has_changes: ${{ steps.detect.outputs.has_changes }}
release_source: ${{ steps.detect.outputs.release_source }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Detect changed driver agents
id: detect
shell: bash
run: |
set -euo pipefail
BASE_REF="${{ github.event.before }}"
if [[ -z "$BASE_REF" || "$BASE_REF" =~ ^0+$ ]]; then
if BASE_REF="$(git rev-parse HEAD^ 2>/dev/null)"; then
:
else
BASE_REF="all"
fi
fi
DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT"
if [ -n "$DRIVERS" ]; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
echo "🧭 Changed driver agents: $DRIVERS"
else
echo "has_changes=false" >> "$GITHUB_OUTPUT"
echo "🧭 No driver-agent changes detected"
fi
echo "release_source=dev-latest" >> "$GITHUB_OUTPUT"
build:
name: Build ${{ matrix.platform }}
needs:
- frontend
- driver_agents
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -87,21 +175,22 @@ jobs:
with:
go-version: '1.24'
- name: Setup Node
uses: actions/setup-node@v4
- name: Download frontend dist
uses: actions/download-artifact@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
name: frontend-dist
path: .
- name: Cache frontend node_modules
uses: actions/cache@v4
with:
path: frontend/node_modules
key: ${{ runner.os }}-node20-frontend-${{ hashFiles('frontend/package-lock.json') }}
- name: Extract frontend dist
shell: bash
run: |
set -euo pipefail
mkdir -p frontend/dist
tar -xf frontend-dist.tar -C frontend/dist
test -s frontend/dist/index.html
- name: Install UPX (Windows)
if: contains(matrix.platform, 'windows')
if: matrix.platform == 'windows/amd64'
shell: pwsh
run: |
$UPX_VERSION = "4.2.4"
@@ -164,7 +253,7 @@ jobs:
- name: Setup MSYS2 Toolchain For DuckDB (Windows AMD64)
id: msys2_duckdb
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
continue-on-error: true
uses: msys2/setup-msys2@v2
with:
@@ -174,7 +263,7 @@ jobs:
mingw-w64-ucrt-x86_64-gcc
- name: Configure DuckDB CGO Toolchain (Windows AMD64)
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
shell: pwsh
run: |
function Find-MingwBin([string[]]$candidates) {
@@ -232,7 +321,7 @@ jobs:
Write-Host "✅ 已配置 DuckDB cgo 编译器: gcc=$gcc g++=$gxx"
- name: Verify DuckDB CGO Toolchain (Windows AMD64)
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
shell: pwsh
run: |
& "$env:CC" --version
@@ -250,25 +339,33 @@ jobs:
- name: Build
shell: bash
env:
CHANGED_DRIVER_AGENTS: ${{ needs.driver_agents.outputs.drivers }}
run: |
set -euo pipefail
DEV_VERSION="${{ steps.version.outputs.version }}"
./tools/generate-driver-agent-revisions.sh --platform "${{ matrix.platform }}"
if [ -n "${{ matrix.wails_tags }}" ]; then
wails build -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -tags "${{ matrix.wails_tags }}" -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}"
if [ -n "$CHANGED_DRIVER_AGENTS" ]; then
./tools/generate-driver-agent-revisions.sh --platform "${{ matrix.platform }}" --drivers "$CHANGED_DRIVER_AGENTS"
else
wails build -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}"
echo "🧭 No driver-agent changes; keeping committed driver revisions"
fi
if [ -n "${{ matrix.wails_tags }}" ]; then
wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -tags "${{ matrix.wails_tags }}" -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}"
else
wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}"
fi
- name: Build Optional Driver Agents
if: ${{ matrix.build_optional_agents }}
if: ${{ matrix.build_optional_agents && needs.driver_agents.outputs.has_changes == 'true' }}
shell: bash
env:
CHANGED_DRIVER_AGENTS: ${{ needs.driver_agents.outputs.drivers }}
run: |
set -euo pipefail
TARGET_PLATFORM="${{ matrix.platform }}"
GOOS="${TARGET_PLATFORM%%/*}"
GOARCH="${TARGET_PLATFORM##*/}"
DRIVERS=(mariadb oceanbase doris starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss mongodb tdengine clickhouse)
IFS=',' read -r -a DRIVERS <<< "$CHANGED_DRIVER_AGENTS"
OUTDIR="drivers/${{ matrix.os_name }}"
mkdir -p "$OUTDIR"
DUCKDB_WINDOWS_LIBRARY_VERSION="v1.4.4"
@@ -547,9 +644,14 @@ jobs:
# 汇总所有产物并发布为 Pre-release
release:
name: Publish Dev Pre-release
needs: build
needs:
- build
- driver_agents
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
@@ -560,18 +662,31 @@ jobs:
- name: List Assets
run: ls -R release-assets
- name: Complete Driver Agent Assets
if: needs.driver_agents.outputs.has_changes == 'true'
env:
DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }}
run: |
python3 tools/complete-driver-release-assets.py \
--assets-dir release-assets \
--source "${{ needs.driver_agents.outputs.release_source }}" \
--require-complete
- name: Package Driver Agents Bundle
id: driver_assets
shell: bash
run: |
set -euo pipefail
cd release-assets
if [ ! -d drivers ]; then
echo "⚠️ 未找到 drivers 目录,跳过驱动总包打包"
echo "has_driver_assets=false" >> "$GITHUB_OUTPUT"
exit 0
fi
if [ -z "$(find drivers -type f 2>/dev/null | head -n 1)" ]; then
echo "⚠️ drivers 目录为空,跳过驱动总包打包"
rm -rf drivers
echo "has_driver_assets=false" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -625,6 +740,7 @@ jobs:
PY
rm -rf drivers
echo "has_driver_assets=true" >> "$GITHUB_OUTPUT"
- name: Generate SHA256SUMS
shell: bash
@@ -644,6 +760,7 @@ jobs:
fi
- name: Generate Driver SHA256SUMS
if: steps.driver_assets.outputs.has_driver_assets == 'true'
shell: bash
run: |
cd driver-release-assets
@@ -726,6 +843,7 @@ jobs:
}
- name: Reset Previous Driver Dev Release
if: steps.driver_assets.outputs.has_driver_assets == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.DRIVER_RELEASE_TOKEN }}
@@ -770,6 +888,7 @@ jobs:
}
- name: Create Dev Driver Agents Pre-release
if: steps.driver_assets.outputs.has_driver_assets == 'true'
uses: softprops/action-gh-release@v2
with:
repository: Syngnat/GoNavi-DriverAgents

View File

@@ -12,9 +12,98 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
frontend:
name: Build frontend
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.24'
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
- name: Cache frontend node_modules
uses: actions/cache@v4
with:
path: frontend/node_modules
key: ${{ runner.os }}-node20-frontend-${{ hashFiles('frontend/package-lock.json') }}
- name: Install Wails
run: go install -v github.com/wailsapp/wails/v2/cmd/wails@v2.11.0
- name: Build frontend dist
shell: bash
run: |
set -euo pipefail
wails generate module
node frontend/scripts/wails-frontend-install.mjs
npm --prefix frontend run build
- name: Pack frontend dist
shell: bash
run: tar -cf frontend-dist.tar -C frontend/dist .
- name: Upload frontend dist
uses: actions/upload-artifact@v4
with:
name: frontend-dist
path: frontend-dist.tar
if-no-files-found: error
retention-days: 1
driver_agents:
name: Detect changed driver agents
runs-on: ubuntu-latest
outputs:
drivers: ${{ steps.detect.outputs.drivers }}
has_changes: ${{ steps.detect.outputs.has_changes }}
release_source: ${{ steps.detect.outputs.release_source }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Detect changed driver agents
id: detect
shell: bash
run: |
set -euo pipefail
git fetch --force --tags
PREV_TAG="$(git describe --tags --match 'v*' --abbrev=0 "${GITHUB_SHA}^" 2>/dev/null || true)"
if [ -n "$PREV_TAG" ]; then
BASE_REF="$PREV_TAG"
RELEASE_SOURCE="$PREV_TAG"
else
BASE_REF="all"
RELEASE_SOURCE="all"
fi
DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT"
echo "release_source=${RELEASE_SOURCE}" >> "$GITHUB_OUTPUT"
if [ -n "$DRIVERS" ]; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
echo "🧭 Changed driver agents since ${BASE_REF}: $DRIVERS"
else
echo "has_changes=false" >> "$GITHUB_OUTPUT"
echo "🧭 No driver-agent changes since ${BASE_REF}"
fi
# Phase 1: Build in parallel and output artifacts
build:
name: Build ${{ matrix.platform }}
needs:
- frontend
- driver_agents
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -85,21 +174,22 @@ jobs:
with:
go-version: '1.24'
- name: Setup Node
uses: actions/setup-node@v4
- name: Download frontend dist
uses: actions/download-artifact@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: frontend/package-lock.json
name: frontend-dist
path: .
- name: Cache frontend node_modules
uses: actions/cache@v4
with:
path: frontend/node_modules
key: ${{ runner.os }}-node20-frontend-${{ hashFiles('frontend/package-lock.json') }}
- name: Extract frontend dist
shell: bash
run: |
set -euo pipefail
mkdir -p frontend/dist
tar -xf frontend-dist.tar -C frontend/dist
test -s frontend/dist/index.html
- name: Install UPX (Windows)
if: contains(matrix.platform, 'windows')
if: matrix.platform == 'windows/amd64'
shell: pwsh
run: |
$UPX_VERSION = "4.2.4"
@@ -166,7 +256,7 @@ jobs:
- name: Setup MSYS2 Toolchain For DuckDB (Windows AMD64)
id: msys2_duckdb
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
continue-on-error: true
uses: msys2/setup-msys2@v2
with:
@@ -176,7 +266,7 @@ jobs:
mingw-w64-ucrt-x86_64-gcc
- name: Configure DuckDB CGO Toolchain (Windows AMD64)
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
shell: pwsh
run: |
function Find-MingwBin([string[]]$candidates) {
@@ -234,7 +324,7 @@ jobs:
Write-Host "✅ 已配置 DuckDB cgo 编译器: gcc=$gcc g++=$gxx"
- name: Verify DuckDB CGO Toolchain (Windows AMD64)
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' }}
if: ${{ matrix.build_optional_agents && matrix.platform == 'windows/amd64' && contains(format(',{0},', needs.driver_agents.outputs.drivers), ',duckdb,') }}
shell: pwsh
run: |
& "$env:CC" --version
@@ -242,24 +332,32 @@ jobs:
- name: Build
shell: bash
env:
CHANGED_DRIVER_AGENTS: ${{ needs.driver_agents.outputs.drivers }}
run: |
set -euo pipefail
./tools/generate-driver-agent-revisions.sh --platform "${{ matrix.platform }}"
if [ -n "${{ matrix.wails_tags }}" ]; then
wails build -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -tags "${{ matrix.wails_tags }}" -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${{ github.ref_name }}"
if [ -n "$CHANGED_DRIVER_AGENTS" ]; then
./tools/generate-driver-agent-revisions.sh --platform "${{ matrix.platform }}" --drivers "$CHANGED_DRIVER_AGENTS"
else
wails build -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${{ github.ref_name }}"
echo "🧭 No driver-agent changes; keeping committed driver revisions"
fi
if [ -n "${{ matrix.wails_tags }}" ]; then
wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -tags "${{ matrix.wails_tags }}" -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${{ github.ref_name }}"
else
wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${{ github.ref_name }}"
fi
- name: Build Optional Driver Agents
if: ${{ matrix.build_optional_agents }}
if: ${{ matrix.build_optional_agents && needs.driver_agents.outputs.has_changes == 'true' }}
shell: bash
env:
CHANGED_DRIVER_AGENTS: ${{ needs.driver_agents.outputs.drivers }}
run: |
set -euo pipefail
TARGET_PLATFORM="${{ matrix.platform }}"
GOOS="${TARGET_PLATFORM%%/*}"
GOARCH="${TARGET_PLATFORM##*/}"
DRIVERS=(mariadb oceanbase doris starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss mongodb tdengine clickhouse)
IFS=',' read -r -a DRIVERS <<< "$CHANGED_DRIVER_AGENTS"
OUTDIR="drivers/${{ matrix.os_name }}"
mkdir -p "$OUTDIR"
DUCKDB_WINDOWS_LIBRARY_VERSION="v1.4.4"
@@ -555,9 +653,14 @@ jobs:
# Phase 2: Collect all artifacts and Publish Release (Single Job)
release:
name: Publish Release
needs: build
needs:
- build
- driver_agents
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
@@ -568,36 +671,48 @@ jobs:
- name: List Assets
run: ls -R release-assets
- name: Complete Driver Agent Assets
if: needs.driver_agents.outputs.has_changes == 'true' && needs.driver_agents.outputs.release_source != 'all'
env:
DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }}
run: |
python3 tools/complete-driver-release-assets.py \
--assets-dir release-assets \
--source "${{ needs.driver_agents.outputs.release_source }}" \
--require-complete
- name: Verify Optional Driver Assets
if: needs.driver_agents.outputs.has_changes == 'true'
shell: bash
env:
CHANGED_DRIVER_AGENTS: ${{ needs.driver_agents.outputs.drivers }}
run: |
set -euo pipefail
cd release-assets
REQUIRED_FILES=(
"drivers/Windows/duckdb-driver-agent-windows-amd64.exe"
"drivers/Windows/duckdb.dll"
"drivers/MacOS/duckdb-driver-agent-darwin-amd64"
"drivers/MacOS/duckdb-driver-agent-darwin-arm64"
"drivers/Linux/duckdb-driver-agent-linux-amd64"
"drivers/Windows/clickhouse-driver-agent-windows-amd64.exe"
"drivers/MacOS/clickhouse-driver-agent-darwin-amd64"
"drivers/MacOS/clickhouse-driver-agent-darwin-arm64"
"drivers/Linux/clickhouse-driver-agent-linux-amd64"
"drivers/Windows/starrocks-driver-agent-windows-amd64.exe"
"drivers/MacOS/starrocks-driver-agent-darwin-amd64"
"drivers/MacOS/starrocks-driver-agent-darwin-arm64"
"drivers/Linux/starrocks-driver-agent-linux-amd64"
)
missing=0
for file in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$file" ]; then
echo "❌ 缺少驱动资产:$file"
missing=1
IFS=',' read -r -a DRIVERS <<< "$CHANGED_DRIVER_AGENTS"
for driver in "${DRIVERS[@]}"; do
REQUIRED_FILES=(
"drivers/Windows/${driver}-driver-agent-windows-amd64.exe"
"drivers/MacOS/${driver}-driver-agent-darwin-amd64"
"drivers/MacOS/${driver}-driver-agent-darwin-arm64"
"drivers/Linux/${driver}-driver-agent-linux-amd64"
)
if [ "$driver" != "duckdb" ]; then
REQUIRED_FILES+=("drivers/Windows/${driver}-driver-agent-windows-arm64.exe")
else
echo "✅ 已找到驱动资产:$file"
REQUIRED_FILES+=("drivers/Windows/duckdb.dll")
fi
for file in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$file" ]; then
echo "❌ 缺少驱动资产:$file"
missing=1
else
echo "✅ 已找到驱动资产:$file"
fi
done
done
if [ "$missing" -ne 0 ]; then
@@ -606,17 +721,20 @@ jobs:
fi
- name: Package Driver Agents Bundle
id: driver_assets
shell: bash
run: |
set -euo pipefail
cd release-assets
if [ ! -d drivers ]; then
echo "⚠️ 未找到 drivers 目录,跳过驱动总包打包"
echo "has_driver_assets=false" >> "$GITHUB_OUTPUT"
exit 0
fi
if [ -z "$(find drivers -type f 2>/dev/null | head -n 1)" ]; then
echo "⚠️ drivers 目录为空,跳过驱动总包打包"
rm -rf drivers
echo "has_driver_assets=false" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -671,6 +789,7 @@ jobs:
# GoNavi 主仓库只保留主程序包;驱动资产发布到独立仓库。
rm -rf drivers
echo "has_driver_assets=true" >> "$GITHUB_OUTPUT"
- name: Generate SHA256SUMS
shell: bash
@@ -690,6 +809,7 @@ jobs:
fi
- name: Generate Driver SHA256SUMS
if: steps.driver_assets.outputs.has_driver_assets == 'true'
shell: bash
run: |
cd driver-release-assets
@@ -707,7 +827,7 @@ jobs:
- name: Create Driver Agents Release
uses: softprops/action-gh-release@v2
if: startsWith(github.ref, 'refs/tags/')
if: startsWith(github.ref, 'refs/tags/') && steps.driver_assets.outputs.has_driver_assets == 'true'
with:
repository: Syngnat/GoNavi-DriverAgents
tag_name: ${{ github.ref_name }}

View File

@@ -0,0 +1,218 @@
#!/usr/bin/env python3
import argparse
import json
import os
import shutil
import sys
import tempfile
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from pathlib import Path
DRIVERS = [
"mariadb",
"oceanbase",
"doris",
"starrocks",
"sphinx",
"sqlserver",
"sqlite",
"duckdb",
"dameng",
"kingbase",
"highgo",
"vastbase",
"opengauss",
"mongodb",
"tdengine",
"clickhouse",
]
BUNDLE_NAME = "GoNavi-DriverAgents.zip"
def required_assets():
assets = []
for driver in DRIVERS:
assets.extend(
[
("Windows", f"{driver}-driver-agent-windows-amd64.exe"),
("MacOS", f"{driver}-driver-agent-darwin-amd64"),
("MacOS", f"{driver}-driver-agent-darwin-arm64"),
("Linux", f"{driver}-driver-agent-linux-amd64"),
]
)
if driver == "duckdb":
assets.append(("Windows", "duckdb.dll"))
else:
assets.append(("Windows", f"{driver}-driver-agent-windows-arm64.exe"))
return assets
def github_headers():
headers = {
"Accept": "application/vnd.github+json",
"User-Agent": "GoNavi-CI",
}
token = os.environ.get("DRIVER_RELEASE_TOKEN") or os.environ.get("GITHUB_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
return headers
def fetch_json(url):
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8"))
def download_asset(asset, destination):
headers = github_headers()
headers["Accept"] = "application/octet-stream"
req = urllib.request.Request(asset["url"], headers=headers)
with urllib.request.urlopen(req, timeout=120) as resp:
with open(destination, "wb") as out:
shutil.copyfileobj(resp, out)
def load_release(repo, source):
owner_repo = repo.strip()
if not owner_repo:
raise ValueError("repo is required")
if source == "latest":
api_url = f"https://api.github.com/repos/{owner_repo}/releases/latest"
else:
api_url = f"https://api.github.com/repos/{owner_repo}/releases/tags/{urllib.parse.quote(source, safe='')}"
try:
return fetch_json(api_url)
except urllib.error.HTTPError as exc:
if exc.code == 404:
print(f"未找到上一版 driver release{source}", file=sys.stderr)
return None
raise
def asset_map(release):
result = {}
for asset in release.get("assets", []):
name = str(asset.get("name", "")).strip()
if name:
result[name] = asset
return result
def copy_missing_from_bundle(bundle_path, target_root):
copied = 0
required = {
(Path(platform) / file_name).as_posix()
for platform, file_name in required_assets()
}
with zipfile.ZipFile(bundle_path) as zf:
members = {
Path(info.filename).as_posix(): info
for info in zf.infolist()
if not info.is_dir()
}
for item in sorted(required):
source = members.get(item)
if not source:
continue
relative = Path(item)
target = target_root / relative
if target.exists():
continue
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(source) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out)
copied += 1
return copied
def copy_missing_standalone(release_assets, target_root):
copied = 0
with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-") as tmp:
tmp_root = Path(tmp)
for platform, file_name in required_assets():
target = target_root / platform / file_name
if target.exists():
continue
asset = release_assets.get(file_name)
if not asset:
continue
temp_path = tmp_root / file_name
download_asset(asset, temp_path)
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(temp_path, target)
copied += 1
return copied
def verify_complete(target_root):
missing = []
for platform, file_name in required_assets():
if not (target_root / platform / file_name).is_file():
missing.append(f"{platform}/{file_name}")
return missing
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--assets-dir", required=True)
parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
parser.add_argument(
"--source",
action="append",
required=True,
help="latest or release tag such as dev-latest/v1.0.0; may be passed more than once",
)
parser.add_argument("--require-complete", action="store_true")
args = parser.parse_args()
assets_dir = Path(args.assets_dir)
driver_root = assets_dir / "drivers"
driver_root.mkdir(parents=True, exist_ok=True)
found_source = False
total_copied = 0
for source in args.source:
release = load_release(args.repo, source)
if release is None:
continue
found_source = True
releases_assets = asset_map(release)
copied = 0
bundle_asset = releases_assets.get(BUNDLE_NAME)
if bundle_asset:
with tempfile.TemporaryDirectory(prefix="gonavi-driver-release-") as tmp:
bundle_path = Path(tmp) / BUNDLE_NAME
download_asset(bundle_asset, bundle_path)
copied += copy_missing_from_bundle(bundle_path, driver_root)
copied += copy_missing_standalone(releases_assets, driver_root)
total_copied += copied
print(f"已从 {source} 补齐 driver assets{copied} 个文件")
if not verify_complete(driver_root):
break
if not found_source:
print("未找到可补齐的上一版 driver release。", file=sys.stderr)
else:
print(f"driver assets 合计补齐:{total_copied} 个文件")
if args.require_complete:
missing = verify_complete(driver_root)
if missing:
print("driver assets 不完整:", file=sys.stderr)
for item in missing:
print(f" - {item}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,393 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$SCRIPT_DIR"
SCRIPT_DIR_WINDOWS="$(pwd -W 2>/dev/null || true)"
SCRIPT_DIR_WINDOWS="${SCRIPT_DIR_WINDOWS//\\//}"
DEFAULT_DRIVERS=(mariadb oceanbase doris starrocks sphinx sqlserver sqlite duckdb dameng kingbase highgo vastbase opengauss mongodb tdengine clickhouse)
TARGET_PLATFORMS=(darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64)
usage() {
cat <<'EOF'
用法:
./tools/detect-changed-driver-agents.sh --base <ref> [--head <ref>]
输出:
逗号分隔的 driver-agent 列表;没有 driver-agent 相关变更时输出空行。
说明:
通过 go list -deps 计算每个 driver-agent 的真实源码依赖,再与 git diff 文件求交集。
如果无法解析基准或依赖分析失败,会保守输出全部 driver。
EOF
}
join_drivers() {
local IFS=,
echo "$*"
}
all_drivers_csv() {
join_drivers "${DEFAULT_DRIVERS[@]}"
}
is_dependency_source_file() {
case "$1" in
*.go|*.c|*.cc|*.cpp|*.cxx|*.h|*.hpp|*.m|*.mm|*.s|*.S|*.syso)
return 0
;;
*)
return 1
;;
esac
}
normalize_driver() {
local value
value="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
case "$value" in
doris|diros) echo "doris" ;;
open_gauss|open-gauss) echo "opengauss" ;;
mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|opengauss|mongodb|tdengine|clickhouse)
echo "$value"
;;
*)
return 1
;;
esac
}
build_driver_name() {
case "$1" in
doris) echo "diros" ;;
*) echo "$1" ;;
esac
}
driver_build_tags() {
local driver="$1"
local platform="$2"
local goos="${platform%%/*}"
local goarch="${platform##*/}"
local build_driver tag
build_driver="$(build_driver_name "$driver")"
tag="gonavi_${build_driver}_driver"
if [[ "$driver" == "duckdb" && "$goos" == "windows" && "$goarch" == "amd64" ]]; then
tag="$tag duckdb_use_lib"
fi
echo "$tag"
}
all_driver_build_tags() {
local platform="$1"
local -a tags=()
local driver
for driver in "${DEFAULT_DRIVERS[@]}"; do
tags+=("$(driver_build_tags "$driver" "$platform")")
done
local IFS=" "
echo "${tags[*]}"
}
driver_cgo_enabled() {
# Detection only needs the Go dependency graph; keeping CGO off avoids
# cross-platform cgo toolchain requirements in the Ubuntu detection job.
echo 0
}
relative_repo_path() {
local path="$1"
path="${path//\\//}"
case "$path" in
"$SCRIPT_DIR"/*)
path="${path#$SCRIPT_DIR/}"
;;
esac
if [[ -n "$SCRIPT_DIR_WINDOWS" ]]; then
case "$path" in
"$SCRIPT_DIR_WINDOWS"/*)
path="${path#$SCRIPT_DIR_WINDOWS/}"
;;
esac
fi
path="${path#./}"
printf '%s\n' "$path"
}
add_analysis_platform() {
local platform="$1"
if [[ "$analysis_platform_seen" == *"|$platform|"* ]]; then
return 0
fi
analysis_platforms+=("$platform")
analysis_platform_seen="${analysis_platform_seen}${platform}|"
}
add_forced_driver() {
local driver
driver="$(normalize_driver "$1")" || return 0
if [[ "$forced_driver_seen" == *"|$driver|"* ]]; then
return 0
fi
forced_changed_drivers+=("$driver")
forced_driver_seen="${forced_driver_seen}${driver}|"
}
list_dependency_files() {
local tags="$1"
local cgo_enabled="$2"
local goos="$3"
local goarch="$4"
local output="$5"
CGO_ENABLED="$cgo_enabled" GOOS="$goos" GOARCH="$goarch" GOTOOLCHAIN=auto \
go list -deps \
-tags "$tags" \
-f '{{if and (not .Standard) .Module.Main}}{{range .GoFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .CgoFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .CFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .CXXFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .MFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .HFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .SFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{range .SysoFiles}}{{$.Dir}}/{{.}}{{"\n"}}{{end}}{{end}}' \
./cmd/optional-driver-agent | sort -u >"$output"
}
dependency_union_contains_changed_files() {
local platform goos goarch tags tmp file rel
local matched_any=false
for platform in "${TARGET_PLATFORMS[@]}"; do
goos="${platform%%/*}"
goarch="${platform##*/}"
tags="$(all_driver_build_tags "$platform")"
tmp="$(mktemp "${TMPDIR:-/tmp}/gonavi-driver-dep-union.XXXXXX")"
if ! list_dependency_files "$tags" 0 "$goos" "$goarch" "$tmp"; then
rm -f "$tmp"
return 2
fi
while IFS= read -r file; do
[[ -n "$file" ]] || continue
rel="$(relative_repo_path "$file")"
if [[ -n "${changed_file_set[$rel]:-}" ]]; then
dependency_matched_platforms["$rel"]="${dependency_matched_platforms[$rel]:-}|$platform|"
matched_any=true
fi
done <"$tmp"
rm -f "$tmp"
done
if [[ "$matched_any" == "true" ]]; then
return 0
fi
return 1
}
base_ref=""
head_ref="HEAD"
while [[ $# -gt 0 ]]; do
case "$1" in
--base)
base_ref="${2:-}"
shift 2
;;
--head)
head_ref="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "未知参数:$1" >&2
usage >&2
exit 1
;;
esac
done
if [[ -z "$base_ref" ]]; then
echo "缺少 --base 参数。" >&2
usage >&2
exit 1
fi
if [[ "$base_ref" == "all" ]]; then
all_drivers_csv
exit 0
fi
if ! git rev-parse --verify "${head_ref}^{commit}" >/dev/null 2>&1; then
echo "无法解析 head ref$head_ref" >&2
exit 1
fi
head_commit="$(git rev-parse "${head_ref}^{commit}")"
if ! git rev-parse --verify "${base_ref}^{commit}" >/dev/null 2>&1; then
echo "无法解析 base ref$base_ref;保守构建全部 driver-agent。" >&2
all_drivers_csv
exit 0
fi
base_commit="$(git rev-parse "${base_ref}^{commit}")"
declare -A changed_file_set=()
while IFS= read -r -d '' file; do
file="$(relative_repo_path "$file")"
[[ -n "$file" ]] || continue
changed_file_set["$file"]=1
done < <(git diff --name-only -z "$base_commit" "$head_commit")
if [[ ${#changed_file_set[@]} -eq 0 ]]; then
echo ""
exit 0
fi
for file in "${!changed_file_set[@]}"; do
case "$file" in
go.mod|go.sum|build-driver-agents.sh|tools/compress-driver-artifact.sh|tools/generate-driver-agent-revisions.sh|tools/detect-changed-driver-agents.sh)
all_drivers_csv
exit 0
;;
esac
done
declare -a forced_changed_drivers=()
forced_driver_seen="|"
for file in "${!changed_file_set[@]}"; do
case "$file" in
internal/db/duckdb_*.go)
add_forced_driver duckdb
;;
esac
done
has_source_candidate=false
for file in "${!changed_file_set[@]}"; do
if is_dependency_source_file "$file"; then
has_source_candidate=true
break
fi
done
if [[ "$has_source_candidate" != "true" ]]; then
echo ""
exit 0
fi
while IFS= read -r -d '' file; do
file="$(relative_repo_path "$file")"
if is_dependency_source_file "$file"; then
echo "检测到源码依赖候选文件被删除;保守构建全部 driver-agent$file" >&2
all_drivers_csv
exit 0
fi
done < <(git diff --name-only --diff-filter=D -z "$base_commit" "$head_commit")
declare -A dependency_matched_platforms=()
set +e
dependency_union_contains_changed_files
dependency_union_status=$?
set -e
case "$dependency_union_status" in
0)
;;
1)
if [[ ${#forced_changed_drivers[@]} -eq 0 ]]; then
echo ""
exit 0
fi
;;
*)
echo "分析 driver-agent 依赖全集失败;保守构建全部 driver-agent。" >&2
all_drivers_csv
exit 0
;;
esac
declare -a analysis_platforms=()
analysis_platform_seen="|"
for file in "${!dependency_matched_platforms[@]}"; do
matched_platforms="${dependency_matched_platforms[$file]}"
if [[ "$matched_platforms" == *"|linux/amd64|"* ]]; then
add_analysis_platform "linux/amd64"
continue
fi
for platform in "${TARGET_PLATFORMS[@]}"; do
if [[ "$matched_platforms" == *"|$platform|"* ]]; then
add_analysis_platform "$platform"
fi
done
done
if [[ ${#analysis_platforms[@]} -eq 0 ]]; then
analysis_platforms=("${TARGET_PLATFORMS[@]}")
fi
declare -a changed_drivers=()
driver_seen="|"
add_driver() {
local driver
driver="$(normalize_driver "$1")" || return 0
if [[ "$driver_seen" == *"|$driver|"* ]]; then
return 0
fi
changed_drivers+=("$driver")
driver_seen="${driver_seen}${driver}|"
}
for driver in "${forced_changed_drivers[@]}"; do
add_driver "$driver"
done
driver_depends_on_changed_files() {
local driver="$1"
local platform="$2"
local goos="${platform%%/*}"
local goarch="${platform##*/}"
local tags cgo_enabled file rel tmp
tags="$(driver_build_tags "$driver" "$platform")"
cgo_enabled="$(driver_cgo_enabled "$driver")"
tmp="$(mktemp "${TMPDIR:-/tmp}/gonavi-driver-deps.XXXXXX")"
if ! list_dependency_files "$tags" "$cgo_enabled" "$goos" "$goarch" "$tmp"; then
rm -f "$tmp"
return 2
fi
while IFS= read -r file; do
[[ -n "$file" ]] || continue
rel="$(relative_repo_path "$file")"
if [[ -n "${changed_file_set[$rel]:-}" ]]; then
rm -f "$tmp"
return 0
fi
done <"$tmp"
rm -f "$tmp"
return 1
}
for driver in "${DEFAULT_DRIVERS[@]}"; do
driver_changed=false
for platform in "${analysis_platforms[@]}"; do
set +e
driver_depends_on_changed_files "$driver" "$platform"
status=$?
set -e
case "$status" in
0)
add_driver "$driver"
driver_changed=true
break
;;
1)
;;
*)
echo "分析 $driver driver-agent 依赖失败;保守构建全部 driver-agent。" >&2
all_drivers_csv
exit 0
;;
esac
done
done
join_drivers "${changed_drivers[@]}"

View File

@@ -65,6 +65,16 @@ hash_file() {
exit 1
}
files_equal() {
local left="$1"
local right="$2"
if command -v cmp >/dev/null 2>&1; then
cmp -s "$left" "$right"
return
fi
[[ "$(hash_file "$left")" == "$(hash_file "$right")" ]]
}
should_include_internal_db_file() {
local driver="$1"
local identity="$2"
@@ -208,6 +218,33 @@ existing_revision_for() {
return 1
}
detect_revision_jobs() {
local configured="${GONAVI_DRIVER_REVISION_JOBS:-}"
local detected
if [[ "$configured" =~ ^[0-9]+$ && "$configured" -gt 0 ]]; then
echo "$configured"
return
fi
if command -v nproc >/dev/null 2>&1; then
detected="$(nproc)"
elif command -v sysctl >/dev/null 2>&1; then
detected="$(sysctl -n hw.ncpu 2>/dev/null || true)"
else
detected=""
fi
if ! [[ "$detected" =~ ^[0-9]+$ && "$detected" -gt 0 ]]; then
detected=4
fi
if [[ "$detected" -gt 4 ]]; then
detected=4
fi
echo "$detected"
}
declare -a output_drivers=()
if [[ -n "$driver_csv" ]]; then
output_drivers=("${DEFAULT_DRIVERS[@]}")
@@ -267,6 +304,65 @@ fingerprint_driver() {
printf 'src-%s' "$revision"
}
revision_jobs="$(detect_revision_jobs)"
revision_tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-agent-revisions.XXXXXX")"
cleanup_revision_tmp_dir() {
rm -rf "$revision_tmp_dir"
}
trap cleanup_revision_tmp_dir EXIT
declare -a revision_pids=()
declare -a revision_pid_drivers=()
revision_failed=0
wait_for_oldest_revision_job() {
local pid driver
pid="${revision_pids[0]}"
driver="${revision_pid_drivers[0]}"
if ! wait "$pid"; then
echo "❌ 生成 driver-agent revision 失败:$driver ($goos/$goarch)" >&2
revision_failed=1
fi
if [[ ${#revision_pids[@]} -le 1 ]]; then
revision_pids=()
revision_pid_drivers=()
else
revision_pids=("${revision_pids[@]:1}")
revision_pid_drivers=("${revision_pid_drivers[@]:1}")
fi
}
start_revision_job() {
local driver="$1"
{
fingerprint_driver "$driver" >"$revision_tmp_dir/$driver.revision"
} &
revision_pids+=("$!")
revision_pid_drivers+=("$driver")
}
for driver in "${output_drivers[@]}"; do
if [[ -n "$driver_csv" && "$selected_driver_set" != *"|$driver|"* ]] && revision="$(existing_revision_for "$driver")"; then
printf '%s\n' "$revision" >"$revision_tmp_dir/$driver.revision"
continue
fi
while [[ ${#revision_pids[@]} -ge "$revision_jobs" ]]; do
wait_for_oldest_revision_job
done
start_revision_job "$driver"
done
while [[ ${#revision_pids[@]} -gt 0 ]]; do
wait_for_oldest_revision_job
done
if [[ "$revision_failed" -ne 0 ]]; then
exit 1
fi
tmp_output="$(mktemp "${TMPDIR:-/tmp}/gonavi-agent-revisions-go.XXXXXX")"
{
cat <<'EOF'
@@ -278,11 +374,7 @@ func init() {
optionalDriverAgentRevisions = map[string]string{
EOF
for driver in "${output_drivers[@]}"; do
if [[ -n "$driver_csv" && "$selected_driver_set" != *"|$driver|"* ]] && revision="$(existing_revision_for "$driver")"; then
:
else
revision="$(fingerprint_driver "$driver")"
fi
revision="$(<"$revision_tmp_dir/$driver.revision")"
printf '\t\t"%s": "%s",\n' "$driver" "$revision"
done
cat <<'EOF'
@@ -293,7 +385,7 @@ EOF
gofmt -w "$tmp_output"
if [[ -f "$OUTPUT_FILE" ]] && cmp -s "$tmp_output" "$OUTPUT_FILE"; then
if [[ -f "$OUTPUT_FILE" ]] && files_equal "$tmp_output" "$OUTPUT_FILE"; then
rm -f "$tmp_output"
else
mv "$tmp_output" "$OUTPUT_FILE"