From 5ce4dddd7a5d58145f75127a053b4510fda940ca Mon Sep 17 00:00:00 2001 From: Syngnat Date: Wed, 17 Jun 2026 17:44:40 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20perf(ci):=20=E5=87=8F?= =?UTF-8?q?=E5=B0=91=20driver-agent=20=E6=A3=80=E6=B5=8B=E4=B8=8E=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=20diff=20=E7=9A=84=E9=87=8D=E5=A4=8D=E8=AE=A1?= =?UTF-8?q?=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - driver_agents job 一次性产出各平台 driver 集合，build 矩阵直接消费结果\n- detect 阶段补充 setup-go 与缓存，降低 revision 检测的冷启动成本\n- diff-driver-agent-revisions 支持候选 drivers 过滤，并行生成 base/head revisions\n- 补充脚本测试，覆盖候选 driver 过滤与无关 driver 忽略场景 --- .github/workflows/dev-build.yml | 115 ++++++++++++++++------ tools/diff-driver-agent-revisions.sh | 56 ++++++++++- tools/diff-driver-agent-revisions.test.sh | 12 +++ 3 files changed, 147 insertions(+), 36 deletions(-) diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 0410dec..f1f713d 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -68,6 +68,11 @@ jobs: runs-on: ubuntu-latest outputs: drivers: ${{ steps.detect.outputs.drivers }} + drivers_darwin_amd64: ${{ steps.detect.outputs.drivers_darwin_amd64 }} + drivers_darwin_arm64: ${{ steps.detect.outputs.drivers_darwin_arm64 }} + drivers_windows_amd64: ${{ steps.detect.outputs.drivers_windows_amd64 }} + drivers_windows_arm64: ${{ steps.detect.outputs.drivers_windows_arm64 }} + drivers_linux_amd64: ${{ steps.detect.outputs.drivers_linux_amd64 }} has_changes: ${{ steps.detect.outputs.has_changes }} release_source: ${{ steps.detect.outputs.release_source }} compare_base: ${{ steps.detect.outputs.compare_base }} @@ -80,6 +85,12 @@ jobs: with: fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + cache: true + - name: Resolve published driver release source id: published_source env: @@ -149,6 +160,20 @@ jobs: printf '%s\n' "$merged" } + platform_output_key() { + case "$1" in + darwin/amd64) echo "darwin_amd64" ;; + darwin/arm64) echo "darwin_arm64" ;; + windows/amd64) echo "windows_amd64" ;; + windows/arm64) echo "windows_arm64" ;; + linux/amd64) echo "linux_amd64" ;; + *) + echo "unknown platform: $1" >&2 + return 1 + ;; + esac + } + BASE_REF="${{ steps.published_source.outputs.source_commit }}" HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}" MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}" @@ -173,27 +198,53 @@ jobs: BASE_REF="all" fi echo "🧭 Final driver detection base: $BASE_REF" + FORCE_GLOBAL_DRIVER_BUILDS="$(bash ./tools/should-force-global-driver-builds.sh --base "$BASE_REF" --head "$GITHUB_SHA")" DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")" - if [[ "$BASE_REF" != "all" ]]; then + PLATFORM_OUTPUTS=() + PLATFORM_DIFF_FAILED=false + if [[ "$BASE_REF" != "all" && "$FORCE_GLOBAL_DRIVER_BUILDS" != "true" ]]; then REVISION_DRIVERS="" for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do - PLATFORM_DRIVERS="$(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "$PLATFORM")" || { + PLATFORM_KEY="$(platform_output_key "$PLATFORM")" + DIFF_ARGS=(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "$PLATFORM") + if [[ -n "$DRIVERS" ]]; then + DIFF_ARGS+=(--drivers "$DRIVERS") + fi + PLATFORM_DRIVERS="$("${DIFF_ARGS[@]}")" || { echo "⚠️ 平台 revision 差异对比失败（${PLATFORM}），保守回退为全量驱动重建" DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base all --head "$GITHUB_SHA")" REVISION_DRIVERS="" + PLATFORM_DIFF_FAILED=true + PLATFORM_OUTPUTS=() break } + PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=${PLATFORM_DRIVERS}") REVISION_DRIVERS="$(merge_csv "$REVISION_DRIVERS" "$PLATFORM_DRIVERS")" done - if [[ -n "$REVISION_DRIVERS" ]]; then + if [[ "$PLATFORM_DIFF_FAILED" == "false" && -n "$REVISION_DRIVERS" ]]; then echo "🧭 Revision diff union drivers: $REVISION_DRIVERS" DRIVERS="$(merge_csv "$DRIVERS" "$REVISION_DRIVERS")" fi fi - FORCE_GLOBAL_DRIVER_BUILDS="$(bash ./tools/should-force-global-driver-builds.sh --base "$BASE_REF" --head "$GITHUB_SHA")" + if [[ "$BASE_REF" == "all" || "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" || "$PLATFORM_DIFF_FAILED" == "true" ]]; then + PLATFORM_OUTPUTS=() + for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do + PLATFORM_KEY="$(platform_output_key "$PLATFORM")" + PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=${DRIVERS}") + done + elif [[ ${#PLATFORM_OUTPUTS[@]} -eq 0 ]]; then + for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do + PLATFORM_KEY="$(platform_output_key "$PLATFORM")" + PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=") + done + fi + echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT" echo "compare_base=${BASE_REF}" >> "$GITHUB_OUTPUT" echo "force_global_driver_builds=${FORCE_GLOBAL_DRIVER_BUILDS}" >> "$GITHUB_OUTPUT" + for OUTPUT_LINE in "${PLATFORM_OUTPUTS[@]}"; do + echo "$OUTPUT_LINE" >> "$GITHUB_OUTPUT" + done if [ -n "$DRIVERS" ]; then echo "has_changes=true" >> "$GITHUB_OUTPUT" echo "🧭 Changed driver agents: $DRIVERS" @@ -481,47 +532,47 @@ jobs: wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}" fi - - name: Resolve Platform Driver Revision Diff - id: revision_diff + - name: Select Platform Driver Set + id: platform_drivers if: ${{ matrix.build_optional_agents && needs.driver_agents.outputs.has_changes == 'true' }} shell: bash run: | set -euo pipefail - BASE_REF="${{ needs.driver_agents.outputs.compare_base }}" - FALLBACK_DRIVERS="${{ needs.driver_agents.outputs.drivers }}" - FORCE_GLOBAL_DRIVER_BUILDS="${{ needs.driver_agents.outputs.force_global_driver_builds }}" - - if [[ -z "$BASE_REF" || "$BASE_REF" == "all" || "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" ]]; then - if [[ "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" && -n "$BASE_REF" && "$BASE_REF" != "all" ]]; then - echo "⚠️ 当前提交涉及 driver 构建/发布链路，保留全局驱动重建结果：${FALLBACK_DRIVERS}" - else - echo "⚠️ 当前 driver 检测基线不可做平台 diff，回退使用全局检测结果：${FALLBACK_DRIVERS}" - fi - echo "drivers=${FALLBACK_DRIVERS}" >> "$GITHUB_OUTPUT" - else - echo "🧭 对比当前平台 revision：base=${BASE_REF} head=${GITHUB_SHA} platform=${{ matrix.platform }}" - if DRIVERS="$(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "${{ matrix.platform }}")"; then - echo "🧭 当前平台实际需要重建的 driver agents: ${DRIVERS:-}" - echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT" - else - echo "⚠️ revision 差异对比失败，保守回退为全量重建" - ALL_DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base all --head "$GITHUB_SHA")" - echo "drivers=${ALL_DRIVERS}" >> "$GITHUB_OUTPUT" - fi - fi - - DRIVERS="$(sed -n 's/^drivers=//p' "$GITHUB_OUTPUT" | tail -n 1)" + case "${{ matrix.platform }}" in + darwin/amd64) + DRIVERS='${{ needs.driver_agents.outputs.drivers_darwin_amd64 }}' + ;; + darwin/arm64) + DRIVERS='${{ needs.driver_agents.outputs.drivers_darwin_arm64 }}' + ;; + windows/amd64) + DRIVERS='${{ needs.driver_agents.outputs.drivers_windows_amd64 }}' + ;; + windows/arm64) + DRIVERS='${{ needs.driver_agents.outputs.drivers_windows_arm64 }}' + ;; + linux/amd64) + DRIVERS='${{ needs.driver_agents.outputs.drivers_linux_amd64 }}' + ;; + *) + echo "unsupported platform: ${{ matrix.platform }}" >&2 + exit 1 + ;; + esac + echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT" if [[ -n "$DRIVERS" ]]; then echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "🧭 当前平台最终需要重建的 driver agents: ${DRIVERS}" else echo "has_changes=false" >> "$GITHUB_OUTPUT" + echo "🧭 当前平台无需重建 driver agents" fi - name: Build Optional Driver Agents - if: ${{ matrix.build_optional_agents && steps.revision_diff.outputs.has_changes == 'true' }} + if: ${{ matrix.build_optional_agents && steps.platform_drivers.outputs.has_changes == 'true' }} shell: bash env: - CHANGED_DRIVER_AGENTS: ${{ steps.revision_diff.outputs.drivers }} + CHANGED_DRIVER_AGENTS: ${{ steps.platform_drivers.outputs.drivers }} run: | set -euo pipefail TARGET_PLATFORM="${{ matrix.platform }}" diff --git a/tools/diff-driver-agent-revisions.sh b/tools/diff-driver-agent-revisions.sh index d94523d..4fdb0d4 100644 --- a/tools/diff-driver-agent-revisions.sh +++ b/tools/diff-driver-agent-revisions.sh @@ -33,6 +33,20 @@ public_driver_name() { esac } +normalize_driver_name() { + local value + value="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + case "$value" in + doris|diros) echo "diros" ;; + mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|opengauss|gaussdb|iris|mongodb|tdengine|iotdb|clickhouse|elasticsearch) + echo "$value" + ;; + *) + return 1 + ;; + esac +} + extract_revision() { local file="$1" local driver="$2" @@ -49,6 +63,7 @@ extract_revision() { base_ref="" head_ref="" target_platform="" +driver_csv="" while [[ $# -gt 0 ]]; do case "$1" in @@ -64,6 +79,10 @@ while [[ $# -gt 0 ]]; do target_platform="${2:-}" shift 2 ;; + --drivers) + driver_csv="${2:-}" + shift 2 + ;; -h|--help) usage exit 0 @@ -102,6 +121,27 @@ if [[ "$base_commit" == "$head_commit" ]]; then exit 0 fi +declare -a drivers_to_compare=() +if [[ -n "$driver_csv" ]]; then + seen="|" + IFS=',' read -r -a raw_drivers <<< "$driver_csv" + for raw_driver in "${raw_drivers[@]}"; do + driver="$(normalize_driver_name "$raw_driver")" || continue + if [[ "$seen" == *"|$driver|"* ]]; then + continue + fi + drivers_to_compare+=("$driver") + seen="${seen}${driver}|" + done +else + drivers_to_compare=("${DEFAULT_DRIVERS[@]}") +fi + +if [[ ${#drivers_to_compare[@]} -eq 0 ]]; then + echo "" + exit 0 +fi + base_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-base.XXXXXX")" head_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-head.XXXXXX")" @@ -119,19 +159,27 @@ generate_revisions() { local worktree="$1" ( cd "$worktree" - GONAVI_DRIVER_REVISION_JOBS="${GONAVI_DRIVER_REVISION_JOBS:-1}" \ + if [[ -n "$driver_csv" ]]; then + bash ./tools/generate-driver-agent-revisions.sh --platform "$target_platform" --drivers "$driver_csv" >/dev/null + else bash ./tools/generate-driver-agent-revisions.sh --platform "$target_platform" >/dev/null + fi ) } -generate_revisions "$base_worktree" -generate_revisions "$head_worktree" +generate_revisions "$base_worktree" & +base_pid=$! +generate_revisions "$head_worktree" & +head_pid=$! + +wait "$base_pid" +wait "$head_pid" base_file="$base_worktree/internal/db/driver_agent_revisions_gen.go" head_file="$head_worktree/internal/db/driver_agent_revisions_gen.go" declare -a changed_drivers=() -for driver in "${DEFAULT_DRIVERS[@]}"; do +for driver in "${drivers_to_compare[@]}"; do base_revision="$(extract_revision "$base_file" "$driver")" head_revision="$(extract_revision "$head_file" "$driver")" if [[ "$base_revision" != "$head_revision" ]]; then diff --git a/tools/diff-driver-agent-revisions.test.sh b/tools/diff-driver-agent-revisions.test.sh index 8c90285..e7cf7a4 100644 --- a/tools/diff-driver-agent-revisions.test.sh +++ b/tools/diff-driver-agent-revisions.test.sh @@ -35,6 +35,18 @@ rsync -a --exclude .git ./ "$tmpdir/" >/dev/null echo "expected duckdb-specific source change to include duckdb revision rebuild, got: ${actual:-}" >&2 exit 1 fi + + filtered_actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64 --drivers duckdb)" + if [[ "$filtered_actual" != "duckdb" ]]; then + echo "expected --drivers duckdb to keep duckdb revision rebuild only, got: ${filtered_actual:-}" >&2 + exit 1 + fi + + ignored_actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64 --drivers mariadb)" + if [[ -n "$ignored_actual" ]]; then + echo "expected --drivers mariadb to ignore unrelated duckdb revision diff, got: ${ignored_actual}" >&2 + exit 1 + fi ) tmpdir_frontend="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-diff-driver-revisions-frontend.XXXXXX")"