mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-06-22 06:23:43 +08:00
⚡️ perf(ci): 减少 driver-agent 检测与平台 diff 的重复计算
- driver_agents job 一次性产出各平台 driver 集合，build 矩阵直接消费结果\n- detect 阶段补充 setup-go 与缓存，降低 revision 检测的冷启动成本\n- diff-driver-agent-revisions 支持候选 drivers 过滤，并行生成 base/head revisions\n- 补充脚本测试，覆盖候选 driver 过滤与无关 driver 忽略场景
This commit is contained in:
Syngnat
115
.github/workflows/dev-build.yml
vendored
115
.github/workflows/dev-build.yml
vendored
@@ -68,6 +68,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
drivers: ${{ steps.detect.outputs.drivers }}
|
||||
drivers_darwin_amd64: ${{ steps.detect.outputs.drivers_darwin_amd64 }}
|
||||
drivers_darwin_arm64: ${{ steps.detect.outputs.drivers_darwin_arm64 }}
|
||||
drivers_windows_amd64: ${{ steps.detect.outputs.drivers_windows_amd64 }}
|
||||
drivers_windows_arm64: ${{ steps.detect.outputs.drivers_windows_arm64 }}
|
||||
drivers_linux_amd64: ${{ steps.detect.outputs.drivers_linux_amd64 }}
|
||||
has_changes: ${{ steps.detect.outputs.has_changes }}
|
||||
release_source: ${{ steps.detect.outputs.release_source }}
|
||||
compare_base: ${{ steps.detect.outputs.compare_base }}
|
||||
@@ -80,6 +85,12 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: true
|
||||
|
||||
- name: Resolve published driver release source
|
||||
id: published_source
|
||||
env:
|
||||
@@ -149,6 +160,20 @@ jobs:
|
||||
printf '%s\n' "$merged"
|
||||
}
|
||||
|
||||
platform_output_key() {
|
||||
case "$1" in
|
||||
darwin/amd64) echo "darwin_amd64" ;;
|
||||
darwin/arm64) echo "darwin_arm64" ;;
|
||||
windows/amd64) echo "windows_amd64" ;;
|
||||
windows/arm64) echo "windows_arm64" ;;
|
||||
linux/amd64) echo "linux_amd64" ;;
|
||||
*)
|
||||
echo "unknown platform: $1" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
BASE_REF="${{ steps.published_source.outputs.source_commit }}"
|
||||
HAS_MANIFEST="${{ steps.published_source.outputs.has_manifest }}"
|
||||
MANIFEST_VALID="${{ steps.published_source.outputs.manifest_valid }}"
|
||||
@@ -173,27 +198,53 @@ jobs:
|
||||
BASE_REF="all"
|
||||
fi
|
||||
echo "🧭 Final driver detection base: $BASE_REF"
|
||||
FORCE_GLOBAL_DRIVER_BUILDS="$(bash ./tools/should-force-global-driver-builds.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
|
||||
DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
|
||||
if [[ "$BASE_REF" != "all" ]]; then
|
||||
PLATFORM_OUTPUTS=()
|
||||
PLATFORM_DIFF_FAILED=false
|
||||
if [[ "$BASE_REF" != "all" && "$FORCE_GLOBAL_DRIVER_BUILDS" != "true" ]]; then
|
||||
REVISION_DRIVERS=""
|
||||
for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do
|
||||
PLATFORM_DRIVERS="$(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "$PLATFORM")" || {
|
||||
PLATFORM_KEY="$(platform_output_key "$PLATFORM")"
|
||||
DIFF_ARGS=(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "$PLATFORM")
|
||||
if [[ -n "$DRIVERS" ]]; then
|
||||
DIFF_ARGS+=(--drivers "$DRIVERS")
|
||||
fi
|
||||
PLATFORM_DRIVERS="$("${DIFF_ARGS[@]}")" || {
|
||||
echo "⚠️ 平台 revision 差异对比失败（${PLATFORM}），保守回退为全量驱动重建"
|
||||
DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base all --head "$GITHUB_SHA")"
|
||||
REVISION_DRIVERS=""
|
||||
PLATFORM_DIFF_FAILED=true
|
||||
PLATFORM_OUTPUTS=()
|
||||
break
|
||||
}
|
||||
PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=${PLATFORM_DRIVERS}")
|
||||
REVISION_DRIVERS="$(merge_csv "$REVISION_DRIVERS" "$PLATFORM_DRIVERS")"
|
||||
done
|
||||
if [[ -n "$REVISION_DRIVERS" ]]; then
|
||||
if [[ "$PLATFORM_DIFF_FAILED" == "false" && -n "$REVISION_DRIVERS" ]]; then
|
||||
echo "🧭 Revision diff union drivers: $REVISION_DRIVERS"
|
||||
DRIVERS="$(merge_csv "$DRIVERS" "$REVISION_DRIVERS")"
|
||||
fi
|
||||
fi
|
||||
FORCE_GLOBAL_DRIVER_BUILDS="$(bash ./tools/should-force-global-driver-builds.sh --base "$BASE_REF" --head "$GITHUB_SHA")"
|
||||
if [[ "$BASE_REF" == "all" || "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" || "$PLATFORM_DIFF_FAILED" == "true" ]]; then
|
||||
PLATFORM_OUTPUTS=()
|
||||
for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do
|
||||
PLATFORM_KEY="$(platform_output_key "$PLATFORM")"
|
||||
PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=${DRIVERS}")
|
||||
done
|
||||
elif [[ ${#PLATFORM_OUTPUTS[@]} -eq 0 ]]; then
|
||||
for PLATFORM in darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 linux/amd64; do
|
||||
PLATFORM_KEY="$(platform_output_key "$PLATFORM")"
|
||||
PLATFORM_OUTPUTS+=("drivers_${PLATFORM_KEY}=")
|
||||
done
|
||||
fi
|
||||
|
||||
echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT"
|
||||
echo "compare_base=${BASE_REF}" >> "$GITHUB_OUTPUT"
|
||||
echo "force_global_driver_builds=${FORCE_GLOBAL_DRIVER_BUILDS}" >> "$GITHUB_OUTPUT"
|
||||
for OUTPUT_LINE in "${PLATFORM_OUTPUTS[@]}"; do
|
||||
echo "$OUTPUT_LINE" >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
if [ -n "$DRIVERS" ]; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 Changed driver agents: $DRIVERS"
|
||||
@@ -481,47 +532,47 @@ jobs:
|
||||
wails build -s -skipbindings -platform ${{ matrix.platform }} -clean -o ${{ matrix.build_name }} -ldflags "-s -w -X GoNavi-Wails/internal/app.AppVersion=${DEV_VERSION}"
|
||||
fi
|
||||
|
||||
- name: Resolve Platform Driver Revision Diff
|
||||
id: revision_diff
|
||||
- name: Select Platform Driver Set
|
||||
id: platform_drivers
|
||||
if: ${{ matrix.build_optional_agents && needs.driver_agents.outputs.has_changes == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BASE_REF="${{ needs.driver_agents.outputs.compare_base }}"
|
||||
FALLBACK_DRIVERS="${{ needs.driver_agents.outputs.drivers }}"
|
||||
FORCE_GLOBAL_DRIVER_BUILDS="${{ needs.driver_agents.outputs.force_global_driver_builds }}"
|
||||
|
||||
if [[ -z "$BASE_REF" || "$BASE_REF" == "all" || "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" ]]; then
|
||||
if [[ "$FORCE_GLOBAL_DRIVER_BUILDS" == "true" && -n "$BASE_REF" && "$BASE_REF" != "all" ]]; then
|
||||
echo "⚠️ 当前提交涉及 driver 构建/发布链路，保留全局驱动重建结果：${FALLBACK_DRIVERS}"
|
||||
else
|
||||
echo "⚠️ 当前 driver 检测基线不可做平台 diff，回退使用全局检测结果：${FALLBACK_DRIVERS}"
|
||||
fi
|
||||
echo "drivers=${FALLBACK_DRIVERS}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "🧭 对比当前平台 revision：base=${BASE_REF} head=${GITHUB_SHA} platform=${{ matrix.platform }}"
|
||||
if DRIVERS="$(bash ./tools/diff-driver-agent-revisions.sh --base "$BASE_REF" --head "$GITHUB_SHA" --platform "${{ matrix.platform }}")"; then
|
||||
echo "🧭 当前平台实际需要重建的 driver agents: ${DRIVERS:-<empty>}"
|
||||
echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "⚠️ revision 差异对比失败，保守回退为全量重建"
|
||||
ALL_DRIVERS="$(bash ./tools/detect-changed-driver-agents.sh --base all --head "$GITHUB_SHA")"
|
||||
echo "drivers=${ALL_DRIVERS}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fi
|
||||
|
||||
DRIVERS="$(sed -n 's/^drivers=//p' "$GITHUB_OUTPUT" | tail -n 1)"
|
||||
case "${{ matrix.platform }}" in
|
||||
darwin/amd64)
|
||||
DRIVERS='${{ needs.driver_agents.outputs.drivers_darwin_amd64 }}'
|
||||
;;
|
||||
darwin/arm64)
|
||||
DRIVERS='${{ needs.driver_agents.outputs.drivers_darwin_arm64 }}'
|
||||
;;
|
||||
windows/amd64)
|
||||
DRIVERS='${{ needs.driver_agents.outputs.drivers_windows_amd64 }}'
|
||||
;;
|
||||
windows/arm64)
|
||||
DRIVERS='${{ needs.driver_agents.outputs.drivers_windows_arm64 }}'
|
||||
;;
|
||||
linux/amd64)
|
||||
DRIVERS='${{ needs.driver_agents.outputs.drivers_linux_amd64 }}'
|
||||
;;
|
||||
*)
|
||||
echo "unsupported platform: ${{ matrix.platform }}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
echo "drivers=${DRIVERS}" >> "$GITHUB_OUTPUT"
|
||||
if [[ -n "$DRIVERS" ]]; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 当前平台最终需要重建的 driver agents: ${DRIVERS}"
|
||||
else
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "🧭 当前平台无需重建 driver agents"
|
||||
fi
|
||||
|
||||
- name: Build Optional Driver Agents
|
||||
if: ${{ matrix.build_optional_agents && steps.revision_diff.outputs.has_changes == 'true' }}
|
||||
if: ${{ matrix.build_optional_agents && steps.platform_drivers.outputs.has_changes == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
CHANGED_DRIVER_AGENTS: ${{ steps.revision_diff.outputs.drivers }}
|
||||
CHANGED_DRIVER_AGENTS: ${{ steps.platform_drivers.outputs.drivers }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TARGET_PLATFORM="${{ matrix.platform }}"
|
||||
|
||||
@@ -33,6 +33,20 @@ public_driver_name() {
|
||||
esac
|
||||
}
|
||||
|
||||
normalize_driver_name() {
|
||||
local value
|
||||
value="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
|
||||
case "$value" in
|
||||
doris|diros) echo "diros" ;;
|
||||
mariadb|oceanbase|starrocks|sphinx|sqlserver|sqlite|duckdb|dameng|kingbase|highgo|vastbase|opengauss|gaussdb|iris|mongodb|tdengine|iotdb|clickhouse|elasticsearch)
|
||||
echo "$value"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
extract_revision() {
|
||||
local file="$1"
|
||||
local driver="$2"
|
||||
@@ -49,6 +63,7 @@ extract_revision() {
|
||||
base_ref=""
|
||||
head_ref=""
|
||||
target_platform=""
|
||||
driver_csv=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -64,6 +79,10 @@ while [[ $# -gt 0 ]]; do
|
||||
target_platform="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--drivers)
|
||||
driver_csv="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -102,6 +121,27 @@ if [[ "$base_commit" == "$head_commit" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -a drivers_to_compare=()
|
||||
if [[ -n "$driver_csv" ]]; then
|
||||
seen="|"
|
||||
IFS=',' read -r -a raw_drivers <<< "$driver_csv"
|
||||
for raw_driver in "${raw_drivers[@]}"; do
|
||||
driver="$(normalize_driver_name "$raw_driver")" || continue
|
||||
if [[ "$seen" == *"|$driver|"* ]]; then
|
||||
continue
|
||||
fi
|
||||
drivers_to_compare+=("$driver")
|
||||
seen="${seen}${driver}|"
|
||||
done
|
||||
else
|
||||
drivers_to_compare=("${DEFAULT_DRIVERS[@]}")
|
||||
fi
|
||||
|
||||
if [[ ${#drivers_to_compare[@]} -eq 0 ]]; then
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
base_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-base.XXXXXX")"
|
||||
head_worktree="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-driver-rev-head.XXXXXX")"
|
||||
|
||||
@@ -119,19 +159,27 @@ generate_revisions() {
|
||||
local worktree="$1"
|
||||
(
|
||||
cd "$worktree"
|
||||
GONAVI_DRIVER_REVISION_JOBS="${GONAVI_DRIVER_REVISION_JOBS:-1}" \
|
||||
if [[ -n "$driver_csv" ]]; then
|
||||
bash ./tools/generate-driver-agent-revisions.sh --platform "$target_platform" --drivers "$driver_csv" >/dev/null
|
||||
else
|
||||
bash ./tools/generate-driver-agent-revisions.sh --platform "$target_platform" >/dev/null
|
||||
fi
|
||||
)
|
||||
}
|
||||
|
||||
generate_revisions "$base_worktree"
|
||||
generate_revisions "$head_worktree"
|
||||
generate_revisions "$base_worktree" &
|
||||
base_pid=$!
|
||||
generate_revisions "$head_worktree" &
|
||||
head_pid=$!
|
||||
|
||||
wait "$base_pid"
|
||||
wait "$head_pid"
|
||||
|
||||
base_file="$base_worktree/internal/db/driver_agent_revisions_gen.go"
|
||||
head_file="$head_worktree/internal/db/driver_agent_revisions_gen.go"
|
||||
|
||||
declare -a changed_drivers=()
|
||||
for driver in "${DEFAULT_DRIVERS[@]}"; do
|
||||
for driver in "${drivers_to_compare[@]}"; do
|
||||
base_revision="$(extract_revision "$base_file" "$driver")"
|
||||
head_revision="$(extract_revision "$head_file" "$driver")"
|
||||
if [[ "$base_revision" != "$head_revision" ]]; then
|
||||
|
||||
@@ -35,6 +35,18 @@ rsync -a --exclude .git ./ "$tmpdir/" >/dev/null
|
||||
echo "expected duckdb-specific source change to include duckdb revision rebuild, got: ${actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
filtered_actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64 --drivers duckdb)"
|
||||
if [[ "$filtered_actual" != "duckdb" ]]; then
|
||||
echo "expected --drivers duckdb to keep duckdb revision rebuild only, got: ${filtered_actual:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ignored_actual="$(bash ./tools/diff-driver-agent-revisions.sh --base "$base" --head HEAD --platform darwin/arm64 --drivers mariadb)"
|
||||
if [[ -n "$ignored_actual" ]]; then
|
||||
echo "expected --drivers mariadb to ignore unrelated duckdb revision diff, got: ${ignored_actual}" >&2
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
tmpdir_frontend="$(mktemp -d "${TMPDIR:-/tmp}/gonavi-diff-driver-revisions-frontend.XXXXXX")"
|
||||
|
||||
Reference in New Issue
Block a user