diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 140dd266..131f6a4f 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -31,6 +31,7 @@ jobs: bash tools/generate-driver-agent-revisions.test.sh python3 tools/generate-driver-release-manifest.test.py python3 tools/generate-update-latest-manifest.test.py + python3 tools/r2-release-retention.test.py python3 tools/package-driver-release-assets.test.py python3 tools/legal-release-assets.test.py python3 tools/windows-release-artifacts.test.py @@ -1118,6 +1119,11 @@ jobs: needs: - build - driver_agents + # Serialize only the publication stage with stable releases so both + # channels cannot pass the retained-byte preflight concurrently. + concurrency: + group: gonavi-r2-publication + cancel-in-progress: false runs-on: ubuntu-latest steps: - name: Checkout code @@ -1263,6 +1269,8 @@ jobs: --tag dev-latest \ --channel dev \ --name "Dev Build (${DEV_VERSION})" \ + --download-base-url "https://download.syngnat.top/gonavi/dev/releases/download" \ + --download-tag "$DEV_VERSION" \ --output release-assets/latest-dev.json test -s release-assets/latest-dev.json echo "📄 latest-dev.json ready:" @@ -1422,3 +1430,249 @@ jobs: > 每次 push 到 `dev` 分支会自动覆盖此 release。 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Mirror, promote, and retain latest dev release in Cloudflare R2 + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + DEV_VERSION: ${{ steps.version.outputs.version }} + HAS_DRIVER_ASSETS: ${{ steps.driver_assets.outputs.has_driver_assets }} + R2_BUCKET: gonavi-downloads + shell: bash + run: | + set -euo pipefail + if [[ -z "${CLOUDFLARE_ACCOUNT_ID}" || -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then + echo "Cloudflare R2 release credentials are not configured" >&2 + exit 1 + fi + + endpoint="https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" + app_dir="release-assets" + driver_dir="driver-release-assets" + app_prefix="gonavi/dev/releases/download/${DEV_VERSION}" + driver_prefix="drivers/dev/releases/download/${DEV_VERSION}" + + jq -e \ + --arg version "${DEV_VERSION}" \ + --arg mirror_base "https://download.syngnat.top/gonavi/dev/releases/download" \ + --arg github_base "https://github.com/${GITHUB_REPOSITORY}/releases/download/dev-latest" \ + ' + .channel == "dev" + and .tagName == "dev-latest" + and .version == $version + and (.assets | type == "array" and length > 0) + and all(.assets[]; + (.url | type == "string" and startswith($mirror_base + "/" + $version + "/")) + and (.apiUrl | type == "string" and startswith($github_base + "/")) + and (.sha256 | type == "string" and test("^[0-9a-fA-F]{64}$")) + ) + ' "${app_dir}/latest-dev.json" >/dev/null + + driver_latest_file="${RUNNER_TEMP}/driver-dev-latest-index.json" + if [[ "${HAS_DRIVER_ASSETS}" == "true" ]]; then + [[ -s "${driver_dir}/SHA256SUMS" ]] || { + echo "DriverAgents SHA256SUMS is missing" >&2 + exit 1 + } + (cd "${driver_dir}" && sha256sum --check SHA256SUMS) + jq \ + --arg tag "dev-latest" \ + --arg mirror_tag "${DEV_VERSION}" \ + '.tagName = $tag | .mirrorTagName = $mirror_tag | del(.assets["GoNavi-DriverAgents.zip"])' \ + "${driver_dir}/GoNavi-DriverAgents-Index.json" \ + > "${driver_latest_file}" + jq -e '.assets | type == "object" and length > 0' "${driver_latest_file}" >/dev/null + else + current_driver_state="${RUNNER_TEMP}/r2-current-dev-driver-state.json" + if python3 tools/r2-release-retention.py validate-driver-pointer \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --channel dev > "${current_driver_state}"; then + current_driver_keep_prefix="$(jq -r '.keepPrefix' "${current_driver_state}")" + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "drivers/dev/releases/download/" \ + --keep-prefix "${current_driver_keep_prefix}" + else + echo "::warning::Current dev driver pointer could not be fully validated; preserving every dev driver object" + fi + fi + + app_manifest_size="$(stat -c '%s' "${app_dir}/latest-dev.json")" + planned_bytes="$(jq '[.assets[].size] | add // 0' "${app_dir}/latest-dev.json")" + planned_bytes="$((planned_bytes + app_manifest_size * 2))" + budget_args=( + measure + --bucket "${R2_BUCKET}" + --endpoint-url "${endpoint}" + --prefix "gonavi/" + --prefix "drivers/" + --exclude-prefix "gonavi/dev/releases/download/" + --exclude-prefix "gonavi/dev/releases/latest/" + ) + if [[ -s "${driver_latest_file}" ]]; then + driver_index_size="$(stat -c '%s' "${driver_latest_file}")" + driver_asset_bytes="$(jq '[.assets[]] | add // 0' "${driver_latest_file}")" + planned_bytes="$((planned_bytes + driver_asset_bytes + driver_index_size * 2))" + budget_args+=( + --exclude-prefix "drivers/dev/releases/download/" + --exclude-prefix "drivers/dev/releases/latest/" + ) + fi + python3 tools/r2-release-retention.py "${budget_args[@]}" \ + --add-bytes "${planned_bytes}" \ + --max-bytes 9000000000 + + upload_immutable() { + local file="$1" + local key="$2" + local content_type="${3:-}" + local sha size head_json remote_sha remote_size + sha="$(sha256sum "${file}" | awk '{print $1}')" + size="$(stat -c '%s' "${file}")" + if head_json="$(aws s3api head-object --bucket "${R2_BUCKET}" --key "${key}" --endpoint-url "${endpoint}" --output json 2>&1)"; then + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "Refusing to overwrite immutable R2 object ${key}" >&2 + exit 1 + fi + return + elif ! grep -Eq '\(404\)|Not Found|NoSuchKey' <<<"${head_json}"; then + echo "Unable to inspect immutable R2 object ${key}: ${head_json}" >&2 + exit 1 + fi + args=(s3 cp "${file}" "s3://${R2_BUCKET}/${key}" --endpoint-url "${endpoint}" --cache-control "public,max-age=31536000,immutable" --metadata "sha256=${sha}" --only-show-errors) + if [[ -n "${content_type}" ]]; then + args+=(--content-type "${content_type}") + fi + aws "${args[@]}" + head_json="$(aws s3api head-object --bucket "${R2_BUCKET}" --key "${key}" --endpoint-url "${endpoint}" --output json)" + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "R2 verification failed for ${key}" >&2 + exit 1 + fi + } + + while IFS=$'\t' read -r name expected_size expected_sha; do + [[ "${name}" == "$(basename "${name}")" ]] || { + echo "Invalid dev app asset name: ${name}" >&2 + exit 1 + } + file="${app_dir}/${name}" + [[ -f "${file}" ]] || { + echo "Dev app asset is missing: ${name}" >&2 + exit 1 + } + actual_size="$(stat -c '%s' "${file}")" + actual_sha="$(sha256sum "${file}" | awk '{print $1}')" + if [[ "${actual_size}" != "${expected_size}" || "${actual_sha}" != "${expected_sha,,}" ]]; then + echo "Dev app asset does not match latest-dev.json: ${name}" >&2 + exit 1 + fi + upload_immutable "${file}" "${app_prefix}/${name}" + done < <(jq -r '.assets[] | [.name, (.size | tostring), .sha256] | @tsv' "${app_dir}/latest-dev.json") + upload_immutable \ + "${app_dir}/latest-dev.json" \ + "${app_prefix}/latest-dev.json" \ + "application/json; charset=utf-8" + + if [[ "${HAS_DRIVER_ASSETS}" == "true" ]]; then + while IFS=$'\t' read -r name expected_size; do + [[ "${name}" == "$(basename "${name}")" ]] || { + echo "Invalid dev driver asset name: ${name}" >&2 + exit 1 + } + file="${driver_dir}/${name}" + [[ -f "${file}" ]] || { + echo "Dev driver asset is missing: ${name}" >&2 + exit 1 + } + actual_size="$(stat -c '%s' "${file}")" + if [[ "${actual_size}" != "${expected_size}" ]]; then + echo "Dev driver asset size does not match index: ${name}" >&2 + exit 1 + fi + upload_immutable "${file}" "${driver_prefix}/${name}" + done < <(jq -r '.assets | to_entries[] | [.key, (.value | tostring)] | @tsv' "${driver_latest_file}") + upload_immutable \ + "${driver_latest_file}" \ + "${driver_prefix}/GoNavi-DriverAgents-Index.json" \ + "application/json; charset=utf-8" + fi + + promote_mutable() { + local file="$1" + local key="$2" + local sha size head_json remote_sha remote_size + sha="$(sha256sum "${file}" | awk '{print $1}')" + size="$(stat -c '%s' "${file}")" + aws s3 cp "${file}" "s3://${R2_BUCKET}/${key}" \ + --endpoint-url "${endpoint}" \ + --cache-control "no-cache,must-revalidate" \ + --content-type "application/json; charset=utf-8" \ + --metadata "sha256=${sha}" \ + --only-show-errors + head_json="$(aws s3api head-object --bucket "${R2_BUCKET}" --key "${key}" --endpoint-url "${endpoint}" --output json)" + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "R2 mutable object verification failed for ${key}" >&2 + exit 1 + fi + } + + app_latest_key="gonavi/dev/releases/latest/latest-dev.json" + promote_mutable "${app_dir}/latest-dev.json" "${app_latest_key}" + curl --fail --silent --show-error \ + "https://download.syngnat.top/${app_latest_key}?verify=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + | jq -e --arg version "${DEV_VERSION}" '.channel == "dev" and .version == $version' >/dev/null + + if [[ -s "${driver_latest_file}" ]]; then + driver_latest_key="drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json" + promote_mutable "${driver_latest_file}" "${driver_latest_key}" + curl --fail --silent --show-error \ + "https://download.syngnat.top/${driver_latest_key}?verify=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + | jq -e \ + --arg mirror_tag "${DEV_VERSION}" \ + '.tagName == "dev-latest" and .mirrorTagName == $mirror_tag' >/dev/null + fi + + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "gonavi/dev/releases/download/" \ + --keep-prefix "${app_prefix}/" + driver_keep_prefix="" + if [[ -s "${driver_latest_file}" ]]; then + driver_keep_prefix="${driver_prefix}/" + else + current_driver_state="${RUNNER_TEMP}/r2-current-dev-driver-state.json" + if python3 tools/r2-release-retention.py validate-driver-pointer \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --channel dev > "${current_driver_state}"; then + driver_keep_prefix="$(jq -r '.keepPrefix' "${current_driver_state}")" + else + echo "::warning::Skipping dev driver cleanup because the current R2 pointer or retained assets are invalid" + fi + fi + if [[ -n "${driver_keep_prefix}" ]]; then + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "drivers/dev/releases/download/" \ + --keep-prefix "${driver_keep_prefix}" + fi + + python3 tools/r2-release-retention.py measure \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --prefix "gonavi/" \ + --prefix "drivers/" \ + --max-bytes 9000000000 diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 4023d37c..a9f3a202 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -16,17 +16,19 @@ permissions: contents: write concurrency: - group: publish-release-${{ inputs.tag || github.ref_name }} + # Stable and dev publications share the same retained R2 byte budget. + group: gonavi-r2-publication cancel-in-progress: false jobs: publish: - name: Validate and publish draft + name: Validate, mirror, and publish release runs-on: ubuntu-latest env: RELEASE_TAG: ${{ inputs.tag || github.ref_name }} steps: - - name: Validate assets and publish + - name: Validate release + id: validate uses: actions/github-script@v8 with: script: | @@ -83,12 +85,41 @@ jobs: const releases = releaseList.data; const release = releases.find((item) => item.tag_name === tag); if (!release) { - core.setFailed(`Draft release not found for ${tag}`); + core.setFailed(`Release not found for ${tag}`); return; } - if (!release.draft) { - core.setFailed(`Release ${tag} is already published`); + + let latestRelease = null; + try { + latestRelease = (await github.rest.repos.getLatestRelease({ owner, repo })).data; + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + const semverParts = (value) => value.slice(1).split('.').map(Number); + const compareSemver = (left, right) => { + const a = semverParts(left); + const b = semverParts(right); + for (let index = 0; index < 3; index += 1) { + if (a[index] !== b[index]) { + return a[index] - b[index]; + } + } + return 0; + }; + let shouldPublish = release.draft; + if (release.draft) { + const latestTag = String(latestRelease?.tag_name || '').trim(); + if (latestTag && /^v\d+\.\d+\.\d+$/.test(latestTag) && compareSemver(tag, latestTag) <= 0) { + core.setFailed(`Refusing to publish ${tag} over current latest ${latestTag}`); + return; + } + } else if (!latestRelease || latestRelease.id !== release.id) { + core.setFailed(`Published release ${tag} is not the current GitHub latest release`); return; + } else { + shouldPublish = false; } const version = tag.slice(1); @@ -117,10 +148,252 @@ jobs: return; } + core.setOutput('tag', tag); + core.setOutput('release_id', String(release.id)); + core.setOutput('should_publish', String(shouldPublish)); + core.notice(`Validated ${release.draft ? 'draft' : 'published'} release ${tag} and its assets`); + + - name: Checkout release tooling + uses: actions/checkout@v5 + + - name: Mirror and verify stable release in Cloudflare R2 + env: + GH_TOKEN: ${{ github.token }} + DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + RELEASE_TAG: ${{ steps.validate.outputs.tag }} + R2_BUCKET: gonavi-downloads + shell: bash + run: | + set -euo pipefail + if [[ -z "${CLOUDFLARE_ACCOUNT_ID}" || -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" || -z "${DRIVER_RELEASE_TOKEN}" ]]; then + echo "Cloudflare R2 or DriverAgents release credentials are not configured" >&2 + exit 1 + fi + + endpoint="https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" + app_dir="${RUNNER_TEMP}/gonavi-release" + driver_dir="${RUNNER_TEMP}/driver-release" + mkdir -p "${app_dir}" "${driver_dir}" + gh release download "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${app_dir}" + + driver_release_json="${RUNNER_TEMP}/driver-release.json" + driver_status="$(curl --silent --show-error \ + --output "${driver_release_json}" \ + --write-out '%{http_code}' \ + --header "Authorization: Bearer ${DRIVER_RELEASE_TOKEN}" \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/Syngnat/GoNavi-DriverAgents/releases/tags/${RELEASE_TAG}")" + has_driver_release=false + case "${driver_status}" in + 200) + jq -e --arg tag "${RELEASE_TAG}" \ + '.tag_name == $tag and .draft == false' "${driver_release_json}" >/dev/null + GH_TOKEN="${DRIVER_RELEASE_TOKEN}" gh release download "${RELEASE_TAG}" \ + --repo Syngnat/GoNavi-DriverAgents --dir "${driver_dir}" + has_driver_release=true + ;; + 404) + echo "No DriverAgents release for ${RELEASE_TAG}; skipping driver mirror" + ;; + *) + echo "DriverAgents release lookup failed with HTTP ${driver_status}" >&2 + exit 1 + ;; + esac + + jq -e \ + --arg tag "${RELEASE_TAG}" \ + --arg mirror_base "https://download.syngnat.top/gonavi/releases/download" \ + --arg github_base "https://github.com/${GITHUB_REPOSITORY}/releases/download" \ + ' + .tagName == $tag + and (.assets | type == "array" and length > 0) + and all(.assets[]; + (.url | type == "string" and startswith($mirror_base + "/" + $tag + "/")) + and (.apiUrl | type == "string" and startswith($github_base + "/" + $tag + "/")) + and (.sha256 | type == "string" and test("^[0-9a-fA-F]{64}$")) + ) + ' "${app_dir}/latest.json" >/dev/null + if [[ "${has_driver_release}" == true ]]; then + jq -e '.assets | type == "object" and length > 0' \ + "${driver_dir}/GoNavi-DriverAgents-Index.json" >/dev/null + [[ -s "${driver_dir}/SHA256SUMS" ]] || { + echo "DriverAgents SHA256SUMS is missing" >&2 + exit 1 + } + (cd "${driver_dir}" && sha256sum --check SHA256SUMS) + jq --arg tag "${RELEASE_TAG}" \ + '.tagName = $tag' \ + "${driver_dir}/GoNavi-DriverAgents-Index.json" \ + > "${RUNNER_TEMP}/driver-latest-index.json" + fi + + while IFS=$'\t' read -r name expected_size expected_sha; do + [[ "${name}" == "$(basename "${name}")" ]] || { + echo "Invalid app asset name in latest.json: ${name}" >&2 + exit 1 + } + file="${app_dir}/${name}" + [[ -f "${file}" ]] || { + echo "App asset referenced by latest.json is missing: ${name}" >&2 + exit 1 + } + actual_size="$(stat -c '%s' "${file}")" + actual_sha="$(sha256sum "${file}" | awk '{print $1}')" + if [[ "${actual_size}" != "${expected_size}" || "${actual_sha}" != "${expected_sha,,}" ]]; then + echo "App asset does not match latest.json: ${name}" >&2 + exit 1 + fi + done < <(jq -r '.assets[] | [.name, (.size | tostring), .sha256] | @tsv' "${app_dir}/latest.json") + + if [[ "${has_driver_release}" == true ]]; then + while IFS=$'\t' read -r name expected_size; do + [[ "${name}" == "$(basename "${name}")" ]] || { + echo "Invalid driver asset name in index: ${name}" >&2 + exit 1 + } + file="${driver_dir}/${name}" + [[ -f "${file}" ]] || { + echo "Driver asset referenced by index is missing: ${name}" >&2 + exit 1 + } + actual_size="$(stat -c '%s' "${file}")" + if [[ "${actual_size}" != "${expected_size}" ]]; then + echo "Driver asset size does not match index: ${name}" >&2 + exit 1 + fi + done < <(jq -r '.assets | to_entries[] | [.key, (.value | tostring)] | @tsv' \ + "${driver_dir}/GoNavi-DriverAgents-Index.json") + else + current_driver_state="${RUNNER_TEMP}/r2-current-stable-driver-state.json" + if python3 tools/r2-release-retention.py validate-driver-pointer \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --channel stable > "${current_driver_state}"; then + current_driver_keep_prefix="$(jq -r '.keepPrefix' "${current_driver_state}")" + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "drivers/releases/download/" \ + --keep-prefix "${current_driver_keep_prefix}" + else + echo "::warning::Current stable driver pointer could not be fully validated; preserving every stable driver object" + fi + fi + + # Preflight the final retained footprint before uploading or + # publishing. Existing objects in the channel being replaced are + # excluded, then the exact local candidate size is added. + app_manifest_size="$(stat -c '%s' "${app_dir}/latest.json")" + planned_bytes="$(jq '[.assets[].size] | add // 0' "${app_dir}/latest.json")" + planned_bytes="$((planned_bytes + app_manifest_size * 2))" + budget_args=( + measure + --bucket "${R2_BUCKET}" + --endpoint-url "${endpoint}" + --prefix "gonavi/" + --prefix "drivers/" + --exclude-prefix "gonavi/releases/download/" + --exclude-prefix "gonavi/releases/latest/" + ) + if [[ "${has_driver_release}" == true ]]; then + driver_index_size="$(stat -c '%s' "${driver_dir}/GoNavi-DriverAgents-Index.json")" + driver_latest_size="$(stat -c '%s' "${RUNNER_TEMP}/driver-latest-index.json")" + driver_asset_bytes="$(jq '[.assets[]] | add // 0' "${driver_dir}/GoNavi-DriverAgents-Index.json")" + planned_bytes="$((planned_bytes + driver_asset_bytes + driver_index_size + driver_latest_size))" + budget_args+=( + --exclude-prefix "drivers/releases/download/" + --exclude-prefix "drivers/releases/latest/" + ) + fi + python3 tools/r2-release-retention.py "${budget_args[@]}" \ + --add-bytes "${planned_bytes}" \ + --max-bytes 9000000000 + + upload_immutable() { + local file="$1" + local key="$2" + local content_type="${3:-}" + local sha size head_json remote_sha remote_size + sha="$(sha256sum "${file}" | awk '{print $1}')" + size="$(stat -c '%s' "${file}")" + if head_json="$(aws s3api head-object --bucket "${R2_BUCKET}" --key "${key}" --endpoint-url "${endpoint}" --output json 2>&1)"; then + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "Refusing to overwrite immutable R2 object ${key}" >&2 + exit 1 + fi + echo "R2 object already verified: ${key}" + return + elif ! grep -Eq '\(404\)|Not Found|NoSuchKey' <<<"${head_json}"; then + echo "Unable to inspect immutable R2 object ${key}: ${head_json}" >&2 + exit 1 + fi + args=(s3 cp "${file}" "s3://${R2_BUCKET}/${key}" --endpoint-url "${endpoint}" --cache-control "public,max-age=31536000,immutable" --metadata "sha256=${sha}" --only-show-errors) + if [[ -n "${content_type}" ]]; then + args+=(--content-type "${content_type}") + fi + aws "${args[@]}" + head_json="$(aws s3api head-object --bucket "${R2_BUCKET}" --key "${key}" --endpoint-url "${endpoint}" --output json)" + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "R2 verification failed for ${key}" >&2 + exit 1 + fi + } + + while IFS= read -r name; do + file="${app_dir}/${name}" + upload_immutable "${file}" "gonavi/releases/download/${RELEASE_TAG}/${name}" + done < <(jq -r '.assets[].name' "${app_dir}/latest.json") + + if [[ "${has_driver_release}" == true ]]; then + # Upload only the standalone assets referenced by the index. The + # aggregate bundle duplicates them and remains on GitHub fallback. + while IFS= read -r name; do + file="${driver_dir}/${name}" + upload_immutable "${file}" "drivers/releases/download/${RELEASE_TAG}/${name}" + done < <(jq -r '.assets | keys[]' "${driver_dir}/GoNavi-DriverAgents-Index.json") + fi + + # Metadata is the commit point: publish it only after every referenced object is verified. + if [[ "${has_driver_release}" == true ]]; then + upload_immutable \ + "${driver_dir}/GoNavi-DriverAgents-Index.json" \ + "drivers/releases/download/${RELEASE_TAG}/GoNavi-DriverAgents-Index.json" \ + "application/json; charset=utf-8" + fi + upload_immutable \ + "${app_dir}/latest.json" \ + "gonavi/releases/download/${RELEASE_TAG}/latest.json" \ + "application/json; charset=utf-8" + + - name: Publish validated draft + if: steps.validate.outputs.should_publish == 'true' + uses: actions/github-script@v8 + env: + RELEASE_TAG: ${{ steps.validate.outputs.tag }} + RELEASE_ID: ${{ steps.validate.outputs.release_id }} + with: + script: | + const tag = String(process.env.RELEASE_TAG || '').trim(); + const releaseId = Number(process.env.RELEASE_ID || 0); + if (!/^v\d+\.\d+\.\d+$/.test(tag) || !Number.isSafeInteger(releaseId) || releaseId <= 0) { + core.setFailed(`Invalid validated release: tag=${tag} id=${releaseId}`); + return; + } + const { owner, repo } = context.repo; const published = await github.rest.repos.updateRelease({ owner, repo, - release_id: release.id, + release_id: releaseId, draft: false, prerelease: false, make_latest: 'true', @@ -129,5 +402,131 @@ jobs: core.setFailed(`Release ${tag} was not published successfully`); return; } - core.notice(`Published ${tag}: ${published.data.html_url}`); + + - name: Verify GitHub release is published and latest + uses: actions/github-script@v8 + env: + RELEASE_TAG: ${{ steps.validate.outputs.tag }} + RELEASE_ID: ${{ steps.validate.outputs.release_id }} + with: + script: | + const tag = String(process.env.RELEASE_TAG || '').trim(); + const releaseId = Number(process.env.RELEASE_ID || 0); + const { owner, repo } = context.repo; + const [releaseResult, latestResult] = await Promise.all([ + github.rest.repos.getRelease({ owner, repo, release_id: releaseId }), + github.rest.repos.getLatestRelease({ owner, repo }), + ]); + const release = releaseResult.data; + const latest = latestResult.data; + if (release.draft || !release.published_at || release.tag_name !== tag || latest.id !== release.id) { + core.setFailed(`Release ${tag} is not the current published GitHub latest release`); + return; + } + core.notice(`Verified GitHub latest release ${tag}`); + + - name: Promote and verify Cloudflare R2 latest manifest + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + RELEASE_TAG: ${{ steps.validate.outputs.tag }} + R2_BUCKET: gonavi-downloads + shell: bash + run: | + set -euo pipefail + endpoint="https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" + file="${RUNNER_TEMP}/gonavi-release/latest.json" + key="gonavi/releases/latest/latest.json" + sha="$(sha256sum "${file}" | awk '{print $1}')" + size="$(stat -c '%s' "${file}")" + + aws s3 cp "${file}" "s3://${R2_BUCKET}/${key}" \ + --endpoint-url "${endpoint}" \ + --cache-control "no-cache,must-revalidate" \ + --content-type "application/json; charset=utf-8" \ + --metadata "sha256=${sha}" \ + --only-show-errors + + head_json="$(aws s3api head-object \ + --bucket "${R2_BUCKET}" \ + --key "${key}" \ + --endpoint-url "${endpoint}" \ + --output json)" + remote_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${head_json}")" + remote_size="$(jq -r '.ContentLength // 0' <<<"${head_json}")" + if [[ "${remote_sha}" != "${sha}" || "${remote_size}" != "${size}" ]]; then + echo "R2 latest manifest verification failed" >&2 + exit 1 + fi + + curl --fail --silent --show-error \ + "https://download.syngnat.top/gonavi/releases/latest/latest.json?verify=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + | jq -e --arg tag "${RELEASE_TAG}" '.tagName == $tag' >/dev/null + + driver_latest_file="${RUNNER_TEMP}/driver-latest-index.json" + if [[ -s "${driver_latest_file}" ]]; then + driver_key="drivers/releases/latest/GoNavi-DriverAgents-Index.json" + driver_sha="$(sha256sum "${driver_latest_file}" | awk '{print $1}')" + driver_size="$(stat -c '%s' "${driver_latest_file}")" + aws s3 cp "${driver_latest_file}" "s3://${R2_BUCKET}/${driver_key}" \ + --endpoint-url "${endpoint}" \ + --cache-control "no-cache,must-revalidate" \ + --content-type "application/json; charset=utf-8" \ + --metadata "sha256=${driver_sha}" \ + --only-show-errors + driver_head_json="$(aws s3api head-object \ + --bucket "${R2_BUCKET}" \ + --key "${driver_key}" \ + --endpoint-url "${endpoint}" \ + --output json)" + remote_driver_sha="$(jq -r '.Metadata.sha256 // ""' <<<"${driver_head_json}")" + remote_driver_size="$(jq -r '.ContentLength // 0' <<<"${driver_head_json}")" + if [[ "${remote_driver_sha}" != "${driver_sha}" || "${remote_driver_size}" != "${driver_size}" ]]; then + echo "R2 latest driver index verification failed" >&2 + exit 1 + fi + curl --fail --silent --show-error \ + "https://download.syngnat.top/${driver_key}?verify=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + | jq -e --arg tag "${RELEASE_TAG}" '.tagName == $tag' >/dev/null + fi + + # Only prune after the mutable pointers are publicly verified. A + # failed cleanup leaves extra storage but never breaks the active + # updater path; rerunning this job is idempotent. + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "gonavi/releases/download/" \ + --keep-prefix "gonavi/releases/download/${RELEASE_TAG}/" + driver_keep_prefix="" + if [[ -s "${driver_latest_file}" ]]; then + driver_keep_prefix="drivers/releases/download/${RELEASE_TAG}/" + else + current_driver_state="${RUNNER_TEMP}/r2-current-stable-driver-state.json" + if python3 tools/r2-release-retention.py validate-driver-pointer \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --channel stable > "${current_driver_state}"; then + driver_keep_prefix="$(jq -r '.keepPrefix' "${current_driver_state}")" + else + echo "::warning::Skipping stable driver cleanup because the current R2 pointer or retained assets are invalid" + fi + fi + if [[ -n "${driver_keep_prefix}" ]]; then + python3 tools/r2-release-retention.py prune \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --root-prefix "drivers/releases/download/" \ + --keep-prefix "${driver_keep_prefix}" + fi + + # Keep a 1 GB safety margin below the 10 GB-month free allowance. + python3 tools/r2-release-retention.py measure \ + --bucket "${R2_BUCKET}" \ + --endpoint-url "${endpoint}" \ + --prefix "gonavi/" \ + --prefix "drivers/" \ + --max-bytes 9000000000 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 472c9506..ca0dc105 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,6 +28,7 @@ jobs: bash tools/generate-driver-agent-revisions.test.sh python3 tools/generate-driver-release-manifest.test.py python3 tools/generate-update-latest-manifest.test.py + python3 tools/r2-release-retention.test.py python3 tools/package-driver-release-assets.test.py python3 tools/legal-release-assets.test.py python3 tools/windows-release-artifacts.test.py @@ -1246,6 +1247,7 @@ jobs: --version "$VERSION" \ --tag "$TAG" \ --channel latest \ + --download-base-url https://download.syngnat.top/gonavi/releases/download \ --output release-assets/latest.json test -s release-assets/latest.json echo "📄 latest.json ready:" diff --git a/internal/app/methods_driver.go b/internal/app/methods_driver.go index 48b3a9a2..e1e4f431 100644 --- a/internal/app/methods_driver.go +++ b/internal/app/methods_driver.go @@ -250,6 +250,7 @@ type driverNetworkProbeItem struct { const ( driverStatusReasonSlimBuildMissingDriver = "slim_build_missing_driver" + driverNetworkProbeCodeCloudflareR2 = "cloudflare_r2" driverNetworkProbeCodeGitHubAPI = "github_api" driverNetworkProbeCodeGitHubRelease = "github_release" driverNetworkProbeCodeGitHubReleaseAsset = "github_release_asset" @@ -317,10 +318,11 @@ type driverVersionOptionItem struct { } type driverReleaseAssetSizeCacheEntry struct { - LoadedAt time.Time - SizeByKey map[string]int64 - PublishedAssets map[string]bool - Err string + LoadedAt time.Time + SizeByKey map[string]int64 + PublishedAssets map[string]bool + MirrorDownloadURLs map[string]string + Err string } type goModuleLatestVersionCacheEntry struct { @@ -345,47 +347,53 @@ type goModuleVersionMeta struct { } type driverBundleAssetIndex struct { - Assets map[string]int64 `json:"assets"` + TagName string `json:"tagName,omitempty"` + MirrorTagName string `json:"mirrorTagName,omitempty"` + Assets map[string]int64 `json:"assets"` } const ( // 默认使用内置 manifest,避免依赖网络与外部仓库 404。 - defaultDriverManifestURLValue = "builtin://manifest" - driverReleaseRepo = "Syngnat/GoNavi-DriverAgents" - driverReleaseLatestAPIURL = "https://api.github.com/repos/" + driverReleaseRepo + "/releases/latest" - driverReleaseDevTag = "dev-latest" - optionalDriverBundleAssetName = "GoNavi-DriverAgents.zip" - duckDBWindowsDriverZipAssetName = "duckdb-driver.zip" - optionalDriverBundleIndexAssetName = "GoNavi-DriverAgents-Index.json" - optionalDriverBundleDownloadTimeout = 15 * time.Minute - optionalDriverBundleCacheMaxAge = 7 * 24 * time.Hour - optionalDriverBundleCacheMaxFiles = 4 - driverManifestCacheTTL = 5 * time.Minute - driverReleaseAssetSizeCacheTTL = 30 * time.Minute - driverReleaseAssetSizeErrorCacheTTL = 30 * time.Second - driverReleaseAssetSizeProbeTimeout = 4 * time.Second - driverReleaseListProbeTimeout = 6 * time.Second - driverModuleLatestCacheTTL = 6 * time.Hour - driverModuleLatestErrorCacheTTL = 2 * time.Minute - driverModuleLatestProbeTimeout = 4 * time.Second - driverModuleVersionInspectLimit = 30 - driverModuleVersionListMaxSize = 4 << 20 - driverRecentVersionLimit = 5 - driverModuleVersionFetchLimit = 64 - driverVersionWarmupMinInterval = 30 * time.Second - driverBundleIndexMaxSize = 1 << 20 - driverManifestMaxSize = 2 << 20 - driverNetworkProbeTimeout = 4 * time.Second - driverNetworkProbeTCPTimeout = 3 * time.Second - localDriverDirectoryScanMaxEntries = 20000 - driverChecksumPolicyStrict = "strict" - driverChecksumPolicyWarn = "warn" - driverChecksumPolicyOff = "off" - driverEngineGo = "go" - driverEngineExternal = "external" - duckDBWindowsLibraryVersion = "v1.4.4" - duckDBWindowsLibraryArchiveURL = "https://github.com/duckdb/duckdb/releases/download/" + duckDBWindowsLibraryVersion + "/libduckdb-windows-amd64.zip" - duckDBWindowsSupportDLLName = "duckdb.dll" + defaultDriverManifestURLValue = "builtin://manifest" + driverReleaseRepo = "Syngnat/GoNavi-DriverAgents" + driverReleaseMirrorBaseURL = "https://download.syngnat.top/drivers/releases/download" + driverReleaseMirrorLatestIndexURL = "https://download.syngnat.top/drivers/releases/latest/GoNavi-DriverAgents-Index.json" + driverReleaseMirrorDevBaseURL = "https://download.syngnat.top/drivers/dev/releases/download" + driverReleaseMirrorDevLatestIndexURL = "https://download.syngnat.top/drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json" + driverReleaseLatestAPIURL = "https://api.github.com/repos/" + driverReleaseRepo + "/releases/latest" + driverReleaseDevTag = "dev-latest" + optionalDriverBundleAssetName = "GoNavi-DriverAgents.zip" + duckDBWindowsDriverZipAssetName = "duckdb-driver.zip" + optionalDriverBundleIndexAssetName = "GoNavi-DriverAgents-Index.json" + optionalDriverBundleDownloadTimeout = 15 * time.Minute + optionalDriverBundleCacheMaxAge = 7 * 24 * time.Hour + optionalDriverBundleCacheMaxFiles = 4 + driverManifestCacheTTL = 5 * time.Minute + driverReleaseAssetSizeCacheTTL = 30 * time.Minute + driverReleaseAssetSizeErrorCacheTTL = 30 * time.Second + driverReleaseAssetSizeProbeTimeout = 4 * time.Second + driverReleaseListProbeTimeout = 6 * time.Second + driverModuleLatestCacheTTL = 6 * time.Hour + driverModuleLatestErrorCacheTTL = 2 * time.Minute + driverModuleLatestProbeTimeout = 4 * time.Second + driverModuleVersionInspectLimit = 30 + driverModuleVersionListMaxSize = 4 << 20 + driverRecentVersionLimit = 5 + driverModuleVersionFetchLimit = 64 + driverVersionWarmupMinInterval = 30 * time.Second + driverBundleIndexMaxSize = 1 << 20 + driverManifestMaxSize = 2 << 20 + driverNetworkProbeTimeout = 4 * time.Second + driverNetworkProbeTCPTimeout = 3 * time.Second + localDriverDirectoryScanMaxEntries = 20000 + driverChecksumPolicyStrict = "strict" + driverChecksumPolicyWarn = "warn" + driverChecksumPolicyOff = "off" + driverEngineGo = "go" + driverEngineExternal = "external" + duckDBWindowsLibraryVersion = "v1.4.4" + duckDBWindowsLibraryArchiveURL = "https://github.com/duckdb/duckdb/releases/download/" + duckDBWindowsLibraryVersion + "/libduckdb-windows-amd64.zip" + duckDBWindowsSupportDLLName = "duckdb.dll" ) const builtinDriverManifestJSON = `{ @@ -1288,6 +1296,11 @@ func (a *App) GetDriverStatusList(downloadDir string, manifestURL string) connec func (a *App) CheckDriverNetworkStatus() connection.QueryResult { checks := []driverNetworkProbeItem{ + { + ProbeCode: driverNetworkProbeCodeCloudflareR2, + Name: "Cloudflare R2", + URL: "https://download.syngnat.top/health.txt", + }, { ProbeCode: driverNetworkProbeCodeGitHubAPI, Name: "GitHub API", @@ -1326,17 +1339,18 @@ func (a *App) CheckDriverNetworkStatus() connection.QueryResult { } return driverNetworkProbeItem{}, false } + r2Check, _ := findProbe(driverNetworkProbeCodeCloudflareR2) githubAPICheck, _ := findProbe(driverNetworkProbeCodeGitHubAPI) githubReleaseCheck, _ := findProbe(driverNetworkProbeCodeGitHubRelease) releaseAssetsCheck, _ := findProbe(driverNetworkProbeCodeGitHubReleaseAsset) - downloadChainReachable := githubReleaseCheck.Reachable && releaseAssetsCheck.Reachable + downloadChainReachable := r2Check.Reachable || (githubReleaseCheck.Reachable && releaseAssetsCheck.Reachable) proxyEnv := collectDriverProxyEnv() proxyConfigured := len(proxyEnv) > 0 summary := a.appText("driver_manager.network.summary.reachable", nil) if githubAPICheck.Reachable && !downloadChainReachable { summary = a.appText("driver_manager.backend.network.summary.download_chain_unreachable", nil) - } else if !allReachable { + } else if !downloadChainReachable { if proxyConfigured { summary = a.appText("driver_manager.network.summary.unreachable_proxy_configured", nil) } else { @@ -1345,13 +1359,15 @@ func (a *App) CheckDriverNetworkStatus() connection.QueryResult { } data := map[string]interface{}{ - "reachable": allReachable, + "reachable": downloadChainReachable, + "allReachable": allReachable, "summary": summary, - "recommendedProxy": !allReachable, + "recommendedProxy": !downloadChainReachable, "proxyConfigured": proxyConfigured, "proxyEnv": proxyEnv, "downloadChainReachable": downloadChainReachable, "downloadRequiredHosts": []string{ + "download.syngnat.top", "github.com", "api.github.com", "release-assets.githubusercontent.com", @@ -2346,6 +2362,11 @@ func resolvePublishedDriverDownloadURLForTag(definition driverDefinition, select if !ok { return "", false } + if strings.EqualFold(tagName, driverReleaseDevTag) { + if mirrorURL := readReleaseMirrorDownloadURLFromCache("tag:"+tagName, assetName); mirrorURL != "" { + return mirrorURL, true + } + } return driverReleaseDownloadURL(tagName, assetName), true } diff --git a/internal/app/methods_driver_assets.go b/internal/app/methods_driver_assets.go index 9b2c99a8..40053c22 100644 --- a/internal/app/methods_driver_assets.go +++ b/internal/app/methods_driver_assets.go @@ -207,6 +207,89 @@ func driverReleaseDownloadURL(tag string, assetName string) string { return fmt.Sprintf("https://github.com/%s/releases/download/%s/%s", driverReleaseRepo, url.PathEscape(tagName), url.PathEscape(asset)) } +func driverMirrorReleaseDownloadURL(tag string, assetName string) string { + tagName := strings.TrimSpace(tag) + asset := strings.TrimSpace(assetName) + if tagName == "" || asset == "" { + return "" + } + return fmt.Sprintf("%s/%s/%s", driverReleaseMirrorBaseURL, url.PathEscape(tagName), url.PathEscape(asset)) +} + +func driverMirrorDevReleaseDownloadURL(tag string, assetName string) string { + tagName := strings.TrimSpace(tag) + asset := strings.TrimSpace(assetName) + if tagName == "" || asset == "" { + return "" + } + return fmt.Sprintf("%s/%s/%s", driverReleaseMirrorDevBaseURL, url.PathEscape(tagName), url.PathEscape(asset)) +} + +func driverMirrorReleaseDownloadURLForTags(releaseTag string, mirrorTag string, assetName string) string { + logicalTag := strings.TrimSpace(releaseTag) + physicalTag := strings.TrimSpace(mirrorTag) + if physicalTag == "" { + physicalTag = logicalTag + } + if strings.EqualFold(logicalTag, driverReleaseDevTag) { + return driverMirrorDevReleaseDownloadURL(physicalTag, assetName) + } + return driverMirrorReleaseDownloadURL(physicalTag, assetName) +} + +func driverReleaseDownloadCoordinates(rawURL string) (string, string, bool) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return "", "", false + } + segments := strings.Split(strings.Trim(parsed.EscapedPath(), "/"), "/") + host := strings.ToLower(strings.TrimSpace(parsed.Hostname())) + knownSource := false + switch host { + case "github.com": + knownSource = len(segments) >= 2 && strings.EqualFold(segments[0], "Syngnat") && strings.EqualFold(segments[1], "GoNavi-DriverAgents") + case "download.syngnat.top": + knownSource = len(segments) >= 1 && strings.EqualFold(segments[0], "drivers") + } + if !knownSource { + return "", "", false + } + for index := 0; index+3 < len(segments); index++ { + if !strings.EqualFold(segments[index], "releases") || !strings.EqualFold(segments[index+1], "download") { + continue + } + tagName, tagErr := url.PathUnescape(segments[index+2]) + assetName, assetErr := url.PathUnescape(strings.Join(segments[index+3:], "/")) + if tagErr != nil || assetErr != nil || strings.TrimSpace(tagName) == "" || strings.TrimSpace(assetName) == "" { + return "", "", false + } + return tagName, assetName, true + } + return "", "", false +} + +func driverReleaseAssetNameFromURL(rawURL string) string { + if _, assetName, ok := driverReleaseDownloadCoordinates(rawURL); ok { + return assetName + } + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return "" + } + if fragment := strings.TrimSpace(parsed.Fragment); fragment != "" { + return fragment + } + segments := strings.Split(strings.Trim(parsed.EscapedPath(), "/"), "/") + if len(segments) == 0 { + return "" + } + assetName, err := url.PathUnescape(segments[len(segments)-1]) + if err != nil { + return "" + } + return strings.TrimSpace(assetName) +} + func driverReleaseLatestDownloadURL(assetName string) string { asset := strings.TrimSpace(assetName) if asset == "" { @@ -215,6 +298,13 @@ func driverReleaseLatestDownloadURL(assetName string) string { return fmt.Sprintf("https://github.com/%s/releases/latest/download/%s", driverReleaseRepo, url.PathEscape(asset)) } +func driverReleaseLatestDownloadURLForCurrentChannel(assetName string) string { + if strings.EqualFold(currentDriverReleaseTag(), driverReleaseDevTag) { + return driverReleaseDownloadURL(driverReleaseDevTag, assetName) + } + return driverReleaseLatestDownloadURL(assetName) +} + func findReleaseAssetByName(release *githubRelease, assetNames []string) (githubAsset, bool) { if release == nil || len(release.Assets) == 0 || len(assetNames) == 0 { return githubAsset{}, false @@ -326,8 +416,8 @@ func resolveOptionalDriverAssetSizeForVersion(sizeByAsset map[string]int64, driv } func resolveOptionalDriverBundleDownloadURLs() []string { - candidates := make([]string, 0, 2) - seen := make(map[string]struct{}, 2) + candidates := make([]string, 0, 6) + seen := make(map[string]struct{}, 6) appendURL := func(value string) { trimmed := strings.TrimSpace(value) if trimmed == "" { @@ -340,20 +430,22 @@ func resolveOptionalDriverBundleDownloadURLs() []string { candidates = append(candidates, trimmed) } - if tag := currentDriverReleaseTag(); tag != "" { - if release, err := fetchReleaseByTag(tag); err == nil { - if asset, ok := findReleaseAssetByName(release, []string{optionalDriverBundleAssetName}); ok { - appendURL(driverReleaseAssetAPIURL(asset)) + tag := currentDriverReleaseTag() + if tag != "" { + if strings.EqualFold(tag, driverReleaseDevTag) { + if release, err := fetchMirrorDriverReleaseByTag(tag); err == nil { + if asset, ok := findReleaseAssetByName(release, []string{optionalDriverBundleAssetName}); ok { + appendURL(asset.BrowserDownloadURL) + } } + } else { + appendURL(driverMirrorReleaseDownloadURL(tag, optionalDriverBundleAssetName)) } appendURL(driverReleaseDownloadURL(tag, optionalDriverBundleAssetName)) } - if release, err := fetchLatestReleaseForDriverAssets(); err == nil { - if asset, ok := findReleaseAssetByName(release, []string{optionalDriverBundleAssetName}); ok { - appendURL(driverReleaseAssetAPIURL(asset)) - } + if !strings.EqualFold(tag, driverReleaseDevTag) { + appendURL(driverReleaseLatestDownloadURL(optionalDriverBundleAssetName)) } - appendURL(driverReleaseLatestDownloadURL(optionalDriverBundleAssetName)) return candidates } @@ -543,8 +635,8 @@ func acquireOptionalDriverBundlePath(bundleURL string, onProgress func(downloade } func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL string, selectedVersion string) []string { - candidates := make([]string, 0, 3) - seen := make(map[string]struct{}, 3) + candidates := make([]string, 0, 6) + seen := make(map[string]struct{}, 6) driverType := normalizeDriverType(definition.Type) appendURL := func(value string) { trimmed := strings.TrimSpace(value) @@ -559,14 +651,37 @@ func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL } restrictToExplicitArtifact := shouldRestrictToExplicitVersionArtifact(definition, selectedVersion) + appendPublishedURL := func(tag string, publishedURL string) { + releaseTag := strings.TrimSpace(tag) + assetName := driverReleaseAssetNameFromURL(publishedURL) + parsed, _ := url.Parse(strings.TrimSpace(publishedURL)) + if parsed != nil && strings.EqualFold(parsed.Hostname(), "download.syngnat.top") { + appendURL(publishedURL) + appendURL(driverReleaseDownloadURL(releaseTag, assetName)) + return + } + mirrorTag := releaseTag + if publishedTag, publishedAsset, ok := driverReleaseDownloadCoordinates(publishedURL); ok { + mirrorTag = publishedTag + assetName = publishedAsset + } + if mirrorTag != "" && assetName != "" { + if strings.EqualFold(releaseTag, driverReleaseDevTag) { + appendURL(readReleaseMirrorDownloadURLFromCache("tag:"+releaseTag, assetName)) + } else { + appendURL(driverMirrorReleaseDownloadURL(mirrorTag, assetName)) + } + } + appendURL(publishedURL) + } appendPublishedURLs := func() { if tag := currentDriverReleaseTag(); tag != "" { if publishedURL, ok := resolvePublishedDriverDownloadURLForTag(definition, selectedVersion, tag); ok { - appendURL(publishedURL) + appendPublishedURL(tag, publishedURL) } } if publishedURL, ok := resolveLatestPublishedDriverDownloadURLForVersion(definition, selectedVersion); ok { - appendURL(publishedURL) + appendPublishedURL(currentDriverReleaseTag(), publishedURL) } } @@ -577,6 +692,14 @@ func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil { switch strings.ToLower(strings.TrimSpace(parsed.Scheme)) { case "http", "https": + if tag, assetName, ok := driverReleaseDownloadCoordinates(parsed.String()); ok && + !strings.EqualFold(parsed.Hostname(), "download.syngnat.top") { + if strings.EqualFold(tag, driverReleaseDevTag) { + appendURL(readReleaseMirrorDownloadURLFromCache("tag:"+tag, assetName)) + } else { + appendURL(driverMirrorReleaseDownloadURL(tag, assetName)) + } + } appendURL(parsed.String()) } } @@ -1058,15 +1181,17 @@ func loadReleaseAssetSizesCached(cacheKey string, fetch func() (*githubRelease, release, err := fetch() entry := driverReleaseAssetSizeCacheEntry{ - LoadedAt: time.Now(), - SizeByKey: map[string]int64{}, - PublishedAssets: map[string]bool{}, + LoadedAt: time.Now(), + SizeByKey: map[string]int64{}, + PublishedAssets: map[string]bool{}, + MirrorDownloadURLs: map[string]string{}, } if err != nil { entry.Err = err.Error() } else { entry.SizeByKey = buildReleaseAssetSizeMap(release) entry.PublishedAssets = buildReleaseAssetNameMap(release) + entry.MirrorDownloadURLs = buildReleaseMirrorDownloadURLMap(release) if indexSizes, indexErr := fetchDriverBundleAssetSizeIndex(release); indexErr == nil { for name, size := range indexSizes { trimmedName := strings.TrimSpace(name) @@ -1088,6 +1213,38 @@ func loadReleaseAssetSizesCached(cacheKey string, fetch func() (*githubRelease, return entry.SizeByKey, entry.PublishedAssets, nil } +func buildReleaseMirrorDownloadURLMap(release *githubRelease) map[string]string { + urls := make(map[string]string) + if release == nil { + return urls + } + for _, asset := range release.Assets { + name := strings.TrimSpace(asset.Name) + downloadURL := strings.TrimSpace(asset.BrowserDownloadURL) + parsed, err := url.Parse(downloadURL) + if name == "" || err != nil || !strings.EqualFold(parsed.Hostname(), "download.syngnat.top") { + continue + } + urls[name] = downloadURL + } + return urls +} + +func readReleaseMirrorDownloadURLFromCache(cacheKey string, assetName string) string { + key := strings.TrimSpace(cacheKey) + name := strings.TrimSpace(assetName) + if key == "" || name == "" { + return "" + } + driverReleaseSizeMu.RLock() + cached, ok := driverReleaseSizeMap[key] + driverReleaseSizeMu.RUnlock() + if !ok || time.Since(cached.LoadedAt) >= driverReleaseAssetSizeCacheTTL { + return "" + } + return strings.TrimSpace(cached.MirrorDownloadURLs[name]) +} + func readReleaseAssetSizesFromCache(cacheKey string) (map[string]int64, map[string]bool, bool) { key := strings.TrimSpace(cacheKey) if key == "" { @@ -1145,8 +1302,16 @@ func buildReleaseAssetNameMap(release *githubRelease) map[string]bool { } func fetchDriverBundleAssetSizeIndex(release *githubRelease) (map[string]int64, error) { + index, err := fetchDriverBundleAssetIndex(release) + if err != nil { + return nil, err + } + return index.Assets, nil +} + +func fetchDriverBundleAssetIndex(release *githubRelease) (driverBundleAssetIndex, error) { if release == nil { - return nil, newLocalizedDriverBackendError("driver_manager.backend.error.release_empty", nil, nil) + return driverBundleAssetIndex{}, newLocalizedDriverBackendError("driver_manager.backend.error.release_empty", nil, nil) } indexURL := "" for _, asset := range release.Assets { @@ -1156,24 +1321,24 @@ func fetchDriverBundleAssetSizeIndex(release *githubRelease) (map[string]int64, } } if indexURL == "" { - return nil, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_asset_missing", nil, nil) + return driverBundleAssetIndex{}, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_asset_missing", nil, nil) } client := newHTTPClientWithGlobalProxy(driverReleaseAssetSizeProbeTimeout) req, err := http.NewRequest(http.MethodGet, indexURL, nil) if err != nil { - return nil, err + return driverBundleAssetIndex{}, err } req.Header.Set("User-Agent", "GoNavi-DriverManager") req.Header.Set("Accept", "application/json") resp, err := client.Do(req) if err != nil { - return nil, err + return driverBundleAssetIndex{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, newLocalizedDriverBackendError( + return driverBundleAssetIndex{}, newLocalizedDriverBackendError( "driver_manager.backend.error.bundle_index_fetch_failed", nil, fmt.Errorf("HTTP %d", resp.StatusCode), @@ -1184,15 +1349,21 @@ func fetchDriverBundleAssetSizeIndex(release *githubRelease) (map[string]int64, decoder := json.NewDecoder(limited) var index driverBundleAssetIndex if err := decoder.Decode(&index); err != nil { - return nil, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_parse_failed", nil, err) + return driverBundleAssetIndex{}, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_parse_failed", nil, err) } if len(index.Assets) == 0 { - return nil, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_empty", nil, nil) + return driverBundleAssetIndex{}, newLocalizedDriverBackendError("driver_manager.backend.error.bundle_index_empty", nil, nil) } - return index.Assets, nil + return index, nil } func fetchLatestReleaseForDriverAssets() (*githubRelease, error) { + if strings.EqualFold(currentDriverReleaseTag(), driverReleaseDevTag) { + return fetchReleaseByTag(driverReleaseDevTag) + } + if release, err := fetchDriverReleaseIndexByURL("", driverReleaseMirrorLatestIndexURL); err == nil { + return release, nil + } return fetchDriverReleaseByURL(driverReleaseLatestAPIURL) } @@ -1213,7 +1384,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseAssetAPIURL(asset), true } } - return driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName), true + return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true } return "", false } @@ -1228,7 +1399,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseAssetAPIURL(asset), true } } - return driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName), true + return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true } return "", false } @@ -1246,7 +1417,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseAssetAPIURL(asset), true } } - return driverReleaseLatestDownloadURL(assetName), true + return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true } } return "", false @@ -1263,7 +1434,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseAssetAPIURL(asset), true } } - return driverReleaseLatestDownloadURL(assetName), true + return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true } } return "", false @@ -1274,10 +1445,77 @@ func fetchReleaseByTag(tag string) (*githubRelease, error) { if tagName == "" { return nil, newLocalizedDriverBackendError("driver_manager.backend.error.tag_empty", nil, nil) } + if release, err := fetchMirrorDriverReleaseByTag(tagName); err == nil { + return release, nil + } apiURL := fmt.Sprintf("https://api.github.com/repos/%s/releases/tags/%s", driverReleaseRepo, url.PathEscape(tagName)) return fetchDriverReleaseByURL(apiURL) } +func fetchMirrorDriverReleaseByTag(tag string) (*githubRelease, error) { + tagName := strings.TrimSpace(tag) + if tagName == "" { + return nil, newLocalizedDriverBackendError("driver_manager.backend.error.tag_empty", nil, nil) + } + if strings.EqualFold(tagName, driverReleaseDevTag) { + return fetchDriverReleaseIndexByURL(tagName, driverReleaseMirrorDevLatestIndexURL) + } + return fetchDriverReleaseIndexByURL( + tagName, + driverMirrorReleaseDownloadURL(tagName, optionalDriverBundleIndexAssetName), + ) +} + +func fetchDriverReleaseIndexByURL(tag string, indexURL string) (*githubRelease, error) { + fallbackTag := strings.TrimSpace(tag) + indexRelease := &githubRelease{ + TagName: fallbackTag, + Assets: []githubAsset{{ + Name: optionalDriverBundleIndexAssetName, + BrowserDownloadURL: strings.TrimSpace(indexURL), + }}, + } + index, err := fetchDriverBundleAssetIndex(indexRelease) + if err != nil { + return nil, err + } + tagName := strings.TrimSpace(index.TagName) + if tagName == "" { + tagName = fallbackTag + } + if strings.EqualFold(fallbackTag, driverReleaseDevTag) { + // dev alias 的逻辑 GitHub 标签固定为 dev-latest;mirrorTagName 仅控制 R2 物理路径。 + tagName = driverReleaseDevTag + } + if tagName == "" { + return nil, newLocalizedDriverBackendError("driver_manager.backend.error.tag_empty", nil, nil) + } + mirrorTagName := strings.TrimSpace(index.MirrorTagName) + if mirrorTagName == "" { + mirrorTagName = tagName + } + sizes := index.Assets + names := make([]string, 0, len(sizes)) + for name := range sizes { + names = append(names, name) + } + sort.Strings(names) + assets := make([]githubAsset, 0, len(names)) + for _, name := range names { + trimmedName := strings.TrimSpace(name) + if trimmedName == "" { + continue + } + assets = append(assets, githubAsset{ + Name: trimmedName, + BrowserDownloadURL: driverMirrorReleaseDownloadURLForTags(tagName, mirrorTagName, trimmedName), + URL: driverReleaseDownloadURL(tagName, trimmedName), + Size: sizes[name], + }) + } + return &githubRelease{TagName: tagName, Assets: assets}, nil +} + func fetchDriverReleaseByURL(apiURL string) (*githubRelease, error) { urlText := strings.TrimSpace(apiURL) if urlText == "" { diff --git a/internal/app/methods_driver_version_test.go b/internal/app/methods_driver_version_test.go index f5f30039..701cbdba 100644 --- a/internal/app/methods_driver_version_test.go +++ b/internal/app/methods_driver_version_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -133,6 +134,21 @@ func TestCurrentDriverReleaseTagUsesVersionedReleaseForStableBuild(t *testing.T) } } +func TestDriverReleaseLatestDownloadURLForCurrentChannelUsesDevLatest(t *testing.T) { + originalVersion := AppVersion + AppVersion = "dev-a1b2c3d" + t.Cleanup(func() { + AppVersion = originalVersion + }) + + const assetName = "sqlserver-driver-agent-windows-amd64.exe" + got := driverReleaseLatestDownloadURLForCurrentChannel(assetName) + want := driverReleaseDownloadURL(driverReleaseDevTag, assetName) + if got != want { + t.Fatalf("dev latest fallback URL = %q, want %q", got, want) + } +} + func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing.T) { originalVersion := AppVersion AppVersion = "0.7.4" @@ -141,6 +157,7 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing }) urls := resolveOptionalDriverBundleDownloadURLs() + wantMirror := driverMirrorReleaseDownloadURL("v0.7.4", optionalDriverBundleAssetName) wantTagged := driverReleaseDownloadURL("v0.7.4", optionalDriverBundleAssetName) wantLatest := driverReleaseLatestDownloadURL(optionalDriverBundleAssetName) if len(urls) < 2 { @@ -148,7 +165,11 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing } foundTagged := false foundLatest := false + foundMirror := false for _, candidate := range urls { + if candidate == wantMirror { + foundMirror = true + } if candidate == wantTagged { foundTagged = true } @@ -156,8 +177,147 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing foundLatest = true } } - if !foundTagged || !foundLatest { - t.Fatalf("expected bundle URLs to include tagged=%q and latest=%q, got %v", wantTagged, wantLatest, urls) + if !foundMirror || !foundTagged || !foundLatest { + t.Fatalf("expected bundle URLs to include mirror=%q, tagged=%q and latest=%q, got %v", wantMirror, wantTagged, wantLatest, urls) + } + if urls[0] != wantMirror { + t.Fatalf("expected R2 mirror first, got %v", urls) + } +} + +func TestDriverReleaseDownloadCoordinates(t *testing.T) { + tag, asset, ok := driverReleaseDownloadCoordinates("https://github.com/Syngnat/GoNavi-DriverAgents/releases/download/v1.2.3/driver%20agent.exe") + if !ok || tag != "v1.2.3" || asset != "driver agent.exe" { + t.Fatalf("unexpected GitHub coordinates: tag=%q asset=%q ok=%v", tag, asset, ok) + } + tag, asset, ok = driverReleaseDownloadCoordinates(driverMirrorReleaseDownloadURL("dev-latest", "driver agent.exe")) + if !ok || tag != "dev-latest" || asset != "driver agent.exe" { + t.Fatalf("unexpected mirror coordinates: tag=%q asset=%q ok=%v", tag, asset, ok) + } + if _, _, ok := driverReleaseDownloadCoordinates("https://example.com/releases/download/v1.2.3/driver.exe"); ok { + t.Fatal("expected unrelated release URL not to be rewritten to the GoNavi mirror") + } +} + +func TestFetchDriverReleaseIndexByURLBuildsMirrorAssets(t *testing.T) { + for _, name := range []string{ + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", + "http_proxy", "https_proxy", "all_proxy", + } { + t.Setenv(name, "") + } + t.Setenv("NO_PROXY", "127.0.0.1,localhost") + t.Setenv("no_proxy", "127.0.0.1,localhost") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"assets":{"sqlserver-driver-agent-windows-amd64.exe":12345}}`) + })) + defer server.Close() + + release, err := fetchDriverReleaseIndexByURL("v1.2.3", server.URL) + if err != nil { + t.Fatalf("fetch driver index: %v", err) + } + if release.TagName != "v1.2.3" || len(release.Assets) != 1 { + t.Fatalf("unexpected synthetic release: %#v", release) + } + asset := release.Assets[0] + if asset.Name != "sqlserver-driver-agent-windows-amd64.exe" || asset.Size != 12345 { + t.Fatalf("unexpected synthetic asset: %#v", asset) + } + if asset.BrowserDownloadURL != driverMirrorReleaseDownloadURL("v1.2.3", asset.Name) { + t.Fatalf("unexpected mirror URL: %q", asset.BrowserDownloadURL) + } + if asset.URL != driverReleaseDownloadURL("v1.2.3", asset.Name) { + t.Fatalf("unexpected GitHub fallback URL: %q", asset.URL) + } + + latestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"tagName":"v1.2.2","assets":{"sqlserver-driver-agent-windows-amd64.exe":12345}}`) + })) + defer latestServer.Close() + latestRelease, err := fetchDriverReleaseIndexByURL("", latestServer.URL) + if err != nil { + t.Fatalf("fetch latest driver index: %v", err) + } + if latestRelease.TagName != "v1.2.2" || len(latestRelease.Assets) != 1 { + t.Fatalf("unexpected latest synthetic release: %#v", latestRelease) + } + latestAsset := latestRelease.Assets[0] + if latestAsset.BrowserDownloadURL != driverMirrorReleaseDownloadURL("v1.2.2", latestAsset.Name) || + latestAsset.URL != driverReleaseDownloadURL("v1.2.2", latestAsset.Name) { + t.Fatalf("latest pointer did not preserve source tag: %#v", latestAsset) + } + + devServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"tagName":"dev-stale-logical-tag","mirrorTagName":"dev-a1b2c3d","assets":{"sqlserver-driver-agent-windows-amd64.exe":12345}}`) + })) + defer devServer.Close() + devRelease, err := fetchDriverReleaseIndexByURL(driverReleaseDevTag, devServer.URL) + if err != nil { + t.Fatalf("fetch dev driver index: %v", err) + } + if devRelease.TagName != driverReleaseDevTag || len(devRelease.Assets) != 1 { + t.Fatalf("unexpected dev synthetic release: %#v", devRelease) + } + devAsset := devRelease.Assets[0] + if devAsset.BrowserDownloadURL != driverMirrorDevReleaseDownloadURL("dev-a1b2c3d", devAsset.Name) { + t.Fatalf("dev mirror URL did not use the physical mirror tag: %#v", devAsset) + } + if devAsset.URL != driverReleaseDownloadURL(driverReleaseDevTag, devAsset.Name) { + t.Fatalf("dev GitHub fallback did not retain dev-latest: %#v", devAsset) + } +} + +func TestResolvePublishedDriverDownloadURLForTagUsesDevMirrorTag(t *testing.T) { + definition := driverDefinition{Type: "sqlserver"} + assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "") + if len(assetNames) == 0 { + t.Fatal("expected sqlserver release asset names") + } + assetName := assetNames[0] + want := driverMirrorDevReleaseDownloadURL("dev-a1b2c3d", assetName) + + driverReleaseSizeMu.Lock() + original := cloneReleaseAssetSizeCache(driverReleaseSizeMap) + driverReleaseSizeMap["tag:"+driverReleaseDevTag] = driverReleaseAssetSizeCacheEntry{ + LoadedAt: time.Now(), + SizeByKey: map[string]int64{assetName: 12345}, + PublishedAssets: map[string]bool{assetName: true}, + MirrorDownloadURLs: map[string]string{assetName: want}, + } + driverReleaseSizeMu.Unlock() + t.Cleanup(func() { + driverReleaseSizeMu.Lock() + driverReleaseSizeMap = original + driverReleaseSizeMu.Unlock() + }) + + got, ok := resolvePublishedDriverDownloadURLForTag(definition, "", driverReleaseDevTag) + if !ok || got != want { + t.Fatalf("dev published URL = %q, ok=%v, want mirror %q", got, ok, want) + } +} + +func TestResolveLatestPublishedDriverDownloadURLFallsBackWhenMirrorIndexMissesAsset(t *testing.T) { + seedReleaseAssetSizeCache(t, "latest", map[string]int64{ + "unrelated-driver-agent-windows-amd64.exe": 123, + }) + definition := driverDefinition{Type: "sqlserver"} + assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "") + if len(assetNames) == 0 { + t.Fatal("expected sqlserver release asset names") + } + got, ok := resolveLatestPublishedDriverDownloadURLForVersion(definition, "") + if !ok { + t.Fatal("expected GitHub latest fallback for an incomplete mirror index") + } + want := driverReleaseLatestDownloadURL(assetNames[0]) + if got != want { + t.Fatalf("latest fallback URL = %q, want %q", got, want) } } @@ -389,10 +549,10 @@ func TestResolveOptionalDriverAgentDownloadURLsDoesNotFallbackForHistoricalVersi explicitURL, "1.17.4", ) - if len(urls) != 1 { - t.Fatalf("expected only explicit historical URL, got %d candidates: %v", len(urls), urls) + if len(urls) != 2 { + t.Fatalf("expected mirror plus explicit historical URL, got %d candidates: %v", len(urls), urls) } - if urls[0] != explicitURL { + if urls[0] != driverMirrorReleaseDownloadURL("v1.17.4", mongoVersionedReleaseAssetName(1)) || urls[1] != explicitURL { t.Fatalf("unexpected historical URL candidate: %v", urls) } } @@ -2196,15 +2356,27 @@ func cloneReleaseAssetSizeCache(src map[string]driverReleaseAssetSizeCacheEntry) cloned := make(map[string]driverReleaseAssetSizeCacheEntry, len(src)) for key, value := range src { cloned[key] = driverReleaseAssetSizeCacheEntry{ - LoadedAt: value.LoadedAt, - SizeByKey: cloneInt64Map(value.SizeByKey), - PublishedAssets: cloneBoolMap(value.PublishedAssets), - Err: value.Err, + LoadedAt: value.LoadedAt, + SizeByKey: cloneInt64Map(value.SizeByKey), + PublishedAssets: cloneBoolMap(value.PublishedAssets), + MirrorDownloadURLs: cloneDriverStringMap(value.MirrorDownloadURLs), + Err: value.Err, } } return cloned } +func cloneDriverStringMap(src map[string]string) map[string]string { + if len(src) == 0 { + return map[string]string{} + } + cloned := make(map[string]string, len(src)) + for key, value := range src { + cloned[key] = value + } + return cloned +} + func cloneBoolMap(src map[string]bool) map[string]bool { if len(src) == 0 { return map[string]bool{} diff --git a/internal/app/methods_update.go b/internal/app/methods_update.go index 3891b2fb..025d3c9f 100644 --- a/internal/app/methods_update.go +++ b/internal/app/methods_update.go @@ -61,6 +61,8 @@ var ( updateExitProcess = os.Exit ) +var errUpdateChecksumMismatch = errors.New("update package checksum mismatch") + type updateState struct { lastCheck *UpdateInfo downloading bool @@ -502,20 +504,6 @@ func (a *App) downloadAndStageUpdate(info UpdateInfo) connection.QueryResult { } a.emitUpdateDownloadProgress("downloading", downloaded, reportTotal, "") } - actualHash, err := downloadFileWithHash(info.AssetURL, assetPath, progressCB) - if err != nil && strings.TrimSpace(info.AssetAPIURL) != "" && !strings.EqualFold(strings.TrimSpace(info.AssetAPIURL), strings.TrimSpace(info.AssetURL)) { - logger.Warnf("更新包主下载地址失败,尝试 assets API 回退:err=%v", err) - _ = os.Remove(assetPath) - actualHash, err = downloadFileWithHash(info.AssetAPIURL, assetPath, progressCB) - } - if err != nil { - _ = os.Remove(assetPath) - _ = os.RemoveAll(stagedDir) - message := a.localizedUpdateError(err) - a.emitUpdateDownloadProgress("error", 0, info.AssetSize, message) - return connection.QueryResult{Success: false, Message: message} - } - if info.SHA256 == "" { _ = os.Remove(assetPath) _ = os.RemoveAll(stagedDir) @@ -523,10 +511,22 @@ func (a *App) downloadAndStageUpdate(info UpdateInfo) connection.QueryResult { a.emitUpdateDownloadProgress("error", 0, info.AssetSize, message) return connection.QueryResult{Success: false, Message: message} } - if !strings.EqualFold(info.SHA256, actualHash) { + + _, err := downloadUpdateAssetWithFallback( + []string{info.AssetURL, info.AssetAPIURL}, + assetPath, + info.SHA256, + progressCB, + ) + if err != nil { _ = os.Remove(assetPath) _ = os.RemoveAll(stagedDir) - message := a.appText("app.update.backend.message.checksum_failed", nil) + if errors.Is(err, errUpdateChecksumMismatch) { + message := a.appText("app.update.backend.message.checksum_failed", nil) + a.emitUpdateDownloadProgress("error", 0, info.AssetSize, message) + return connection.QueryResult{Success: false, Message: message} + } + message := a.localizedUpdateError(err) a.emitUpdateDownloadProgress("error", 0, info.AssetSize, message) return connection.QueryResult{Success: false, Message: message} } @@ -552,6 +552,52 @@ func (a *App) downloadAndStageUpdate(info UpdateInfo) connection.QueryResult { return connection.QueryResult{Success: true, Message: a.appText("app.update.backend.message.package_downloaded", nil), Data: buildUpdateDownloadResult(info, staged)} } +func downloadUpdateAssetWithFallback( + candidates []string, + assetPath string, + expectedSHA256 string, + onProgress func(downloaded, total int64), +) (string, error) { + seen := make(map[string]struct{}, len(candidates)) + urls := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + trimmed := strings.TrimSpace(candidate) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + urls = append(urls, trimmed) + } + if len(urls) == 0 { + return "", localizedUpdateError{ + key: "app.update.backend.error.download_failed", + params: map[string]any{"detail": "download URL is empty"}, + } + } + + expectedHash := strings.TrimSpace(expectedSHA256) + var lastErr error + for index, candidate := range urls { + _ = os.Remove(assetPath) + actualHash, err := downloadFileWithHash(candidate, assetPath, onProgress) + if err == nil && expectedHash != "" && !strings.EqualFold(expectedHash, actualHash) { + err = errUpdateChecksumMismatch + } + if err == nil { + return actualHash, nil + } + lastErr = err + if index+1 < len(urls) { + logger.Warnf("更新包下载源失败,尝试下一下载源:attempt=%d err=%v", index+1, err) + } + } + _ = os.Remove(assetPath) + return "", lastErr +} + func fetchLatestUpdateInfo(channel updateChannel) (UpdateInfo, error) { return fetchLatestUpdateInfoWithOptions(channel, true) } diff --git a/internal/app/update_manifest.go b/internal/app/update_manifest.go index 239457e2..7b7bee50 100644 --- a/internal/app/update_manifest.go +++ b/internal/app/update_manifest.go @@ -5,8 +5,10 @@ import ( "fmt" "io" "net/http" + neturl "net/url" "os" "path/filepath" + stdRuntime "runtime" "strings" "sync" "time" @@ -16,10 +18,11 @@ import ( ) const ( - // 静态清单:挂在 GitHub Release 资产上,走 github.com 下载链路,不消耗 api.github.com 配额。 - // 发版时由 CI/build-release 生成 latest.json 并上传到当前 latest 发布。 - updateLatestManifestURL = "https://github.com/" + updateRepo + "/releases/latest/download/latest.json" - updateDevManifestURL = "https://github.com/" + updateRepo + "/releases/download/" + updateDevReleaseTag + "/latest-dev.json" + // 静态清单优先走 R2 自定义域名,GitHub Release 作为故障回退。 + updateMirrorLatestManifestURL = "https://download.syngnat.top/gonavi/releases/latest/latest.json" + updateMirrorDevManifestURL = "https://download.syngnat.top/gonavi/dev/releases/latest/latest-dev.json" + updateGitHubLatestManifestURL = "https://github.com/" + updateRepo + "/releases/latest/download/latest.json" + updateGitHubDevManifestURL = "https://github.com/" + updateRepo + "/releases/download/" + updateDevReleaseTag + "/latest-dev.json" updateManifestSchemaVersion = 1 updateManifestFileName = "latest.json" @@ -32,16 +35,16 @@ const ( // updateReleaseManifest 是面向终端用户的静态更新清单(不依赖 GitHub REST API)。 type updateReleaseManifest struct { - SchemaVersion int `json:"schemaVersion"` - Channel string `json:"channel"` - TagName string `json:"tagName"` - Version string `json:"version"` - Name string `json:"name,omitempty"` - HTMLURL string `json:"htmlUrl,omitempty"` - PublishedAt string `json:"publishedAt,omitempty"` - Assets []updateManifestAsset `json:"assets"` - FetchedAt time.Time `json:"fetchedAt,omitempty"` // 仅本地缓存写入 - Source string `json:"source,omitempty"` // static | api | disk-cache + SchemaVersion int `json:"schemaVersion"` + Channel string `json:"channel"` + TagName string `json:"tagName"` + Version string `json:"version"` + Name string `json:"name,omitempty"` + HTMLURL string `json:"htmlUrl,omitempty"` + PublishedAt string `json:"publishedAt,omitempty"` + Assets []updateManifestAsset `json:"assets"` + FetchedAt time.Time `json:"fetchedAt,omitempty"` // 仅本地缓存写入 + Source string `json:"source,omitempty"` // static | api | disk-cache } type updateManifestAsset struct { @@ -71,11 +74,11 @@ func swapUpdateFetchStaticManifest(next func(updateChannel) (*githubRelease, err } } -func updateManifestRemoteURL(channel updateChannel) string { +func updateManifestRemoteURLs(channel updateChannel) []string { if channel == updateChannelDev { - return updateDevManifestURL + return []string{updateMirrorDevManifestURL, updateGitHubDevManifestURL} } - return updateLatestManifestURL + return []string{updateMirrorLatestManifestURL, updateGitHubLatestManifestURL} } func updateManifestCachePath(channel updateChannel) string { @@ -220,7 +223,31 @@ func storeDiskUpdateManifest(channel updateChannel, manifest *updateReleaseManif } func fetchStaticUpdateManifest(channel updateChannel) (*githubRelease, error) { - url := updateManifestRemoteURL(channel) + return fetchStaticUpdateManifestFromURLs(channel, updateManifestRemoteURLs(channel)) +} + +func fetchStaticUpdateManifestFromURLs(channel updateChannel, manifestURLs []string) (*githubRelease, error) { + var failures []string + for _, manifestURL := range manifestURLs { + release, err := fetchStaticUpdateManifestFromURL(channel, manifestURL) + if err == nil && release != nil { + return release, nil + } + if err != nil { + failures = append(failures, fmt.Sprintf("%s: %v", manifestURL, err)) + } + } + if len(failures) == 0 { + return nil, fmt.Errorf("no static update manifest URL configured") + } + return nil, fmt.Errorf("static update manifests unavailable: %s", strings.Join(failures, "; ")) +} + +func fetchStaticUpdateManifestFromURL(channel updateChannel, manifestURL string) (*githubRelease, error) { + url := strings.TrimSpace(manifestURL) + if url == "" { + return nil, fmt.Errorf("static update manifest URL is empty") + } client := newHTTPClientWithGlobalProxy(15 * time.Second) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -257,6 +284,9 @@ func fetchStaticUpdateManifest(channel updateChannel) (*githubRelease, error) { if manifest.SchemaVersion != 0 && manifest.SchemaVersion != updateManifestSchemaVersion { return nil, fmt.Errorf("unsupported update manifest schema: %d", manifest.SchemaVersion) } + if err := validateRemoteUpdateManifest(channel, &manifest); err != nil { + return nil, fmt.Errorf("invalid static update manifest %s: %w", url, err) + } manifest.FetchedAt = time.Now().UTC() manifest.Source = "static" if strings.TrimSpace(manifest.Channel) == "" { @@ -273,6 +303,73 @@ func fetchStaticUpdateManifest(channel updateChannel) (*githubRelease, error) { return release, nil } +func validateRemoteUpdateManifest(channel updateChannel, manifest *updateReleaseManifest) error { + if manifest == nil { + return fmt.Errorf("manifest is empty") + } + expectedChannel := string(updateChannelLatest) + if channel == updateChannelDev { + expectedChannel = string(updateChannelDev) + } + manifestChannel := strings.TrimSpace(manifest.Channel) + if manifestChannel != "" && !strings.EqualFold(manifestChannel, expectedChannel) { + return fmt.Errorf("channel %q does not match requested channel %q", manifestChannel, expectedChannel) + } + if len(manifest.Assets) == 0 { + return fmt.Errorf("manifest has no assets") + } + + assetNames := make(map[string]struct{}, len(manifest.Assets)) + for _, asset := range manifest.Assets { + name := strings.TrimSpace(asset.Name) + if name == "" { + return fmt.Errorf("manifest contains an unnamed asset") + } + nameKey := strings.ToLower(name) + if _, exists := assetNames[nameKey]; exists { + return fmt.Errorf("manifest contains duplicate asset %q", name) + } + assetNames[nameKey] = struct{}{} + if asset.Size <= 0 { + return fmt.Errorf("asset %q has invalid size", name) + } + if normalizeGitHubAssetSHA256(asset.SHA256) == "" { + return fmt.Errorf("asset %q has invalid sha256", name) + } + for label, rawURL := range map[string]string{"url": asset.URL, "apiUrl": asset.APIURL} { + value := strings.TrimSpace(rawURL) + if value == "" { + if label == "url" { + return fmt.Errorf("asset %q has no download URL", name) + } + continue + } + parsed, err := neturl.ParseRequestURI(value) + if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return fmt.Errorf("asset %q has invalid %s", name, label) + } + } + } + + assetVersion := strings.TrimSpace(manifest.TagName) + if assetVersion == "" || strings.EqualFold(normalizeVersion(assetVersion), updateDevReleaseTag) { + assetVersion = strings.TrimSpace(manifest.Version) + } + expectedAsset, err := expectedAssetNameForInstallMode( + stdRuntime.GOOS, + stdRuntime.GOARCH, + assetVersion, + updateResolveInstallMode(), + ) + if err != nil { + return err + } + if _, exists := assetNames[strings.ToLower(expectedAsset)]; !exists { + return fmt.Errorf("manifest does not contain current platform asset %q", expectedAsset) + } + return nil +} + // fetchReleaseForChannel prefers static manifest → GitHub API → disk cache. // forceNetwork=false 时:静默检查若距上次成功拉网过近,直接用磁盘缓存。 func fetchReleaseForChannelPreferringStatic(channel updateChannel, forceNetwork bool) (*githubRelease, error) { diff --git a/internal/app/update_manifest_test.go b/internal/app/update_manifest_test.go index a43bd3bf..ed469d20 100644 --- a/internal/app/update_manifest_test.go +++ b/internal/app/update_manifest_test.go @@ -1,15 +1,56 @@ package app import ( + "crypto/sha256" "encoding/json" "errors" + "fmt" + "net/http" + "net/http/httptest" "os" + "path/filepath" + stdRuntime "runtime" "strings" "sync" + "sync/atomic" "testing" "time" ) +func TestDownloadUpdateAssetWithFallbackRetriesChecksumMismatch(t *testing.T) { + goodPayload := []byte("verified update package") + expectedHash := fmt.Sprintf("%x", sha256.Sum256(goodPayload)) + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("corrupted mirror object")) + })) + defer primary.Close() + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(goodPayload) + })) + defer fallback.Close() + + assetPath := filepath.Join(t.TempDir(), "GoNavi.bin") + actualHash, err := downloadUpdateAssetWithFallback( + []string{primary.URL, fallback.URL}, + assetPath, + expectedHash, + nil, + ) + if err != nil { + t.Fatalf("expected checksum mismatch fallback to succeed: %v", err) + } + if actualHash != expectedHash { + t.Fatalf("actual hash = %q, want %q", actualHash, expectedHash) + } + payload, err := os.ReadFile(assetPath) + if err != nil { + t.Fatalf("read downloaded asset: %v", err) + } + if string(payload) != string(goodPayload) { + t.Fatalf("downloaded payload = %q", payload) + } +} + func TestReleaseFromUpdateManifestMapsAssets(t *testing.T) { release := releaseFromUpdateManifest(&updateReleaseManifest{ TagName: "v1.2.3", @@ -40,6 +81,111 @@ func TestReleaseFromUpdateManifestMapsAssets(t *testing.T) { } } +func TestUpdateManifestRemoteURLsPreferR2ThenGitHub(t *testing.T) { + tests := []struct { + channel updateChannel + want []string + }{ + {updateChannelLatest, []string{updateMirrorLatestManifestURL, updateGitHubLatestManifestURL}}, + {updateChannelDev, []string{updateMirrorDevManifestURL, updateGitHubDevManifestURL}}, + } + for _, test := range tests { + got := updateManifestRemoteURLs(test.channel) + if len(got) != len(test.want) { + t.Fatalf("channel %s URLs = %v", test.channel, got) + } + for index := range test.want { + if got[index] != test.want[index] { + t.Fatalf("channel %s URLs = %v, want %v", test.channel, got, test.want) + } + } + } +} + +func TestFetchStaticUpdateManifestFromURLsFallsBackFromInvalidR2Manifests(t *testing.T) { + for _, name := range []string{ + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", + "http_proxy", "https_proxy", "all_proxy", + } { + t.Setenv(name, "") + } + t.Setenv("NO_PROXY", "127.0.0.1,localhost") + t.Setenv("no_proxy", "127.0.0.1,localhost") + t.Setenv("GONAVI_DATA_ROOT", t.TempDir()) + + const version = "9.9.9" + expectedAsset, err := expectedAssetNameForInstallMode( + stdRuntime.GOOS, + stdRuntime.GOARCH, + version, + updateResolveInstallMode(), + ) + if err != nil { + t.Fatalf("resolve expected update asset: %v", err) + } + valid := updateReleaseManifest{ + SchemaVersion: updateManifestSchemaVersion, + Channel: string(updateChannelLatest), + TagName: "v" + version, + Version: version, + Assets: []updateManifestAsset{{ + Name: expectedAsset, + URL: "https://download.example.test/" + expectedAsset, + APIURL: "https://github.example.test/" + expectedAsset, + Size: 123, + SHA256: strings.Repeat("a", 64), + }}, + } + wrongChannel := valid + wrongChannel.Channel = string(updateChannelDev) + missingTarget := valid + missingTarget.Assets = []updateManifestAsset{{ + Name: "GoNavi-9.9.9-Other-Platform.bin", + URL: "https://download.example.test/other.bin", + Size: 123, + SHA256: strings.Repeat("b", 64), + }} + + serveManifest := func(manifest updateReleaseManifest, hits *atomic.Int32) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(manifest); err != nil { + t.Errorf("encode manifest: %v", err) + } + })) + } + var wrongChannelHits atomic.Int32 + var missingTargetHits atomic.Int32 + var validHits atomic.Int32 + wrongChannelServer := serveManifest(wrongChannel, &wrongChannelHits) + defer wrongChannelServer.Close() + missingTargetServer := serveManifest(missingTarget, &missingTargetHits) + defer missingTargetServer.Close() + validServer := serveManifest(valid, &validHits) + defer validServer.Close() + + release, err := fetchStaticUpdateManifestFromURLs(updateChannelLatest, []string{ + wrongChannelServer.URL, + missingTargetServer.URL, + validServer.URL, + }) + if err != nil { + t.Fatalf("expected valid fallback manifest: %v", err) + } + if release == nil || release.TagName != "v"+version || len(release.Assets) != 1 || release.Assets[0].Name != expectedAsset { + t.Fatalf("unexpected fallback release: %#v", release) + } + if wrongChannelHits.Load() != 1 || missingTargetHits.Load() != 1 || validHits.Load() != 1 { + t.Fatalf( + "unexpected manifest request counts: wrong-channel=%d missing-target=%d valid=%d", + wrongChannelHits.Load(), + missingTargetHits.Load(), + validHits.Load(), + ) + } +} + func TestDiskUpdateManifestRoundTrip(t *testing.T) { root := t.TempDir() t.Setenv("GONAVI_DATA_ROOT", root) diff --git a/tools/generate-update-latest-manifest.py b/tools/generate-update-latest-manifest.py index 22396c8f..c57bc90f 100755 --- a/tools/generate-update-latest-manifest.py +++ b/tools/generate-update-latest-manifest.py @@ -29,6 +29,7 @@ import re import sys from datetime import datetime, timezone from pathlib import Path +from urllib.parse import quote REPO = "Syngnat/GoNavi" SCHEMA_VERSION = 1 @@ -72,11 +73,22 @@ def browser_download_url(tag: str, asset_name: str) -> str: return f"https://github.com/{REPO}/releases/download/{tag}/{name}" +def mirror_download_url(base_url: str, tag: str, asset_name: str) -> str: + base = (base_url or "").strip().rstrip("/") + return f"{base}/{quote(tag.strip(), safe='')}/{quote(asset_name.strip(), safe='')}" + + def html_url(tag: str) -> str: return f"https://github.com/{REPO}/releases/tag/{tag.strip()}" -def collect_assets(assets_dir: Path, tag: str, hashes: dict[str, str]) -> list[dict]: +def collect_assets( + assets_dir: Path, + tag: str, + hashes: dict[str, str], + download_base_url: str = "", + download_tag: str = "", +) -> list[dict]: assets: list[dict] = [] for path in sorted(assets_dir.iterdir()): if not path.is_file(): @@ -86,11 +98,16 @@ def collect_assets(assets_dir: Path, tag: str, hashes: dict[str, str]) -> list[d continue if name.startswith("."): continue + github_url = browser_download_url(tag, name) item = { "name": name, - "url": browser_download_url(tag, name), + "url": mirror_download_url(download_base_url, download_tag or tag, name) + if download_base_url + else github_url, "size": path.stat().st_size, } + if download_base_url: + item["apiUrl"] = github_url sha = hashes.get(name, "").strip().lower() if sha: item["sha256"] = sha @@ -106,11 +123,13 @@ def build_manifest( assets_dir: Path, name: str | None, published_at: str | None, + download_base_url: str = "", + download_tag: str = "", ) -> dict: hashes = parse_sha256sums(assets_dir / "SHA256SUMS") tag = tag.strip() or f"v{normalize_version(version)}" version = normalize_version(version) or normalize_version(tag) - assets = collect_assets(assets_dir, tag, hashes) + assets = collect_assets(assets_dir, tag, hashes, download_base_url, download_tag) if not assets: raise SystemExit(f"no release assets found under {assets_dir}") @@ -139,6 +158,16 @@ def main() -> int: ) parser.add_argument("--name", default="", help="Release display name") parser.add_argument("--published-at", default="", help="ISO8601 published time") + parser.add_argument( + "--download-base-url", + default="", + help="Primary release download base URL; GitHub is retained in apiUrl as fallback", + ) + parser.add_argument( + "--download-tag", + default="", + help="Optional tag/path segment for the primary download URL; manifest and GitHub tag stay unchanged", + ) parser.add_argument( "--output", default="", @@ -169,6 +198,8 @@ def main() -> int: assets_dir=assets_dir, name=args.name or None, published_at=args.published_at or None, + download_base_url=args.download_base_url, + download_tag=args.download_tag, ) output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(f"wrote {output} ({len(manifest['assets'])} assets, version={manifest['version']})") diff --git a/tools/generate-update-latest-manifest.test.py b/tools/generate-update-latest-manifest.test.py index 271c9422..86038053 100644 --- a/tools/generate-update-latest-manifest.test.py +++ b/tools/generate-update-latest-manifest.test.py @@ -38,6 +38,8 @@ class GenerateUpdateLatestManifestTest(unittest.TestCase): "v1.2.3", "--channel", "latest", + "--download-base-url", + "https://download.syngnat.top/gonavi/releases/download/", "--output", str(out), ], @@ -59,12 +61,20 @@ class GenerateUpdateLatestManifestTest(unittest.TestCase): portable_asset = assets_by_name["GoNavi-1.2.3-Windows-Amd64-Portable.exe"] self.assertEqual( portable_asset["url"], + "https://download.syngnat.top/gonavi/releases/download/v1.2.3/GoNavi-1.2.3-Windows-Amd64-Portable.exe", + ) + self.assertEqual( + portable_asset["apiUrl"], "https://github.com/Syngnat/GoNavi/releases/download/v1.2.3/GoNavi-1.2.3-Windows-Amd64-Portable.exe", ) self.assertEqual(portable_asset["sha256"], "a" * 64) installer_asset = assets_by_name["GoNavi-1.2.3-Windows-Amd64-Installer.msi"] self.assertEqual( installer_asset["url"], + "https://download.syngnat.top/gonavi/releases/download/v1.2.3/GoNavi-1.2.3-Windows-Amd64-Installer.msi", + ) + self.assertEqual( + installer_asset["apiUrl"], "https://github.com/Syngnat/GoNavi/releases/download/v1.2.3/GoNavi-1.2.3-Windows-Amd64-Installer.msi", ) self.assertEqual(installer_asset["sha256"], "b" * 64) @@ -72,6 +82,53 @@ class GenerateUpdateLatestManifestTest(unittest.TestCase): self.assertNotIn("LICENSE", [a["name"] for a in data["assets"]]) self.assertNotIn("NOTICE", [a["name"] for a in data["assets"]]) + def test_dev_manifest_keeps_github_tag_but_uses_unique_mirror_tag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + assets = Path(tmp) + asset_name = "GoNavi-dev-a1b2c3d-Windows-Amd64-Portable.exe" + (assets / asset_name).write_bytes(b"fake-dev-binary") + (assets / "SHA256SUMS").write_text( + f"{'c' * 64} {asset_name}\n", + encoding="utf-8", + ) + out = assets / "latest-dev.json" + subprocess.check_call( + [ + sys.executable, + str(SCRIPT), + "--assets-dir", + str(assets), + "--version", + "dev-a1b2c3d", + "--tag", + "dev-latest", + "--channel", + "dev", + "--download-base-url", + "https://download.syngnat.top/gonavi/dev/releases/download", + "--download-tag", + "dev-a1b2c3d", + "--output", + str(out), + ], + cwd=str(ROOT), + ) + + data = json.loads(out.read_text(encoding="utf-8")) + self.assertEqual(data["channel"], "dev") + self.assertEqual(data["tagName"], "dev-latest") + self.assertEqual(data["htmlUrl"], "https://github.com/Syngnat/GoNavi/releases/tag/dev-latest") + self.assertEqual(len(data["assets"]), 1) + asset = data["assets"][0] + self.assertEqual( + asset["url"], + f"https://download.syngnat.top/gonavi/dev/releases/download/dev-a1b2c3d/{asset_name}", + ) + self.assertEqual( + asset["apiUrl"], + f"https://github.com/Syngnat/GoNavi/releases/download/dev-latest/{asset_name}", + ) + if __name__ == "__main__": unittest.main() diff --git a/tools/r2-release-retention.py b/tools/r2-release-retention.py new file mode 100644 index 00000000..4e1adddd --- /dev/null +++ b/tools/r2-release-retention.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""Prune obsolete GoNavi R2 release prefixes and report retained usage. + +The script intentionally accepts only the four release roots used by GoNavi. +That guard keeps a malformed workflow variable from turning a retention cleanup +into a bucket-wide delete. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from collections.abc import Iterable, Sequence +from pathlib import PurePosixPath + + +ALLOWED_RELEASE_ROOTS = frozenset( + { + "gonavi/releases/download/", + "gonavi/dev/releases/download/", + "drivers/releases/download/", + "drivers/dev/releases/download/", + } +) + +DRIVER_CHANNELS = { + "stable": { + "root": "drivers/releases/download/", + "pointer": "drivers/releases/latest/GoNavi-DriverAgents-Index.json", + }, + "dev": { + "root": "drivers/dev/releases/download/", + "pointer": "drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json", + }, +} +DRIVER_INDEX_NAME = "GoNavi-DriverAgents-Index.json" +MAX_DRIVER_INDEX_BYTES = 1 << 20 + + +def normalize_prefix(value: str) -> str: + prefix = (value or "").strip().replace("\\", "/").lstrip("/") + if not prefix.endswith("/"): + prefix += "/" + segments = prefix.rstrip("/").split("/") + if not prefix or any(segment in {"", ".", ".."} for segment in segments): + raise ValueError(f"invalid R2 prefix: {value!r}") + return prefix + + +def validate_prune_scope(root_prefix: str, keep_prefix: str) -> tuple[str, str]: + root = normalize_prefix(root_prefix) + keep = normalize_prefix(keep_prefix) + if root not in ALLOWED_RELEASE_ROOTS: + raise ValueError(f"refusing unsupported R2 release root: {root}") + if keep == root or not keep.startswith(root): + raise ValueError(f"keep prefix {keep!r} must be a child of {root!r}") + return root, keep + + +def select_obsolete_keys(keys: Iterable[str], root_prefix: str, keep_prefix: str) -> list[str]: + root, keep = validate_prune_scope(root_prefix, keep_prefix) + return sorted( + key + for key in keys + if key.startswith(root) and not key.startswith(keep) + ) + + +def chunks(values: Sequence[str], size: int = 1000) -> Iterable[list[str]]: + if size <= 0: + raise ValueError("chunk size must be positive") + for start in range(0, len(values), size): + yield list(values[start : start + size]) + + +def calculate_retained_bytes( + objects_by_key: dict[str, int], + exclude_prefixes: Sequence[str] = (), + add_bytes: int = 0, +) -> int: + if add_bytes < 0: + raise ValueError("added bytes must not be negative") + excluded = tuple(normalize_prefix(prefix) for prefix in exclude_prefixes) + retained = sum( + size + for key, size in objects_by_key.items() + if not any(key.startswith(prefix) for prefix in excluded) + ) + return retained + add_bytes + + +def run_aws(arguments: list[str]) -> dict: + proc = subprocess.run( + ["aws", *arguments, "--no-cli-pager"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if proc.returncode != 0: + detail = proc.stderr.strip() or proc.stdout.strip() or f"exit {proc.returncode}" + raise RuntimeError(f"AWS CLI failed: {detail}") + if not proc.stdout.strip(): + return {} + payload = json.loads(proc.stdout) + if not isinstance(payload, dict): + raise RuntimeError("AWS CLI returned a non-object JSON response") + return payload + + +def get_object_json(bucket: str, endpoint_url: str, key: str) -> dict: + handle, path = tempfile.mkstemp(prefix="gonavi-r2-index-", suffix=".json") + os.close(handle) + os.unlink(path) + try: + run_aws( + [ + "s3api", + "get-object", + "--bucket", + bucket, + "--key", + key, + "--endpoint-url", + endpoint_url, + path, + "--output", + "json", + ] + ) + if os.path.getsize(path) > MAX_DRIVER_INDEX_BYTES: + raise ValueError(f"R2 driver index is too large: {key}") + with open(path, encoding="utf-8") as stream: + payload = json.load(stream) + if not isinstance(payload, dict): + raise ValueError(f"R2 driver index is not a JSON object: {key}") + return payload + finally: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def list_objects(bucket: str, endpoint_url: str, prefix: str) -> list[dict]: + objects: list[dict] = [] + continuation_token = "" + while True: + args = [ + "s3api", + "list-objects-v2", + "--bucket", + bucket, + "--prefix", + prefix, + "--max-keys", + "1000", + "--endpoint-url", + endpoint_url, + "--output", + "json", + ] + if continuation_token: + args.extend(["--continuation-token", continuation_token]) + payload = run_aws(args) + for item in payload.get("Contents") or []: + if not isinstance(item, dict): + continue + key = str(item.get("Key") or "") + if key.startswith(prefix): + objects.append({"Key": key, "Size": int(item.get("Size") or 0)}) + if not payload.get("IsTruncated"): + break + continuation_token = str(payload.get("NextContinuationToken") or "") + if not continuation_token: + raise RuntimeError("R2 listing was truncated without a continuation token") + return objects + + +def delete_objects(bucket: str, endpoint_url: str, keys: Sequence[str]) -> None: + for batch in chunks(keys): + payload = run_aws( + [ + "s3api", + "delete-objects", + "--bucket", + bucket, + "--delete", + json.dumps( + {"Objects": [{"Key": key} for key in batch], "Quiet": True}, + separators=(",", ":"), + ), + "--endpoint-url", + endpoint_url, + "--output", + "json", + ] + ) + errors = payload.get("Errors") or [] + if errors: + raise RuntimeError(f"R2 delete returned errors: {json.dumps(errors)}") + + +def validate_driver_pointer_payloads( + channel: str, + pointer: dict, + versioned: dict, + objects_by_key: dict[str, int], +) -> dict: + config = DRIVER_CHANNELS.get(channel) + if config is None: + raise ValueError(f"unsupported driver channel: {channel}") + logical_tag = str(pointer.get("tagName") or "").strip() + if channel == "stable": + if not re.fullmatch(r"v\d+\.\d+\.\d+", logical_tag): + raise ValueError("stable driver pointer has an invalid tagName") + physical_tag = logical_tag + else: + if logical_tag != "dev-latest": + raise ValueError("dev driver pointer tagName must be dev-latest") + physical_tag = str(pointer.get("mirrorTagName") or "").strip() + if not re.fullmatch(r"dev-[0-9a-f]{7,40}", physical_tag): + raise ValueError("dev driver pointer has an invalid mirrorTagName") + + pointer_assets = pointer.get("assets") + versioned_assets = versioned.get("assets") + if not isinstance(pointer_assets, dict) or not pointer_assets: + raise ValueError("driver pointer has no assets") + if versioned_assets != pointer_assets: + raise ValueError("versioned driver index does not match the mutable pointer") + + root = str(config["root"]) + keep_prefix = f"{root}{physical_tag}/" + index_key = keep_prefix + DRIVER_INDEX_NAME + if objects_by_key.get(index_key, 0) <= 0: + raise ValueError("versioned driver index is missing from the retained prefix") + for name, size in pointer_assets.items(): + if not isinstance(name, str) or not name or PurePosixPath(name).name != name: + raise ValueError(f"invalid driver asset name in pointer: {name!r}") + if type(size) is not int or size <= 0: + raise ValueError(f"invalid driver asset size in pointer: {name!r}") + if objects_by_key.get(keep_prefix + name) != size: + raise ValueError(f"driver asset is missing or has the wrong size: {name}") + return { + "channel": channel, + "logicalTag": logical_tag, + "physicalTag": physical_tag, + "keepPrefix": keep_prefix, + "assetCount": len(pointer_assets), + } + + +def command_prune(args: argparse.Namespace) -> int: + root, keep = validate_prune_scope(args.root_prefix, args.keep_prefix) + objects = list_objects(args.bucket, args.endpoint_url, root) + obsolete = select_obsolete_keys((str(item["Key"]) for item in objects), root, keep) + obsolete_set = set(obsolete) + reclaimed_bytes = sum( + int(item["Size"]) + for item in objects + if str(item["Key"]) in obsolete_set + ) + if obsolete and not args.dry_run: + delete_objects(args.bucket, args.endpoint_url, obsolete) + remaining = list_objects(args.bucket, args.endpoint_url, root) + still_obsolete = select_obsolete_keys( + (str(item["Key"]) for item in remaining), + root, + keep, + ) + if still_obsolete: + raise RuntimeError( + f"R2 retention verification found {len(still_obsolete)} obsolete objects" + ) + print( + json.dumps( + { + "rootPrefix": root, + "keepPrefix": keep, + "deletedObjects": len(obsolete), + "reclaimedBytes": reclaimed_bytes, + "dryRun": bool(args.dry_run), + }, + separators=(",", ":"), + ) + ) + return 0 + + +def command_measure(args: argparse.Namespace) -> int: + prefixes = [normalize_prefix(prefix) for prefix in args.prefix] + exclude_prefixes = [normalize_prefix(prefix) for prefix in args.exclude_prefix] + objects_by_key: dict[str, int] = {} + for prefix in prefixes: + for item in list_objects(args.bucket, args.endpoint_url, prefix): + objects_by_key[str(item["Key"])] = int(item["Size"]) + retained_bytes = calculate_retained_bytes( + objects_by_key, + exclude_prefixes, + args.add_bytes, + ) + print( + json.dumps( + { + "prefixes": prefixes, + "excludedPrefixes": exclude_prefixes, + "retainedObjects": len(objects_by_key), + "retainedBytes": retained_bytes, + "addedBytes": args.add_bytes, + "maxBytes": args.max_bytes, + }, + separators=(",", ":"), + ) + ) + if args.max_bytes is not None and retained_bytes > args.max_bytes: + print( + f"retained R2 release data {retained_bytes} exceeds budget {args.max_bytes}", + file=sys.stderr, + ) + return 3 + return 0 + + +def command_validate_driver_pointer(args: argparse.Namespace) -> int: + config = DRIVER_CHANNELS[args.channel] + root = str(config["root"]) + pointer = get_object_json(args.bucket, args.endpoint_url, str(config["pointer"])) + if args.channel == "stable": + physical_tag = str(pointer.get("tagName") or "").strip() + else: + physical_tag = str(pointer.get("mirrorTagName") or "").strip() + # Validate the tag before constructing a key from untrusted pointer data. + if args.channel == "stable" and not re.fullmatch(r"v\d+\.\d+\.\d+", physical_tag): + raise ValueError("stable driver pointer has an invalid tagName") + if args.channel == "dev" and not re.fullmatch(r"dev-[0-9a-f]{7,40}", physical_tag): + raise ValueError("dev driver pointer has an invalid mirrorTagName") + keep_prefix = f"{root}{physical_tag}/" + versioned = get_object_json( + args.bucket, + args.endpoint_url, + keep_prefix + DRIVER_INDEX_NAME, + ) + objects = list_objects(args.bucket, args.endpoint_url, keep_prefix) + result = validate_driver_pointer_payloads( + args.channel, + pointer, + versioned, + {str(item["Key"]): int(item["Size"]) for item in objects}, + ) + print(json.dumps(result, separators=(",", ":"))) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--bucket", required=True) + common.add_argument("--endpoint-url", required=True) + + prune = subparsers.add_parser("prune", parents=[common]) + prune.add_argument("--root-prefix", required=True) + prune.add_argument("--keep-prefix", required=True) + prune.add_argument("--dry-run", action="store_true") + prune.set_defaults(handler=command_prune) + + measure = subparsers.add_parser("measure", parents=[common]) + measure.add_argument("--prefix", action="append", required=True) + measure.add_argument("--exclude-prefix", action="append", default=[]) + measure.add_argument("--add-bytes", type=int, default=0) + measure.add_argument("--max-bytes", type=int) + measure.set_defaults(handler=command_measure) + + validate = subparsers.add_parser("validate-driver-pointer", parents=[common]) + validate.add_argument("--channel", choices=tuple(DRIVER_CHANNELS), required=True) + validate.set_defaults(handler=command_validate_driver_pointer) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + return int(args.handler(args)) + except (RuntimeError, ValueError, OSError, json.JSONDecodeError) as error: + print(str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/r2-release-retention.test.py b/tools/r2-release-retention.test.py new file mode 100644 index 00000000..19cf17a7 --- /dev/null +++ b/tools/r2-release-retention.test.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "tools" / "r2-release-retention.py" +SPEC = importlib.util.spec_from_file_location("r2_release_retention", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class R2ReleaseRetentionTest(unittest.TestCase): + def test_selects_only_obsolete_keys_inside_the_requested_release_root(self) -> None: + keys = [ + "gonavi/releases/download/v0.8.6/GoNavi.exe", + "gonavi/releases/download/v0.8.7/GoNavi.exe", + "gonavi/releases/latest/latest.json", + "gonavi/dev/releases/download/dev-abc/GoNavi.exe", + ] + + self.assertEqual( + MODULE.select_obsolete_keys( + keys, + "gonavi/releases/download/", + "gonavi/releases/download/v0.8.7/", + ), + ["gonavi/releases/download/v0.8.6/GoNavi.exe"], + ) + + def test_rejects_bucket_wide_or_unknown_prune_roots(self) -> None: + for root in ("", "/", "gonavi/", "unrelated/releases/download/"): + with self.subTest(root=root): + with self.assertRaises(ValueError): + MODULE.validate_prune_scope(root, "gonavi/releases/download/v0.8.7/") + + def test_rejects_keep_prefix_outside_or_equal_to_root(self) -> None: + root = "drivers/dev/releases/download/" + for keep in (root, "drivers/releases/download/v0.8.7/"): + with self.subTest(keep=keep): + with self.assertRaises(ValueError): + MODULE.validate_prune_scope(root, keep) + + def test_chunks_deletes_at_most_one_thousand_objects_per_request(self) -> None: + values = [f"key-{index}" for index in range(2001)] + batches = list(MODULE.chunks(values)) + + self.assertEqual([len(batch) for batch in batches], [1000, 1000, 1]) + self.assertEqual([item for batch in batches for item in batch], values) + + def test_projected_usage_replaces_the_current_channel_before_adding_candidate(self) -> None: + objects = { + "gonavi/releases/download/v0.8.7/app.exe": 250, + "drivers/releases/download/v0.8.7/driver.exe": 3_500, + "gonavi/dev/releases/download/dev-old/app.exe": 300, + "drivers/dev/releases/download/dev-old/driver.exe": 3_600, + } + + retained = MODULE.calculate_retained_bytes( + objects, + [ + "gonavi/dev/releases/download/", + "drivers/dev/releases/download/", + ], + add_bytes=4_000, + ) + + self.assertEqual(retained, 7_750) + + def test_validates_stable_driver_pointer_before_pruning(self) -> None: + asset_name = "sqlserver-driver-agent-windows-amd64.exe" + pointer = {"tagName": "v0.8.7", "assets": {asset_name: 123}} + prefix = "drivers/releases/download/v0.8.7/" + objects = { + prefix + MODULE.DRIVER_INDEX_NAME: 456, + prefix + asset_name: 123, + } + + result = MODULE.validate_driver_pointer_payloads("stable", pointer, pointer, objects) + + self.assertEqual(result["keepPrefix"], prefix) + self.assertEqual(result["assetCount"], 1) + + def test_rejects_incomplete_driver_pointer_target_before_pruning(self) -> None: + asset_name = "sqlserver-driver-agent-windows-amd64.exe" + pointer = { + "tagName": "dev-latest", + "mirrorTagName": "dev-a1b2c3d", + "assets": {asset_name: 123}, + } + prefix = "drivers/dev/releases/download/dev-a1b2c3d/" + objects = {prefix + MODULE.DRIVER_INDEX_NAME: 456} + + with self.assertRaisesRegex(ValueError, "missing or has the wrong size"): + MODULE.validate_driver_pointer_payloads("dev", pointer, pointer, objects) + + def test_workflows_keep_stable_and_dev_roots_isolated(self) -> None: + stable = (ROOT / ".github" / "workflows" / "publish-release.yml").read_text(encoding="utf-8") + dev = (ROOT / ".github" / "workflows" / "dev-build.yml").read_text(encoding="utf-8") + + self.assertIn('--root-prefix "gonavi/releases/download/"', stable) + self.assertIn('--root-prefix "drivers/releases/download/"', stable) + self.assertNotIn('--root-prefix "gonavi/dev/releases/download/"', stable) + self.assertIn('validate-driver-pointer', stable) + self.assertIn('r2-current-stable-driver-state.json', stable) + self.assertIn('--root-prefix "gonavi/dev/releases/download/"', dev) + self.assertIn('--root-prefix "drivers/dev/releases/download/"', dev) + self.assertIn('validate-driver-pointer', dev) + self.assertIn('r2-current-dev-driver-state.json', dev) + + def test_workflows_exclude_the_duplicate_driver_bundle_and_enforce_budget(self) -> None: + stable = (ROOT / ".github" / "workflows" / "publish-release.yml").read_text(encoding="utf-8") + dev = (ROOT / ".github" / "workflows" / "dev-build.yml").read_text(encoding="utf-8") + + self.assertIn(".assets | keys[]", stable) + self.assertNotIn('.assets["GoNavi-DriverAgents.zip"] =', stable) + self.assertIn('del(.assets["GoNavi-DriverAgents.zip"])', dev) + for workflow in (stable, dev): + self.assertIn("--max-bytes 9000000000", workflow) + self.assertIn("--exclude-prefix", workflow) + self.assertIn("--add-bytes", workflow) + + def test_workflows_serialize_stable_and_dev_r2_publication(self) -> None: + stable = (ROOT / ".github" / "workflows" / "publish-release.yml").read_text(encoding="utf-8") + dev = (ROOT / ".github" / "workflows" / "dev-build.yml").read_text(encoding="utf-8") + + shared_group = "group: gonavi-r2-publication" + self.assertEqual(stable.count(shared_group), 1) + self.assertEqual(dev.count(shared_group), 1) + + +if __name__ == "__main__": + unittest.main()