diff --git a/.github/actions/publish-vps-mirror/action.yml b/.github/actions/publish-vps-mirror/action.yml index 424a3fdb..aefe7504 100644 --- a/.github/actions/publish-vps-mirror/action.yml +++ b/.github/actions/publish-vps-mirror/action.yml @@ -14,6 +14,9 @@ inputs: app-manifest: description: Application latest manifest path required: true + driver-enabled: + description: Whether this deployment must publish driver assets + required: true driver-tag: description: Physical driver release directory name required: false @@ -56,6 +59,7 @@ runs: MIRROR_APP_TAG: ${{ inputs.app-tag }} MIRROR_APP_DIR: ${{ inputs.app-dir }} MIRROR_APP_MANIFEST: ${{ inputs.app-manifest }} + MIRROR_DRIVER_ENABLED: ${{ inputs.driver-enabled }} MIRROR_DRIVER_TAG: ${{ inputs.driver-tag }} MIRROR_DRIVER_DIR: ${{ inputs.driver-dir }} MIRROR_DRIVER_VERSION_INDEX: ${{ inputs.driver-version-index }} @@ -68,11 +72,36 @@ runs: run: | set -euo pipefail mirror_root="/srv/gonavi-downloads" + [[ "${MIRROR_CHANNEL}" == "stable" || "${MIRROR_CHANNEL}" == "dev" ]] || { + echo "Invalid mirror channel: ${MIRROR_CHANNEL}" >&2 + exit 1 + } + [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[0-9]+$ ]] || { + echo "Invalid GitHub run identity" >&2 + exit 1 + } deployment_id="${MIRROR_CHANNEL}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" stage_dir="${RUNNER_TEMP}/gonavi-vps-mirror-${deployment_id}" ssh_dir="${RUNNER_TEMP}/gonavi-vps-ssh-${deployment_id}" remote_stage="${mirror_root}/.incoming/${deployment_id}" effective_driver_tag="" + remote="" + remote_cleanup_enabled=false + ssh_options=() + + cleanup() { + local exit_code=$? + local cleanup_remote_command="" + trap - EXIT + set +e + if [[ "${remote_cleanup_enabled}" == true && -n "${remote}" ]]; then + printf -v cleanup_remote_command 'rm -rf -- %q' "${remote_stage}" + ssh "${ssh_options[@]}" "${remote}" "${cleanup_remote_command}" >/dev/null 2>&1 || true + fi + rm -rf -- "${ssh_dir}" "${stage_dir}" + exit "${exit_code}" + } + trap cleanup EXIT [[ "${MIRROR_SSH_PORT}" =~ ^[0-9]+$ ]] || { echo "Invalid mirror SSH port" >&2 @@ -92,15 +121,44 @@ runs: --app-manifest "${MIRROR_APP_MANIFEST}" --output "${stage_dir}" ) - if [[ -n "${MIRROR_DRIVER_LATEST_INDEX}" && -s "${MIRROR_DRIVER_LATEST_INDEX}" ]]; then - effective_driver_tag="${MIRROR_DRIVER_TAG}" - prepare_args+=( - --driver-tag "${effective_driver_tag}" - --driver-dir "${MIRROR_DRIVER_DIR}" - --driver-version-index "${MIRROR_DRIVER_VERSION_INDEX}" - --driver-latest-index "${MIRROR_DRIVER_LATEST_INDEX}" - ) - fi + case "${MIRROR_DRIVER_ENABLED}" in + true) + [[ -n "${MIRROR_DRIVER_TAG}" ]] || { + echo "Driver mirror publication is enabled but driver-tag is empty" >&2 + exit 1 + } + [[ -n "${MIRROR_DRIVER_DIR}" && -d "${MIRROR_DRIVER_DIR}" ]] || { + echo "Driver mirror publication is enabled but driver-dir is missing" >&2 + exit 1 + } + [[ -n "$(find "${MIRROR_DRIVER_DIR}" -maxdepth 1 -type f -print -quit)" ]] || { + echo "Driver mirror publication is enabled but driver-dir is empty" >&2 + exit 1 + } + [[ -n "${MIRROR_DRIVER_VERSION_INDEX}" && -s "${MIRROR_DRIVER_VERSION_INDEX}" ]] || { + echo "Driver mirror publication is enabled but driver-version-index is missing or empty" >&2 + exit 1 + } + [[ -n "${MIRROR_DRIVER_LATEST_INDEX}" && -s "${MIRROR_DRIVER_LATEST_INDEX}" ]] || { + echo "Driver mirror publication is enabled but driver-latest-index is missing or empty" >&2 + exit 1 + } + effective_driver_tag="${MIRROR_DRIVER_TAG}" + prepare_args+=( + --driver-tag "${effective_driver_tag}" + --driver-dir "${MIRROR_DRIVER_DIR}" + --driver-version-index "${MIRROR_DRIVER_VERSION_INDEX}" + --driver-latest-index "${MIRROR_DRIVER_LATEST_INDEX}" + ) + ;; + false) + echo "Driver mirror publication explicitly disabled" + ;; + *) + echo "Invalid driver-enabled value: ${MIRROR_DRIVER_ENABLED}" >&2 + exit 1 + ;; + esac python3 "${GITHUB_WORKSPACE}/tools/prepare-vps-release-payload.py" "${prepare_args[@]}" cp "${GITHUB_WORKSPACE}/tools/vps-release-commit.sh" "${stage_dir}/vps-release-commit.sh" chmod 0755 "${stage_dir}/vps-release-commit.sh" @@ -114,6 +172,17 @@ runs: -o "UserKnownHostsFile=${ssh_dir}/known_hosts" ) remote="${MIRROR_SSH_USER}@${MIRROR_SSH_HOST}" + printf -v stale_cleanup_script '%s\n' \ + 'set -euo pipefail' \ + "root=$(printf '%q' "${mirror_root}")" \ + 'marker="$(cat "${root}/.gonavi-mirror-root" 2>/dev/null || true)"' \ + '[[ "${marker}" == "gonavi-download-mirror-v1" ]] || { echo "mirror root marker is missing or invalid" >&2; exit 1; }' \ + 'incoming="${root}/.incoming"' \ + 'mkdir -p "${incoming}"' \ + 'find "${incoming}" -mindepth 1 -maxdepth 1 -type d -mmin +1440 -exec rm -rf -- {} +' + printf -v stale_cleanup_command 'bash -c %q' "${stale_cleanup_script}" + ssh "${ssh_options[@]}" "${remote}" "${stale_cleanup_command}" + payload_bytes="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["payloadBytes"])' "${stage_dir}/deployment.json")" available_kib="$(ssh "${ssh_options[@]}" "${remote}" "LC_ALL=C df -Pk '${mirror_root}' | awk 'NR == 2 { print \$4 }'")" [[ "${available_kib}" =~ ^[0-9]+$ ]] || { @@ -127,6 +196,7 @@ runs: exit 1 fi + remote_cleanup_enabled=true ssh "${ssh_options[@]}" "${remote}" "mkdir -p '${remote_stage}'" rsync -rlt --delete --partial \ --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r \ @@ -141,5 +211,3 @@ runs: "${MIRROR_APP_TAG}" \ "${effective_driver_tag}" ssh "${ssh_options[@]}" "${remote}" "${remote_command}" - - rm -rf "${ssh_dir}" "${stage_dir}" diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index fa19f588..ebc7d9e7 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -1492,6 +1492,7 @@ jobs: app-tag: ${{ steps.version.outputs.version }} app-dir: release-assets app-manifest: release-assets/latest-dev.json + driver-enabled: ${{ steps.driver_assets.outputs.has_driver_assets }} driver-tag: ${{ steps.version.outputs.version }} driver-dir: driver-release-assets driver-version-index: ${{ runner.temp }}/driver-dev-latest-index.json diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 018f01f5..3d2a268e 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -159,6 +159,7 @@ jobs: uses: actions/checkout@v5 - name: Prepare and verify stable mirror payload + id: mirror_payload env: GH_TOKEN: ${{ github.token }} DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }} @@ -201,6 +202,7 @@ jobs: exit 1 ;; esac + echo "has_driver_release=${has_driver_release}" >> "$GITHUB_OUTPUT" jq -e \ --arg tag "${RELEASE_TAG}" \ @@ -325,6 +327,7 @@ jobs: app-tag: ${{ steps.validate.outputs.tag }} app-dir: ${{ runner.temp }}/gonavi-release app-manifest: ${{ runner.temp }}/gonavi-release/latest.json + driver-enabled: ${{ steps.mirror_payload.outputs.has_driver_release }} driver-tag: ${{ steps.validate.outputs.tag }} driver-dir: ${{ runner.temp }}/driver-release driver-version-index: ${{ runner.temp }}/driver-release/GoNavi-DriverAgents-Index.json diff --git a/internal/app/methods_driver.go b/internal/app/methods_driver.go index 763e5f9d..127df468 100644 --- a/internal/app/methods_driver.go +++ b/internal/app/methods_driver.go @@ -364,7 +364,6 @@ const ( driverReleaseLatestAPIURL = "https://api.github.com/repos/" + driverReleaseRepo + "/releases/latest" driverReleaseDevTag = "dev-latest" optionalDriverBundleAssetName = "GoNavi-DriverAgents.zip" - duckDBWindowsDriverZipAssetName = "duckdb-driver.zip" optionalDriverBundleIndexAssetName = "GoNavi-DriverAgents-Index.json" optionalDriverBundleDownloadTimeout = 15 * time.Minute optionalDriverBundleCacheMaxAge = 7 * 24 * time.Hour @@ -1160,7 +1159,7 @@ func (a *App) GetDriverVersionPackageSize(driverType string, version string) con if err := a.localizeDriverSelectionError(definition, validateDriverSelectedVersion(definition, normalizedVersion)); err != nil { return connection.QueryResult{Success: false, Message: err.Error()} } - assetName := optionalDriverReleaseAssetNameForVersion(normalizedType, normalizedVersion) + assetName := optionalDriverReleaseZipAssetNameForVersion(normalizedType, normalizedVersion) if strings.TrimSpace(assetName) == "" { return connection.QueryResult{Success: false, Message: a.appText("driver_manager.backend.error.asset_name_empty", nil)} } @@ -2372,28 +2371,7 @@ func resolvePublishedDriverDownloadURLForTag(definition driverDefinition, select } func resolvePublishedDriverReleaseAssetName(driverType string, version string, tag string) (string, bool) { - if shouldUseDuckDBWindowsDynamicLibrary(driverType) { - cacheKey := "tag:" + strings.TrimSpace(tag) - if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache(cacheKey); ok { - if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 { - return duckDBWindowsDriverZipAssetName, true - } - return "", false - } - - sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached(cacheKey, func() (*githubRelease, error) { - return fetchReleaseByTag(tag) - }) - if err != nil { - return "", false - } - if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 { - return duckDBWindowsDriverZipAssetName, true - } - return "", false - } - - assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version) + assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version) if len(assetNames) == 0 { return "", false } @@ -2435,7 +2413,7 @@ func resolveDriverVersionPackageSizeBytes(definition driverDefinition, option dr if version == "" { return 0 } - assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version) + assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version) if len(assetNames) == 0 { return 0 } @@ -2767,8 +2745,8 @@ func resolveDriverVersionOptionsFromReleases(definition driverDefinition) []driv if tag == "" || version == "" { continue } - assetName := optionalDriverReleaseAssetNameForVersion(driverType, version) - assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version) + assetName := optionalDriverReleaseZipAssetNameForVersion(driverType, version) + assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version) if !releaseContainsAnyAsset(release, assetNames) { continue } @@ -4154,12 +4132,13 @@ func ensureOptionalDriverAgentBinary(a *App, definition driverDefinition, execut } if !forceSourceBuild { + downloadURLs = reorderOptionalDriverDownloadURLsBySpeed(downloadURLs) if len(downloadURLs) > 0 { for _, candidateURL := range downloadURLs { if a != nil { a.emitDriverDownloadProgress(driverType, "downloading", 20, 100, a.appText("driver_manager.progress.download_prebuilt_agent", map[string]any{"name": displayName})) } - hash, dlErr := downloadOptionalDriverAgentBinary(a, definition, candidateURL, executablePath) + hash, dlErr := downloadOptionalDriverAgentBinary(a, definition, candidateURL, executablePath, selectedVersion) if dlErr == nil { if revisionErr := validateCandidateRevision(); revisionErr != nil { logger.Warnf("预编译 %s 驱动代理 revision 校验失败，url=%s err=%v", displayName, candidateURL, revisionErr) @@ -4264,13 +4243,10 @@ func formatOptionalDriverAttemptError(a *App, source string, err error) string { } func shouldUseOptionalDriverBundleFallback(driverType string, restrictToExplicitArtifact bool, directURLCount int) bool { - if restrictToExplicitArtifact { - return false - } - if shouldSkipDirectOptionalDriverDownloads(driverType) { - return true - } - return directURLCount == 0 + _ = driverType + _ = restrictToExplicitArtifact + _ = directURLCount + return false } func isOptionalDriverDownloadZipURL(urlText string) bool { @@ -4290,7 +4266,7 @@ func isOptionalDriverDownloadZipURL(urlText string) bool { return strings.EqualFold(filepath.Ext(trimmedURL), ".zip") } -func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlText string, executablePath string) (string, error) { +func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlText string, executablePath string, selectedVersion string) (string, error) { driverType := normalizeDriverType(definition.Type) displayName := resolveDriverDisplayName(definition) trimmedURL := strings.TrimSpace(urlText) @@ -4312,7 +4288,7 @@ func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlT return "", newLocalizedDriverBackendError("driver_manager.backend.error.download_failed", nil, err) } - if _, err := installOptionalDriverAgentFromLocalZip(tempPath, definition, executablePath, ""); err != nil { + if _, err := installOptionalDriverAgentFromLocalZip(tempPath, definition, executablePath, selectedVersion); err != nil { _ = os.Remove(tempPath) _ = os.Remove(executablePath) for _, supportName := range optionalDriverSupportFileNames(driverType) { diff --git a/internal/app/methods_driver_assets.go b/internal/app/methods_driver_assets.go index 40053c22..8dec1cd4 100644 --- a/internal/app/methods_driver_assets.go +++ b/internal/app/methods_driver_assets.go @@ -2,6 +2,7 @@ package app import ( "archive/zip" + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -21,6 +22,22 @@ import ( "GoNavi-Wails/internal/logger" ) +const ( + optionalDriverDownloadProbeBytes = 256 << 10 + optionalDriverDownloadProbeMinBytes = 64 << 10 + optionalDriverDownloadProbeTimeout = 4 * time.Second + optionalDriverDownloadProbeSpeedRatio = 1.5 +) + +type optionalDriverDownloadProbeResult struct { + URL string + Bytes int64 + Duration time.Duration + OK bool +} + +type optionalDriverDownloadProbeFunc func(context.Context, *http.Client, string) optionalDriverDownloadProbeResult + func optionalDriverPublicTypeName(driverType string) string { switch normalizeDriverType(driverType) { case "diros": @@ -54,6 +71,17 @@ func optionalDriverReleaseAssetNameForType(typeName string, goos string, goarch return name } +func optionalDriverReleaseZipAssetName(assetName string) string { + name := strings.TrimSpace(assetName) + if name == "" { + return "" + } + if strings.EqualFold(filepath.Ext(name), ".exe") { + name = name[:len(name)-len(filepath.Ext(name))] + } + return name + ".zip" +} + func optionalDriverNameStemCandidates(driverType string, selectedVersion string) []string { candidates := make([]string, 0, 3) seen := make(map[string]struct{}, 3) @@ -147,6 +175,36 @@ func optionalDriverReleaseAssetNames(driverType string) []string { return optionalDriverReleaseAssetNamesForVersion(driverType, "") } +func optionalDriverReleaseZipAssetNamesForVersion(driverType string, selectedVersion string) []string { + rawNames := optionalDriverReleaseAssetNamesForVersion(driverType, selectedVersion) + names := make([]string, 0, len(rawNames)) + seen := make(map[string]struct{}, len(rawNames)) + for _, rawName := range rawNames { + name := optionalDriverReleaseZipAssetName(rawName) + if name == "" { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + return names +} + +func optionalDriverReleaseZipAssetNames(driverType string) []string { + return optionalDriverReleaseZipAssetNamesForVersion(driverType, "") +} + +func optionalDriverReleaseZipAssetNameForVersion(driverType string, selectedVersion string) string { + names := optionalDriverReleaseZipAssetNamesForVersion(driverType, selectedVersion) + if len(names) == 0 { + return optionalDriverReleaseZipAssetName(optionalDriverReleaseAssetNameForType("", stdRuntime.GOOS, stdRuntime.GOARCH)) + } + return names[0] +} + func optionalDriverExecutableBaseName(driverType string) string { names := optionalDriverExecutableBaseNames(driverType) if len(names) == 0 { @@ -393,7 +451,7 @@ func resolveOptionalDriverAssetSize(sizeByAsset map[string]int64, driverType str if len(sizeByAsset) == 0 { return 0 } - for _, assetName := range optionalDriverReleaseAssetNames(driverType) { + for _, assetName := range optionalDriverReleaseZipAssetNames(driverType) { sizeBytes := sizeByAsset[assetName] if sizeBytes > 0 { return sizeBytes @@ -406,7 +464,7 @@ func resolveOptionalDriverAssetSizeForVersion(sizeByAsset map[string]int64, driv if len(sizeByAsset) == 0 { return 0 } - for _, assetName := range optionalDriverReleaseAssetNamesForVersion(driverType, version) { + for _, assetName := range optionalDriverReleaseZipAssetNamesForVersion(driverType, version) { sizeBytes := sizeByAsset[assetName] if sizeBytes > 0 { return sizeBytes @@ -634,13 +692,131 @@ func acquireOptionalDriverBundlePath(bundleURL string, onProgress func(downloade } } +func optionalDriverDownloadSource(rawURL string) string { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return "" + } + switch strings.ToLower(strings.TrimSpace(parsed.Hostname())) { + case "download.syngnat.top": + return "mirror" + case "github.com", "api.github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com": + return "github" + default: + return "" + } +} + +func probeOptionalDriverDownloadURL(ctx context.Context, client *http.Client, rawURL string) optionalDriverDownloadProbeResult { + result := optionalDriverDownloadProbeResult{URL: strings.TrimSpace(rawURL)} + if result.URL == "" || client == nil { + return result + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, result.URL, nil) + if err != nil { + return result + } + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", optionalDriverDownloadProbeBytes-1)) + applyGitHubDownloadRequestHeaders(req, isGitHubReleaseAssetAPIURL(result.URL)) + + startedAt := time.Now() + resp, err := doUpdateRequest(client, req) + if err != nil { + return result + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return result + } + + written, err := io.Copy(io.Discard, io.LimitReader(resp.Body, optionalDriverDownloadProbeBytes)) + result.Duration = time.Since(startedAt) + result.Bytes = written + result.OK = err == nil && written >= optionalDriverDownloadProbeMinBytes && result.Duration > 0 + return result +} + +func reorderOptionalDriverDownloadURLsBySpeedWithProbe(urls []string, probe optionalDriverDownloadProbeFunc) []string { + ordered := append([]string(nil), urls...) + if len(ordered) < 2 || probe == nil { + return ordered + } + + mirrorURL := "" + githubURL := "" + for _, candidate := range ordered { + if !isOptionalDriverDownloadZipURL(candidate) { + continue + } + switch optionalDriverDownloadSource(candidate) { + case "mirror": + if mirrorURL == "" { + mirrorURL = candidate + } + case "github": + if githubURL == "" { + githubURL = candidate + } + } + } + if mirrorURL == "" || githubURL == "" { + return ordered + } + + ctx, cancel := context.WithTimeout(context.Background(), optionalDriverDownloadProbeTimeout) + defer cancel() + client := newHTTPClientWithGlobalProxy(optionalDriverDownloadProbeTimeout) + results := make(chan optionalDriverDownloadProbeResult, 2) + for _, candidate := range []string{mirrorURL, githubURL} { + candidateURL := candidate + go func() { + results <- probe(ctx, client, candidateURL) + }() + } + + measured := make(map[string]optionalDriverDownloadProbeResult, 2) + for remaining := 2; remaining > 0; remaining-- { + select { + case result := <-results: + measured[result.URL] = result + case <-ctx.Done(): + remaining = 0 + } + } + + mirrorResult := measured[mirrorURL] + githubResult := measured[githubURL] + preferGitHub := githubResult.OK && !mirrorResult.OK + if githubResult.OK && mirrorResult.OK { + mirrorSpeed := float64(mirrorResult.Bytes) / mirrorResult.Duration.Seconds() + githubSpeed := float64(githubResult.Bytes) / githubResult.Duration.Seconds() + preferGitHub = githubSpeed >= mirrorSpeed*optionalDriverDownloadProbeSpeedRatio + } + if !preferGitHub { + return ordered + } + + reordered := make([]string, 0, len(ordered)) + reordered = append(reordered, githubURL) + for _, candidate := range ordered { + if candidate != githubURL { + reordered = append(reordered, candidate) + } + } + return reordered +} + +func reorderOptionalDriverDownloadURLsBySpeed(urls []string) []string { + return reorderOptionalDriverDownloadURLsBySpeedWithProbe(urls, probeOptionalDriverDownloadURL) +} + func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL string, selectedVersion string) []string { candidates := make([]string, 0, 6) seen := make(map[string]struct{}, 6) driverType := normalizeDriverType(definition.Type) appendURL := func(value string) { trimmed := strings.TrimSpace(value) - if trimmed == "" { + if trimmed == "" || !isOptionalDriverDownloadZipURL(trimmed) { return } if _, ok := seen[trimmed]; ok { @@ -689,7 +865,7 @@ func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL appendPublishedURLs() } - if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil { + if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil && isOptionalDriverDownloadZipURL(parsed.String()) { switch strings.ToLower(strings.TrimSpace(parsed.Scheme)) { case "http", "https": if tag, assetName, ok := driverReleaseDownloadCoordinates(parsed.String()); ok && @@ -1376,35 +1552,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti if driverType == "" { return "", false } - if shouldUseDuckDBWindowsDynamicLibrary(driverType) { - if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache("latest"); ok { - if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 { - if release, err := fetchLatestReleaseForDriverAssets(); err == nil { - if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found { - return driverReleaseAssetAPIURL(asset), true - } - } - return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true - } - return "", false - } - - sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached("latest", fetchLatestReleaseForDriverAssets) - if err != nil { - return "", false - } - if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 { - if release, relErr := fetchLatestReleaseForDriverAssets(); relErr == nil { - if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found { - return driverReleaseAssetAPIURL(asset), true - } - } - return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true - } - return "", false - } - - assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, selectedVersion) + assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, selectedVersion) if len(assetNames) == 0 { return "", false } @@ -1420,12 +1568,12 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true } } - return "", false + return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true } sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached("latest", fetchLatestReleaseForDriverAssets) if err != nil { - return "", false + return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true } for _, assetName := range assetNames { if publishedAssets[assetName] && sizeByAsset[assetName] > 0 { @@ -1437,7 +1585,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true } } - return "", false + return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true } func fetchReleaseByTag(tag string) (*githubRelease, error) { @@ -1484,7 +1632,7 @@ func fetchDriverReleaseIndexByURL(tag string, indexURL string) (*githubRelease, tagName = fallbackTag } if strings.EqualFold(fallbackTag, driverReleaseDevTag) { - // dev alias 的逻辑 GitHub 标签固定为 dev-latest；mirrorTagName 仅控制 R2 物理路径。 + // dev alias 的逻辑 GitHub 标签固定为 dev-latest；mirrorTagName 仅控制镜像物理路径。 tagName = driverReleaseDevTag } if tagName == "" { diff --git a/internal/app/methods_driver_version_test.go b/internal/app/methods_driver_version_test.go index 1936b92a..7c8a48be 100644 --- a/internal/app/methods_driver_version_test.go +++ b/internal/app/methods_driver_version_test.go @@ -2,6 +2,7 @@ package app import ( "archive/zip" + "context" "encoding/json" "fmt" "io" @@ -83,7 +84,7 @@ func TestResolveVersionedDriverOptionUsesPublishedMongoV1Release(t *testing.T) { } version := "1.17.4" - assetName := mongoVersionedReleaseAssetName(1) + assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1)) seedReleaseAssetSizeCache(t, "tag:v"+version, map[string]int64{ assetName: 24 << 20, }) @@ -170,6 +171,78 @@ func TestDriverReleaseLatestDownloadURLForCurrentChannelUsesDevLatest(t *testing } } +func TestOptionalDriverReleaseZipAssetNamesArePlatformNeutralArchives(t *testing.T) { + if got := optionalDriverReleaseZipAssetName("mariadb-driver-agent-windows-amd64.exe"); got != "mariadb-driver-agent-windows-amd64.zip" { + t.Fatalf("unexpected Windows ZIP asset name: %q", got) + } + if got := optionalDriverReleaseZipAssetName("mariadb-driver-agent-darwin-arm64"); got != "mariadb-driver-agent-darwin-arm64.zip" { + t.Fatalf("unexpected Darwin ZIP asset name: %q", got) + } + MongoNames := optionalDriverReleaseZipAssetNamesForVersion("mongodb", "1.17.9") + if len(MongoNames) != 1 || !strings.Contains(MongoNames[0], "mongodb-driver-agent-v1-") || !strings.HasSuffix(MongoNames[0], ".zip") { + t.Fatalf("expected MongoDB v1 to resolve one versioned ZIP, got %v", MongoNames) + } +} + +func TestReorderOptionalDriverDownloadURLsBySpeedPrefersClearlyFasterGitHub(t *testing.T) { + mirrorURL := driverMirrorReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip") + githubURL := driverReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip") + got := reorderOptionalDriverDownloadURLsBySpeedWithProbe( + []string{mirrorURL, githubURL}, + func(_ context.Context, _ *http.Client, rawURL string) optionalDriverDownloadProbeResult { + duration := 2 * time.Second + if optionalDriverDownloadSource(rawURL) == "github" { + duration = 500 * time.Millisecond + } + return optionalDriverDownloadProbeResult{URL: rawURL, Bytes: optionalDriverDownloadProbeBytes, Duration: duration, OK: true} + }, + ) + if len(got) != 2 || got[0] != githubURL || got[1] != mirrorURL { + t.Fatalf("expected faster GitHub ZIP first without dropping mirror fallback, got %v", got) + } +} + +func TestReorderOptionalDriverDownloadURLsBySpeedKeepsMirrorForSimilarOrFailedGitHub(t *testing.T) { + mirrorURL := driverMirrorReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip") + githubURL := driverReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip") + for name, githubResult := range map[string]optionalDriverDownloadProbeResult{ + "similar": {Bytes: optionalDriverDownloadProbeBytes, Duration: 900 * time.Millisecond, OK: true}, + "failed": {}, + } { + t.Run(name, func(t *testing.T) { + got := reorderOptionalDriverDownloadURLsBySpeedWithProbe( + []string{mirrorURL, githubURL}, + func(_ context.Context, _ *http.Client, rawURL string) optionalDriverDownloadProbeResult { + if optionalDriverDownloadSource(rawURL) == "github" { + githubResult.URL = rawURL + return githubResult + } + return optionalDriverDownloadProbeResult{URL: rawURL, Bytes: optionalDriverDownloadProbeBytes, Duration: time.Second, OK: true} + }, + ) + if len(got) != 2 || got[0] != mirrorURL || got[1] != githubURL { + t.Fatalf("expected mirror-first order to remain unchanged, got %v", got) + } + }) + } +} + +func TestProbeOptionalDriverDownloadURLUsesBoundedRange(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Range"); got != "bytes=0-262143" { + t.Errorf("unexpected Range header: %q", got) + } + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(make([]byte, optionalDriverDownloadProbeBytes+1024)) + })) + defer server.Close() + + result := probeOptionalDriverDownloadURL(context.Background(), server.Client(), server.URL+"/driver.zip") + if !result.OK || result.Bytes != optionalDriverDownloadProbeBytes { + t.Fatalf("expected a valid bounded probe, got %#v", result) + } +} + func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing.T) { originalVersion := AppVersion AppVersion = "0.7.4" @@ -202,7 +275,7 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing t.Fatalf("expected bundle URLs to include mirror=%q, tagged=%q and latest=%q, got %v", wantMirror, wantTagged, wantLatest, urls) } if urls[0] != wantMirror { - t.Fatalf("expected R2 mirror first, got %v", urls) + t.Fatalf("expected mirror first, got %v", urls) } } @@ -295,7 +368,7 @@ func TestFetchDriverReleaseIndexByURLBuildsMirrorAssets(t *testing.T) { func TestResolvePublishedDriverDownloadURLForTagUsesDevMirrorTag(t *testing.T) { definition := driverDefinition{Type: "sqlserver"} - assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "") + assetNames := optionalDriverReleaseZipAssetNamesForVersion(definition.Type, "") if len(assetNames) == 0 { t.Fatal("expected sqlserver release asset names") } @@ -325,10 +398,10 @@ func TestResolvePublishedDriverDownloadURLForTagUsesDevMirrorTag(t *testing.T) { func TestResolveLatestPublishedDriverDownloadURLFallsBackWhenMirrorIndexMissesAsset(t *testing.T) { seedReleaseAssetSizeCache(t, "latest", map[string]int64{ - "unrelated-driver-agent-windows-amd64.exe": 123, + "unrelated-driver-agent-windows-amd64.zip": 123, }) definition := driverDefinition{Type: "sqlserver"} - assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "") + assetNames := optionalDriverReleaseZipAssetNamesForVersion(definition.Type, "") if len(assetNames) == 0 { t.Fatal("expected sqlserver release asset names") } @@ -543,7 +616,7 @@ func TestResolveDriverVersionPackageSizeBytesReadsMongoV1VersionedAsset(t *testi } version := "1.17.4" - assetName := mongoVersionedReleaseAssetName(1) + assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1)) const wantSize int64 = 31 << 20 seedReleaseAssetSizeCache(t, "tag:v"+version, map[string]int64{ assetName: wantSize, @@ -564,7 +637,8 @@ func TestResolveOptionalDriverAgentDownloadURLsDoesNotFallbackForHistoricalVersi t.Fatal("expected mongodb driver definition") } - explicitURL := driverReleaseDownloadURL("v1.17.4", mongoVersionedReleaseAssetName(1)) + zipAssetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1)) + explicitURL := driverReleaseDownloadURL("v1.17.4", zipAssetName) urls := resolveOptionalDriverAgentDownloadURLs( definition, explicitURL, @@ -573,7 +647,7 @@ func TestResolveOptionalDriverAgentDownloadURLsDoesNotFallbackForHistoricalVersi if len(urls) != 2 { t.Fatalf("expected mirror plus explicit historical URL, got %d candidates: %v", len(urls), urls) } - if urls[0] != driverMirrorReleaseDownloadURL("v1.17.4", mongoVersionedReleaseAssetName(1)) || urls[1] != explicitURL { + if urls[0] != driverMirrorReleaseDownloadURL("v1.17.4", zipAssetName) || urls[1] != explicitURL { t.Fatalf("unexpected historical URL candidate: %v", urls) } } @@ -590,7 +664,7 @@ func TestResolveOptionalDriverAgentDownloadURLsUsesMongoV1AssetForCompatibleDefa AppVersion = originalVersion }) - assetName := mongoVersionedReleaseAssetName(1) + assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1)) seedReleaseAssetSizeCache(t, "tag:v0.7.9", map[string]int64{ assetName: 24 << 20, }) @@ -662,24 +736,29 @@ func TestMongoDBVersionedAssetNamesDoNotFallbackToBaseForV1(t *testing.T) { } } -func TestResolveOptionalDriverAgentDownloadURLsSkipsBundleOnlyDamengAsset(t *testing.T) { +func TestResolveOptionalDriverAgentDownloadURLsUsesDamengZipAsset(t *testing.T) { definition, ok := resolveDriverDefinition("dameng") if !ok { t.Fatal("expected dameng driver definition") } version := normalizeVersion(definition.PinnedVersion) - assetName := optionalDriverReleaseAssetNameForVersion("dameng", version) + assetName := optionalDriverReleaseZipAssetNameForVersion("dameng", version) seedReleaseAssetCacheEntry(t, "tag:v"+version, map[string]int64{ assetName: 23 << 20, - }, nil) + }, map[string]int64{assetName: 23 << 20}) seedReleaseAssetCacheEntry(t, "latest", map[string]int64{ assetName: 23 << 20, - }, nil) + }, map[string]int64{assetName: 23 << 20}) urls := resolveOptionalDriverAgentDownloadURLs(definition, "builtin://activate/dameng", version) - if len(urls) != 0 { - t.Fatalf("expected bundle-only dameng install to skip direct asset URLs, got %v", urls) + if len(urls) == 0 { + t.Fatal("expected Dameng install to use the published standalone zip") + } + for _, candidate := range urls { + if !isOptionalDriverDownloadZipURL(candidate) { + t.Fatalf("expected zip-only Dameng candidates, got %v", urls) + } } } @@ -689,9 +768,9 @@ func TestShouldUseOptionalDriverBundleFallbackSkipsWhenDirectAssetExists(t *test } } -func TestShouldUseOptionalDriverBundleFallbackKeepsBundleWhenDirectAssetMissing(t *testing.T) { - if !shouldUseOptionalDriverBundleFallback("dameng", false, 0) { - t.Fatal("expected missing single-file driver asset to keep bundle fallback") +func TestShouldUseOptionalDriverBundleFallbackStaysDisabledWhenZipMissing(t *testing.T) { + if shouldUseOptionalDriverBundleFallback("dameng", false, 0) { + t.Fatal("expected ZIP-only installs not to download the CI driver bundle") } if shouldUseOptionalDriverBundleFallback("dameng", true, 0) { t.Fatal("expected explicit version artifact installs to skip bundle fallback") @@ -1228,22 +1307,20 @@ func TestDuckDBWindowsBuildUsesDynamicLibraryTag(t *testing.T) { if !shouldSkipReusableAgentCandidate("duckdb", "") { t.Fatal("expected DuckDB Windows install to skip reusable static agent candidates") } + zipAssetName := optionalDriverReleaseZipAssetNameForVersion("duckdb", "") seedReleaseAssetCacheEntry(t, "latest", map[string]int64{ - duckDBWindowsDriverZipAssetName: 19 << 20, + zipAssetName: 19 << 20, }, map[string]int64{ - duckDBWindowsDriverZipAssetName: 19 << 20, + zipAssetName: 19 << 20, }) legacyDirectURL := "https://example.com/duckdb-driver-agent-windows-amd64.exe" urls := resolveOptionalDriverAgentDownloadURLs(driverDefinition{Type: "duckdb"}, legacyDirectURL, "") - if len(urls) < 2 { - t.Fatalf("expected DuckDB Windows install to keep dedicated zip ahead of legacy direct candidate, got %v", urls) + if len(urls) != 1 { + t.Fatalf("expected DuckDB Windows install to use only the dedicated zip, got %v", urls) } - if urls[0] != driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName) { + if urls[0] != driverReleaseLatestDownloadURL(zipAssetName) { t.Fatalf("expected DuckDB Windows dedicated zip candidate first, got %v", urls) } - if urls[1] != legacyDirectURL { - t.Fatalf("expected DuckDB Windows to keep legacy direct candidate after dedicated zip, got %v", urls) - } } func TestDuckDBWindowsDynamicLibraryCGOLDFlagsIncludeSupportLibraries(t *testing.T) { @@ -1328,7 +1405,8 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing. }) tmpDir := t.TempDir() - zipPath := filepath.Join(tmpDir, duckDBWindowsDriverZipAssetName) + zipAssetName := optionalDriverReleaseZipAssetNameForVersion("duckdb", "") + zipPath := filepath.Join(tmpDir, zipAssetName) zipFile, err := os.Create(zipPath) if err != nil { t.Fatalf("create zip failed: %v", err) @@ -1370,7 +1448,7 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing. t.Fatalf("create install dir failed: %v", err) } - hash, err := downloadOptionalDriverAgentBinary(nil, driverDefinition{Type: "duckdb", Name: "DuckDB"}, server.URL+"/"+duckDBWindowsDriverZipAssetName+"?source=release", target) + hash, err := downloadOptionalDriverAgentBinary(nil, driverDefinition{Type: "duckdb", Name: "DuckDB"}, server.URL+"/"+zipAssetName+"?source=release", target, "") if err != nil { t.Fatalf("download dedicated zip failed: %v", err) } @@ -1389,6 +1467,69 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing. } } +func TestDownloadOptionalDriverAgentBinaryPreservesMongoSelectedVersion(t *testing.T) { + originalValidateFunc := validateOptionalDriverAgentExecutableFunc + validateOptionalDriverAgentExecutableFunc = func(driverType string, executablePath string) error { + return nil + } + t.Cleanup(func() { + validateOptionalDriverAgentExecutableFunc = originalValidateFunc + }) + disableGlobalProxyForTest(t) + for _, name := range []string{"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"} { + t.Setenv(name, "") + } + + tmpDir := t.TempDir() + rawAssetName := mongoVersionedReleaseAssetName(1) + zipAssetName := optionalDriverReleaseZipAssetName(rawAssetName) + zipPath := filepath.Join(tmpDir, zipAssetName) + zipFile, err := os.Create(zipPath) + if err != nil { + t.Fatalf("create MongoDB ZIP: %v", err) + } + zw := zip.NewWriter(zipFile) + entryPath := filepath.ToSlash(filepath.Join(optionalDriverBundlePlatformDir(runtime.GOOS), rawAssetName)) + entry, err := zw.Create(entryPath) + if err != nil { + t.Fatalf("create MongoDB v1 entry: %v", err) + } + if _, err := entry.Write([]byte("mongodb-v1-agent")); err != nil { + t.Fatalf("write MongoDB v1 entry: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("close MongoDB ZIP: %v", err) + } + if err := zipFile.Close(); err != nil { + t.Fatalf("close MongoDB ZIP file: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, zipPath) + })) + defer server.Close() + target := filepath.Join(tmpDir, "install", optionalDriverExecutableBaseNameForType("mongodb")) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatalf("create MongoDB install dir: %v", err) + } + if _, err := downloadOptionalDriverAgentBinary( + nil, + driverDefinition{Type: "mongodb", Name: "MongoDB"}, + server.URL+"/"+zipAssetName, + target, + "1.17.9", + ); err != nil { + t.Fatalf("download MongoDB v1 ZIP: %v", err) + } + installed, err := os.ReadFile(target) + if err != nil { + t.Fatalf("read installed MongoDB v1 agent: %v", err) + } + if string(installed) != "mongodb-v1-agent" { + t.Fatalf("unexpected installed MongoDB v1 agent: %q", string(installed)) + } +} + func TestOptionalDriverDownloadZipURLAcceptsQueryString(t *testing.T) { if !isOptionalDriverDownloadZipURL("https://example.com/duckdb-driver.zip?token=abc") { t.Fatal("expected signed zip URL to be treated as zip download") @@ -1759,7 +1900,7 @@ func TestResolveOptionalDriverAgentDownloadURLsIncludesPublishedKingbaseAsset(t } version := normalizeVersion(definition.PinnedVersion) - assetName := optionalDriverReleaseAssetNameForVersion("kingbase", version) + assetName := optionalDriverReleaseZipAssetNameForVersion("kingbase", version) publishedAssets := map[string]int64{ assetName: 18 << 20, } diff --git a/tools/package-driver-release-assets.py b/tools/package-driver-release-assets.py index 3fe30b9c..192d959e 100644 --- a/tools/package-driver-release-assets.py +++ b/tools/package-driver-release-assets.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 +import hashlib import json +import re import shutil import sys import zipfile @@ -9,6 +11,53 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent LEGAL_FILENAMES = ("LICENSE", "NOTICE") +DRIVER_AGENT_RE = re.compile( + r"^.+-driver-agent(?:-v[0-9]+)?-(?:darwin|linux|windows)-(?:amd64|arm64)(?:\.exe)?$" +) +DUCKDB_WINDOWS_AGENT = "duckdb-driver-agent-windows-amd64.exe" +DUCKDB_WINDOWS_LIBRARY = "duckdb.dll" + + +def individual_archive_name(asset_name: str) -> str: + if asset_name.lower().endswith(".exe"): + return asset_name[:-4] + ".zip" + return asset_name + ".zip" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_individual_archive( + archive_path: Path, + asset: Path, + arcname: str, + legal_files: list[Path], +) -> list[tuple[Path, str]]: + entries = [(asset, arcname)] + if asset.name == DUCKDB_WINDOWS_AGENT: + support_file = asset.with_name(DUCKDB_WINDOWS_LIBRARY) + if not support_file.is_file(): + raise RuntimeError( + f"DuckDB Windows runtime dependency not found: {support_file}" + ) + entries.append( + ( + support_file, + (Path(arcname).parent / DUCKDB_WINDOWS_LIBRARY).as_posix(), + ) + ) + + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for source, entry_name in entries: + zf.write(source, entry_name) + for legal_file in legal_files: + zf.write(legal_file, legal_file.name) + return entries def main(): @@ -40,21 +89,24 @@ def main(): manifest_path = output_dir / manifest_name size_index = {} - standalone_assets = [] + archive_sha256_index = {} + entry_index = {} + individual_archives = [] + source_assets = [] with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: for asset in sorted(drivers_dir.rglob("*")): if not asset.is_file(): continue arcname = asset.relative_to(drivers_dir).as_posix() - if asset.name in size_index: + if any(existing.name == asset.name for existing, _ in source_assets): raise RuntimeError(f"driver asset name conflict: {asset.name}") + # Dedicated source packages are regenerated below from their raw + # agent and support files. Keeping a zip inside the CI bundle only + # wastes space and is not needed by the completion step. + if asset.suffix.lower() == ".zip": + continue zf.write(asset, arcname) - size_index[asset.name] = asset.stat().st_size - standalone_path = output_dir / asset.name - if standalone_path.exists(): - raise RuntimeError(f"release asset already exists: {standalone_path}") - shutil.copy2(asset, standalone_path) - standalone_assets.append(standalone_path.name) + source_assets.append((asset, arcname)) for legal_file in legal_files: zf.write(legal_file, legal_file.name) @@ -63,14 +115,53 @@ def main(): raise RuntimeError(f"release asset already exists: {standalone_path}") shutil.copy2(legal_file, standalone_path) + for asset, arcname in source_assets: + if not DRIVER_AGENT_RE.fullmatch(asset.name): + continue + archive_name = individual_archive_name(asset.name) + archive_path = output_dir / archive_name + if archive_path.exists(): + raise RuntimeError(f"release asset already exists: {archive_path}") + archive_entries = write_individual_archive( + archive_path, + asset, + arcname, + legal_files, + ) + size_index[archive_name] = archive_path.stat().st_size + archive_sha256_index[archive_name] = sha256_file(archive_path) + for source, entry_path in archive_entries: + if source.name in entry_index: + raise RuntimeError(f"driver archive entry name conflict: {source.name}") + entry_index[source.name] = { + "archive": archive_name, + "path": entry_path, + "size": source.stat().st_size, + "sha256": sha256_file(source), + } + individual_archives.append(archive_name) + + if not individual_archives: + raise RuntimeError("no driver agent binaries found") + index_path.write_text( - json.dumps({"assets": size_index}, ensure_ascii=False, indent=2) + "\n", + json.dumps( + { + "assets": size_index, + "assetSha256": archive_sha256_index, + "entries": entry_index, + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", encoding="utf-8", ) print(f"created {out_name} size={out_path.stat().st_size} bytes") print(f"created {index_name} entries={len(size_index)}") - print(f"published standalone driver assets={len(standalone_assets)}") + print(f"published individual driver archives={len(individual_archives)}") print(f"bundled legal files={len(legal_files)}") print(f"reserved manifest output path: {manifest_path}") return 0 diff --git a/tools/package-driver-release-assets.test.py b/tools/package-driver-release-assets.test.py index 48627d4c..c7a51f91 100644 --- a/tools/package-driver-release-assets.test.py +++ b/tools/package-driver-release-assets.test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import hashlib import json import subprocess import tempfile @@ -13,7 +14,7 @@ SCRIPT = ROOT / "tools" / "package-driver-release-assets.py" class PackageDriverReleaseAssetsTest(unittest.TestCase): - def test_packages_bundle_and_standalone_assets(self): + def test_packages_ci_bundle_and_individual_archives_without_raw_assets(self): with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp: tmpdir = Path(tmp) drivers_dir = tmpdir / "drivers" @@ -37,14 +38,78 @@ class PackageDriverReleaseAssetsTest(unittest.TestCase): self.assertIn("created GoNavi-DriverAgents.zip", proc.stdout) self.assertTrue((output_dir / "GoNavi-DriverAgents.zip").is_file()) - self.assertTrue((output_dir / windows_asset.name).is_file()) - self.assertTrue((output_dir / darwin_asset.name).is_file()) + windows_archive = output_dir / "clickhouse-driver-agent-windows-amd64.zip" + darwin_archive = output_dir / "clickhouse-driver-agent-darwin-arm64.zip" + self.assertTrue(windows_archive.is_file()) + self.assertTrue(darwin_archive.is_file()) + self.assertFalse((output_dir / windows_asset.name).exists()) + self.assertFalse((output_dir / darwin_asset.name).exists()) self.assertEqual((output_dir / "LICENSE").read_bytes(), (ROOT / "LICENSE").read_bytes()) self.assertEqual((output_dir / "NOTICE").read_bytes(), (ROOT / "NOTICE").read_bytes()) index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8")) - self.assertEqual(index["assets"][windows_asset.name], len(b"windows-asset")) - self.assertEqual(index["assets"][darwin_asset.name], len(b"darwin-asset")) + self.assertEqual(set(index), {"assets", "assetSha256", "entries"}) + self.assertEqual( + index["assets"], + { + windows_archive.name: windows_archive.stat().st_size, + darwin_archive.name: darwin_archive.stat().st_size, + }, + ) + self.assertEqual( + index["assetSha256"], + { + windows_archive.name: hashlib.sha256(windows_archive.read_bytes()).hexdigest(), + darwin_archive.name: hashlib.sha256(darwin_archive.read_bytes()).hexdigest(), + }, + ) + self.assertEqual( + index["entries"], + { + windows_asset.name: { + "archive": windows_archive.name, + "path": "Windows/clickhouse-driver-agent-windows-amd64.exe", + "size": len(b"windows-asset"), + "sha256": hashlib.sha256(b"windows-asset").hexdigest(), + }, + darwin_asset.name: { + "archive": darwin_archive.name, + "path": "MacOS/clickhouse-driver-agent-darwin-arm64", + "size": len(b"darwin-asset"), + "sha256": hashlib.sha256(b"darwin-asset").hexdigest(), + }, + }, + ) + self.assertNotIn("GoNavi-DriverAgents.zip", index["assets"]) + self.assertNotIn("GoNavi-DriverAgents.zip", index["assetSha256"]) + + with zipfile.ZipFile(windows_archive) as zf: + self.assertEqual( + sorted(zf.namelist()), + [ + "LICENSE", + "NOTICE", + "Windows/clickhouse-driver-agent-windows-amd64.exe", + ], + ) + self.assertEqual( + zf.read("Windows/clickhouse-driver-agent-windows-amd64.exe"), + b"windows-asset", + ) + + with zipfile.ZipFile(darwin_archive) as zf: + self.assertEqual( + sorted(zf.namelist()), + [ + "LICENSE", + "MacOS/clickhouse-driver-agent-darwin-arm64", + "NOTICE", + ], + ) + self.assertEqual( + zf.read("MacOS/clickhouse-driver-agent-darwin-arm64"), + b"darwin-asset", + ) with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf: self.assertEqual( @@ -59,6 +124,76 @@ class PackageDriverReleaseAssetsTest(unittest.TestCase): self.assertEqual(zf.read("LICENSE"), (ROOT / "LICENSE").read_bytes()) self.assertEqual(zf.read("NOTICE"), (ROOT / "NOTICE").read_bytes()) + def test_rebuilds_duckdb_windows_archive_with_agent_and_library(self): + with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp: + tmpdir = Path(tmp) + drivers_dir = tmpdir / "drivers" + output_dir = tmpdir / "driver-release-assets" + windows_dir = drivers_dir / "Windows" + windows_dir.mkdir(parents=True) + + agent = windows_dir / "duckdb-driver-agent-windows-amd64.exe" + library = windows_dir / "duckdb.dll" + agent.write_bytes(b"duckdb-agent") + library.write_bytes(b"duckdb-library") + (windows_dir / "duckdb-driver.zip").write_bytes(b"stale-source-package") + + subprocess.run( + ["python3", str(SCRIPT), str(drivers_dir), str(output_dir)], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + + archive = output_dir / "duckdb-driver-agent-windows-amd64.zip" + self.assertTrue(archive.is_file()) + self.assertFalse((output_dir / agent.name).exists()) + self.assertFalse((output_dir / library.name).exists()) + index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8")) + self.assertEqual(index["assets"], {archive.name: archive.stat().st_size}) + self.assertEqual( + index["assetSha256"], + {archive.name: hashlib.sha256(archive.read_bytes()).hexdigest()}, + ) + self.assertEqual( + index["entries"], + { + agent.name: { + "archive": archive.name, + "path": "Windows/duckdb-driver-agent-windows-amd64.exe", + "size": len(b"duckdb-agent"), + "sha256": hashlib.sha256(b"duckdb-agent").hexdigest(), + }, + library.name: { + "archive": archive.name, + "path": "Windows/duckdb.dll", + "size": len(b"duckdb-library"), + "sha256": hashlib.sha256(b"duckdb-library").hexdigest(), + }, + }, + ) + + with zipfile.ZipFile(archive) as zf: + self.assertEqual( + sorted(zf.namelist()), + [ + "LICENSE", + "NOTICE", + "Windows/duckdb-driver-agent-windows-amd64.exe", + "Windows/duckdb.dll", + ], + ) + self.assertEqual( + zf.read("Windows/duckdb-driver-agent-windows-amd64.exe"), + b"duckdb-agent", + ) + self.assertEqual(zf.read("Windows/duckdb.dll"), b"duckdb-library") + + with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf: + self.assertNotIn("Windows/duckdb-driver.zip", zf.namelist()) + if __name__ == "__main__": unittest.main() diff --git a/tools/prepare-vps-release-payload.py b/tools/prepare-vps-release-payload.py index b5fd6dcd..fe2e9373 100644 --- a/tools/prepare-vps-release-payload.py +++ b/tools/prepare-vps-release-payload.py @@ -16,6 +16,9 @@ from typing import Any TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +WINDOWS_ABSOLUTE_PATH_RE = re.compile(r"^[A-Za-z]:/") +DRIVER_CI_BUNDLE_NAME = "GoNavi-DriverAgents.zip" +DRIVER_MUTABLE_INDEX_FIELDS = frozenset(("tagName", "mirrorTagName")) def parse_args() -> argparse.Namespace: @@ -60,6 +63,89 @@ def validate_asset_name(value: Any, label: str) -> str: return value +def is_nonnegative_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def validate_sha256(value: Any, label: str) -> str: + if not isinstance(value, str) or not SHA256_RE.fullmatch(value): + fail(f"invalid {label} sha256: {value!r}") + return value.lower() + + +def validate_driver_entry_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or "\\" in value: + fail(f"invalid {label} path: {value!r}") + parts = value.split("/") + if ( + value.startswith("/") + or WINDOWS_ABSOLUTE_PATH_RE.match(value) + or any(part in {"", ".", ".."} for part in parts) + ): + fail(f"invalid {label} path: {value!r}") + return value + + +def validate_driver_index(index: dict[str, Any], label: str) -> None: + assets = index.get("assets") + asset_sha256 = index.get("assetSha256") + entries = index.get("entries") + if not isinstance(assets, dict) or not assets: + fail(f"{label} assets must be a non-empty object") + if not isinstance(asset_sha256, dict) or not asset_sha256: + fail(f"{label} assetSha256 must be a non-empty object") + if not isinstance(entries, dict) or not entries: + fail(f"{label} entries must be a non-empty object") + if set(asset_sha256) != set(assets): + fail(f"{label} assets and assetSha256 reference different archives") + + archive_names: set[str] = set() + for raw_name, expected_size in sorted(assets.items()): + name = validate_asset_name(raw_name, f"{label} driver") + if name == DRIVER_CI_BUNDLE_NAME: + fail(f"driver CI bundle must not be mirrored: {name}") + if Path(name).suffix.lower() != ".zip": + fail(f"driver mirror asset must be a zip archive: {name}") + if not is_nonnegative_int(expected_size): + fail(f"invalid driver asset size for {name}") + validate_sha256(asset_sha256.get(name), f"driver asset {name}") + archive_names.add(name) + + referenced_archives: set[str] = set() + for raw_name, raw_entry in sorted(entries.items()): + name = validate_asset_name(raw_name, f"{label} driver entry") + if not isinstance(raw_entry, dict): + fail(f"invalid {label} driver entry metadata for {name}") + archive = validate_asset_name( + raw_entry.get("archive"), + f"{label} driver entry archive", + ) + if archive not in archive_names: + fail(f"driver entry references an unknown archive: {name} -> {archive}") + validate_driver_entry_path( + raw_entry.get("path"), + f"{label} driver entry {name}", + ) + if not is_nonnegative_int(raw_entry.get("size")): + fail(f"invalid {label} driver entry size for {name}") + validate_sha256( + raw_entry.get("sha256"), + f"{label} driver entry {name}", + ) + referenced_archives.add(archive) + + if referenced_archives != archive_names: + fail(f"{label} entries do not cover every driver archive") + + +def normalized_driver_index(index: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in index.items() + if key not in DRIVER_MUTABLE_INDEX_FIELDS + } + + def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -114,19 +200,18 @@ def verify_and_copy_driver_assets( index: dict[str, Any], destination: Path, ) -> None: - assets = index.get("assets") - if not isinstance(assets, dict) or not assets: - fail("driver index assets must be a non-empty object") + assets = index["assets"] + asset_sha256 = index["assetSha256"] for raw_name, expected_size in sorted(assets.items()): name = validate_asset_name(raw_name, "driver") - if not isinstance(expected_size, int) or expected_size < 0: - fail(f"invalid driver asset size for {name}") source = driver_dir / name if not source.is_file(): fail(f"driver asset is missing: {source}") if source.stat().st_size != expected_size: fail(f"driver asset size mismatch: {name}") + if sha256_file(source) != asset_sha256[name].lower(): + fail(f"driver asset sha256 mismatch: {name}") copy_file(source, destination / name) @@ -198,8 +283,10 @@ def build(args: argparse.Namespace) -> dict[str, Any]: driver_tag = validate_tag(args.driver_tag, "driver tag") version_index = load_object(args.driver_version_index, "driver version index") latest_index = load_object(args.driver_latest_index, "driver latest index") - if version_index.get("assets") != latest_index.get("assets"): - fail("driver version and latest indexes reference different assets") + validate_driver_index(version_index, "driver version index") + validate_driver_index(latest_index, "driver latest index") + if normalized_driver_index(version_index) != normalized_driver_index(latest_index): + fail("driver version and latest indexes differ outside tag metadata") driver_version_dir, driver_latest_path = driver_layout(args.channel, driver_tag) verify_and_copy_driver_assets( diff --git a/tools/prepare-vps-release-payload.test.py b/tools/prepare-vps-release-payload.test.py index aac3f7c5..7522e80e 100644 --- a/tools/prepare-vps-release-payload.test.py +++ b/tools/prepare-vps-release-payload.test.py @@ -8,17 +8,47 @@ import subprocess import sys import tempfile import unittest +import zipfile from pathlib import Path SCRIPT = Path(__file__).with_name("prepare-vps-release-payload.py") PUBLISH_ACTION = SCRIPT.parents[1] / ".github/actions/publish-vps-mirror/action.yml" +DEV_WORKFLOW = SCRIPT.parents[1] / ".github/workflows/dev-build.yml" +PUBLISH_RELEASE_WORKFLOW = SCRIPT.parents[1] / ".github/workflows/publish-release.yml" def sha256(value: bytes) -> str: return hashlib.sha256(value).hexdigest() +def write_driver_archive(path: Path, entry_path: str, value: bytes) -> bytes: + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(entry_path, value) + return path.read_bytes() + + +def driver_index( + archive_name: str, + archive_bytes: bytes, + entry_name: str, + entry_path: str, + entry_bytes: bytes, +) -> dict[str, object]: + return { + "assets": {archive_name: len(archive_bytes)}, + "assetSha256": {archive_name: sha256(archive_bytes)}, + "entries": { + entry_name: { + "archive": archive_name, + "path": entry_path, + "size": len(entry_bytes), + "sha256": sha256(entry_bytes), + } + }, + } + + class PrepareVPSReleasePayloadTest(unittest.TestCase): def run_script(self, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -55,17 +85,38 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase): encoding="utf-8", ) - driver_bytes = b"driver" - (driver_dir / "mysql.exe").write_bytes(driver_bytes) - assets = {"mysql.exe": len(driver_bytes)} + driver_entry_bytes = b"driver binary" + driver_entry_name = "mysql-driver-agent-windows-amd64.exe" + driver_entry_path = f"Windows/{driver_entry_name}" + driver_name = "mysql-driver-agent-windows-amd64.zip" + driver_bytes = write_driver_archive( + driver_dir / driver_name, + driver_entry_path, + driver_entry_bytes, + ) + (driver_dir / "mysql-driver-agent-windows-amd64.exe").write_bytes(b"raw driver") + (driver_dir / "GoNavi-DriverAgents.zip").write_bytes(b"CI bundle") + index = driver_index( + driver_name, + driver_bytes, + driver_entry_name, + driver_entry_path, + driver_entry_bytes, + ) version_index = driver_dir / "version-index.json" latest_index = driver_dir / "latest-index.json" version_index.write_text( - json.dumps({"tagName": "v1.2.3", "assets": assets}), + json.dumps({**index, "tagName": "v1.2.3"}), encoding="utf-8", ) latest_index.write_text( - json.dumps({"tagName": "v1.2.3", "assets": assets}), + json.dumps( + { + **index, + "tagName": "latest", + "mirrorTagName": "v1.2.3", + } + ), encoding="utf-8", ) @@ -100,9 +151,21 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase): self.assertTrue( ( output - / "payload/drivers/releases/download/v1.2.3/mysql.exe" + / f"payload/drivers/releases/download/v1.2.3/{driver_name}" ).is_file() ) + self.assertFalse( + ( + output + / "payload/drivers/releases/download/v1.2.3/mysql-driver-agent-windows-amd64.exe" + ).exists() + ) + self.assertFalse( + ( + output + / "payload/drivers/releases/download/v1.2.3/GoNavi-DriverAgents.zip" + ).exists() + ) self.assertTrue( ( output @@ -117,6 +180,308 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase): metadata["fileCount"], ) + def test_rejects_non_zip_driver_asset_in_index(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + app_dir = root / "app" + driver_dir = root / "driver" + app_dir.mkdir() + driver_dir.mkdir() + + app_bytes = b"portable zip" + (app_dir / "GoNavi.zip").write_bytes(app_bytes) + app_manifest = app_dir / "latest.json" + app_manifest.write_text( + json.dumps( + { + "tagName": "v1.2.3", + "assets": [ + { + "name": "GoNavi.zip", + "size": len(app_bytes), + "sha256": sha256(app_bytes), + } + ], + } + ), + encoding="utf-8", + ) + + raw_name = "mysql-driver-agent-windows-amd64.exe" + raw_bytes = b"raw driver" + (driver_dir / raw_name).write_bytes(raw_bytes) + index = { + "assets": {raw_name: len(raw_bytes)}, + "assetSha256": {raw_name: sha256(raw_bytes)}, + "entries": { + raw_name: { + "archive": raw_name, + "path": f"Windows/{raw_name}", + "size": len(raw_bytes), + "sha256": sha256(raw_bytes), + } + }, + } + version_index = driver_dir / "version-index.json" + latest_index = driver_dir / "latest-index.json" + version_index.write_text(json.dumps(index), encoding="utf-8") + latest_index.write_text(json.dumps(index), encoding="utf-8") + + result = self.run_script( + "--channel", + "stable", + "--app-tag", + "v1.2.3", + "--app-dir", + str(app_dir), + "--app-manifest", + str(app_manifest), + "--driver-tag", + "v1.2.3", + "--driver-dir", + str(driver_dir), + "--driver-version-index", + str(version_index), + "--driver-latest-index", + str(latest_index), + "--output", + str(root / "output"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("driver mirror asset must be a zip archive", result.stderr) + + def test_rejects_driver_archive_sha256_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + app_dir = root / "app" + driver_dir = root / "driver" + app_dir.mkdir() + driver_dir.mkdir() + + app_bytes = b"portable zip" + (app_dir / "GoNavi.zip").write_bytes(app_bytes) + app_manifest = app_dir / "latest.json" + app_manifest.write_text( + json.dumps( + { + "tagName": "v1.2.3", + "assets": [ + { + "name": "GoNavi.zip", + "size": len(app_bytes), + "sha256": sha256(app_bytes), + } + ], + } + ), + encoding="utf-8", + ) + + entry_name = "mysql-driver-agent-windows-amd64.exe" + entry_path = f"Windows/{entry_name}" + archive_name = "mysql-driver-agent-windows-amd64.zip" + entry_bytes = b"driver binary" + archive_bytes = write_driver_archive( + driver_dir / archive_name, + entry_path, + entry_bytes, + ) + index = driver_index( + archive_name, + archive_bytes, + entry_name, + entry_path, + entry_bytes, + ) + index["assetSha256"] = {archive_name: "0" * 64} + version_index = driver_dir / "version-index.json" + latest_index = driver_dir / "latest-index.json" + version_index.write_text(json.dumps(index), encoding="utf-8") + latest_index.write_text(json.dumps(index), encoding="utf-8") + + result = self.run_script( + "--channel", + "stable", + "--app-tag", + "v1.2.3", + "--app-dir", + str(app_dir), + "--app-manifest", + str(app_manifest), + "--driver-tag", + "v1.2.3", + "--driver-dir", + str(driver_dir), + "--driver-version-index", + str(version_index), + "--driver-latest-index", + str(latest_index), + "--output", + str(root / "output"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("driver asset sha256 mismatch", result.stderr) + + def test_rejects_driver_index_metadata_drift_outside_tags(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + app_dir = root / "app" + driver_dir = root / "driver" + app_dir.mkdir() + driver_dir.mkdir() + + app_bytes = b"portable zip" + (app_dir / "GoNavi.zip").write_bytes(app_bytes) + app_manifest = app_dir / "latest.json" + app_manifest.write_text( + json.dumps( + { + "tagName": "v1.2.3", + "assets": [ + { + "name": "GoNavi.zip", + "size": len(app_bytes), + "sha256": sha256(app_bytes), + } + ], + } + ), + encoding="utf-8", + ) + + entry_name = "mysql-driver-agent-windows-amd64.exe" + entry_path = f"Windows/{entry_name}" + archive_name = "mysql-driver-agent-windows-amd64.zip" + entry_bytes = b"driver binary" + archive_bytes = write_driver_archive( + driver_dir / archive_name, + entry_path, + entry_bytes, + ) + version_payload = driver_index( + archive_name, + archive_bytes, + entry_name, + entry_path, + entry_bytes, + ) + latest_payload = json.loads(json.dumps(version_payload)) + latest_payload["entries"][entry_name]["path"] = f"Linux/{entry_name}" + version_index = driver_dir / "version-index.json" + latest_index = driver_dir / "latest-index.json" + version_index.write_text( + json.dumps({**version_payload, "tagName": "v1.2.3"}), + encoding="utf-8", + ) + latest_index.write_text( + json.dumps( + { + **latest_payload, + "tagName": "latest", + "mirrorTagName": "v1.2.3", + } + ), + encoding="utf-8", + ) + + result = self.run_script( + "--channel", + "stable", + "--app-tag", + "v1.2.3", + "--app-dir", + str(app_dir), + "--app-manifest", + str(app_manifest), + "--driver-tag", + "v1.2.3", + "--driver-dir", + str(driver_dir), + "--driver-version-index", + str(version_index), + "--driver-latest-index", + str(latest_index), + "--output", + str(root / "output"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("driver version and latest indexes differ outside tag metadata", result.stderr) + + def test_rejects_windows_drive_absolute_driver_entry_path(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + app_dir = root / "app" + driver_dir = root / "driver" + app_dir.mkdir() + driver_dir.mkdir() + + app_bytes = b"portable zip" + (app_dir / "GoNavi.zip").write_bytes(app_bytes) + app_manifest = app_dir / "latest.json" + app_manifest.write_text( + json.dumps( + { + "tagName": "v1.2.3", + "assets": [ + { + "name": "GoNavi.zip", + "size": len(app_bytes), + "sha256": sha256(app_bytes), + } + ], + } + ), + encoding="utf-8", + ) + + entry_name = "mysql-driver-agent-windows-amd64.exe" + entry_path = f"Windows/{entry_name}" + archive_name = "mysql-driver-agent-windows-amd64.zip" + entry_bytes = b"driver binary" + archive_bytes = write_driver_archive( + driver_dir / archive_name, + entry_path, + entry_bytes, + ) + index = driver_index( + archive_name, + archive_bytes, + entry_name, + f"C:/{entry_name}", + entry_bytes, + ) + version_index = driver_dir / "version-index.json" + latest_index = driver_dir / "latest-index.json" + version_index.write_text(json.dumps(index), encoding="utf-8") + latest_index.write_text(json.dumps(index), encoding="utf-8") + + result = self.run_script( + "--channel", + "stable", + "--app-tag", + "v1.2.3", + "--app-dir", + str(app_dir), + "--app-manifest", + str(app_manifest), + "--driver-tag", + "v1.2.3", + "--driver-dir", + str(driver_dir), + "--driver-version-index", + str(version_index), + "--driver-latest-index", + str(latest_index), + "--output", + str(root / "output"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid driver version index driver entry", result.stderr) + def test_rejects_manifest_path_traversal(self) -> None: with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) @@ -162,6 +527,34 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase): self.assertIn("available_kib", source) self.assertIn("available_kib * 1024", source) + def test_publish_action_requires_explicit_driver_mode_and_cleans_staging(self) -> None: + source = PUBLISH_ACTION.read_text(encoding="utf-8") + + self.assertIn("driver-enabled:", source) + self.assertIn("MIRROR_DRIVER_ENABLED", source) + self.assertIn('case "${MIRROR_DRIVER_ENABLED}"', source) + self.assertIn("trap cleanup EXIT", source) + self.assertIn(".gonavi-mirror-root", source) + self.assertIn("-mindepth 1 -maxdepth 1 -type d -mmin +1440", source) + + def test_workflows_pass_explicit_driver_mode(self) -> None: + dev_source = DEV_WORKFLOW.read_text(encoding="utf-8") + stable_source = PUBLISH_RELEASE_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn( + "driver-enabled: ${{ steps.driver_assets.outputs.has_driver_assets }}", + dev_source, + ) + self.assertIn("id: mirror_payload", stable_source) + self.assertIn( + 'echo "has_driver_release=${has_driver_release}" >> "$GITHUB_OUTPUT"', + stable_source, + ) + self.assertIn( + "driver-enabled: ${{ steps.mirror_payload.outputs.has_driver_release }}", + stable_source, + ) + if __name__ == "__main__": unittest.main() diff --git a/tools/validate-driver-release-assets.py b/tools/validate-driver-release-assets.py index f023a2aa..46ea230d 100644 --- a/tools/validate-driver-release-assets.py +++ b/tools/validate-driver-release-assets.py @@ -13,10 +13,15 @@ import tempfile import urllib.error import urllib.parse import urllib.request +import zipfile from pathlib import Path MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json" +INDEX_ASSET_NAME = "GoNavi-DriverAgents-Index.json" +CI_BUNDLE_ASSET_NAME = "GoNavi-DriverAgents.zip" +DUCKDB_WINDOWS_AGENT_NAME = "duckdb-driver-agent-windows-amd64.exe" +DUCKDB_WINDOWS_LIBRARY_NAME = "duckdb.dll" def github_headers(binary: bool = False): @@ -110,6 +115,36 @@ def infer_asset_path(name: str): return None +def infer_driver_zip_asset_name(name: str): + trimmed = str(name or "").strip() + if not trimmed or infer_asset_path(trimmed) is None or "-driver-agent-" not in trimmed: + return None + if trimmed.lower().endswith(".exe"): + trimmed = trimmed[:-4] + return f"{trimmed}.zip" + + +def extract_zip_entry(zip_path: Path, entry_name: str, destination: Path): + expected = str(entry_name or "").strip().replace("\\", "/") + if not expected: + return False + with zipfile.ZipFile(zip_path) as archive: + entry = None + for candidate in archive.infolist(): + normalized = candidate.filename.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + if not candidate.is_dir() and normalized == expected: + entry = candidate + break + if entry is None: + return False + destination.parent.mkdir(parents=True, exist_ok=True) + with archive.open(entry) as source, open(destination, "wb") as output: + shutil.copyfileobj(source, output) + return True + + def normalize_machine(value: str): machine = str(value or "").strip().lower() if machine in {"x86_64", "amd64"}: @@ -161,7 +196,7 @@ def probe_metadata_revision(path: Path): return str(((payload.get("data") or {}).get("agentRevision") or "")).strip() -def validate_release_assets(release: dict, manifest: dict, runtime_platform=None): +def _validate_legacy_release_assets(release: dict, manifest: dict, runtime_platform=None): assets = asset_map(release) manifest_assets = manifest.get("assets") or {} if not isinstance(manifest_assets, dict) or not manifest_assets: @@ -178,8 +213,11 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None for name, meta in sorted(manifest_assets.items()): if name == MANIFEST_ASSET_NAME: continue - asset = assets.get(name) - if asset is None: + path_hint = infer_asset_path(name) + zip_asset_name = infer_driver_zip_asset_name(name) + zip_asset = assets.get(zip_asset_name) if zip_asset_name else None + legacy_asset = assets.get(name) + if zip_asset is None and legacy_asset is None: mismatches.append((name, "missing_release_asset", "", "present in manifest")) continue @@ -188,16 +226,31 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None asset_platform = str(meta.get("platform") or "").strip().lower() local_path = None - actual_sha = asset_sha256_digest(asset) - if expected_sha and not actual_sha: - local_path = tmp_root / name - download_url(str(asset.get("browser_download_url") or "").strip(), local_path) + actual_sha = "" + if zip_asset is not None: + archive_path = tmp_root / "archives" / zip_asset_name + archive_path.parent.mkdir(parents=True, exist_ok=True) + download_url(str(zip_asset.get("browser_download_url") or "").strip(), archive_path) + local_path = tmp_root / "extracted" / name + try: + found_entry = extract_zip_entry(archive_path, path_hint, local_path) + except zipfile.BadZipFile as exc: + mismatches.append((name, "zip", str(exc), "valid zip archive")) + continue + if not found_entry: + mismatches.append((name, "zip_entry", "", path_hint or name)) + continue actual_sha = sha256_file(local_path).lower() + else: + actual_sha = asset_sha256_digest(legacy_asset) + if expected_sha and not actual_sha: + local_path = tmp_root / name + download_url(str(legacy_asset.get("browser_download_url") or "").strip(), local_path) + actual_sha = sha256_file(local_path).lower() if expected_sha and actual_sha != expected_sha: mismatches.append((name, "sha256", actual_sha, expected_sha)) continue - path_hint = infer_asset_path(name) if path_hint is None: skipped.append(name) continue @@ -205,7 +258,7 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None if expected_revision and asset_platform == runtime_platform: if local_path is None: local_path = tmp_root / name - download_url(str(asset.get("browser_download_url") or "").strip(), local_path) + download_url(str(legacy_asset.get("browser_download_url") or "").strip(), local_path) actual_revision = probe_metadata_revision(local_path) if actual_revision != expected_revision: mismatches.append((name, "revision", actual_revision, expected_revision)) @@ -213,6 +266,301 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None return mismatches, skipped +def is_nonnegative_int(value): + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def is_sha256(value): + normalized = str(value or "").strip().lower() + return len(normalized) == 64 and all(char in "0123456789abcdef" for char in normalized) + + +def is_basename(value): + normalized = str(value or "").strip() + return normalized not in {"", ".", ".."} and "/" not in normalized and "\\" not in normalized + + +def is_safe_archive_entry_path(value): + normalized = str(value or "").strip().replace("\\", "/") + if not normalized or normalized.startswith("/"): + return False + if len(normalized) >= 3 and normalized[0].isalpha() and normalized[1:3] == ":/": + return False + parts = normalized.split("/") + return all(part not in {"", ".", ".."} for part in parts) + + +def indexed_release_maps(release_index): + if release_index is None: + return None + if not isinstance(release_index, dict): + raise RuntimeError("release index must be an object") + if "entries" not in release_index and "assetSha256" not in release_index: + return None + + result = [] + for key in ("assets", "assetSha256", "entries"): + value = release_index.get(key) + if not isinstance(value, dict) or not value: + raise RuntimeError(f"release index {key} must be a non-empty object") + result.append(value) + return tuple(result) + + +def _validate_indexed_release_assets( + release: dict, + manifest: dict, + runtime_platform: str, + index_assets: dict, + index_asset_sha256: dict, + index_entries: dict, +): + release_assets = asset_map(release) + manifest_assets = manifest.get("assets") or {} + if not isinstance(manifest_assets, dict) or not manifest_assets: + raise RuntimeError("manifest assets is empty") + + mismatches = [] + skipped = [] + archive_metadata_valid = {} + + for archive_name, expected_size in sorted(index_assets.items()): + archive = str(archive_name or "").strip() + valid = True + if not is_basename(archive): + mismatches.append((archive, "index_asset_name", archive, "archive basename")) + archive_metadata_valid[archive] = False + continue + if archive == CI_BUNDLE_ASSET_NAME: + mismatches.append((archive, "index_ci_bundle", "present in assets", "absent")) + valid = False + if not is_nonnegative_int(expected_size): + mismatches.append((archive, "index_archive_size", expected_size, "non-negative integer")) + valid = False + + expected_sha = str(index_asset_sha256.get(archive) or "").strip().lower() + if not is_sha256(expected_sha): + mismatches.append((archive, "index_archive_sha256", expected_sha, "sha256")) + valid = False + + release_asset = release_assets.get(archive) + if release_asset is None: + mismatches.append((archive, "missing_release_asset", "", "present in release index")) + archive_metadata_valid[archive] = False + continue + + actual_size = release_asset.get("size") + if actual_size != expected_size: + mismatches.append((archive, "archive_size", actual_size, expected_size)) + valid = False + actual_sha = asset_sha256_digest(release_asset) + if actual_sha != expected_sha: + mismatches.append((archive, "archive_sha256", actual_sha, expected_sha)) + valid = False + archive_metadata_valid[archive] = valid + + for archive_name in sorted(index_asset_sha256): + if archive_name == CI_BUNDLE_ASSET_NAME: + mismatches.append((archive_name, "index_ci_bundle_sha256", "present in assetSha256", "absent")) + if archive_name not in index_assets: + mismatches.append((archive_name, "index_asset", "assetSha256", "assets")) + + normalized_entries = {} + entry_metadata_valid = {} + for raw_name, raw_entry in sorted(index_entries.items(), key=lambda item: str(item[0])): + name = str(raw_name or "").strip() + valid = True + if not is_basename(name): + mismatches.append((name, "index_entry_name", name, "basename")) + entry_metadata_valid[name] = False + continue + if not isinstance(raw_entry, dict): + mismatches.append((name, "index_entry", type(raw_entry).__name__, "object")) + entry_metadata_valid[name] = False + continue + + archive = str(raw_entry.get("archive") or "").strip() + entry_path = str(raw_entry.get("path") or "").strip().replace("\\", "/") + entry_size = raw_entry.get("size") + entry_sha = str(raw_entry.get("sha256") or "").strip().lower() + normalized_entries[name] = { + "archive": archive, + "path": entry_path, + "size": entry_size, + "sha256": entry_sha, + } + + if not is_basename(archive): + mismatches.append((name, "index_archive", archive, "archive basename")) + valid = False + elif archive == CI_BUNDLE_ASSET_NAME: + mismatches.append((name, "index_ci_bundle_entry", archive, "individual driver archive")) + valid = False + if archive not in index_assets: + mismatches.append((name, "index_asset", archive, "present in assets")) + valid = False + if archive not in index_asset_sha256: + mismatches.append((name, "index_asset_sha256", archive, "present in assetSha256")) + valid = False + if not is_safe_archive_entry_path(entry_path): + mismatches.append((name, "index_entry_path", entry_path, "safe relative archive path")) + valid = False + expected_path = infer_asset_path(name) + if expected_path is not None and entry_path != expected_path: + mismatches.append((name, "index_path", entry_path, expected_path)) + valid = False + if not is_nonnegative_int(entry_size): + mismatches.append((name, "index_entry_size", entry_size, "non-negative integer")) + valid = False + if not is_sha256(entry_sha): + mismatches.append((name, "index_entry_sha256", entry_sha, "sha256")) + valid = False + if name == DUCKDB_WINDOWS_LIBRARY_NAME: + expected_archive = infer_driver_zip_asset_name(DUCKDB_WINDOWS_AGENT_NAME) + if archive != expected_archive: + mismatches.append((name, "index_archive", archive, expected_archive)) + valid = False + entry_metadata_valid[name] = valid + + if DUCKDB_WINDOWS_AGENT_NAME in normalized_entries and DUCKDB_WINDOWS_LIBRARY_NAME not in normalized_entries: + mismatches.append( + ( + DUCKDB_WINDOWS_LIBRARY_NAME, + "index_entry", + "", + f"present with {DUCKDB_WINDOWS_AGENT_NAME}", + ) + ) + + runtime_manifest_entries = [] + for name, meta in sorted(manifest_assets.items()): + if name == MANIFEST_ASSET_NAME: + continue + if not isinstance(meta, dict): + mismatches.append((name, "manifest_entry", type(meta).__name__, "object")) + continue + + path_hint = infer_asset_path(name) + archive_hint = infer_driver_zip_asset_name(name) + if path_hint is None or archive_hint is None: + skipped.append(name) + continue + + entry = normalized_entries.get(name) + if entry is None: + mismatches.append((name, "index_entry", "", "present in release index")) + continue + + entry_valid = entry_metadata_valid.get(name, False) + indexed_archive = entry["archive"] + indexed_path = entry["path"] + indexed_size = entry.get("size") + indexed_sha = entry["sha256"] + manifest_size = meta.get("size") + manifest_sha = str(meta.get("sha256") or "").strip().lower() + + if indexed_archive != archive_hint: + mismatches.append((name, "index_archive", indexed_archive, archive_hint)) + entry_valid = False + if indexed_path != path_hint: + mismatches.append((name, "index_path", indexed_path, path_hint)) + entry_valid = False + if not is_nonnegative_int(manifest_size): + mismatches.append((name, "manifest_size", manifest_size, "non-negative integer")) + entry_valid = False + elif indexed_size != manifest_size: + mismatches.append((name, "index_size", indexed_size, manifest_size)) + entry_valid = False + if not is_sha256(manifest_sha): + mismatches.append((name, "manifest_sha256", manifest_sha, "sha256")) + entry_valid = False + elif indexed_sha != manifest_sha: + mismatches.append((name, "index_sha256", indexed_sha, manifest_sha)) + entry_valid = False + asset_platform = str(meta.get("platform") or "").strip().lower() + if entry_valid and asset_platform == runtime_platform and archive_metadata_valid.get(indexed_archive, False): + runtime_manifest_entries.append((name, meta, indexed_archive)) + + runtime_by_archive = {} + for name, meta, archive in runtime_manifest_entries: + runtime_by_archive.setdefault(archive, []).append((name, meta)) + + with tempfile.TemporaryDirectory(prefix="gonavi-release-assets-") as tmp: + tmp_root = Path(tmp) + for archive, manifest_entries in sorted(runtime_by_archive.items()): + release_asset = release_assets[archive] + archive_path = tmp_root / "archives" / archive + archive_path.parent.mkdir(parents=True, exist_ok=True) + download_url(str(release_asset.get("browser_download_url") or "").strip(), archive_path) + + expected_archive_size = index_assets[archive] + actual_archive_size = archive_path.stat().st_size + if actual_archive_size != expected_archive_size: + mismatches.append((archive, "download_size", actual_archive_size, expected_archive_size)) + continue + expected_archive_sha = str(index_asset_sha256[archive]).strip().lower() + actual_archive_sha = sha256_file(archive_path).lower() + if actual_archive_sha != expected_archive_sha: + mismatches.append((archive, "download_sha256", actual_archive_sha, expected_archive_sha)) + continue + + archive_entries = { + raw_name: entry + for raw_name, entry in normalized_entries.items() + if entry_metadata_valid.get(raw_name, False) and entry["archive"] == archive + } + extracted = {} + try: + for raw_name, entry in sorted(archive_entries.items()): + entry_path = entry["path"] + entry_size = entry.get("size") + entry_sha = entry["sha256"] + local_path = tmp_root / "extracted" / archive / raw_name + if not extract_zip_entry(archive_path, entry_path, local_path): + mismatches.append((raw_name, "zip_entry", "", entry_path)) + continue + actual_size = local_path.stat().st_size + if actual_size != entry_size: + mismatches.append((raw_name, "entry_size", actual_size, entry_size)) + continue + actual_sha = sha256_file(local_path).lower() + if actual_sha != entry_sha: + mismatches.append((raw_name, "entry_sha256", actual_sha, entry_sha)) + continue + extracted[raw_name] = local_path + except zipfile.BadZipFile as exc: + mismatches.append((archive, "zip", str(exc), "valid zip archive")) + continue + + for name, meta in manifest_entries: + local_path = extracted.get(name) + if local_path is None: + continue + expected_revision = str(meta.get("revision") or "").strip() + if not expected_revision: + continue + actual_revision = probe_metadata_revision(local_path) + if actual_revision != expected_revision: + mismatches.append((name, "revision", actual_revision, expected_revision)) + + return mismatches, skipped + + +def validate_release_assets(release: dict, manifest: dict, runtime_platform=None, release_index=None): + if runtime_platform is None: + runtime_platform = current_runtime_platform() + normalized_runtime = str(runtime_platform or "").strip().lower() + index_maps = indexed_release_maps(release_index) + if index_maps is None: + return _validate_legacy_release_assets(release, manifest, normalized_runtime) + return _validate_indexed_release_assets( + release, + manifest, + normalized_runtime, + *index_maps, + ) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents") @@ -224,13 +572,20 @@ def main(): manifest_asset = assets.get(MANIFEST_ASSET_NAME) if manifest_asset is None: raise SystemExit(f"release {args.repo}@{args.tag} missing {MANIFEST_ASSET_NAME}") + index_asset = assets.get(INDEX_ASSET_NAME) with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-") as tmp: - manifest_path = Path(tmp) / MANIFEST_ASSET_NAME + metadata_root = Path(tmp) + manifest_path = metadata_root / MANIFEST_ASSET_NAME download_url(str(manifest_asset.get("browser_download_url") or "").strip(), manifest_path) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + release_index = None + if index_asset is not None: + index_path = metadata_root / INDEX_ASSET_NAME + download_url(str(index_asset.get("browser_download_url") or "").strip(), index_path) + release_index = json.loads(index_path.read_text(encoding="utf-8")) - mismatches, skipped = validate_release_assets(release, manifest) + mismatches, skipped = validate_release_assets(release, manifest, release_index=release_index) if mismatches: print("published driver release assets mismatch manifest:", file=sys.stderr) for name, field, actual, expected in mismatches: diff --git a/tools/validate-driver-release-assets.test.py b/tools/validate-driver-release-assets.test.py index 27bc18ff..1a244f5f 100644 --- a/tools/validate-driver-release-assets.test.py +++ b/tools/validate-driver-release-assets.test.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 +import contextlib import hashlib import importlib.util +import io +import json import pathlib +import sys import tempfile import unittest +import zipfile MODULE_PATH = pathlib.Path(__file__).with_name("validate-driver-release-assets.py") @@ -39,6 +44,878 @@ class ValidateDriverReleaseAssetsTests(unittest.TestCase): self.assertEqual(MODULE.infer_asset_path("duckdb.dll"), "Windows/duckdb.dll") self.assertIsNone(MODULE.infer_asset_path("duckdb-driver.zip")) + def test_infer_driver_zip_asset_name(self): + self.assertEqual( + MODULE.infer_driver_zip_asset_name("clickhouse-driver-agent-darwin-arm64"), + "clickhouse-driver-agent-darwin-arm64.zip", + ) + self.assertEqual( + MODULE.infer_driver_zip_asset_name("mariadb-driver-agent-windows-amd64.exe"), + "mariadb-driver-agent-windows-amd64.zip", + ) + self.assertEqual( + MODULE.infer_driver_zip_asset_name("mongodb-driver-agent-v1-windows-arm64.exe"), + "mongodb-driver-agent-v1-windows-arm64.zip", + ) + self.assertEqual( + MODULE.infer_driver_zip_asset_name("duckdb-driver-agent-windows-amd64.exe"), + "duckdb-driver-agent-windows-amd64.zip", + ) + self.assertIsNone(MODULE.infer_driver_zip_asset_name("duckdb.dll")) + + def test_validate_release_assets_extracts_independent_zip(self): + name = "clickhouse-driver-agent-darwin-arm64" + zip_name = f"{name}.zip" + payload = b"zipped-driver-agent" + release = { + "assets": [ + { + "name": zip_name, + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "darwin/arm64", + "revision": "src-expected", + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + + with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp: + archive_path = pathlib.Path(tmp) / zip_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(f"MacOS/{name}", payload) + archive_bytes = archive_path.read_bytes() + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + release["assets"][0]["size"] = len(archive_bytes) + release["assets"][0]["digest"] = f"sha256:{archive_sha}" + manifest["assets"][name]["size"] = len(payload) + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"MacOS/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + downloaded_urls = [] + probed_payloads = [] + + def fake_download(url, destination): + downloaded_urls.append(url) + destination.write_bytes(archive_path.read_bytes()) + + def fake_probe(path): + probed_payloads.append(path.read_bytes()) + return "src-expected" + + original_download = MODULE.download_url + original_probe = MODULE.probe_metadata_revision + try: + MODULE.download_url = fake_download + MODULE.probe_metadata_revision = fake_probe + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + MODULE.probe_metadata_revision = original_probe + + self.assertEqual(mismatches, []) + self.assertEqual(skipped, []) + self.assertEqual(downloaded_urls, [f"https://example.test/{zip_name}"]) + self.assertEqual(probed_payloads, [payload]) + + def test_indexed_duckdb_archive_verifies_runtime_library_entry(self): + name = "duckdb-driver-agent-windows-amd64.exe" + zip_name = "duckdb-driver-agent-windows-amd64.zip" + agent_payload = b"duckdb-agent" + library_payload = b"wrong-duckdb-library" + expected_library_sha = hashlib.sha256(b"expected-duckdb-library").hexdigest() + manifest = { + "assets": { + name: { + "platform": "windows/amd64", + "revision": "src-expected", + "size": len(agent_payload), + "sha256": hashlib.sha256(agent_payload).hexdigest(), + } + } + } + + with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp: + archive_path = pathlib.Path(tmp) / zip_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(f"Windows/{name}", agent_payload) + archive.writestr("Windows/duckdb.dll", library_payload) + archive_bytes = archive_path.read_bytes() + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + release = { + "assets": [ + { + "name": zip_name, + "size": len(archive_bytes), + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Windows/{name}", + "size": len(agent_payload), + "sha256": hashlib.sha256(agent_payload).hexdigest(), + }, + "duckdb.dll": { + "archive": zip_name, + "path": "Windows/duckdb.dll", + "size": len(library_payload), + "sha256": expected_library_sha, + }, + }, + } + + def fake_download(_url, destination): + destination.write_bytes(archive_bytes) + + original_download = MODULE.download_url + original_probe = MODULE.probe_metadata_revision + try: + MODULE.download_url = fake_download + MODULE.probe_metadata_revision = lambda _path: "src-expected" + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="windows/amd64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + MODULE.probe_metadata_revision = original_probe + + self.assertEqual(skipped, []) + self.assertEqual( + mismatches, + [ + ( + "duckdb.dll", + "entry_sha256", + hashlib.sha256(library_payload).hexdigest(), + expected_library_sha, + ) + ], + ) + + def test_indexed_validation_rejects_ci_bundle_archive_maps(self): + name = "clickhouse-driver-agent-linux-amd64" + zip_name = f"{name}.zip" + archive_bytes = b"independent-archive" + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + bundle_bytes = b"ci-completion-bundle" + bundle_sha = hashlib.sha256(bundle_bytes).hexdigest() + payload = b"driver" + release = { + "assets": [ + { + "name": zip_name, + "size": len(archive_bytes), + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + }, + { + "name": MODULE.CI_BUNDLE_ASSET_NAME, + "size": len(bundle_bytes), + "digest": f"sha256:{bundle_sha}", + "browser_download_url": f"https://example.test/{MODULE.CI_BUNDLE_ASSET_NAME}", + }, + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/amd64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": { + zip_name: len(archive_bytes), + MODULE.CI_BUNDLE_ASSET_NAME: len(bundle_bytes), + }, + "assetSha256": { + zip_name: archive_sha, + MODULE.CI_BUNDLE_ASSET_NAME: bundle_sha, + }, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertIn( + (MODULE.CI_BUNDLE_ASSET_NAME, "index_ci_bundle", "present in assets", "absent"), + mismatches, + ) + self.assertIn( + ( + MODULE.CI_BUNDLE_ASSET_NAME, + "index_ci_bundle_sha256", + "present in assetSha256", + "absent", + ), + mismatches, + ) + + def test_indexed_validation_rejects_entry_backed_by_ci_bundle(self): + name = "clickhouse-driver-agent-linux-amd64" + zip_name = f"{name}.zip" + archive_bytes = b"independent-archive" + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + payload = b"driver" + release = { + "assets": [ + { + "name": zip_name, + "size": len(archive_bytes), + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/amd64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + }, + "rogue-driver-agent": { + "archive": MODULE.CI_BUNDLE_ASSET_NAME, + "path": "Linux/rogue-driver-agent", + "size": 1, + "sha256": "a" * 64, + }, + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertIn( + ( + "rogue-driver-agent", + "index_ci_bundle_entry", + MODULE.CI_BUNDLE_ASSET_NAME, + "individual driver archive", + ), + mismatches, + ) + + def test_indexed_validation_rejects_malformed_cross_platform_duckdb_library(self): + name = MODULE.DUCKDB_WINDOWS_AGENT_NAME + zip_name = MODULE.infer_driver_zip_asset_name(name) + archive_bytes = b"duckdb-archive" + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + payload = b"duckdb-agent" + release = { + "assets": [ + { + "name": zip_name, + "size": len(archive_bytes), + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "windows/amd64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Windows/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + }, + MODULE.DUCKDB_WINDOWS_LIBRARY_NAME: { + "archive": "missing.zip", + "path": "../Windows/duckdb.dll", + "size": -1, + "sha256": "invalid", + }, + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertIn( + (MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_asset", "missing.zip", "present in assets"), + mismatches, + ) + self.assertIn( + ( + MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, + "index_entry_path", + "../Windows/duckdb.dll", + "safe relative archive path", + ), + mismatches, + ) + self.assertIn( + (MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_entry_size", -1, "non-negative integer"), + mismatches, + ) + self.assertIn( + (MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_entry_sha256", "invalid", "sha256"), + mismatches, + ) + self.assertIn( + (MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_archive", "missing.zip", zip_name), + mismatches, + ) + + def test_indexed_validation_requires_duckdb_library_mapping(self): + name = MODULE.DUCKDB_WINDOWS_AGENT_NAME + zip_name = MODULE.infer_driver_zip_asset_name(name) + archive_bytes = b"duckdb-archive" + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + payload = b"duckdb-agent" + release = { + "assets": [ + { + "name": zip_name, + "size": len(archive_bytes), + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "windows/amd64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Windows/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertIn( + ( + MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, + "index_entry", + "", + f"present with {MODULE.DUCKDB_WINDOWS_AGENT_NAME}", + ), + mismatches, + ) + + def test_validate_release_assets_reports_independent_zip_raw_sha_mismatch(self): + name = "mariadb-driver-agent-windows-amd64.exe" + zip_name = "mariadb-driver-agent-windows-amd64.zip" + payload = b"unexpected-driver-agent" + expected_manifest_sha = "d" * 64 + release = { + "assets": [ + { + "name": zip_name, + "browser_download_url": f"https://example.test/{zip_name}", + "digest": f"sha256:{hashlib.sha256(b'archive-bytes').hexdigest()}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "windows/amd64", + "revision": "src-expected", + "sha256": expected_manifest_sha, + } + } + } + + with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp: + archive_path = pathlib.Path(tmp) / zip_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(f"Windows/{name}", payload) + archive_bytes = archive_path.read_bytes() + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + release["assets"][0]["size"] = len(archive_bytes) + release["assets"][0]["digest"] = f"sha256:{archive_sha}" + manifest["assets"][name]["size"] = len(payload) + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Windows/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + original_probe = MODULE.probe_metadata_revision + try: + MODULE.download_url = fail_download + MODULE.probe_metadata_revision = lambda _path: "src-expected" + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="linux/amd64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + MODULE.probe_metadata_revision = original_probe + + self.assertEqual(skipped, []) + self.assertEqual( + mismatches, + [(name, "index_sha256", hashlib.sha256(payload).hexdigest(), expected_manifest_sha)], + ) + + def test_validate_release_assets_skips_cross_platform_zip_revision_probe(self): + name = "mongodb-driver-agent-v1-linux-arm64" + zip_name = f"{name}.zip" + payload = b"cross-platform-driver-agent" + release = { + "assets": [ + { + "name": zip_name, + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/arm64", + "revision": "src-expected", + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + + with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp: + archive_path = pathlib.Path(tmp) / zip_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(f"Linux/{name}", payload) + archive_bytes = archive_path.read_bytes() + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + release["assets"][0]["size"] = len(archive_bytes) + release["assets"][0]["digest"] = f"sha256:{archive_sha}" + manifest["assets"][name]["size"] = len(payload) + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + def fail_probe(_path): + raise AssertionError("cross-platform zip entry should not be executed") + + original_download = MODULE.download_url + original_probe = MODULE.probe_metadata_revision + try: + MODULE.download_url = fail_download + MODULE.probe_metadata_revision = fail_probe + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + MODULE.probe_metadata_revision = original_probe + + self.assertEqual(mismatches, []) + self.assertEqual(skipped, []) + + def test_validate_release_assets_reports_missing_independent_zip_entry(self): + name = "clickhouse-driver-agent-linux-amd64" + zip_name = f"{name}.zip" + release = { + "assets": [ + { + "name": zip_name, + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/amd64", + "revision": "src-expected", + "sha256": hashlib.sha256(b"driver").hexdigest(), + } + } + } + + with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp: + archive_path = pathlib.Path(tmp) / zip_name + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(name, b"driver") + archive_bytes = archive_path.read_bytes() + archive_sha = hashlib.sha256(archive_bytes).hexdigest() + release["assets"][0]["size"] = len(archive_bytes) + release["assets"][0]["digest"] = f"sha256:{archive_sha}" + manifest["assets"][name]["size"] = len(b"driver") + release_index = { + "assets": {zip_name: len(archive_bytes)}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(b"driver"), + "sha256": hashlib.sha256(b"driver").hexdigest(), + } + }, + } + + def fake_download(_url, destination): + destination.write_bytes(archive_path.read_bytes()) + + original_download = MODULE.download_url + try: + MODULE.download_url = fake_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="linux/amd64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertEqual( + mismatches, + [(name, "zip_entry", "", f"Linux/{name}")], + ) + + def test_indexed_validation_checks_release_archive_metadata_without_download(self): + name = "clickhouse-driver-agent-linux-amd64" + zip_name = f"{name}.zip" + payload = b"driver" + expected_archive = b"indexed-archive" + expected_archive_sha = hashlib.sha256(expected_archive).hexdigest() + release = { + "assets": [ + { + "name": zip_name, + "size": len(expected_archive) + 1, + "digest": f"sha256:{hashlib.sha256(b'wrong-archive').hexdigest()}", + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/amd64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": {zip_name: len(expected_archive)}, + "assetSha256": {zip_name: expected_archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + + def fail_download(_url, _destination): + raise AssertionError("cross-platform indexed asset should not be downloaded") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + mismatches, skipped = MODULE.validate_release_assets( + release, + manifest, + runtime_platform="darwin/arm64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + self.assertEqual(skipped, []) + self.assertEqual( + mismatches, + [ + (zip_name, "archive_size", len(expected_archive) + 1, len(expected_archive)), + ( + zip_name, + "archive_sha256", + hashlib.sha256(b"wrong-archive").hexdigest(), + expected_archive_sha, + ), + ], + ) + + def test_malformed_new_index_does_not_fall_back_to_zip_downloads(self): + name = "clickhouse-driver-agent-linux-amd64" + zip_name = f"{name}.zip" + release = { + "assets": [ + { + "name": zip_name, + "browser_download_url": f"https://example.test/{zip_name}", + } + ] + } + manifest = { + "assets": { + name: { + "platform": "linux/amd64", + "revision": "src-expected", + "size": 6, + "sha256": hashlib.sha256(b"driver").hexdigest(), + } + } + } + release_index = {"assets": {zip_name: 123}, "entries": {}} + + def fail_download(_url, _destination): + raise AssertionError("malformed new index must fail before downloading archives") + + original_download = MODULE.download_url + try: + MODULE.download_url = fail_download + with self.assertRaisesRegex(RuntimeError, "assetSha256"): + MODULE.validate_release_assets( + release, + manifest, + runtime_platform="linux/amd64", + release_index=release_index, + ) + finally: + MODULE.download_url = original_download + + def test_main_downloads_index_and_avoids_cross_platform_archive(self): + name = "mongodb-driver-agent-v1-linux-arm64" + zip_name = f"{name}.zip" + payload = b"driver" + archive_size = 123 + archive_sha = hashlib.sha256(b"archive").hexdigest() + manifest = { + "assets": { + name: { + "platform": "linux/arm64", + "revision": "src-expected", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + } + } + release_index = { + "assets": {zip_name: archive_size}, + "assetSha256": {zip_name: archive_sha}, + "entries": { + name: { + "archive": zip_name, + "path": f"Linux/{name}", + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + }, + } + release = { + "assets": [ + { + "name": MODULE.MANIFEST_ASSET_NAME, + "browser_download_url": "https://example.test/manifest", + }, + { + "name": MODULE.INDEX_ASSET_NAME, + "browser_download_url": "https://example.test/index", + }, + { + "name": zip_name, + "size": archive_size, + "digest": f"sha256:{archive_sha}", + "browser_download_url": f"https://example.test/{zip_name}", + }, + ] + } + downloaded_urls = [] + + def fake_download(url, destination): + downloaded_urls.append(url) + if url.endswith("/manifest"): + destination.write_text(json.dumps(manifest), encoding="utf-8") + elif url.endswith("/index"): + destination.write_text(json.dumps(release_index), encoding="utf-8") + else: + raise AssertionError("main should not download a cross-platform archive") + + original_load_release = MODULE.load_release + original_download = MODULE.download_url + original_runtime = MODULE.current_runtime_platform + original_argv = sys.argv + try: + MODULE.load_release = lambda _repo, _tag: release + MODULE.download_url = fake_download + MODULE.current_runtime_platform = lambda: "darwin/arm64" + sys.argv = [str(MODULE_PATH), "--tag", "dev-latest"] + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(MODULE.main(), 0) + finally: + MODULE.load_release = original_load_release + MODULE.download_url = original_download + MODULE.current_runtime_platform = original_runtime + sys.argv = original_argv + + self.assertEqual( + downloaded_urls, + ["https://example.test/manifest", "https://example.test/index"], + ) + def test_validate_release_assets_reports_sha_mismatch(self): release = { "assets": [ diff --git a/tools/vps-release-commit.sh b/tools/vps-release-commit.sh index 893fe912..dd0da67f 100644 --- a/tools/vps-release-commit.sh +++ b/tools/vps-release-commit.sh @@ -48,6 +48,19 @@ case "${staging_dir}" in ;; esac +lock_dir="" +cleanup() { + local exit_code=$? + trap - EXIT + set +e + if [[ -n "${lock_dir}" ]]; then + rmdir "${lock_dir}" 2>/dev/null || true + fi + rm -rf -- "${staging_dir}" + exit "${exit_code}" +} +trap cleanup EXIT + [[ "${channel}" == "stable" || "${channel}" == "dev" ]] || { echo "invalid channel: ${channel}" >&2 exit 1 @@ -77,7 +90,6 @@ if command -v flock >/dev/null 2>&1; then else lock_dir="${root}/.deploy.lock.d" mkdir "${lock_dir}" - trap 'rmdir "${lock_dir}" 2>/dev/null || true' EXIT fi if command -v sha256sum >/dev/null 2>&1; then diff --git a/tools/vps-release-commit.test.sh b/tools/vps-release-commit.test.sh index 08ad7c2d..770d5291 100644 --- a/tools/vps-release-commit.test.sh +++ b/tools/vps-release-commit.test.sh @@ -5,18 +5,23 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEMP_DIR="$(mktemp -d)" trap 'rm -rf "${TEMP_DIR}"' EXIT +# The no-flock fallback must share the staging cleanup trap instead of replacing it. +test "$(grep -c '^trap cleanup EXIT$' "${SCRIPT_DIR}/vps-release-commit.sh")" -eq 1 +test "$(grep -c "trap 'rmdir" "${SCRIPT_DIR}/vps-release-commit.sh")" -eq 0 + ROOT="${TEMP_DIR}/mirror" SOURCE="${TEMP_DIR}/source" mkdir -p "${ROOT}/.incoming" "${SOURCE}/app" "${SOURCE}/driver" printf '%s\n' 'gonavi-download-mirror-v1' > "${ROOT}/.gonavi-mirror-root" printf '%s' 'portable zip' > "${SOURCE}/app/GoNavi.zip" -printf '%s' 'driver binary' > "${SOURCE}/driver/mysql.exe" +printf '%s' 'driver archive' > "${SOURCE}/driver/mysql-driver-agent-windows-amd64.zip" python3 - "${SOURCE}" <<'PY' import hashlib import json import pathlib import sys +import zipfile source = pathlib.Path(sys.argv[1]) app = (source / "app/GoNavi.zip").read_bytes() @@ -28,9 +33,27 @@ app = (source / "app/GoNavi.zip").read_bytes() "sha256": hashlib.sha256(app).hexdigest(), }], })) -assets = {"mysql.exe": (source / "driver/mysql.exe").stat().st_size} -(source / "driver/version.json").write_text(json.dumps({"tagName": "v1.2.3", "assets": assets})) -(source / "driver/latest.json").write_text(json.dumps({"tagName": "v1.2.3", "assets": assets})) +driver_name = "mysql-driver-agent-windows-amd64.zip" +driver_entry_name = "mysql-driver-agent-windows-amd64.exe" +driver_entry_path = f"Windows/{driver_entry_name}" +driver_entry = b"driver binary" +with zipfile.ZipFile(source / "driver" / driver_name, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(driver_entry_path, driver_entry) +driver_archive = (source / "driver" / driver_name).read_bytes() +index = { + "assets": {driver_name: len(driver_archive)}, + "assetSha256": {driver_name: hashlib.sha256(driver_archive).hexdigest()}, + "entries": { + driver_entry_name: { + "archive": driver_name, + "path": driver_entry_path, + "size": len(driver_entry), + "sha256": hashlib.sha256(driver_entry).hexdigest(), + } + }, +} +(source / "driver/version.json").write_text(json.dumps({**index, "tagName": "v1.2.3"})) +(source / "driver/latest.json").write_text(json.dumps({**index, "tagName": "latest", "mirrorTagName": "v1.2.3"})) PY prepare_stable() { @@ -55,7 +78,7 @@ bash "${SCRIPT_DIR}/vps-release-commit.sh" \ test -f "${ROOT}/gonavi/releases/download/v1.2.3/GoNavi.zip" test -f "${ROOT}/gonavi/releases/latest/latest.json" -test -f "${ROOT}/drivers/releases/download/v1.2.3/mysql.exe" +test -f "${ROOT}/drivers/releases/download/v1.2.3/mysql-driver-agent-windows-amd64.zip" test -f "${ROOT}/drivers/releases/latest/GoNavi-DriverAgents-Index.json" test ! -e "${ROOT}/gonavi/releases/download/v0.9.0" test ! -e "${ROOT}/drivers/releases/download/v0.9.0" @@ -98,5 +121,29 @@ if bash "${SCRIPT_DIR}/vps-release-commit.sh" \ exit 1 fi test ! -e "${ROOT}/gonavi/dev/releases/latest/latest-dev.json" +test ! -e "${ROOT}/.incoming/dev-1" +test ! -e "${ROOT}/.deploy.lock.d" + +# A driver-disabled dev deployment must not prune stable or existing dev driver assets. +mkdir -p "${ROOT}/gonavi/dev/releases/download/dev-old" \ + "${ROOT}/drivers/dev/releases/download/dev-driver-old" \ + "${ROOT}/drivers/dev/releases/latest" +printf '%s' 'old driver pointer' > "${ROOT}/drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json" +python3 "${SCRIPT_DIR}/prepare-vps-release-payload.py" \ + --channel dev \ + --app-tag dev-abc123 \ + --app-dir "${SOURCE}/app" \ + --app-manifest "${SOURCE}/app/latest-dev.json" \ + --output "${ROOT}/.incoming/dev-2" >/dev/null +bash "${SCRIPT_DIR}/vps-release-commit.sh" \ + "${ROOT}" "${ROOT}/.incoming/dev-2" dev dev-abc123 '' >/dev/null + +test -f "${ROOT}/gonavi/dev/releases/download/dev-abc123/GoNavi-dev.zip" +test ! -e "${ROOT}/gonavi/dev/releases/download/dev-old" +test -d "${ROOT}/drivers/dev/releases/download/dev-driver-old" +test "$(cat "${ROOT}/drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json")" = 'old driver pointer' +test -d "${ROOT}/gonavi/releases/download/v1.2.3" +test -d "${ROOT}/drivers/releases/download/v1.2.3" +test ! -e "${ROOT}/.incoming/dev-2" echo "vps release commit tests passed"