mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-12 09:34:05 +08:00
⚡️ perf(driver/release): 优化驱动镜像下载与发布链路
- 将各平台驱动拆分为独立 ZIP,并为 DuckDB 同包携带运行库 - 仅生成 ZIP 下载候选,按 Gatewaysentry 与 GitHub 短测速结果调整优先级 - 扩展发布索引的归档摘要与条目元数据,强化跨平台资产校验 - 禁止裸驱动与 CI 总包进入 VPS 镜像,并校验载荷摘要和索引一致性 - 显式控制驱动镜像发布,自动清理失败暂存、过期目录和旧版本
This commit is contained in:
90
.github/actions/publish-vps-mirror/action.yml
vendored
90
.github/actions/publish-vps-mirror/action.yml
vendored
@@ -14,6 +14,9 @@ inputs:
|
||||
app-manifest:
|
||||
description: Application latest manifest path
|
||||
required: true
|
||||
driver-enabled:
|
||||
description: Whether this deployment must publish driver assets
|
||||
required: true
|
||||
driver-tag:
|
||||
description: Physical driver release directory name
|
||||
required: false
|
||||
@@ -56,6 +59,7 @@ runs:
|
||||
MIRROR_APP_TAG: ${{ inputs.app-tag }}
|
||||
MIRROR_APP_DIR: ${{ inputs.app-dir }}
|
||||
MIRROR_APP_MANIFEST: ${{ inputs.app-manifest }}
|
||||
MIRROR_DRIVER_ENABLED: ${{ inputs.driver-enabled }}
|
||||
MIRROR_DRIVER_TAG: ${{ inputs.driver-tag }}
|
||||
MIRROR_DRIVER_DIR: ${{ inputs.driver-dir }}
|
||||
MIRROR_DRIVER_VERSION_INDEX: ${{ inputs.driver-version-index }}
|
||||
@@ -68,11 +72,36 @@ runs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mirror_root="/srv/gonavi-downloads"
|
||||
[[ "${MIRROR_CHANNEL}" == "stable" || "${MIRROR_CHANNEL}" == "dev" ]] || {
|
||||
echo "Invalid mirror channel: ${MIRROR_CHANNEL}" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[0-9]+$ ]] || {
|
||||
echo "Invalid GitHub run identity" >&2
|
||||
exit 1
|
||||
}
|
||||
deployment_id="${MIRROR_CHANNEL}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
stage_dir="${RUNNER_TEMP}/gonavi-vps-mirror-${deployment_id}"
|
||||
ssh_dir="${RUNNER_TEMP}/gonavi-vps-ssh-${deployment_id}"
|
||||
remote_stage="${mirror_root}/.incoming/${deployment_id}"
|
||||
effective_driver_tag=""
|
||||
remote=""
|
||||
remote_cleanup_enabled=false
|
||||
ssh_options=()
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
local cleanup_remote_command=""
|
||||
trap - EXIT
|
||||
set +e
|
||||
if [[ "${remote_cleanup_enabled}" == true && -n "${remote}" ]]; then
|
||||
printf -v cleanup_remote_command 'rm -rf -- %q' "${remote_stage}"
|
||||
ssh "${ssh_options[@]}" "${remote}" "${cleanup_remote_command}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf -- "${ssh_dir}" "${stage_dir}"
|
||||
exit "${exit_code}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
[[ "${MIRROR_SSH_PORT}" =~ ^[0-9]+$ ]] || {
|
||||
echo "Invalid mirror SSH port" >&2
|
||||
@@ -92,15 +121,44 @@ runs:
|
||||
--app-manifest "${MIRROR_APP_MANIFEST}"
|
||||
--output "${stage_dir}"
|
||||
)
|
||||
if [[ -n "${MIRROR_DRIVER_LATEST_INDEX}" && -s "${MIRROR_DRIVER_LATEST_INDEX}" ]]; then
|
||||
effective_driver_tag="${MIRROR_DRIVER_TAG}"
|
||||
prepare_args+=(
|
||||
--driver-tag "${effective_driver_tag}"
|
||||
--driver-dir "${MIRROR_DRIVER_DIR}"
|
||||
--driver-version-index "${MIRROR_DRIVER_VERSION_INDEX}"
|
||||
--driver-latest-index "${MIRROR_DRIVER_LATEST_INDEX}"
|
||||
)
|
||||
fi
|
||||
case "${MIRROR_DRIVER_ENABLED}" in
|
||||
true)
|
||||
[[ -n "${MIRROR_DRIVER_TAG}" ]] || {
|
||||
echo "Driver mirror publication is enabled but driver-tag is empty" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -n "${MIRROR_DRIVER_DIR}" && -d "${MIRROR_DRIVER_DIR}" ]] || {
|
||||
echo "Driver mirror publication is enabled but driver-dir is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -n "$(find "${MIRROR_DRIVER_DIR}" -maxdepth 1 -type f -print -quit)" ]] || {
|
||||
echo "Driver mirror publication is enabled but driver-dir is empty" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -n "${MIRROR_DRIVER_VERSION_INDEX}" && -s "${MIRROR_DRIVER_VERSION_INDEX}" ]] || {
|
||||
echo "Driver mirror publication is enabled but driver-version-index is missing or empty" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -n "${MIRROR_DRIVER_LATEST_INDEX}" && -s "${MIRROR_DRIVER_LATEST_INDEX}" ]] || {
|
||||
echo "Driver mirror publication is enabled but driver-latest-index is missing or empty" >&2
|
||||
exit 1
|
||||
}
|
||||
effective_driver_tag="${MIRROR_DRIVER_TAG}"
|
||||
prepare_args+=(
|
||||
--driver-tag "${effective_driver_tag}"
|
||||
--driver-dir "${MIRROR_DRIVER_DIR}"
|
||||
--driver-version-index "${MIRROR_DRIVER_VERSION_INDEX}"
|
||||
--driver-latest-index "${MIRROR_DRIVER_LATEST_INDEX}"
|
||||
)
|
||||
;;
|
||||
false)
|
||||
echo "Driver mirror publication explicitly disabled"
|
||||
;;
|
||||
*)
|
||||
echo "Invalid driver-enabled value: ${MIRROR_DRIVER_ENABLED}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
python3 "${GITHUB_WORKSPACE}/tools/prepare-vps-release-payload.py" "${prepare_args[@]}"
|
||||
cp "${GITHUB_WORKSPACE}/tools/vps-release-commit.sh" "${stage_dir}/vps-release-commit.sh"
|
||||
chmod 0755 "${stage_dir}/vps-release-commit.sh"
|
||||
@@ -114,6 +172,17 @@ runs:
|
||||
-o "UserKnownHostsFile=${ssh_dir}/known_hosts"
|
||||
)
|
||||
remote="${MIRROR_SSH_USER}@${MIRROR_SSH_HOST}"
|
||||
printf -v stale_cleanup_script '%s\n' \
|
||||
'set -euo pipefail' \
|
||||
"root=$(printf '%q' "${mirror_root}")" \
|
||||
'marker="$(cat "${root}/.gonavi-mirror-root" 2>/dev/null || true)"' \
|
||||
'[[ "${marker}" == "gonavi-download-mirror-v1" ]] || { echo "mirror root marker is missing or invalid" >&2; exit 1; }' \
|
||||
'incoming="${root}/.incoming"' \
|
||||
'mkdir -p "${incoming}"' \
|
||||
'find "${incoming}" -mindepth 1 -maxdepth 1 -type d -mmin +1440 -exec rm -rf -- {} +'
|
||||
printf -v stale_cleanup_command 'bash -c %q' "${stale_cleanup_script}"
|
||||
ssh "${ssh_options[@]}" "${remote}" "${stale_cleanup_command}"
|
||||
|
||||
payload_bytes="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["payloadBytes"])' "${stage_dir}/deployment.json")"
|
||||
available_kib="$(ssh "${ssh_options[@]}" "${remote}" "LC_ALL=C df -Pk '${mirror_root}' | awk 'NR == 2 { print \$4 }'")"
|
||||
[[ "${available_kib}" =~ ^[0-9]+$ ]] || {
|
||||
@@ -127,6 +196,7 @@ runs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
remote_cleanup_enabled=true
|
||||
ssh "${ssh_options[@]}" "${remote}" "mkdir -p '${remote_stage}'"
|
||||
rsync -rlt --delete --partial \
|
||||
--chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r \
|
||||
@@ -141,5 +211,3 @@ runs:
|
||||
"${MIRROR_APP_TAG}" \
|
||||
"${effective_driver_tag}"
|
||||
ssh "${ssh_options[@]}" "${remote}" "${remote_command}"
|
||||
|
||||
rm -rf "${ssh_dir}" "${stage_dir}"
|
||||
|
||||
1
.github/workflows/dev-build.yml
vendored
1
.github/workflows/dev-build.yml
vendored
@@ -1492,6 +1492,7 @@ jobs:
|
||||
app-tag: ${{ steps.version.outputs.version }}
|
||||
app-dir: release-assets
|
||||
app-manifest: release-assets/latest-dev.json
|
||||
driver-enabled: ${{ steps.driver_assets.outputs.has_driver_assets }}
|
||||
driver-tag: ${{ steps.version.outputs.version }}
|
||||
driver-dir: driver-release-assets
|
||||
driver-version-index: ${{ runner.temp }}/driver-dev-latest-index.json
|
||||
|
||||
3
.github/workflows/publish-release.yml
vendored
3
.github/workflows/publish-release.yml
vendored
@@ -159,6 +159,7 @@ jobs:
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Prepare and verify stable mirror payload
|
||||
id: mirror_payload
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
DRIVER_RELEASE_TOKEN: ${{ secrets.DRIVER_RELEASE_TOKEN }}
|
||||
@@ -201,6 +202,7 @@ jobs:
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
echo "has_driver_release=${has_driver_release}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
jq -e \
|
||||
--arg tag "${RELEASE_TAG}" \
|
||||
@@ -325,6 +327,7 @@ jobs:
|
||||
app-tag: ${{ steps.validate.outputs.tag }}
|
||||
app-dir: ${{ runner.temp }}/gonavi-release
|
||||
app-manifest: ${{ runner.temp }}/gonavi-release/latest.json
|
||||
driver-enabled: ${{ steps.mirror_payload.outputs.has_driver_release }}
|
||||
driver-tag: ${{ steps.validate.outputs.tag }}
|
||||
driver-dir: ${{ runner.temp }}/driver-release
|
||||
driver-version-index: ${{ runner.temp }}/driver-release/GoNavi-DriverAgents-Index.json
|
||||
|
||||
@@ -364,7 +364,6 @@ const (
|
||||
driverReleaseLatestAPIURL = "https://api.github.com/repos/" + driverReleaseRepo + "/releases/latest"
|
||||
driverReleaseDevTag = "dev-latest"
|
||||
optionalDriverBundleAssetName = "GoNavi-DriverAgents.zip"
|
||||
duckDBWindowsDriverZipAssetName = "duckdb-driver.zip"
|
||||
optionalDriverBundleIndexAssetName = "GoNavi-DriverAgents-Index.json"
|
||||
optionalDriverBundleDownloadTimeout = 15 * time.Minute
|
||||
optionalDriverBundleCacheMaxAge = 7 * 24 * time.Hour
|
||||
@@ -1160,7 +1159,7 @@ func (a *App) GetDriverVersionPackageSize(driverType string, version string) con
|
||||
if err := a.localizeDriverSelectionError(definition, validateDriverSelectedVersion(definition, normalizedVersion)); err != nil {
|
||||
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||
}
|
||||
assetName := optionalDriverReleaseAssetNameForVersion(normalizedType, normalizedVersion)
|
||||
assetName := optionalDriverReleaseZipAssetNameForVersion(normalizedType, normalizedVersion)
|
||||
if strings.TrimSpace(assetName) == "" {
|
||||
return connection.QueryResult{Success: false, Message: a.appText("driver_manager.backend.error.asset_name_empty", nil)}
|
||||
}
|
||||
@@ -2372,28 +2371,7 @@ func resolvePublishedDriverDownloadURLForTag(definition driverDefinition, select
|
||||
}
|
||||
|
||||
func resolvePublishedDriverReleaseAssetName(driverType string, version string, tag string) (string, bool) {
|
||||
if shouldUseDuckDBWindowsDynamicLibrary(driverType) {
|
||||
cacheKey := "tag:" + strings.TrimSpace(tag)
|
||||
if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache(cacheKey); ok {
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
return duckDBWindowsDriverZipAssetName, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached(cacheKey, func() (*githubRelease, error) {
|
||||
return fetchReleaseByTag(tag)
|
||||
})
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
return duckDBWindowsDriverZipAssetName, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version)
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version)
|
||||
if len(assetNames) == 0 {
|
||||
return "", false
|
||||
}
|
||||
@@ -2435,7 +2413,7 @@ func resolveDriverVersionPackageSizeBytes(definition driverDefinition, option dr
|
||||
if version == "" {
|
||||
return 0
|
||||
}
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version)
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version)
|
||||
if len(assetNames) == 0 {
|
||||
return 0
|
||||
}
|
||||
@@ -2767,8 +2745,8 @@ func resolveDriverVersionOptionsFromReleases(definition driverDefinition) []driv
|
||||
if tag == "" || version == "" {
|
||||
continue
|
||||
}
|
||||
assetName := optionalDriverReleaseAssetNameForVersion(driverType, version)
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, version)
|
||||
assetName := optionalDriverReleaseZipAssetNameForVersion(driverType, version)
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, version)
|
||||
if !releaseContainsAnyAsset(release, assetNames) {
|
||||
continue
|
||||
}
|
||||
@@ -4154,12 +4132,13 @@ func ensureOptionalDriverAgentBinary(a *App, definition driverDefinition, execut
|
||||
}
|
||||
|
||||
if !forceSourceBuild {
|
||||
downloadURLs = reorderOptionalDriverDownloadURLsBySpeed(downloadURLs)
|
||||
if len(downloadURLs) > 0 {
|
||||
for _, candidateURL := range downloadURLs {
|
||||
if a != nil {
|
||||
a.emitDriverDownloadProgress(driverType, "downloading", 20, 100, a.appText("driver_manager.progress.download_prebuilt_agent", map[string]any{"name": displayName}))
|
||||
}
|
||||
hash, dlErr := downloadOptionalDriverAgentBinary(a, definition, candidateURL, executablePath)
|
||||
hash, dlErr := downloadOptionalDriverAgentBinary(a, definition, candidateURL, executablePath, selectedVersion)
|
||||
if dlErr == nil {
|
||||
if revisionErr := validateCandidateRevision(); revisionErr != nil {
|
||||
logger.Warnf("预编译 %s 驱动代理 revision 校验失败,url=%s err=%v", displayName, candidateURL, revisionErr)
|
||||
@@ -4264,13 +4243,10 @@ func formatOptionalDriverAttemptError(a *App, source string, err error) string {
|
||||
}
|
||||
|
||||
func shouldUseOptionalDriverBundleFallback(driverType string, restrictToExplicitArtifact bool, directURLCount int) bool {
|
||||
if restrictToExplicitArtifact {
|
||||
return false
|
||||
}
|
||||
if shouldSkipDirectOptionalDriverDownloads(driverType) {
|
||||
return true
|
||||
}
|
||||
return directURLCount == 0
|
||||
_ = driverType
|
||||
_ = restrictToExplicitArtifact
|
||||
_ = directURLCount
|
||||
return false
|
||||
}
|
||||
|
||||
func isOptionalDriverDownloadZipURL(urlText string) bool {
|
||||
@@ -4290,7 +4266,7 @@ func isOptionalDriverDownloadZipURL(urlText string) bool {
|
||||
return strings.EqualFold(filepath.Ext(trimmedURL), ".zip")
|
||||
}
|
||||
|
||||
func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlText string, executablePath string) (string, error) {
|
||||
func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlText string, executablePath string, selectedVersion string) (string, error) {
|
||||
driverType := normalizeDriverType(definition.Type)
|
||||
displayName := resolveDriverDisplayName(definition)
|
||||
trimmedURL := strings.TrimSpace(urlText)
|
||||
@@ -4312,7 +4288,7 @@ func downloadOptionalDriverAgentBinary(a *App, definition driverDefinition, urlT
|
||||
return "", newLocalizedDriverBackendError("driver_manager.backend.error.download_failed", nil, err)
|
||||
}
|
||||
|
||||
if _, err := installOptionalDriverAgentFromLocalZip(tempPath, definition, executablePath, ""); err != nil {
|
||||
if _, err := installOptionalDriverAgentFromLocalZip(tempPath, definition, executablePath, selectedVersion); err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
_ = os.Remove(executablePath)
|
||||
for _, supportName := range optionalDriverSupportFileNames(driverType) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@@ -21,6 +22,22 @@ import (
|
||||
"GoNavi-Wails/internal/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
optionalDriverDownloadProbeBytes = 256 << 10
|
||||
optionalDriverDownloadProbeMinBytes = 64 << 10
|
||||
optionalDriverDownloadProbeTimeout = 4 * time.Second
|
||||
optionalDriverDownloadProbeSpeedRatio = 1.5
|
||||
)
|
||||
|
||||
type optionalDriverDownloadProbeResult struct {
|
||||
URL string
|
||||
Bytes int64
|
||||
Duration time.Duration
|
||||
OK bool
|
||||
}
|
||||
|
||||
type optionalDriverDownloadProbeFunc func(context.Context, *http.Client, string) optionalDriverDownloadProbeResult
|
||||
|
||||
func optionalDriverPublicTypeName(driverType string) string {
|
||||
switch normalizeDriverType(driverType) {
|
||||
case "diros":
|
||||
@@ -54,6 +71,17 @@ func optionalDriverReleaseAssetNameForType(typeName string, goos string, goarch
|
||||
return name
|
||||
}
|
||||
|
||||
func optionalDriverReleaseZipAssetName(assetName string) string {
|
||||
name := strings.TrimSpace(assetName)
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.EqualFold(filepath.Ext(name), ".exe") {
|
||||
name = name[:len(name)-len(filepath.Ext(name))]
|
||||
}
|
||||
return name + ".zip"
|
||||
}
|
||||
|
||||
func optionalDriverNameStemCandidates(driverType string, selectedVersion string) []string {
|
||||
candidates := make([]string, 0, 3)
|
||||
seen := make(map[string]struct{}, 3)
|
||||
@@ -147,6 +175,36 @@ func optionalDriverReleaseAssetNames(driverType string) []string {
|
||||
return optionalDriverReleaseAssetNamesForVersion(driverType, "")
|
||||
}
|
||||
|
||||
func optionalDriverReleaseZipAssetNamesForVersion(driverType string, selectedVersion string) []string {
|
||||
rawNames := optionalDriverReleaseAssetNamesForVersion(driverType, selectedVersion)
|
||||
names := make([]string, 0, len(rawNames))
|
||||
seen := make(map[string]struct{}, len(rawNames))
|
||||
for _, rawName := range rawNames {
|
||||
name := optionalDriverReleaseZipAssetName(rawName)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func optionalDriverReleaseZipAssetNames(driverType string) []string {
|
||||
return optionalDriverReleaseZipAssetNamesForVersion(driverType, "")
|
||||
}
|
||||
|
||||
func optionalDriverReleaseZipAssetNameForVersion(driverType string, selectedVersion string) string {
|
||||
names := optionalDriverReleaseZipAssetNamesForVersion(driverType, selectedVersion)
|
||||
if len(names) == 0 {
|
||||
return optionalDriverReleaseZipAssetName(optionalDriverReleaseAssetNameForType("", stdRuntime.GOOS, stdRuntime.GOARCH))
|
||||
}
|
||||
return names[0]
|
||||
}
|
||||
|
||||
func optionalDriverExecutableBaseName(driverType string) string {
|
||||
names := optionalDriverExecutableBaseNames(driverType)
|
||||
if len(names) == 0 {
|
||||
@@ -393,7 +451,7 @@ func resolveOptionalDriverAssetSize(sizeByAsset map[string]int64, driverType str
|
||||
if len(sizeByAsset) == 0 {
|
||||
return 0
|
||||
}
|
||||
for _, assetName := range optionalDriverReleaseAssetNames(driverType) {
|
||||
for _, assetName := range optionalDriverReleaseZipAssetNames(driverType) {
|
||||
sizeBytes := sizeByAsset[assetName]
|
||||
if sizeBytes > 0 {
|
||||
return sizeBytes
|
||||
@@ -406,7 +464,7 @@ func resolveOptionalDriverAssetSizeForVersion(sizeByAsset map[string]int64, driv
|
||||
if len(sizeByAsset) == 0 {
|
||||
return 0
|
||||
}
|
||||
for _, assetName := range optionalDriverReleaseAssetNamesForVersion(driverType, version) {
|
||||
for _, assetName := range optionalDriverReleaseZipAssetNamesForVersion(driverType, version) {
|
||||
sizeBytes := sizeByAsset[assetName]
|
||||
if sizeBytes > 0 {
|
||||
return sizeBytes
|
||||
@@ -634,13 +692,131 @@ func acquireOptionalDriverBundlePath(bundleURL string, onProgress func(downloade
|
||||
}
|
||||
}
|
||||
|
||||
func optionalDriverDownloadSource(rawURL string) string {
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(parsed.Hostname())) {
|
||||
case "download.syngnat.top":
|
||||
return "mirror"
|
||||
case "github.com", "api.github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com":
|
||||
return "github"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func probeOptionalDriverDownloadURL(ctx context.Context, client *http.Client, rawURL string) optionalDriverDownloadProbeResult {
|
||||
result := optionalDriverDownloadProbeResult{URL: strings.TrimSpace(rawURL)}
|
||||
if result.URL == "" || client == nil {
|
||||
return result
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, result.URL, nil)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", optionalDriverDownloadProbeBytes-1))
|
||||
applyGitHubDownloadRequestHeaders(req, isGitHubReleaseAssetAPIURL(result.URL))
|
||||
|
||||
startedAt := time.Now()
|
||||
resp, err := doUpdateRequest(client, req)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return result
|
||||
}
|
||||
|
||||
written, err := io.Copy(io.Discard, io.LimitReader(resp.Body, optionalDriverDownloadProbeBytes))
|
||||
result.Duration = time.Since(startedAt)
|
||||
result.Bytes = written
|
||||
result.OK = err == nil && written >= optionalDriverDownloadProbeMinBytes && result.Duration > 0
|
||||
return result
|
||||
}
|
||||
|
||||
func reorderOptionalDriverDownloadURLsBySpeedWithProbe(urls []string, probe optionalDriverDownloadProbeFunc) []string {
|
||||
ordered := append([]string(nil), urls...)
|
||||
if len(ordered) < 2 || probe == nil {
|
||||
return ordered
|
||||
}
|
||||
|
||||
mirrorURL := ""
|
||||
githubURL := ""
|
||||
for _, candidate := range ordered {
|
||||
if !isOptionalDriverDownloadZipURL(candidate) {
|
||||
continue
|
||||
}
|
||||
switch optionalDriverDownloadSource(candidate) {
|
||||
case "mirror":
|
||||
if mirrorURL == "" {
|
||||
mirrorURL = candidate
|
||||
}
|
||||
case "github":
|
||||
if githubURL == "" {
|
||||
githubURL = candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
if mirrorURL == "" || githubURL == "" {
|
||||
return ordered
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), optionalDriverDownloadProbeTimeout)
|
||||
defer cancel()
|
||||
client := newHTTPClientWithGlobalProxy(optionalDriverDownloadProbeTimeout)
|
||||
results := make(chan optionalDriverDownloadProbeResult, 2)
|
||||
for _, candidate := range []string{mirrorURL, githubURL} {
|
||||
candidateURL := candidate
|
||||
go func() {
|
||||
results <- probe(ctx, client, candidateURL)
|
||||
}()
|
||||
}
|
||||
|
||||
measured := make(map[string]optionalDriverDownloadProbeResult, 2)
|
||||
for remaining := 2; remaining > 0; remaining-- {
|
||||
select {
|
||||
case result := <-results:
|
||||
measured[result.URL] = result
|
||||
case <-ctx.Done():
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
|
||||
mirrorResult := measured[mirrorURL]
|
||||
githubResult := measured[githubURL]
|
||||
preferGitHub := githubResult.OK && !mirrorResult.OK
|
||||
if githubResult.OK && mirrorResult.OK {
|
||||
mirrorSpeed := float64(mirrorResult.Bytes) / mirrorResult.Duration.Seconds()
|
||||
githubSpeed := float64(githubResult.Bytes) / githubResult.Duration.Seconds()
|
||||
preferGitHub = githubSpeed >= mirrorSpeed*optionalDriverDownloadProbeSpeedRatio
|
||||
}
|
||||
if !preferGitHub {
|
||||
return ordered
|
||||
}
|
||||
|
||||
reordered := make([]string, 0, len(ordered))
|
||||
reordered = append(reordered, githubURL)
|
||||
for _, candidate := range ordered {
|
||||
if candidate != githubURL {
|
||||
reordered = append(reordered, candidate)
|
||||
}
|
||||
}
|
||||
return reordered
|
||||
}
|
||||
|
||||
func reorderOptionalDriverDownloadURLsBySpeed(urls []string) []string {
|
||||
return reorderOptionalDriverDownloadURLsBySpeedWithProbe(urls, probeOptionalDriverDownloadURL)
|
||||
}
|
||||
|
||||
func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL string, selectedVersion string) []string {
|
||||
candidates := make([]string, 0, 6)
|
||||
seen := make(map[string]struct{}, 6)
|
||||
driverType := normalizeDriverType(definition.Type)
|
||||
appendURL := func(value string) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
if trimmed == "" || !isOptionalDriverDownloadZipURL(trimmed) {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[trimmed]; ok {
|
||||
@@ -689,7 +865,7 @@ func resolveOptionalDriverAgentDownloadURLs(definition driverDefinition, rawURL
|
||||
appendPublishedURLs()
|
||||
}
|
||||
|
||||
if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {
|
||||
if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil && isOptionalDriverDownloadZipURL(parsed.String()) {
|
||||
switch strings.ToLower(strings.TrimSpace(parsed.Scheme)) {
|
||||
case "http", "https":
|
||||
if tag, assetName, ok := driverReleaseDownloadCoordinates(parsed.String()); ok &&
|
||||
@@ -1376,35 +1552,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti
|
||||
if driverType == "" {
|
||||
return "", false
|
||||
}
|
||||
if shouldUseDuckDBWindowsDynamicLibrary(driverType) {
|
||||
if sizeByAsset, publishedAssets, ok := readReleaseAssetSizesFromCache("latest"); ok {
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
if release, err := fetchLatestReleaseForDriverAssets(); err == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached("latest", fetchLatestReleaseForDriverAssets)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if publishedAssets[duckDBWindowsDriverZipAssetName] && sizeByAsset[duckDBWindowsDriverZipAssetName] > 0 {
|
||||
if release, relErr := fetchLatestReleaseForDriverAssets(); relErr == nil {
|
||||
if asset, found := findReleaseAssetByName(release, []string{duckDBWindowsDriverZipAssetName}); found {
|
||||
return driverReleaseAssetAPIURL(asset), true
|
||||
}
|
||||
}
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(duckDBWindowsDriverZipAssetName), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(driverType, selectedVersion)
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(driverType, selectedVersion)
|
||||
if len(assetNames) == 0 {
|
||||
return "", false
|
||||
}
|
||||
@@ -1420,12 +1568,12 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true
|
||||
}
|
||||
|
||||
sizeByAsset, publishedAssets, err := loadReleaseAssetSizesCached("latest", fetchLatestReleaseForDriverAssets)
|
||||
if err != nil {
|
||||
return "", false
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true
|
||||
}
|
||||
for _, assetName := range assetNames {
|
||||
if publishedAssets[assetName] && sizeByAsset[assetName] > 0 {
|
||||
@@ -1437,7 +1585,7 @@ func resolveLatestPublishedDriverDownloadURLForVersion(definition driverDefiniti
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(assetName), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
return driverReleaseLatestDownloadURLForCurrentChannel(assetNames[0]), true
|
||||
}
|
||||
|
||||
func fetchReleaseByTag(tag string) (*githubRelease, error) {
|
||||
@@ -1484,7 +1632,7 @@ func fetchDriverReleaseIndexByURL(tag string, indexURL string) (*githubRelease,
|
||||
tagName = fallbackTag
|
||||
}
|
||||
if strings.EqualFold(fallbackTag, driverReleaseDevTag) {
|
||||
// dev alias 的逻辑 GitHub 标签固定为 dev-latest;mirrorTagName 仅控制 R2 物理路径。
|
||||
// dev alias 的逻辑 GitHub 标签固定为 dev-latest;mirrorTagName 仅控制镜像物理路径。
|
||||
tagName = driverReleaseDevTag
|
||||
}
|
||||
if tagName == "" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -83,7 +84,7 @@ func TestResolveVersionedDriverOptionUsesPublishedMongoV1Release(t *testing.T) {
|
||||
}
|
||||
|
||||
version := "1.17.4"
|
||||
assetName := mongoVersionedReleaseAssetName(1)
|
||||
assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1))
|
||||
seedReleaseAssetSizeCache(t, "tag:v"+version, map[string]int64{
|
||||
assetName: 24 << 20,
|
||||
})
|
||||
@@ -170,6 +171,78 @@ func TestDriverReleaseLatestDownloadURLForCurrentChannelUsesDevLatest(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionalDriverReleaseZipAssetNamesArePlatformNeutralArchives(t *testing.T) {
|
||||
if got := optionalDriverReleaseZipAssetName("mariadb-driver-agent-windows-amd64.exe"); got != "mariadb-driver-agent-windows-amd64.zip" {
|
||||
t.Fatalf("unexpected Windows ZIP asset name: %q", got)
|
||||
}
|
||||
if got := optionalDriverReleaseZipAssetName("mariadb-driver-agent-darwin-arm64"); got != "mariadb-driver-agent-darwin-arm64.zip" {
|
||||
t.Fatalf("unexpected Darwin ZIP asset name: %q", got)
|
||||
}
|
||||
MongoNames := optionalDriverReleaseZipAssetNamesForVersion("mongodb", "1.17.9")
|
||||
if len(MongoNames) != 1 || !strings.Contains(MongoNames[0], "mongodb-driver-agent-v1-") || !strings.HasSuffix(MongoNames[0], ".zip") {
|
||||
t.Fatalf("expected MongoDB v1 to resolve one versioned ZIP, got %v", MongoNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReorderOptionalDriverDownloadURLsBySpeedPrefersClearlyFasterGitHub(t *testing.T) {
|
||||
mirrorURL := driverMirrorReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip")
|
||||
githubURL := driverReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip")
|
||||
got := reorderOptionalDriverDownloadURLsBySpeedWithProbe(
|
||||
[]string{mirrorURL, githubURL},
|
||||
func(_ context.Context, _ *http.Client, rawURL string) optionalDriverDownloadProbeResult {
|
||||
duration := 2 * time.Second
|
||||
if optionalDriverDownloadSource(rawURL) == "github" {
|
||||
duration = 500 * time.Millisecond
|
||||
}
|
||||
return optionalDriverDownloadProbeResult{URL: rawURL, Bytes: optionalDriverDownloadProbeBytes, Duration: duration, OK: true}
|
||||
},
|
||||
)
|
||||
if len(got) != 2 || got[0] != githubURL || got[1] != mirrorURL {
|
||||
t.Fatalf("expected faster GitHub ZIP first without dropping mirror fallback, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReorderOptionalDriverDownloadURLsBySpeedKeepsMirrorForSimilarOrFailedGitHub(t *testing.T) {
|
||||
mirrorURL := driverMirrorReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip")
|
||||
githubURL := driverReleaseDownloadURL("v1.2.3", "mariadb-driver-agent-darwin-arm64.zip")
|
||||
for name, githubResult := range map[string]optionalDriverDownloadProbeResult{
|
||||
"similar": {Bytes: optionalDriverDownloadProbeBytes, Duration: 900 * time.Millisecond, OK: true},
|
||||
"failed": {},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got := reorderOptionalDriverDownloadURLsBySpeedWithProbe(
|
||||
[]string{mirrorURL, githubURL},
|
||||
func(_ context.Context, _ *http.Client, rawURL string) optionalDriverDownloadProbeResult {
|
||||
if optionalDriverDownloadSource(rawURL) == "github" {
|
||||
githubResult.URL = rawURL
|
||||
return githubResult
|
||||
}
|
||||
return optionalDriverDownloadProbeResult{URL: rawURL, Bytes: optionalDriverDownloadProbeBytes, Duration: time.Second, OK: true}
|
||||
},
|
||||
)
|
||||
if len(got) != 2 || got[0] != mirrorURL || got[1] != githubURL {
|
||||
t.Fatalf("expected mirror-first order to remain unchanged, got %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeOptionalDriverDownloadURLUsesBoundedRange(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Range"); got != "bytes=0-262143" {
|
||||
t.Errorf("unexpected Range header: %q", got)
|
||||
}
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(make([]byte, optionalDriverDownloadProbeBytes+1024))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result := probeOptionalDriverDownloadURL(context.Background(), server.Client(), server.URL+"/driver.zip")
|
||||
if !result.OK || result.Bytes != optionalDriverDownloadProbeBytes {
|
||||
t.Fatalf("expected a valid bounded probe, got %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing.T) {
|
||||
originalVersion := AppVersion
|
||||
AppVersion = "0.7.4"
|
||||
@@ -202,7 +275,7 @@ func TestResolveOptionalDriverBundleDownloadURLsUsesDriverReleaseRepo(t *testing
|
||||
t.Fatalf("expected bundle URLs to include mirror=%q, tagged=%q and latest=%q, got %v", wantMirror, wantTagged, wantLatest, urls)
|
||||
}
|
||||
if urls[0] != wantMirror {
|
||||
t.Fatalf("expected R2 mirror first, got %v", urls)
|
||||
t.Fatalf("expected mirror first, got %v", urls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,7 +368,7 @@ func TestFetchDriverReleaseIndexByURLBuildsMirrorAssets(t *testing.T) {
|
||||
|
||||
func TestResolvePublishedDriverDownloadURLForTagUsesDevMirrorTag(t *testing.T) {
|
||||
definition := driverDefinition{Type: "sqlserver"}
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "")
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(definition.Type, "")
|
||||
if len(assetNames) == 0 {
|
||||
t.Fatal("expected sqlserver release asset names")
|
||||
}
|
||||
@@ -325,10 +398,10 @@ func TestResolvePublishedDriverDownloadURLForTagUsesDevMirrorTag(t *testing.T) {
|
||||
|
||||
func TestResolveLatestPublishedDriverDownloadURLFallsBackWhenMirrorIndexMissesAsset(t *testing.T) {
|
||||
seedReleaseAssetSizeCache(t, "latest", map[string]int64{
|
||||
"unrelated-driver-agent-windows-amd64.exe": 123,
|
||||
"unrelated-driver-agent-windows-amd64.zip": 123,
|
||||
})
|
||||
definition := driverDefinition{Type: "sqlserver"}
|
||||
assetNames := optionalDriverReleaseAssetNamesForVersion(definition.Type, "")
|
||||
assetNames := optionalDriverReleaseZipAssetNamesForVersion(definition.Type, "")
|
||||
if len(assetNames) == 0 {
|
||||
t.Fatal("expected sqlserver release asset names")
|
||||
}
|
||||
@@ -543,7 +616,7 @@ func TestResolveDriverVersionPackageSizeBytesReadsMongoV1VersionedAsset(t *testi
|
||||
}
|
||||
|
||||
version := "1.17.4"
|
||||
assetName := mongoVersionedReleaseAssetName(1)
|
||||
assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1))
|
||||
const wantSize int64 = 31 << 20
|
||||
seedReleaseAssetSizeCache(t, "tag:v"+version, map[string]int64{
|
||||
assetName: wantSize,
|
||||
@@ -564,7 +637,8 @@ func TestResolveOptionalDriverAgentDownloadURLsDoesNotFallbackForHistoricalVersi
|
||||
t.Fatal("expected mongodb driver definition")
|
||||
}
|
||||
|
||||
explicitURL := driverReleaseDownloadURL("v1.17.4", mongoVersionedReleaseAssetName(1))
|
||||
zipAssetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1))
|
||||
explicitURL := driverReleaseDownloadURL("v1.17.4", zipAssetName)
|
||||
urls := resolveOptionalDriverAgentDownloadURLs(
|
||||
definition,
|
||||
explicitURL,
|
||||
@@ -573,7 +647,7 @@ func TestResolveOptionalDriverAgentDownloadURLsDoesNotFallbackForHistoricalVersi
|
||||
if len(urls) != 2 {
|
||||
t.Fatalf("expected mirror plus explicit historical URL, got %d candidates: %v", len(urls), urls)
|
||||
}
|
||||
if urls[0] != driverMirrorReleaseDownloadURL("v1.17.4", mongoVersionedReleaseAssetName(1)) || urls[1] != explicitURL {
|
||||
if urls[0] != driverMirrorReleaseDownloadURL("v1.17.4", zipAssetName) || urls[1] != explicitURL {
|
||||
t.Fatalf("unexpected historical URL candidate: %v", urls)
|
||||
}
|
||||
}
|
||||
@@ -590,7 +664,7 @@ func TestResolveOptionalDriverAgentDownloadURLsUsesMongoV1AssetForCompatibleDefa
|
||||
AppVersion = originalVersion
|
||||
})
|
||||
|
||||
assetName := mongoVersionedReleaseAssetName(1)
|
||||
assetName := optionalDriverReleaseZipAssetName(mongoVersionedReleaseAssetName(1))
|
||||
seedReleaseAssetSizeCache(t, "tag:v0.7.9", map[string]int64{
|
||||
assetName: 24 << 20,
|
||||
})
|
||||
@@ -662,24 +736,29 @@ func TestMongoDBVersionedAssetNamesDoNotFallbackToBaseForV1(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveOptionalDriverAgentDownloadURLsSkipsBundleOnlyDamengAsset(t *testing.T) {
|
||||
func TestResolveOptionalDriverAgentDownloadURLsUsesDamengZipAsset(t *testing.T) {
|
||||
definition, ok := resolveDriverDefinition("dameng")
|
||||
if !ok {
|
||||
t.Fatal("expected dameng driver definition")
|
||||
}
|
||||
|
||||
version := normalizeVersion(definition.PinnedVersion)
|
||||
assetName := optionalDriverReleaseAssetNameForVersion("dameng", version)
|
||||
assetName := optionalDriverReleaseZipAssetNameForVersion("dameng", version)
|
||||
seedReleaseAssetCacheEntry(t, "tag:v"+version, map[string]int64{
|
||||
assetName: 23 << 20,
|
||||
}, nil)
|
||||
}, map[string]int64{assetName: 23 << 20})
|
||||
seedReleaseAssetCacheEntry(t, "latest", map[string]int64{
|
||||
assetName: 23 << 20,
|
||||
}, nil)
|
||||
}, map[string]int64{assetName: 23 << 20})
|
||||
|
||||
urls := resolveOptionalDriverAgentDownloadURLs(definition, "builtin://activate/dameng", version)
|
||||
if len(urls) != 0 {
|
||||
t.Fatalf("expected bundle-only dameng install to skip direct asset URLs, got %v", urls)
|
||||
if len(urls) == 0 {
|
||||
t.Fatal("expected Dameng install to use the published standalone zip")
|
||||
}
|
||||
for _, candidate := range urls {
|
||||
if !isOptionalDriverDownloadZipURL(candidate) {
|
||||
t.Fatalf("expected zip-only Dameng candidates, got %v", urls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -689,9 +768,9 @@ func TestShouldUseOptionalDriverBundleFallbackSkipsWhenDirectAssetExists(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldUseOptionalDriverBundleFallbackKeepsBundleWhenDirectAssetMissing(t *testing.T) {
|
||||
if !shouldUseOptionalDriverBundleFallback("dameng", false, 0) {
|
||||
t.Fatal("expected missing single-file driver asset to keep bundle fallback")
|
||||
func TestShouldUseOptionalDriverBundleFallbackStaysDisabledWhenZipMissing(t *testing.T) {
|
||||
if shouldUseOptionalDriverBundleFallback("dameng", false, 0) {
|
||||
t.Fatal("expected ZIP-only installs not to download the CI driver bundle")
|
||||
}
|
||||
if shouldUseOptionalDriverBundleFallback("dameng", true, 0) {
|
||||
t.Fatal("expected explicit version artifact installs to skip bundle fallback")
|
||||
@@ -1228,22 +1307,20 @@ func TestDuckDBWindowsBuildUsesDynamicLibraryTag(t *testing.T) {
|
||||
if !shouldSkipReusableAgentCandidate("duckdb", "") {
|
||||
t.Fatal("expected DuckDB Windows install to skip reusable static agent candidates")
|
||||
}
|
||||
zipAssetName := optionalDriverReleaseZipAssetNameForVersion("duckdb", "")
|
||||
seedReleaseAssetCacheEntry(t, "latest", map[string]int64{
|
||||
duckDBWindowsDriverZipAssetName: 19 << 20,
|
||||
zipAssetName: 19 << 20,
|
||||
}, map[string]int64{
|
||||
duckDBWindowsDriverZipAssetName: 19 << 20,
|
||||
zipAssetName: 19 << 20,
|
||||
})
|
||||
legacyDirectURL := "https://example.com/duckdb-driver-agent-windows-amd64.exe"
|
||||
urls := resolveOptionalDriverAgentDownloadURLs(driverDefinition{Type: "duckdb"}, legacyDirectURL, "")
|
||||
if len(urls) < 2 {
|
||||
t.Fatalf("expected DuckDB Windows install to keep dedicated zip ahead of legacy direct candidate, got %v", urls)
|
||||
if len(urls) != 1 {
|
||||
t.Fatalf("expected DuckDB Windows install to use only the dedicated zip, got %v", urls)
|
||||
}
|
||||
if urls[0] != driverReleaseLatestDownloadURL(duckDBWindowsDriverZipAssetName) {
|
||||
if urls[0] != driverReleaseLatestDownloadURL(zipAssetName) {
|
||||
t.Fatalf("expected DuckDB Windows dedicated zip candidate first, got %v", urls)
|
||||
}
|
||||
if urls[1] != legacyDirectURL {
|
||||
t.Fatalf("expected DuckDB Windows to keep legacy direct candidate after dedicated zip, got %v", urls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDBWindowsDynamicLibraryCGOLDFlagsIncludeSupportLibraries(t *testing.T) {
|
||||
@@ -1328,7 +1405,8 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing.
|
||||
})
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
zipPath := filepath.Join(tmpDir, duckDBWindowsDriverZipAssetName)
|
||||
zipAssetName := optionalDriverReleaseZipAssetNameForVersion("duckdb", "")
|
||||
zipPath := filepath.Join(tmpDir, zipAssetName)
|
||||
zipFile, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
t.Fatalf("create zip failed: %v", err)
|
||||
@@ -1370,7 +1448,7 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing.
|
||||
t.Fatalf("create install dir failed: %v", err)
|
||||
}
|
||||
|
||||
hash, err := downloadOptionalDriverAgentBinary(nil, driverDefinition{Type: "duckdb", Name: "DuckDB"}, server.URL+"/"+duckDBWindowsDriverZipAssetName+"?source=release", target)
|
||||
hash, err := downloadOptionalDriverAgentBinary(nil, driverDefinition{Type: "duckdb", Name: "DuckDB"}, server.URL+"/"+zipAssetName+"?source=release", target, "")
|
||||
if err != nil {
|
||||
t.Fatalf("download dedicated zip failed: %v", err)
|
||||
}
|
||||
@@ -1389,6 +1467,69 @@ func TestDownloadOptionalDriverAgentBinaryInstallsDuckDBDedicatedZip(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadOptionalDriverAgentBinaryPreservesMongoSelectedVersion(t *testing.T) {
|
||||
originalValidateFunc := validateOptionalDriverAgentExecutableFunc
|
||||
validateOptionalDriverAgentExecutableFunc = func(driverType string, executablePath string) error {
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
validateOptionalDriverAgentExecutableFunc = originalValidateFunc
|
||||
})
|
||||
disableGlobalProxyForTest(t)
|
||||
for _, name := range []string{"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
rawAssetName := mongoVersionedReleaseAssetName(1)
|
||||
zipAssetName := optionalDriverReleaseZipAssetName(rawAssetName)
|
||||
zipPath := filepath.Join(tmpDir, zipAssetName)
|
||||
zipFile, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
t.Fatalf("create MongoDB ZIP: %v", err)
|
||||
}
|
||||
zw := zip.NewWriter(zipFile)
|
||||
entryPath := filepath.ToSlash(filepath.Join(optionalDriverBundlePlatformDir(runtime.GOOS), rawAssetName))
|
||||
entry, err := zw.Create(entryPath)
|
||||
if err != nil {
|
||||
t.Fatalf("create MongoDB v1 entry: %v", err)
|
||||
}
|
||||
if _, err := entry.Write([]byte("mongodb-v1-agent")); err != nil {
|
||||
t.Fatalf("write MongoDB v1 entry: %v", err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("close MongoDB ZIP: %v", err)
|
||||
}
|
||||
if err := zipFile.Close(); err != nil {
|
||||
t.Fatalf("close MongoDB ZIP file: %v", err)
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeFile(w, r, zipPath)
|
||||
}))
|
||||
defer server.Close()
|
||||
target := filepath.Join(tmpDir, "install", optionalDriverExecutableBaseNameForType("mongodb"))
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
t.Fatalf("create MongoDB install dir: %v", err)
|
||||
}
|
||||
if _, err := downloadOptionalDriverAgentBinary(
|
||||
nil,
|
||||
driverDefinition{Type: "mongodb", Name: "MongoDB"},
|
||||
server.URL+"/"+zipAssetName,
|
||||
target,
|
||||
"1.17.9",
|
||||
); err != nil {
|
||||
t.Fatalf("download MongoDB v1 ZIP: %v", err)
|
||||
}
|
||||
installed, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatalf("read installed MongoDB v1 agent: %v", err)
|
||||
}
|
||||
if string(installed) != "mongodb-v1-agent" {
|
||||
t.Fatalf("unexpected installed MongoDB v1 agent: %q", string(installed))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionalDriverDownloadZipURLAcceptsQueryString(t *testing.T) {
|
||||
if !isOptionalDriverDownloadZipURL("https://example.com/duckdb-driver.zip?token=abc") {
|
||||
t.Fatal("expected signed zip URL to be treated as zip download")
|
||||
@@ -1759,7 +1900,7 @@ func TestResolveOptionalDriverAgentDownloadURLsIncludesPublishedKingbaseAsset(t
|
||||
}
|
||||
|
||||
version := normalizeVersion(definition.PinnedVersion)
|
||||
assetName := optionalDriverReleaseAssetNameForVersion("kingbase", version)
|
||||
assetName := optionalDriverReleaseZipAssetNameForVersion("kingbase", version)
|
||||
publishedAssets := map[string]int64{
|
||||
assetName: 18 << 20,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
@@ -9,6 +11,53 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
LEGAL_FILENAMES = ("LICENSE", "NOTICE")
|
||||
DRIVER_AGENT_RE = re.compile(
|
||||
r"^.+-driver-agent(?:-v[0-9]+)?-(?:darwin|linux|windows)-(?:amd64|arm64)(?:\.exe)?$"
|
||||
)
|
||||
DUCKDB_WINDOWS_AGENT = "duckdb-driver-agent-windows-amd64.exe"
|
||||
DUCKDB_WINDOWS_LIBRARY = "duckdb.dll"
|
||||
|
||||
|
||||
def individual_archive_name(asset_name: str) -> str:
|
||||
if asset_name.lower().endswith(".exe"):
|
||||
return asset_name[:-4] + ".zip"
|
||||
return asset_name + ".zip"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_individual_archive(
|
||||
archive_path: Path,
|
||||
asset: Path,
|
||||
arcname: str,
|
||||
legal_files: list[Path],
|
||||
) -> list[tuple[Path, str]]:
|
||||
entries = [(asset, arcname)]
|
||||
if asset.name == DUCKDB_WINDOWS_AGENT:
|
||||
support_file = asset.with_name(DUCKDB_WINDOWS_LIBRARY)
|
||||
if not support_file.is_file():
|
||||
raise RuntimeError(
|
||||
f"DuckDB Windows runtime dependency not found: {support_file}"
|
||||
)
|
||||
entries.append(
|
||||
(
|
||||
support_file,
|
||||
(Path(arcname).parent / DUCKDB_WINDOWS_LIBRARY).as_posix(),
|
||||
)
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for source, entry_name in entries:
|
||||
zf.write(source, entry_name)
|
||||
for legal_file in legal_files:
|
||||
zf.write(legal_file, legal_file.name)
|
||||
return entries
|
||||
|
||||
|
||||
def main():
|
||||
@@ -40,21 +89,24 @@ def main():
|
||||
manifest_path = output_dir / manifest_name
|
||||
|
||||
size_index = {}
|
||||
standalone_assets = []
|
||||
archive_sha256_index = {}
|
||||
entry_index = {}
|
||||
individual_archives = []
|
||||
source_assets = []
|
||||
with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for asset in sorted(drivers_dir.rglob("*")):
|
||||
if not asset.is_file():
|
||||
continue
|
||||
arcname = asset.relative_to(drivers_dir).as_posix()
|
||||
if asset.name in size_index:
|
||||
if any(existing.name == asset.name for existing, _ in source_assets):
|
||||
raise RuntimeError(f"driver asset name conflict: {asset.name}")
|
||||
# Dedicated source packages are regenerated below from their raw
|
||||
# agent and support files. Keeping a zip inside the CI bundle only
|
||||
# wastes space and is not needed by the completion step.
|
||||
if asset.suffix.lower() == ".zip":
|
||||
continue
|
||||
zf.write(asset, arcname)
|
||||
size_index[asset.name] = asset.stat().st_size
|
||||
standalone_path = output_dir / asset.name
|
||||
if standalone_path.exists():
|
||||
raise RuntimeError(f"release asset already exists: {standalone_path}")
|
||||
shutil.copy2(asset, standalone_path)
|
||||
standalone_assets.append(standalone_path.name)
|
||||
source_assets.append((asset, arcname))
|
||||
|
||||
for legal_file in legal_files:
|
||||
zf.write(legal_file, legal_file.name)
|
||||
@@ -63,14 +115,53 @@ def main():
|
||||
raise RuntimeError(f"release asset already exists: {standalone_path}")
|
||||
shutil.copy2(legal_file, standalone_path)
|
||||
|
||||
for asset, arcname in source_assets:
|
||||
if not DRIVER_AGENT_RE.fullmatch(asset.name):
|
||||
continue
|
||||
archive_name = individual_archive_name(asset.name)
|
||||
archive_path = output_dir / archive_name
|
||||
if archive_path.exists():
|
||||
raise RuntimeError(f"release asset already exists: {archive_path}")
|
||||
archive_entries = write_individual_archive(
|
||||
archive_path,
|
||||
asset,
|
||||
arcname,
|
||||
legal_files,
|
||||
)
|
||||
size_index[archive_name] = archive_path.stat().st_size
|
||||
archive_sha256_index[archive_name] = sha256_file(archive_path)
|
||||
for source, entry_path in archive_entries:
|
||||
if source.name in entry_index:
|
||||
raise RuntimeError(f"driver archive entry name conflict: {source.name}")
|
||||
entry_index[source.name] = {
|
||||
"archive": archive_name,
|
||||
"path": entry_path,
|
||||
"size": source.stat().st_size,
|
||||
"sha256": sha256_file(source),
|
||||
}
|
||||
individual_archives.append(archive_name)
|
||||
|
||||
if not individual_archives:
|
||||
raise RuntimeError("no driver agent binaries found")
|
||||
|
||||
index_path.write_text(
|
||||
json.dumps({"assets": size_index}, ensure_ascii=False, indent=2) + "\n",
|
||||
json.dumps(
|
||||
{
|
||||
"assets": size_index,
|
||||
"assetSha256": archive_sha256_index,
|
||||
"entries": entry_index,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"created {out_name} size={out_path.stat().st_size} bytes")
|
||||
print(f"created {index_name} entries={len(size_index)}")
|
||||
print(f"published standalone driver assets={len(standalone_assets)}")
|
||||
print(f"published individual driver archives={len(individual_archives)}")
|
||||
print(f"bundled legal files={len(legal_files)}")
|
||||
print(f"reserved manifest output path: {manifest_path}")
|
||||
return 0
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -13,7 +14,7 @@ SCRIPT = ROOT / "tools" / "package-driver-release-assets.py"
|
||||
|
||||
|
||||
class PackageDriverReleaseAssetsTest(unittest.TestCase):
|
||||
def test_packages_bundle_and_standalone_assets(self):
|
||||
def test_packages_ci_bundle_and_individual_archives_without_raw_assets(self):
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
drivers_dir = tmpdir / "drivers"
|
||||
@@ -37,14 +38,78 @@ class PackageDriverReleaseAssetsTest(unittest.TestCase):
|
||||
|
||||
self.assertIn("created GoNavi-DriverAgents.zip", proc.stdout)
|
||||
self.assertTrue((output_dir / "GoNavi-DriverAgents.zip").is_file())
|
||||
self.assertTrue((output_dir / windows_asset.name).is_file())
|
||||
self.assertTrue((output_dir / darwin_asset.name).is_file())
|
||||
windows_archive = output_dir / "clickhouse-driver-agent-windows-amd64.zip"
|
||||
darwin_archive = output_dir / "clickhouse-driver-agent-darwin-arm64.zip"
|
||||
self.assertTrue(windows_archive.is_file())
|
||||
self.assertTrue(darwin_archive.is_file())
|
||||
self.assertFalse((output_dir / windows_asset.name).exists())
|
||||
self.assertFalse((output_dir / darwin_asset.name).exists())
|
||||
self.assertEqual((output_dir / "LICENSE").read_bytes(), (ROOT / "LICENSE").read_bytes())
|
||||
self.assertEqual((output_dir / "NOTICE").read_bytes(), (ROOT / "NOTICE").read_bytes())
|
||||
|
||||
index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(index["assets"][windows_asset.name], len(b"windows-asset"))
|
||||
self.assertEqual(index["assets"][darwin_asset.name], len(b"darwin-asset"))
|
||||
self.assertEqual(set(index), {"assets", "assetSha256", "entries"})
|
||||
self.assertEqual(
|
||||
index["assets"],
|
||||
{
|
||||
windows_archive.name: windows_archive.stat().st_size,
|
||||
darwin_archive.name: darwin_archive.stat().st_size,
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
index["assetSha256"],
|
||||
{
|
||||
windows_archive.name: hashlib.sha256(windows_archive.read_bytes()).hexdigest(),
|
||||
darwin_archive.name: hashlib.sha256(darwin_archive.read_bytes()).hexdigest(),
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
index["entries"],
|
||||
{
|
||||
windows_asset.name: {
|
||||
"archive": windows_archive.name,
|
||||
"path": "Windows/clickhouse-driver-agent-windows-amd64.exe",
|
||||
"size": len(b"windows-asset"),
|
||||
"sha256": hashlib.sha256(b"windows-asset").hexdigest(),
|
||||
},
|
||||
darwin_asset.name: {
|
||||
"archive": darwin_archive.name,
|
||||
"path": "MacOS/clickhouse-driver-agent-darwin-arm64",
|
||||
"size": len(b"darwin-asset"),
|
||||
"sha256": hashlib.sha256(b"darwin-asset").hexdigest(),
|
||||
},
|
||||
},
|
||||
)
|
||||
self.assertNotIn("GoNavi-DriverAgents.zip", index["assets"])
|
||||
self.assertNotIn("GoNavi-DriverAgents.zip", index["assetSha256"])
|
||||
|
||||
with zipfile.ZipFile(windows_archive) as zf:
|
||||
self.assertEqual(
|
||||
sorted(zf.namelist()),
|
||||
[
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
"Windows/clickhouse-driver-agent-windows-amd64.exe",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
zf.read("Windows/clickhouse-driver-agent-windows-amd64.exe"),
|
||||
b"windows-asset",
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(darwin_archive) as zf:
|
||||
self.assertEqual(
|
||||
sorted(zf.namelist()),
|
||||
[
|
||||
"LICENSE",
|
||||
"MacOS/clickhouse-driver-agent-darwin-arm64",
|
||||
"NOTICE",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
zf.read("MacOS/clickhouse-driver-agent-darwin-arm64"),
|
||||
b"darwin-asset",
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf:
|
||||
self.assertEqual(
|
||||
@@ -59,6 +124,76 @@ class PackageDriverReleaseAssetsTest(unittest.TestCase):
|
||||
self.assertEqual(zf.read("LICENSE"), (ROOT / "LICENSE").read_bytes())
|
||||
self.assertEqual(zf.read("NOTICE"), (ROOT / "NOTICE").read_bytes())
|
||||
|
||||
def test_rebuilds_duckdb_windows_archive_with_agent_and_library(self):
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-driver-assets-test-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
drivers_dir = tmpdir / "drivers"
|
||||
output_dir = tmpdir / "driver-release-assets"
|
||||
windows_dir = drivers_dir / "Windows"
|
||||
windows_dir.mkdir(parents=True)
|
||||
|
||||
agent = windows_dir / "duckdb-driver-agent-windows-amd64.exe"
|
||||
library = windows_dir / "duckdb.dll"
|
||||
agent.write_bytes(b"duckdb-agent")
|
||||
library.write_bytes(b"duckdb-library")
|
||||
(windows_dir / "duckdb-driver.zip").write_bytes(b"stale-source-package")
|
||||
|
||||
subprocess.run(
|
||||
["python3", str(SCRIPT), str(drivers_dir), str(output_dir)],
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
archive = output_dir / "duckdb-driver-agent-windows-amd64.zip"
|
||||
self.assertTrue(archive.is_file())
|
||||
self.assertFalse((output_dir / agent.name).exists())
|
||||
self.assertFalse((output_dir / library.name).exists())
|
||||
index = json.loads((output_dir / "GoNavi-DriverAgents-Index.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(index["assets"], {archive.name: archive.stat().st_size})
|
||||
self.assertEqual(
|
||||
index["assetSha256"],
|
||||
{archive.name: hashlib.sha256(archive.read_bytes()).hexdigest()},
|
||||
)
|
||||
self.assertEqual(
|
||||
index["entries"],
|
||||
{
|
||||
agent.name: {
|
||||
"archive": archive.name,
|
||||
"path": "Windows/duckdb-driver-agent-windows-amd64.exe",
|
||||
"size": len(b"duckdb-agent"),
|
||||
"sha256": hashlib.sha256(b"duckdb-agent").hexdigest(),
|
||||
},
|
||||
library.name: {
|
||||
"archive": archive.name,
|
||||
"path": "Windows/duckdb.dll",
|
||||
"size": len(b"duckdb-library"),
|
||||
"sha256": hashlib.sha256(b"duckdb-library").hexdigest(),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
self.assertEqual(
|
||||
sorted(zf.namelist()),
|
||||
[
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
"Windows/duckdb-driver-agent-windows-amd64.exe",
|
||||
"Windows/duckdb.dll",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
zf.read("Windows/duckdb-driver-agent-windows-amd64.exe"),
|
||||
b"duckdb-agent",
|
||||
)
|
||||
self.assertEqual(zf.read("Windows/duckdb.dll"), b"duckdb-library")
|
||||
|
||||
with zipfile.ZipFile(output_dir / "GoNavi-DriverAgents.zip") as zf:
|
||||
self.assertNotIn("Windows/duckdb-driver.zip", zf.namelist())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -16,6 +16,9 @@ from typing import Any
|
||||
|
||||
TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
WINDOWS_ABSOLUTE_PATH_RE = re.compile(r"^[A-Za-z]:/")
|
||||
DRIVER_CI_BUNDLE_NAME = "GoNavi-DriverAgents.zip"
|
||||
DRIVER_MUTABLE_INDEX_FIELDS = frozenset(("tagName", "mirrorTagName"))
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -60,6 +63,89 @@ def validate_asset_name(value: Any, label: str) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def is_nonnegative_int(value: Any) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
|
||||
|
||||
def validate_sha256(value: Any, label: str) -> str:
|
||||
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
|
||||
fail(f"invalid {label} sha256: {value!r}")
|
||||
return value.lower()
|
||||
|
||||
|
||||
def validate_driver_entry_path(value: Any, label: str) -> str:
|
||||
if not isinstance(value, str) or not value or "\\" in value:
|
||||
fail(f"invalid {label} path: {value!r}")
|
||||
parts = value.split("/")
|
||||
if (
|
||||
value.startswith("/")
|
||||
or WINDOWS_ABSOLUTE_PATH_RE.match(value)
|
||||
or any(part in {"", ".", ".."} for part in parts)
|
||||
):
|
||||
fail(f"invalid {label} path: {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
def validate_driver_index(index: dict[str, Any], label: str) -> None:
|
||||
assets = index.get("assets")
|
||||
asset_sha256 = index.get("assetSha256")
|
||||
entries = index.get("entries")
|
||||
if not isinstance(assets, dict) or not assets:
|
||||
fail(f"{label} assets must be a non-empty object")
|
||||
if not isinstance(asset_sha256, dict) or not asset_sha256:
|
||||
fail(f"{label} assetSha256 must be a non-empty object")
|
||||
if not isinstance(entries, dict) or not entries:
|
||||
fail(f"{label} entries must be a non-empty object")
|
||||
if set(asset_sha256) != set(assets):
|
||||
fail(f"{label} assets and assetSha256 reference different archives")
|
||||
|
||||
archive_names: set[str] = set()
|
||||
for raw_name, expected_size in sorted(assets.items()):
|
||||
name = validate_asset_name(raw_name, f"{label} driver")
|
||||
if name == DRIVER_CI_BUNDLE_NAME:
|
||||
fail(f"driver CI bundle must not be mirrored: {name}")
|
||||
if Path(name).suffix.lower() != ".zip":
|
||||
fail(f"driver mirror asset must be a zip archive: {name}")
|
||||
if not is_nonnegative_int(expected_size):
|
||||
fail(f"invalid driver asset size for {name}")
|
||||
validate_sha256(asset_sha256.get(name), f"driver asset {name}")
|
||||
archive_names.add(name)
|
||||
|
||||
referenced_archives: set[str] = set()
|
||||
for raw_name, raw_entry in sorted(entries.items()):
|
||||
name = validate_asset_name(raw_name, f"{label} driver entry")
|
||||
if not isinstance(raw_entry, dict):
|
||||
fail(f"invalid {label} driver entry metadata for {name}")
|
||||
archive = validate_asset_name(
|
||||
raw_entry.get("archive"),
|
||||
f"{label} driver entry archive",
|
||||
)
|
||||
if archive not in archive_names:
|
||||
fail(f"driver entry references an unknown archive: {name} -> {archive}")
|
||||
validate_driver_entry_path(
|
||||
raw_entry.get("path"),
|
||||
f"{label} driver entry {name}",
|
||||
)
|
||||
if not is_nonnegative_int(raw_entry.get("size")):
|
||||
fail(f"invalid {label} driver entry size for {name}")
|
||||
validate_sha256(
|
||||
raw_entry.get("sha256"),
|
||||
f"{label} driver entry {name}",
|
||||
)
|
||||
referenced_archives.add(archive)
|
||||
|
||||
if referenced_archives != archive_names:
|
||||
fail(f"{label} entries do not cover every driver archive")
|
||||
|
||||
|
||||
def normalized_driver_index(index: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: value
|
||||
for key, value in index.items()
|
||||
if key not in DRIVER_MUTABLE_INDEX_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
@@ -114,19 +200,18 @@ def verify_and_copy_driver_assets(
|
||||
index: dict[str, Any],
|
||||
destination: Path,
|
||||
) -> None:
|
||||
assets = index.get("assets")
|
||||
if not isinstance(assets, dict) or not assets:
|
||||
fail("driver index assets must be a non-empty object")
|
||||
assets = index["assets"]
|
||||
asset_sha256 = index["assetSha256"]
|
||||
|
||||
for raw_name, expected_size in sorted(assets.items()):
|
||||
name = validate_asset_name(raw_name, "driver")
|
||||
if not isinstance(expected_size, int) or expected_size < 0:
|
||||
fail(f"invalid driver asset size for {name}")
|
||||
source = driver_dir / name
|
||||
if not source.is_file():
|
||||
fail(f"driver asset is missing: {source}")
|
||||
if source.stat().st_size != expected_size:
|
||||
fail(f"driver asset size mismatch: {name}")
|
||||
if sha256_file(source) != asset_sha256[name].lower():
|
||||
fail(f"driver asset sha256 mismatch: {name}")
|
||||
copy_file(source, destination / name)
|
||||
|
||||
|
||||
@@ -198,8 +283,10 @@ def build(args: argparse.Namespace) -> dict[str, Any]:
|
||||
driver_tag = validate_tag(args.driver_tag, "driver tag")
|
||||
version_index = load_object(args.driver_version_index, "driver version index")
|
||||
latest_index = load_object(args.driver_latest_index, "driver latest index")
|
||||
if version_index.get("assets") != latest_index.get("assets"):
|
||||
fail("driver version and latest indexes reference different assets")
|
||||
validate_driver_index(version_index, "driver version index")
|
||||
validate_driver_index(latest_index, "driver latest index")
|
||||
if normalized_driver_index(version_index) != normalized_driver_index(latest_index):
|
||||
fail("driver version and latest indexes differ outside tag metadata")
|
||||
|
||||
driver_version_dir, driver_latest_path = driver_layout(args.channel, driver_tag)
|
||||
verify_and_copy_driver_assets(
|
||||
|
||||
@@ -8,17 +8,47 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("prepare-vps-release-payload.py")
|
||||
PUBLISH_ACTION = SCRIPT.parents[1] / ".github/actions/publish-vps-mirror/action.yml"
|
||||
DEV_WORKFLOW = SCRIPT.parents[1] / ".github/workflows/dev-build.yml"
|
||||
PUBLISH_RELEASE_WORKFLOW = SCRIPT.parents[1] / ".github/workflows/publish-release.yml"
|
||||
|
||||
|
||||
def sha256(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def write_driver_archive(path: Path, entry_path: str, value: bytes) -> bytes:
|
||||
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(entry_path, value)
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def driver_index(
|
||||
archive_name: str,
|
||||
archive_bytes: bytes,
|
||||
entry_name: str,
|
||||
entry_path: str,
|
||||
entry_bytes: bytes,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"assets": {archive_name: len(archive_bytes)},
|
||||
"assetSha256": {archive_name: sha256(archive_bytes)},
|
||||
"entries": {
|
||||
entry_name: {
|
||||
"archive": archive_name,
|
||||
"path": entry_path,
|
||||
"size": len(entry_bytes),
|
||||
"sha256": sha256(entry_bytes),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class PrepareVPSReleasePayloadTest(unittest.TestCase):
|
||||
def run_script(self, *args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
@@ -55,17 +85,38 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
driver_bytes = b"driver"
|
||||
(driver_dir / "mysql.exe").write_bytes(driver_bytes)
|
||||
assets = {"mysql.exe": len(driver_bytes)}
|
||||
driver_entry_bytes = b"driver binary"
|
||||
driver_entry_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
driver_entry_path = f"Windows/{driver_entry_name}"
|
||||
driver_name = "mysql-driver-agent-windows-amd64.zip"
|
||||
driver_bytes = write_driver_archive(
|
||||
driver_dir / driver_name,
|
||||
driver_entry_path,
|
||||
driver_entry_bytes,
|
||||
)
|
||||
(driver_dir / "mysql-driver-agent-windows-amd64.exe").write_bytes(b"raw driver")
|
||||
(driver_dir / "GoNavi-DriverAgents.zip").write_bytes(b"CI bundle")
|
||||
index = driver_index(
|
||||
driver_name,
|
||||
driver_bytes,
|
||||
driver_entry_name,
|
||||
driver_entry_path,
|
||||
driver_entry_bytes,
|
||||
)
|
||||
version_index = driver_dir / "version-index.json"
|
||||
latest_index = driver_dir / "latest-index.json"
|
||||
version_index.write_text(
|
||||
json.dumps({"tagName": "v1.2.3", "assets": assets}),
|
||||
json.dumps({**index, "tagName": "v1.2.3"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
latest_index.write_text(
|
||||
json.dumps({"tagName": "v1.2.3", "assets": assets}),
|
||||
json.dumps(
|
||||
{
|
||||
**index,
|
||||
"tagName": "latest",
|
||||
"mirrorTagName": "v1.2.3",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@@ -100,9 +151,21 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase):
|
||||
self.assertTrue(
|
||||
(
|
||||
output
|
||||
/ "payload/drivers/releases/download/v1.2.3/mysql.exe"
|
||||
/ f"payload/drivers/releases/download/v1.2.3/{driver_name}"
|
||||
).is_file()
|
||||
)
|
||||
self.assertFalse(
|
||||
(
|
||||
output
|
||||
/ "payload/drivers/releases/download/v1.2.3/mysql-driver-agent-windows-amd64.exe"
|
||||
).exists()
|
||||
)
|
||||
self.assertFalse(
|
||||
(
|
||||
output
|
||||
/ "payload/drivers/releases/download/v1.2.3/GoNavi-DriverAgents.zip"
|
||||
).exists()
|
||||
)
|
||||
self.assertTrue(
|
||||
(
|
||||
output
|
||||
@@ -117,6 +180,308 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase):
|
||||
metadata["fileCount"],
|
||||
)
|
||||
|
||||
def test_rejects_non_zip_driver_asset_in_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
app_dir = root / "app"
|
||||
driver_dir = root / "driver"
|
||||
app_dir.mkdir()
|
||||
driver_dir.mkdir()
|
||||
|
||||
app_bytes = b"portable zip"
|
||||
(app_dir / "GoNavi.zip").write_bytes(app_bytes)
|
||||
app_manifest = app_dir / "latest.json"
|
||||
app_manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tagName": "v1.2.3",
|
||||
"assets": [
|
||||
{
|
||||
"name": "GoNavi.zip",
|
||||
"size": len(app_bytes),
|
||||
"sha256": sha256(app_bytes),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
raw_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
raw_bytes = b"raw driver"
|
||||
(driver_dir / raw_name).write_bytes(raw_bytes)
|
||||
index = {
|
||||
"assets": {raw_name: len(raw_bytes)},
|
||||
"assetSha256": {raw_name: sha256(raw_bytes)},
|
||||
"entries": {
|
||||
raw_name: {
|
||||
"archive": raw_name,
|
||||
"path": f"Windows/{raw_name}",
|
||||
"size": len(raw_bytes),
|
||||
"sha256": sha256(raw_bytes),
|
||||
}
|
||||
},
|
||||
}
|
||||
version_index = driver_dir / "version-index.json"
|
||||
latest_index = driver_dir / "latest-index.json"
|
||||
version_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
latest_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
|
||||
result = self.run_script(
|
||||
"--channel",
|
||||
"stable",
|
||||
"--app-tag",
|
||||
"v1.2.3",
|
||||
"--app-dir",
|
||||
str(app_dir),
|
||||
"--app-manifest",
|
||||
str(app_manifest),
|
||||
"--driver-tag",
|
||||
"v1.2.3",
|
||||
"--driver-dir",
|
||||
str(driver_dir),
|
||||
"--driver-version-index",
|
||||
str(version_index),
|
||||
"--driver-latest-index",
|
||||
str(latest_index),
|
||||
"--output",
|
||||
str(root / "output"),
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("driver mirror asset must be a zip archive", result.stderr)
|
||||
|
||||
def test_rejects_driver_archive_sha256_mismatch(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
app_dir = root / "app"
|
||||
driver_dir = root / "driver"
|
||||
app_dir.mkdir()
|
||||
driver_dir.mkdir()
|
||||
|
||||
app_bytes = b"portable zip"
|
||||
(app_dir / "GoNavi.zip").write_bytes(app_bytes)
|
||||
app_manifest = app_dir / "latest.json"
|
||||
app_manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tagName": "v1.2.3",
|
||||
"assets": [
|
||||
{
|
||||
"name": "GoNavi.zip",
|
||||
"size": len(app_bytes),
|
||||
"sha256": sha256(app_bytes),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
entry_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
entry_path = f"Windows/{entry_name}"
|
||||
archive_name = "mysql-driver-agent-windows-amd64.zip"
|
||||
entry_bytes = b"driver binary"
|
||||
archive_bytes = write_driver_archive(
|
||||
driver_dir / archive_name,
|
||||
entry_path,
|
||||
entry_bytes,
|
||||
)
|
||||
index = driver_index(
|
||||
archive_name,
|
||||
archive_bytes,
|
||||
entry_name,
|
||||
entry_path,
|
||||
entry_bytes,
|
||||
)
|
||||
index["assetSha256"] = {archive_name: "0" * 64}
|
||||
version_index = driver_dir / "version-index.json"
|
||||
latest_index = driver_dir / "latest-index.json"
|
||||
version_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
latest_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
|
||||
result = self.run_script(
|
||||
"--channel",
|
||||
"stable",
|
||||
"--app-tag",
|
||||
"v1.2.3",
|
||||
"--app-dir",
|
||||
str(app_dir),
|
||||
"--app-manifest",
|
||||
str(app_manifest),
|
||||
"--driver-tag",
|
||||
"v1.2.3",
|
||||
"--driver-dir",
|
||||
str(driver_dir),
|
||||
"--driver-version-index",
|
||||
str(version_index),
|
||||
"--driver-latest-index",
|
||||
str(latest_index),
|
||||
"--output",
|
||||
str(root / "output"),
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("driver asset sha256 mismatch", result.stderr)
|
||||
|
||||
def test_rejects_driver_index_metadata_drift_outside_tags(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
app_dir = root / "app"
|
||||
driver_dir = root / "driver"
|
||||
app_dir.mkdir()
|
||||
driver_dir.mkdir()
|
||||
|
||||
app_bytes = b"portable zip"
|
||||
(app_dir / "GoNavi.zip").write_bytes(app_bytes)
|
||||
app_manifest = app_dir / "latest.json"
|
||||
app_manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tagName": "v1.2.3",
|
||||
"assets": [
|
||||
{
|
||||
"name": "GoNavi.zip",
|
||||
"size": len(app_bytes),
|
||||
"sha256": sha256(app_bytes),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
entry_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
entry_path = f"Windows/{entry_name}"
|
||||
archive_name = "mysql-driver-agent-windows-amd64.zip"
|
||||
entry_bytes = b"driver binary"
|
||||
archive_bytes = write_driver_archive(
|
||||
driver_dir / archive_name,
|
||||
entry_path,
|
||||
entry_bytes,
|
||||
)
|
||||
version_payload = driver_index(
|
||||
archive_name,
|
||||
archive_bytes,
|
||||
entry_name,
|
||||
entry_path,
|
||||
entry_bytes,
|
||||
)
|
||||
latest_payload = json.loads(json.dumps(version_payload))
|
||||
latest_payload["entries"][entry_name]["path"] = f"Linux/{entry_name}"
|
||||
version_index = driver_dir / "version-index.json"
|
||||
latest_index = driver_dir / "latest-index.json"
|
||||
version_index.write_text(
|
||||
json.dumps({**version_payload, "tagName": "v1.2.3"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
latest_index.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**latest_payload,
|
||||
"tagName": "latest",
|
||||
"mirrorTagName": "v1.2.3",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = self.run_script(
|
||||
"--channel",
|
||||
"stable",
|
||||
"--app-tag",
|
||||
"v1.2.3",
|
||||
"--app-dir",
|
||||
str(app_dir),
|
||||
"--app-manifest",
|
||||
str(app_manifest),
|
||||
"--driver-tag",
|
||||
"v1.2.3",
|
||||
"--driver-dir",
|
||||
str(driver_dir),
|
||||
"--driver-version-index",
|
||||
str(version_index),
|
||||
"--driver-latest-index",
|
||||
str(latest_index),
|
||||
"--output",
|
||||
str(root / "output"),
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("driver version and latest indexes differ outside tag metadata", result.stderr)
|
||||
|
||||
def test_rejects_windows_drive_absolute_driver_entry_path(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
app_dir = root / "app"
|
||||
driver_dir = root / "driver"
|
||||
app_dir.mkdir()
|
||||
driver_dir.mkdir()
|
||||
|
||||
app_bytes = b"portable zip"
|
||||
(app_dir / "GoNavi.zip").write_bytes(app_bytes)
|
||||
app_manifest = app_dir / "latest.json"
|
||||
app_manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tagName": "v1.2.3",
|
||||
"assets": [
|
||||
{
|
||||
"name": "GoNavi.zip",
|
||||
"size": len(app_bytes),
|
||||
"sha256": sha256(app_bytes),
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
entry_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
entry_path = f"Windows/{entry_name}"
|
||||
archive_name = "mysql-driver-agent-windows-amd64.zip"
|
||||
entry_bytes = b"driver binary"
|
||||
archive_bytes = write_driver_archive(
|
||||
driver_dir / archive_name,
|
||||
entry_path,
|
||||
entry_bytes,
|
||||
)
|
||||
index = driver_index(
|
||||
archive_name,
|
||||
archive_bytes,
|
||||
entry_name,
|
||||
f"C:/{entry_name}",
|
||||
entry_bytes,
|
||||
)
|
||||
version_index = driver_dir / "version-index.json"
|
||||
latest_index = driver_dir / "latest-index.json"
|
||||
version_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
latest_index.write_text(json.dumps(index), encoding="utf-8")
|
||||
|
||||
result = self.run_script(
|
||||
"--channel",
|
||||
"stable",
|
||||
"--app-tag",
|
||||
"v1.2.3",
|
||||
"--app-dir",
|
||||
str(app_dir),
|
||||
"--app-manifest",
|
||||
str(app_manifest),
|
||||
"--driver-tag",
|
||||
"v1.2.3",
|
||||
"--driver-dir",
|
||||
str(driver_dir),
|
||||
"--driver-version-index",
|
||||
str(version_index),
|
||||
"--driver-latest-index",
|
||||
str(latest_index),
|
||||
"--output",
|
||||
str(root / "output"),
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("invalid driver version index driver entry", result.stderr)
|
||||
|
||||
def test_rejects_manifest_path_traversal(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
@@ -162,6 +527,34 @@ class PrepareVPSReleasePayloadTest(unittest.TestCase):
|
||||
self.assertIn("available_kib", source)
|
||||
self.assertIn("available_kib * 1024", source)
|
||||
|
||||
def test_publish_action_requires_explicit_driver_mode_and_cleans_staging(self) -> None:
|
||||
source = PUBLISH_ACTION.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("driver-enabled:", source)
|
||||
self.assertIn("MIRROR_DRIVER_ENABLED", source)
|
||||
self.assertIn('case "${MIRROR_DRIVER_ENABLED}"', source)
|
||||
self.assertIn("trap cleanup EXIT", source)
|
||||
self.assertIn(".gonavi-mirror-root", source)
|
||||
self.assertIn("-mindepth 1 -maxdepth 1 -type d -mmin +1440", source)
|
||||
|
||||
def test_workflows_pass_explicit_driver_mode(self) -> None:
|
||||
dev_source = DEV_WORKFLOW.read_text(encoding="utf-8")
|
||||
stable_source = PUBLISH_RELEASE_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn(
|
||||
"driver-enabled: ${{ steps.driver_assets.outputs.has_driver_assets }}",
|
||||
dev_source,
|
||||
)
|
||||
self.assertIn("id: mirror_payload", stable_source)
|
||||
self.assertIn(
|
||||
'echo "has_driver_release=${has_driver_release}" >> "$GITHUB_OUTPUT"',
|
||||
stable_source,
|
||||
)
|
||||
self.assertIn(
|
||||
"driver-enabled: ${{ steps.mirror_payload.outputs.has_driver_release }}",
|
||||
stable_source,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -13,10 +13,15 @@ import tempfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MANIFEST_ASSET_NAME = "GoNavi-DriverAgents-Manifest.json"
|
||||
INDEX_ASSET_NAME = "GoNavi-DriverAgents-Index.json"
|
||||
CI_BUNDLE_ASSET_NAME = "GoNavi-DriverAgents.zip"
|
||||
DUCKDB_WINDOWS_AGENT_NAME = "duckdb-driver-agent-windows-amd64.exe"
|
||||
DUCKDB_WINDOWS_LIBRARY_NAME = "duckdb.dll"
|
||||
|
||||
|
||||
def github_headers(binary: bool = False):
|
||||
@@ -110,6 +115,36 @@ def infer_asset_path(name: str):
|
||||
return None
|
||||
|
||||
|
||||
def infer_driver_zip_asset_name(name: str):
|
||||
trimmed = str(name or "").strip()
|
||||
if not trimmed or infer_asset_path(trimmed) is None or "-driver-agent-" not in trimmed:
|
||||
return None
|
||||
if trimmed.lower().endswith(".exe"):
|
||||
trimmed = trimmed[:-4]
|
||||
return f"{trimmed}.zip"
|
||||
|
||||
|
||||
def extract_zip_entry(zip_path: Path, entry_name: str, destination: Path):
|
||||
expected = str(entry_name or "").strip().replace("\\", "/")
|
||||
if not expected:
|
||||
return False
|
||||
with zipfile.ZipFile(zip_path) as archive:
|
||||
entry = None
|
||||
for candidate in archive.infolist():
|
||||
normalized = candidate.filename.replace("\\", "/")
|
||||
while normalized.startswith("./"):
|
||||
normalized = normalized[2:]
|
||||
if not candidate.is_dir() and normalized == expected:
|
||||
entry = candidate
|
||||
break
|
||||
if entry is None:
|
||||
return False
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with archive.open(entry) as source, open(destination, "wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
return True
|
||||
|
||||
|
||||
def normalize_machine(value: str):
|
||||
machine = str(value or "").strip().lower()
|
||||
if machine in {"x86_64", "amd64"}:
|
||||
@@ -161,7 +196,7 @@ def probe_metadata_revision(path: Path):
|
||||
return str(((payload.get("data") or {}).get("agentRevision") or "")).strip()
|
||||
|
||||
|
||||
def validate_release_assets(release: dict, manifest: dict, runtime_platform=None):
|
||||
def _validate_legacy_release_assets(release: dict, manifest: dict, runtime_platform=None):
|
||||
assets = asset_map(release)
|
||||
manifest_assets = manifest.get("assets") or {}
|
||||
if not isinstance(manifest_assets, dict) or not manifest_assets:
|
||||
@@ -178,8 +213,11 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None
|
||||
for name, meta in sorted(manifest_assets.items()):
|
||||
if name == MANIFEST_ASSET_NAME:
|
||||
continue
|
||||
asset = assets.get(name)
|
||||
if asset is None:
|
||||
path_hint = infer_asset_path(name)
|
||||
zip_asset_name = infer_driver_zip_asset_name(name)
|
||||
zip_asset = assets.get(zip_asset_name) if zip_asset_name else None
|
||||
legacy_asset = assets.get(name)
|
||||
if zip_asset is None and legacy_asset is None:
|
||||
mismatches.append((name, "missing_release_asset", "", "present in manifest"))
|
||||
continue
|
||||
|
||||
@@ -188,16 +226,31 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None
|
||||
asset_platform = str(meta.get("platform") or "").strip().lower()
|
||||
|
||||
local_path = None
|
||||
actual_sha = asset_sha256_digest(asset)
|
||||
if expected_sha and not actual_sha:
|
||||
local_path = tmp_root / name
|
||||
download_url(str(asset.get("browser_download_url") or "").strip(), local_path)
|
||||
actual_sha = ""
|
||||
if zip_asset is not None:
|
||||
archive_path = tmp_root / "archives" / zip_asset_name
|
||||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
download_url(str(zip_asset.get("browser_download_url") or "").strip(), archive_path)
|
||||
local_path = tmp_root / "extracted" / name
|
||||
try:
|
||||
found_entry = extract_zip_entry(archive_path, path_hint, local_path)
|
||||
except zipfile.BadZipFile as exc:
|
||||
mismatches.append((name, "zip", str(exc), "valid zip archive"))
|
||||
continue
|
||||
if not found_entry:
|
||||
mismatches.append((name, "zip_entry", "", path_hint or name))
|
||||
continue
|
||||
actual_sha = sha256_file(local_path).lower()
|
||||
else:
|
||||
actual_sha = asset_sha256_digest(legacy_asset)
|
||||
if expected_sha and not actual_sha:
|
||||
local_path = tmp_root / name
|
||||
download_url(str(legacy_asset.get("browser_download_url") or "").strip(), local_path)
|
||||
actual_sha = sha256_file(local_path).lower()
|
||||
if expected_sha and actual_sha != expected_sha:
|
||||
mismatches.append((name, "sha256", actual_sha, expected_sha))
|
||||
continue
|
||||
|
||||
path_hint = infer_asset_path(name)
|
||||
if path_hint is None:
|
||||
skipped.append(name)
|
||||
continue
|
||||
@@ -205,7 +258,7 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None
|
||||
if expected_revision and asset_platform == runtime_platform:
|
||||
if local_path is None:
|
||||
local_path = tmp_root / name
|
||||
download_url(str(asset.get("browser_download_url") or "").strip(), local_path)
|
||||
download_url(str(legacy_asset.get("browser_download_url") or "").strip(), local_path)
|
||||
actual_revision = probe_metadata_revision(local_path)
|
||||
if actual_revision != expected_revision:
|
||||
mismatches.append((name, "revision", actual_revision, expected_revision))
|
||||
@@ -213,6 +266,301 @@ def validate_release_assets(release: dict, manifest: dict, runtime_platform=None
|
||||
return mismatches, skipped
|
||||
|
||||
|
||||
def is_nonnegative_int(value):
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
|
||||
|
||||
def is_sha256(value):
|
||||
normalized = str(value or "").strip().lower()
|
||||
return len(normalized) == 64 and all(char in "0123456789abcdef" for char in normalized)
|
||||
|
||||
|
||||
def is_basename(value):
|
||||
normalized = str(value or "").strip()
|
||||
return normalized not in {"", ".", ".."} and "/" not in normalized and "\\" not in normalized
|
||||
|
||||
|
||||
def is_safe_archive_entry_path(value):
|
||||
normalized = str(value or "").strip().replace("\\", "/")
|
||||
if not normalized or normalized.startswith("/"):
|
||||
return False
|
||||
if len(normalized) >= 3 and normalized[0].isalpha() and normalized[1:3] == ":/":
|
||||
return False
|
||||
parts = normalized.split("/")
|
||||
return all(part not in {"", ".", ".."} for part in parts)
|
||||
|
||||
|
||||
def indexed_release_maps(release_index):
|
||||
if release_index is None:
|
||||
return None
|
||||
if not isinstance(release_index, dict):
|
||||
raise RuntimeError("release index must be an object")
|
||||
if "entries" not in release_index and "assetSha256" not in release_index:
|
||||
return None
|
||||
|
||||
result = []
|
||||
for key in ("assets", "assetSha256", "entries"):
|
||||
value = release_index.get(key)
|
||||
if not isinstance(value, dict) or not value:
|
||||
raise RuntimeError(f"release index {key} must be a non-empty object")
|
||||
result.append(value)
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _validate_indexed_release_assets(
|
||||
release: dict,
|
||||
manifest: dict,
|
||||
runtime_platform: str,
|
||||
index_assets: dict,
|
||||
index_asset_sha256: dict,
|
||||
index_entries: dict,
|
||||
):
|
||||
release_assets = asset_map(release)
|
||||
manifest_assets = manifest.get("assets") or {}
|
||||
if not isinstance(manifest_assets, dict) or not manifest_assets:
|
||||
raise RuntimeError("manifest assets is empty")
|
||||
|
||||
mismatches = []
|
||||
skipped = []
|
||||
archive_metadata_valid = {}
|
||||
|
||||
for archive_name, expected_size in sorted(index_assets.items()):
|
||||
archive = str(archive_name or "").strip()
|
||||
valid = True
|
||||
if not is_basename(archive):
|
||||
mismatches.append((archive, "index_asset_name", archive, "archive basename"))
|
||||
archive_metadata_valid[archive] = False
|
||||
continue
|
||||
if archive == CI_BUNDLE_ASSET_NAME:
|
||||
mismatches.append((archive, "index_ci_bundle", "present in assets", "absent"))
|
||||
valid = False
|
||||
if not is_nonnegative_int(expected_size):
|
||||
mismatches.append((archive, "index_archive_size", expected_size, "non-negative integer"))
|
||||
valid = False
|
||||
|
||||
expected_sha = str(index_asset_sha256.get(archive) or "").strip().lower()
|
||||
if not is_sha256(expected_sha):
|
||||
mismatches.append((archive, "index_archive_sha256", expected_sha, "sha256"))
|
||||
valid = False
|
||||
|
||||
release_asset = release_assets.get(archive)
|
||||
if release_asset is None:
|
||||
mismatches.append((archive, "missing_release_asset", "", "present in release index"))
|
||||
archive_metadata_valid[archive] = False
|
||||
continue
|
||||
|
||||
actual_size = release_asset.get("size")
|
||||
if actual_size != expected_size:
|
||||
mismatches.append((archive, "archive_size", actual_size, expected_size))
|
||||
valid = False
|
||||
actual_sha = asset_sha256_digest(release_asset)
|
||||
if actual_sha != expected_sha:
|
||||
mismatches.append((archive, "archive_sha256", actual_sha, expected_sha))
|
||||
valid = False
|
||||
archive_metadata_valid[archive] = valid
|
||||
|
||||
for archive_name in sorted(index_asset_sha256):
|
||||
if archive_name == CI_BUNDLE_ASSET_NAME:
|
||||
mismatches.append((archive_name, "index_ci_bundle_sha256", "present in assetSha256", "absent"))
|
||||
if archive_name not in index_assets:
|
||||
mismatches.append((archive_name, "index_asset", "assetSha256", "assets"))
|
||||
|
||||
normalized_entries = {}
|
||||
entry_metadata_valid = {}
|
||||
for raw_name, raw_entry in sorted(index_entries.items(), key=lambda item: str(item[0])):
|
||||
name = str(raw_name or "").strip()
|
||||
valid = True
|
||||
if not is_basename(name):
|
||||
mismatches.append((name, "index_entry_name", name, "basename"))
|
||||
entry_metadata_valid[name] = False
|
||||
continue
|
||||
if not isinstance(raw_entry, dict):
|
||||
mismatches.append((name, "index_entry", type(raw_entry).__name__, "object"))
|
||||
entry_metadata_valid[name] = False
|
||||
continue
|
||||
|
||||
archive = str(raw_entry.get("archive") or "").strip()
|
||||
entry_path = str(raw_entry.get("path") or "").strip().replace("\\", "/")
|
||||
entry_size = raw_entry.get("size")
|
||||
entry_sha = str(raw_entry.get("sha256") or "").strip().lower()
|
||||
normalized_entries[name] = {
|
||||
"archive": archive,
|
||||
"path": entry_path,
|
||||
"size": entry_size,
|
||||
"sha256": entry_sha,
|
||||
}
|
||||
|
||||
if not is_basename(archive):
|
||||
mismatches.append((name, "index_archive", archive, "archive basename"))
|
||||
valid = False
|
||||
elif archive == CI_BUNDLE_ASSET_NAME:
|
||||
mismatches.append((name, "index_ci_bundle_entry", archive, "individual driver archive"))
|
||||
valid = False
|
||||
if archive not in index_assets:
|
||||
mismatches.append((name, "index_asset", archive, "present in assets"))
|
||||
valid = False
|
||||
if archive not in index_asset_sha256:
|
||||
mismatches.append((name, "index_asset_sha256", archive, "present in assetSha256"))
|
||||
valid = False
|
||||
if not is_safe_archive_entry_path(entry_path):
|
||||
mismatches.append((name, "index_entry_path", entry_path, "safe relative archive path"))
|
||||
valid = False
|
||||
expected_path = infer_asset_path(name)
|
||||
if expected_path is not None and entry_path != expected_path:
|
||||
mismatches.append((name, "index_path", entry_path, expected_path))
|
||||
valid = False
|
||||
if not is_nonnegative_int(entry_size):
|
||||
mismatches.append((name, "index_entry_size", entry_size, "non-negative integer"))
|
||||
valid = False
|
||||
if not is_sha256(entry_sha):
|
||||
mismatches.append((name, "index_entry_sha256", entry_sha, "sha256"))
|
||||
valid = False
|
||||
if name == DUCKDB_WINDOWS_LIBRARY_NAME:
|
||||
expected_archive = infer_driver_zip_asset_name(DUCKDB_WINDOWS_AGENT_NAME)
|
||||
if archive != expected_archive:
|
||||
mismatches.append((name, "index_archive", archive, expected_archive))
|
||||
valid = False
|
||||
entry_metadata_valid[name] = valid
|
||||
|
||||
if DUCKDB_WINDOWS_AGENT_NAME in normalized_entries and DUCKDB_WINDOWS_LIBRARY_NAME not in normalized_entries:
|
||||
mismatches.append(
|
||||
(
|
||||
DUCKDB_WINDOWS_LIBRARY_NAME,
|
||||
"index_entry",
|
||||
"",
|
||||
f"present with {DUCKDB_WINDOWS_AGENT_NAME}",
|
||||
)
|
||||
)
|
||||
|
||||
runtime_manifest_entries = []
|
||||
for name, meta in sorted(manifest_assets.items()):
|
||||
if name == MANIFEST_ASSET_NAME:
|
||||
continue
|
||||
if not isinstance(meta, dict):
|
||||
mismatches.append((name, "manifest_entry", type(meta).__name__, "object"))
|
||||
continue
|
||||
|
||||
path_hint = infer_asset_path(name)
|
||||
archive_hint = infer_driver_zip_asset_name(name)
|
||||
if path_hint is None or archive_hint is None:
|
||||
skipped.append(name)
|
||||
continue
|
||||
|
||||
entry = normalized_entries.get(name)
|
||||
if entry is None:
|
||||
mismatches.append((name, "index_entry", "", "present in release index"))
|
||||
continue
|
||||
|
||||
entry_valid = entry_metadata_valid.get(name, False)
|
||||
indexed_archive = entry["archive"]
|
||||
indexed_path = entry["path"]
|
||||
indexed_size = entry.get("size")
|
||||
indexed_sha = entry["sha256"]
|
||||
manifest_size = meta.get("size")
|
||||
manifest_sha = str(meta.get("sha256") or "").strip().lower()
|
||||
|
||||
if indexed_archive != archive_hint:
|
||||
mismatches.append((name, "index_archive", indexed_archive, archive_hint))
|
||||
entry_valid = False
|
||||
if indexed_path != path_hint:
|
||||
mismatches.append((name, "index_path", indexed_path, path_hint))
|
||||
entry_valid = False
|
||||
if not is_nonnegative_int(manifest_size):
|
||||
mismatches.append((name, "manifest_size", manifest_size, "non-negative integer"))
|
||||
entry_valid = False
|
||||
elif indexed_size != manifest_size:
|
||||
mismatches.append((name, "index_size", indexed_size, manifest_size))
|
||||
entry_valid = False
|
||||
if not is_sha256(manifest_sha):
|
||||
mismatches.append((name, "manifest_sha256", manifest_sha, "sha256"))
|
||||
entry_valid = False
|
||||
elif indexed_sha != manifest_sha:
|
||||
mismatches.append((name, "index_sha256", indexed_sha, manifest_sha))
|
||||
entry_valid = False
|
||||
asset_platform = str(meta.get("platform") or "").strip().lower()
|
||||
if entry_valid and asset_platform == runtime_platform and archive_metadata_valid.get(indexed_archive, False):
|
||||
runtime_manifest_entries.append((name, meta, indexed_archive))
|
||||
|
||||
runtime_by_archive = {}
|
||||
for name, meta, archive in runtime_manifest_entries:
|
||||
runtime_by_archive.setdefault(archive, []).append((name, meta))
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-assets-") as tmp:
|
||||
tmp_root = Path(tmp)
|
||||
for archive, manifest_entries in sorted(runtime_by_archive.items()):
|
||||
release_asset = release_assets[archive]
|
||||
archive_path = tmp_root / "archives" / archive
|
||||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
download_url(str(release_asset.get("browser_download_url") or "").strip(), archive_path)
|
||||
|
||||
expected_archive_size = index_assets[archive]
|
||||
actual_archive_size = archive_path.stat().st_size
|
||||
if actual_archive_size != expected_archive_size:
|
||||
mismatches.append((archive, "download_size", actual_archive_size, expected_archive_size))
|
||||
continue
|
||||
expected_archive_sha = str(index_asset_sha256[archive]).strip().lower()
|
||||
actual_archive_sha = sha256_file(archive_path).lower()
|
||||
if actual_archive_sha != expected_archive_sha:
|
||||
mismatches.append((archive, "download_sha256", actual_archive_sha, expected_archive_sha))
|
||||
continue
|
||||
|
||||
archive_entries = {
|
||||
raw_name: entry
|
||||
for raw_name, entry in normalized_entries.items()
|
||||
if entry_metadata_valid.get(raw_name, False) and entry["archive"] == archive
|
||||
}
|
||||
extracted = {}
|
||||
try:
|
||||
for raw_name, entry in sorted(archive_entries.items()):
|
||||
entry_path = entry["path"]
|
||||
entry_size = entry.get("size")
|
||||
entry_sha = entry["sha256"]
|
||||
local_path = tmp_root / "extracted" / archive / raw_name
|
||||
if not extract_zip_entry(archive_path, entry_path, local_path):
|
||||
mismatches.append((raw_name, "zip_entry", "", entry_path))
|
||||
continue
|
||||
actual_size = local_path.stat().st_size
|
||||
if actual_size != entry_size:
|
||||
mismatches.append((raw_name, "entry_size", actual_size, entry_size))
|
||||
continue
|
||||
actual_sha = sha256_file(local_path).lower()
|
||||
if actual_sha != entry_sha:
|
||||
mismatches.append((raw_name, "entry_sha256", actual_sha, entry_sha))
|
||||
continue
|
||||
extracted[raw_name] = local_path
|
||||
except zipfile.BadZipFile as exc:
|
||||
mismatches.append((archive, "zip", str(exc), "valid zip archive"))
|
||||
continue
|
||||
|
||||
for name, meta in manifest_entries:
|
||||
local_path = extracted.get(name)
|
||||
if local_path is None:
|
||||
continue
|
||||
expected_revision = str(meta.get("revision") or "").strip()
|
||||
if not expected_revision:
|
||||
continue
|
||||
actual_revision = probe_metadata_revision(local_path)
|
||||
if actual_revision != expected_revision:
|
||||
mismatches.append((name, "revision", actual_revision, expected_revision))
|
||||
|
||||
return mismatches, skipped
|
||||
|
||||
|
||||
def validate_release_assets(release: dict, manifest: dict, runtime_platform=None, release_index=None):
|
||||
if runtime_platform is None:
|
||||
runtime_platform = current_runtime_platform()
|
||||
normalized_runtime = str(runtime_platform or "").strip().lower()
|
||||
index_maps = indexed_release_maps(release_index)
|
||||
if index_maps is None:
|
||||
return _validate_legacy_release_assets(release, manifest, normalized_runtime)
|
||||
return _validate_indexed_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
normalized_runtime,
|
||||
*index_maps,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repo", default="Syngnat/GoNavi-DriverAgents")
|
||||
@@ -224,13 +572,20 @@ def main():
|
||||
manifest_asset = assets.get(MANIFEST_ASSET_NAME)
|
||||
if manifest_asset is None:
|
||||
raise SystemExit(f"release {args.repo}@{args.tag} missing {MANIFEST_ASSET_NAME}")
|
||||
index_asset = assets.get(INDEX_ASSET_NAME)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-release-manifest-") as tmp:
|
||||
manifest_path = Path(tmp) / MANIFEST_ASSET_NAME
|
||||
metadata_root = Path(tmp)
|
||||
manifest_path = metadata_root / MANIFEST_ASSET_NAME
|
||||
download_url(str(manifest_asset.get("browser_download_url") or "").strip(), manifest_path)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
release_index = None
|
||||
if index_asset is not None:
|
||||
index_path = metadata_root / INDEX_ASSET_NAME
|
||||
download_url(str(index_asset.get("browser_download_url") or "").strip(), index_path)
|
||||
release_index = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
|
||||
mismatches, skipped = validate_release_assets(release, manifest)
|
||||
mismatches, skipped = validate_release_assets(release, manifest, release_index=release_index)
|
||||
if mismatches:
|
||||
print("published driver release assets mismatch manifest:", file=sys.stderr)
|
||||
for name, field, actual, expected in mismatches:
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
|
||||
|
||||
MODULE_PATH = pathlib.Path(__file__).with_name("validate-driver-release-assets.py")
|
||||
@@ -39,6 +44,878 @@ class ValidateDriverReleaseAssetsTests(unittest.TestCase):
|
||||
self.assertEqual(MODULE.infer_asset_path("duckdb.dll"), "Windows/duckdb.dll")
|
||||
self.assertIsNone(MODULE.infer_asset_path("duckdb-driver.zip"))
|
||||
|
||||
def test_infer_driver_zip_asset_name(self):
|
||||
self.assertEqual(
|
||||
MODULE.infer_driver_zip_asset_name("clickhouse-driver-agent-darwin-arm64"),
|
||||
"clickhouse-driver-agent-darwin-arm64.zip",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_driver_zip_asset_name("mariadb-driver-agent-windows-amd64.exe"),
|
||||
"mariadb-driver-agent-windows-amd64.zip",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_driver_zip_asset_name("mongodb-driver-agent-v1-windows-arm64.exe"),
|
||||
"mongodb-driver-agent-v1-windows-arm64.zip",
|
||||
)
|
||||
self.assertEqual(
|
||||
MODULE.infer_driver_zip_asset_name("duckdb-driver-agent-windows-amd64.exe"),
|
||||
"duckdb-driver-agent-windows-amd64.zip",
|
||||
)
|
||||
self.assertIsNone(MODULE.infer_driver_zip_asset_name("duckdb.dll"))
|
||||
|
||||
def test_validate_release_assets_extracts_independent_zip(self):
|
||||
name = "clickhouse-driver-agent-darwin-arm64"
|
||||
zip_name = f"{name}.zip"
|
||||
payload = b"zipped-driver-agent"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "darwin/arm64",
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
archive_path = pathlib.Path(tmp) / zip_name
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(f"MacOS/{name}", payload)
|
||||
archive_bytes = archive_path.read_bytes()
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
release["assets"][0]["size"] = len(archive_bytes)
|
||||
release["assets"][0]["digest"] = f"sha256:{archive_sha}"
|
||||
manifest["assets"][name]["size"] = len(payload)
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"MacOS/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
downloaded_urls = []
|
||||
probed_payloads = []
|
||||
|
||||
def fake_download(url, destination):
|
||||
downloaded_urls.append(url)
|
||||
destination.write_bytes(archive_path.read_bytes())
|
||||
|
||||
def fake_probe(path):
|
||||
probed_payloads.append(path.read_bytes())
|
||||
return "src-expected"
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = fake_probe
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(mismatches, [])
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(downloaded_urls, [f"https://example.test/{zip_name}"])
|
||||
self.assertEqual(probed_payloads, [payload])
|
||||
|
||||
def test_indexed_duckdb_archive_verifies_runtime_library_entry(self):
|
||||
name = "duckdb-driver-agent-windows-amd64.exe"
|
||||
zip_name = "duckdb-driver-agent-windows-amd64.zip"
|
||||
agent_payload = b"duckdb-agent"
|
||||
library_payload = b"wrong-duckdb-library"
|
||||
expected_library_sha = hashlib.sha256(b"expected-duckdb-library").hexdigest()
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "windows/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(agent_payload),
|
||||
"sha256": hashlib.sha256(agent_payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
archive_path = pathlib.Path(tmp) / zip_name
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(f"Windows/{name}", agent_payload)
|
||||
archive.writestr("Windows/duckdb.dll", library_payload)
|
||||
archive_bytes = archive_path.read_bytes()
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(archive_bytes),
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Windows/{name}",
|
||||
"size": len(agent_payload),
|
||||
"sha256": hashlib.sha256(agent_payload).hexdigest(),
|
||||
},
|
||||
"duckdb.dll": {
|
||||
"archive": zip_name,
|
||||
"path": "Windows/duckdb.dll",
|
||||
"size": len(library_payload),
|
||||
"sha256": expected_library_sha,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def fake_download(_url, destination):
|
||||
destination.write_bytes(archive_bytes)
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-expected"
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="windows/amd64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(
|
||||
mismatches,
|
||||
[
|
||||
(
|
||||
"duckdb.dll",
|
||||
"entry_sha256",
|
||||
hashlib.sha256(library_payload).hexdigest(),
|
||||
expected_library_sha,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
def test_indexed_validation_rejects_ci_bundle_archive_maps(self):
|
||||
name = "clickhouse-driver-agent-linux-amd64"
|
||||
zip_name = f"{name}.zip"
|
||||
archive_bytes = b"independent-archive"
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
bundle_bytes = b"ci-completion-bundle"
|
||||
bundle_sha = hashlib.sha256(bundle_bytes).hexdigest()
|
||||
payload = b"driver"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(archive_bytes),
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
},
|
||||
{
|
||||
"name": MODULE.CI_BUNDLE_ASSET_NAME,
|
||||
"size": len(bundle_bytes),
|
||||
"digest": f"sha256:{bundle_sha}",
|
||||
"browser_download_url": f"https://example.test/{MODULE.CI_BUNDLE_ASSET_NAME}",
|
||||
},
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {
|
||||
zip_name: len(archive_bytes),
|
||||
MODULE.CI_BUNDLE_ASSET_NAME: len(bundle_bytes),
|
||||
},
|
||||
"assetSha256": {
|
||||
zip_name: archive_sha,
|
||||
MODULE.CI_BUNDLE_ASSET_NAME: bundle_sha,
|
||||
},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertIn(
|
||||
(MODULE.CI_BUNDLE_ASSET_NAME, "index_ci_bundle", "present in assets", "absent"),
|
||||
mismatches,
|
||||
)
|
||||
self.assertIn(
|
||||
(
|
||||
MODULE.CI_BUNDLE_ASSET_NAME,
|
||||
"index_ci_bundle_sha256",
|
||||
"present in assetSha256",
|
||||
"absent",
|
||||
),
|
||||
mismatches,
|
||||
)
|
||||
|
||||
def test_indexed_validation_rejects_entry_backed_by_ci_bundle(self):
|
||||
name = "clickhouse-driver-agent-linux-amd64"
|
||||
zip_name = f"{name}.zip"
|
||||
archive_bytes = b"independent-archive"
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
payload = b"driver"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(archive_bytes),
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
},
|
||||
"rogue-driver-agent": {
|
||||
"archive": MODULE.CI_BUNDLE_ASSET_NAME,
|
||||
"path": "Linux/rogue-driver-agent",
|
||||
"size": 1,
|
||||
"sha256": "a" * 64,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertIn(
|
||||
(
|
||||
"rogue-driver-agent",
|
||||
"index_ci_bundle_entry",
|
||||
MODULE.CI_BUNDLE_ASSET_NAME,
|
||||
"individual driver archive",
|
||||
),
|
||||
mismatches,
|
||||
)
|
||||
|
||||
def test_indexed_validation_rejects_malformed_cross_platform_duckdb_library(self):
|
||||
name = MODULE.DUCKDB_WINDOWS_AGENT_NAME
|
||||
zip_name = MODULE.infer_driver_zip_asset_name(name)
|
||||
archive_bytes = b"duckdb-archive"
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
payload = b"duckdb-agent"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(archive_bytes),
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "windows/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Windows/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
},
|
||||
MODULE.DUCKDB_WINDOWS_LIBRARY_NAME: {
|
||||
"archive": "missing.zip",
|
||||
"path": "../Windows/duckdb.dll",
|
||||
"size": -1,
|
||||
"sha256": "invalid",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertIn(
|
||||
(MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_asset", "missing.zip", "present in assets"),
|
||||
mismatches,
|
||||
)
|
||||
self.assertIn(
|
||||
(
|
||||
MODULE.DUCKDB_WINDOWS_LIBRARY_NAME,
|
||||
"index_entry_path",
|
||||
"../Windows/duckdb.dll",
|
||||
"safe relative archive path",
|
||||
),
|
||||
mismatches,
|
||||
)
|
||||
self.assertIn(
|
||||
(MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_entry_size", -1, "non-negative integer"),
|
||||
mismatches,
|
||||
)
|
||||
self.assertIn(
|
||||
(MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_entry_sha256", "invalid", "sha256"),
|
||||
mismatches,
|
||||
)
|
||||
self.assertIn(
|
||||
(MODULE.DUCKDB_WINDOWS_LIBRARY_NAME, "index_archive", "missing.zip", zip_name),
|
||||
mismatches,
|
||||
)
|
||||
|
||||
def test_indexed_validation_requires_duckdb_library_mapping(self):
|
||||
name = MODULE.DUCKDB_WINDOWS_AGENT_NAME
|
||||
zip_name = MODULE.infer_driver_zip_asset_name(name)
|
||||
archive_bytes = b"duckdb-archive"
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
payload = b"duckdb-agent"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(archive_bytes),
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "windows/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Windows/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertIn(
|
||||
(
|
||||
MODULE.DUCKDB_WINDOWS_LIBRARY_NAME,
|
||||
"index_entry",
|
||||
"",
|
||||
f"present with {MODULE.DUCKDB_WINDOWS_AGENT_NAME}",
|
||||
),
|
||||
mismatches,
|
||||
)
|
||||
|
||||
def test_validate_release_assets_reports_independent_zip_raw_sha_mismatch(self):
|
||||
name = "mariadb-driver-agent-windows-amd64.exe"
|
||||
zip_name = "mariadb-driver-agent-windows-amd64.zip"
|
||||
payload = b"unexpected-driver-agent"
|
||||
expected_manifest_sha = "d" * 64
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
"digest": f"sha256:{hashlib.sha256(b'archive-bytes').hexdigest()}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "windows/amd64",
|
||||
"revision": "src-expected",
|
||||
"sha256": expected_manifest_sha,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
archive_path = pathlib.Path(tmp) / zip_name
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(f"Windows/{name}", payload)
|
||||
archive_bytes = archive_path.read_bytes()
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
release["assets"][0]["size"] = len(archive_bytes)
|
||||
release["assets"][0]["digest"] = f"sha256:{archive_sha}"
|
||||
manifest["assets"][name]["size"] = len(payload)
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Windows/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
MODULE.probe_metadata_revision = lambda _path: "src-expected"
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="linux/amd64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(
|
||||
mismatches,
|
||||
[(name, "index_sha256", hashlib.sha256(payload).hexdigest(), expected_manifest_sha)],
|
||||
)
|
||||
|
||||
def test_validate_release_assets_skips_cross_platform_zip_revision_probe(self):
|
||||
name = "mongodb-driver-agent-v1-linux-arm64"
|
||||
zip_name = f"{name}.zip"
|
||||
payload = b"cross-platform-driver-agent"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/arm64",
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
archive_path = pathlib.Path(tmp) / zip_name
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(f"Linux/{name}", payload)
|
||||
archive_bytes = archive_path.read_bytes()
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
release["assets"][0]["size"] = len(archive_bytes)
|
||||
release["assets"][0]["digest"] = f"sha256:{archive_sha}"
|
||||
manifest["assets"][name]["size"] = len(payload)
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
def fail_probe(_path):
|
||||
raise AssertionError("cross-platform zip entry should not be executed")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
original_probe = MODULE.probe_metadata_revision
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
MODULE.probe_metadata_revision = fail_probe
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
MODULE.probe_metadata_revision = original_probe
|
||||
|
||||
self.assertEqual(mismatches, [])
|
||||
self.assertEqual(skipped, [])
|
||||
|
||||
def test_validate_release_assets_reports_missing_independent_zip_entry(self):
|
||||
name = "clickhouse-driver-agent-linux-amd64"
|
||||
zip_name = f"{name}.zip"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/amd64",
|
||||
"revision": "src-expected",
|
||||
"sha256": hashlib.sha256(b"driver").hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="gonavi-validate-release-assets-") as tmp:
|
||||
archive_path = pathlib.Path(tmp) / zip_name
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(name, b"driver")
|
||||
archive_bytes = archive_path.read_bytes()
|
||||
archive_sha = hashlib.sha256(archive_bytes).hexdigest()
|
||||
release["assets"][0]["size"] = len(archive_bytes)
|
||||
release["assets"][0]["digest"] = f"sha256:{archive_sha}"
|
||||
manifest["assets"][name]["size"] = len(b"driver")
|
||||
release_index = {
|
||||
"assets": {zip_name: len(archive_bytes)},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(b"driver"),
|
||||
"sha256": hashlib.sha256(b"driver").hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fake_download(_url, destination):
|
||||
destination.write_bytes(archive_path.read_bytes())
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fake_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="linux/amd64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(
|
||||
mismatches,
|
||||
[(name, "zip_entry", "", f"Linux/{name}")],
|
||||
)
|
||||
|
||||
def test_indexed_validation_checks_release_archive_metadata_without_download(self):
|
||||
name = "clickhouse-driver-agent-linux-amd64"
|
||||
zip_name = f"{name}.zip"
|
||||
payload = b"driver"
|
||||
expected_archive = b"indexed-archive"
|
||||
expected_archive_sha = hashlib.sha256(expected_archive).hexdigest()
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": len(expected_archive) + 1,
|
||||
"digest": f"sha256:{hashlib.sha256(b'wrong-archive').hexdigest()}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: len(expected_archive)},
|
||||
"assetSha256": {zip_name: expected_archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("cross-platform indexed asset should not be downloaded")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
mismatches, skipped = MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="darwin/arm64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual(
|
||||
mismatches,
|
||||
[
|
||||
(zip_name, "archive_size", len(expected_archive) + 1, len(expected_archive)),
|
||||
(
|
||||
zip_name,
|
||||
"archive_sha256",
|
||||
hashlib.sha256(b"wrong-archive").hexdigest(),
|
||||
expected_archive_sha,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
def test_malformed_new_index_does_not_fall_back_to_zip_downloads(self):
|
||||
name = "clickhouse-driver-agent-linux-amd64"
|
||||
zip_name = f"{name}.zip"
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": zip_name,
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
}
|
||||
]
|
||||
}
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/amd64",
|
||||
"revision": "src-expected",
|
||||
"size": 6,
|
||||
"sha256": hashlib.sha256(b"driver").hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {"assets": {zip_name: 123}, "entries": {}}
|
||||
|
||||
def fail_download(_url, _destination):
|
||||
raise AssertionError("malformed new index must fail before downloading archives")
|
||||
|
||||
original_download = MODULE.download_url
|
||||
try:
|
||||
MODULE.download_url = fail_download
|
||||
with self.assertRaisesRegex(RuntimeError, "assetSha256"):
|
||||
MODULE.validate_release_assets(
|
||||
release,
|
||||
manifest,
|
||||
runtime_platform="linux/amd64",
|
||||
release_index=release_index,
|
||||
)
|
||||
finally:
|
||||
MODULE.download_url = original_download
|
||||
|
||||
def test_main_downloads_index_and_avoids_cross_platform_archive(self):
|
||||
name = "mongodb-driver-agent-v1-linux-arm64"
|
||||
zip_name = f"{name}.zip"
|
||||
payload = b"driver"
|
||||
archive_size = 123
|
||||
archive_sha = hashlib.sha256(b"archive").hexdigest()
|
||||
manifest = {
|
||||
"assets": {
|
||||
name: {
|
||||
"platform": "linux/arm64",
|
||||
"revision": "src-expected",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
}
|
||||
}
|
||||
release_index = {
|
||||
"assets": {zip_name: archive_size},
|
||||
"assetSha256": {zip_name: archive_sha},
|
||||
"entries": {
|
||||
name: {
|
||||
"archive": zip_name,
|
||||
"path": f"Linux/{name}",
|
||||
"size": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
release = {
|
||||
"assets": [
|
||||
{
|
||||
"name": MODULE.MANIFEST_ASSET_NAME,
|
||||
"browser_download_url": "https://example.test/manifest",
|
||||
},
|
||||
{
|
||||
"name": MODULE.INDEX_ASSET_NAME,
|
||||
"browser_download_url": "https://example.test/index",
|
||||
},
|
||||
{
|
||||
"name": zip_name,
|
||||
"size": archive_size,
|
||||
"digest": f"sha256:{archive_sha}",
|
||||
"browser_download_url": f"https://example.test/{zip_name}",
|
||||
},
|
||||
]
|
||||
}
|
||||
downloaded_urls = []
|
||||
|
||||
def fake_download(url, destination):
|
||||
downloaded_urls.append(url)
|
||||
if url.endswith("/manifest"):
|
||||
destination.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
elif url.endswith("/index"):
|
||||
destination.write_text(json.dumps(release_index), encoding="utf-8")
|
||||
else:
|
||||
raise AssertionError("main should not download a cross-platform archive")
|
||||
|
||||
original_load_release = MODULE.load_release
|
||||
original_download = MODULE.download_url
|
||||
original_runtime = MODULE.current_runtime_platform
|
||||
original_argv = sys.argv
|
||||
try:
|
||||
MODULE.load_release = lambda _repo, _tag: release
|
||||
MODULE.download_url = fake_download
|
||||
MODULE.current_runtime_platform = lambda: "darwin/arm64"
|
||||
sys.argv = [str(MODULE_PATH), "--tag", "dev-latest"]
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertEqual(MODULE.main(), 0)
|
||||
finally:
|
||||
MODULE.load_release = original_load_release
|
||||
MODULE.download_url = original_download
|
||||
MODULE.current_runtime_platform = original_runtime
|
||||
sys.argv = original_argv
|
||||
|
||||
self.assertEqual(
|
||||
downloaded_urls,
|
||||
["https://example.test/manifest", "https://example.test/index"],
|
||||
)
|
||||
|
||||
def test_validate_release_assets_reports_sha_mismatch(self):
|
||||
release = {
|
||||
"assets": [
|
||||
|
||||
@@ -48,6 +48,19 @@ case "${staging_dir}" in
|
||||
;;
|
||||
esac
|
||||
|
||||
lock_dir=""
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
if [[ -n "${lock_dir}" ]]; then
|
||||
rmdir "${lock_dir}" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf -- "${staging_dir}"
|
||||
exit "${exit_code}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
[[ "${channel}" == "stable" || "${channel}" == "dev" ]] || {
|
||||
echo "invalid channel: ${channel}" >&2
|
||||
exit 1
|
||||
@@ -77,7 +90,6 @@ if command -v flock >/dev/null 2>&1; then
|
||||
else
|
||||
lock_dir="${root}/.deploy.lock.d"
|
||||
mkdir "${lock_dir}"
|
||||
trap 'rmdir "${lock_dir}" 2>/dev/null || true' EXIT
|
||||
fi
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
|
||||
@@ -5,18 +5,23 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TEMP_DIR}"' EXIT
|
||||
|
||||
# The no-flock fallback must share the staging cleanup trap instead of replacing it.
|
||||
test "$(grep -c '^trap cleanup EXIT$' "${SCRIPT_DIR}/vps-release-commit.sh")" -eq 1
|
||||
test "$(grep -c "trap 'rmdir" "${SCRIPT_DIR}/vps-release-commit.sh")" -eq 0
|
||||
|
||||
ROOT="${TEMP_DIR}/mirror"
|
||||
SOURCE="${TEMP_DIR}/source"
|
||||
mkdir -p "${ROOT}/.incoming" "${SOURCE}/app" "${SOURCE}/driver"
|
||||
printf '%s\n' 'gonavi-download-mirror-v1' > "${ROOT}/.gonavi-mirror-root"
|
||||
printf '%s' 'portable zip' > "${SOURCE}/app/GoNavi.zip"
|
||||
printf '%s' 'driver binary' > "${SOURCE}/driver/mysql.exe"
|
||||
printf '%s' 'driver archive' > "${SOURCE}/driver/mysql-driver-agent-windows-amd64.zip"
|
||||
|
||||
python3 - "${SOURCE}" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
source = pathlib.Path(sys.argv[1])
|
||||
app = (source / "app/GoNavi.zip").read_bytes()
|
||||
@@ -28,9 +33,27 @@ app = (source / "app/GoNavi.zip").read_bytes()
|
||||
"sha256": hashlib.sha256(app).hexdigest(),
|
||||
}],
|
||||
}))
|
||||
assets = {"mysql.exe": (source / "driver/mysql.exe").stat().st_size}
|
||||
(source / "driver/version.json").write_text(json.dumps({"tagName": "v1.2.3", "assets": assets}))
|
||||
(source / "driver/latest.json").write_text(json.dumps({"tagName": "v1.2.3", "assets": assets}))
|
||||
driver_name = "mysql-driver-agent-windows-amd64.zip"
|
||||
driver_entry_name = "mysql-driver-agent-windows-amd64.exe"
|
||||
driver_entry_path = f"Windows/{driver_entry_name}"
|
||||
driver_entry = b"driver binary"
|
||||
with zipfile.ZipFile(source / "driver" / driver_name, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr(driver_entry_path, driver_entry)
|
||||
driver_archive = (source / "driver" / driver_name).read_bytes()
|
||||
index = {
|
||||
"assets": {driver_name: len(driver_archive)},
|
||||
"assetSha256": {driver_name: hashlib.sha256(driver_archive).hexdigest()},
|
||||
"entries": {
|
||||
driver_entry_name: {
|
||||
"archive": driver_name,
|
||||
"path": driver_entry_path,
|
||||
"size": len(driver_entry),
|
||||
"sha256": hashlib.sha256(driver_entry).hexdigest(),
|
||||
}
|
||||
},
|
||||
}
|
||||
(source / "driver/version.json").write_text(json.dumps({**index, "tagName": "v1.2.3"}))
|
||||
(source / "driver/latest.json").write_text(json.dumps({**index, "tagName": "latest", "mirrorTagName": "v1.2.3"}))
|
||||
PY
|
||||
|
||||
prepare_stable() {
|
||||
@@ -55,7 +78,7 @@ bash "${SCRIPT_DIR}/vps-release-commit.sh" \
|
||||
|
||||
test -f "${ROOT}/gonavi/releases/download/v1.2.3/GoNavi.zip"
|
||||
test -f "${ROOT}/gonavi/releases/latest/latest.json"
|
||||
test -f "${ROOT}/drivers/releases/download/v1.2.3/mysql.exe"
|
||||
test -f "${ROOT}/drivers/releases/download/v1.2.3/mysql-driver-agent-windows-amd64.zip"
|
||||
test -f "${ROOT}/drivers/releases/latest/GoNavi-DriverAgents-Index.json"
|
||||
test ! -e "${ROOT}/gonavi/releases/download/v0.9.0"
|
||||
test ! -e "${ROOT}/drivers/releases/download/v0.9.0"
|
||||
@@ -98,5 +121,29 @@ if bash "${SCRIPT_DIR}/vps-release-commit.sh" \
|
||||
exit 1
|
||||
fi
|
||||
test ! -e "${ROOT}/gonavi/dev/releases/latest/latest-dev.json"
|
||||
test ! -e "${ROOT}/.incoming/dev-1"
|
||||
test ! -e "${ROOT}/.deploy.lock.d"
|
||||
|
||||
# A driver-disabled dev deployment must not prune stable or existing dev driver assets.
|
||||
mkdir -p "${ROOT}/gonavi/dev/releases/download/dev-old" \
|
||||
"${ROOT}/drivers/dev/releases/download/dev-driver-old" \
|
||||
"${ROOT}/drivers/dev/releases/latest"
|
||||
printf '%s' 'old driver pointer' > "${ROOT}/drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json"
|
||||
python3 "${SCRIPT_DIR}/prepare-vps-release-payload.py" \
|
||||
--channel dev \
|
||||
--app-tag dev-abc123 \
|
||||
--app-dir "${SOURCE}/app" \
|
||||
--app-manifest "${SOURCE}/app/latest-dev.json" \
|
||||
--output "${ROOT}/.incoming/dev-2" >/dev/null
|
||||
bash "${SCRIPT_DIR}/vps-release-commit.sh" \
|
||||
"${ROOT}" "${ROOT}/.incoming/dev-2" dev dev-abc123 '' >/dev/null
|
||||
|
||||
test -f "${ROOT}/gonavi/dev/releases/download/dev-abc123/GoNavi-dev.zip"
|
||||
test ! -e "${ROOT}/gonavi/dev/releases/download/dev-old"
|
||||
test -d "${ROOT}/drivers/dev/releases/download/dev-driver-old"
|
||||
test "$(cat "${ROOT}/drivers/dev/releases/latest/GoNavi-DriverAgents-Index.json")" = 'old driver pointer'
|
||||
test -d "${ROOT}/gonavi/releases/download/v1.2.3"
|
||||
test -d "${ROOT}/drivers/releases/download/v1.2.3"
|
||||
test ! -e "${ROOT}/.incoming/dev-2"
|
||||
|
||||
echo "vps release commit tests passed"
|
||||
|
||||
Reference in New Issue
Block a user