From 51f1909a73fda1b577b2e9cae56f108f260ec872 Mon Sep 17 00:00:00 2001 From: Awuqing <3184394176@qq.com> Date: Tue, 31 Mar 2026 07:46:12 +0800 Subject: [PATCH 1/5] feat: add SHA-256 checksum verification for backup integrity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses community feedback about 0KB corrupted backup files going undetected after upload. Implementation: - Compute SHA-256 hash of final artifact (after compress/encrypt) before upload - After each storage target upload, download the file back and verify the hash matches the local checksum - If verification fails: mark that target as failed, auto-delete the corrupted remote file, and log detailed mismatch info - Store checksum in BackupRecord model (new `checksum` column) - Display truncated SHA-256 with copy button in backup records UI Verification flow per storage target: local SHA-256 → upload → download → remote SHA-256 → compare - match: mark success - mismatch: mark failed + delete corrupted remote file --- server/internal/backup/helpers.go | 26 ++++++++++++ server/internal/model/backup_record.go | 1 + .../service/backup_execution_service.go | 40 +++++++++++++++++-- .../internal/service/backup_record_service.go | 2 + .../backup-records/BackupRecordsPage.tsx | 5 +++ web/src/types/backup-records.ts | 1 + 6 files changed, 72 insertions(+), 3 deletions(-) diff --git a/server/internal/backup/helpers.go b/server/internal/backup/helpers.go index c8d7a93..90d21a1 100644 --- a/server/internal/backup/helpers.go +++ b/server/internal/backup/helpers.go @@ -1,7 +1,10 @@ package backup import ( + "crypto/sha256" + "encoding/hex" "fmt" + "io" "os" "path/filepath" "strings" @@ -21,6 +24,29 @@ func createTempArtifact(baseDir, taskName string, extension string) (string, str return tempDir, filepath.Join(tempDir, fileName), nil } +// SHA256File 计算文件的 SHA-256 哈希值，返回十六进制字符串 +func SHA256File(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open file for checksum: %w", err) + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", fmt.Errorf("compute checksum: %w", err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// SHA256Reader 计算 reader 的 SHA-256 哈希值，返回十六进制字符串 +func SHA256Reader(reader io.Reader) (string, error) { + hash := sha256.New() + if _, err := io.Copy(hash, reader); err != nil { + return "", fmt.Errorf("compute checksum: %w", err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + func sanitizeFileName(value string) string { builder := strings.Builder{} for _, char := range strings.TrimSpace(value) { diff --git a/server/internal/model/backup_record.go b/server/internal/model/backup_record.go index 5cab79a..4d4c7a8 100644 --- a/server/internal/model/backup_record.go +++ b/server/internal/model/backup_record.go @@ -17,6 +17,7 @@ type BackupRecord struct { Status string `gorm:"size:20;index;not null" json:"status"` FileName string `gorm:"column:file_name;size:255" json:"fileName"` FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"` + Checksum string `gorm:"column:checksum;size:64" json:"checksum"` StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"` StorageUploadResults string `gorm:"column:storage_upload_results;type:text" json:"-"` DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"` diff --git a/server/internal/service/backup_execution_service.go b/server/internal/service/backup_execution_service.go index 6076e22..8cd60b9 100644 --- a/server/internal/service/backup_execution_service.go +++ b/server/internal/service/backup_execution_service.go @@ -253,10 +253,11 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba errMessage := "" var fileName string var fileSize int64 + var checksum string var storagePath string var uploadResults []StorageUploadResultItem completeRecord := func() { - if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, storagePath); finalizeErr != nil { + if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath); finalizeErr != nil { logger.Errorf("写回备份记录失败：%v", finalizeErr) } // 写入多目标上传结果 @@ -325,6 +326,17 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba fileName = filepath.Base(finalPath) storagePath = backup.BuildStorageKey(task.Type, startedAt, fileName) + // 计算文件 SHA-256 哈希（上传前） + logger.Infof("计算备份文件校验和...") + localChecksum, checksumErr := backup.SHA256File(finalPath) + if checksumErr != nil { + errMessage = checksumErr.Error() + logger.Errorf("计算文件校验和失败：%v", checksumErr) + return + } + checksum = localChecksum + logger.Infof("文件校验和: SHA-256=%s, 大小=%d bytes", checksum, fileSize) + // 收集所有存储目标 targetIDs := collectTargetIDs(task) if len(targetIDs) == 0 { @@ -364,8 +376,29 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba logger.Warnf("存储目标 %s 上传失败：%v", targetName, uploadErr) return } + // 上传后完整性校验：下载并验证 SHA-256 + verifyReader, verifyErr := provider.Download(ctx, storagePath) + if verifyErr != nil { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("上传后校验失败（无法下载验证）: %v", verifyErr)} + logger.Warnf("存储目标 %s 上传后下载验证失败：%v", targetName, verifyErr) + return + } + remoteChecksum, hashErr := backup.SHA256Reader(verifyReader) + verifyReader.Close() + if hashErr != nil { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("上传后校验失败（哈希计算错误）: %v", hashErr)} + logger.Warnf("存储目标 %s 上传后哈希计算失败：%v", targetName, hashErr) + return + } + if remoteChecksum != localChecksum { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 本地=%s, 远端=%s", localChecksum, remoteChecksum)} + logger.Errorf("存储目标 %s 完整性校验失败：本地 SHA-256=%s, 远端 SHA-256=%s", targetName, localChecksum, remoteChecksum) + // 删除损坏的远端文件 + _ = provider.Delete(ctx, storagePath) + return + } uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "success", StoragePath: storagePath, FileSize: fileSize} - logger.Infof("存储目标 %s 上传成功", targetName) + logger.Infof("存储目标 %s 上传成功，完整性校验通过 (SHA-256=%s)", targetName, localChecksum) // 每个成功目标独立执行保留策略 if s.retention != nil { cleanupResult, cleanupErr := s.retention.Cleanup(ctx, task, provider) @@ -403,7 +436,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba } } -func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, status string, errorMessage string, logContent string, fileName string, fileSize int64, storagePath string) error { +func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, status string, errorMessage string, logContent string, fileName string, fileSize int64, checksum string, storagePath string) error { record, err := s.records.FindByID(ctx, recordID) if err != nil { return err @@ -415,6 +448,7 @@ func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model record.Status = status record.FileName = fileName record.FileSize = fileSize + record.Checksum = checksum record.StoragePath = storagePath record.DurationSeconds = int(completedAt.Sub(startedAt).Seconds()) record.ErrorMessage = strings.TrimSpace(errorMessage) diff --git a/server/internal/service/backup_record_service.go b/server/internal/service/backup_record_service.go index 4a3e53c..d5ae215 100644 --- a/server/internal/service/backup_record_service.go +++ b/server/internal/service/backup_record_service.go @@ -30,6 +30,7 @@ type BackupRecordSummary struct { Status string `json:"status"` FileName string `json:"fileName"` FileSize int64 `json:"fileSize"` + Checksum string `json:"checksum"` StoragePath string `json:"storagePath"` DurationSeconds int `json:"durationSeconds"` ErrorMessage string `json:"errorMessage"` @@ -111,6 +112,7 @@ func toBackupRecordSummary(item *model.BackupRecord) BackupRecordSummary { Status: item.Status, FileName: item.FileName, FileSize: item.FileSize, + Checksum: item.Checksum, StoragePath: item.StoragePath, DurationSeconds: item.DurationSeconds, ErrorMessage: item.ErrorMessage, diff --git a/web/src/pages/backup-records/BackupRecordsPage.tsx b/web/src/pages/backup-records/BackupRecordsPage.tsx index 6e69ddb..2543dcc 100644 --- a/web/src/pages/backup-records/BackupRecordsPage.tsx +++ b/web/src/pages/backup-records/BackupRecordsPage.tsx @@ -98,6 +98,11 @@ export function BackupRecordsPage() { {record.fileName || '-'} {formatBytes(record.fileSize)} + {record.checksum && ( + + SHA-256: {record.checksum.substring(0, 16)}... + + )} ), }, diff --git a/web/src/types/backup-records.ts b/web/src/types/backup-records.ts index ae7b8d0..2a59dc0 100644 --- a/web/src/types/backup-records.ts +++ b/web/src/types/backup-records.ts @@ -19,6 +19,7 @@ export interface BackupRecordSummary { status: BackupRecordStatus fileName: string fileSize: number + checksum: string storagePath: string durationSeconds: number errorMessage: string From e5a4aaadb27010f1cb73a5949682cbb465104a63 Mon Sep 17 00:00:00 2001 From: Awuqing <3184394176@qq.com> Date: Tue, 31 Mar 2026 12:36:29 +0800 Subject: [PATCH 2/5] refactor: replace download-based hash verification with lightweight size check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous approach downloaded the entire backup file after upload to compute a remote SHA-256, which doubles bandwidth cost for every backup. New approach: - Local SHA-256 is still computed before upload (stored in record for audit) - After upload, use provider.List() to check remote file size (single API call) - If remote size is 0 or mismatches local size → mark failed + auto-delete - If List() fails, log a warning but don't block (file may have uploaded fine) This catches 0KB corrupted uploads with zero download overhead. --- server/internal/backup/helpers.go | 9 ---- .../service/backup_execution_service.go | 47 ++++++++++--------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/server/internal/backup/helpers.go b/server/internal/backup/helpers.go index 90d21a1..5ae055c 100644 --- a/server/internal/backup/helpers.go +++ b/server/internal/backup/helpers.go @@ -38,15 +38,6 @@ func SHA256File(path string) (string, error) { return hex.EncodeToString(hash.Sum(nil)), nil } -// SHA256Reader 计算 reader 的 SHA-256 哈希值，返回十六进制字符串 -func SHA256Reader(reader io.Reader) (string, error) { - hash := sha256.New() - if _, err := io.Copy(hash, reader); err != nil { - return "", fmt.Errorf("compute checksum: %w", err) - } - return hex.EncodeToString(hash.Sum(nil)), nil -} - func sanitizeFileName(value string) string { builder := strings.Builder{} for _, char := range strings.TrimSpace(value) { diff --git a/server/internal/service/backup_execution_service.go b/server/internal/service/backup_execution_service.go index 8cd60b9..fcbe813 100644 --- a/server/internal/service/backup_execution_service.go +++ b/server/internal/service/backup_execution_service.go @@ -376,29 +376,34 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba logger.Warnf("存储目标 %s 上传失败：%v", targetName, uploadErr) return } - // 上传后完整性校验：下载并验证 SHA-256 - verifyReader, verifyErr := provider.Download(ctx, storagePath) - if verifyErr != nil { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("上传后校验失败（无法下载验证）: %v", verifyErr)} - logger.Warnf("存储目标 %s 上传后下载验证失败：%v", targetName, verifyErr) - return - } - remoteChecksum, hashErr := backup.SHA256Reader(verifyReader) - verifyReader.Close() - if hashErr != nil { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("上传后校验失败（哈希计算错误）: %v", hashErr)} - logger.Warnf("存储目标 %s 上传后哈希计算失败：%v", targetName, hashErr) - return - } - if remoteChecksum != localChecksum { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 本地=%s, 远端=%s", localChecksum, remoteChecksum)} - logger.Errorf("存储目标 %s 完整性校验失败：本地 SHA-256=%s, 远端 SHA-256=%s", targetName, localChecksum, remoteChecksum) - // 删除损坏的远端文件 - _ = provider.Delete(ctx, storagePath) - return + // 上传后轻量级完整性校验：通过 List 检查远端文件大小 + remoteObjects, listErr := provider.List(ctx, storagePath) + if listErr != nil { + // List 失败不阻断，仅警告（文件可能已上传成功） + logger.Warnf("存储目标 %s 上传后大小校验跳过（List 失败）：%v", targetName, listErr) + } else { + remoteSize := int64(0) + for _, obj := range remoteObjects { + if obj.Key == storagePath { + remoteSize = obj.Size + break + } + } + if remoteSize == 0 { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: "完整性校验失败: 远端文件大小为 0（上传损坏）"} + logger.Errorf("存储目标 %s 完整性校验失败：远端文件大小为 0", targetName) + _ = provider.Delete(ctx, storagePath) + return + } + if remoteSize != fileSize { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 本地=%d bytes, 远端=%d bytes", fileSize, remoteSize)} + logger.Errorf("存储目标 %s 完整性校验失败：本地 %d bytes, 远端 %d bytes", targetName, fileSize, remoteSize) + _ = provider.Delete(ctx, storagePath) + return + } } uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "success", StoragePath: storagePath, FileSize: fileSize} - logger.Infof("存储目标 %s 上传成功，完整性校验通过 (SHA-256=%s)", targetName, localChecksum) + logger.Infof("存储目标 %s 上传成功，大小校验通过 (%d bytes, SHA-256=%s)", targetName, fileSize, localChecksum) // 每个成功目标独立执行保留策略 if s.retention != nil { cleanupResult, cleanupErr := s.retention.Cleanup(ctx, task, provider) From 7568d8a2a2c79c1216014d6567fe74cedcc92a48 Mon Sep 17 00:00:00 2001 From: Awuqing <3184394176@qq.com> Date: Tue, 31 Mar 2026 12:40:12 +0800 Subject: [PATCH 3/5] refactor: use CountingReader for upload integrity instead of List API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit List()-based size check depends on the storage backend returning accurate file sizes, which is not guaranteed (some WebDAV/Google Drive impls may return 0 or omit the size field). New approach: wrap the upload io.Reader with a CountingReader that counts bytes as they flow through during upload. After upload completes, compare counter.n against the expected fileSize. This is: - Zero extra network calls (no List, no Download) - Zero extra CPU/memory overhead (just an int64 increment per Read) - Storage-backend agnostic (works with any provider) If bytes transmitted != expected size → mark failed + auto-delete remote. --- .../service/backup_execution_service.go | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/server/internal/service/backup_execution_service.go b/server/internal/service/backup_execution_service.go index fcbe813..296394e 100644 --- a/server/internal/service/backup_execution_service.go +++ b/server/internal/service/backup_execution_service.go @@ -371,39 +371,22 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba } defer artifact.Close() logger.Infof("开始上传备份到存储目标：%s", targetName) - if uploadErr := provider.Upload(ctx, storagePath, artifact, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil { + // 用 CountingReader 包装，上传过程中统计实际传输字节数（零额外开销） + counter := &countingReader{reader: artifact} + if uploadErr := provider.Upload(ctx, storagePath, counter, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil { uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()} logger.Warnf("存储目标 %s 上传失败：%v", targetName, uploadErr) return } - // 上传后轻量级完整性校验：通过 List 检查远端文件大小 - remoteObjects, listErr := provider.List(ctx, storagePath) - if listErr != nil { - // List 失败不阻断，仅警告（文件可能已上传成功） - logger.Warnf("存储目标 %s 上传后大小校验跳过（List 失败）：%v", targetName, listErr) - } else { - remoteSize := int64(0) - for _, obj := range remoteObjects { - if obj.Key == storagePath { - remoteSize = obj.Size - break - } - } - if remoteSize == 0 { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: "完整性校验失败: 远端文件大小为 0（上传损坏）"} - logger.Errorf("存储目标 %s 完整性校验失败：远端文件大小为 0", targetName) - _ = provider.Delete(ctx, storagePath) - return - } - if remoteSize != fileSize { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 本地=%d bytes, 远端=%d bytes", fileSize, remoteSize)} - logger.Errorf("存储目标 %s 完整性校验失败：本地 %d bytes, 远端 %d bytes", targetName, fileSize, remoteSize) - _ = provider.Delete(ctx, storagePath) - return - } + // 完整性校验：对比实际传输字节数与本地文件大小 + if counter.n != fileSize { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 预期 %d bytes, 实际传输 %d bytes", fileSize, counter.n)} + logger.Errorf("存储目标 %s 完整性校验失败：预期 %d bytes, 实际传输 %d bytes", targetName, fileSize, counter.n) + _ = provider.Delete(ctx, storagePath) + return } uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "success", StoragePath: storagePath, FileSize: fileSize} - logger.Infof("存储目标 %s 上传成功，大小校验通过 (%d bytes, SHA-256=%s)", targetName, fileSize, localChecksum) + logger.Infof("存储目标 %s 上传成功 (%d bytes, SHA-256=%s)", targetName, fileSize, localChecksum) // 每个成功目标独立执行保留策略 if s.retention != nil { cleanupResult, cleanupErr := s.retention.Cleanup(ctx, task, provider) @@ -617,3 +600,15 @@ func buildStorageProviderFromRepos(ctx context.Context, storageTargetID uint, st } return provider, target, nil } + +// countingReader 包装 io.Reader，统计实际读取字节数 +type countingReader struct { + reader io.Reader + n int64 +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.n += int64(n) + return n, err +} From ad5c25f38e5339a4784ea18c08602d07104f8e41 Mon Sep 17 00:00:00 2001 From: Awuqing <3184394176@qq.com> Date: Tue, 31 Mar 2026 13:08:10 +0800 Subject: [PATCH 4/5] refactor: single-pass hashing during upload via TeeReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous approach read the file twice (once for SHA-256, once for upload), doubling disk I/O. Under concurrent multi-target uploads this becomes a bottleneck. New design — hashingReader wraps io.TeeReader + sha256.Hash: file.Read() → TeeReader → sha256.Write() (hash) + provider (upload) Single read pass yields both byte count and SHA-256 simultaneously. Each upload goroutine independently opens the file and computes its own hash. The first successful target writes checksum to the record via sync.Once. Zero extra disk I/O, zero extra memory copies, fully concurrent-safe. --- server/internal/backup/helpers.go | 17 ------ .../service/backup_execution_service.go | 53 +++++++++++-------- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/server/internal/backup/helpers.go b/server/internal/backup/helpers.go index 5ae055c..c8d7a93 100644 --- a/server/internal/backup/helpers.go +++ b/server/internal/backup/helpers.go @@ -1,10 +1,7 @@ package backup import ( - "crypto/sha256" - "encoding/hex" "fmt" - "io" "os" "path/filepath" "strings" @@ -24,20 +21,6 @@ func createTempArtifact(baseDir, taskName string, extension string) (string, str return tempDir, filepath.Join(tempDir, fileName), nil } -// SHA256File 计算文件的 SHA-256 哈希值，返回十六进制字符串 -func SHA256File(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", fmt.Errorf("open file for checksum: %w", err) - } - defer file.Close() - hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { - return "", fmt.Errorf("compute checksum: %w", err) - } - return hex.EncodeToString(hash.Sum(nil)), nil -} - func sanitizeFileName(value string) string { builder := strings.Builder{} for _, char := range strings.TrimSpace(value) { diff --git a/server/internal/service/backup_execution_service.go b/server/internal/service/backup_execution_service.go index 296394e..1cf575a 100644 --- a/server/internal/service/backup_execution_service.go +++ b/server/internal/service/backup_execution_service.go @@ -2,8 +2,11 @@ package service import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "hash" "io" "os" "path/filepath" @@ -326,17 +329,6 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba fileName = filepath.Base(finalPath) storagePath = backup.BuildStorageKey(task.Type, startedAt, fileName) - // 计算文件 SHA-256 哈希（上传前） - logger.Infof("计算备份文件校验和...") - localChecksum, checksumErr := backup.SHA256File(finalPath) - if checksumErr != nil { - errMessage = checksumErr.Error() - logger.Errorf("计算文件校验和失败：%v", checksumErr) - return - } - checksum = localChecksum - logger.Infof("文件校验和: SHA-256=%s, 大小=%d bytes", checksum, fileSize) - // 收集所有存储目标 targetIDs := collectTargetIDs(task) if len(targetIDs) == 0 { @@ -347,6 +339,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba // 并行上传到所有目标 uploadResults = make([]StorageUploadResultItem, len(targetIDs)) + var checksumOnce sync.Once var wg sync.WaitGroup for i, tid := range targetIDs { wg.Add(1) @@ -371,22 +364,25 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba } defer artifact.Close() logger.Infof("开始上传备份到存储目标：%s", targetName) - // 用 CountingReader 包装，上传过程中统计实际传输字节数（零额外开销） - counter := &countingReader{reader: artifact} - if uploadErr := provider.Upload(ctx, storagePath, counter, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil { + // hashingReader: 上传过程中同步计算字节数 + SHA-256，单次读取零额外 I/O + hr := newHashingReader(artifact) + if uploadErr := provider.Upload(ctx, storagePath, hr, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil { uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()} logger.Warnf("存储目标 %s 上传失败：%v", targetName, uploadErr) return } - // 完整性校验：对比实际传输字节数与本地文件大小 - if counter.n != fileSize { - uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 预期 %d bytes, 实际传输 %d bytes", fileSize, counter.n)} - logger.Errorf("存储目标 %s 完整性校验失败：预期 %d bytes, 实际传输 %d bytes", targetName, fileSize, counter.n) + // 完整性校验：对比实际传输字节数 + if hr.n != fileSize { + uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: fmt.Sprintf("完整性校验失败: 预期 %d bytes, 实际传输 %d bytes", fileSize, hr.n)} + logger.Errorf("存储目标 %s 完整性校验失败：预期 %d bytes, 实际传输 %d bytes", targetName, fileSize, hr.n) _ = provider.Delete(ctx, storagePath) return } + // 取第一个成功目标的哈希写入 record（所有目标读同一文件，哈希一定相同） + targetChecksum := hr.Sum() + checksumOnce.Do(func() { checksum = targetChecksum }) uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "success", StoragePath: storagePath, FileSize: fileSize} - logger.Infof("存储目标 %s 上传成功 (%d bytes, SHA-256=%s)", targetName, fileSize, localChecksum) + logger.Infof("存储目标 %s 上传成功 (%d bytes, SHA-256=%s)", targetName, fileSize, targetChecksum) // 每个成功目标独立执行保留策略 if s.retention != nil { cleanupResult, cleanupErr := s.retention.Cleanup(ctx, task, provider) @@ -601,14 +597,27 @@ func buildStorageProviderFromRepos(ctx context.Context, storageTargetID uint, st return provider, target, nil } -// countingReader 包装 io.Reader，统计实际读取字节数 -type countingReader struct { +// hashingReader 在上传过程中同步计算字节数和 SHA-256，零额外 I/O +type hashingReader struct { reader io.Reader + hash hash.Hash n int64 } -func (r *countingReader) Read(p []byte) (int, error) { +func newHashingReader(reader io.Reader) *hashingReader { + h := sha256.New() + return &hashingReader{ + reader: io.TeeReader(reader, h), + hash: h, + } +} + +func (r *hashingReader) Read(p []byte) (int, error) { n, err := r.reader.Read(p) r.n += int64(n) return n, err } + +func (r *hashingReader) Sum() string { + return hex.EncodeToString(r.hash.Sum(nil)) +} From deb7cf9a5ecf51c592ad0207027225a1a1c1322a Mon Sep 17 00:00:00 2001 From: Awuqing <3184394176@qq.com> Date: Tue, 31 Mar 2026 13:20:11 +0800 Subject: [PATCH 5/5] fix(test): use test TempDir for backup execution tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test passed an empty tempDir which defaulted to /tmp/backupx — a directory that does not exist in CI runners. Use t.TempDir() based path instead so the test is self-contained. --- server/internal/service/backup_execution_service_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/internal/service/backup_execution_service_test.go b/server/internal/service/backup_execution_service_test.go index db89e15..057539f 100644 --- a/server/internal/service/backup_execution_service_test.go +++ b/server/internal/service/backup_execution_service_test.go @@ -55,7 +55,11 @@ func newExecutionTestServices(t *testing.T) (*BackupExecutionService, *BackupRec runnerRegistry := backup.NewRegistry(backup.NewFileRunner(), backup.NewMySQLRunner(nil), backup.NewSQLiteRunner(), backup.NewPostgreSQLRunner(nil)) storageRegistry := storage.NewRegistry(localdisk.NewFactory()) retentionService := backupretention.NewService(records) - executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, "", 2) + tempDir := filepath.Join(baseDir, "tmp") + if err := os.MkdirAll(tempDir, 0o755); err != nil { + t.Fatalf("MkdirAll tempDir returned error: %v", err) + } + executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, tempDir, 2) recordService := NewBackupRecordService(records, executionService, logHub) return executionService, recordService, tasks, targets, records, sourceDir, storageDir }