feat: 解决 CDC 去重、集中备份与首次初始化问题 (#105)

实现 CDC 内容寻址仓库、远程 Agent 中央中转备份与首次初始化体验，并补充安全校验、测试及双语文档。 Closes #94 Closes #101 Closes #104
2026-09-07 08:26:42 +08:00 · 2026-08-07 23:01:29 +08:00
parent 50ce6587d8
commit 00151e466c
74 changed files with 4507 additions and 400 deletions
@@ -2,11 +2,13 @@ package retention

 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
 	"time"

+	"backupx/server/internal/backup"
 	"backupx/server/internal/model"
 	"backupx/server/internal/repository"
 	"backupx/server/internal/storage"
@@ -40,16 +42,48 @@ type CleanupResult struct {
 	Warnings       []string
 }

-type Service struct {
-	records repository.BackupRecordRepository
-	now     func() time.Time
+type cleanupObject struct {
+	targetID uint
+	path     string
 }

-func NewService(records repository.BackupRecordRepository) *Service {
-	return &Service{records: records, now: func() time.Time { return time.Now().UTC() }}
+type storedUploadResult struct {
+	StorageTargetID uint   `json:"storageTargetId"`
+	Status          string `json:"status"`
+	StoragePath     string `json:"storagePath"`
+}
+
+type Service struct {
+	records       repository.BackupRecordRepository
+	repositoryKey []byte
+	now           func() time.Time
+}
+
+func NewService(records repository.BackupRecordRepository, repositoryKey ...[]byte) *Service {
+	var key []byte
+	if len(repositoryKey) > 0 {
+		key = append([]byte(nil), repositoryKey[0]...)
+	}
+	return &Service{records: records, repositoryKey: key, now: func() time.Time { return time.Now().UTC() }}
 }

 func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider storage.StorageProvider) (*CleanupResult, error) {
+	return s.cleanup(ctx, task, func(uint) (storage.StorageProvider, bool) {
+		return provider, provider != nil
+	})
+}
+
+// CleanupProviders applies one retention decision to every successful copy of
+// a record before deleting its database row. This prevents multi-target tasks
+// from leaving stale objects after the first target removes the shared record.
+func (s *Service) CleanupProviders(ctx context.Context, task *model.BackupTask, providers map[uint]storage.StorageProvider) (*CleanupResult, error) {
+	return s.cleanup(ctx, task, func(targetID uint) (storage.StorageProvider, bool) {
+		provider, ok := providers[targetID]
+		return provider, ok && provider != nil
+	})
+}
+
+func (s *Service) cleanup(ctx context.Context, task *model.BackupTask, resolveProvider func(uint) (storage.StorageProvider, bool)) (*CleanupResult, error) {
 	if task == nil {
 		return nil, fmt.Errorf("backup task is required")
 	}
@@ -67,17 +101,35 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
 	// 差异链保护：保留仍被存活差异依赖的全量，避免删除基线后差异无法恢复。
 	candidates = protectDifferentialBases(records, candidates)
 	result := &CleanupResult{}
+	repositoryProviders := make(map[uint]storage.StorageProvider)
+	touchedProviders := make(map[uint]storage.StorageProvider)
 	for _, record := range candidates {
-		if strings.TrimSpace(record.StoragePath) != "" {
-			if provider == nil {
-				result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider for cleanup", record.ID))
+		objects, objectErr := cleanupObjectsForRecord(record)
+		if objectErr != nil {
+			result.Warnings = append(result.Warnings, fmt.Sprintf("decode storage copies for record %d failed: %v", record.ID, objectErr))
+			continue
+		}
+		allObjectsDeleted := true
+		for _, object := range objects {
+			provider, ok := resolveProvider(object.targetID)
+			if !ok {
+				result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider %d for cleanup", record.ID, object.targetID))
+				allObjectsDeleted = false
 				continue
 			}
-			if err := provider.Delete(ctx, record.StoragePath); err != nil {
-				result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s failed: %v", record.StoragePath, err))
+			if err := provider.Delete(ctx, object.path); err != nil {
+				result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s from target %d failed: %v", object.path, object.targetID, err))
+				allObjectsDeleted = false
 				continue
 			}
 			result.DeletedObjects++
+			touchedProviders[object.targetID] = provider
+			if record.BackupKind == model.BackupKindRepository {
+				repositoryProviders[object.targetID] = provider
+			}
+		}
+		if !allObjectsDeleted {
+			continue
 		}
 		if err := s.records.Delete(ctx, record.ID); err != nil {
 			result.Warnings = append(result.Warnings, fmt.Sprintf("delete backup record %d failed: %v", record.ID, err))
@@ -85,13 +137,25 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
 		}
 		result.DeletedRecords++
 	}
+	for targetID, provider := range repositoryProviders {
+		pruned, pruneErr := backup.NewRepositoryStore(s.repositoryKey).Prune(ctx, provider)
+		if pruneErr != nil {
+			result.Warnings = append(result.Warnings, fmt.Sprintf("prune CDC repository on target %d failed: %v", targetID, pruneErr))
+		} else {
+			result.DeletedObjects += pruned.DeletedPacks + pruned.DeletedIndexes
+		}
+	}

 	// 清理空目录：收集被删除文件的父目录，尝试移除空目录
-	if dirCleaner, ok := provider.(storage.StorageDirCleaner); ok && result.DeletedObjects > 0 {
+	for targetID, provider := range touchedProviders {
+		dirCleaner, ok := provider.(storage.StorageDirCleaner)
+		if !ok {
+			continue
+		}
 		prefixes := collectDirPrefixes(candidates)
 		for _, prefix := range prefixes {
 			if err := dirCleaner.RemoveEmptyDirs(ctx, prefix); err != nil {
-				result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s: %v", prefix, err))
+				result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s on target %d: %v", prefix, targetID, err))
 			}
 		}
 	}
@@ -99,6 +163,43 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
 	return result, nil
 }

+func cleanupObjectsForRecord(record model.BackupRecord) ([]cleanupObject, error) {
+	defaultPath := strings.TrimSpace(record.StoragePath)
+	if strings.TrimSpace(record.StorageUploadResults) == "" {
+		if defaultPath == "" {
+			return nil, nil
+		}
+		return []cleanupObject{{targetID: record.StorageTargetID, path: defaultPath}}, nil
+	}
+	var results []storedUploadResult
+	if err := json.Unmarshal([]byte(record.StorageUploadResults), &results); err != nil {
+		return nil, err
+	}
+	objects := make([]cleanupObject, 0, len(results))
+	seen := make(map[uint]struct{}, len(results))
+	for _, result := range results {
+		if !strings.EqualFold(strings.TrimSpace(result.Status), model.BackupRecordStatusSuccess) {
+			continue
+		}
+		objectPath := strings.TrimSpace(result.StoragePath)
+		if objectPath == "" {
+			objectPath = defaultPath
+		}
+		if objectPath == "" {
+			continue
+		}
+		if _, exists := seen[result.StorageTargetID]; exists {
+			continue
+		}
+		seen[result.StorageTargetID] = struct{}{}
+		objects = append(objects, cleanupObject{targetID: result.StorageTargetID, path: objectPath})
+	}
+	if len(objects) == 0 && defaultPath != "" {
+		return nil, fmt.Errorf("successful record has no successful storage copy")
+	}
+	return objects, nil
+}
+
 // protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」，
 // 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。
 func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord {