mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-09-05 07:26:43 +08:00
feat(backup): 新增 CDC 内容寻址仓库模式
使用内容定义分块、不可变 pack 与快照索引实现跨文件和跨快照去重。 接入多目标上传、软配额、恢复、下载、校验、保留和安全垃圾回收，并补充前端入口、中英文文档及端到端测试。 Closes #94
This commit is contained in:
Awuqing
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,131 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
repositoryChunkMin = 512 << 10
|
||||
repositoryChunkAvg = 1 << 20
|
||||
repositoryChunkMax = 4 << 20
|
||||
)
|
||||
|
||||
// contentDefinedChunker implements the normalized FastCDC cut-point strategy.
|
||||
// The rolling Gear hash only retains the latest 64 bytes through uint64
|
||||
// overflow, so chunk boundaries re-synchronize after insertions or deletions.
|
||||
type contentDefinedChunker struct {
|
||||
minSize int
|
||||
avgSize int
|
||||
maxSize int
|
||||
smallMask uint64
|
||||
largeMask uint64
|
||||
gear [256]uint64
|
||||
}
|
||||
|
||||
func newContentDefinedChunker() *contentDefinedChunker {
|
||||
chunker := &contentDefinedChunker{
|
||||
minSize: repositoryChunkMin,
|
||||
avgSize: repositoryChunkAvg,
|
||||
maxSize: repositoryChunkMax,
|
||||
smallMask: (1 << 21) - 1,
|
||||
largeMask: (1 << 19) - 1,
|
||||
}
|
||||
|
||||
// SplitMix64 produces a stable, well-distributed Gear table. The seed and
|
||||
// generation algorithm are part of repository format v1 and must not change.
|
||||
seed := uint64(0x6a09e667f3bcc909)
|
||||
for i := range chunker.gear {
|
||||
seed += 0x9e3779b97f4a7c15
|
||||
value := seed
|
||||
value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9
|
||||
value = (value ^ (value >> 27)) * 0x94d049bb133111eb
|
||||
chunker.gear[i] = value ^ (value >> 31)
|
||||
}
|
||||
return chunker
|
||||
}
|
||||
|
||||
func (c *contentDefinedChunker) Split(ctx context.Context, reader io.Reader, emit func([]byte) error) error {
|
||||
if reader == nil || emit == nil {
|
||||
return fmt.Errorf("chunk reader and emitter are required")
|
||||
}
|
||||
|
||||
pending := make([]byte, 0, c.maxSize+(256<<10))
|
||||
readBuffer := make([]byte, 256<<10)
|
||||
eof := false
|
||||
for {
|
||||
if !eof {
|
||||
readCount, readErr := reader.Read(readBuffer)
|
||||
if readCount > 0 {
|
||||
pending = append(pending, readBuffer[:readCount]...)
|
||||
}
|
||||
switch readErr {
|
||||
case nil:
|
||||
case io.EOF:
|
||||
eof = true
|
||||
default:
|
||||
return fmt.Errorf("read source for chunking: %w", readErr)
|
||||
}
|
||||
if readCount == 0 && readErr == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for len(pending) > 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
cut := c.findCutPoint(pending, eof)
|
||||
if cut == 0 {
|
||||
break
|
||||
}
|
||||
chunk := make([]byte, cut)
|
||||
copy(chunk, pending[:cut])
|
||||
if err := emit(chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
copy(pending, pending[cut:])
|
||||
pending = pending[:len(pending)-cut]
|
||||
}
|
||||
|
||||
if eof {
|
||||
if len(pending) != 0 {
|
||||
return fmt.Errorf("chunker stopped with %d buffered bytes", len(pending))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *contentDefinedChunker) findCutPoint(data []byte, eof bool) int {
|
||||
if len(data) < c.minSize {
|
||||
if eof {
|
||||
return len(data)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
limit := len(data)
|
||||
if limit > c.maxSize {
|
||||
limit = c.maxSize
|
||||
}
|
||||
var hash uint64
|
||||
for index := c.minSize; index < limit; index++ {
|
||||
hash = (hash << 1) + c.gear[data[index]]
|
||||
mask := c.largeMask
|
||||
if index < c.avgSize {
|
||||
mask = c.smallMask
|
||||
}
|
||||
if hash&mask == 0 {
|
||||
return index + 1
|
||||
}
|
||||
}
|
||||
if len(data) >= c.maxSize {
|
||||
return c.maxSize
|
||||
}
|
||||
if eof {
|
||||
return len(data)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/storage"
|
||||
)
|
||||
|
||||
func TestContentDefinedChunkerResynchronizesAfterInsertion(t *testing.T) {
|
||||
source := make([]byte, 8<<20)
|
||||
if _, err := rand.New(rand.NewSource(42)).Read(source); err != nil {
|
||||
t.Fatalf("generate source: %v", err)
|
||||
}
|
||||
modified := make([]byte, 0, len(source)+4096)
|
||||
modified = append(modified, source[:2<<20]...)
|
||||
modified = append(modified, bytes.Repeat([]byte("inserted"), 512)...)
|
||||
modified = append(modified, source[2<<20:]...)
|
||||
|
||||
chunker := newContentDefinedChunker()
|
||||
collect := func(data []byte) map[string]struct{} {
|
||||
t.Helper()
|
||||
ids := make(map[string]struct{})
|
||||
err := chunker.Split(context.Background(), bytes.NewReader(data), func(chunk []byte) error {
|
||||
digest := sha256.Sum256(chunk)
|
||||
ids[fmt.Sprintf("%x", digest[:])] = struct{}{}
|
||||
if len(chunk) > repositoryChunkMax {
|
||||
return fmt.Errorf("chunk exceeds maximum: %d", len(chunk))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("split chunks: %v", err)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
originalChunks := collect(source)
|
||||
modifiedChunks := collect(modified)
|
||||
shared := 0
|
||||
for chunkID := range originalChunks {
|
||||
if _, ok := modifiedChunks[chunkID]; ok {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
if shared < len(originalChunks)/2 {
|
||||
t.Fatalf("content-defined boundaries did not resynchronize: shared=%d original=%d", shared, len(originalChunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryRoundTripDedupAndPrune(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tempDir := t.TempDir()
|
||||
sourceDir := filepath.Join(tempDir, "dataset")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "empty"), 0o755); err != nil {
|
||||
t.Fatalf("create source: %v", err)
|
||||
}
|
||||
original := make([]byte, 6<<20)
|
||||
if _, err := rand.New(rand.NewSource(7)).Read(original); err != nil {
|
||||
t.Fatalf("generate fixture: %v", err)
|
||||
}
|
||||
primaryPath := filepath.Join(sourceDir, "primary.bin")
|
||||
duplicatePath := filepath.Join(sourceDir, "duplicate.bin")
|
||||
if err := os.WriteFile(primaryPath, original, 0o640); err != nil {
|
||||
t.Fatalf("write primary: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(duplicatePath, original, 0o640); err != nil {
|
||||
t.Fatalf("write duplicate: %v", err)
|
||||
}
|
||||
|
||||
key := sha256.Sum256([]byte("repository-test-key"))
|
||||
store := NewRepositoryStore(key[:])
|
||||
provider := newMemoryRepositoryProvider()
|
||||
task := TaskSpec{
|
||||
ID: 12,
|
||||
Name: "repository-test",
|
||||
Type: "file",
|
||||
SourcePaths: []string{sourceDir},
|
||||
Compression: "zstd",
|
||||
Encrypt: true,
|
||||
StartedAt: time.Date(2026, 8, 6, 1, 2, 3, 0, time.UTC),
|
||||
TempDir: tempDir,
|
||||
}
|
||||
|
||||
firstPlan, err := store.BuildPlan(ctx, task, NopLogWriter{})
|
||||
if err != nil {
|
||||
t.Fatalf("build first plan: %v", err)
|
||||
}
|
||||
firstKey := store.SnapshotKey(task.ID, 1, task.StartedAt)
|
||||
firstResult, err := store.Upload(ctx, provider, firstPlan, firstKey)
|
||||
if closeErr := firstPlan.Close(); closeErr != nil {
|
||||
t.Fatalf("close first plan: %v", closeErr)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("upload first snapshot: %v", err)
|
||||
}
|
||||
if firstResult.NewChunks == 0 || firstResult.UniqueChunks == 0 {
|
||||
t.Fatalf("first upload did not create chunks: %+v", firstResult)
|
||||
}
|
||||
if firstPlan.UniqueSize >= firstPlan.LogicalSize {
|
||||
t.Fatalf("duplicate file was not deduplicated within plan: unique=%d logical=%d", firstPlan.UniqueSize, firstPlan.LogicalSize)
|
||||
}
|
||||
|
||||
modified := make([]byte, 0, len(original)+4096)
|
||||
modified = append(modified, original[:2<<20]...)
|
||||
modified = append(modified, bytes.Repeat([]byte("changed!"), 512)...)
|
||||
modified = append(modified, original[2<<20:]...)
|
||||
if err := os.WriteFile(primaryPath, modified, 0o640); err != nil {
|
||||
t.Fatalf("modify primary: %v", err)
|
||||
}
|
||||
task.StartedAt = task.StartedAt.Add(time.Hour)
|
||||
secondPlan, err := store.BuildPlan(ctx, task, NopLogWriter{})
|
||||
if err != nil {
|
||||
t.Fatalf("build second plan: %v", err)
|
||||
}
|
||||
secondKey := store.SnapshotKey(task.ID, 2, task.StartedAt)
|
||||
secondResult, err := store.Upload(ctx, provider, secondPlan, secondKey)
|
||||
if closeErr := secondPlan.Close(); closeErr != nil {
|
||||
t.Fatalf("close second plan: %v", closeErr)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("upload second snapshot: %v", err)
|
||||
}
|
||||
if secondResult.ReusedBytes <= secondResult.LogicalSize/3 {
|
||||
t.Fatalf("second snapshot reused too little data: %+v", secondResult)
|
||||
}
|
||||
if secondResult.UploadedBytes >= secondResult.LogicalSize {
|
||||
t.Fatalf("incremental upload was not smaller than logical data: %+v", secondResult)
|
||||
}
|
||||
|
||||
verify, err := store.Verify(ctx, provider, secondKey, secondResult.Checksum)
|
||||
if err != nil {
|
||||
t.Fatalf("verify repository: %v", err)
|
||||
}
|
||||
if verify.Chunks == 0 || verify.Bytes == 0 {
|
||||
t.Fatalf("empty verification result: %+v", verify)
|
||||
}
|
||||
|
||||
restoreRoot := filepath.Join(tempDir, "restore")
|
||||
restoreTask := task
|
||||
restoreTask.RestoreTargetPath = restoreRoot
|
||||
if err := store.Restore(ctx, provider, secondKey, restoreTask, NopLogWriter{}); err != nil {
|
||||
t.Fatalf("restore snapshot: %v", err)
|
||||
}
|
||||
restored, err := os.ReadFile(filepath.Join(restoreRoot, filepath.Base(sourceDir), "primary.bin"))
|
||||
if err != nil {
|
||||
t.Fatalf("read restored primary: %v", err)
|
||||
}
|
||||
if !bytes.Equal(restored, modified) {
|
||||
t.Fatalf("restored primary differs from source")
|
||||
}
|
||||
if info, err := os.Stat(filepath.Join(restoreRoot, filepath.Base(sourceDir), "empty")); err != nil || !info.IsDir() {
|
||||
t.Fatalf("empty directory was not restored: info=%v err=%v", info, err)
|
||||
}
|
||||
|
||||
if err := provider.Delete(ctx, firstKey); err != nil {
|
||||
t.Fatalf("delete first snapshot: %v", err)
|
||||
}
|
||||
if _, err := store.Prune(ctx, provider); err != nil {
|
||||
t.Fatalf("prune with live snapshot: %v", err)
|
||||
}
|
||||
if err := provider.Delete(ctx, secondKey); err != nil {
|
||||
t.Fatalf("delete second snapshot: %v", err)
|
||||
}
|
||||
pruned, err := store.Prune(ctx, provider)
|
||||
if err != nil {
|
||||
t.Fatalf("prune empty repository: %v", err)
|
||||
}
|
||||
if pruned.DeletedPacks == 0 || pruned.DeletedIndexes == 0 {
|
||||
t.Fatalf("prune did not reclaim repository data: %+v", pruned)
|
||||
}
|
||||
if objects, err := provider.List(ctx, repositoryPackPrefix); err != nil || len(objects) != 0 {
|
||||
t.Fatalf("packs remain after prune: objects=%v err=%v", objects, err)
|
||||
}
|
||||
}
|
||||
|
||||
type memoryRepositoryProvider struct {
|
||||
mu sync.RWMutex
|
||||
objects map[string][]byte
|
||||
times map[string]time.Time
|
||||
}
|
||||
|
||||
func newMemoryRepositoryProvider() *memoryRepositoryProvider {
|
||||
return &memoryRepositoryProvider{objects: make(map[string][]byte), times: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
func (p *memoryRepositoryProvider) Type() storage.ProviderType { return "memory" }
|
||||
func (p *memoryRepositoryProvider) TestConnection(context.Context) error { return nil }
|
||||
|
||||
func (p *memoryRepositoryProvider) Upload(_ context.Context, key string, reader io.Reader, size int64, _ map[string]string) error {
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(len(data)) != size {
|
||||
return fmt.Errorf("size mismatch for %s: %d != %d", key, len(data), size)
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.objects[key] = append([]byte(nil), data...)
|
||||
p.times[key] = time.Now().UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *memoryRepositoryProvider) Download(_ context.Context, key string) (io.ReadCloser, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
data, ok := p.objects[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("object %s not found", key)
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(append([]byte(nil), data...))), nil
|
||||
}
|
||||
|
||||
func (p *memoryRepositoryProvider) DownloadRange(_ context.Context, key string, offset, length int64) (io.ReadCloser, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
data, ok := p.objects[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("object %s not found", key)
|
||||
}
|
||||
if offset < 0 || length <= 0 || offset+length > int64(len(data)) {
|
||||
return nil, fmt.Errorf("invalid range %d:%d for %s", offset, length, key)
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(append([]byte(nil), data[offset:offset+length]...))), nil
|
||||
}
|
||||
|
||||
func (p *memoryRepositoryProvider) Delete(_ context.Context, key string) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if _, ok := p.objects[key]; !ok {
|
||||
return fmt.Errorf("object %s not found", key)
|
||||
}
|
||||
delete(p.objects, key)
|
||||
delete(p.times, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *memoryRepositoryProvider) List(_ context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
result := make([]storage.ObjectInfo, 0)
|
||||
for key, data := range p.objects {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
result = append(result, storage.ObjectInfo{Key: key, Size: int64(len(data)), UpdatedAt: p.times[key]})
|
||||
}
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].Key < result[j].Key })
|
||||
return result, nil
|
||||
}
|
||||
@@ -2,11 +2,13 @@ package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/backup"
|
||||
"backupx/server/internal/model"
|
||||
"backupx/server/internal/repository"
|
||||
"backupx/server/internal/storage"
|
||||
@@ -40,16 +42,48 @@ type CleanupResult struct {
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
records repository.BackupRecordRepository
|
||||
now func() time.Time
|
||||
type cleanupObject struct {
|
||||
targetID uint
|
||||
path string
|
||||
}
|
||||
|
||||
func NewService(records repository.BackupRecordRepository) *Service {
|
||||
return &Service{records: records, now: func() time.Time { return time.Now().UTC() }}
|
||||
type storedUploadResult struct {
|
||||
StorageTargetID uint `json:"storageTargetId"`
|
||||
Status string `json:"status"`
|
||||
StoragePath string `json:"storagePath"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
records repository.BackupRecordRepository
|
||||
repositoryKey []byte
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewService(records repository.BackupRecordRepository, repositoryKey ...[]byte) *Service {
|
||||
var key []byte
|
||||
if len(repositoryKey) > 0 {
|
||||
key = append([]byte(nil), repositoryKey[0]...)
|
||||
}
|
||||
return &Service{records: records, repositoryKey: key, now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider storage.StorageProvider) (*CleanupResult, error) {
|
||||
return s.cleanup(ctx, task, func(uint) (storage.StorageProvider, bool) {
|
||||
return provider, provider != nil
|
||||
})
|
||||
}
|
||||
|
||||
// CleanupProviders applies one retention decision to every successful copy of
|
||||
// a record before deleting its database row. This prevents multi-target tasks
|
||||
// from leaving stale objects after the first target removes the shared record.
|
||||
func (s *Service) CleanupProviders(ctx context.Context, task *model.BackupTask, providers map[uint]storage.StorageProvider) (*CleanupResult, error) {
|
||||
return s.cleanup(ctx, task, func(targetID uint) (storage.StorageProvider, bool) {
|
||||
provider, ok := providers[targetID]
|
||||
return provider, ok && provider != nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) cleanup(ctx context.Context, task *model.BackupTask, resolveProvider func(uint) (storage.StorageProvider, bool)) (*CleanupResult, error) {
|
||||
if task == nil {
|
||||
return nil, fmt.Errorf("backup task is required")
|
||||
}
|
||||
@@ -67,17 +101,35 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
|
||||
// 差异链保护：保留仍被存活差异依赖的全量，避免删除基线后差异无法恢复。
|
||||
candidates = protectDifferentialBases(records, candidates)
|
||||
result := &CleanupResult{}
|
||||
repositoryProviders := make(map[uint]storage.StorageProvider)
|
||||
touchedProviders := make(map[uint]storage.StorageProvider)
|
||||
for _, record := range candidates {
|
||||
if strings.TrimSpace(record.StoragePath) != "" {
|
||||
if provider == nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider for cleanup", record.ID))
|
||||
objects, objectErr := cleanupObjectsForRecord(record)
|
||||
if objectErr != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("decode storage copies for record %d failed: %v", record.ID, objectErr))
|
||||
continue
|
||||
}
|
||||
allObjectsDeleted := true
|
||||
for _, object := range objects {
|
||||
provider, ok := resolveProvider(object.targetID)
|
||||
if !ok {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider %d for cleanup", record.ID, object.targetID))
|
||||
allObjectsDeleted = false
|
||||
continue
|
||||
}
|
||||
if err := provider.Delete(ctx, record.StoragePath); err != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s failed: %v", record.StoragePath, err))
|
||||
if err := provider.Delete(ctx, object.path); err != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s from target %d failed: %v", object.path, object.targetID, err))
|
||||
allObjectsDeleted = false
|
||||
continue
|
||||
}
|
||||
result.DeletedObjects++
|
||||
touchedProviders[object.targetID] = provider
|
||||
if record.BackupKind == model.BackupKindRepository {
|
||||
repositoryProviders[object.targetID] = provider
|
||||
}
|
||||
}
|
||||
if !allObjectsDeleted {
|
||||
continue
|
||||
}
|
||||
if err := s.records.Delete(ctx, record.ID); err != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("delete backup record %d failed: %v", record.ID, err))
|
||||
@@ -85,13 +137,25 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
|
||||
}
|
||||
result.DeletedRecords++
|
||||
}
|
||||
for targetID, provider := range repositoryProviders {
|
||||
pruned, pruneErr := backup.NewRepositoryStore(s.repositoryKey).Prune(ctx, provider)
|
||||
if pruneErr != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("prune CDC repository on target %d failed: %v", targetID, pruneErr))
|
||||
} else {
|
||||
result.DeletedObjects += pruned.DeletedPacks + pruned.DeletedIndexes
|
||||
}
|
||||
}
|
||||
|
||||
// 清理空目录：收集被删除文件的父目录，尝试移除空目录
|
||||
if dirCleaner, ok := provider.(storage.StorageDirCleaner); ok && result.DeletedObjects > 0 {
|
||||
for targetID, provider := range touchedProviders {
|
||||
dirCleaner, ok := provider.(storage.StorageDirCleaner)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
prefixes := collectDirPrefixes(candidates)
|
||||
for _, prefix := range prefixes {
|
||||
if err := dirCleaner.RemoveEmptyDirs(ctx, prefix); err != nil {
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s: %v", prefix, err))
|
||||
result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s on target %d: %v", prefix, targetID, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,6 +163,43 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func cleanupObjectsForRecord(record model.BackupRecord) ([]cleanupObject, error) {
|
||||
defaultPath := strings.TrimSpace(record.StoragePath)
|
||||
if strings.TrimSpace(record.StorageUploadResults) == "" {
|
||||
if defaultPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return []cleanupObject{{targetID: record.StorageTargetID, path: defaultPath}}, nil
|
||||
}
|
||||
var results []storedUploadResult
|
||||
if err := json.Unmarshal([]byte(record.StorageUploadResults), &results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects := make([]cleanupObject, 0, len(results))
|
||||
seen := make(map[uint]struct{}, len(results))
|
||||
for _, result := range results {
|
||||
if !strings.EqualFold(strings.TrimSpace(result.Status), model.BackupRecordStatusSuccess) {
|
||||
continue
|
||||
}
|
||||
objectPath := strings.TrimSpace(result.StoragePath)
|
||||
if objectPath == "" {
|
||||
objectPath = defaultPath
|
||||
}
|
||||
if objectPath == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[result.StorageTargetID]; exists {
|
||||
continue
|
||||
}
|
||||
seen[result.StorageTargetID] = struct{}{}
|
||||
objects = append(objects, cleanupObject{targetID: result.StorageTargetID, path: objectPath})
|
||||
}
|
||||
if len(objects) == 0 && defaultPath != "" {
|
||||
return nil, fmt.Errorf("successful record has no successful storage copy")
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」，
|
||||
// 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。
|
||||
func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord {
|
||||
|
||||
@@ -221,3 +221,66 @@ func TestCleanupDeletesExpiredRecords(t *testing.T) {
|
||||
t.Fatalf("unexpected deleted objects: %#v", provider.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupProvidersDeletesEverySuccessfulCopyBeforeRecord(t *testing.T) {
|
||||
now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC)
|
||||
completedNew := now.Add(-time.Hour)
|
||||
completedOld := now.Add(-24 * time.Hour)
|
||||
repo := &fakeRecordRepository{records: []model.BackupRecord{
|
||||
{ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew},
|
||||
{
|
||||
ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld,
|
||||
StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"},{"storageTargetId":13,"status":"failed"}]`,
|
||||
},
|
||||
}}
|
||||
first := &fakeProvider{}
|
||||
second := &fakeProvider{}
|
||||
service := NewService(repo)
|
||||
service.now = func() time.Time { return now }
|
||||
|
||||
result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{
|
||||
11: first,
|
||||
12: second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupProviders returned error: %v", err)
|
||||
}
|
||||
if result.DeletedRecords != 1 || result.DeletedObjects != 2 || len(result.Warnings) != 0 {
|
||||
t.Fatalf("unexpected cleanup result: %#v", result)
|
||||
}
|
||||
if len(repo.deleted) != 1 || repo.deleted[0] != 1 {
|
||||
t.Fatalf("unexpected deleted records: %#v", repo.deleted)
|
||||
}
|
||||
if len(first.deleted) != 1 || first.deleted[0] != "first/1" {
|
||||
t.Fatalf("unexpected first-target deletes: %#v", first.deleted)
|
||||
}
|
||||
if len(second.deleted) != 1 || second.deleted[0] != "second/1" {
|
||||
t.Fatalf("unexpected second-target deletes: %#v", second.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupProvidersKeepsRecordWhenCopyProviderIsUnavailable(t *testing.T) {
|
||||
now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC)
|
||||
completedNew := now.Add(-time.Hour)
|
||||
completedOld := now.Add(-24 * time.Hour)
|
||||
repo := &fakeRecordRepository{records: []model.BackupRecord{
|
||||
{ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew},
|
||||
{
|
||||
ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld,
|
||||
StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"}]`,
|
||||
},
|
||||
}}
|
||||
first := &fakeProvider{}
|
||||
service := NewService(repo)
|
||||
|
||||
result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{11: first})
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupProviders returned error: %v", err)
|
||||
}
|
||||
if result.DeletedRecords != 0 || result.DeletedObjects != 1 || len(result.Warnings) != 1 {
|
||||
t.Fatalf("unexpected safe partial-cleanup result: %#v", result)
|
||||
}
|
||||
if len(repo.deleted) != 0 {
|
||||
t.Fatalf("record must remain until all copies are deleted: %#v", repo.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user