feat(backup): 新增差异备份(differential)模式 (#88)

文件备份新增差异模式:仅打包自上次全量以来的变更并记录删除,恢复自动按全量+差异链还原。含基线解析、链式恢复、保留链保护与本机文件任务校验;清单/比对/删除/往返/保留保护单测全覆盖。
This commit is contained in:
Wu Qing
2026-05-27 19:03:40 +08:00
committed by GitHub
parent f584a0802a
commit 90b58d58d6
17 changed files with 761 additions and 60 deletions

View File

@@ -3,6 +3,7 @@ package backup
import (
"archive/tar"
"context"
"encoding/json"
"fmt"
"io"
"os"
@@ -52,6 +53,20 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
defer tw.Close()
excludes := normalizeExcludePatterns(task.ExcludePatterns)
// 差异备份:基于上次全量清单仅打包新增/变更条目并记录删除;
// 全量备份记录完整清单manifest供后续差异比对。
differential := task.Differential && len(task.BaseManifest.Entries) > 0
baseIndex := map[string]ManifestEntry{}
seen := map[string]struct{}{}
var manifest *Manifest
if differential {
baseIndex = task.BaseManifest.index()
writer.WriteLine(fmt.Sprintf("差异备份模式:基线含 %d 个条目", len(baseIndex)))
} else {
manifest = &Manifest{Entries: make([]ManifestEntry, 0)}
}
totalFileCount := 0
totalDirCount := 0
@@ -88,6 +103,16 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
return nil
}
entry := entryFromInfo(archiveName, currentInfo)
if differential {
seen[entry.Path] = struct{}{}
if !changedSince(baseIndex, entry) {
return nil // 自全量以来未变更,跳过
}
} else {
manifest.Entries = append(manifest.Entries, entry)
}
if currentInfo.IsDir() {
dirCount++
writer.WriteLine(fmt.Sprintf("📁 进入目录 %s", archiveName))
@@ -130,10 +155,16 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
totalDirCount += dirCount
}
if len(sourcePaths) > 1 {
if differential {
deletions := deletedPaths(baseIndex, seen)
if err := writeDeletionsEntry(tw, deletions); err != nil {
return nil, err
}
writer.WriteLine(fmt.Sprintf("差异备份完成(%d 个目录、%d 个文件变更,删除 %d 项)", totalDirCount, totalFileCount, len(deletions)))
} else if len(sourcePaths) > 1 {
writer.WriteLine(fmt.Sprintf("全部源路径打包完成(共 %d 个目录,%d 个文件)", totalDirCount, totalFileCount))
}
return &RunResult{ArtifactPath: artifactPath, FileName: filepath.Base(artifactPath), TempDir: tempDir}, nil
return &RunResult{ArtifactPath: artifactPath, FileName: filepath.Base(artifactPath), TempDir: tempDir, Manifest: manifest}, nil
}
func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath string, writer LogWriter) error {
@@ -151,6 +182,7 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
if err := os.MkdirAll(targetParent, 0o755); err != nil {
return fmt.Errorf("create restore parent: %w", err)
}
var pendingDeletions []string
tr := tar.NewReader(artifactFile)
for {
header, err := tr.Next()
@@ -160,13 +192,23 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
if err != nil {
return fmt.Errorf("read tar entry: %w", err)
}
// 差异归档的删除清单不落地,留待提取完成后统一应用(避免被同批新增条目误删)。
if header.Name == deletionsEntryName {
data, readErr := io.ReadAll(tr)
if readErr != nil {
return fmt.Errorf("read deletions entry: %w", readErr)
}
if jsonErr := json.Unmarshal(data, &pendingDeletions); jsonErr != nil {
return fmt.Errorf("parse deletions entry: %w", jsonErr)
}
continue
}
cleanName := path.Clean(strings.TrimSpace(header.Name))
if cleanName == "." || cleanName == "" {
continue
}
targetPath := filepath.Clean(filepath.Join(targetParent, filepath.FromSlash(cleanName)))
parentWithSep := filepath.Clean(targetParent) + string(filepath.Separator)
if targetPath != filepath.Clean(targetParent) && !strings.HasPrefix(targetPath, parentWithSep) {
targetPath, ok := resolveWithinParent(targetParent, cleanName)
if !ok {
return fmt.Errorf("tar entry escapes restore path")
}
switch header.Typeflag {
@@ -191,10 +233,65 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
}
}
}
if err := applyDeletions(targetParent, pendingDeletions, writer); err != nil {
return err
}
writer.WriteLine("文件恢复完成")
return nil
}
// resolveWithinParent 将归档相对名安全解析为 targetParent 下的绝对路径;
// 越界(路径穿越)时返回 ok=false。提取与删除共用此校验杜绝逃逸。
func resolveWithinParent(targetParent, name string) (string, bool) {
targetPath := filepath.Clean(filepath.Join(targetParent, filepath.FromSlash(name)))
cleanParent := filepath.Clean(targetParent)
if targetPath == cleanParent {
return targetPath, true
}
if !strings.HasPrefix(targetPath, cleanParent+string(filepath.Separator)) {
return "", false
}
return targetPath, true
}
// writeDeletionsEntry 将差异备份的删除路径列表写入归档特殊条目。
func writeDeletionsEntry(tw *tar.Writer, deletions []string) error {
payload, err := json.Marshal(deletions)
if err != nil {
return fmt.Errorf("marshal deletions: %w", err)
}
header := &tar.Header{Name: deletionsEntryName, Mode: 0o600, Size: int64(len(payload)), Typeflag: tar.TypeReg}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write deletions header: %w", err)
}
if _, err := tw.Write(payload); err != nil {
return fmt.Errorf("write deletions body: %w", err)
}
return nil
}
// applyDeletions 在基线恢复之上删除差异归档记录的路径(仅差异备份恢复时存在)。
// 每个路径经 resolveWithinParent 校验,越界即报错;目标不存在视为已删除。
func applyDeletions(targetParent string, deletions []string, writer LogWriter) error {
for _, name := range deletions {
clean := path.Clean(strings.TrimSpace(name))
if clean == "." || clean == "" {
continue
}
targetPath, ok := resolveWithinParent(targetParent, clean)
if !ok {
return fmt.Errorf("deletion entry escapes restore path")
}
if err := os.RemoveAll(targetPath); err != nil {
return fmt.Errorf("apply deletion %s: %w", clean, err)
}
}
if len(deletions) > 0 {
writer.WriteLine(fmt.Sprintf("已应用差异删除 %d 项", len(deletions)))
}
return nil
}
func normalizeExcludePatterns(items []string) []string {
result := make([]string, 0, len(items))
for _, item := range items {

View File

@@ -0,0 +1,141 @@
package backup
import (
"archive/tar"
"context"
"io"
"os"
"path/filepath"
"testing"
)
func diffWrite(t *testing.T, p, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(p), err)
}
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatalf("write %s: %v", p, err)
}
}
func diffAssertContent(t *testing.T, p, want string) {
t.Helper()
got, err := os.ReadFile(p)
if err != nil {
t.Fatalf("read %s: %v", p, err)
}
if string(got) != want {
t.Fatalf("%s content = %q, want %q", p, string(got), want)
}
}
func diffAssertAbsent(t *testing.T, p string) {
t.Helper()
if _, err := os.Stat(p); !os.IsNotExist(err) {
t.Fatalf("expected %s to be absent, stat err=%v", p, err)
}
}
func diffArchiveNames(t *testing.T, artifactPath string) map[string]bool {
t.Helper()
f, err := os.Open(artifactPath)
if err != nil {
t.Fatalf("open artifact: %v", err)
}
defer f.Close()
names := map[string]bool{}
tr := tar.NewReader(f)
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("read tar: %v", err)
}
names[h.Name] = true
}
return names
}
// TestFileRunnerDifferentialRoundTrip 验证差异备份的端到端正确性:
// 全量 → 修改源(变更/删除/新增)→ 差异 → 链式恢复(全量+差异)→ 结果与修改后源一致。
func TestFileRunnerDifferentialRoundTrip(t *testing.T) {
work := t.TempDir()
src := filepath.Join(work, "src")
diffWrite(t, filepath.Join(src, "a.txt"), "alpha")
diffWrite(t, filepath.Join(src, "b.txt"), "bravo")
diffWrite(t, filepath.Join(src, "sub", "c.txt"), "charlie")
runner := NewFileRunner()
full, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir()}, NopLogWriter{})
if err != nil {
t.Fatalf("full Run: %v", err)
}
if full.Manifest == nil || len(full.Manifest.Entries) == 0 {
t.Fatalf("full backup must produce a manifest, got %#v", full.Manifest)
}
// 变更 a.txt内容变长 → size 差异必被检出)、删除 b.txt、新增 d.txtsub/c.txt 不变
diffWrite(t, filepath.Join(src, "a.txt"), "ALPHA-modified-and-longer")
if err := os.Remove(filepath.Join(src, "b.txt")); err != nil {
t.Fatalf("remove b.txt: %v", err)
}
diffWrite(t, filepath.Join(src, "d.txt"), "delta")
diff, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir(), Differential: true, BaseManifest: *full.Manifest}, NopLogWriter{})
if err != nil {
t.Fatalf("differential Run: %v", err)
}
if diff.Manifest != nil {
t.Fatalf("differential backup must not produce a manifest")
}
// 差异归档应包含变更/新增条目与删除清单,但不含未变更的 sub/c.txt
names := diffArchiveNames(t, diff.ArtifactPath)
if !names["src/a.txt"] || !names["src/d.txt"] {
t.Fatalf("differential archive missing changed/new entries: %v", names)
}
if names["src/sub/c.txt"] {
t.Fatalf("differential archive should not contain unchanged file sub/c.txt")
}
if !names[deletionsEntryName] {
t.Fatalf("differential archive missing deletions entry: %v", names)
}
// 链式恢复到全新目标
restoreRoot := t.TempDir()
restoreSrc := filepath.Join(restoreRoot, "src")
restoreTask := TaskSpec{Name: "diff", Type: "file", SourcePath: restoreSrc}
if err := runner.Restore(context.Background(), restoreTask, full.ArtifactPath, NopLogWriter{}); err != nil {
t.Fatalf("restore full: %v", err)
}
if err := runner.Restore(context.Background(), restoreTask, diff.ArtifactPath, NopLogWriter{}); err != nil {
t.Fatalf("restore differential: %v", err)
}
diffAssertContent(t, filepath.Join(restoreSrc, "a.txt"), "ALPHA-modified-and-longer")
diffAssertContent(t, filepath.Join(restoreSrc, "sub", "c.txt"), "charlie")
diffAssertContent(t, filepath.Join(restoreSrc, "d.txt"), "delta")
diffAssertAbsent(t, filepath.Join(restoreSrc, "b.txt"))
}
// TestFileRunnerDifferentialWithoutBaseIsFull 验证无基线时差异请求回退为全量(产出清单、含全部文件)。
func TestFileRunnerDifferentialWithoutBaseIsFull(t *testing.T) {
src := filepath.Join(t.TempDir(), "src")
diffWrite(t, filepath.Join(src, "a.txt"), "alpha")
runner := NewFileRunner()
res, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir(), Differential: true}, NopLogWriter{})
if err != nil {
t.Fatalf("Run: %v", err)
}
if res.Manifest == nil {
t.Fatalf("differential without base must fall back to full and produce a manifest")
}
if names := diffArchiveNames(t, res.ArtifactPath); !names["src/a.txt"] || names[deletionsEntryName] {
t.Fatalf("fallback-full archive unexpected: %v", names)
}
}

View File

@@ -0,0 +1,92 @@
package backup
import (
"encoding/json"
"os"
"path/filepath"
"sort"
)
// deletionsEntryName 是差异备份归档中记录「自全量以来被删除路径」的特殊条目名。
// 恢复时该条目不落地为文件,而是用于在基线之上删除对应路径。
const deletionsEntryName = ".backupx/deletions.json"
// ManifestEntry 记录一次全量备份中单个归档条目(文件或目录)的指纹,
// 供差异备份比对「自全量以来的变化」。Path 为归档内相对名slash 分隔,
// 与 tar header.Name 一致)。字段使用短键以压缩清单体积。
type ManifestEntry struct {
Path string `json:"p"`
Size int64 `json:"s"`
ModTimeNs int64 `json:"m"`
Mode uint32 `json:"o"`
IsDir bool `json:"d,omitempty"`
}
// Manifest 是一次全量备份的完整条目清单(文件与目录)。
type Manifest struct {
Entries []ManifestEntry `json:"entries"`
}
// EncodeManifest 将清单序列化为紧凑 JSON。
func EncodeManifest(m Manifest) ([]byte, error) {
return json.Marshal(m)
}
// DecodeManifest 反序列化清单;空输入返回空清单(视为「无基线」)。
func DecodeManifest(data []byte) (Manifest, error) {
m := Manifest{}
if len(data) == 0 {
return m, nil
}
if err := json.Unmarshal(data, &m); err != nil {
return Manifest{}, err
}
return m, nil
}
// index 构建 path -> entry 映射,便于差异比对 O(1) 查找。
func (m Manifest) index() map[string]ManifestEntry {
idx := make(map[string]ManifestEntry, len(m.Entries))
for _, e := range m.Entries {
idx[e.Path] = e
}
return idx
}
// entryFromInfo 由归档名与文件信息构造指纹条目。
func entryFromInfo(archiveName string, info os.FileInfo) ManifestEntry {
return ManifestEntry{
Path: filepath.ToSlash(archiveName),
Size: info.Size(),
ModTimeNs: info.ModTime().UnixNano(),
Mode: uint32(info.Mode().Perm()),
IsDir: info.IsDir(),
}
}
// changedSince 判断当前条目相对基线是否为「新增或变更」(即应纳入差异归档)。
// - 不在基线中 → 新增,纳入;
// - 已存在的目录 → 不携带数据,跳过(其下变更文件会各自判定);
// - 文件大小或 mtime 变化 → 变更纳入rsync 风格启发式)。
func changedSince(base map[string]ManifestEntry, cur ManifestEntry) bool {
prev, ok := base[cur.Path]
if !ok {
return true
}
if cur.IsDir {
return false
}
return prev.Size != cur.Size || prev.ModTimeNs != cur.ModTimeNs
}
// deletedPaths 返回基线中存在、但本次遍历未出现的路径(被删除的条目),按路径升序。
func deletedPaths(base map[string]ManifestEntry, seen map[string]struct{}) []string {
deleted := make([]string, 0)
for p := range base {
if _, ok := seen[p]; !ok {
deleted = append(deleted, p)
}
}
sort.Strings(deleted)
return deleted
}

View File

@@ -0,0 +1,79 @@
package backup
import (
"reflect"
"testing"
)
func TestEncodeDecodeManifestRoundTrip(t *testing.T) {
m := Manifest{Entries: []ManifestEntry{
{Path: "src/a.txt", Size: 10, ModTimeNs: 100, Mode: 0o644},
{Path: "src", Size: 0, ModTimeNs: 50, Mode: 0o755, IsDir: true},
}}
data, err := EncodeManifest(m)
if err != nil {
t.Fatalf("EncodeManifest: %v", err)
}
got, err := DecodeManifest(data)
if err != nil {
t.Fatalf("DecodeManifest: %v", err)
}
if !reflect.DeepEqual(got, m) {
t.Fatalf("roundtrip mismatch:\n got %#v\nwant %#v", got, m)
}
}
func TestDecodeManifestEmpty(t *testing.T) {
got, err := DecodeManifest(nil)
if err != nil {
t.Fatalf("DecodeManifest(nil): %v", err)
}
if len(got.Entries) != 0 {
t.Fatalf("expected empty manifest, got %#v", got)
}
}
func TestChangedSince(t *testing.T) {
base := Manifest{Entries: []ManifestEntry{
{Path: "a.txt", Size: 10, ModTimeNs: 100},
{Path: "dir", IsDir: true, ModTimeNs: 100},
}}.index()
cases := []struct {
name string
cur ManifestEntry
want bool
}{
{"unchanged file", ManifestEntry{Path: "a.txt", Size: 10, ModTimeNs: 100}, false},
{"size changed", ManifestEntry{Path: "a.txt", Size: 11, ModTimeNs: 100}, true},
{"mtime changed", ManifestEntry{Path: "a.txt", Size: 10, ModTimeNs: 200}, true},
{"new file", ManifestEntry{Path: "b.txt", Size: 1, ModTimeNs: 1}, true},
{"existing dir skipped", ManifestEntry{Path: "dir", IsDir: true, ModTimeNs: 999}, false},
{"new dir included", ManifestEntry{Path: "newdir", IsDir: true, ModTimeNs: 1}, true},
}
for _, tc := range cases {
if got := changedSince(base, tc.cur); got != tc.want {
t.Errorf("%s: changedSince=%v want %v", tc.name, got, tc.want)
}
}
}
func TestDeletedPaths(t *testing.T) {
base := Manifest{Entries: []ManifestEntry{
{Path: "a"}, {Path: "b"}, {Path: "c"},
}}.index()
seen := map[string]struct{}{"a": {}, "c": {}}
got := deletedPaths(base, seen)
want := []string{"b"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("deletedPaths=%v want %v", got, want)
}
}
func TestDeletedPathsNoneWhenAllSeen(t *testing.T) {
base := Manifest{Entries: []ManifestEntry{{Path: "a"}, {Path: "b"}}}.index()
seen := map[string]struct{}{"a": {}, "b": {}}
if got := deletedPaths(base, seen); len(got) != 0 {
t.Fatalf("expected no deletions, got %v", got)
}
}

View File

@@ -0,0 +1,55 @@
package retention
import (
"testing"
"backupx/server/internal/model"
)
func retentionRecIDs(records []model.BackupRecord) []uint {
ids := make([]uint, 0, len(records))
for _, r := range records {
ids = append(ids, r.ID)
}
return ids
}
// 基线全量仍被「不在删除集合中的差异」依赖 → 必须保留,否则差异无法恢复。
func TestProtectDifferentialBasesKeepsBaseWithSurvivingDiff(t *testing.T) {
all := []model.BackupRecord{
{ID: 1, BackupKind: model.BackupKindFull},
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
}
candidates := []model.BackupRecord{{ID: 1, BackupKind: model.BackupKindFull}}
if got := protectDifferentialBases(all, candidates); len(got) != 0 {
t.Fatalf("base with surviving diff must be protected, got %v", retentionRecIDs(got))
}
}
// 基线全量与其全部差异都在删除集合中 → 可一并删除(无残留差异失去基线)。
func TestProtectDifferentialBasesDeletesBaseWhenDiffAlsoDeleted(t *testing.T) {
all := []model.BackupRecord{
{ID: 1, BackupKind: model.BackupKindFull},
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
}
candidates := []model.BackupRecord{
{ID: 1, BackupKind: model.BackupKindFull},
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
}
if got := protectDifferentialBases(all, candidates); len(got) != 2 {
t.Fatalf("base+diff both expired should both be deleted, got %v", retentionRecIDs(got))
}
}
// 无差异备份时原样透传(不影响既有全量保留逻辑)。
func TestProtectDifferentialBasesNoDiffsPassThrough(t *testing.T) {
all := []model.BackupRecord{
{ID: 1, BackupKind: model.BackupKindFull},
{ID: 2, BackupKind: model.BackupKindFull},
}
candidates := []model.BackupRecord{{ID: 1, BackupKind: model.BackupKindFull}}
got := protectDifferentialBases(all, candidates)
if len(got) != 1 || got[0].ID != 1 {
t.Fatalf("no diffs should pass through unchanged, got %v", retentionRecIDs(got))
}
}

View File

@@ -64,6 +64,8 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
} else {
candidates = selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now())
}
// 差异链保护:保留仍被存活差异依赖的全量,避免删除基线后差异无法恢复。
candidates = protectDifferentialBases(records, candidates)
result := &CleanupResult{}
for _, record := range candidates {
if strings.TrimSpace(record.StoragePath) != "" {
@@ -97,6 +99,38 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
return result, nil
}
// protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」,
// 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。
func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord {
deleting := make(map[uint]struct{}, len(candidates))
for _, r := range candidates {
deleting[r.ID] = struct{}{}
}
protected := make(map[uint]struct{})
for _, r := range all {
if r.BackupKind != model.BackupKindDifferential || r.BaseRecordID == 0 {
continue
}
if _, beingDeleted := deleting[r.ID]; beingDeleted {
continue // 该差异本身也将被删除,无需保护其基线
}
protected[r.BaseRecordID] = struct{}{}
}
if len(protected) == 0 {
return candidates
}
filtered := make([]model.BackupRecord, 0, len(candidates))
for _, r := range candidates {
if r.BackupKind == model.BackupKindFull {
if _, keep := protected[r.ID]; keep {
continue
}
}
filtered = append(filtered, r)
}
return filtered
}
func selectRecordsToDelete(records []model.BackupRecord, retentionDays int, maxBackups int, now time.Time) []model.BackupRecord {
// 保留锁定(法律保留)的记录永不参与清理:先从候选集中剔除,
// 锁定备份既不被删除,也不占用 maxBackups 轮转名额。

View File

@@ -36,6 +36,10 @@ type TaskSpec struct {
MaxBackups int
StartedAt time.Time
TempDir string
// Differential 为 true 时执行差异备份:仅打包自 BaseManifest 以来新增/变更的条目,
// 并记录被删除的路径。仅文件类型任务支持BaseManifest 为空时回退为全量。
Differential bool
BaseManifest Manifest
}
type RunResult struct {
@@ -44,6 +48,8 @@ type RunResult struct {
TempDir string
Size int64
StorageKey string
// Manifest 为全量备份产出的条目清单,供后续差异备份比对;差异备份运行时为 nil。
Manifest *Manifest
}
type LogEvent struct {
@@ -62,7 +68,7 @@ type ProgressInfo struct {
BytesSent int64 `json:"bytesSent"`
TotalBytes int64 `json:"totalBytes"`
Percent float64 `json:"percent"`
SpeedBps float64 `json:"speedBps"` // bytes/sec
SpeedBps float64 `json:"speedBps"` // bytes/sec
TargetName string `json:"targetName"`
}