mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-06-12 21:29:35 +08:00
feat(backup): 新增差异备份(differential)模式 (#88)
文件备份新增差异模式:仅打包自上次全量以来的变更并记录删除,恢复自动按全量+差异链还原。含基线解析、链式恢复、保留链保护与本机文件任务校验;清单/比对/删除/往返/保留保护单测全覆盖。
This commit is contained in:
@@ -3,6 +3,7 @@ package backup
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -52,6 +53,20 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
|
||||
defer tw.Close()
|
||||
|
||||
excludes := normalizeExcludePatterns(task.ExcludePatterns)
|
||||
|
||||
// 差异备份:基于上次全量清单仅打包新增/变更条目并记录删除;
|
||||
// 全量备份:记录完整清单(manifest)供后续差异比对。
|
||||
differential := task.Differential && len(task.BaseManifest.Entries) > 0
|
||||
baseIndex := map[string]ManifestEntry{}
|
||||
seen := map[string]struct{}{}
|
||||
var manifest *Manifest
|
||||
if differential {
|
||||
baseIndex = task.BaseManifest.index()
|
||||
writer.WriteLine(fmt.Sprintf("差异备份模式:基线含 %d 个条目", len(baseIndex)))
|
||||
} else {
|
||||
manifest = &Manifest{Entries: make([]ManifestEntry, 0)}
|
||||
}
|
||||
|
||||
totalFileCount := 0
|
||||
totalDirCount := 0
|
||||
|
||||
@@ -88,6 +103,16 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
|
||||
return nil
|
||||
}
|
||||
|
||||
entry := entryFromInfo(archiveName, currentInfo)
|
||||
if differential {
|
||||
seen[entry.Path] = struct{}{}
|
||||
if !changedSince(baseIndex, entry) {
|
||||
return nil // 自全量以来未变更,跳过
|
||||
}
|
||||
} else {
|
||||
manifest.Entries = append(manifest.Entries, entry)
|
||||
}
|
||||
|
||||
if currentInfo.IsDir() {
|
||||
dirCount++
|
||||
writer.WriteLine(fmt.Sprintf("📁 进入目录 %s", archiveName))
|
||||
@@ -130,10 +155,16 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
|
||||
totalDirCount += dirCount
|
||||
}
|
||||
|
||||
if len(sourcePaths) > 1 {
|
||||
if differential {
|
||||
deletions := deletedPaths(baseIndex, seen)
|
||||
if err := writeDeletionsEntry(tw, deletions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writer.WriteLine(fmt.Sprintf("差异备份完成(%d 个目录、%d 个文件变更,删除 %d 项)", totalDirCount, totalFileCount, len(deletions)))
|
||||
} else if len(sourcePaths) > 1 {
|
||||
writer.WriteLine(fmt.Sprintf("全部源路径打包完成(共 %d 个目录,%d 个文件)", totalDirCount, totalFileCount))
|
||||
}
|
||||
return &RunResult{ArtifactPath: artifactPath, FileName: filepath.Base(artifactPath), TempDir: tempDir}, nil
|
||||
return &RunResult{ArtifactPath: artifactPath, FileName: filepath.Base(artifactPath), TempDir: tempDir, Manifest: manifest}, nil
|
||||
}
|
||||
|
||||
func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath string, writer LogWriter) error {
|
||||
@@ -151,6 +182,7 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
|
||||
if err := os.MkdirAll(targetParent, 0o755); err != nil {
|
||||
return fmt.Errorf("create restore parent: %w", err)
|
||||
}
|
||||
var pendingDeletions []string
|
||||
tr := tar.NewReader(artifactFile)
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
@@ -160,13 +192,23 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
|
||||
if err != nil {
|
||||
return fmt.Errorf("read tar entry: %w", err)
|
||||
}
|
||||
// 差异归档的删除清单不落地,留待提取完成后统一应用(避免被同批新增条目误删)。
|
||||
if header.Name == deletionsEntryName {
|
||||
data, readErr := io.ReadAll(tr)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("read deletions entry: %w", readErr)
|
||||
}
|
||||
if jsonErr := json.Unmarshal(data, &pendingDeletions); jsonErr != nil {
|
||||
return fmt.Errorf("parse deletions entry: %w", jsonErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
cleanName := path.Clean(strings.TrimSpace(header.Name))
|
||||
if cleanName == "." || cleanName == "" {
|
||||
continue
|
||||
}
|
||||
targetPath := filepath.Clean(filepath.Join(targetParent, filepath.FromSlash(cleanName)))
|
||||
parentWithSep := filepath.Clean(targetParent) + string(filepath.Separator)
|
||||
if targetPath != filepath.Clean(targetParent) && !strings.HasPrefix(targetPath, parentWithSep) {
|
||||
targetPath, ok := resolveWithinParent(targetParent, cleanName)
|
||||
if !ok {
|
||||
return fmt.Errorf("tar entry escapes restore path")
|
||||
}
|
||||
switch header.Typeflag {
|
||||
@@ -191,10 +233,65 @@ func (r *FileRunner) Restore(_ context.Context, task TaskSpec, artifactPath stri
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := applyDeletions(targetParent, pendingDeletions, writer); err != nil {
|
||||
return err
|
||||
}
|
||||
writer.WriteLine("文件恢复完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveWithinParent 将归档相对名安全解析为 targetParent 下的绝对路径;
|
||||
// 越界(路径穿越)时返回 ok=false。提取与删除共用此校验,杜绝逃逸。
|
||||
func resolveWithinParent(targetParent, name string) (string, bool) {
|
||||
targetPath := filepath.Clean(filepath.Join(targetParent, filepath.FromSlash(name)))
|
||||
cleanParent := filepath.Clean(targetParent)
|
||||
if targetPath == cleanParent {
|
||||
return targetPath, true
|
||||
}
|
||||
if !strings.HasPrefix(targetPath, cleanParent+string(filepath.Separator)) {
|
||||
return "", false
|
||||
}
|
||||
return targetPath, true
|
||||
}
|
||||
|
||||
// writeDeletionsEntry 将差异备份的删除路径列表写入归档特殊条目。
|
||||
func writeDeletionsEntry(tw *tar.Writer, deletions []string) error {
|
||||
payload, err := json.Marshal(deletions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal deletions: %w", err)
|
||||
}
|
||||
header := &tar.Header{Name: deletionsEntryName, Mode: 0o600, Size: int64(len(payload)), Typeflag: tar.TypeReg}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return fmt.Errorf("write deletions header: %w", err)
|
||||
}
|
||||
if _, err := tw.Write(payload); err != nil {
|
||||
return fmt.Errorf("write deletions body: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyDeletions 在基线恢复之上删除差异归档记录的路径(仅差异备份恢复时存在)。
|
||||
// 每个路径经 resolveWithinParent 校验,越界即报错;目标不存在视为已删除。
|
||||
func applyDeletions(targetParent string, deletions []string, writer LogWriter) error {
|
||||
for _, name := range deletions {
|
||||
clean := path.Clean(strings.TrimSpace(name))
|
||||
if clean == "." || clean == "" {
|
||||
continue
|
||||
}
|
||||
targetPath, ok := resolveWithinParent(targetParent, clean)
|
||||
if !ok {
|
||||
return fmt.Errorf("deletion entry escapes restore path")
|
||||
}
|
||||
if err := os.RemoveAll(targetPath); err != nil {
|
||||
return fmt.Errorf("apply deletion %s: %w", clean, err)
|
||||
}
|
||||
}
|
||||
if len(deletions) > 0 {
|
||||
writer.WriteLine(fmt.Sprintf("已应用差异删除 %d 项", len(deletions)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeExcludePatterns(items []string) []string {
|
||||
result := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
|
||||
141
server/internal/backup/file_runner_diff_test.go
Normal file
141
server/internal/backup/file_runner_diff_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func diffWrite(t *testing.T, p, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", filepath.Dir(p), err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
func diffAssertContent(t *testing.T, p, want string) {
|
||||
t.Helper()
|
||||
got, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", p, err)
|
||||
}
|
||||
if string(got) != want {
|
||||
t.Fatalf("%s content = %q, want %q", p, string(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func diffAssertAbsent(t *testing.T, p string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %s to be absent, stat err=%v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
func diffArchiveNames(t *testing.T, artifactPath string) map[string]bool {
|
||||
t.Helper()
|
||||
f, err := os.Open(artifactPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open artifact: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
names := map[string]bool{}
|
||||
tr := tar.NewReader(f)
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read tar: %v", err)
|
||||
}
|
||||
names[h.Name] = true
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// TestFileRunnerDifferentialRoundTrip 验证差异备份的端到端正确性:
|
||||
// 全量 → 修改源(变更/删除/新增)→ 差异 → 链式恢复(全量+差异)→ 结果与修改后源一致。
|
||||
func TestFileRunnerDifferentialRoundTrip(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
src := filepath.Join(work, "src")
|
||||
diffWrite(t, filepath.Join(src, "a.txt"), "alpha")
|
||||
diffWrite(t, filepath.Join(src, "b.txt"), "bravo")
|
||||
diffWrite(t, filepath.Join(src, "sub", "c.txt"), "charlie")
|
||||
|
||||
runner := NewFileRunner()
|
||||
|
||||
full, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir()}, NopLogWriter{})
|
||||
if err != nil {
|
||||
t.Fatalf("full Run: %v", err)
|
||||
}
|
||||
if full.Manifest == nil || len(full.Manifest.Entries) == 0 {
|
||||
t.Fatalf("full backup must produce a manifest, got %#v", full.Manifest)
|
||||
}
|
||||
|
||||
// 变更 a.txt(内容变长 → size 差异必被检出)、删除 b.txt、新增 d.txt;sub/c.txt 不变
|
||||
diffWrite(t, filepath.Join(src, "a.txt"), "ALPHA-modified-and-longer")
|
||||
if err := os.Remove(filepath.Join(src, "b.txt")); err != nil {
|
||||
t.Fatalf("remove b.txt: %v", err)
|
||||
}
|
||||
diffWrite(t, filepath.Join(src, "d.txt"), "delta")
|
||||
|
||||
diff, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir(), Differential: true, BaseManifest: *full.Manifest}, NopLogWriter{})
|
||||
if err != nil {
|
||||
t.Fatalf("differential Run: %v", err)
|
||||
}
|
||||
if diff.Manifest != nil {
|
||||
t.Fatalf("differential backup must not produce a manifest")
|
||||
}
|
||||
|
||||
// 差异归档应包含变更/新增条目与删除清单,但不含未变更的 sub/c.txt
|
||||
names := diffArchiveNames(t, diff.ArtifactPath)
|
||||
if !names["src/a.txt"] || !names["src/d.txt"] {
|
||||
t.Fatalf("differential archive missing changed/new entries: %v", names)
|
||||
}
|
||||
if names["src/sub/c.txt"] {
|
||||
t.Fatalf("differential archive should not contain unchanged file sub/c.txt")
|
||||
}
|
||||
if !names[deletionsEntryName] {
|
||||
t.Fatalf("differential archive missing deletions entry: %v", names)
|
||||
}
|
||||
|
||||
// 链式恢复到全新目标
|
||||
restoreRoot := t.TempDir()
|
||||
restoreSrc := filepath.Join(restoreRoot, "src")
|
||||
restoreTask := TaskSpec{Name: "diff", Type: "file", SourcePath: restoreSrc}
|
||||
if err := runner.Restore(context.Background(), restoreTask, full.ArtifactPath, NopLogWriter{}); err != nil {
|
||||
t.Fatalf("restore full: %v", err)
|
||||
}
|
||||
if err := runner.Restore(context.Background(), restoreTask, diff.ArtifactPath, NopLogWriter{}); err != nil {
|
||||
t.Fatalf("restore differential: %v", err)
|
||||
}
|
||||
|
||||
diffAssertContent(t, filepath.Join(restoreSrc, "a.txt"), "ALPHA-modified-and-longer")
|
||||
diffAssertContent(t, filepath.Join(restoreSrc, "sub", "c.txt"), "charlie")
|
||||
diffAssertContent(t, filepath.Join(restoreSrc, "d.txt"), "delta")
|
||||
diffAssertAbsent(t, filepath.Join(restoreSrc, "b.txt"))
|
||||
}
|
||||
|
||||
// TestFileRunnerDifferentialWithoutBaseIsFull 验证无基线时差异请求回退为全量(产出清单、含全部文件)。
|
||||
func TestFileRunnerDifferentialWithoutBaseIsFull(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "src")
|
||||
diffWrite(t, filepath.Join(src, "a.txt"), "alpha")
|
||||
|
||||
runner := NewFileRunner()
|
||||
res, err := runner.Run(context.Background(), TaskSpec{Name: "diff", Type: "file", SourcePath: src, TempDir: t.TempDir(), Differential: true}, NopLogWriter{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if res.Manifest == nil {
|
||||
t.Fatalf("differential without base must fall back to full and produce a manifest")
|
||||
}
|
||||
if names := diffArchiveNames(t, res.ArtifactPath); !names["src/a.txt"] || names[deletionsEntryName] {
|
||||
t.Fatalf("fallback-full archive unexpected: %v", names)
|
||||
}
|
||||
}
|
||||
92
server/internal/backup/manifest.go
Normal file
92
server/internal/backup/manifest.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// deletionsEntryName 是差异备份归档中记录「自全量以来被删除路径」的特殊条目名。
|
||||
// 恢复时该条目不落地为文件,而是用于在基线之上删除对应路径。
|
||||
const deletionsEntryName = ".backupx/deletions.json"
|
||||
|
||||
// ManifestEntry 记录一次全量备份中单个归档条目(文件或目录)的指纹,
|
||||
// 供差异备份比对「自全量以来的变化」。Path 为归档内相对名(slash 分隔,
|
||||
// 与 tar header.Name 一致)。字段使用短键以压缩清单体积。
|
||||
type ManifestEntry struct {
|
||||
Path string `json:"p"`
|
||||
Size int64 `json:"s"`
|
||||
ModTimeNs int64 `json:"m"`
|
||||
Mode uint32 `json:"o"`
|
||||
IsDir bool `json:"d,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest 是一次全量备份的完整条目清单(文件与目录)。
|
||||
type Manifest struct {
|
||||
Entries []ManifestEntry `json:"entries"`
|
||||
}
|
||||
|
||||
// EncodeManifest 将清单序列化为紧凑 JSON。
|
||||
func EncodeManifest(m Manifest) ([]byte, error) {
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
// DecodeManifest 反序列化清单;空输入返回空清单(视为「无基线」)。
|
||||
func DecodeManifest(data []byte) (Manifest, error) {
|
||||
m := Manifest{}
|
||||
if len(data) == 0 {
|
||||
return m, nil
|
||||
}
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// index 构建 path -> entry 映射,便于差异比对 O(1) 查找。
|
||||
func (m Manifest) index() map[string]ManifestEntry {
|
||||
idx := make(map[string]ManifestEntry, len(m.Entries))
|
||||
for _, e := range m.Entries {
|
||||
idx[e.Path] = e
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// entryFromInfo 由归档名与文件信息构造指纹条目。
|
||||
func entryFromInfo(archiveName string, info os.FileInfo) ManifestEntry {
|
||||
return ManifestEntry{
|
||||
Path: filepath.ToSlash(archiveName),
|
||||
Size: info.Size(),
|
||||
ModTimeNs: info.ModTime().UnixNano(),
|
||||
Mode: uint32(info.Mode().Perm()),
|
||||
IsDir: info.IsDir(),
|
||||
}
|
||||
}
|
||||
|
||||
// changedSince 判断当前条目相对基线是否为「新增或变更」(即应纳入差异归档)。
|
||||
// - 不在基线中 → 新增,纳入;
|
||||
// - 已存在的目录 → 不携带数据,跳过(其下变更文件会各自判定);
|
||||
// - 文件大小或 mtime 变化 → 变更,纳入(rsync 风格启发式)。
|
||||
func changedSince(base map[string]ManifestEntry, cur ManifestEntry) bool {
|
||||
prev, ok := base[cur.Path]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if cur.IsDir {
|
||||
return false
|
||||
}
|
||||
return prev.Size != cur.Size || prev.ModTimeNs != cur.ModTimeNs
|
||||
}
|
||||
|
||||
// deletedPaths 返回基线中存在、但本次遍历未出现的路径(被删除的条目),按路径升序。
|
||||
func deletedPaths(base map[string]ManifestEntry, seen map[string]struct{}) []string {
|
||||
deleted := make([]string, 0)
|
||||
for p := range base {
|
||||
if _, ok := seen[p]; !ok {
|
||||
deleted = append(deleted, p)
|
||||
}
|
||||
}
|
||||
sort.Strings(deleted)
|
||||
return deleted
|
||||
}
|
||||
79
server/internal/backup/manifest_test.go
Normal file
79
server/internal/backup/manifest_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEncodeDecodeManifestRoundTrip(t *testing.T) {
|
||||
m := Manifest{Entries: []ManifestEntry{
|
||||
{Path: "src/a.txt", Size: 10, ModTimeNs: 100, Mode: 0o644},
|
||||
{Path: "src", Size: 0, ModTimeNs: 50, Mode: 0o755, IsDir: true},
|
||||
}}
|
||||
data, err := EncodeManifest(m)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeManifest: %v", err)
|
||||
}
|
||||
got, err := DecodeManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeManifest: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, m) {
|
||||
t.Fatalf("roundtrip mismatch:\n got %#v\nwant %#v", got, m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeManifestEmpty(t *testing.T) {
|
||||
got, err := DecodeManifest(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeManifest(nil): %v", err)
|
||||
}
|
||||
if len(got.Entries) != 0 {
|
||||
t.Fatalf("expected empty manifest, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedSince(t *testing.T) {
|
||||
base := Manifest{Entries: []ManifestEntry{
|
||||
{Path: "a.txt", Size: 10, ModTimeNs: 100},
|
||||
{Path: "dir", IsDir: true, ModTimeNs: 100},
|
||||
}}.index()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cur ManifestEntry
|
||||
want bool
|
||||
}{
|
||||
{"unchanged file", ManifestEntry{Path: "a.txt", Size: 10, ModTimeNs: 100}, false},
|
||||
{"size changed", ManifestEntry{Path: "a.txt", Size: 11, ModTimeNs: 100}, true},
|
||||
{"mtime changed", ManifestEntry{Path: "a.txt", Size: 10, ModTimeNs: 200}, true},
|
||||
{"new file", ManifestEntry{Path: "b.txt", Size: 1, ModTimeNs: 1}, true},
|
||||
{"existing dir skipped", ManifestEntry{Path: "dir", IsDir: true, ModTimeNs: 999}, false},
|
||||
{"new dir included", ManifestEntry{Path: "newdir", IsDir: true, ModTimeNs: 1}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := changedSince(base, tc.cur); got != tc.want {
|
||||
t.Errorf("%s: changedSince=%v want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletedPaths(t *testing.T) {
|
||||
base := Manifest{Entries: []ManifestEntry{
|
||||
{Path: "a"}, {Path: "b"}, {Path: "c"},
|
||||
}}.index()
|
||||
seen := map[string]struct{}{"a": {}, "c": {}}
|
||||
got := deletedPaths(base, seen)
|
||||
want := []string{"b"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("deletedPaths=%v want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletedPathsNoneWhenAllSeen(t *testing.T) {
|
||||
base := Manifest{Entries: []ManifestEntry{{Path: "a"}, {Path: "b"}}}.index()
|
||||
seen := map[string]struct{}{"a": {}, "b": {}}
|
||||
if got := deletedPaths(base, seen); len(got) != 0 {
|
||||
t.Fatalf("expected no deletions, got %v", got)
|
||||
}
|
||||
}
|
||||
55
server/internal/backup/retention/differential_test.go
Normal file
55
server/internal/backup/retention/differential_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"backupx/server/internal/model"
|
||||
)
|
||||
|
||||
func retentionRecIDs(records []model.BackupRecord) []uint {
|
||||
ids := make([]uint, 0, len(records))
|
||||
for _, r := range records {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// 基线全量仍被「不在删除集合中的差异」依赖 → 必须保留,否则差异无法恢复。
|
||||
func TestProtectDifferentialBasesKeepsBaseWithSurvivingDiff(t *testing.T) {
|
||||
all := []model.BackupRecord{
|
||||
{ID: 1, BackupKind: model.BackupKindFull},
|
||||
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
|
||||
}
|
||||
candidates := []model.BackupRecord{{ID: 1, BackupKind: model.BackupKindFull}}
|
||||
if got := protectDifferentialBases(all, candidates); len(got) != 0 {
|
||||
t.Fatalf("base with surviving diff must be protected, got %v", retentionRecIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
// 基线全量与其全部差异都在删除集合中 → 可一并删除(无残留差异失去基线)。
|
||||
func TestProtectDifferentialBasesDeletesBaseWhenDiffAlsoDeleted(t *testing.T) {
|
||||
all := []model.BackupRecord{
|
||||
{ID: 1, BackupKind: model.BackupKindFull},
|
||||
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
|
||||
}
|
||||
candidates := []model.BackupRecord{
|
||||
{ID: 1, BackupKind: model.BackupKindFull},
|
||||
{ID: 2, BackupKind: model.BackupKindDifferential, BaseRecordID: 1},
|
||||
}
|
||||
if got := protectDifferentialBases(all, candidates); len(got) != 2 {
|
||||
t.Fatalf("base+diff both expired should both be deleted, got %v", retentionRecIDs(got))
|
||||
}
|
||||
}
|
||||
|
||||
// 无差异备份时原样透传(不影响既有全量保留逻辑)。
|
||||
func TestProtectDifferentialBasesNoDiffsPassThrough(t *testing.T) {
|
||||
all := []model.BackupRecord{
|
||||
{ID: 1, BackupKind: model.BackupKindFull},
|
||||
{ID: 2, BackupKind: model.BackupKindFull},
|
||||
}
|
||||
candidates := []model.BackupRecord{{ID: 1, BackupKind: model.BackupKindFull}}
|
||||
got := protectDifferentialBases(all, candidates)
|
||||
if len(got) != 1 || got[0].ID != 1 {
|
||||
t.Fatalf("no diffs should pass through unchanged, got %v", retentionRecIDs(got))
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,8 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
|
||||
} else {
|
||||
candidates = selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now())
|
||||
}
|
||||
// 差异链保护:保留仍被存活差异依赖的全量,避免删除基线后差异无法恢复。
|
||||
candidates = protectDifferentialBases(records, candidates)
|
||||
result := &CleanupResult{}
|
||||
for _, record := range candidates {
|
||||
if strings.TrimSpace(record.StoragePath) != "" {
|
||||
@@ -97,6 +99,38 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」,
|
||||
// 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。
|
||||
func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord {
|
||||
deleting := make(map[uint]struct{}, len(candidates))
|
||||
for _, r := range candidates {
|
||||
deleting[r.ID] = struct{}{}
|
||||
}
|
||||
protected := make(map[uint]struct{})
|
||||
for _, r := range all {
|
||||
if r.BackupKind != model.BackupKindDifferential || r.BaseRecordID == 0 {
|
||||
continue
|
||||
}
|
||||
if _, beingDeleted := deleting[r.ID]; beingDeleted {
|
||||
continue // 该差异本身也将被删除,无需保护其基线
|
||||
}
|
||||
protected[r.BaseRecordID] = struct{}{}
|
||||
}
|
||||
if len(protected) == 0 {
|
||||
return candidates
|
||||
}
|
||||
filtered := make([]model.BackupRecord, 0, len(candidates))
|
||||
for _, r := range candidates {
|
||||
if r.BackupKind == model.BackupKindFull {
|
||||
if _, keep := protected[r.ID]; keep {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, r)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func selectRecordsToDelete(records []model.BackupRecord, retentionDays int, maxBackups int, now time.Time) []model.BackupRecord {
|
||||
// 保留锁定(法律保留)的记录永不参与清理:先从候选集中剔除,
|
||||
// 锁定备份既不被删除,也不占用 maxBackups 轮转名额。
|
||||
|
||||
@@ -36,6 +36,10 @@ type TaskSpec struct {
|
||||
MaxBackups int
|
||||
StartedAt time.Time
|
||||
TempDir string
|
||||
// Differential 为 true 时执行差异备份:仅打包自 BaseManifest 以来新增/变更的条目,
|
||||
// 并记录被删除的路径。仅文件类型任务支持;BaseManifest 为空时回退为全量。
|
||||
Differential bool
|
||||
BaseManifest Manifest
|
||||
}
|
||||
|
||||
type RunResult struct {
|
||||
@@ -44,6 +48,8 @@ type RunResult struct {
|
||||
TempDir string
|
||||
Size int64
|
||||
StorageKey string
|
||||
// Manifest 为全量备份产出的条目清单,供后续差异备份比对;差异备份运行时为 nil。
|
||||
Manifest *Manifest
|
||||
}
|
||||
|
||||
type LogEvent struct {
|
||||
@@ -62,7 +68,7 @@ type ProgressInfo struct {
|
||||
BytesSent int64 `json:"bytesSent"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
Percent float64 `json:"percent"`
|
||||
SpeedBps float64 `json:"speedBps"` // bytes/sec
|
||||
SpeedBps float64 `json:"speedBps"` // bytes/sec
|
||||
TargetName string `json:"targetName"`
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,12 @@ const (
|
||||
BackupRecordStatusFailed = "failed"
|
||||
)
|
||||
|
||||
const (
|
||||
// BackupKindFull 全量备份;BackupKindDifferential 差异备份(仅含自基线全量以来的变更)。
|
||||
BackupKindFull = "full"
|
||||
BackupKindDifferential = "differential"
|
||||
)
|
||||
|
||||
type BackupRecord struct {
|
||||
ID uint `gorm:"primaryKey" json:"id"`
|
||||
TaskID uint `gorm:"column:task_id;index;not null" json:"taskId"`
|
||||
@@ -26,7 +32,13 @@ type BackupRecord struct {
|
||||
DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"`
|
||||
// Locked 保留锁定(法律保留):为 true 时该备份不参与保留期/数量自动清理,
|
||||
// 且禁止手动删除,直到显式解锁。用于保护合规快照、迁移前基线等关键备份。
|
||||
Locked bool `gorm:"column:locked;not null;default:false;index" json:"locked"`
|
||||
Locked bool `gorm:"column:locked;not null;default:false;index" json:"locked"`
|
||||
// BackupKind 备份类型:full(全量)/ differential(差异)。
|
||||
BackupKind string `gorm:"column:backup_kind;size:16;not null;default:'full';index" json:"backupKind"`
|
||||
// BaseRecordID 差异备份所基于的全量备份记录 ID(全量记录为 0)。
|
||||
BaseRecordID uint `gorm:"column:base_record_id;index;not null;default:0" json:"baseRecordId"`
|
||||
// Manifest 全量备份的条目清单(JSON),供后续差异备份比对;差异记录为空。
|
||||
Manifest string `gorm:"column:manifest;type:text" json:"-"`
|
||||
ErrorMessage string `gorm:"column:error_message;size:2000" json:"errorMessage"`
|
||||
LogContent string `gorm:"column:log_content;type:text" json:"logContent"`
|
||||
StartedAt time.Time `gorm:"column:started_at;index;not null" json:"startedAt"`
|
||||
|
||||
@@ -11,6 +11,12 @@ const (
|
||||
BackupTaskTypeMongoDB = "mongodb"
|
||||
)
|
||||
|
||||
const (
|
||||
// BackupModeFull 全量模式(默认);BackupModeDifferential 差异模式(仅文件类型本机任务)。
|
||||
BackupModeFull = "full"
|
||||
BackupModeDifferential = "differential"
|
||||
)
|
||||
|
||||
const (
|
||||
BackupTaskStatusIdle = "idle"
|
||||
BackupTaskStatusRunning = "running"
|
||||
@@ -49,6 +55,11 @@ type BackupTask struct {
|
||||
Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"`
|
||||
Encrypt bool `gorm:"not null;default:false" json:"encrypt"`
|
||||
MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"`
|
||||
// BackupMode 备份模式:full(全量,默认)/ differential(差异)。差异仅支持本机文件任务。
|
||||
BackupMode string `gorm:"column:backup_mode;size:16;not null;default:'full'" json:"backupMode"`
|
||||
// DiffFullIntervalDays 差异模式下强制全量的间隔(天):最近全量超过该天数则本次自动改为全量,
|
||||
// 限制差异链跨度与单个差异体积。默认 7。
|
||||
DiffFullIntervalDays int `gorm:"column:diff_full_interval_days;not null;default:7" json:"diffFullIntervalDays"`
|
||||
// GFS(祖父-父-子)保留:分别保留最近 N 天 / M 周 / K 月 / Y 年的代表性备份(每周期保留最新一份)。
|
||||
// 任一 > 0 即启用 GFS,取代 RetentionDays/MaxBackups 简单策略;全为 0 时维持简单策略(向后兼容)。
|
||||
KeepDaily int `gorm:"column:keep_daily;not null;default:0" json:"keepDaily"`
|
||||
|
||||
@@ -580,6 +580,40 @@ func (s *BackupExecutionService) resolveRemoteNode(ctx context.Context, nodeID u
|
||||
return resolveRemoteExecutionNode(ctx, s.nodeRepo, s.agentDispatcher != nil, nodeID)
|
||||
}
|
||||
|
||||
// resolveDifferentialBase 为差异备份解析基线全量:仅本机(NodeID=0)文件任务且 BackupMode=differential 时生效。
|
||||
// 返回最近一次「成功、含清单、未超过 DiffFullIntervalDays」的全量记录 ID 及其清单;
|
||||
// 无合适基线(首次备份 / 最近全量已过期 / 清单缺失)时 ok=false,调用方回退为全量。
|
||||
func (s *BackupExecutionService) resolveDifferentialBase(ctx context.Context, task *model.BackupTask) (uint, backup.Manifest, bool) {
|
||||
if task.Type != model.BackupTaskTypeFile || task.NodeID != 0 || !strings.EqualFold(task.BackupMode, model.BackupModeDifferential) {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
records, err := s.records.ListSuccessfulByTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
intervalDays := task.DiffFullIntervalDays
|
||||
if intervalDays <= 0 {
|
||||
intervalDays = 7
|
||||
}
|
||||
cutoff := time.Now().Add(-time.Duration(intervalDays) * 24 * time.Hour)
|
||||
for i := range records {
|
||||
rec := records[i]
|
||||
if rec.BackupKind != model.BackupKindFull || strings.TrimSpace(rec.Manifest) == "" {
|
||||
continue
|
||||
}
|
||||
// 最近的全量已超过强制全量间隔 → 触发新全量,限制差异链跨度与单个差异体积。
|
||||
if rec.StartedAt.Before(cutoff) {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
manifest, decErr := backup.DecodeManifest([]byte(rec.Manifest))
|
||||
if decErr != nil || len(manifest.Entries) == 0 {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
return rec.ID, manifest, true
|
||||
}
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) {
|
||||
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0,
|
||||
// 该节点上所有任务共享一个节点专属 semaphore,互相排队
|
||||
@@ -604,6 +638,10 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
var storagePath string
|
||||
selectedStorageTargetID := task.StorageTargetID
|
||||
var uploadResults []StorageUploadResultItem
|
||||
// 差异备份链信息:实际类型(全量/差异)、基线全量 ID、全量清单 JSON。
|
||||
backupKind := model.BackupKindFull
|
||||
var baseRecordID uint
|
||||
var manifestJSON string
|
||||
completeRecord := func() {
|
||||
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
|
||||
logger.Errorf("写回备份记录失败:%v", finalizeErr)
|
||||
@@ -619,6 +657,17 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
}
|
||||
}
|
||||
// 持久化差异链信息:全量记录其清单(供后续差异比对),差异记录其基线全量 ID。
|
||||
if status == model.BackupRecordStatusSuccess && (backupKind != model.BackupKindFull || baseRecordID != 0 || manifestJSON != "") {
|
||||
if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil {
|
||||
record.BackupKind = backupKind
|
||||
record.BaseRecordID = baseRecordID
|
||||
record.Manifest = manifestJSON
|
||||
if updErr := s.records.Update(ctx, record); updErr != nil {
|
||||
logger.Warnf("写回差异链信息失败:%v", updErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.shouldNotify(ctx, task, status) {
|
||||
if err := s.notifier.NotifyBackupResult(ctx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
|
||||
logger.Warnf("发送备份通知失败:%v", err)
|
||||
@@ -636,6 +685,13 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
logger.Errorf("构建任务运行时配置失败:%v", err)
|
||||
return
|
||||
}
|
||||
// 差异备份:解析基线全量,命中则切换为差异模式(仅本机文件任务)。
|
||||
if baseID, baseManifest, ok := s.resolveDifferentialBase(ctx, task); ok {
|
||||
spec.Differential = true
|
||||
spec.BaseManifest = baseManifest
|
||||
baseRecordID = baseID
|
||||
logger.Infof("差异备份模式:基于全量备份 #%d 仅打包变更", baseID)
|
||||
}
|
||||
runner, err := s.runnerRegistry.Runner(spec.Type)
|
||||
if err != nil {
|
||||
errMessage = err.Error()
|
||||
@@ -649,6 +705,17 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(result.TempDir)
|
||||
// 依据运行器产出判定实际类型:产出清单 → 全量(记录清单供后续差异比对);否则为差异。
|
||||
if result.Manifest != nil {
|
||||
backupKind = model.BackupKindFull
|
||||
if data, encErr := backup.EncodeManifest(*result.Manifest); encErr == nil {
|
||||
manifestJSON = string(data)
|
||||
} else {
|
||||
logger.Warnf("备份清单序列化失败(不影响本次备份,但将禁用后续差异):%v", encErr)
|
||||
}
|
||||
} else {
|
||||
backupKind = model.BackupKindDifferential
|
||||
}
|
||||
finalPath := result.ArtifactPath
|
||||
if strings.EqualFold(task.Compression, "gzip") && !strings.HasSuffix(strings.ToLower(finalPath), ".gz") {
|
||||
logger.Infof("开始压缩备份文件")
|
||||
|
||||
@@ -37,6 +37,7 @@ type BackupRecordSummary struct {
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
Locked bool `json:"locked"`
|
||||
BackupKind string `json:"backupKind"`
|
||||
}
|
||||
|
||||
type BackupRecordDetail struct {
|
||||
@@ -139,6 +140,7 @@ func toBackupRecordSummary(item *model.BackupRecord) BackupRecordSummary {
|
||||
StartedAt: item.StartedAt,
|
||||
CompletedAt: item.CompletedAt,
|
||||
Locked: item.Locked,
|
||||
BackupKind: item.BackupKind,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,9 @@ type BackupTaskUpsertInput struct {
|
||||
KeepWeekly int `json:"keepWeekly"`
|
||||
KeepMonthly int `json:"keepMonthly"`
|
||||
KeepYearly int `json:"keepYearly"`
|
||||
// BackupMode 备份模式:full(默认)/ differential(差异,仅文件类型本机任务)
|
||||
BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential"`
|
||||
DiffFullIntervalDays int `json:"diffFullIntervalDays"`
|
||||
// 备份复制目标存储 ID 列表(3-2-1 规则)
|
||||
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
|
||||
// 维护窗口(CSV,详见 backup/window.go)
|
||||
@@ -99,6 +102,8 @@ type BackupTaskSummary struct {
|
||||
KeepWeekly int `json:"keepWeekly"`
|
||||
KeepMonthly int `json:"keepMonthly"`
|
||||
KeepYearly int `json:"keepYearly"`
|
||||
BackupMode string `json:"backupMode"`
|
||||
DiffFullIntervalDays int `json:"diffFullIntervalDays"`
|
||||
// 备份复制目标(3-2-1)
|
||||
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
|
||||
MaintenanceWindows string `json:"maintenanceWindows"`
|
||||
@@ -517,6 +522,14 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B
|
||||
return apperror.BadRequest("BACKUP_TASK_REMOTE_ENCRYPT_UNSUPPORTED",
|
||||
"远程节点暂不支持加密备份。请关闭加密,或将任务固定在 Master 本机执行。", nil)
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(input.BackupMode), model.BackupModeDifferential) {
|
||||
if input.Type != model.BackupTaskTypeFile {
|
||||
return apperror.BadRequest("BACKUP_TASK_DIFF_UNSUPPORTED", "差异备份仅支持文件目录类型任务", nil)
|
||||
}
|
||||
if strings.TrimSpace(input.NodePoolTag) != "" || (fixedNode != nil && !fixedNode.IsLocal) {
|
||||
return apperror.BadRequest("BACKUP_TASK_DIFF_REMOTE_UNSUPPORTED", "差异备份当前仅支持本机 Master 执行,请将任务固定在本机或改用全量备份。", nil)
|
||||
}
|
||||
}
|
||||
if input.RetentionDays < 0 {
|
||||
return apperror.BadRequest("BACKUP_TASK_INVALID", "保留天数不能小于 0", nil)
|
||||
}
|
||||
@@ -687,6 +700,8 @@ func (s *BackupTaskService) buildTask(existing *model.BackupTask, input BackupTa
|
||||
KeepWeekly: maxInt(0, input.KeepWeekly),
|
||||
KeepMonthly: maxInt(0, input.KeepMonthly),
|
||||
KeepYearly: maxInt(0, input.KeepYearly),
|
||||
BackupMode: normalizeBackupMode(input.BackupMode, input.Type),
|
||||
DiffFullIntervalDays: diffFullInterval(input.DiffFullIntervalDays),
|
||||
ReplicationTargetIDs: encodeUintCSV(input.ReplicationTargetIDs),
|
||||
MaintenanceWindows: strings.TrimSpace(input.MaintenanceWindows),
|
||||
DependsOnTaskIDs: encodeUintCSV(input.DependsOnTaskIDs),
|
||||
@@ -783,6 +798,8 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary {
|
||||
KeepWeekly: item.KeepWeekly,
|
||||
KeepMonthly: item.KeepMonthly,
|
||||
KeepYearly: item.KeepYearly,
|
||||
BackupMode: item.BackupMode,
|
||||
DiffFullIntervalDays: item.DiffFullIntervalDays,
|
||||
ReplicationTargetIDs: parseUintCSV(item.ReplicationTargetIDs),
|
||||
MaintenanceWindows: item.MaintenanceWindows,
|
||||
DependsOnTaskIDs: parseUintCSV(item.DependsOnTaskIDs),
|
||||
@@ -918,6 +935,22 @@ func decodeExtraConfig(value string) (map[string]any, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeBackupMode 归一化备份模式:仅文件类型可启用差异,其余一律全量(双保险,防绕过校验)。
|
||||
func normalizeBackupMode(mode, taskType string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(mode), model.BackupModeDifferential) && normalizeBackupTaskType(taskType) == model.BackupTaskTypeFile {
|
||||
return model.BackupModeDifferential
|
||||
}
|
||||
return model.BackupModeFull
|
||||
}
|
||||
|
||||
// diffFullInterval 归一化差异模式下的强制全量间隔(天),非正值回退默认 7。
|
||||
func diffFullInterval(days int) int {
|
||||
if days <= 0 {
|
||||
return 7
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
func normalizeBackupTaskType(value string) string {
|
||||
normalized := strings.TrimSpace(strings.ToLower(value))
|
||||
if normalized == "pgsql" {
|
||||
|
||||
@@ -223,10 +223,17 @@ func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, tas
|
||||
}()
|
||||
|
||||
logger.Infof("开始在本地执行恢复(备份记录 #%d)", backupRecord.ID)
|
||||
provider, providerErr := s.resolveProvider(ctx, backupRecord.StorageTargetID)
|
||||
if providerErr != nil {
|
||||
errMessage = providerErr.Error()
|
||||
logger.Errorf("创建存储客户端失败:%v", providerErr)
|
||||
|
||||
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
|
||||
if specErr != nil {
|
||||
errMessage = specErr.Error()
|
||||
logger.Errorf("构建恢复规格失败:%v", specErr)
|
||||
return
|
||||
}
|
||||
runner, runnerErr := s.runnerRegistry.Runner(spec.Type)
|
||||
if runnerErr != nil {
|
||||
errMessage = runnerErr.Error()
|
||||
logger.Errorf("不支持的备份类型:%v", runnerErr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -243,63 +250,89 @@ func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, tas
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
fileName := backupRecord.FileName
|
||||
if strings.TrimSpace(fileName) == "" {
|
||||
fileName = filepath.Base(backupRecord.StoragePath)
|
||||
}
|
||||
artifactPath := filepath.Join(tempDir, filepath.Base(fileName))
|
||||
logger.Infof("开始下载备份文件:%s", backupRecord.StoragePath)
|
||||
reader, downloadErr := provider.Download(ctx, backupRecord.StoragePath)
|
||||
if downloadErr != nil {
|
||||
errMessage = downloadErr.Error()
|
||||
logger.Errorf("下载备份文件失败:%v", downloadErr)
|
||||
// 恢复链:全量 → [自身];差异 → [基线全量, 自身],按序应用(全量铺底,差异覆盖并删除)。
|
||||
chain, chainErr := s.buildRestoreChain(ctx, backupRecord)
|
||||
if chainErr != nil {
|
||||
errMessage = chainErr.Error()
|
||||
logger.Errorf("%v", chainErr)
|
||||
return
|
||||
}
|
||||
if writeErr := writeReaderToFile(artifactPath, reader); writeErr != nil {
|
||||
errMessage = writeErr.Error()
|
||||
logger.Errorf("写入恢复文件失败:%v", writeErr)
|
||||
return
|
||||
}
|
||||
// 完整性校验:在解密/解压前比对下载对象的 SHA-256,拒绝还原损坏或被篡改的备份。
|
||||
// 早期未记录 checksum 的备份会跳过(向后兼容)。
|
||||
if backupRecord.Checksum != "" {
|
||||
logger.Infof("校验备份完整性(SHA-256)")
|
||||
if csErr := verifyArtifactChecksum(artifactPath, backupRecord.Checksum); csErr != nil {
|
||||
errMessage = csErr.Error()
|
||||
logger.Errorf("完整性校验失败:%v", csErr)
|
||||
logger.Infof("开始执行 %s 恢复(恢复链含 %d 个备份)", spec.Type, len(chain))
|
||||
for idx := range chain {
|
||||
rec := chain[idx]
|
||||
if len(chain) > 1 {
|
||||
logger.Infof("恢复链 [%d/%d]:应用备份记录 #%d(%s)", idx+1, len(chain), rec.ID, backupKindLabel(rec.BackupKind))
|
||||
}
|
||||
if err := s.restoreArtifact(ctx, &rec, spec, runner, tempDir, logger); err != nil {
|
||||
errMessage = err.Error()
|
||||
logger.Errorf("恢复执行失败:%v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof("完整性校验通过")
|
||||
}
|
||||
preparedPath, prepareErr := s.prepareArtifact(artifactPath, logger)
|
||||
if prepareErr != nil {
|
||||
errMessage = prepareErr.Error()
|
||||
logger.Errorf("准备恢复文件失败:%v", prepareErr)
|
||||
return
|
||||
}
|
||||
|
||||
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
|
||||
if specErr != nil {
|
||||
errMessage = specErr.Error()
|
||||
logger.Errorf("构建恢复规格失败:%v", specErr)
|
||||
return
|
||||
}
|
||||
runner, runnerErr := s.runnerRegistry.Runner(spec.Type)
|
||||
if runnerErr != nil {
|
||||
errMessage = runnerErr.Error()
|
||||
logger.Errorf("不支持的备份类型:%v", runnerErr)
|
||||
return
|
||||
}
|
||||
logger.Infof("开始执行 %s 恢复", spec.Type)
|
||||
if restoreErr := runner.Restore(ctx, spec, preparedPath, logger); restoreErr != nil {
|
||||
errMessage = restoreErr.Error()
|
||||
logger.Errorf("恢复执行失败:%v", restoreErr)
|
||||
return
|
||||
}
|
||||
status = model.RestoreRecordStatusSuccess
|
||||
logger.Infof("恢复执行成功")
|
||||
}
|
||||
|
||||
// restoreArtifact 下载、完整性校验、解密解压并通过 runner 应用单个备份记录的归档。
|
||||
// 每个记录使用独立子目录,避免恢复链中基线/差异的同名归档相互覆盖。
|
||||
func (s *RestoreService) restoreArtifact(ctx context.Context, record *model.BackupRecord, spec backup.TaskSpec, runner backup.BackupRunner, parentTempDir string, logger *backup.ExecutionLogger) error {
|
||||
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建存储客户端失败:%w", err)
|
||||
}
|
||||
recDir, err := os.MkdirTemp(parentTempDir, fmt.Sprintf("rec-%d-*", record.ID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建恢复子目录失败:%w", err)
|
||||
}
|
||||
fileName := record.FileName
|
||||
if strings.TrimSpace(fileName) == "" {
|
||||
fileName = filepath.Base(record.StoragePath)
|
||||
}
|
||||
artifactPath := filepath.Join(recDir, filepath.Base(fileName))
|
||||
logger.Infof("开始下载备份文件:%s", record.StoragePath)
|
||||
reader, err := provider.Download(ctx, record.StoragePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("下载备份文件失败:%w", err)
|
||||
}
|
||||
if err := writeReaderToFile(artifactPath, reader); err != nil {
|
||||
return fmt.Errorf("写入恢复文件失败:%w", err)
|
||||
}
|
||||
// 完整性校验:解密/解压前比对 SHA-256;早期无 checksum 的备份跳过(向后兼容)。
|
||||
if record.Checksum != "" {
|
||||
if err := verifyArtifactChecksum(artifactPath, record.Checksum); err != nil {
|
||||
return fmt.Errorf("完整性校验失败:%w", err)
|
||||
}
|
||||
}
|
||||
preparedPath, err := s.prepareArtifact(artifactPath, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("准备恢复文件失败:%w", err)
|
||||
}
|
||||
return runner.Restore(ctx, spec, preparedPath, logger)
|
||||
}
|
||||
|
||||
// buildRestoreChain 返回恢复某记录所需、按应用顺序排列的记录链:
|
||||
// 全量 → [自身];差异 → [基线全量, 自身]。基线缺失/不可用时报错,杜绝残缺恢复。
|
||||
func (s *RestoreService) buildRestoreChain(ctx context.Context, record *model.BackupRecord) ([]model.BackupRecord, error) {
|
||||
if record.BackupKind != model.BackupKindDifferential || record.BaseRecordID == 0 {
|
||||
return []model.BackupRecord{*record}, nil
|
||||
}
|
||||
base, err := s.records.FindByID(ctx, record.BaseRecordID)
|
||||
if err != nil || base == nil {
|
||||
return nil, fmt.Errorf("差异备份的基线全量 #%d 不存在,无法恢复", record.BaseRecordID)
|
||||
}
|
||||
if base.Status != model.BackupRecordStatusSuccess || strings.TrimSpace(base.StoragePath) == "" {
|
||||
return nil, fmt.Errorf("差异备份的基线全量 #%d 不可用,无法恢复", record.BaseRecordID)
|
||||
}
|
||||
return []model.BackupRecord{*base, *record}, nil
|
||||
}
|
||||
|
||||
func backupKindLabel(kind string) string {
|
||||
if kind == model.BackupKindDifferential {
|
||||
return "差异"
|
||||
}
|
||||
return "全量"
|
||||
}
|
||||
|
||||
// dispatchRestoreEvent 按终态向事件总线派发 restore_success 或 restore_failed。
|
||||
// eventDispatcher 未注入时静默忽略,保持向后兼容。
|
||||
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { useEffect, useMemo, useState } from 'react'
|
||||
import { CronInput } from '../CronInput'
|
||||
import type { StorageTargetDetail, StorageTargetPayload, StorageTargetSummary } from '../../types/storage-targets'
|
||||
import type { StorageConnectionTestResult } from '../../types/storage-targets'
|
||||
import type { BackupTaskDetail, BackupTaskPayload, BackupTaskType } from '../../types/backup-tasks'
|
||||
import type { BackupMode, BackupTaskDetail, BackupTaskPayload, BackupTaskType } from '../../types/backup-tasks'
|
||||
import type { NodeSummary } from '../../types/nodes'
|
||||
import { DatabasePicker } from '../common/DatabasePicker'
|
||||
import { DirectoryPicker } from '../common/DirectoryPicker'
|
||||
@@ -69,6 +69,8 @@ function createEmptyDraft(storageTargets?: StorageTargetSummary[]): BackupTaskPa
|
||||
keepWeekly: 0,
|
||||
keepMonthly: 0,
|
||||
keepYearly: 0,
|
||||
backupMode: 'full',
|
||||
diffFullIntervalDays: 7,
|
||||
extraConfig: undefined,
|
||||
verifyEnabled: false,
|
||||
verifyCronExpr: '',
|
||||
@@ -142,6 +144,8 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa
|
||||
keepWeekly: initialValue.keepWeekly ?? 0,
|
||||
keepMonthly: initialValue.keepMonthly ?? 0,
|
||||
keepYearly: initialValue.keepYearly ?? 0,
|
||||
backupMode: initialValue.backupMode ?? 'full',
|
||||
diffFullIntervalDays: initialValue.diffFullIntervalDays ?? 7,
|
||||
extraConfig: initialValue.extraConfig,
|
||||
verifyEnabled: initialValue.verifyEnabled ?? false,
|
||||
verifyCronExpr: initialValue.verifyCronExpr ?? '',
|
||||
@@ -588,6 +592,34 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa
|
||||
<Typography.Text>压缩策略</Typography.Text>
|
||||
<Select value={draft.compression} options={backupCompressionOptions as unknown as { label: string; value: string }[]} onChange={(value) => updateDraft({ compression: value as BackupTaskPayload['compression'] })} />
|
||||
</div>
|
||||
{isFileBackupTask(draft.type) && (
|
||||
<div>
|
||||
<Typography.Text>备份模式</Typography.Text>
|
||||
<Select
|
||||
value={draft.backupMode}
|
||||
options={[
|
||||
{ label: '全量备份', value: 'full' },
|
||||
{ label: '差异备份(仅文件、本机)', value: 'differential' },
|
||||
]}
|
||||
onChange={(value) => updateDraft({ backupMode: value as BackupMode })}
|
||||
/>
|
||||
{draft.backupMode === 'differential' && (
|
||||
<div style={{ marginTop: 8 }}>
|
||||
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
|
||||
差异备份仅打包自上次全量以来的变更,显著减小体积;超过下列天数将自动重做一次全量以控制差异链长度。恢复时自动按「全量 + 差异」链还原。
|
||||
</Typography.Text>
|
||||
<InputNumber
|
||||
style={{ width: '100%', marginTop: 4 }}
|
||||
placeholder="强制全量间隔(天)"
|
||||
prefix="全量间隔(天)"
|
||||
min={1}
|
||||
value={draft.diffFullIntervalDays}
|
||||
onChange={(value) => updateDraft({ diffFullIntervalDays: Number(value ?? 7) })}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
<Typography.Text>保留天数</Typography.Text>
|
||||
<InputNumber style={{ width: '100%' }} value={draft.retentionDays} min={0} onChange={(value) => updateDraft({ retentionDays: Number(value ?? 0) })} />
|
||||
|
||||
@@ -113,6 +113,7 @@ export function BackupRecordsPage() {
|
||||
<Space size={4}>
|
||||
<Typography.Text>{record.fileName || '-'}</Typography.Text>
|
||||
{record.locked && <Tag color="orange" size="small" bordered>已锁定</Tag>}
|
||||
{record.backupKind === 'differential' && <Tag color="purple" size="small" bordered>差异</Tag>}
|
||||
</Space>
|
||||
<Typography.Text type="secondary">{formatBytes(record.fileSize)}</Typography.Text>
|
||||
{record.checksum && (
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface BackupRecordSummary {
|
||||
startedAt: string
|
||||
completedAt?: string
|
||||
locked: boolean
|
||||
backupKind: 'full' | 'differential'
|
||||
}
|
||||
|
||||
export interface StorageUploadResultItem {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export type BackupTaskType = 'file' | 'mysql' | 'sqlite' | 'postgresql' | 'saphana' | 'mongodb'
|
||||
export type BackupTaskStatus = 'idle' | 'running' | 'success' | 'failed'
|
||||
export type BackupCompression = 'gzip' | 'none'
|
||||
export type BackupMode = 'full' | 'differential'
|
||||
|
||||
export interface BackupTaskSummary {
|
||||
id: number
|
||||
@@ -25,6 +26,8 @@ export interface BackupTaskSummary {
|
||||
keepWeekly: number
|
||||
keepMonthly: number
|
||||
keepYearly: number
|
||||
backupMode: BackupMode
|
||||
diffFullIntervalDays: number
|
||||
lastRunAt?: string
|
||||
lastStatus: BackupTaskStatus
|
||||
verifyEnabled: boolean
|
||||
@@ -81,6 +84,8 @@ export interface BackupTaskPayload {
|
||||
keepWeekly: number
|
||||
keepMonthly: number
|
||||
keepYearly: number
|
||||
backupMode: BackupMode
|
||||
diffFullIntervalDays: number
|
||||
/** 类型特有的扩展配置(如 SAP HANA 的 backupLevel/backupChannels 等) */
|
||||
extraConfig?: Record<string, unknown>
|
||||
verifyEnabled: boolean
|
||||
|
||||
Reference in New Issue
Block a user