mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-07-12 16:02:18 +08:00
feat(backup): 新增差异备份(differential)模式 (#88)
文件备份新增差异模式:仅打包自上次全量以来的变更并记录删除,恢复自动按全量+差异链还原。含基线解析、链式恢复、保留链保护与本机文件任务校验;清单/比对/删除/往返/保留保护单测全覆盖。
This commit is contained in:
@@ -580,6 +580,40 @@ func (s *BackupExecutionService) resolveRemoteNode(ctx context.Context, nodeID u
|
||||
return resolveRemoteExecutionNode(ctx, s.nodeRepo, s.agentDispatcher != nil, nodeID)
|
||||
}
|
||||
|
||||
// resolveDifferentialBase 为差异备份解析基线全量:仅本机(NodeID=0)文件任务且 BackupMode=differential 时生效。
|
||||
// 返回最近一次「成功、含清单、未超过 DiffFullIntervalDays」的全量记录 ID 及其清单;
|
||||
// 无合适基线(首次备份 / 最近全量已过期 / 清单缺失)时 ok=false,调用方回退为全量。
|
||||
func (s *BackupExecutionService) resolveDifferentialBase(ctx context.Context, task *model.BackupTask) (uint, backup.Manifest, bool) {
|
||||
if task.Type != model.BackupTaskTypeFile || task.NodeID != 0 || !strings.EqualFold(task.BackupMode, model.BackupModeDifferential) {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
records, err := s.records.ListSuccessfulByTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
intervalDays := task.DiffFullIntervalDays
|
||||
if intervalDays <= 0 {
|
||||
intervalDays = 7
|
||||
}
|
||||
cutoff := time.Now().Add(-time.Duration(intervalDays) * 24 * time.Hour)
|
||||
for i := range records {
|
||||
rec := records[i]
|
||||
if rec.BackupKind != model.BackupKindFull || strings.TrimSpace(rec.Manifest) == "" {
|
||||
continue
|
||||
}
|
||||
// 最近的全量已超过强制全量间隔 → 触发新全量,限制差异链跨度与单个差异体积。
|
||||
if rec.StartedAt.Before(cutoff) {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
manifest, decErr := backup.DecodeManifest([]byte(rec.Manifest))
|
||||
if decErr != nil || len(manifest.Entries) == 0 {
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
return rec.ID, manifest, true
|
||||
}
|
||||
return 0, backup.Manifest{}, false
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) {
|
||||
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0,
|
||||
// 该节点上所有任务共享一个节点专属 semaphore,互相排队
|
||||
@@ -604,6 +638,10 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
var storagePath string
|
||||
selectedStorageTargetID := task.StorageTargetID
|
||||
var uploadResults []StorageUploadResultItem
|
||||
// 差异备份链信息:实际类型(全量/差异)、基线全量 ID、全量清单 JSON。
|
||||
backupKind := model.BackupKindFull
|
||||
var baseRecordID uint
|
||||
var manifestJSON string
|
||||
completeRecord := func() {
|
||||
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
|
||||
logger.Errorf("写回备份记录失败:%v", finalizeErr)
|
||||
@@ -619,6 +657,17 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
}
|
||||
}
|
||||
// 持久化差异链信息:全量记录其清单(供后续差异比对),差异记录其基线全量 ID。
|
||||
if status == model.BackupRecordStatusSuccess && (backupKind != model.BackupKindFull || baseRecordID != 0 || manifestJSON != "") {
|
||||
if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil {
|
||||
record.BackupKind = backupKind
|
||||
record.BaseRecordID = baseRecordID
|
||||
record.Manifest = manifestJSON
|
||||
if updErr := s.records.Update(ctx, record); updErr != nil {
|
||||
logger.Warnf("写回差异链信息失败:%v", updErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.shouldNotify(ctx, task, status) {
|
||||
if err := s.notifier.NotifyBackupResult(ctx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
|
||||
logger.Warnf("发送备份通知失败:%v", err)
|
||||
@@ -636,6 +685,13 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
logger.Errorf("构建任务运行时配置失败:%v", err)
|
||||
return
|
||||
}
|
||||
// 差异备份:解析基线全量,命中则切换为差异模式(仅本机文件任务)。
|
||||
if baseID, baseManifest, ok := s.resolveDifferentialBase(ctx, task); ok {
|
||||
spec.Differential = true
|
||||
spec.BaseManifest = baseManifest
|
||||
baseRecordID = baseID
|
||||
logger.Infof("差异备份模式:基于全量备份 #%d 仅打包变更", baseID)
|
||||
}
|
||||
runner, err := s.runnerRegistry.Runner(spec.Type)
|
||||
if err != nil {
|
||||
errMessage = err.Error()
|
||||
@@ -649,6 +705,17 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(result.TempDir)
|
||||
// 依据运行器产出判定实际类型:产出清单 → 全量(记录清单供后续差异比对);否则为差异。
|
||||
if result.Manifest != nil {
|
||||
backupKind = model.BackupKindFull
|
||||
if data, encErr := backup.EncodeManifest(*result.Manifest); encErr == nil {
|
||||
manifestJSON = string(data)
|
||||
} else {
|
||||
logger.Warnf("备份清单序列化失败(不影响本次备份,但将禁用后续差异):%v", encErr)
|
||||
}
|
||||
} else {
|
||||
backupKind = model.BackupKindDifferential
|
||||
}
|
||||
finalPath := result.ArtifactPath
|
||||
if strings.EqualFold(task.Compression, "gzip") && !strings.HasSuffix(strings.ToLower(finalPath), ".gz") {
|
||||
logger.Infof("开始压缩备份文件")
|
||||
|
||||
@@ -37,6 +37,7 @@ type BackupRecordSummary struct {
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
Locked bool `json:"locked"`
|
||||
BackupKind string `json:"backupKind"`
|
||||
}
|
||||
|
||||
type BackupRecordDetail struct {
|
||||
@@ -139,6 +140,7 @@ func toBackupRecordSummary(item *model.BackupRecord) BackupRecordSummary {
|
||||
StartedAt: item.StartedAt,
|
||||
CompletedAt: item.CompletedAt,
|
||||
Locked: item.Locked,
|
||||
BackupKind: item.BackupKind,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,9 @@ type BackupTaskUpsertInput struct {
|
||||
KeepWeekly int `json:"keepWeekly"`
|
||||
KeepMonthly int `json:"keepMonthly"`
|
||||
KeepYearly int `json:"keepYearly"`
|
||||
// BackupMode 备份模式:full(默认)/ differential(差异,仅文件类型本机任务)
|
||||
BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential"`
|
||||
DiffFullIntervalDays int `json:"diffFullIntervalDays"`
|
||||
// 备份复制目标存储 ID 列表(3-2-1 规则)
|
||||
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
|
||||
// 维护窗口(CSV,详见 backup/window.go)
|
||||
@@ -99,6 +102,8 @@ type BackupTaskSummary struct {
|
||||
KeepWeekly int `json:"keepWeekly"`
|
||||
KeepMonthly int `json:"keepMonthly"`
|
||||
KeepYearly int `json:"keepYearly"`
|
||||
BackupMode string `json:"backupMode"`
|
||||
DiffFullIntervalDays int `json:"diffFullIntervalDays"`
|
||||
// 备份复制目标(3-2-1)
|
||||
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
|
||||
MaintenanceWindows string `json:"maintenanceWindows"`
|
||||
@@ -517,6 +522,14 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B
|
||||
return apperror.BadRequest("BACKUP_TASK_REMOTE_ENCRYPT_UNSUPPORTED",
|
||||
"远程节点暂不支持加密备份。请关闭加密,或将任务固定在 Master 本机执行。", nil)
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(input.BackupMode), model.BackupModeDifferential) {
|
||||
if input.Type != model.BackupTaskTypeFile {
|
||||
return apperror.BadRequest("BACKUP_TASK_DIFF_UNSUPPORTED", "差异备份仅支持文件目录类型任务", nil)
|
||||
}
|
||||
if strings.TrimSpace(input.NodePoolTag) != "" || (fixedNode != nil && !fixedNode.IsLocal) {
|
||||
return apperror.BadRequest("BACKUP_TASK_DIFF_REMOTE_UNSUPPORTED", "差异备份当前仅支持本机 Master 执行,请将任务固定在本机或改用全量备份。", nil)
|
||||
}
|
||||
}
|
||||
if input.RetentionDays < 0 {
|
||||
return apperror.BadRequest("BACKUP_TASK_INVALID", "保留天数不能小于 0", nil)
|
||||
}
|
||||
@@ -687,6 +700,8 @@ func (s *BackupTaskService) buildTask(existing *model.BackupTask, input BackupTa
|
||||
KeepWeekly: maxInt(0, input.KeepWeekly),
|
||||
KeepMonthly: maxInt(0, input.KeepMonthly),
|
||||
KeepYearly: maxInt(0, input.KeepYearly),
|
||||
BackupMode: normalizeBackupMode(input.BackupMode, input.Type),
|
||||
DiffFullIntervalDays: diffFullInterval(input.DiffFullIntervalDays),
|
||||
ReplicationTargetIDs: encodeUintCSV(input.ReplicationTargetIDs),
|
||||
MaintenanceWindows: strings.TrimSpace(input.MaintenanceWindows),
|
||||
DependsOnTaskIDs: encodeUintCSV(input.DependsOnTaskIDs),
|
||||
@@ -783,6 +798,8 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary {
|
||||
KeepWeekly: item.KeepWeekly,
|
||||
KeepMonthly: item.KeepMonthly,
|
||||
KeepYearly: item.KeepYearly,
|
||||
BackupMode: item.BackupMode,
|
||||
DiffFullIntervalDays: item.DiffFullIntervalDays,
|
||||
ReplicationTargetIDs: parseUintCSV(item.ReplicationTargetIDs),
|
||||
MaintenanceWindows: item.MaintenanceWindows,
|
||||
DependsOnTaskIDs: parseUintCSV(item.DependsOnTaskIDs),
|
||||
@@ -918,6 +935,22 @@ func decodeExtraConfig(value string) (map[string]any, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeBackupMode 归一化备份模式:仅文件类型可启用差异,其余一律全量(双保险,防绕过校验)。
|
||||
func normalizeBackupMode(mode, taskType string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(mode), model.BackupModeDifferential) && normalizeBackupTaskType(taskType) == model.BackupTaskTypeFile {
|
||||
return model.BackupModeDifferential
|
||||
}
|
||||
return model.BackupModeFull
|
||||
}
|
||||
|
||||
// diffFullInterval 归一化差异模式下的强制全量间隔(天),非正值回退默认 7。
|
||||
func diffFullInterval(days int) int {
|
||||
if days <= 0 {
|
||||
return 7
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
func normalizeBackupTaskType(value string) string {
|
||||
normalized := strings.TrimSpace(strings.ToLower(value))
|
||||
if normalized == "pgsql" {
|
||||
|
||||
@@ -223,10 +223,17 @@ func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, tas
|
||||
}()
|
||||
|
||||
logger.Infof("开始在本地执行恢复(备份记录 #%d)", backupRecord.ID)
|
||||
provider, providerErr := s.resolveProvider(ctx, backupRecord.StorageTargetID)
|
||||
if providerErr != nil {
|
||||
errMessage = providerErr.Error()
|
||||
logger.Errorf("创建存储客户端失败:%v", providerErr)
|
||||
|
||||
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
|
||||
if specErr != nil {
|
||||
errMessage = specErr.Error()
|
||||
logger.Errorf("构建恢复规格失败:%v", specErr)
|
||||
return
|
||||
}
|
||||
runner, runnerErr := s.runnerRegistry.Runner(spec.Type)
|
||||
if runnerErr != nil {
|
||||
errMessage = runnerErr.Error()
|
||||
logger.Errorf("不支持的备份类型:%v", runnerErr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -243,63 +250,89 @@ func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, tas
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
fileName := backupRecord.FileName
|
||||
if strings.TrimSpace(fileName) == "" {
|
||||
fileName = filepath.Base(backupRecord.StoragePath)
|
||||
}
|
||||
artifactPath := filepath.Join(tempDir, filepath.Base(fileName))
|
||||
logger.Infof("开始下载备份文件:%s", backupRecord.StoragePath)
|
||||
reader, downloadErr := provider.Download(ctx, backupRecord.StoragePath)
|
||||
if downloadErr != nil {
|
||||
errMessage = downloadErr.Error()
|
||||
logger.Errorf("下载备份文件失败:%v", downloadErr)
|
||||
// 恢复链:全量 → [自身];差异 → [基线全量, 自身],按序应用(全量铺底,差异覆盖并删除)。
|
||||
chain, chainErr := s.buildRestoreChain(ctx, backupRecord)
|
||||
if chainErr != nil {
|
||||
errMessage = chainErr.Error()
|
||||
logger.Errorf("%v", chainErr)
|
||||
return
|
||||
}
|
||||
if writeErr := writeReaderToFile(artifactPath, reader); writeErr != nil {
|
||||
errMessage = writeErr.Error()
|
||||
logger.Errorf("写入恢复文件失败:%v", writeErr)
|
||||
return
|
||||
}
|
||||
// 完整性校验:在解密/解压前比对下载对象的 SHA-256,拒绝还原损坏或被篡改的备份。
|
||||
// 早期未记录 checksum 的备份会跳过(向后兼容)。
|
||||
if backupRecord.Checksum != "" {
|
||||
logger.Infof("校验备份完整性(SHA-256)")
|
||||
if csErr := verifyArtifactChecksum(artifactPath, backupRecord.Checksum); csErr != nil {
|
||||
errMessage = csErr.Error()
|
||||
logger.Errorf("完整性校验失败:%v", csErr)
|
||||
logger.Infof("开始执行 %s 恢复(恢复链含 %d 个备份)", spec.Type, len(chain))
|
||||
for idx := range chain {
|
||||
rec := chain[idx]
|
||||
if len(chain) > 1 {
|
||||
logger.Infof("恢复链 [%d/%d]:应用备份记录 #%d(%s)", idx+1, len(chain), rec.ID, backupKindLabel(rec.BackupKind))
|
||||
}
|
||||
if err := s.restoreArtifact(ctx, &rec, spec, runner, tempDir, logger); err != nil {
|
||||
errMessage = err.Error()
|
||||
logger.Errorf("恢复执行失败:%v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof("完整性校验通过")
|
||||
}
|
||||
preparedPath, prepareErr := s.prepareArtifact(artifactPath, logger)
|
||||
if prepareErr != nil {
|
||||
errMessage = prepareErr.Error()
|
||||
logger.Errorf("准备恢复文件失败:%v", prepareErr)
|
||||
return
|
||||
}
|
||||
|
||||
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
|
||||
if specErr != nil {
|
||||
errMessage = specErr.Error()
|
||||
logger.Errorf("构建恢复规格失败:%v", specErr)
|
||||
return
|
||||
}
|
||||
runner, runnerErr := s.runnerRegistry.Runner(spec.Type)
|
||||
if runnerErr != nil {
|
||||
errMessage = runnerErr.Error()
|
||||
logger.Errorf("不支持的备份类型:%v", runnerErr)
|
||||
return
|
||||
}
|
||||
logger.Infof("开始执行 %s 恢复", spec.Type)
|
||||
if restoreErr := runner.Restore(ctx, spec, preparedPath, logger); restoreErr != nil {
|
||||
errMessage = restoreErr.Error()
|
||||
logger.Errorf("恢复执行失败:%v", restoreErr)
|
||||
return
|
||||
}
|
||||
status = model.RestoreRecordStatusSuccess
|
||||
logger.Infof("恢复执行成功")
|
||||
}
|
||||
|
||||
// restoreArtifact 下载、完整性校验、解密解压并通过 runner 应用单个备份记录的归档。
|
||||
// 每个记录使用独立子目录,避免恢复链中基线/差异的同名归档相互覆盖。
|
||||
func (s *RestoreService) restoreArtifact(ctx context.Context, record *model.BackupRecord, spec backup.TaskSpec, runner backup.BackupRunner, parentTempDir string, logger *backup.ExecutionLogger) error {
|
||||
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建存储客户端失败:%w", err)
|
||||
}
|
||||
recDir, err := os.MkdirTemp(parentTempDir, fmt.Sprintf("rec-%d-*", record.ID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建恢复子目录失败:%w", err)
|
||||
}
|
||||
fileName := record.FileName
|
||||
if strings.TrimSpace(fileName) == "" {
|
||||
fileName = filepath.Base(record.StoragePath)
|
||||
}
|
||||
artifactPath := filepath.Join(recDir, filepath.Base(fileName))
|
||||
logger.Infof("开始下载备份文件:%s", record.StoragePath)
|
||||
reader, err := provider.Download(ctx, record.StoragePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("下载备份文件失败:%w", err)
|
||||
}
|
||||
if err := writeReaderToFile(artifactPath, reader); err != nil {
|
||||
return fmt.Errorf("写入恢复文件失败:%w", err)
|
||||
}
|
||||
// 完整性校验:解密/解压前比对 SHA-256;早期无 checksum 的备份跳过(向后兼容)。
|
||||
if record.Checksum != "" {
|
||||
if err := verifyArtifactChecksum(artifactPath, record.Checksum); err != nil {
|
||||
return fmt.Errorf("完整性校验失败:%w", err)
|
||||
}
|
||||
}
|
||||
preparedPath, err := s.prepareArtifact(artifactPath, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("准备恢复文件失败:%w", err)
|
||||
}
|
||||
return runner.Restore(ctx, spec, preparedPath, logger)
|
||||
}
|
||||
|
||||
// buildRestoreChain 返回恢复某记录所需、按应用顺序排列的记录链:
|
||||
// 全量 → [自身];差异 → [基线全量, 自身]。基线缺失/不可用时报错,杜绝残缺恢复。
|
||||
func (s *RestoreService) buildRestoreChain(ctx context.Context, record *model.BackupRecord) ([]model.BackupRecord, error) {
|
||||
if record.BackupKind != model.BackupKindDifferential || record.BaseRecordID == 0 {
|
||||
return []model.BackupRecord{*record}, nil
|
||||
}
|
||||
base, err := s.records.FindByID(ctx, record.BaseRecordID)
|
||||
if err != nil || base == nil {
|
||||
return nil, fmt.Errorf("差异备份的基线全量 #%d 不存在,无法恢复", record.BaseRecordID)
|
||||
}
|
||||
if base.Status != model.BackupRecordStatusSuccess || strings.TrimSpace(base.StoragePath) == "" {
|
||||
return nil, fmt.Errorf("差异备份的基线全量 #%d 不可用,无法恢复", record.BaseRecordID)
|
||||
}
|
||||
return []model.BackupRecord{*base, *record}, nil
|
||||
}
|
||||
|
||||
func backupKindLabel(kind string) string {
|
||||
if kind == model.BackupKindDifferential {
|
||||
return "差异"
|
||||
}
|
||||
return "全量"
|
||||
}
|
||||
|
||||
// dispatchRestoreEvent 按终态向事件总线派发 restore_success 或 restore_failed。
|
||||
// eventDispatcher 未注入时静默忽略,保持向后兼容。
|
||||
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) {
|
||||
|
||||
Reference in New Issue
Block a user