From 992fc241508db463d16b05242c57f4b1d2a36984 Mon Sep 17 00:00:00 2001 From: Wu Qing <3184394176@qq.com> Date: Wed, 27 May 2026 15:36:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(backup):=20GFS=20=E5=88=86=E5=B1=82?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E7=AD=96=E7=95=A5=EF=BC=88=E7=A5=96=E7=88=B6?= =?UTF-8?q?-=E7=88=B6-=E5=AD=90=EF=BC=89+=20=E4=BB=BB=E5=8A=A1=E8=A1=A8?= =?UTF-8?q?=E5=8D=95=E9=85=8D=E7=BD=AE=20(#85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GFS 分层保留：按天/周/月/年保留代表性备份，各层级并集；任一>0 启用、全0 维持原策略（兼容），锁定记录豁免。后端 retention 算法+任务字段贯通，前端任务表单 GFS 配置。go test、tsc+vite 通过。 --- server/internal/backup/retention/service.go | 78 ++++++++++++++++++- .../internal/backup/retention/service_test.go | 75 ++++++++++++++++++ server/internal/model/backup_task.go | 74 ++++++++++-------- .../internal/service/backup_task_service.go | 17 ++++ .../backup-tasks/BackupTaskFormDrawer.tsx | 27 +++++++ web/src/types/backup-tasks.ts | 8 ++ 6 files changed, 244 insertions(+), 35 deletions(-) diff --git a/server/internal/backup/retention/service.go b/server/internal/backup/retention/service.go index cee8b6b..6e4fc53 100644 --- a/server/internal/backup/retention/service.go +++ b/server/internal/backup/retention/service.go @@ -3,6 +3,7 @@ package retention import ( "context" "fmt" + "sort" "strings" "time" @@ -56,7 +57,13 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider if err != nil { return nil, fmt.Errorf("list successful records: %w", err) } - candidates := selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now()) + var candidates []model.BackupRecord + if gfsEnabled(task) { + // GFS 策略：按天/周/月/年分层保留代表性备份，取代简单的天数/数量策略。 + candidates = selectGFSToDelete(records, task.KeepDaily, task.KeepWeekly, task.KeepMonthly, task.KeepYearly) + } else { + candidates = selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now()) + } result := &CleanupResult{} for _, record := range candidates { if strings.TrimSpace(record.StoragePath) != "" { @@ -133,3 +140,72 @@ func hasLocked(records []model.BackupRecord) bool { } return false } + +// gfsEnabled 判定任务是否启用 GFS 分层保留（任一层级 > 0）。 +func gfsEnabled(task *model.BackupTask) bool { + return task.KeepDaily > 0 || task.KeepWeekly > 0 || task.KeepMonthly > 0 || task.KeepYearly > 0 +} + +func recordTime(r *model.BackupRecord) time.Time { + if r.CompletedAt != nil { + return *r.CompletedAt + } + return r.StartedAt +} + +func isoWeekKey(t time.Time) string { + y, w := t.ISOWeek() + return fmt.Sprintf("%d-W%02d", y, w) +} + +// selectGFSToDelete 按 GFS（祖父-父-子）策略选出应删除的记录。 +// +// 规则：对每个层级（天/周/月/年），在按时间降序排列后，保留最近 keep 个不同周期中 +// 每个周期最新的一份备份；各层级保留集合取并集即「保留集」，其余删除。 +// 锁定（法律保留）的记录始终排除在删除候选之外。 +func selectGFSToDelete(records []model.BackupRecord, daily, weekly, monthly, yearly int) []model.BackupRecord { + active := make([]model.BackupRecord, 0, len(records)) + for i := range records { + if !records[i].Locked { + active = append(active, records[i]) + } + } + sort.SliceStable(active, func(i, j int) bool { + return recordTime(&active[i]).After(recordTime(&active[j])) + }) + + keep := make(map[uint]bool, len(active)) + keepTier := func(count int, key func(time.Time) string) { + if count <= 0 { + return + } + periods := 0 + lastPeriod := "" + havePrev := false + for i := range active { + p := key(recordTime(&active[i])) + if havePrev && p == lastPeriod { + continue // 同周期已保留代表（最新一份） + } + if periods >= count { + break // 该层级已保留足够多的周期 + } + keep[active[i].ID] = true + lastPeriod = p + havePrev = true + periods++ + } + } + keepTier(daily, func(t time.Time) string { return t.Format("2006-01-02") }) + keepTier(weekly, isoWeekKey) + keepTier(monthly, func(t time.Time) string { return t.Format("2006-01") }) + keepTier(yearly, func(t time.Time) string { return t.Format("2006") }) + + del := make([]model.BackupRecord, 0) + for i := range active { + if !keep[active[i].ID] { + del = append(del, active[i]) + } + } + return del +} diff --git a/server/internal/backup/retention/service_test.go b/server/internal/backup/retention/service_test.go index 8661ddf..a3b7631 100644 --- a/server/internal/backup/retention/service_test.go +++ b/server/internal/backup/retention/service_test.go @@ -93,6 +93,81 @@ func TestSelectRecordsToDelete(t *testing.T) { } } +func gfsRecord(id uint, ts time.Time, locked bool) model.BackupRecord { + completed := ts + return model.BackupRecord{ID: id, StartedAt: ts, CompletedAt: &completed, Locked: locked} +} + +func gfsDay(y, m, d, h int) time.Time { + return time.Date(y, time.Month(m), d, h, 0, 0, 0, time.UTC) +} + +func deletedIDSet(records []model.BackupRecord) map[uint]bool { + out := make(map[uint]bool, len(records)) + for i := range records { + out[records[i].ID] = true + } + return out +} + +func assertDeleted(t *testing.T, del []model.BackupRecord, want ...uint) { + t.Helper() + got := deletedIDSet(del) + if len(got) != len(want) { + t.Fatalf("deleted set size = %d %v, want %d %v", len(got), got, len(want), want) + } + for _, id := range want { + if !got[id] { + t.Fatalf("expected id %d to be deleted; got %v", id, got) + } + } +} + +// TestSelectGFSToDelete_DailyTier 验证按天分层：每天仅保留最新一份，且只保留最近 N 天。 +func TestSelectGFSToDelete_DailyTier(t *testing.T) { + records := []model.BackupRecord{ + gfsRecord(5, gfsDay(2026, 3, 7, 12), false), // 今天，最新 → 保留 + gfsRecord(4, gfsDay(2026, 3, 7, 6), false), // 今天，较早 → 删除（非当天代表） + gfsRecord(3, gfsDay(2026, 3, 6, 12), false), // 昨天 → 保留 + gfsRecord(2, gfsDay(2026, 3, 5, 12), false), // 前天 → 超出 daily=2 → 删除 + gfsRecord(1, gfsDay(2026, 3, 4, 12), false), // 更早 → 删除 + } + del := selectGFSToDelete(records, 2, 0, 0, 0) + assertDeleted(t, del, 4, 2, 1) +} + +// TestSelectGFSToDelete_TierUnion 验证多层级取并集：月度层级保留日度层级会删除的旧备份。 +func TestSelectGFSToDelete_TierUnion(t *testing.T) { + records := []model.BackupRecord{ + gfsRecord(3, gfsDay(2026, 3, 7, 12), false), // 3 月（最新） + gfsRecord(2, gfsDay(2026, 2, 15, 12), false), // 2 月 + gfsRecord(1, gfsDay(2026, 1, 15, 12), false), // 1 月 + } + // daily=1 只留 ID3；monthly=2 留最近两个月（3 月=ID3、2 月=ID2）。并集={3,2}，删除 ID1。 + del := selectGFSToDelete(records, 1, 0, 2, 0) + assertDeleted(t, del, 1) +} + +// TestSelectGFSToDelete_SkipsLocked 验证锁定记录即使超出所有层级也永不删除。 +func TestSelectGFSToDelete_SkipsLocked(t *testing.T) { + records := []model.BackupRecord{ + gfsRecord(3, gfsDay(2026, 3, 7, 12), false), + gfsRecord(2, gfsDay(2026, 3, 6, 12), false), + gfsRecord(1, gfsDay(2020, 1, 1, 12), true), // 远超 daily=1 但已锁定 → 不删 + } + del := selectGFSToDelete(records, 1, 0, 0, 0) + assertDeleted(t, del, 2) // 仅 ID2 被删；ID1 锁定豁免，ID3 为当日代表 +} + +func TestGFSEnabled(t *testing.T) { + if gfsEnabled(&model.BackupTask{}) { + t.Fatal("empty GFS config should be disabled") + } + if !gfsEnabled(&model.BackupTask{KeepWeekly: 4}) { + t.Fatal("KeepWeekly>0 should enable GFS") + } +} + // TestSelectRecordsToDelete_SkipsLocked 验证保留锁定（法律保留）的记录永不被选中删除， // 即使它既超过保留期、又超过 maxBackups 名额。 func TestSelectRecordsToDelete_SkipsLocked(t *testing.T) { diff --git a/server/internal/model/backup_task.go b/server/internal/model/backup_task.go index 1e77e75..5305ed9 100644 --- a/server/internal/model/backup_task.go +++ b/server/internal/model/backup_task.go @@ -18,42 +18,48 @@ const ( ) type BackupTask struct { - ID uint `gorm:"primaryKey" json:"id"` - Name string `gorm:"size:100;uniqueIndex;not null" json:"name"` - Type string `gorm:"size:20;index;not null" json:"type"` - Enabled bool `gorm:"not null;default:true" json:"enabled"` - CronExpr string `gorm:"column:cron_expr;size:64" json:"cronExpr"` - SourcePath string `gorm:"column:source_path;size:500" json:"sourcePath"` - SourcePaths string `gorm:"column:source_paths;type:text" json:"sourcePaths"` - ExcludePatterns string `gorm:"column:exclude_patterns;type:text" json:"excludePatterns"` - DBHost string `gorm:"column:db_host;size:255" json:"dbHost"` - DBPort int `gorm:"column:db_port" json:"dbPort"` - DBUser string `gorm:"column:db_user;size:100" json:"dbUser"` - DBPasswordCiphertext string `gorm:"column:db_password_ciphertext;type:text" json:"-"` - DBName string `gorm:"column:db_name;size:255" json:"dbName"` - DBPath string `gorm:"column:db_path;size:500" json:"dbPath"` + ID uint `gorm:"primaryKey" json:"id"` + Name string `gorm:"size:100;uniqueIndex;not null" json:"name"` + Type string `gorm:"size:20;index;not null" json:"type"` + Enabled bool `gorm:"not null;default:true" json:"enabled"` + CronExpr string `gorm:"column:cron_expr;size:64" json:"cronExpr"` + SourcePath string `gorm:"column:source_path;size:500" json:"sourcePath"` + SourcePaths string `gorm:"column:source_paths;type:text" json:"sourcePaths"` + ExcludePatterns string `gorm:"column:exclude_patterns;type:text" json:"excludePatterns"` + DBHost string `gorm:"column:db_host;size:255" json:"dbHost"` + DBPort int `gorm:"column:db_port" json:"dbPort"` + DBUser string `gorm:"column:db_user;size:100" json:"dbUser"` + DBPasswordCiphertext string `gorm:"column:db_password_ciphertext;type:text" json:"-"` + DBName string `gorm:"column:db_name;size:255" json:"dbName"` + DBPath string `gorm:"column:db_path;size:500" json:"dbPath"` // ExtraConfig 类型特有的扩展配置（JSON），如 SAP HANA 的 backupLevel / backupChannels 等 - ExtraConfig string `gorm:"column:extra_config;type:text" json:"extraConfig"` - StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"` // deprecated: 保留兼容 - StorageTarget StorageTarget `json:"storageTarget,omitempty"` // deprecated: 保留兼容 - StorageTargets []StorageTarget `gorm:"many2many:backup_task_storage_targets" json:"storageTargets,omitempty"` - NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"` - Node Node `json:"node,omitempty"` + ExtraConfig string `gorm:"column:extra_config;type:text" json:"extraConfig"` + StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"` // deprecated: 保留兼容 + StorageTarget StorageTarget `json:"storageTarget,omitempty"` // deprecated: 保留兼容 + StorageTargets []StorageTarget `gorm:"many2many:backup_task_storage_targets" json:"storageTargets,omitempty"` + NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"` + Node Node `json:"node,omitempty"` // NodePoolTag 节点池标签（可选）。非空且 NodeID=0 时，调度器会从 Node.Labels 包含该 tag // 的在线节点中动态挑选一台执行（按运行中任务数最少原则），失败会 best-effort 切换到下一个候选。 // 典型场景：NodePoolTag="db" 让 MySQL 备份任务在任意标有 "db" 的数据库节点执行。 - NodePoolTag string `gorm:"column:node_pool_tag;size:64;index" json:"nodePoolTag"` - Tags string `gorm:"column:tags;size:500" json:"tags"` - RetentionDays int `gorm:"column:retention_days;not null;default:30" json:"retentionDays"` - Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"` - Encrypt bool `gorm:"not null;default:false" json:"encrypt"` - MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"` - LastRunAt *time.Time `gorm:"column:last_run_at" json:"lastRunAt,omitempty"` - LastStatus string `gorm:"column:last_status;size:20;not null;default:'idle'" json:"lastStatus"` + NodePoolTag string `gorm:"column:node_pool_tag;size:64;index" json:"nodePoolTag"` + Tags string `gorm:"column:tags;size:500" json:"tags"` + RetentionDays int `gorm:"column:retention_days;not null;default:30" json:"retentionDays"` + Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"` + Encrypt bool `gorm:"not null;default:false" json:"encrypt"` + MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"` + // GFS（祖父-父-子）保留：分别保留最近 N 天 / M 周 / K 月 / Y 年的代表性备份（每周期保留最新一份）。 + // 任一 > 0 即启用 GFS，取代 RetentionDays/MaxBackups 简单策略；全为 0 时维持简单策略（向后兼容）。 + KeepDaily int `gorm:"column:keep_daily;not null;default:0" json:"keepDaily"` + KeepWeekly int `gorm:"column:keep_weekly;not null;default:0" json:"keepWeekly"` + KeepMonthly int `gorm:"column:keep_monthly;not null;default:0" json:"keepMonthly"` + KeepYearly int `gorm:"column:keep_yearly;not null;default:0" json:"keepYearly"` + LastRunAt *time.Time `gorm:"column:last_run_at" json:"lastRunAt,omitempty"` + LastStatus string `gorm:"column:last_status;size:20;not null;default:'idle'" json:"lastStatus"` // 验证（恢复演练）配置 — 定期自动校验备份可恢复性 - VerifyEnabled bool `gorm:"column:verify_enabled;not null;default:false" json:"verifyEnabled"` - VerifyCronExpr string `gorm:"column:verify_cron_expr;size:64" json:"verifyCronExpr"` - VerifyMode string `gorm:"column:verify_mode;size:20;not null;default:'quick'" json:"verifyMode"` + VerifyEnabled bool `gorm:"column:verify_enabled;not null;default:false" json:"verifyEnabled"` + VerifyCronExpr string `gorm:"column:verify_cron_expr;size:64" json:"verifyCronExpr"` + VerifyMode string `gorm:"column:verify_mode;size:20;not null;default:'quick'" json:"verifyMode"` // SLA 配置 — RPO（期望最长未备份间隔）与告警阈值 SLAHoursRPO int `gorm:"column:sla_hours_rpo;not null;default:0" json:"slaHoursRpo"` AlertOnConsecutiveFails int `gorm:"column:alert_on_consecutive_fails;not null;default:1" json:"alertOnConsecutiveFails"` @@ -68,9 +74,9 @@ type BackupTask struct { // 语义：上游任务成功后自动触发本任务，形成工作流（如 DB 备份完成 → 归档压缩）。 // 调度器继续按本任务自己的 cron 触发，仅"自动触发"路径响应依赖完成事件。 // 循环依赖检查在 service 层完成，避免配置阶段即出错。 - DependsOnTaskIDs string `gorm:"column:depends_on_task_ids;size:500" json:"dependsOnTaskIds"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + DependsOnTaskIDs string `gorm:"column:depends_on_task_ids;size:500" json:"dependsOnTaskIds"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } func (BackupTask) TableName() string { diff --git a/server/internal/service/backup_task_service.go b/server/internal/service/backup_task_service.go index 1f40f2d..4b31711 100644 --- a/server/internal/service/backup_task_service.go +++ b/server/internal/service/backup_task_service.go @@ -52,6 +52,11 @@ type BackupTaskUpsertInput struct { // SLA 配置 SLAHoursRPO int `json:"slaHoursRpo"` AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails"` + // GFS 分层保留（任一 > 0 启用，取代 RetentionDays/MaxBackups） + KeepDaily int `json:"keepDaily"` + KeepWeekly int `json:"keepWeekly"` + KeepMonthly int `json:"keepMonthly"` + KeepYearly int `json:"keepYearly"` // 备份复制目标存储 ID 列表（3-2-1 规则） ReplicationTargetIDs []uint `json:"replicationTargetIds"` // 维护窗口（CSV，详见 backup/window.go） @@ -90,6 +95,10 @@ type BackupTaskSummary struct { VerifyMode string `json:"verifyMode"` SLAHoursRPO int `json:"slaHoursRpo"` AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails"` + KeepDaily int `json:"keepDaily"` + KeepWeekly int `json:"keepWeekly"` + KeepMonthly int `json:"keepMonthly"` + KeepYearly int `json:"keepYearly"` // 备份复制目标（3-2-1） ReplicationTargetIDs []uint `json:"replicationTargetIds"` MaintenanceWindows string `json:"maintenanceWindows"` @@ -674,6 +683,10 @@ func (s *BackupTaskService) buildTask(existing *model.BackupTask, input BackupTa VerifyMode: normalizeVerifyMode(input.VerifyMode), SLAHoursRPO: maxInt(0, input.SLAHoursRPO), AlertOnConsecutiveFails: alertThreshold(input.AlertOnConsecutiveFails), + KeepDaily: maxInt(0, input.KeepDaily), + KeepWeekly: maxInt(0, input.KeepWeekly), + KeepMonthly: maxInt(0, input.KeepMonthly), + KeepYearly: maxInt(0, input.KeepYearly), ReplicationTargetIDs: encodeUintCSV(input.ReplicationTargetIDs), MaintenanceWindows: strings.TrimSpace(input.MaintenanceWindows), DependsOnTaskIDs: encodeUintCSV(input.DependsOnTaskIDs), @@ -766,6 +779,10 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary { VerifyMode: item.VerifyMode, SLAHoursRPO: item.SLAHoursRPO, AlertOnConsecutiveFails: item.AlertOnConsecutiveFails, + KeepDaily: item.KeepDaily, + KeepWeekly: item.KeepWeekly, + KeepMonthly: item.KeepMonthly, + KeepYearly: item.KeepYearly, ReplicationTargetIDs: parseUintCSV(item.ReplicationTargetIDs), MaintenanceWindows: item.MaintenanceWindows, DependsOnTaskIDs: parseUintCSV(item.DependsOnTaskIDs), diff --git a/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx b/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx index d0a2233..1e30293 100644 --- a/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx +++ b/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx @@ -65,6 +65,10 @@ function createEmptyDraft(storageTargets?: StorageTargetSummary[]): BackupTaskPa compression: 'gzip', encrypt: false, maxBackups: 10, + keepDaily: 0, + keepWeekly: 0, + keepMonthly: 0, + keepYearly: 0, extraConfig: undefined, verifyEnabled: false, verifyCronExpr: '', @@ -134,6 +138,10 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa compression: initialValue.compression, encrypt: initialValue.encrypt, maxBackups: initialValue.maxBackups, + keepDaily: initialValue.keepDaily ?? 0, + keepWeekly: initialValue.keepWeekly ?? 0, + keepMonthly: initialValue.keepMonthly ?? 0, + keepYearly: initialValue.keepYearly ?? 0, extraConfig: initialValue.extraConfig, verifyEnabled: initialValue.verifyEnabled ?? false, verifyCronExpr: initialValue.verifyCronExpr ?? '', @@ -588,6 +596,25 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa 最大保留份数 updateDraft({ maxBackups: Number(value ?? 0) })} /> +
+ + GFS 分层保留（任一 > 0 即启用，覆盖上面的天数/份数；每天/周/月/年仅保留最新一份） + + + + updateDraft({ keepDaily: Number(value ?? 0) })} /> + + + updateDraft({ keepWeekly: Number(value ?? 0) })} /> + + + updateDraft({ keepMonthly: Number(value ?? 0) })} /> + + + updateDraft({ keepYearly: Number(value ?? 0) })} /> + + +
标签（逗号分隔，用于分组与筛选） verifyEnabled: boolean