feat(backup): GFS 分层保留策略(祖父-父-子)+ 任务表单配置 (#85)

新增 GFS 分层保留:按天/周/月/年保留代表性备份,各层级并集;任一>0 启用、全0 维持原策略(兼容),锁定记录豁免。后端 retention 算法+任务字段贯通,前端任务表单 GFS 配置。go test、tsc+vite 通过。
This commit is contained in:
Wu Qing
2026-05-27 15:36:58 +08:00
committed by GitHub
parent 386f12a11b
commit 992fc24150
6 changed files with 244 additions and 35 deletions

View File

@@ -3,6 +3,7 @@ package retention
import (
"context"
"fmt"
"sort"
"strings"
"time"
@@ -56,7 +57,13 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
if err != nil {
return nil, fmt.Errorf("list successful records: %w", err)
}
candidates := selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now())
var candidates []model.BackupRecord
if gfsEnabled(task) {
// GFS 策略:按天/周/月/年分层保留代表性备份,取代简单的天数/数量策略。
candidates = selectGFSToDelete(records, task.KeepDaily, task.KeepWeekly, task.KeepMonthly, task.KeepYearly)
} else {
candidates = selectRecordsToDelete(records, task.RetentionDays, task.MaxBackups, s.now())
}
result := &CleanupResult{}
for _, record := range candidates {
if strings.TrimSpace(record.StoragePath) != "" {
@@ -133,3 +140,72 @@ func hasLocked(records []model.BackupRecord) bool {
}
return false
}
// gfsEnabled 判定任务是否启用 GFS 分层保留(任一层级 > 0
func gfsEnabled(task *model.BackupTask) bool {
return task.KeepDaily > 0 || task.KeepWeekly > 0 || task.KeepMonthly > 0 || task.KeepYearly > 0
}
func recordTime(r *model.BackupRecord) time.Time {
if r.CompletedAt != nil {
return *r.CompletedAt
}
return r.StartedAt
}
func isoWeekKey(t time.Time) string {
y, w := t.ISOWeek()
return fmt.Sprintf("%d-W%02d", y, w)
}
// selectGFSToDelete 按 GFS祖父-父-子)策略选出应删除的记录。
//
// 规则:对每个层级(天/周/月/年),在按时间降序排列后,保留最近 keep 个不同周期中
// 每个周期最新的一份备份;各层级保留集合取并集即「保留集」,其余删除。
// 锁定(法律保留)的记录始终排除在删除候选之外。
func selectGFSToDelete(records []model.BackupRecord, daily, weekly, monthly, yearly int) []model.BackupRecord {
active := make([]model.BackupRecord, 0, len(records))
for i := range records {
if !records[i].Locked {
active = append(active, records[i])
}
}
sort.SliceStable(active, func(i, j int) bool {
return recordTime(&active[i]).After(recordTime(&active[j]))
})
keep := make(map[uint]bool, len(active))
keepTier := func(count int, key func(time.Time) string) {
if count <= 0 {
return
}
periods := 0
lastPeriod := ""
havePrev := false
for i := range active {
p := key(recordTime(&active[i]))
if havePrev && p == lastPeriod {
continue // 同周期已保留代表(最新一份)
}
if periods >= count {
break // 该层级已保留足够多的周期
}
keep[active[i].ID] = true
lastPeriod = p
havePrev = true
periods++
}
}
keepTier(daily, func(t time.Time) string { return t.Format("2006-01-02") })
keepTier(weekly, isoWeekKey)
keepTier(monthly, func(t time.Time) string { return t.Format("2006-01") })
keepTier(yearly, func(t time.Time) string { return t.Format("2006") })
del := make([]model.BackupRecord, 0)
for i := range active {
if !keep[active[i].ID] {
del = append(del, active[i])
}
}
return del
}

View File

@@ -93,6 +93,81 @@ func TestSelectRecordsToDelete(t *testing.T) {
}
}
func gfsRecord(id uint, ts time.Time, locked bool) model.BackupRecord {
completed := ts
return model.BackupRecord{ID: id, StartedAt: ts, CompletedAt: &completed, Locked: locked}
}
func gfsDay(y, m, d, h int) time.Time {
return time.Date(y, time.Month(m), d, h, 0, 0, 0, time.UTC)
}
func deletedIDSet(records []model.BackupRecord) map[uint]bool {
out := make(map[uint]bool, len(records))
for i := range records {
out[records[i].ID] = true
}
return out
}
func assertDeleted(t *testing.T, del []model.BackupRecord, want ...uint) {
t.Helper()
got := deletedIDSet(del)
if len(got) != len(want) {
t.Fatalf("deleted set size = %d %v, want %d %v", len(got), got, len(want), want)
}
for _, id := range want {
if !got[id] {
t.Fatalf("expected id %d to be deleted; got %v", id, got)
}
}
}
// TestSelectGFSToDelete_DailyTier 验证按天分层:每天仅保留最新一份,且只保留最近 N 天。
func TestSelectGFSToDelete_DailyTier(t *testing.T) {
records := []model.BackupRecord{
gfsRecord(5, gfsDay(2026, 3, 7, 12), false), // 今天,最新 → 保留
gfsRecord(4, gfsDay(2026, 3, 7, 6), false), // 今天,较早 → 删除(非当天代表)
gfsRecord(3, gfsDay(2026, 3, 6, 12), false), // 昨天 → 保留
gfsRecord(2, gfsDay(2026, 3, 5, 12), false), // 前天 → 超出 daily=2 → 删除
gfsRecord(1, gfsDay(2026, 3, 4, 12), false), // 更早 → 删除
}
del := selectGFSToDelete(records, 2, 0, 0, 0)
assertDeleted(t, del, 4, 2, 1)
}
// TestSelectGFSToDelete_TierUnion 验证多层级取并集:月度层级保留日度层级会删除的旧备份。
func TestSelectGFSToDelete_TierUnion(t *testing.T) {
records := []model.BackupRecord{
gfsRecord(3, gfsDay(2026, 3, 7, 12), false), // 3 月(最新)
gfsRecord(2, gfsDay(2026, 2, 15, 12), false), // 2 月
gfsRecord(1, gfsDay(2026, 1, 15, 12), false), // 1 月
}
// daily=1 只留 ID3monthly=2 留最近两个月3 月=ID3、2 月=ID2。并集={3,2},删除 ID1。
del := selectGFSToDelete(records, 1, 0, 2, 0)
assertDeleted(t, del, 1)
}
// TestSelectGFSToDelete_SkipsLocked 验证锁定记录即使超出所有层级也永不删除。
func TestSelectGFSToDelete_SkipsLocked(t *testing.T) {
records := []model.BackupRecord{
gfsRecord(3, gfsDay(2026, 3, 7, 12), false),
gfsRecord(2, gfsDay(2026, 3, 6, 12), false),
gfsRecord(1, gfsDay(2020, 1, 1, 12), true), // 远超 daily=1 但已锁定 → 不删
}
del := selectGFSToDelete(records, 1, 0, 0, 0)
assertDeleted(t, del, 2) // 仅 ID2 被删ID1 锁定豁免ID3 为当日代表
}
func TestGFSEnabled(t *testing.T) {
if gfsEnabled(&model.BackupTask{}) {
t.Fatal("empty GFS config should be disabled")
}
if !gfsEnabled(&model.BackupTask{KeepWeekly: 4}) {
t.Fatal("KeepWeekly>0 should enable GFS")
}
}
// TestSelectRecordsToDelete_SkipsLocked 验证保留锁定(法律保留)的记录永不被选中删除,
// 即使它既超过保留期、又超过 maxBackups 名额。
func TestSelectRecordsToDelete_SkipsLocked(t *testing.T) {

View File

@@ -18,42 +18,48 @@ const (
)
type BackupTask struct {
ID uint `gorm:"primaryKey" json:"id"`
Name string `gorm:"size:100;uniqueIndex;not null" json:"name"`
Type string `gorm:"size:20;index;not null" json:"type"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
CronExpr string `gorm:"column:cron_expr;size:64" json:"cronExpr"`
SourcePath string `gorm:"column:source_path;size:500" json:"sourcePath"`
SourcePaths string `gorm:"column:source_paths;type:text" json:"sourcePaths"`
ExcludePatterns string `gorm:"column:exclude_patterns;type:text" json:"excludePatterns"`
DBHost string `gorm:"column:db_host;size:255" json:"dbHost"`
DBPort int `gorm:"column:db_port" json:"dbPort"`
DBUser string `gorm:"column:db_user;size:100" json:"dbUser"`
DBPasswordCiphertext string `gorm:"column:db_password_ciphertext;type:text" json:"-"`
DBName string `gorm:"column:db_name;size:255" json:"dbName"`
DBPath string `gorm:"column:db_path;size:500" json:"dbPath"`
ID uint `gorm:"primaryKey" json:"id"`
Name string `gorm:"size:100;uniqueIndex;not null" json:"name"`
Type string `gorm:"size:20;index;not null" json:"type"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
CronExpr string `gorm:"column:cron_expr;size:64" json:"cronExpr"`
SourcePath string `gorm:"column:source_path;size:500" json:"sourcePath"`
SourcePaths string `gorm:"column:source_paths;type:text" json:"sourcePaths"`
ExcludePatterns string `gorm:"column:exclude_patterns;type:text" json:"excludePatterns"`
DBHost string `gorm:"column:db_host;size:255" json:"dbHost"`
DBPort int `gorm:"column:db_port" json:"dbPort"`
DBUser string `gorm:"column:db_user;size:100" json:"dbUser"`
DBPasswordCiphertext string `gorm:"column:db_password_ciphertext;type:text" json:"-"`
DBName string `gorm:"column:db_name;size:255" json:"dbName"`
DBPath string `gorm:"column:db_path;size:500" json:"dbPath"`
// ExtraConfig 类型特有的扩展配置JSON如 SAP HANA 的 backupLevel / backupChannels 等
ExtraConfig string `gorm:"column:extra_config;type:text" json:"extraConfig"`
StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"` // deprecated: 保留兼容
StorageTarget StorageTarget `json:"storageTarget,omitempty"` // deprecated: 保留兼容
StorageTargets []StorageTarget `gorm:"many2many:backup_task_storage_targets" json:"storageTargets,omitempty"`
NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"`
Node Node `json:"node,omitempty"`
ExtraConfig string `gorm:"column:extra_config;type:text" json:"extraConfig"`
StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"` // deprecated: 保留兼容
StorageTarget StorageTarget `json:"storageTarget,omitempty"` // deprecated: 保留兼容
StorageTargets []StorageTarget `gorm:"many2many:backup_task_storage_targets" json:"storageTargets,omitempty"`
NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"`
Node Node `json:"node,omitempty"`
// NodePoolTag 节点池标签(可选)。非空且 NodeID=0 时,调度器会从 Node.Labels 包含该 tag
// 的在线节点中动态挑选一台执行(按运行中任务数最少原则),失败会 best-effort 切换到下一个候选。
// 典型场景NodePoolTag="db" 让 MySQL 备份任务在任意标有 "db" 的数据库节点执行。
NodePoolTag string `gorm:"column:node_pool_tag;size:64;index" json:"nodePoolTag"`
Tags string `gorm:"column:tags;size:500" json:"tags"`
RetentionDays int `gorm:"column:retention_days;not null;default:30" json:"retentionDays"`
Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"`
Encrypt bool `gorm:"not null;default:false" json:"encrypt"`
MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"`
LastRunAt *time.Time `gorm:"column:last_run_at" json:"lastRunAt,omitempty"`
LastStatus string `gorm:"column:last_status;size:20;not null;default:'idle'" json:"lastStatus"`
NodePoolTag string `gorm:"column:node_pool_tag;size:64;index" json:"nodePoolTag"`
Tags string `gorm:"column:tags;size:500" json:"tags"`
RetentionDays int `gorm:"column:retention_days;not null;default:30" json:"retentionDays"`
Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"`
Encrypt bool `gorm:"not null;default:false" json:"encrypt"`
MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"`
// GFS祖父-父-子)保留:分别保留最近 N 天 / M 周 / K 月 / Y 年的代表性备份(每周期保留最新一份)。
// 任一 > 0 即启用 GFS取代 RetentionDays/MaxBackups 简单策略;全为 0 时维持简单策略(向后兼容)。
KeepDaily int `gorm:"column:keep_daily;not null;default:0" json:"keepDaily"`
KeepWeekly int `gorm:"column:keep_weekly;not null;default:0" json:"keepWeekly"`
KeepMonthly int `gorm:"column:keep_monthly;not null;default:0" json:"keepMonthly"`
KeepYearly int `gorm:"column:keep_yearly;not null;default:0" json:"keepYearly"`
LastRunAt *time.Time `gorm:"column:last_run_at" json:"lastRunAt,omitempty"`
LastStatus string `gorm:"column:last_status;size:20;not null;default:'idle'" json:"lastStatus"`
// 验证(恢复演练)配置 — 定期自动校验备份可恢复性
VerifyEnabled bool `gorm:"column:verify_enabled;not null;default:false" json:"verifyEnabled"`
VerifyCronExpr string `gorm:"column:verify_cron_expr;size:64" json:"verifyCronExpr"`
VerifyMode string `gorm:"column:verify_mode;size:20;not null;default:'quick'" json:"verifyMode"`
VerifyEnabled bool `gorm:"column:verify_enabled;not null;default:false" json:"verifyEnabled"`
VerifyCronExpr string `gorm:"column:verify_cron_expr;size:64" json:"verifyCronExpr"`
VerifyMode string `gorm:"column:verify_mode;size:20;not null;default:'quick'" json:"verifyMode"`
// SLA 配置 — RPO期望最长未备份间隔与告警阈值
SLAHoursRPO int `gorm:"column:sla_hours_rpo;not null;default:0" json:"slaHoursRpo"`
AlertOnConsecutiveFails int `gorm:"column:alert_on_consecutive_fails;not null;default:1" json:"alertOnConsecutiveFails"`
@@ -68,9 +74,9 @@ type BackupTask struct {
// 语义:上游任务成功后自动触发本任务,形成工作流(如 DB 备份完成 → 归档压缩)。
// 调度器继续按本任务自己的 cron 触发,仅"自动触发"路径响应依赖完成事件。
// 循环依赖检查在 service 层完成,避免配置阶段即出错。
DependsOnTaskIDs string `gorm:"column:depends_on_task_ids;size:500" json:"dependsOnTaskIds"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DependsOnTaskIDs string `gorm:"column:depends_on_task_ids;size:500" json:"dependsOnTaskIds"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (BackupTask) TableName() string {

View File

@@ -52,6 +52,11 @@ type BackupTaskUpsertInput struct {
// SLA 配置
SLAHoursRPO int `json:"slaHoursRpo"`
AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails"`
// GFS 分层保留(任一 > 0 启用,取代 RetentionDays/MaxBackups
KeepDaily int `json:"keepDaily"`
KeepWeekly int `json:"keepWeekly"`
KeepMonthly int `json:"keepMonthly"`
KeepYearly int `json:"keepYearly"`
// 备份复制目标存储 ID 列表3-2-1 规则)
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
// 维护窗口CSV详见 backup/window.go
@@ -90,6 +95,10 @@ type BackupTaskSummary struct {
VerifyMode string `json:"verifyMode"`
SLAHoursRPO int `json:"slaHoursRpo"`
AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails"`
KeepDaily int `json:"keepDaily"`
KeepWeekly int `json:"keepWeekly"`
KeepMonthly int `json:"keepMonthly"`
KeepYearly int `json:"keepYearly"`
// 备份复制目标3-2-1
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
MaintenanceWindows string `json:"maintenanceWindows"`
@@ -674,6 +683,10 @@ func (s *BackupTaskService) buildTask(existing *model.BackupTask, input BackupTa
VerifyMode: normalizeVerifyMode(input.VerifyMode),
SLAHoursRPO: maxInt(0, input.SLAHoursRPO),
AlertOnConsecutiveFails: alertThreshold(input.AlertOnConsecutiveFails),
KeepDaily: maxInt(0, input.KeepDaily),
KeepWeekly: maxInt(0, input.KeepWeekly),
KeepMonthly: maxInt(0, input.KeepMonthly),
KeepYearly: maxInt(0, input.KeepYearly),
ReplicationTargetIDs: encodeUintCSV(input.ReplicationTargetIDs),
MaintenanceWindows: strings.TrimSpace(input.MaintenanceWindows),
DependsOnTaskIDs: encodeUintCSV(input.DependsOnTaskIDs),
@@ -766,6 +779,10 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary {
VerifyMode: item.VerifyMode,
SLAHoursRPO: item.SLAHoursRPO,
AlertOnConsecutiveFails: item.AlertOnConsecutiveFails,
KeepDaily: item.KeepDaily,
KeepWeekly: item.KeepWeekly,
KeepMonthly: item.KeepMonthly,
KeepYearly: item.KeepYearly,
ReplicationTargetIDs: parseUintCSV(item.ReplicationTargetIDs),
MaintenanceWindows: item.MaintenanceWindows,
DependsOnTaskIDs: parseUintCSV(item.DependsOnTaskIDs),

View File

@@ -65,6 +65,10 @@ function createEmptyDraft(storageTargets?: StorageTargetSummary[]): BackupTaskPa
compression: 'gzip',
encrypt: false,
maxBackups: 10,
keepDaily: 0,
keepWeekly: 0,
keepMonthly: 0,
keepYearly: 0,
extraConfig: undefined,
verifyEnabled: false,
verifyCronExpr: '',
@@ -134,6 +138,10 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa
compression: initialValue.compression,
encrypt: initialValue.encrypt,
maxBackups: initialValue.maxBackups,
keepDaily: initialValue.keepDaily ?? 0,
keepWeekly: initialValue.keepWeekly ?? 0,
keepMonthly: initialValue.keepMonthly ?? 0,
keepYearly: initialValue.keepYearly ?? 0,
extraConfig: initialValue.extraConfig,
verifyEnabled: initialValue.verifyEnabled ?? false,
verifyCronExpr: initialValue.verifyCronExpr ?? '',
@@ -588,6 +596,25 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa
<Typography.Text></Typography.Text>
<InputNumber style={{ width: '100%' }} value={draft.maxBackups} min={0} onChange={(value) => updateDraft({ maxBackups: Number(value ?? 0) })} />
</div>
<div>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
GFS &gt; 0 ////
</Typography.Text>
<Grid.Row gutter={8} style={{ marginTop: 4 }}>
<Grid.Col span={6}>
<InputNumber style={{ width: '100%' }} placeholder="日" prefix="日" min={0} value={draft.keepDaily} onChange={(value) => updateDraft({ keepDaily: Number(value ?? 0) })} />
</Grid.Col>
<Grid.Col span={6}>
<InputNumber style={{ width: '100%' }} placeholder="周" prefix="周" min={0} value={draft.keepWeekly} onChange={(value) => updateDraft({ keepWeekly: Number(value ?? 0) })} />
</Grid.Col>
<Grid.Col span={6}>
<InputNumber style={{ width: '100%' }} placeholder="月" prefix="月" min={0} value={draft.keepMonthly} onChange={(value) => updateDraft({ keepMonthly: Number(value ?? 0) })} />
</Grid.Col>
<Grid.Col span={6}>
<InputNumber style={{ width: '100%' }} placeholder="年" prefix="年" min={0} value={draft.keepYearly} onChange={(value) => updateDraft({ keepYearly: Number(value ?? 0) })} />
</Grid.Col>
</Grid.Row>
</div>
<div>
<Typography.Text></Typography.Text>
<Input

View File

@@ -21,6 +21,10 @@ export interface BackupTaskSummary {
compression: BackupCompression
encrypt: boolean
maxBackups: number
keepDaily: number
keepWeekly: number
keepMonthly: number
keepYearly: number
lastRunAt?: string
lastStatus: BackupTaskStatus
verifyEnabled: boolean
@@ -73,6 +77,10 @@ export interface BackupTaskPayload {
compression: BackupCompression
encrypt: boolean
maxBackups: number
keepDaily: number
keepWeekly: number
keepMonthly: number
keepYearly: number
/** 类型特有的扩展配置(如 SAP HANA 的 backupLevel/backupChannels 等) */
extraConfig?: Record<string, unknown>
verifyEnabled: boolean