GithubBackup/BackupX
1
0
Fork 0
mirror of https://github.com/Awuqing/BackupX.git synced 2026-09-05 23:47:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(BackupX): harden agent cluster backup workflow

Browse Source 
Squash merge PR #61
This commit is contained in:
Wu Qing
2026-05-13 14:24:45 +08:00
committed by GitHub
parent 7a6ffd4ddd
commit 7084d47c4b
30 changed files with 1360 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/metrics/collector.go
+25 -10
View File
@@ -13,16 +13,18 @@ type SampleSource interface {
ListStorageTargets(ctx context.Context) ([]model.StorageTarget, error)
StorageUsage(ctx context.Context) ([]repository.BackupStorageUsageItem, error)
ListNodes(ctx context.Context) ([]model.Node, error)
AgentQueueSummaries(ctx context.Context) (map[uint]repository.AgentCommandQueueSummary, error)
CountSLABreach(ctx context.Context) (int, error)
}
// repoSource 把 repository 适配到 SampleSource。
type repoSource struct {
targets repository.StorageTargetRepository
records repository.BackupRecordRepository
nodes repository.NodeRepository
tasks repository.BackupTaskRepository
now func() time.Time
targets repository.StorageTargetRepository
records repository.BackupRecordRepository
nodes repository.NodeRepository
tasks repository.BackupTaskRepository
commands repository.AgentCommandRepository
now func() time.Time
}
// NewRepoSource 用仓储实例构造 SampleSource。
@@ -31,13 +33,15 @@ func NewRepoSource(
records repository.BackupRecordRepository,
nodes repository.NodeRepository,
tasks repository.BackupTaskRepository,
commands repository.AgentCommandRepository,
) SampleSource {
return &repoSource{
targets: targets,
records: records,
nodes: nodes,
tasks: tasks,
now: func() time.Time { return time.Now().UTC() },
targets: targets,
records: records,
nodes: nodes,
tasks: tasks,
commands: commands,
now: func() time.Time { return time.Now().UTC() },
}
}
@@ -53,6 +57,13 @@ func (s *repoSource) ListNodes(ctx context.Context) ([]model.Node, error) {
return s.nodes.List(ctx)
}
func (s *repoSource) AgentQueueSummaries(ctx context.Context) (map[uint]repository.AgentCommandQueueSummary, error) {
if s.commands == nil {
return nil, nil
}
return s.commands.NodeQueueSummaries(ctx)
}
// CountSLABreach 统计当前违反 RPO 的任务：
// - 任务启用且配置了 SLAHoursRPO > 0
// - 最近一次成功备份距今超出 SLA 时间窗，或从未成功过
@@ -136,7 +147,9 @@ func (c *Collector) collect(ctx context.Context) {
}
// 节点在线状态：role 约定为 master / agent
if nodes, err := c.source.ListNodes(ctx); err == nil {
queueByNode, _ := c.source.AgentQueueSummaries(ctx)
c.metrics.ResetNodeOnline()
c.metrics.ResetAgentQueue()
for i := range nodes {
n := &nodes[i]
role := "agent"
@@ -144,6 +157,8 @@ func (c *Collector) collect(ctx context.Context) {
role = "master"
}
c.metrics.SetNodeOnline(n.Name, role, n.Status == model.NodeStatusOnline)
queue := queueByNode[n.ID]
c.metrics.SetAgentQueue(n.Name, role, queue.Depth, queue.Running, queue.Timeouts)
}
}
if breach, err := c.source.CountSLABreach(ctx); err == nil {
server/internal/metrics/registry.go
+39
View File
@@ -31,6 +31,12 @@ type Metrics struct {
StorageUsedBytes *prometheus.GaugeVec
// 节点在线状态（labels: node_name, role；value: 0/1）
NodeOnline *prometheus.GaugeVec
// Agent 命令队列深度（labels: node_name, role）
AgentCommandQueueDepth *prometheus.GaugeVec
// Agent 正在执行的长命令数（labels: node_name, role）
AgentCommandRunning *prometheus.GaugeVec
// Agent 命令超时累计数快照（labels: node_name, role）
AgentCommandTimeoutTotal *prometheus.GaugeVec
// 验证演练结果（labels: status）
VerifyRunTotal *prometheus.CounterVec
// 恢复操作结果（labels: status）
@@ -78,6 +84,18 @@ func New(version string) *Metrics {
Name: "backupx_node_online",
Help: "集群节点在线状态（1 在线 / 0 离线）",
}, []string{"node_name", "role"}),
AgentCommandQueueDepth: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backupx_agent_command_queue_depth",
Help: "Agent 当前 pending/dispatched 命令总数",
}, []string{"node_name", "role"}),
AgentCommandRunning: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backupx_agent_command_running",
Help: "Agent 当前正在执行的长命令数",
}, []string{"node_name", "role"}),
AgentCommandTimeoutTotal: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backupx_agent_command_timeout_total",
Help: "Agent 已超时命令数快照",
}, []string{"node_name", "role"}),
VerifyRunTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "backupx_verify_run_total",
Help: "备份验证演练执行总数",
@@ -106,6 +124,9 @@ func New(version string) *Metrics {
m.TaskRunningGauge,
m.StorageUsedBytes,
m.NodeOnline,
m.AgentCommandQueueDepth,
m.AgentCommandRunning,
m.AgentCommandTimeoutTotal,
m.VerifyRunTotal,
m.RestoreRunTotal,
m.ReplicationRunTotal,
@@ -208,6 +229,24 @@ func (m *Metrics) ResetNodeOnline() {
m.NodeOnline.Reset()
}
func (m *Metrics) SetAgentQueue(name, role string, depth, running, timeoutCount int) {
if m == nil {
return
}
m.AgentCommandQueueDepth.WithLabelValues(name, role).Set(float64(depth))
m.AgentCommandRunning.WithLabelValues(name, role).Set(float64(running))
m.AgentCommandTimeoutTotal.WithLabelValues(name, role).Set(float64(timeoutCount))
}
func (m *Metrics) ResetAgentQueue() {
if m == nil {
return
}
m.AgentCommandQueueDepth.Reset()
m.AgentCommandRunning.Reset()
m.AgentCommandTimeoutTotal.Reset()
}
// ResetStorageUsed 清空存储目标 gauge。
func (m *Metrics) ResetStorageUsed() {
if m == nil {
server/internal/metrics/registry_test.go
+6
View File
@@ -41,9 +41,11 @@ func TestObserveTaskRun_NilReceiverIsSafe(t *testing.T) {
m.DecTaskRunning()
m.SetStorageUsed("a", "s3", 1)
m.SetNodeOnline("n1", "master", true)
m.SetAgentQueue("n1", "agent", 2, 1, 3)
m.SetSLABreach(3)
m.ResetNodeOnline()
m.ResetStorageUsed()
m.ResetAgentQueue()
// no panic -> pass
}
@@ -51,6 +53,7 @@ func TestHandler_ExposesBackupxMetrics(t *testing.T) {
m := New("0.0.0-test")
m.ObserveTaskRun("file", "success", 1.0, 2048)
m.SetNodeOnline("n1", "master", true)
m.SetAgentQueue("edge-a", "agent", 3, 1, 2)
m.SetSLABreach(1)
recorder := httptest.NewRecorder()
@@ -66,6 +69,9 @@ func TestHandler_ExposesBackupxMetrics(t *testing.T) {
"backupx_task_run_total",
"backupx_task_run_duration_seconds",
"backupx_node_online",
"backupx_agent_command_queue_depth",
"backupx_agent_command_running",
"backupx_agent_command_timeout_total",
"backupx_sla_breach_tasks",
"backupx_app_info",
} {