feat(BackupX): 修复跨节点备份恢复终态处理 (#60)

* feat(BackupX): 修复集群部署管理逻辑

* feat(BackupX): 修复节点池任务运行归属

* feat(BackupX): 修复跨节点恢复路由

* feat(BackupX): 修复跨节点备份恢复终态处理

* test(BackupX): 稳定安装流HTTP测试
This commit is contained in:
Wu Qing
2026-05-09 23:03:25 +08:00
committed by GitHub
parent 61709dd4c9
commit 7a6ffd4ddd
27 changed files with 2311 additions and 247 deletions

View File

@@ -118,7 +118,8 @@ func (s *AgentService) SubmitCommandResult(ctx context.Context, node *model.Node
cmd.Result = string(result.Result)
}
cmd.CompletedAt = &now
return s.cmdRepo.Update(ctx, cmd)
_, err = s.cmdRepo.CompleteDispatched(ctx, cmd)
return err
}
// AgentTaskSpec 给 Agent 返回的任务规格,包含解密后的存储配置,供 Agent 直接执行。
@@ -159,8 +160,8 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID
if task == nil {
return nil, apperror.New(404, "BACKUP_TASK_NOT_FOUND", "任务不存在", nil)
}
if task.NodeID != node.ID {
return nil, apperror.Unauthorized("BACKUP_TASK_FORBIDDEN", "任务不属于当前节点", nil)
if err := s.ensureTaskSpecAccess(ctx, node, task); err != nil {
return nil, err
}
// 解密数据库密码(若有)
dbPassword := ""
@@ -213,6 +214,20 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID
}, nil
}
func (s *AgentService) ensureTaskSpecAccess(ctx context.Context, node *model.Node, task *model.BackupTask) error {
if task.NodeID == node.ID {
return nil
}
record, err := s.recordRepo.FindRunningByTaskAndNode(ctx, task.ID, node.ID)
if err != nil {
return err
}
if record == nil {
return apperror.Unauthorized("BACKUP_TASK_FORBIDDEN", "任务不属于当前节点", nil)
}
return nil
}
// AgentRecordUpdate Agent 上报备份记录的最终状态。
type AgentRecordUpdate struct {
Status string `json:"status"` // running | success | failed
@@ -233,14 +248,16 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
if record == nil {
return apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "记录不存在", nil)
}
// 通过 task.NodeID 判断是否属于当前 agent
task, err := s.taskRepo.FindByID(ctx, record.TaskID)
if err != nil {
return err
}
if task == nil || task.NodeID != node.ID {
if task == nil || !recordBelongsToNode(record, task, node.ID) {
return apperror.Unauthorized("BACKUP_RECORD_FORBIDDEN", "记录不属于当前节点", nil)
}
if isBackupRecordTerminal(record.Status) {
return nil
}
if update.Status != "" {
record.Status = update.Status
}
@@ -282,6 +299,17 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
return nil
}
func recordBelongsToNode(record *model.BackupRecord, task *model.BackupTask, nodeID uint) bool {
if record.NodeID != 0 {
return record.NodeID == nodeID
}
return task.NodeID == nodeID
}
func isBackupRecordTerminal(status string) bool {
return status == model.BackupRecordStatusSuccess || status == model.BackupRecordStatusFailed
}
// EnqueueCommand Master 端调用:给指定节点插入一条待执行命令。
// 返回命令 ID。
func (s *AgentService) EnqueueCommand(ctx context.Context, nodeID uint, cmdType string, payload any) (uint, error) {
@@ -356,25 +384,84 @@ func (s *AgentService) StartCommandTimeoutMonitor(ctx context.Context, interval
}()
}
// processStaleCommands 扫描已超时的 dispatched 命令并联动关联记录。
// 流程:先取超时候选 → 对每条联动 backup/restore 记录 → 把命令置为 timeout
// processStaleCommands 扫描已超时的 pending/dispatched 命令并联动关联记录。
// 流程:先取超时候选 → 条件式把命令置为 timeout → 对抢到的命令联动 backup/restore 记录。
// 单条失败不影响后续处理。
func (s *AgentService) processStaleCommands(ctx context.Context, threshold time.Time) {
commands, err := s.cmdRepo.ListStaleDispatched(ctx, threshold)
commands, err := s.cmdRepo.ListStaleActive(ctx, threshold)
if err != nil || len(commands) == 0 {
return
}
for i := range commands {
cmd := commands[i]
s.failLinkedRecord(ctx, &cmd)
if s.commandStillActive(ctx, &cmd, threshold) {
continue
}
now := time.Now().UTC()
cmd.Status = model.AgentCommandStatusTimeout
cmd.ErrorMessage = "agent did not report result before timeout"
cmd.CompletedAt = &now
_ = s.cmdRepo.Update(ctx, &cmd)
timedOut, err := s.cmdRepo.TimeoutActive(ctx, &cmd)
if err != nil || !timedOut {
continue
}
s.failLinkedRecord(ctx, &cmd)
}
}
// commandStillActive 用关联记录状态、记录更新时间和节点心跳作为长任务续租信号。
// 仅 run_task / restore_record 允许续租,避免短 RPC 命令被在线节点长期保留。
func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentCommand, threshold time.Time) bool {
if cmd.Status != model.AgentCommandStatusDispatched {
return false
}
switch cmd.Type {
case model.AgentCommandTypeRunTask:
var payload struct {
RecordID uint `json:"recordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RecordID == 0 {
return false
}
record, err := s.recordRepo.FindByID(ctx, payload.RecordID)
if err != nil || record == nil || record.Status != model.BackupRecordStatusRunning {
return false
}
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
return true
}
return record.UpdatedAt.After(threshold)
case model.AgentCommandTypeRestoreRecord:
if s.restoreRepo == nil {
return false
}
var payload struct {
RestoreRecordID uint `json:"restoreRecordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RestoreRecordID == 0 {
return false
}
restore, err := s.restoreRepo.FindByID(ctx, payload.RestoreRecordID)
if err != nil || restore == nil || restore.Status != model.RestoreRecordStatusRunning {
return false
}
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
return true
}
return restore.UpdatedAt.After(threshold)
default:
return false
}
}
func (s *AgentService) nodeRecentlySeen(ctx context.Context, nodeID uint, threshold time.Time) bool {
node, err := s.nodeRepo.FindByID(ctx, nodeID)
if err != nil || node == nil {
return false
}
return node.Status == model.NodeStatusOnline && node.LastSeen.After(threshold)
}
// failLinkedRecord 根据命令类型把关联记录标记为 failed。
// 只对仍然处于 running 状态的记录生效,避免覆盖已完成的结果。
func (s *AgentService) failLinkedRecord(ctx context.Context, cmd *model.AgentCommand) {

View File

@@ -0,0 +1,589 @@
package service
import (
"context"
"path/filepath"
"testing"
"time"
"backupx/server/internal/config"
"backupx/server/internal/database"
"backupx/server/internal/logger"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"backupx/server/internal/storage/codec"
"gorm.io/gorm"
)
func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repository.BackupRecordRepository, repository.AgentCommandRepository, *model.Node, *model.Node) {
t.Helper()
log, err := logger.New(config.LogConfig{Level: "error"})
if err != nil {
t.Fatalf("logger.New returned error: %v", err)
}
db, err := database.Open(config.DatabaseConfig{Path: filepath.Join(t.TempDir(), "backupx.db")}, log)
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
cipher := codec.NewConfigCipher("agent-service-secret")
nodeRepo := repository.NewNodeRepository(db)
taskRepo := repository.NewBackupTaskRepository(db)
recordRepo := repository.NewBackupRecordRepository(db)
storageRepo := repository.NewStorageTargetRepository(db)
cmdRepo := repository.NewAgentCommandRepository(db)
owner := &model.Node{Name: "edge-owner", Token: "owner-token", Status: model.NodeStatusOnline, IsLocal: false, LastSeen: time.Now().UTC()}
other := &model.Node{Name: "edge-other", Token: "other-token", Status: model.NodeStatusOnline, IsLocal: false, LastSeen: time.Now().UTC()}
if err := nodeRepo.Create(context.Background(), owner); err != nil {
t.Fatalf("create owner node: %v", err)
}
if err := nodeRepo.Create(context.Background(), other); err != nil {
t.Fatalf("create other node: %v", err)
}
targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()})
if err != nil {
t.Fatalf("EncryptJSON returned error: %v", err)
}
target := &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: targetConfig, ConfigVersion: 1, LastTestStatus: "unknown"}
if err := storageRepo.Create(context.Background(), target); err != nil {
t.Fatalf("create storage target: %v", err)
}
task := &model.BackupTask{
Name: "pooled-task",
Type: "file",
Enabled: true,
SourcePath: "/srv/data",
StorageTargetID: target.ID,
NodeID: 0,
NodePoolTag: "db",
RetentionDays: 30,
Compression: "gzip",
MaxBackups: 10,
LastStatus: "running",
}
if err := taskRepo.Create(context.Background(), task); err != nil {
t.Fatalf("create task: %v", err)
}
record := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: target.ID,
NodeID: owner.ID,
Status: model.BackupRecordStatusRunning,
StartedAt: time.Now().UTC(),
}
if err := recordRepo.Create(context.Background(), record); err != nil {
t.Fatalf("create record: %v", err)
}
return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher), db, recordRepo, cmdRepo, owner, other
}
func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) {
svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t)
ctx := context.Background()
spec, err := svc.GetTaskSpec(ctx, owner, 1)
if err != nil {
t.Fatalf("owner GetTaskSpec returned error: %v", err)
}
if spec.TaskID != 1 || len(spec.StorageTargets) != 1 {
t.Fatalf("unexpected spec: %#v", spec)
}
if _, err := svc.GetTaskSpec(ctx, other, 1); err == nil {
t.Fatal("expected non-owner node to be forbidden from pooled task spec")
}
if err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{
Status: model.BackupRecordStatusSuccess,
FileName: "backup.tar.gz",
FileSize: 123,
StoragePath: "tasks/1/backup.tar.gz",
}); err != nil {
t.Fatalf("owner UpdateRecord returned error: %v", err)
}
updated, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID returned error: %v", err)
}
if updated.Status != model.BackupRecordStatusSuccess || updated.NodeID != owner.ID {
t.Fatalf("unexpected updated record: %#v", updated)
}
if err := svc.UpdateRecord(ctx, other, 1, AgentRecordUpdate{LogAppend: "bad"}); err == nil {
t.Fatal("expected non-owner node to be forbidden from record update")
}
}
func TestAgentServiceProcessStaleCommandsFailsPendingRunTaskRecord(t *testing.T) {
svc, _, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusPending,
Payload: `{"recordId":1}`,
CreatedAt: time.Now().UTC().Add(-time.Hour),
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected command timeout, got %#v", updatedCommand)
}
updatedRecord, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if updatedRecord.Status != model.BackupRecordStatusFailed {
t.Fatalf("expected record failed, got %#v", updatedRecord)
}
if updatedRecord.CompletedAt == nil {
t.Fatal("expected failed record completedAt to be set")
}
}
func TestAgentServiceProcessStaleCommandsFailsPendingRestoreRecord(t *testing.T) {
svc, db, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
restoreRepo := repository.NewRestoreRecordRepository(db)
restore := &model.RestoreRecord{
BackupRecordID: 1,
TaskID: 1,
NodeID: owner.ID,
Status: model.RestoreRecordStatusRunning,
StartedAt: time.Now().UTC().Add(-time.Hour),
}
if err := restoreRepo.Create(ctx, restore); err != nil {
t.Fatalf("Create restore returned error: %v", err)
}
svc.SetRestoreRepository(restoreRepo)
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRestoreRecord,
Status: model.AgentCommandStatusPending,
Payload: `{"restoreRecordId":1}`,
CreatedAt: time.Now().UTC().Add(-time.Hour),
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected command timeout, got %#v", updatedCommand)
}
updatedRestore, err := restoreRepo.FindByID(ctx, restore.ID)
if err != nil {
t.Fatalf("FindByID restore returned error: %v", err)
}
if updatedRestore.Status != model.RestoreRecordStatusFailed {
t.Fatalf("expected restore failed, got %#v", updatedRestore)
}
if updatedRestore.CompletedAt == nil {
t.Fatal("expected failed restore completedAt to be set")
}
}
func TestAgentServiceProcessStaleCommandsKeepsActiveDispatchedRunTaskRecord(t *testing.T) {
svc, _, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC().Add(-time.Hour)
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusDispatched,
Payload: `{"recordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusDispatched {
t.Fatalf("expected active command to remain dispatched, got %#v", updatedCommand)
}
updatedRecord, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if updatedRecord.Status != model.BackupRecordStatusRunning {
t.Fatalf("expected active record to remain running, got %#v", updatedRecord)
}
}
func TestAgentServiceProcessStaleCommandsKeepsDispatchedRunTaskWhenNodeHeartbeatIsFresh(t *testing.T) {
svc, db, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := setBackupRecordUpdatedAt(db, 1, dispatchedAt); err != nil {
t.Fatalf("set backup record updated_at: %v", err)
}
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", time.Now().UTC()).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusDispatched,
Payload: `{"recordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusDispatched {
t.Fatalf("expected command to remain dispatched while node heartbeat is fresh, got %#v", updatedCommand)
}
updatedRecord, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if updatedRecord.Status != model.BackupRecordStatusRunning {
t.Fatalf("expected record to remain running while node heartbeat is fresh, got %#v", updatedRecord)
}
}
func TestAgentServiceProcessStaleCommandsTimesOutShortCommandEvenWhenNodeHeartbeatIsFresh(t *testing.T) {
svc, db, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", time.Now().UTC()).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeListDir,
Status: model.AgentCommandStatusDispatched,
Payload: `{"path":"/srv"}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected stale short command timeout, got %#v", updatedCommand)
}
}
func TestAgentServiceProcessStaleCommandsTimesOutDispatchedRunTaskWhenRecordIsTerminalEvenWithFreshHeartbeat(t *testing.T) {
svc, db, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", time.Now().UTC()).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
completedAt := time.Now().UTC().Add(-time.Minute)
record.Status = model.BackupRecordStatusFailed
record.CompletedAt = &completedAt
if err := records.Update(ctx, record); err != nil {
t.Fatalf("Update terminal record returned error: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusDispatched,
Payload: `{"recordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected command timeout when linked record is terminal, got %#v", updatedCommand)
}
}
func TestAgentServiceProcessStaleCommandsTimesOutInactiveDispatchedRunTaskRecord(t *testing.T) {
svc, db, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := setBackupRecordUpdatedAt(db, 1, dispatchedAt); err != nil {
t.Fatalf("set backup record updated_at: %v", err)
}
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", dispatchedAt).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusDispatched,
Payload: `{"recordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected inactive command timeout, got %#v", updatedCommand)
}
updatedRecord, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if updatedRecord.Status != model.BackupRecordStatusFailed {
t.Fatalf("expected inactive record failed, got %#v", updatedRecord)
}
}
func TestAgentServiceProcessStaleCommandsKeepsActiveDispatchedRestoreRecord(t *testing.T) {
svc, db, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
restoreRepo := repository.NewRestoreRecordRepository(db)
restore := createAgentServiceRestoreRecord(t, restoreRepo, owner.ID)
svc.SetRestoreRepository(restoreRepo)
dispatchedAt := time.Now().UTC().Add(-time.Hour)
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRestoreRecord,
Status: model.AgentCommandStatusDispatched,
Payload: `{"restoreRecordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusDispatched {
t.Fatalf("expected active restore command to remain dispatched, got %#v", updatedCommand)
}
updatedRestore, err := restoreRepo.FindByID(ctx, restore.ID)
if err != nil {
t.Fatalf("FindByID restore returned error: %v", err)
}
if updatedRestore.Status != model.RestoreRecordStatusRunning {
t.Fatalf("expected active restore to remain running, got %#v", updatedRestore)
}
}
func TestAgentServiceProcessStaleCommandsKeepsDispatchedRestoreWhenNodeHeartbeatIsFresh(t *testing.T) {
svc, db, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
restoreRepo := repository.NewRestoreRecordRepository(db)
restore := createAgentServiceRestoreRecord(t, restoreRepo, owner.ID)
svc.SetRestoreRepository(restoreRepo)
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := setRestoreRecordUpdatedAt(db, restore.ID, dispatchedAt); err != nil {
t.Fatalf("set restore record updated_at: %v", err)
}
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", time.Now().UTC()).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRestoreRecord,
Status: model.AgentCommandStatusDispatched,
Payload: `{"restoreRecordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusDispatched {
t.Fatalf("expected restore command to remain dispatched while node heartbeat is fresh, got %#v", updatedCommand)
}
}
func TestAgentServiceProcessStaleCommandsTimesOutInactiveDispatchedRestoreRecord(t *testing.T) {
svc, db, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
restoreRepo := repository.NewRestoreRecordRepository(db)
restore := createAgentServiceRestoreRecord(t, restoreRepo, owner.ID)
svc.SetRestoreRepository(restoreRepo)
dispatchedAt := time.Now().UTC().Add(-time.Hour)
if err := setRestoreRecordUpdatedAt(db, restore.ID, dispatchedAt); err != nil {
t.Fatalf("set restore record updated_at: %v", err)
}
if err := db.Model(&model.Node{}).Where("id = ?", owner.ID).UpdateColumn("last_seen", dispatchedAt).Error; err != nil {
t.Fatalf("set owner last_seen: %v", err)
}
oldCommand := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRestoreRecord,
Status: model.AgentCommandStatusDispatched,
Payload: `{"restoreRecordId":1}`,
CreatedAt: dispatchedAt,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, oldCommand); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
svc.processStaleCommands(ctx, time.Now().UTC().Add(-30*time.Minute))
updatedCommand, err := commands.FindByID(ctx, oldCommand.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected inactive restore command timeout, got %#v", updatedCommand)
}
updatedRestore, err := restoreRepo.FindByID(ctx, restore.ID)
if err != nil {
t.Fatalf("FindByID restore returned error: %v", err)
}
if updatedRestore.Status != model.RestoreRecordStatusFailed {
t.Fatalf("expected inactive restore failed, got %#v", updatedRestore)
}
}
func TestAgentServiceSubmitCommandResultDoesNotOverwriteTerminalCommand(t *testing.T) {
svc, _, _, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
completedAt := time.Now().UTC().Add(-time.Minute)
command := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusTimeout,
Payload: `{"recordId":1}`,
ErrorMessage: "timeout",
CompletedAt: &completedAt,
}
if err := commands.Create(ctx, command); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
if err := svc.SubmitCommandResult(ctx, owner, command.ID, AgentCommandResult{Success: true, Result: []byte(`{"ok":true}`)}); err != nil {
t.Fatalf("SubmitCommandResult returned error: %v", err)
}
updatedCommand, err := commands.FindByID(ctx, command.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusTimeout {
t.Fatalf("expected terminal command status to remain timeout, got %#v", updatedCommand)
}
if updatedCommand.Result != "" {
t.Fatalf("expected terminal command result to remain empty, got %q", updatedCommand.Result)
}
}
func TestAgentServiceUpdateRecordDoesNotOverwriteTerminalRecord(t *testing.T) {
svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
completedAt := time.Now().UTC().Add(-time.Minute)
record.Status = model.BackupRecordStatusFailed
record.ErrorMessage = "timeout"
record.CompletedAt = &completedAt
if err := records.Update(ctx, record); err != nil {
t.Fatalf("Update record returned error: %v", err)
}
if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{
Status: model.BackupRecordStatusSuccess,
FileName: "late.tar.gz",
FileSize: 42,
Checksum: "late",
StoragePath: "late/path",
ErrorMessage: "late success",
LogAppend: "late log\n",
}); err != nil {
t.Fatalf("UpdateRecord returned error: %v", err)
}
updatedRecord, err := records.FindByID(ctx, record.ID)
if err != nil {
t.Fatalf("FindByID updated record returned error: %v", err)
}
if updatedRecord.Status != model.BackupRecordStatusFailed {
t.Fatalf("expected terminal record status to remain failed, got %#v", updatedRecord)
}
if updatedRecord.FileName != "" || updatedRecord.StoragePath != "" || updatedRecord.ErrorMessage != "timeout" {
t.Fatalf("expected terminal record fields to remain unchanged, got %#v", updatedRecord)
}
}
func createAgentServiceRestoreRecord(t *testing.T, repo repository.RestoreRecordRepository, nodeID uint) *model.RestoreRecord {
t.Helper()
restore := &model.RestoreRecord{
BackupRecordID: 1,
TaskID: 1,
NodeID: nodeID,
Status: model.RestoreRecordStatusRunning,
StartedAt: time.Now().UTC().Add(-time.Hour),
}
if err := repo.Create(context.Background(), restore); err != nil {
t.Fatalf("Create restore returned error: %v", err)
}
return restore
}
func setBackupRecordUpdatedAt(db *gorm.DB, id uint, updatedAt time.Time) error {
return db.Model(&model.BackupRecord{}).Where("id = ?", id).UpdateColumn("updated_at", updatedAt).Error
}
func setRestoreRecordUpdatedAt(db *gorm.DB, id uint, updatedAt time.Time) error {
return db.Model(&model.RestoreRecord{}).Where("id = ?", id).UpdateColumn("updated_at", updatedAt).Error
}

View File

@@ -73,28 +73,28 @@ func collectTargetIDs(task *model.BackupTask) []uint {
}
type BackupExecutionService struct {
tasks repository.BackupTaskRepository
records repository.BackupRecordRepository
targets repository.StorageTargetRepository
nodeRepo repository.NodeRepository
storageRegistry *storage.Registry
runnerRegistry *backup.Registry
logHub *backup.LogHub
retention *backupretention.Service
cipher *codec.ConfigCipher
tasks repository.BackupTaskRepository
records repository.BackupRecordRepository
targets repository.StorageTargetRepository
nodeRepo repository.NodeRepository
storageRegistry *storage.Registry
runnerRegistry *backup.Registry
logHub *backup.LogHub
retention *backupretention.Service
cipher *codec.ConfigCipher
notifier BackupResultNotifier
agentDispatcher AgentDispatcher
replicationHook ReplicationTrigger
dependentsResolver DependentsResolver
async func(func())
now func() time.Time
tempDir string
semaphore chan struct{}
async func(func())
now func() time.Time
tempDir string
semaphore chan struct{}
// nodeSemaphores 节点级并发限制(按 NodeID 映射)。
// 没命中的 NodeID 走全局 semaphore节点配置 MaxConcurrent>0 时按该节点独立排队。
nodeSemaphores sync.Map
retries int // rclone 底层重试次数
bandwidthLimit string // rclone 带宽限制(全局默认,节点配置可覆盖)
retries int // rclone 底层重试次数
bandwidthLimit string // rclone 带宽限制(全局默认,节点配置可覆盖)
metrics *metrics.Metrics
}
@@ -270,11 +270,9 @@ func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint
if record == nil {
return apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "备份记录不存在", fmt.Errorf("backup record %d not found", recordID))
}
// 集群场景保护:跨节点 local_disk 文件 Master 无法远程删除,拒绝操作以避免存储泄漏的错觉
if err := s.validateClusterAccessible(ctx, record); err != nil {
if remote, err := s.deleteRemoteLocalDiskObject(ctx, record); err != nil {
return err
}
if strings.TrimSpace(record.StoragePath) != "" {
} else if !remote && strings.TrimSpace(record.StoragePath) != "" {
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
if err != nil {
return err
@@ -289,6 +287,40 @@ func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint
return nil
}
func (s *BackupExecutionService) deleteRemoteLocalDiskObject(ctx context.Context, record *model.BackupRecord) (bool, error) {
if strings.TrimSpace(record.StoragePath) == "" || s.nodeRepo == nil {
return false, nil
}
node, err := s.nodeRepo.FindByID(ctx, record.NodeID)
if err != nil || node == nil || node.IsLocal {
return false, nil
}
target, err := s.targets.FindByID(ctx, record.StorageTargetID)
if err != nil {
return false, apperror.Internal("BACKUP_STORAGE_TARGET_GET_FAILED", "无法获取存储目标详情", err)
}
if target == nil || !strings.EqualFold(target.Type, "local_disk") {
return false, nil
}
if s.agentDispatcher == nil {
return true, apperror.BadRequest("BACKUP_RECORD_CROSS_NODE_LOCAL_DISK",
fmt.Sprintf("该备份位于节点 %s 的本地磁盘local_diskMaster 无法跨节点删除。请确保 Agent 在线后再操作。", node.Name),
nil)
}
configMap := map[string]any{}
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
return true, apperror.Internal("BACKUP_STORAGE_TARGET_DECRYPT_FAILED", "无法解密存储目标配置", err)
}
if _, err := s.agentDispatcher.EnqueueCommand(ctx, record.NodeID, model.AgentCommandTypeDeleteStorageObject, map[string]any{
"targetType": target.Type,
"targetConfig": configMap,
"storagePath": record.StoragePath,
}); err != nil {
return true, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发远程备份文件删除命令", err)
}
return true, nil
}
// validateClusterAccessible 在跨节点 + local_disk 场景下拒绝 Master 端直接访问。
// 场景说明:远程 Agent 把备份写到其本机磁盘local_disk basePathMaster 的
// provider 指向的是 Master 本机的同名路径,访问会静默取错文件或 404。明确拒绝
@@ -356,8 +388,8 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
if err := s.records.Create(ctx, record); err != nil {
return nil, apperror.Internal("BACKUP_RECORD_CREATE_FAILED", "无法创建备份记录", err)
}
// 用池选出的节点 ID 复写 task 副本,使后续路由/执行沿用
task.NodeID = resolvedNodeID
runTask := *task
runTask.NodeID = resolvedNodeID
task.LastRunAt = &startedAt
task.LastStatus = "running"
if err := s.tasks.Update(ctx, task); err != nil {
@@ -365,27 +397,27 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
}
// 多节点路由task.NodeID 指向远程节点时,把执行任务入队给 Agent
// NodeID=0 或本机节点时由 Master 直接执行。
if remoteNode := s.resolveRemoteNode(ctx, task.NodeID); remoteNode != nil {
if remoteNode := s.resolveRemoteNode(ctx, resolvedNodeID); remoteNode != nil {
// 节点离线 → 立即把刚创建的 running 记录标记 failed返回明确错误
if remoteNode.Status != model.NodeStatusOnline {
offlineMsg := fmt.Sprintf("节点 %s 当前离线,无法执行备份任务", remoteNode.Name)
_ = s.finalizeRecord(ctx, task, record.ID, startedAt, model.BackupRecordStatusFailed,
offlineMsg, "", "", 0, "", "")
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
offlineMsg, "", "", 0, "", "", primaryTargetID)
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
}
if _, enqueueErr := s.agentDispatcher.EnqueueCommand(ctx, task.NodeID, model.AgentCommandTypeRunTask, map[string]any{
if _, enqueueErr := s.agentDispatcher.EnqueueCommand(ctx, resolvedNodeID, model.AgentCommandTypeRunTask, map[string]any{
"taskId": task.ID,
"recordId": record.ID,
}); enqueueErr != nil {
// 入队失败 → 在记录中标记失败,继续返回详情
_ = s.finalizeRecord(ctx, task, record.ID, startedAt, model.BackupRecordStatusFailed,
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "")
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "", primaryTargetID)
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发任务到远程节点", enqueueErr)
}
return s.getRecordDetail(ctx, record.ID)
}
run := func() {
s.executeTask(context.Background(), task, record.ID, startedAt)
s.executeTask(context.Background(), &runTask, record.ID, startedAt)
}
if async {
s.async(run)
@@ -561,9 +593,10 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
var fileSize int64
var checksum string
var storagePath string
selectedStorageTargetID := task.StorageTargetID
var uploadResults []StorageUploadResultItem
completeRecord := func() {
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath); finalizeErr != nil {
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
logger.Errorf("写回备份记录失败:%v", finalizeErr)
}
// 采集任务执行结果到 Prometheus耗时 + 产出字节 + 状态计数)
@@ -759,6 +792,9 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
for _, r := range uploadResults {
if r.Status == "success" {
anySuccess = true
if selectedStorageTargetID == task.StorageTargetID {
selectedStorageTargetID = r.StorageTargetID
}
} else if r.Error != "" {
failedMessages = append(failedMessages, fmt.Sprintf("%s: %s", r.StorageTargetName, r.Error))
}
@@ -791,7 +827,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
record := &model.BackupRecord{
ID: recordID,
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
StorageTargetID: selectedStorageTargetID,
NodeID: task.NodeID,
Status: "success",
FileName: fileName,
@@ -816,7 +852,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
}
}
func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, status string, errorMessage string, logContent string, fileName string, fileSize int64, checksum string, storagePath string) error {
func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, status string, errorMessage string, logContent string, fileName string, fileSize int64, checksum string, storagePath string, storageTargetID uint) error {
record, err := s.records.FindByID(ctx, recordID)
if err != nil {
return err
@@ -826,6 +862,9 @@ func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model
}
completedAt := s.now()
record.Status = status
if storageTargetID > 0 {
record.StorageTargetID = storageTargetID
}
record.FileName = fileName
record.FileSize = fileSize
record.Checksum = checksum
@@ -957,6 +996,9 @@ func (s *BackupExecutionService) loadRecordProvider(ctx context.Context, recordI
if record == nil {
return nil, nil, apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "备份记录不存在", fmt.Errorf("backup record %d not found", recordID))
}
if err := s.validateClusterAccessible(ctx, record); err != nil {
return nil, nil, err
}
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
if err != nil {
return nil, nil, err

View File

@@ -2,9 +2,13 @@ package service
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"backupx/server/internal/backup"
backupretention "backupx/server/internal/backup/retention"
@@ -18,6 +22,62 @@ import (
storageRclone "backupx/server/internal/storage/rclone"
)
type testStorageFactory struct {
providers map[string]*testStorageProvider
}
func (f *testStorageFactory) Type() storage.ProviderType {
return "test_storage"
}
func (f *testStorageFactory) New(_ context.Context, config map[string]any) (storage.StorageProvider, error) {
name, _ := config["name"].(string)
provider := f.providers[name]
if provider == nil {
return nil, fmt.Errorf("unknown provider %q", name)
}
return provider, nil
}
type testStorageProvider struct {
name string
failUpload bool
objects map[string][]byte
}
func (p *testStorageProvider) Type() storage.ProviderType { return "test_storage" }
func (p *testStorageProvider) TestConnection(context.Context) error {
return nil
}
func (p *testStorageProvider) Upload(_ context.Context, objectKey string, reader io.Reader, _ int64, _ map[string]string) error {
if p.failUpload {
return fmt.Errorf("upload failed for %s", p.name)
}
data, err := io.ReadAll(reader)
if err != nil {
return err
}
if p.objects == nil {
p.objects = map[string][]byte{}
}
p.objects[objectKey] = data
return nil
}
func (p *testStorageProvider) Download(_ context.Context, objectKey string) (io.ReadCloser, error) {
data, ok := p.objects[objectKey]
if !ok {
return nil, fmt.Errorf("object %s not found", objectKey)
}
return io.NopCloser(strings.NewReader(string(data))), nil
}
func (p *testStorageProvider) Delete(_ context.Context, objectKey string) error {
delete(p.objects, objectKey)
return nil
}
func (p *testStorageProvider) List(context.Context, string) ([]storage.ObjectInfo, error) {
return nil, nil
}
func newExecutionTestServices(t *testing.T) (*BackupExecutionService, *BackupRecordService, repository.BackupTaskRepository, repository.StorageTargetRepository, repository.BackupRecordRepository, string, string) {
t.Helper()
baseDir := t.TempDir()
@@ -85,6 +145,195 @@ func TestBackupExecutionServiceRunTaskByIDSync(t *testing.T) {
}
}
func TestBackupExecutionServiceNodePoolSelectionDoesNotPersistTaskNodeID(t *testing.T) {
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
nodeRepo := &nodeRepoStub{nodes: []model.Node{
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline, Labels: "prod,db"},
{ID: 11, Name: "edge-b", Token: "edge-b-token", Status: model.NodeStatusOnline, Labels: "prod,db"},
}}
dispatcher := &fakeDispatcher{}
executionService.SetClusterDependencies(nodeRepo, dispatcher)
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID returned error: %v", err)
}
task.NodeID = 0
task.NodePoolTag = "db"
if err := tasks.Update(ctx, task); err != nil {
t.Fatalf("Update task returned error: %v", err)
}
detail, err := executionService.RunTaskByID(ctx, 1)
if err != nil {
t.Fatalf("RunTaskByID returned error: %v", err)
}
storedTask, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID after run returned error: %v", err)
}
if storedTask.NodeID != 0 {
t.Fatalf("expected pooled task NodeID to remain 0, got %d", storedTask.NodeID)
}
if storedTask.NodePoolTag != "db" {
t.Fatalf("expected pooled task tag to remain db, got %q", storedTask.NodePoolTag)
}
storedRecord, err := records.FindByID(ctx, detail.ID)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if storedRecord == nil || storedRecord.NodeID != 10 {
t.Fatalf("expected record to keep selected node 10, got %#v", storedRecord)
}
calls := dispatcher.snapshot()
if len(calls) != 1 || calls[0].NodeID != 10 || calls[0].CmdType != model.AgentCommandTypeRunTask {
t.Fatalf("unexpected dispatcher calls: %#v", calls)
}
}
func TestBackupExecutionServiceDeleteRecordDispatchesRemoteLocalDiskCleanup(t *testing.T) {
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
nodeRepo := &nodeRepoStub{nodes: []model.Node{
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline},
}}
dispatcher := &fakeDispatcher{}
executionService.SetClusterDependencies(nodeRepo, dispatcher)
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
completedAt := time.Now().UTC()
record := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: 10,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: "file/2026/05/09/remote.tar.gz",
StartedAt: completedAt.Add(-time.Second),
CompletedAt: &completedAt,
}
if err := records.Create(ctx, record); err != nil {
t.Fatalf("Create record returned error: %v", err)
}
if err := executionService.DeleteRecord(ctx, record.ID); err != nil {
t.Fatalf("DeleteRecord returned error: %v", err)
}
deleted, err := records.FindByID(ctx, record.ID)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if deleted != nil {
t.Fatalf("expected record deleted, got %#v", deleted)
}
calls := dispatcher.snapshot()
if len(calls) != 1 {
t.Fatalf("expected one dispatcher call, got %#v", calls)
}
if calls[0].NodeID != 10 || calls[0].CmdType != model.AgentCommandTypeDeleteStorageObject {
t.Fatalf("unexpected dispatcher call: %#v", calls[0])
}
if calls[0].Payload["storagePath"] != record.StoragePath {
t.Fatalf("expected storagePath %q, got %#v", record.StoragePath, calls[0].Payload)
}
if calls[0].Payload["targetType"] != string(storage.ProviderTypeLocalDisk) {
t.Fatalf("expected local_disk targetType, got %#v", calls[0].Payload)
}
if _, ok := calls[0].Payload["targetConfig"].(map[string]any); !ok {
t.Fatalf("expected targetConfig map, got %#v", calls[0].Payload["targetConfig"])
}
}
func TestBackupExecutionServiceRestoreRecordRejectsRemoteLocalDisk(t *testing.T) {
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
executionService.SetClusterDependencies(&nodeRepoStub{nodes: []model.Node{
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline},
}}, &fakeDispatcher{})
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
completedAt := time.Now().UTC()
record := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: 10,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: "file/2026/05/09/remote.tar.gz",
StartedAt: completedAt.Add(-time.Second),
CompletedAt: &completedAt,
}
if err := records.Create(ctx, record); err != nil {
t.Fatalf("Create record returned error: %v", err)
}
err = executionService.RestoreRecord(ctx, record.ID)
if err == nil {
t.Fatal("expected remote local_disk restore to be rejected")
}
if !strings.Contains(err.Error(), "Master 无法跨节点访问") {
t.Fatalf("expected cross-node local_disk error, got %v", err)
}
}
func TestBackupExecutionServiceRecordsFirstSuccessfulStorageTarget(t *testing.T) {
executionService, _, tasks, targets, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
second := &testStorageProvider{name: "second", objects: map[string][]byte{}}
executionService.storageRegistry = storage.NewRegistry(&testStorageFactory{providers: map[string]*testStorageProvider{
"second": second,
}})
cipher := codec.NewConfigCipher("execution-secret")
firstConfig, err := cipher.EncryptJSON(map[string]any{"name": "missing"})
if err != nil {
t.Fatalf("EncryptJSON first returned error: %v", err)
}
secondConfig, err := cipher.EncryptJSON(map[string]any{"name": "second"})
if err != nil {
t.Fatalf("EncryptJSON second returned error: %v", err)
}
if err := targets.Create(ctx, &model.StorageTarget{Name: "first", Type: "test_storage", Enabled: true, ConfigCiphertext: firstConfig, ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
t.Fatalf("Create first target returned error: %v", err)
}
if err := targets.Create(ctx, &model.StorageTarget{Name: "second", Type: "test_storage", Enabled: true, ConfigCiphertext: secondConfig, ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
t.Fatalf("Create second target returned error: %v", err)
}
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
task.StorageTargetID = 2
task.StorageTargets = []model.StorageTarget{{ID: 2}, {ID: 3}}
if err := tasks.Update(ctx, task); err != nil {
t.Fatalf("Update task returned error: %v", err)
}
detail, err := executionService.RunTaskByIDSync(ctx, 1)
if err != nil {
t.Fatalf("RunTaskByIDSync returned error: %v", err)
}
if detail.Status != model.BackupRecordStatusSuccess {
t.Fatalf("expected success, got %#v", detail)
}
storedRecord, err := records.FindByID(ctx, detail.ID)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if storedRecord.StorageTargetID != 3 {
t.Fatalf("expected record StorageTargetID to point at successful target 3, got %d", storedRecord.StorageTargetID)
}
if _, ok := second.objects[storedRecord.StoragePath]; !ok {
t.Fatalf("expected object in successful provider at %q", storedRecord.StoragePath)
}
}
func TestBackupRecordServiceRestore(t *testing.T) {
executionService, recordService, _, _, _, sourceDir, _ := newExecutionTestServices(t)
detail, err := executionService.RunTaskByIDSync(context.Background(), 1)

View File

@@ -3,12 +3,14 @@ package service
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"fmt"
"strings"
"time"
"backupx/server/internal/apperror"
"backupx/server/internal/installscript"
"backupx/server/internal/model"
"backupx/server/internal/repository"
)
@@ -42,6 +44,25 @@ type InstallTokenOutput struct {
Record *model.AgentInstallToken
}
// InstallCommandInput 生成可展示安装命令所需的完整业务输入。
type InstallCommandInput struct {
InstallTokenInput
MasterURL string
}
// InstallCommandOutput 是 UI 生成安装命令所需的完整业务输出。
type InstallCommandOutput struct {
Token string
ExpiresAt time.Time
Node *model.Node
Record *model.AgentInstallToken
URL string
FallbackURL string
ComposeURL string
FallbackComposeURL string
ScriptBase64 string
}
// ConsumedInstallToken 消费成功后返回给 handler 的组合体。
type ConsumedInstallToken struct {
Record *model.AgentInstallToken
@@ -106,6 +127,67 @@ func (s *InstallTokenService) Create(ctx context.Context, in InstallTokenInput)
return &InstallTokenOutput{Token: token, ExpiresAt: expiresAt, Node: node, Record: record}, nil
}
// CreateCommand 创建 install token并返回 UI 展示安装命令所需的 URL 与嵌入式脚本。
func (s *InstallTokenService) CreateCommand(ctx context.Context, in InstallCommandInput) (*InstallCommandOutput, error) {
masterURL := strings.TrimRight(strings.TrimSpace(in.MasterURL), "/")
if masterURL == "" {
return nil, apperror.BadRequest("INSTALL_TOKEN_INVALID", "masterURL 必填", nil)
}
if err := s.validate(in.InstallTokenInput); err != nil {
return nil, err
}
node, err := s.nodeRepo.FindByID(ctx, in.NodeID)
if err != nil {
return nil, err
}
if node == nil {
return nil, apperror.New(404, "NODE_NOT_FOUND", "节点不存在", nil)
}
if _, err := renderInstallCommandScript(masterURL, node, &model.AgentInstallToken{
Mode: in.Mode,
Arch: in.Arch,
AgentVer: in.AgentVersion,
DownloadSrc: in.DownloadSrc,
}); err != nil {
return nil, err
}
out, err := s.Create(ctx, in.InstallTokenInput)
if err != nil {
return nil, err
}
script, err := renderInstallCommandScript(masterURL, out.Node, out.Record)
if err != nil {
return nil, err
}
result := &InstallCommandOutput{
Token: out.Token,
ExpiresAt: out.ExpiresAt,
Node: out.Node,
Record: out.Record,
URL: masterURL + "/api/install/" + out.Token,
FallbackURL: masterURL + "/install/" + out.Token,
ScriptBase64: base64.StdEncoding.EncodeToString([]byte(script)),
}
if out.Record.Mode == model.InstallModeDocker {
result.ComposeURL = masterURL + "/api/install/" + out.Token + "/compose.yml"
result.FallbackComposeURL = masterURL + "/install/" + out.Token + "/compose.yml"
}
return result, nil
}
func renderInstallCommandScript(masterURL string, node *model.Node, record *model.AgentInstallToken) (string, error) {
return installscript.RenderScript(installscript.Context{
MasterURL: masterURL,
AgentToken: node.Token,
AgentVersion: record.AgentVer,
Mode: record.Mode,
Arch: record.Arch,
DownloadBase: installscript.DownloadBaseFor(record.DownloadSrc),
InstallPrefix: "/opt/backupx-agent",
NodeID: node.ID,
})
}
// Consume 原子消费令牌。未命中/已过期/已消费均返回 (nil, nil)。
func (s *InstallTokenService) Consume(ctx context.Context, token string) (*ConsumedInstallToken, error) {
if strings.TrimSpace(token) == "" {
@@ -170,8 +252,8 @@ func (s *InstallTokenService) validate(in InstallTokenInput) error {
if !validInstallSources[in.DownloadSrc] {
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "downloadSrc 非法", nil)
}
if strings.TrimSpace(in.AgentVersion) == "" {
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "agentVersion 必填", nil)
if err := validateInstallAgentVersion(in.AgentVersion); err != nil {
return err
}
if in.TTLSeconds < InstallTokenMinTTL || in.TTLSeconds > InstallTokenMaxTTL {
return apperror.BadRequest("INSTALL_TOKEN_INVALID",
@@ -180,6 +262,27 @@ func (s *InstallTokenService) validate(in InstallTokenInput) error {
return nil
}
func validateInstallAgentVersion(v string) error {
v = strings.TrimSpace(v)
if v == "" {
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "agentVersion 必填", nil)
}
if len(v) > 64 {
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "agentVersion 不能超过 64 字符", nil)
}
for _, c := range v {
switch {
case c >= '0' && c <= '9':
case c >= 'a' && c <= 'z':
case c >= 'A' && c <= 'Z':
case c == '.' || c == '-' || c == '_' || c == '+':
default:
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "agentVersion 包含非法字符", nil)
}
}
return nil
}
func generateInstallToken() (string, error) {
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {

View File

@@ -131,6 +131,79 @@ func TestInstallTokenServiceValidatesInput(t *testing.T) {
}
}
func TestInstallTokenServiceRejectsInvalidAgentVersionBeforeCreate(t *testing.T) {
db := openInstallTokenTestDB(t)
nodeRepo := repository.NewNodeRepository(db)
node := &model.Node{Name: "invalid-version", Token: "feedface"}
if err := nodeRepo.Create(context.Background(), node); err != nil {
t.Fatalf("create node: %v", err)
}
tokenRepo := repository.NewAgentInstallTokenRepository(db)
svc := NewInstallTokenService(tokenRepo, nodeRepo)
_, err := svc.Create(context.Background(), InstallTokenInput{
NodeID: node.ID,
Mode: model.InstallModeSystemd,
Arch: model.InstallArchAuto,
AgentVersion: "v1 && rm -rf /",
DownloadSrc: model.InstallSourceGitHub,
TTLSeconds: 900,
CreatedByID: 1,
})
if err == nil {
t.Fatalf("expected invalid version error")
}
count, err := tokenRepo.CountCreatedSince(context.Background(), node.ID, time.Now().UTC().Add(-time.Hour))
if err != nil {
t.Fatalf("count: %v", err)
}
if count != 0 {
t.Fatalf("invalid request created %d token records", count)
}
}
func TestInstallTokenServiceCreateCommandBuildsURLsAndScript(t *testing.T) {
db := openInstallTokenTestDB(t)
nodeRepo := repository.NewNodeRepository(db)
node := &model.Node{
Name: "command-node",
Token: "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef",
}
if err := nodeRepo.Create(context.Background(), node); err != nil {
t.Fatalf("create node: %v", err)
}
tokenRepo := repository.NewAgentInstallTokenRepository(db)
svc := NewInstallTokenService(tokenRepo, nodeRepo)
out, err := svc.CreateCommand(context.Background(), InstallCommandInput{
InstallTokenInput: InstallTokenInput{
NodeID: node.ID,
Mode: model.InstallModeDocker,
Arch: model.InstallArchAuto,
AgentVersion: "v1.7.0",
DownloadSrc: model.InstallSourceGitHub,
TTLSeconds: 900,
CreatedByID: 1,
},
MasterURL: "https://public.example.com/base",
})
if err != nil {
t.Fatalf("create command: %v", err)
}
if out.Token == "" || out.ScriptBase64 == "" {
t.Fatalf("missing token or script: %+v", out)
}
if out.URL != "https://public.example.com/base/api/install/"+out.Token {
t.Fatalf("bad url: %s", out.URL)
}
if out.FallbackURL != "https://public.example.com/base/install/"+out.Token {
t.Fatalf("bad fallback url: %s", out.FallbackURL)
}
if out.ComposeURL != "https://public.example.com/base/api/install/"+out.Token+"/compose.yml" {
t.Fatalf("bad compose url: %s", out.ComposeURL)
}
}
func TestInstallTokenServiceRateLimit(t *testing.T) {
db := openInstallTokenTestDB(t)
nodeRepo := repository.NewNodeRepository(db)

View File

@@ -141,10 +141,11 @@ func (s *RestoreService) Start(ctx context.Context, backupRecordID uint, trigger
}
startedAt := s.now()
restoreNodeID := s.resolveRestoreNodeID(record, task)
restore := &model.RestoreRecord{
BackupRecordID: backupRecordID,
TaskID: record.TaskID,
NodeID: task.NodeID,
NodeID: restoreNodeID,
Status: model.RestoreRecordStatusRunning,
StartedAt: startedAt,
TriggeredBy: strings.TrimSpace(triggeredBy),
@@ -154,7 +155,7 @@ func (s *RestoreService) Start(ctx context.Context, backupRecordID uint, trigger
}
// 远程节点路由
if remoteNode := s.resolveRemoteNode(ctx, task.NodeID); remoteNode != nil {
if remoteNode := s.resolveRemoteNode(ctx, restoreNodeID); remoteNode != nil {
if s.dispatcher == nil {
return nil, apperror.Internal("RESTORE_DISPATCH_UNAVAILABLE", "Agent 下发通道未就绪", nil)
}
@@ -166,14 +167,14 @@ func (s *RestoreService) Start(ctx context.Context, backupRecordID uint, trigger
s.logHub.Complete(restore.ID, model.RestoreRecordStatusFailed)
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
}
if _, dispatchErr := s.dispatcher.EnqueueCommand(ctx, task.NodeID, model.AgentCommandTypeRestoreRecord, map[string]any{
if _, dispatchErr := s.dispatcher.EnqueueCommand(ctx, restoreNodeID, model.AgentCommandTypeRestoreRecord, map[string]any{
"restoreRecordId": restore.ID,
}); dispatchErr != nil {
_ = s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed,
"下发恢复任务到远程节点失败: "+dispatchErr.Error())
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发恢复任务到远程节点", dispatchErr)
}
s.logHub.Append(restore.ID, "info", fmt.Sprintf("已下发恢复任务到节点 %s#%d等待 Agent 执行", remoteNode.Name, task.NodeID))
s.logHub.Append(restore.ID, "info", fmt.Sprintf("已下发恢复任务到节点 %s#%d等待 Agent 执行", remoteNode.Name, restoreNodeID))
return s.getDetail(ctx, restore.ID)
}
@@ -185,6 +186,16 @@ func (s *RestoreService) Start(ctx context.Context, backupRecordID uint, trigger
return s.getDetail(ctx, restore.ID)
}
func (s *RestoreService) resolveRestoreNodeID(record *model.BackupRecord, task *model.BackupTask) uint {
if record != nil && record.NodeID != 0 {
return record.NodeID
}
if task != nil {
return task.NodeID
}
return 0
}
// isRemoteNode 判断 NodeID 是否指向有效的远程节点。
func (s *RestoreService) isRemoteNode(ctx context.Context, nodeID uint) bool {
return s.resolveRemoteNode(ctx, nodeID) != nil
@@ -629,6 +640,9 @@ func (s *RestoreService) UpdateAgentRestore(ctx context.Context, node *model.Nod
if restore.NodeID != node.ID {
return apperror.Unauthorized("RESTORE_RECORD_FORBIDDEN", "恢复记录不属于当前节点", nil)
}
if isRestoreRecordTerminal(restore.Status) {
return nil
}
// 追加日志到 LogHub + DB
if strings.TrimSpace(update.LogAppend) != "" {
for _, line := range strings.Split(update.LogAppend, "\n") {
@@ -667,6 +681,10 @@ func (s *RestoreService) UpdateAgentRestore(ctx context.Context, node *model.Nod
return nil
}
func isRestoreRecordTerminal(status string) bool {
return status == model.RestoreRecordStatusSuccess || status == model.RestoreRecordStatusFailed
}
// --- 内部辅助 ---
func (s *RestoreService) getDetail(ctx context.Context, restoreID uint) (*RestoreRecordDetail, error) {

View File

@@ -51,15 +51,15 @@ func (f *fakeDispatcher) snapshot() []dispatcherCall {
}
type restoreTestHarness struct {
service *RestoreService
execution *BackupExecutionService
records repository.BackupRecordRepository
restores repository.RestoreRecordRepository
tasks repository.BackupTaskRepository
nodes repository.NodeRepository
dispatcher *fakeDispatcher
sourceDir string
storageDir string
service *RestoreService
execution *BackupExecutionService
records repository.BackupRecordRepository
restores repository.RestoreRecordRepository
tasks repository.BackupTaskRepository
nodes repository.NodeRepository
dispatcher *fakeDispatcher
sourceDir string
storageDir string
}
func newRestoreTestHarness(t *testing.T, remoteNode bool) *restoreTestHarness {
@@ -228,6 +228,179 @@ func TestRestoreServiceStart_RemoteNodeEnqueuesCommand(t *testing.T) {
}
}
func TestRestoreServiceStart_UsesBackupRecordNodeForPooledTask(t *testing.T) {
h := newRestoreTestHarness(t, true)
ctx := context.Background()
task, err := h.tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task: %v", err)
}
remoteNodeID := task.NodeID
task.NodeID = 0
task.NodePoolTag = "db"
if err := h.tasks.Update(ctx, task); err != nil {
t.Fatalf("Update task: %v", err)
}
storedTask, err := h.tasks.FindByID(ctx, task.ID)
if err != nil {
t.Fatalf("FindByID stored task: %v", err)
}
if storedTask.NodeID != 0 {
t.Fatalf("expected stored task NodeID to be reset to 0, got %d", storedTask.NodeID)
}
startedAt := time.Now().UTC()
completedAt := startedAt.Add(time.Second)
backupRecord := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: remoteNodeID,
Status: model.BackupRecordStatusSuccess,
FileName: "pooled.tar.gz",
StoragePath: "file/2026/05/09/pooled.tar.gz",
StartedAt: startedAt,
CompletedAt: &completedAt,
}
if err := h.records.Create(ctx, backupRecord); err != nil {
t.Fatalf("Create backup record: %v", err)
}
detail, err := h.service.Start(ctx, backupRecord.ID, "tester-pool")
if err != nil {
t.Fatalf("Start: %v", err)
}
if detail.NodeID != remoteNodeID {
t.Fatalf("expected restore node %d, got %d", remoteNodeID, detail.NodeID)
}
calls := h.dispatcher.snapshot()
if len(calls) != 1 {
t.Fatalf("expected exactly 1 dispatcher call, got %d", len(calls))
}
if calls[0].NodeID != remoteNodeID {
t.Fatalf("expected dispatch to node %d, got %d", remoteNodeID, calls[0].NodeID)
}
}
func TestRestoreServiceAgentRestoreAccessUsesRestoreRecordNode(t *testing.T) {
h := newRestoreTestHarness(t, true)
ctx := context.Background()
task, err := h.tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task: %v", err)
}
owner, err := h.nodes.FindByID(ctx, task.NodeID)
if err != nil {
t.Fatalf("FindByID owner node: %v", err)
}
other := &model.Node{Name: "edge-2", Token: "other-token", Status: model.NodeStatusOnline, IsLocal: false, LastSeen: time.Now().UTC()}
if err := h.nodes.Create(ctx, other); err != nil {
t.Fatalf("Create other node: %v", err)
}
startedAt := time.Now().UTC()
completedAt := startedAt.Add(time.Second)
backupRecord := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: owner.ID,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: "file/2026/05/09/remote.tar.gz",
StartedAt: startedAt,
CompletedAt: &completedAt,
}
if err := h.records.Create(ctx, backupRecord); err != nil {
t.Fatalf("Create backup record: %v", err)
}
restore := &model.RestoreRecord{
BackupRecordID: backupRecord.ID,
TaskID: task.ID,
NodeID: owner.ID,
Status: model.RestoreRecordStatusRunning,
StartedAt: startedAt,
TriggeredBy: "agent-test",
}
if err := h.restores.Create(ctx, restore); err != nil {
t.Fatalf("Create restore record: %v", err)
}
spec, err := h.service.GetAgentRestoreSpec(ctx, owner, restore.ID)
if err != nil {
t.Fatalf("owner GetAgentRestoreSpec returned error: %v", err)
}
if spec.RestoreRecordID != restore.ID || spec.StoragePath != backupRecord.StoragePath {
t.Fatalf("unexpected restore spec: %#v", spec)
}
if _, err := h.service.GetAgentRestoreSpec(ctx, other, restore.ID); err == nil {
t.Fatal("expected non-owner node to be forbidden from restore spec")
}
if err := h.service.UpdateAgentRestore(ctx, owner, restore.ID, AgentRestoreUpdate{
Status: model.RestoreRecordStatusSuccess,
LogAppend: "done\n",
}); err != nil {
t.Fatalf("owner UpdateAgentRestore returned error: %v", err)
}
updated, err := h.restores.FindByID(ctx, restore.ID)
if err != nil {
t.Fatalf("FindByID restore returned error: %v", err)
}
if updated.Status != model.RestoreRecordStatusSuccess || updated.NodeID != owner.ID {
t.Fatalf("unexpected updated restore record: %#v", updated)
}
if err := h.service.UpdateAgentRestore(ctx, other, restore.ID, AgentRestoreUpdate{LogAppend: "bad\n"}); err == nil {
t.Fatal("expected non-owner node to be forbidden from restore update")
}
}
func TestRestoreServiceUpdateAgentRestoreDoesNotOverwriteTerminalRecord(t *testing.T) {
h := newRestoreTestHarness(t, true)
ctx := context.Background()
task, err := h.tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task: %v", err)
}
owner, err := h.nodes.FindByID(ctx, task.NodeID)
if err != nil {
t.Fatalf("FindByID owner node: %v", err)
}
startedAt := time.Now().UTC().Add(-time.Hour)
completedAt := time.Now().UTC().Add(-time.Minute)
restore := &model.RestoreRecord{
BackupRecordID: 1,
TaskID: task.ID,
NodeID: owner.ID,
Status: model.RestoreRecordStatusFailed,
ErrorMessage: "timeout",
StartedAt: startedAt,
CompletedAt: &completedAt,
TriggeredBy: "agent-test",
}
if err := h.restores.Create(ctx, restore); err != nil {
t.Fatalf("Create restore record: %v", err)
}
if err := h.service.UpdateAgentRestore(ctx, owner, restore.ID, AgentRestoreUpdate{
Status: model.RestoreRecordStatusSuccess,
ErrorMessage: "late success",
LogAppend: "late log\n",
}); err != nil {
t.Fatalf("UpdateAgentRestore returned error: %v", err)
}
updated, err := h.restores.FindByID(ctx, restore.ID)
if err != nil {
t.Fatalf("FindByID restore returned error: %v", err)
}
if updated.Status != model.RestoreRecordStatusFailed {
t.Fatalf("expected terminal restore status to remain failed, got %#v", updated)
}
if updated.ErrorMessage != "timeout" {
t.Fatalf("expected terminal restore error to remain unchanged, got %q", updated.ErrorMessage)
}
}
func TestRestoreServiceStart_FailsOnNonSuccessBackup(t *testing.T) {
h := newRestoreTestHarness(t, false)
ctx := context.Background()