mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-09-05 15:37:03 +08:00
refactor: simplify architecture and harden lifecycle
Remove obsolete implementations, centralize background task ownership and terminal-state recovery, consolidate frontend routing and log streaming, and enforce project-wide verification in CI.
This commit is contained in:
@@ -19,6 +19,7 @@ import (
|
||||
"backupx/server/internal/repository"
|
||||
"backupx/server/internal/storage"
|
||||
"backupx/server/internal/storage/codec"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// AgentService 实现 Master 端 Agent 协议,提供给远程 Agent 通过 HTTP 调用。
|
||||
@@ -32,6 +33,8 @@ type AgentService struct {
|
||||
restoreRepo repository.RestoreRecordRepository
|
||||
registry *storage.Registry
|
||||
cipher *codec.ConfigCipher
|
||||
logger *zap.Logger
|
||||
background BackgroundRunner
|
||||
}
|
||||
|
||||
func NewAgentService(
|
||||
@@ -51,9 +54,24 @@ func NewAgentService(
|
||||
cmdRepo: cmdRepo,
|
||||
registry: registry,
|
||||
cipher: cipher,
|
||||
logger: zap.NewNop(),
|
||||
}
|
||||
}
|
||||
|
||||
// SetLogger attaches the application logger used by background command
|
||||
// reconciliation. The no-op default keeps the service safe in tests.
|
||||
func (s *AgentService) SetLogger(logger *zap.Logger) {
|
||||
if logger != nil {
|
||||
s.logger = logger
|
||||
}
|
||||
}
|
||||
|
||||
// SetBackgroundRunner makes the command timeout monitor part of the
|
||||
// application lifecycle so shutdown waits for an in-flight reconciliation.
|
||||
func (s *AgentService) SetBackgroundRunner(background BackgroundRunner) {
|
||||
s.background = background
|
||||
}
|
||||
|
||||
// SetRestoreRepository 注入恢复记录仓储,用于命令超时时联动 restore_record 状态。
|
||||
// 可选注入:未注入时恢复命令超时仅标记命令 timeout,记录需另行查验。
|
||||
func (s *AgentService) SetRestoreRepository(repo repository.RestoreRecordRepository) {
|
||||
@@ -117,6 +135,16 @@ func (s *AgentService) SubmitCommandResult(ctx context.Context, node *model.Node
|
||||
if cmd.NodeID != node.ID {
|
||||
return apperror.Unauthorized("AGENT_COMMAND_FORBIDDEN", "命令不属于当前节点", nil)
|
||||
}
|
||||
// A failed terminal report may be retried after the command row was already
|
||||
// completed but before its linked business record was updated. Re-run that
|
||||
// idempotent convergence step so a transient database error cannot leave a
|
||||
// backup or restore record stuck in running forever.
|
||||
if cmd.Status == model.AgentCommandStatusFailed {
|
||||
if result.Success {
|
||||
return nil
|
||||
}
|
||||
return s.failLinkedRecord(ctx, cmd, agentCommandFailureMessage(result.ErrorMessage))
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if result.Success {
|
||||
cmd.Status = model.AgentCommandStatusSucceeded
|
||||
@@ -128,8 +156,21 @@ func (s *AgentService) SubmitCommandResult(ctx context.Context, node *model.Node
|
||||
cmd.Result = string(result.Result)
|
||||
}
|
||||
cmd.CompletedAt = &now
|
||||
_, err = s.cmdRepo.CompleteDispatched(ctx, cmd)
|
||||
return err
|
||||
completed, err := s.cmdRepo.CompleteDispatched(ctx, cmd)
|
||||
if err != nil || !completed || result.Success {
|
||||
return err
|
||||
}
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
defer cancel()
|
||||
return s.failLinkedRecord(persistCtx, cmd, agentCommandFailureMessage(result.ErrorMessage))
|
||||
}
|
||||
|
||||
func agentCommandFailureMessage(message string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
return "Agent 命令执行失败"
|
||||
}
|
||||
return "Agent 命令执行失败:" + message
|
||||
}
|
||||
|
||||
// AgentTaskSpec 给 Agent 返回的任务规格,包含解密后的存储配置,供 Agent 直接执行。
|
||||
@@ -616,19 +657,29 @@ func (s *AgentService) StartCommandTimeoutMonitor(ctx context.Context, interval
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Minute
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
monitor := func(runCtx context.Context) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
threshold := time.Now().UTC().Add(-timeout)
|
||||
s.processStaleCommands(ctx, threshold)
|
||||
s.processStaleCommands(runCtx, threshold)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if s.background != nil {
|
||||
if !s.background.Go(monitor) {
|
||||
s.logger.Warn("agent command timeout monitor not started: application is shutting down")
|
||||
}
|
||||
return
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
go monitor(ctx)
|
||||
}
|
||||
|
||||
// processStaleCommands 扫描已超时的 pending/dispatched 命令并联动关联记录。
|
||||
@@ -636,12 +687,24 @@ func (s *AgentService) StartCommandTimeoutMonitor(ctx context.Context, interval
|
||||
// 单条失败不影响后续处理。
|
||||
func (s *AgentService) processStaleCommands(ctx context.Context, threshold time.Time) {
|
||||
commands, err := s.cmdRepo.ListStaleActive(ctx, threshold)
|
||||
if err != nil || len(commands) == 0 {
|
||||
if err != nil {
|
||||
s.logger.Error("list stale agent commands failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if len(commands) == 0 {
|
||||
return
|
||||
}
|
||||
for i := range commands {
|
||||
cmd := commands[i]
|
||||
if s.commandStillActive(ctx, &cmd, threshold) {
|
||||
stillActive, activeErr := s.commandStillActive(ctx, &cmd, threshold)
|
||||
if activeErr != nil {
|
||||
s.logger.Warn("check stale agent command activity failed",
|
||||
zap.Uint("command_id", cmd.ID),
|
||||
zap.String("command_type", cmd.Type),
|
||||
zap.Error(activeErr))
|
||||
continue
|
||||
}
|
||||
if stillActive {
|
||||
continue
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
@@ -649,18 +712,33 @@ func (s *AgentService) processStaleCommands(ctx context.Context, threshold time.
|
||||
cmd.ErrorMessage = "agent did not report result before timeout"
|
||||
cmd.CompletedAt = &now
|
||||
timedOut, err := s.cmdRepo.TimeoutActive(ctx, &cmd)
|
||||
if err != nil || !timedOut {
|
||||
if err != nil {
|
||||
s.logger.Error("mark agent command timed out failed",
|
||||
zap.Uint("command_id", cmd.ID),
|
||||
zap.String("command_type", cmd.Type),
|
||||
zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.failLinkedRecord(ctx, &cmd)
|
||||
if !timedOut {
|
||||
continue
|
||||
}
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
failErr := s.failLinkedRecord(persistCtx, &cmd)
|
||||
cancel()
|
||||
if failErr != nil {
|
||||
s.logger.Error("mark timed-out agent command record failed",
|
||||
zap.Uint("command_id", cmd.ID),
|
||||
zap.String("command_type", cmd.Type),
|
||||
zap.Error(failErr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// commandStillActive 用关联记录状态、记录更新时间和节点心跳作为长任务续租信号。
|
||||
// 仅 run_task / restore_record 允许续租,避免短 RPC 命令被在线节点长期保留。
|
||||
func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentCommand, threshold time.Time) bool {
|
||||
func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentCommand, threshold time.Time) (bool, error) {
|
||||
if cmd.Status != model.AgentCommandStatusDispatched {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
switch cmd.Type {
|
||||
case model.AgentCommandTypeRunTask:
|
||||
@@ -668,90 +746,121 @@ func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentC
|
||||
RecordID uint `json:"recordId"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RecordID == 0 {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
record, err := s.recordRepo.FindByID(ctx, payload.RecordID)
|
||||
if err != nil || record == nil || record.Status != model.BackupRecordStatusRunning {
|
||||
return false
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("find backup record %d: %w", payload.RecordID, err)
|
||||
}
|
||||
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
|
||||
return true
|
||||
if record == nil || record.Status != model.BackupRecordStatusRunning {
|
||||
return false, nil
|
||||
}
|
||||
return record.UpdatedAt.After(threshold)
|
||||
nodeActive, err := s.nodeRecentlySeen(ctx, cmd.NodeID, threshold)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return nodeActive || record.UpdatedAt.After(threshold), nil
|
||||
case model.AgentCommandTypeRestoreRecord:
|
||||
if s.restoreRepo == nil {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
var payload struct {
|
||||
RestoreRecordID uint `json:"restoreRecordId"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RestoreRecordID == 0 {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
restore, err := s.restoreRepo.FindByID(ctx, payload.RestoreRecordID)
|
||||
if err != nil || restore == nil || restore.Status != model.RestoreRecordStatusRunning {
|
||||
return false
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("find restore record %d: %w", payload.RestoreRecordID, err)
|
||||
}
|
||||
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
|
||||
return true
|
||||
if restore == nil || restore.Status != model.RestoreRecordStatusRunning {
|
||||
return false, nil
|
||||
}
|
||||
return restore.UpdatedAt.After(threshold)
|
||||
nodeActive, err := s.nodeRecentlySeen(ctx, cmd.NodeID, threshold)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return nodeActive || restore.UpdatedAt.After(threshold), nil
|
||||
default:
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AgentService) nodeRecentlySeen(ctx context.Context, nodeID uint, threshold time.Time) bool {
|
||||
func (s *AgentService) nodeRecentlySeen(ctx context.Context, nodeID uint, threshold time.Time) (bool, error) {
|
||||
node, err := s.nodeRepo.FindByID(ctx, nodeID)
|
||||
if err != nil || node == nil {
|
||||
return false
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("find agent node %d: %w", nodeID, err)
|
||||
}
|
||||
return node.Status == model.NodeStatusOnline && node.LastSeen.After(threshold)
|
||||
if node == nil {
|
||||
return false, nil
|
||||
}
|
||||
return node.Status == model.NodeStatusOnline && node.LastSeen.After(threshold), nil
|
||||
}
|
||||
|
||||
// failLinkedRecord 根据命令类型把关联记录标记为 failed。
|
||||
// 只对仍然处于 running 状态的记录生效,避免覆盖已完成的结果。
|
||||
func (s *AgentService) failLinkedRecord(ctx context.Context, cmd *model.AgentCommand) {
|
||||
const failureMessage = "Agent 未在超时前回传状态(节点可能已离线或崩溃)"
|
||||
func (s *AgentService) failLinkedRecord(ctx context.Context, cmd *model.AgentCommand, messages ...string) error {
|
||||
failureMessage := "Agent 未在超时前回传状态(节点可能已离线或崩溃)"
|
||||
if len(messages) > 0 && strings.TrimSpace(messages[0]) != "" {
|
||||
failureMessage = strings.TrimSpace(messages[0])
|
||||
}
|
||||
switch cmd.Type {
|
||||
case model.AgentCommandTypeRunTask:
|
||||
var payload struct {
|
||||
RecordID uint `json:"recordId"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RecordID == 0 {
|
||||
return
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil {
|
||||
return fmt.Errorf("decode run_task payload: %w", err)
|
||||
}
|
||||
if payload.RecordID == 0 {
|
||||
return errors.New("run_task payload has no recordId")
|
||||
}
|
||||
record, err := s.recordRepo.FindByID(ctx, payload.RecordID)
|
||||
if err != nil || record == nil || record.Status != model.BackupRecordStatusRunning {
|
||||
return
|
||||
if err != nil {
|
||||
return fmt.Errorf("find backup record %d: %w", payload.RecordID, err)
|
||||
}
|
||||
if record == nil || record.Status != model.BackupRecordStatusRunning {
|
||||
return nil
|
||||
}
|
||||
completedAt := time.Now().UTC()
|
||||
record.Status = model.BackupRecordStatusFailed
|
||||
record.ErrorMessage = failureMessage
|
||||
record.CompletedAt = &completedAt
|
||||
record.DurationSeconds = int(completedAt.Sub(record.StartedAt).Seconds())
|
||||
_ = s.recordRepo.Update(ctx, record)
|
||||
if err := s.recordRepo.Update(ctx, record); err != nil {
|
||||
return fmt.Errorf("update backup record %d: %w", record.ID, err)
|
||||
}
|
||||
case model.AgentCommandTypeRestoreRecord:
|
||||
if s.restoreRepo == nil {
|
||||
return
|
||||
return errors.New("restore record repository is not configured")
|
||||
}
|
||||
var payload struct {
|
||||
RestoreRecordID uint `json:"restoreRecordId"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RestoreRecordID == 0 {
|
||||
return
|
||||
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil {
|
||||
return fmt.Errorf("decode restore_record payload: %w", err)
|
||||
}
|
||||
if payload.RestoreRecordID == 0 {
|
||||
return errors.New("restore_record payload has no restoreRecordId")
|
||||
}
|
||||
restore, err := s.restoreRepo.FindByID(ctx, payload.RestoreRecordID)
|
||||
if err != nil || restore == nil || restore.Status != model.RestoreRecordStatusRunning {
|
||||
return
|
||||
if err != nil {
|
||||
return fmt.Errorf("find restore record %d: %w", payload.RestoreRecordID, err)
|
||||
}
|
||||
if restore == nil || restore.Status != model.RestoreRecordStatusRunning {
|
||||
return nil
|
||||
}
|
||||
completedAt := time.Now().UTC()
|
||||
restore.Status = model.RestoreRecordStatusFailed
|
||||
restore.ErrorMessage = failureMessage
|
||||
restore.CompletedAt = &completedAt
|
||||
restore.DurationSeconds = int(completedAt.Sub(restore.StartedAt).Seconds())
|
||||
_ = s.restoreRepo.Update(ctx, restore)
|
||||
if err := s.restoreRepo.Update(ctx, restore); err != nil {
|
||||
return fmt.Errorf("update restore record %d: %w", restore.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AgentSelfStatus 是 /api/v1/agent/self 端点返回给 Agent 的轻量状态摘要。
|
||||
|
||||
@@ -24,6 +24,15 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type failingUpdateBackupRecordRepository struct {
|
||||
repository.BackupRecordRepository
|
||||
updateErr error
|
||||
}
|
||||
|
||||
func (r *failingUpdateBackupRecordRepository) Update(context.Context, *model.BackupRecord) error {
|
||||
return r.updateErr
|
||||
}
|
||||
|
||||
func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repository.BackupRecordRepository, repository.AgentCommandRepository, *model.Node, *model.Node) {
|
||||
t.Helper()
|
||||
log, err := logger.New(config.LogConfig{Level: "error"})
|
||||
@@ -34,6 +43,7 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open returned error: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
cipher := codec.NewConfigCipher("agent-service-secret")
|
||||
nodeRepo := repository.NewNodeRepository(db)
|
||||
taskRepo := repository.NewBackupTaskRepository(db)
|
||||
@@ -87,6 +97,23 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
|
||||
return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher, storageRegistry), db, recordRepo, cmdRepo, owner, other
|
||||
}
|
||||
|
||||
func TestAgentServiceFailLinkedRecordPropagatesTerminalUpdateError(t *testing.T) {
|
||||
svc, _, records, _, _, _ := newAgentServicePoolTestHarness(t)
|
||||
wantErr := errors.New("record update failed")
|
||||
svc.recordRepo = &failingUpdateBackupRecordRepository{
|
||||
BackupRecordRepository: records,
|
||||
updateErr: wantErr,
|
||||
}
|
||||
|
||||
err := svc.failLinkedRecord(context.Background(), &model.AgentCommand{
|
||||
Type: model.AgentCommandTypeRunTask,
|
||||
Payload: `{"recordId":1}`,
|
||||
})
|
||||
if err == nil || !errors.Is(err, wantErr) {
|
||||
t.Fatalf("failLinkedRecord error = %v, want wrapped update error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) {
|
||||
svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t)
|
||||
ctx := context.Background()
|
||||
@@ -752,6 +779,44 @@ func TestAgentServiceSubmitCommandResultDoesNotOverwriteTerminalCommand(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServiceSubmitFailedCommandConvergesLinkedRecord(t *testing.T) {
|
||||
svc, _, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
|
||||
ctx := context.Background()
|
||||
dispatchedAt := time.Now().UTC()
|
||||
command := &model.AgentCommand{
|
||||
NodeID: owner.ID,
|
||||
Type: model.AgentCommandTypeRunTask,
|
||||
Status: model.AgentCommandStatusDispatched,
|
||||
Payload: `{"recordId":1}`,
|
||||
DispatchedAt: &dispatchedAt,
|
||||
}
|
||||
if err := commands.Create(ctx, command); err != nil {
|
||||
t.Fatalf("Create command returned error: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.SubmitCommandResult(ctx, owner, command.ID, AgentCommandResult{
|
||||
Success: false,
|
||||
ErrorMessage: "terminal update could not reach Master",
|
||||
}); err != nil {
|
||||
t.Fatalf("SubmitCommandResult returned error: %v", err)
|
||||
}
|
||||
|
||||
record, err := records.FindByID(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("FindByID record returned error: %v", err)
|
||||
}
|
||||
if record.Status != model.BackupRecordStatusFailed || !strings.Contains(record.ErrorMessage, "terminal update") {
|
||||
t.Fatalf("linked record did not converge: %#v", record)
|
||||
}
|
||||
updatedCommand, err := commands.FindByID(ctx, command.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("FindByID command returned error: %v", err)
|
||||
}
|
||||
if updatedCommand.Status != model.AgentCommandStatusFailed {
|
||||
t.Fatalf("command status = %q, want failed", updatedCommand.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServiceUpdateRecordDoesNotOverwriteTerminalRecord(t *testing.T) {
|
||||
svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -24,6 +24,7 @@ func newApiKeyTestService(t *testing.T) *ApiKeyService {
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
return NewApiKeyService(repository.NewApiKeyRepository(db))
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,11 @@ func TestAuditRetention(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
auditRepo := repository.NewAuditLogRepository(db)
|
||||
configRepo := repository.NewSystemConfigRepository(db)
|
||||
svc := NewAuditService(auditRepo)
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -18,6 +17,7 @@ import (
|
||||
"backupx/server/internal/apperror"
|
||||
"backupx/server/internal/model"
|
||||
"backupx/server/internal/repository"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// AuditEntry 是记录审计日志的输入结构
|
||||
@@ -41,14 +41,35 @@ type AuditService struct {
|
||||
webhookURL string
|
||||
webhookSecret string
|
||||
httpClient *http.Client
|
||||
async func(func(context.Context)) bool
|
||||
logger *zap.Logger
|
||||
inFlight chan struct{}
|
||||
}
|
||||
|
||||
const maxAuditInFlight = 64
|
||||
|
||||
func NewAuditService(repo repository.AuditLogRepository) *AuditService {
|
||||
return &AuditService{
|
||||
repo: repo,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 3 * time.Second, // 短超时:审计 webhook 不应拖慢业务
|
||||
},
|
||||
async: runDetached,
|
||||
logger: zap.NewNop(),
|
||||
inFlight: make(chan struct{}, maxAuditInFlight),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AuditService) SetLogger(logger *zap.Logger) {
|
||||
if logger != nil {
|
||||
s.logger = logger
|
||||
}
|
||||
}
|
||||
|
||||
// SetBackgroundRunner binds audit persistence and webhook delivery to the application lifecycle.
|
||||
func (s *AuditService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
if runner != nil {
|
||||
s.async = runner.Go
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,37 +92,55 @@ func (s *AuditService) StartRetentionMonitor(ctx context.Context, configs reposi
|
||||
if interval <= 0 {
|
||||
interval = 6 * time.Hour
|
||||
}
|
||||
go func() {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
accepted := s.async(func(workerCtx context.Context) {
|
||||
monitorCtx, cancel := context.WithCancel(workerCtx)
|
||||
defer cancel()
|
||||
stopLink := context.AfterFunc(ctx, cancel)
|
||||
defer stopLink()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
s.runRetentionOnce(ctx, configs) // 启动后立即跑一次
|
||||
s.runRetentionOnce(monitorCtx, configs) // 启动后立即跑一次
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-monitorCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.runRetentionOnce(ctx, configs)
|
||||
s.runRetentionOnce(monitorCtx, configs)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
if !accepted {
|
||||
s.logger.Warn("audit retention monitor not started: application is shutting down")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AuditService) runRetentionOnce(ctx context.Context, configs repository.SystemConfigRepository) {
|
||||
cfg, err := configs.GetByKey(ctx, SettingKeyAuditRetentionDays)
|
||||
if err != nil || cfg == nil {
|
||||
if err != nil {
|
||||
s.logger.Warn("read audit retention setting failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
days, err := strconv.Atoi(strings.TrimSpace(cfg.Value))
|
||||
if err != nil || days <= 0 {
|
||||
if err != nil {
|
||||
s.logger.Warn("invalid audit retention setting", zap.String("value", cfg.Value), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if days <= 0 {
|
||||
return
|
||||
}
|
||||
deleted, err := s.PurgeOlderThan(ctx, days)
|
||||
if err != nil {
|
||||
log.Printf("[audit] retention purge failed: %v", err)
|
||||
s.logger.Warn("audit retention purge failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if deleted > 0 {
|
||||
log.Printf("[audit] retention purge: deleted %d logs older than %d days", deleted, days)
|
||||
s.logger.Info("audit retention purge completed", zap.Int64("deleted", deleted), zap.Int("retention_days", days))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,12 +162,24 @@ func (s *AuditService) SetWebhook(url, secret string) {
|
||||
s.webhookSecret = strings.TrimSpace(secret)
|
||||
}
|
||||
|
||||
// Record 异步 fire-and-forget 写入审计日志,不阻塞业务逻辑
|
||||
// Record asynchronously persists an audit event without blocking the request.
|
||||
func (s *AuditService) Record(entry AuditEntry) {
|
||||
if s == nil || s.repo == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
select {
|
||||
case s.inFlight <- struct{}{}:
|
||||
default:
|
||||
s.logger.Error("audit event rejected: in-flight limit reached",
|
||||
zap.Int("limit", cap(s.inFlight)),
|
||||
zap.String("category", entry.Category),
|
||||
zap.String("action", entry.Action))
|
||||
return
|
||||
}
|
||||
accepted := s.async(func(workerCtx context.Context) {
|
||||
defer func() { <-s.inFlight }()
|
||||
persistCtx, cancel := finalizationContext(workerCtx)
|
||||
defer cancel()
|
||||
record := &model.AuditLog{
|
||||
UserID: entry.UserID,
|
||||
Username: entry.Username,
|
||||
@@ -140,24 +191,30 @@ func (s *AuditService) Record(entry AuditEntry) {
|
||||
Detail: entry.Detail,
|
||||
ClientIP: entry.ClientIP,
|
||||
}
|
||||
if err := s.repo.Create(context.Background(), record); err != nil {
|
||||
log.Printf("[audit] failed to write audit log: %v", err)
|
||||
if err := s.repo.Create(persistCtx, record); err != nil {
|
||||
s.logger.Error("failed to write audit log", zap.String("category", entry.Category), zap.String("action", entry.Action), zap.Error(err))
|
||||
}
|
||||
s.fireWebhook(record)
|
||||
}()
|
||||
if err := s.fireWebhook(persistCtx, record); err != nil {
|
||||
s.logger.Warn("audit webhook delivery failed", zap.String("category", entry.Category), zap.String("action", entry.Action), zap.Error(err))
|
||||
}
|
||||
})
|
||||
if !accepted {
|
||||
<-s.inFlight
|
||||
s.logger.Warn("audit event rejected: application is shutting down", zap.String("category", entry.Category), zap.String("action", entry.Action))
|
||||
}
|
||||
}
|
||||
|
||||
// fireWebhook 异步向外部系统转发审计事件。失败降级到本地日志,永不影响主流程。
|
||||
func (s *AuditService) fireWebhook(record *model.AuditLog) {
|
||||
// fireWebhook forwards an audit event. The caller owns asynchronous execution.
|
||||
func (s *AuditService) fireWebhook(ctx context.Context, record *model.AuditLog) error {
|
||||
if s == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
s.webhookMu.RLock()
|
||||
url := s.webhookURL
|
||||
secret := s.webhookSecret
|
||||
s.webhookMu.RUnlock()
|
||||
if url == "" {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
payload := map[string]any{
|
||||
"eventType": "audit.log",
|
||||
@@ -176,13 +233,11 @@ func (s *AuditService) fireWebhook(record *model.AuditLog) {
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
log.Printf("[audit] webhook marshal failed: %v", err)
|
||||
return
|
||||
return fmt.Errorf("marshal audit webhook: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, bytes.NewReader(body))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("[audit] webhook build request failed: %v", err)
|
||||
return
|
||||
return fmt.Errorf("build audit webhook request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "BackupX-Audit/1.0")
|
||||
@@ -193,13 +248,13 @@ func (s *AuditService) fireWebhook(record *model.AuditLog) {
|
||||
}
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("[audit] webhook POST failed: %v", err)
|
||||
return
|
||||
return fmt.Errorf("post audit webhook: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
log.Printf("[audit] webhook returned status %d", resp.StatusCode)
|
||||
return fmt.Errorf("audit webhook returned status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List 分页查询审计日志
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/lifecycle"
|
||||
"backupx/server/internal/model"
|
||||
"backupx/server/internal/repository"
|
||||
)
|
||||
@@ -131,3 +132,40 @@ func TestAuditService_WebhookDisabledWhenURLEmpty(t *testing.T) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
// 无显式断言:能不 panic 即算通过
|
||||
}
|
||||
|
||||
func TestAuditServiceSupervisorShutdownFlushesAcceptedRecord(t *testing.T) {
|
||||
repo := newFakeAuditRepo()
|
||||
supervisor := lifecycle.NewSupervisor(context.Background())
|
||||
svc := NewAuditService(repo)
|
||||
svc.SetBackgroundRunner(supervisor)
|
||||
|
||||
svc.Record(AuditEntry{Username: "alice", Category: "auth", Action: "logout"})
|
||||
waitCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := supervisor.Shutdown(waitCtx); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-repo.created:
|
||||
default:
|
||||
t.Fatal("accepted audit record was not flushed during shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditServiceBoundsInFlightWork(t *testing.T) {
|
||||
repo := newFakeAuditRepo()
|
||||
svc := NewAuditService(repo)
|
||||
svc.inFlight = make(chan struct{}, 1)
|
||||
|
||||
accepted := 0
|
||||
svc.async = func(func(context.Context)) bool {
|
||||
accepted++
|
||||
return true
|
||||
}
|
||||
|
||||
svc.Record(AuditEntry{Category: "auth", Action: "first"})
|
||||
svc.Record(AuditEntry{Category: "auth", Action: "second"})
|
||||
if accepted != 1 {
|
||||
t.Fatalf("accepted tasks = %d, want 1", accepted)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type capturingMonitorRunner struct {
|
||||
tasks []func(context.Context)
|
||||
}
|
||||
|
||||
func (r *capturingMonitorRunner) Go(task func(context.Context)) bool {
|
||||
r.tasks = append(r.tasks, task)
|
||||
return true
|
||||
}
|
||||
|
||||
func TestLongRunningMonitorsUseConfiguredBackgroundRunner(t *testing.T) {
|
||||
runner := &capturingMonitorRunner{}
|
||||
|
||||
nodes := NewNodeService(nil, "test")
|
||||
nodes.SetBackgroundRunner(runner)
|
||||
nodes.StartOfflineMonitor(context.Background(), time.Hour)
|
||||
|
||||
installTokens := NewInstallTokenService(nil, nil)
|
||||
installTokens.SetBackgroundRunner(runner)
|
||||
installTokens.StartGC(context.Background(), time.Hour)
|
||||
|
||||
dashboard := NewDashboardService(nil, nil, nil)
|
||||
dashboard.SetBackgroundRunner(runner)
|
||||
dashboard.StartSLAMonitor(context.Background(), nil, time.Hour, time.Hour)
|
||||
|
||||
versions := NewClusterVersionMonitor(nil, "test")
|
||||
versions.SetBackgroundRunner(runner)
|
||||
versions.Start(context.Background(), time.Hour, time.Hour)
|
||||
|
||||
storageTargets := NewStorageTargetService(nil, nil, nil, nil)
|
||||
storageTargets.SetBackgroundRunner(runner)
|
||||
storageTargets.StartHealthMonitor(context.Background(), nil, time.Hour)
|
||||
|
||||
if len(runner.tasks) != 5 {
|
||||
t.Fatalf("background runner received %d tasks, want 5", len(runner.tasks))
|
||||
}
|
||||
|
||||
// The monitor must listen to the supervisor-provided context, not retain
|
||||
// the context passed to Start. Running one captured task is sufficient to
|
||||
// lock this ownership contract for the shared helper.
|
||||
runCtx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runner.tasks[0](runCtx)
|
||||
close(done)
|
||||
}()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("monitor did not stop when background runner context was canceled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackgroundMonitorFallbackUsesCallerContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
started := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
if !startBackgroundMonitor(nil, ctx, func(runCtx context.Context) {
|
||||
close(started)
|
||||
<-runCtx.Done()
|
||||
close(done)
|
||||
}) {
|
||||
t.Fatal("fallback monitor was rejected")
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("fallback monitor did not start")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("fallback monitor did not stop when caller context was canceled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/apperror"
|
||||
)
|
||||
|
||||
// BackgroundRunner is the narrow lifecycle dependency used by asynchronous
|
||||
// services. lifecycle.Supervisor implements it at the application boundary.
|
||||
type BackgroundRunner interface {
|
||||
Go(func(context.Context)) bool
|
||||
}
|
||||
|
||||
func runDetached(task func(context.Context)) bool {
|
||||
if task == nil {
|
||||
return false
|
||||
}
|
||||
go task(context.Background())
|
||||
return true
|
||||
}
|
||||
|
||||
// startBackgroundMonitor keeps the legacy caller-owned context when no
|
||||
// lifecycle runner is configured, while allowing the application supervisor
|
||||
// to own and wait for long-running monitors in production.
|
||||
func startBackgroundMonitor(runner BackgroundRunner, fallbackCtx context.Context, task func(context.Context)) bool {
|
||||
if task == nil {
|
||||
return false
|
||||
}
|
||||
if runner != nil {
|
||||
return runner.Go(task)
|
||||
}
|
||||
if fallbackCtx == nil {
|
||||
fallbackCtx = context.Background()
|
||||
}
|
||||
go task(fallbackCtx)
|
||||
return true
|
||||
}
|
||||
|
||||
func backgroundTaskUnavailable(code string) *apperror.AppError {
|
||||
return apperror.New(http.StatusServiceUnavailable, code, "服务正在关闭,无法启动新的后台任务", context.Canceled)
|
||||
}
|
||||
|
||||
// finalizationContext lets a canceled task persist its terminal state. It is
|
||||
// intentionally short-lived so shutdown cannot wait forever on cleanup I/O.
|
||||
func finalizationContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
}
|
||||
|
||||
func acquireBackgroundSlot(ctx context.Context, semaphore chan struct{}) bool {
|
||||
select {
|
||||
case semaphore <- struct{}{}:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -105,7 +105,7 @@ type BackupExecutionService struct {
|
||||
agentDispatcher AgentDispatcher
|
||||
replicationHook ReplicationTrigger
|
||||
dependentsResolver DependentsResolver
|
||||
async func(func())
|
||||
async func(func(context.Context)) bool
|
||||
now func() time.Time
|
||||
tempDir string
|
||||
semaphore chan struct{}
|
||||
@@ -127,6 +127,13 @@ func (s *BackupExecutionService) SetMetrics(m *metrics.Metrics) {
|
||||
s.metrics = m
|
||||
}
|
||||
|
||||
// SetBackgroundRunner binds local asynchronous executions to the application lifecycle.
|
||||
func (s *BackupExecutionService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
if runner != nil {
|
||||
s.async = runner.Go
|
||||
}
|
||||
}
|
||||
|
||||
// ReplicationTrigger 抽象备份成功后的副本派发(实现者:ReplicationService)。
|
||||
type ReplicationTrigger interface {
|
||||
TriggerAutoReplication(ctx context.Context, task *model.BackupTask, record *model.BackupRecord)
|
||||
@@ -194,14 +201,12 @@ func NewBackupExecutionService(
|
||||
retention: retention,
|
||||
cipher: cipher,
|
||||
notifier: notifier,
|
||||
async: func(job func()) {
|
||||
go job()
|
||||
},
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
retries: retries,
|
||||
bandwidthLimit: bandwidthLimit,
|
||||
async: runDetached,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
retries: retries,
|
||||
bandwidthLimit: bandwidthLimit,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,62 +264,6 @@ func (s *BackupExecutionService) DownloadRecord(ctx context.Context, recordID ui
|
||||
return &DownloadedArtifact{FileName: fileName, Reader: reader}, nil
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) RestoreRecord(ctx context.Context, recordID uint) error {
|
||||
record, provider, err := s.loadRecordProvider(ctx, recordID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := s.tasks.FindByID(ctx, record.TaskID)
|
||||
if err != nil {
|
||||
return apperror.Internal("BACKUP_TASK_GET_FAILED", "无法获取关联备份任务", err)
|
||||
}
|
||||
if task == nil {
|
||||
return apperror.New(404, "BACKUP_TASK_NOT_FOUND", "关联的备份任务不存在,无法执行恢复", fmt.Errorf("backup task %d not found", record.TaskID))
|
||||
}
|
||||
if record.BackupKind == model.BackupKindRepository {
|
||||
spec, specErr := s.buildTaskSpec(task, record.StartedAt)
|
||||
if specErr != nil {
|
||||
return specErr
|
||||
}
|
||||
if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, backup.NopLogWriter{}); err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "从 CDC 仓库恢复备份失败", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
tempDir, err := os.MkdirTemp("", "backupx-restore-*")
|
||||
if err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法创建恢复目录", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
artifactPath := filepath.Join(tempDir, filepath.Base(record.FileName))
|
||||
if strings.TrimSpace(filepath.Base(record.FileName)) == "" {
|
||||
artifactPath = filepath.Join(tempDir, filepath.Base(record.StoragePath))
|
||||
}
|
||||
reader, err := provider.Download(ctx, record.StoragePath)
|
||||
if err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法下载备份文件", err)
|
||||
}
|
||||
if err := writeReaderToFile(artifactPath, reader); err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法写入恢复文件", err)
|
||||
}
|
||||
preparedPath, err := s.prepareArtifactForRestore(artifactPath)
|
||||
if err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法准备恢复文件", err)
|
||||
}
|
||||
spec, err := s.buildTaskSpec(task, record.StartedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runner, err := s.runnerRegistry.Runner(spec.Type)
|
||||
if err != nil {
|
||||
return apperror.BadRequest("BACKUP_TASK_INVALID", "不支持的备份任务类型", err)
|
||||
}
|
||||
if err := runner.Restore(ctx, spec, preparedPath, backup.NopLogWriter{}); err != nil {
|
||||
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "恢复备份失败", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint) error {
|
||||
record, err := s.records.FindByID(ctx, recordID)
|
||||
if err != nil {
|
||||
@@ -504,6 +453,11 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
|
||||
task.LastRunAt = &startedAt
|
||||
task.LastStatus = "running"
|
||||
if err := s.tasks.Update(ctx, task); err != nil {
|
||||
finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
"无法更新任务状态: "+err.Error(), "", "", 0, "", "", primaryTargetID)
|
||||
if finalizeErr != nil {
|
||||
err = errors.Join(err, finalizeErr)
|
||||
}
|
||||
return nil, apperror.Internal("BACKUP_TASK_UPDATE_FAILED", "无法更新任务状态", err)
|
||||
}
|
||||
// 多节点路由:task.NodeID 指向远程节点时,把执行任务入队给 Agent;
|
||||
@@ -512,8 +466,10 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
|
||||
// 节点离线 → 立即把刚创建的 running 记录标记 failed,返回明确错误
|
||||
if remoteNode.Status != model.NodeStatusOnline {
|
||||
offlineMsg := fmt.Sprintf("节点 %s 当前离线,无法执行备份任务", remoteNode.Name)
|
||||
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
offlineMsg, "", "", 0, "", "", primaryTargetID)
|
||||
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
offlineMsg, "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
|
||||
return nil, apperror.Internal("BACKUP_RECORD_FINALIZE_FAILED", "无法写回备份失败状态", finalizeErr)
|
||||
}
|
||||
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
|
||||
}
|
||||
if _, enqueueErr := s.agentDispatcher.EnqueueCommand(ctx, resolvedNodeID, model.AgentCommandTypeRunTask, map[string]any{
|
||||
@@ -521,19 +477,28 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
|
||||
"recordId": record.ID,
|
||||
}); enqueueErr != nil {
|
||||
// 入队失败 → 在记录中标记失败,继续返回详情
|
||||
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "", primaryTargetID)
|
||||
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
|
||||
enqueueErr = errors.Join(enqueueErr, finalizeErr)
|
||||
}
|
||||
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发任务到远程节点", enqueueErr)
|
||||
}
|
||||
return s.getRecordDetail(ctx, record.ID)
|
||||
}
|
||||
run := func() {
|
||||
s.executeTask(context.Background(), &runTask, record.ID, startedAt)
|
||||
run := func(runCtx context.Context) {
|
||||
s.executeTask(runCtx, &runTask, record.ID, startedAt)
|
||||
}
|
||||
if async {
|
||||
s.async(run)
|
||||
if !s.async(run) {
|
||||
message := "服务正在关闭,备份任务未启动"
|
||||
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
|
||||
message, "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
|
||||
return nil, apperror.Internal("BACKUP_RECORD_FINALIZE_FAILED", "无法写回备份失败状态", finalizeErr)
|
||||
}
|
||||
return nil, backgroundTaskUnavailable("BACKUP_SERVICE_SHUTTING_DOWN")
|
||||
}
|
||||
} else {
|
||||
run()
|
||||
run(ctx)
|
||||
}
|
||||
return s.getRecordDetail(ctx, record.ID)
|
||||
}
|
||||
@@ -844,20 +809,23 @@ func (s *BackupExecutionService) executeRepositoryTask(ctx context.Context, task
|
||||
logger.Warnf("部分存储目标 CDC 仓库同步失败:%s", strings.Join(failures, "; "))
|
||||
}
|
||||
if s.dependentsResolver != nil {
|
||||
go func(upstreamID uint, upstreamName string) {
|
||||
dependents, resolveErr := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID)
|
||||
accepted := s.async(func(runCtx context.Context) {
|
||||
dependents, resolveErr := s.dependentsResolver.TriggerDependents(runCtx, task.ID)
|
||||
if resolveErr != nil {
|
||||
logger.Warnf("解析任务 %s 的下游依赖失败:%v", upstreamName, resolveErr)
|
||||
logger.Warnf("解析任务 %s 的下游依赖失败:%v", task.Name, resolveErr)
|
||||
return
|
||||
}
|
||||
for _, dependentID := range dependents {
|
||||
if _, runErr := s.RunTaskByID(context.Background(), dependentID); runErr != nil {
|
||||
logger.Warnf("触发下游任务 #%d 失败(上游: %s):%v", dependentID, upstreamName, runErr)
|
||||
if _, runErr := s.RunTaskByID(runCtx, dependentID); runErr != nil {
|
||||
logger.Warnf("触发下游任务 #%d 失败(上游: %s):%v", dependentID, task.Name, runErr)
|
||||
} else {
|
||||
logger.Infof("已触发下游任务 #%d(上游: %s)", dependentID, upstreamName)
|
||||
logger.Infof("已触发下游任务 #%d(上游: %s)", dependentID, task.Name)
|
||||
}
|
||||
}
|
||||
}(task.ID, task.Name)
|
||||
})
|
||||
if !accepted {
|
||||
logger.Warnf("服务正在关闭,跳过触发任务 %s 的下游依赖", task.Name)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -871,20 +839,6 @@ func (s *BackupExecutionService) acquireRepositoryLock(targetID uint) func() {
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) {
|
||||
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0,
|
||||
// 该节点上所有任务共享一个节点专属 semaphore,互相排队
|
||||
nodeSem := s.acquireNodeSemaphore(ctx, task.NodeID)
|
||||
if nodeSem != nil {
|
||||
nodeSem <- struct{}{}
|
||||
defer func() { <-nodeSem }()
|
||||
}
|
||||
s.semaphore <- struct{}{}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
// Prometheus: running gauge + 完成时 observe 耗时/字节/状态
|
||||
s.metrics.IncTaskRunning()
|
||||
defer s.metrics.DecTaskRunning()
|
||||
|
||||
logger := backup.NewExecutionLogger(recordID, s.logHub)
|
||||
status := "failed"
|
||||
errMessage := ""
|
||||
@@ -900,8 +854,10 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
var manifestJSON string
|
||||
var repositoryProviders map[uint]storage.StorageProvider
|
||||
completeRecord := func() {
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
defer cancel()
|
||||
readyForRepositoryRetention := status == model.BackupRecordStatusSuccess
|
||||
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
|
||||
if finalizeErr := s.finalizeRecord(persistCtx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
|
||||
logger.Errorf("写回备份记录失败:%v", finalizeErr)
|
||||
readyForRepositoryRetention = false
|
||||
}
|
||||
@@ -913,7 +869,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
if marshalErr != nil {
|
||||
logger.Warnf("序列化多目标上传结果失败:%v", marshalErr)
|
||||
readyForRepositoryRetention = false
|
||||
} else if record, findErr := s.records.FindByID(ctx, recordID); findErr != nil || record == nil {
|
||||
} else if record, findErr := s.records.FindByID(persistCtx, recordID); findErr != nil || record == nil {
|
||||
if findErr != nil {
|
||||
logger.Warnf("读取备份记录以写回多目标结果失败:%v", findErr)
|
||||
} else {
|
||||
@@ -922,7 +878,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
readyForRepositoryRetention = false
|
||||
} else {
|
||||
record.StorageUploadResults = string(resultsJSON)
|
||||
if updateErr := s.records.Update(ctx, record); updateErr != nil {
|
||||
if updateErr := s.records.Update(persistCtx, record); updateErr != nil {
|
||||
logger.Warnf("写回多目标上传结果失败:%v", updateErr)
|
||||
readyForRepositoryRetention = false
|
||||
}
|
||||
@@ -930,11 +886,11 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
// 持久化差异链信息:全量记录其清单(供后续差异比对),差异记录其基线全量 ID。
|
||||
if status == model.BackupRecordStatusSuccess && (backupKind != model.BackupKindFull || baseRecordID != 0 || manifestJSON != "") {
|
||||
if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil {
|
||||
if record, findErr := s.records.FindByID(persistCtx, recordID); findErr == nil && record != nil {
|
||||
record.BackupKind = backupKind
|
||||
record.BaseRecordID = baseRecordID
|
||||
record.Manifest = manifestJSON
|
||||
if updErr := s.records.Update(ctx, record); updErr != nil {
|
||||
if updErr := s.records.Update(persistCtx, record); updErr != nil {
|
||||
logger.Warnf("写回差异链信息失败:%v", updErr)
|
||||
readyForRepositoryRetention = false
|
||||
}
|
||||
@@ -947,7 +903,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
readyForRepositoryRetention = false
|
||||
}
|
||||
}
|
||||
if readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 {
|
||||
if ctx.Err() == nil && readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 {
|
||||
targetIDs := make([]uint, 0, len(repositoryProviders))
|
||||
for targetID := range repositoryProviders {
|
||||
targetIDs = append(targetIDs, targetID)
|
||||
@@ -969,8 +925,8 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.shouldNotify(ctx, task, status) {
|
||||
if err := s.notifier.NotifyBackupResult(ctx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
|
||||
if s.shouldNotify(persistCtx, task, status) {
|
||||
if err := s.notifier.NotifyBackupResult(persistCtx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
|
||||
logger.Warnf("发送备份通知失败:%v", err)
|
||||
}
|
||||
} else {
|
||||
@@ -980,6 +936,28 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
defer completeRecord()
|
||||
|
||||
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0,
|
||||
// 该节点上所有任务共享一个节点专属 semaphore,互相排队。
|
||||
nodeSem := s.acquireNodeSemaphore(ctx, task.NodeID)
|
||||
if nodeSem != nil {
|
||||
if !acquireBackgroundSlot(ctx, nodeSem) {
|
||||
errMessage = ctx.Err().Error()
|
||||
logger.Warnf("等待节点执行槽时任务被取消:%v", ctx.Err())
|
||||
return
|
||||
}
|
||||
defer func() { <-nodeSem }()
|
||||
}
|
||||
if !acquireBackgroundSlot(ctx, s.semaphore) {
|
||||
errMessage = ctx.Err().Error()
|
||||
logger.Warnf("等待全局执行槽时任务被取消:%v", ctx.Err())
|
||||
return
|
||||
}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
// Prometheus: running gauge + 完成时 observe 耗时/字节/状态
|
||||
s.metrics.IncTaskRunning()
|
||||
defer s.metrics.DecTaskRunning()
|
||||
|
||||
spec, err := s.buildTaskSpec(task, startedAt)
|
||||
if err != nil {
|
||||
errMessage = err.Error()
|
||||
@@ -1217,20 +1195,24 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
// 自动派发复制(3-2-1):任务配置 ReplicationTargetIDs 且本次有任意目标成功时生效
|
||||
// 触发下游依赖任务(best-effort,失败仅 warn)
|
||||
if s.dependentsResolver != nil {
|
||||
go func(upstreamID uint, upstreamName string) {
|
||||
dependents, err := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID)
|
||||
accepted := s.async(func(runCtx context.Context) {
|
||||
dependents, err := s.dependentsResolver.TriggerDependents(runCtx, task.ID)
|
||||
if err != nil {
|
||||
logger.Warnf("解析任务 %s 的下游依赖失败:%v", task.Name, err)
|
||||
return
|
||||
}
|
||||
for _, depID := range dependents {
|
||||
_, runErr := s.RunTaskByID(context.Background(), depID)
|
||||
_, runErr := s.RunTaskByID(runCtx, depID)
|
||||
if runErr != nil {
|
||||
logger.Warnf("触发下游任务 #%d 失败(上游: %s): %v", depID, upstreamName, runErr)
|
||||
logger.Warnf("触发下游任务 #%d 失败(上游: %s): %v", depID, task.Name, runErr)
|
||||
} else {
|
||||
logger.Infof("已触发下游任务 #%d(上游: %s)", depID, upstreamName)
|
||||
logger.Infof("已触发下游任务 #%d(上游: %s)", depID, task.Name)
|
||||
}
|
||||
}
|
||||
}(task.ID, task.Name)
|
||||
})
|
||||
if !accepted {
|
||||
logger.Warnf("服务正在关闭,跳过触发任务 %s 的下游依赖", task.Name)
|
||||
}
|
||||
}
|
||||
if s.replicationHook != nil && strings.TrimSpace(task.ReplicationTargetIDs) != "" {
|
||||
record := &model.BackupRecord{
|
||||
@@ -1253,7 +1235,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
}
|
||||
}
|
||||
logger.Infof("触发自动复制(3-2-1 规则):%s", task.ReplicationTargetIDs)
|
||||
s.replicationHook.TriggerAutoReplication(context.Background(), task, record)
|
||||
s.replicationHook.TriggerAutoReplication(ctx, task, record)
|
||||
}
|
||||
} else {
|
||||
errMessage = strings.Join(failedMessages, "; ")
|
||||
@@ -1356,28 +1338,6 @@ func applyHANAExtraConfig(spec *backup.DatabaseSpec, extra map[string]any) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) loadRecordProvider(ctx context.Context, recordID uint) (*model.BackupRecord, storage.StorageProvider, error) {
|
||||
record, err := s.records.FindByID(ctx, recordID)
|
||||
if err != nil {
|
||||
return nil, nil, apperror.Internal("BACKUP_RECORD_GET_FAILED", "无法获取备份记录详情", err)
|
||||
}
|
||||
if record == nil {
|
||||
return nil, nil, apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "备份记录不存在", fmt.Errorf("backup record %d not found", recordID))
|
||||
}
|
||||
if err := s.validateClusterAccessible(ctx, record); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return record, provider, nil
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) prepareArtifactForRestore(artifactPath string) (string, error) {
|
||||
return prepareBackupArtifact(s.cipher, artifactPath, nil)
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) getRecordDetail(ctx context.Context, recordID uint) (*BackupRecordDetail, error) {
|
||||
record, err := s.records.FindByID(ctx, recordID)
|
||||
if err != nil {
|
||||
@@ -1410,25 +1370,6 @@ func buildOptionalError(message string) error {
|
||||
return fmt.Errorf("%s", message)
|
||||
}
|
||||
|
||||
func buildStorageProviderFromRepos(ctx context.Context, storageTargetID uint, storageTargets repository.StorageTargetRepository, storageRegistry *storage.Registry, cipher *codec.ConfigCipher) (storage.StorageProvider, *model.StorageTarget, error) {
|
||||
target, err := storageTargets.FindByID(ctx, storageTargetID)
|
||||
if err != nil {
|
||||
return nil, nil, apperror.Internal("BACKUP_STORAGE_TARGET_LOOKUP_FAILED", "无法读取存储目标", err)
|
||||
}
|
||||
if target == nil {
|
||||
return nil, nil, apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil)
|
||||
}
|
||||
var configMap map[string]any
|
||||
if err := cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
|
||||
return nil, nil, apperror.Internal("BACKUP_STORAGE_TARGET_DECRYPT_FAILED", "无法解密存储目标配置", err)
|
||||
}
|
||||
provider, err := storageRegistry.Create(ctx, storage.ParseProviderType(target.Type), configMap)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return provider, target, nil
|
||||
}
|
||||
|
||||
// hashingReader 在上传过程中同步计算字节数和 SHA-256,零额外 I/O
|
||||
type hashingReader struct {
|
||||
reader io.Reader
|
||||
|
||||
@@ -33,6 +33,8 @@ func (f *testStorageFactory) Type() storage.ProviderType {
|
||||
return "test_storage"
|
||||
}
|
||||
|
||||
func (f *testStorageFactory) SensitiveFields() []string { return nil }
|
||||
|
||||
func (f *testStorageFactory) New(_ context.Context, config map[string]any) (storage.StorageProvider, error) {
|
||||
name, _ := config["name"].(string)
|
||||
provider := f.providers[name]
|
||||
@@ -250,20 +252,6 @@ func TestBackupExecutionServiceRepositoryModeRoundTrip(t *testing.T) {
|
||||
t.Fatalf("unexpected repository export: name=%s size=%d", download.FileName, len(exported))
|
||||
}
|
||||
|
||||
if err := os.WriteFile(largePath, bytes.Repeat([]byte{0}, len(large)), 0o640); err != nil {
|
||||
t.Fatalf("damage source before restore: %v", err)
|
||||
}
|
||||
if err := executionService.RestoreRecord(ctx, second.ID); err != nil {
|
||||
t.Fatalf("restore repository record returned error: %v", err)
|
||||
}
|
||||
restored, err := os.ReadFile(largePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read restored source: %v", err)
|
||||
}
|
||||
if !bytes.Equal(restored, large) {
|
||||
t.Fatalf("repository restore did not reproduce the source")
|
||||
}
|
||||
|
||||
if err := recordService.Delete(ctx, first.ID); err != nil {
|
||||
t.Fatalf("delete first repository record: %v", err)
|
||||
}
|
||||
@@ -428,40 +416,6 @@ func TestBackupExecutionServiceDeleteRecordDispatchesRemoteLocalDiskCleanup(t *t
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupExecutionServiceRestoreRecordRejectsRemoteLocalDisk(t *testing.T) {
|
||||
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
|
||||
ctx := context.Background()
|
||||
executionService.SetClusterDependencies(&nodeRepoStub{nodes: []model.Node{
|
||||
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline},
|
||||
}}, &fakeDispatcher{})
|
||||
task, err := tasks.FindByID(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("FindByID task returned error: %v", err)
|
||||
}
|
||||
completedAt := time.Now().UTC()
|
||||
record := &model.BackupRecord{
|
||||
TaskID: task.ID,
|
||||
StorageTargetID: task.StorageTargetID,
|
||||
NodeID: 10,
|
||||
Status: model.BackupRecordStatusSuccess,
|
||||
FileName: "remote.tar.gz",
|
||||
StoragePath: "file/2026/05/09/remote.tar.gz",
|
||||
StartedAt: completedAt.Add(-time.Second),
|
||||
CompletedAt: &completedAt,
|
||||
}
|
||||
if err := records.Create(ctx, record); err != nil {
|
||||
t.Fatalf("Create record returned error: %v", err)
|
||||
}
|
||||
|
||||
err = executionService.RestoreRecord(ctx, record.ID)
|
||||
if err == nil {
|
||||
t.Fatal("expected remote local_disk restore to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "Master 无法跨节点访问") {
|
||||
t.Fatalf("expected cross-node local_disk error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupExecutionServiceDownloadsMasterRelayedLocalDiskRecord(t *testing.T) {
|
||||
executionService, _, tasks, _, records, _, storageDir := newExecutionTestServices(t)
|
||||
ctx := context.Background()
|
||||
@@ -712,27 +666,6 @@ func TestBackupExecutionServiceContinuesWhenStorageUsageSnapshotFails(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRecordServiceRestore(t *testing.T) {
|
||||
executionService, recordService, _, _, _, sourceDir, _ := newExecutionTestServices(t)
|
||||
detail, err := executionService.RunTaskByIDSync(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTaskByIDSync returned error: %v", err)
|
||||
}
|
||||
if err := os.RemoveAll(sourceDir); err != nil {
|
||||
t.Fatalf("RemoveAll returned error: %v", err)
|
||||
}
|
||||
if err := recordService.Restore(context.Background(), detail.ID); err != nil {
|
||||
t.Fatalf("Restore returned error: %v", err)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(sourceDir, "index.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile returned error: %v", err)
|
||||
}
|
||||
if string(content) != "hello" {
|
||||
t.Fatalf("unexpected restored content: %s", string(content))
|
||||
}
|
||||
}
|
||||
|
||||
type storageUsageCountingRecordRepo struct {
|
||||
repository.BackupRecordRepository
|
||||
mu sync.Mutex
|
||||
|
||||
@@ -37,6 +37,7 @@ func newLockTestHarness(t *testing.T) (*BackupRecordService, *BackupExecutionSer
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
cipher := codec.NewConfigCipher("lock-secret")
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
|
||||
@@ -156,10 +156,6 @@ func (s *BackupRecordService) Download(ctx context.Context, id uint) (*Downloade
|
||||
return s.execution.DownloadRecord(ctx, id)
|
||||
}
|
||||
|
||||
func (s *BackupRecordService) Restore(ctx context.Context, id uint) error {
|
||||
return s.execution.RestoreRecord(ctx, id)
|
||||
}
|
||||
|
||||
func (s *BackupRecordService) Delete(ctx context.Context, id uint) error {
|
||||
return s.execution.DeleteRecord(ctx, id)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -389,7 +390,15 @@ func (s *BackupTaskService) Delete(ctx context.Context, id uint) (*DeleteResult,
|
||||
return nil, apperror.New(http.StatusNotFound, "BACKUP_TASK_NOT_FOUND", "备份任务不存在", fmt.Errorf("backup task %d not found", id))
|
||||
}
|
||||
if s.scheduler != nil {
|
||||
_ = s.scheduler.RemoveTask(ctx, id)
|
||||
if err := s.scheduler.RemoveTask(ctx, id); err != nil {
|
||||
rollbackCtx, cancel := finalizationContext(ctx)
|
||||
rollbackErr := s.scheduler.SyncTask(rollbackCtx, existing)
|
||||
cancel()
|
||||
if rollbackErr != nil {
|
||||
err = errors.Join(err, fmt.Errorf("restore task schedule: %w", rollbackErr))
|
||||
}
|
||||
return nil, apperror.Internal("BACKUP_TASK_UNSCHEDULE_FAILED", "无法移除备份任务调度", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 清理远端存储文件(尽力而为,不阻止删除)
|
||||
@@ -397,6 +406,14 @@ func (s *BackupTaskService) Delete(ctx context.Context, id uint) (*DeleteResult,
|
||||
result.RecordCount, result.CleanedFiles = s.cleanupRemoteFiles(ctx, id)
|
||||
|
||||
if err := s.tasks.Delete(ctx, id); err != nil {
|
||||
if s.scheduler != nil {
|
||||
rollbackCtx, cancel := finalizationContext(ctx)
|
||||
rollbackErr := s.scheduler.SyncTask(rollbackCtx, existing)
|
||||
cancel()
|
||||
if rollbackErr != nil {
|
||||
err = errors.Join(err, fmt.Errorf("restore task schedule: %w", rollbackErr))
|
||||
}
|
||||
}
|
||||
return nil, apperror.Internal("BACKUP_TASK_DELETE_FAILED", "无法删除备份任务", err)
|
||||
}
|
||||
return result, nil
|
||||
|
||||
@@ -2,10 +2,12 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"backupx/server/internal/apperror"
|
||||
"backupx/server/internal/config"
|
||||
"backupx/server/internal/database"
|
||||
"backupx/server/internal/logger"
|
||||
@@ -14,6 +16,34 @@ import (
|
||||
"backupx/server/internal/storage/codec"
|
||||
)
|
||||
|
||||
type backupTaskSchedulerStub struct {
|
||||
removeErr error
|
||||
syncErr error
|
||||
removedIDs []uint
|
||||
syncedTasks []model.BackupTask
|
||||
}
|
||||
|
||||
func (s *backupTaskSchedulerStub) SyncTask(_ context.Context, task *model.BackupTask) error {
|
||||
if task != nil {
|
||||
s.syncedTasks = append(s.syncedTasks, *task)
|
||||
}
|
||||
return s.syncErr
|
||||
}
|
||||
|
||||
func (s *backupTaskSchedulerStub) RemoveTask(_ context.Context, taskID uint) error {
|
||||
s.removedIDs = append(s.removedIDs, taskID)
|
||||
return s.removeErr
|
||||
}
|
||||
|
||||
type failingDeleteBackupTaskRepository struct {
|
||||
repository.BackupTaskRepository
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
func (r *failingDeleteBackupTaskRepository) Delete(context.Context, uint) error {
|
||||
return r.deleteErr
|
||||
}
|
||||
|
||||
func newBackupTaskServiceForTest(t *testing.T) (*BackupTaskService, repository.StorageTargetRepository, repository.BackupTaskRepository) {
|
||||
t.Helper()
|
||||
log, err := logger.New(config.LogConfig{Level: "error"})
|
||||
@@ -24,6 +54,7 @@ func newBackupTaskServiceForTest(t *testing.T) (*BackupTaskService, repository.S
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open returned error: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
service := NewBackupTaskService(tasks, targets, codec.NewConfigCipher("task-service-secret"))
|
||||
@@ -138,6 +169,76 @@ func TestBackupTaskServiceCreateAndGet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupTaskServiceDeleteKeepsTaskWhenUnscheduleFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
service, targets, tasks := newBackupTaskServiceForTest(t)
|
||||
if err := targets.Create(ctx, &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "ciphertext", ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created, err := service.Create(ctx, BackupTaskUpsertInput{
|
||||
Name: "unschedule-failure", Type: "file", Enabled: true, SourcePath: "/srv/data",
|
||||
StorageTargetID: 1, RetentionDays: 7, Compression: "gzip", MaxBackups: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scheduler := &backupTaskSchedulerStub{removeErr: errors.New("remove failed")}
|
||||
service.SetScheduler(scheduler)
|
||||
|
||||
if _, err := service.Delete(ctx, created.ID); err == nil {
|
||||
t.Fatal("Delete should fail when the scheduler cannot remove the task")
|
||||
} else {
|
||||
var appErr *apperror.AppError
|
||||
if !errors.As(err, &appErr) || appErr.Code != "BACKUP_TASK_UNSCHEDULE_FAILED" {
|
||||
t.Fatalf("Delete error = %#v", err)
|
||||
}
|
||||
}
|
||||
stored, err := tasks.FindByID(ctx, created.ID)
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("task should remain after unschedule failure: task=%#v err=%v", stored, err)
|
||||
}
|
||||
if len(scheduler.removedIDs) != 1 || len(scheduler.syncedTasks) != 1 {
|
||||
t.Fatalf("scheduler calls = remove:%v sync:%d", scheduler.removedIDs, len(scheduler.syncedTasks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupTaskServiceDeleteRestoresScheduleWhenPersistenceFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
service, targets, tasks := newBackupTaskServiceForTest(t)
|
||||
if err := targets.Create(ctx, &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "ciphertext", ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created, err := service.Create(ctx, BackupTaskUpsertInput{
|
||||
Name: "delete-failure", Type: "file", Enabled: true, SourcePath: "/srv/data",
|
||||
StorageTargetID: 1, RetentionDays: 7, Compression: "gzip", MaxBackups: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service.tasks = &failingDeleteBackupTaskRepository{
|
||||
BackupTaskRepository: tasks,
|
||||
deleteErr: errors.New("database delete failed"),
|
||||
}
|
||||
scheduler := &backupTaskSchedulerStub{}
|
||||
service.SetScheduler(scheduler)
|
||||
|
||||
if _, err := service.Delete(ctx, created.ID); err == nil {
|
||||
t.Fatal("Delete should fail when persistence fails")
|
||||
} else {
|
||||
var appErr *apperror.AppError
|
||||
if !errors.As(err, &appErr) || appErr.Code != "BACKUP_TASK_DELETE_FAILED" {
|
||||
t.Fatalf("Delete error = %#v", err)
|
||||
}
|
||||
}
|
||||
if len(scheduler.removedIDs) != 1 || len(scheduler.syncedTasks) != 1 || scheduler.syncedTasks[0].ID != created.ID {
|
||||
t.Fatalf("scheduler rollback calls = remove:%v sync:%#v", scheduler.removedIDs, scheduler.syncedTasks)
|
||||
}
|
||||
stored, err := tasks.FindByID(ctx, created.ID)
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("task should remain after persistence failure: task=%#v err=%v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupTaskServiceKeepsMaskedPasswordOnUpdate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
service, targets, tasks := newBackupTaskServiceForTest(t)
|
||||
|
||||
@@ -21,6 +21,7 @@ type ClusterVersionMonitor struct {
|
||||
nodeRepo repository.NodeRepository
|
||||
eventDispatcher EventDispatcher
|
||||
masterVersion string
|
||||
background BackgroundRunner
|
||||
mu sync.Mutex
|
||||
notified map[uint]time.Time
|
||||
}
|
||||
@@ -37,6 +38,10 @@ func (m *ClusterVersionMonitor) SetEventDispatcher(dispatcher EventDispatcher) {
|
||||
m.eventDispatcher = dispatcher
|
||||
}
|
||||
|
||||
func (m *ClusterVersionMonitor) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
m.background = runner
|
||||
}
|
||||
|
||||
// Start 启动后台扫描。ctx 取消时退出。
|
||||
// scanInterval 建议 30 分钟;resetInterval 建议 24 小时。
|
||||
func (m *ClusterVersionMonitor) Start(ctx context.Context, scanInterval, resetInterval time.Duration) {
|
||||
@@ -47,19 +52,19 @@ func (m *ClusterVersionMonitor) Start(ctx context.Context, scanInterval, resetIn
|
||||
resetInterval = 24 * time.Hour
|
||||
}
|
||||
// 启动立即跑一次,让控制台尽快看到
|
||||
go func() {
|
||||
m.scan(ctx, resetInterval)
|
||||
startBackgroundMonitor(m.background, ctx, func(runCtx context.Context) {
|
||||
m.scan(runCtx, resetInterval)
|
||||
ticker := time.NewTicker(scanInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.scan(ctx, resetInterval)
|
||||
m.scan(runCtx, resetInterval)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (m *ClusterVersionMonitor) scan(ctx context.Context, resetInterval time.Duration) {
|
||||
|
||||
@@ -45,6 +45,7 @@ func newDashboardNotificationTestDeps(t *testing.T) (*DashboardService, *Notific
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open returned error: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
records := repository.NewBackupRecordRepository(db)
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
|
||||
@@ -34,6 +34,7 @@ type DashboardService struct {
|
||||
targets repository.StorageTargetRepository
|
||||
nodes repository.NodeRepository
|
||||
masterVersion string
|
||||
background BackgroundRunner
|
||||
// slaMonitor 内部跟踪已告警的违约任务,避免每次扫描重复派发事件
|
||||
slaNotified map[uint]time.Time
|
||||
slaMu sync.Mutex
|
||||
@@ -43,6 +44,10 @@ func NewDashboardService(tasks repository.BackupTaskRepository, records reposito
|
||||
return &DashboardService{tasks: tasks, records: records, targets: targets, slaNotified: map[uint]time.Time{}}
|
||||
}
|
||||
|
||||
func (s *DashboardService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
s.background = runner
|
||||
}
|
||||
|
||||
// SetClusterDependencies 注入节点仓储与 Master 版本,启用集群概览。
|
||||
func (s *DashboardService) SetClusterDependencies(nodes repository.NodeRepository, masterVersion string) {
|
||||
s.nodes = nodes
|
||||
@@ -561,18 +566,18 @@ func (s *DashboardService) StartSLAMonitor(ctx context.Context, dispatcher Event
|
||||
if resetInterval <= 0 {
|
||||
resetInterval = 6 * time.Hour
|
||||
}
|
||||
ticker := time.NewTicker(scanInterval)
|
||||
go func() {
|
||||
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
|
||||
ticker := time.NewTicker(scanInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.scanAndDispatchSLA(ctx, dispatcher, resetInterval)
|
||||
s.scanAndDispatchSLA(runCtx, dispatcher, resetInterval)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// scanAndDispatchSLA 执行一次 SLA 违约扫描并按需派发事件。
|
||||
|
||||
@@ -27,6 +27,7 @@ func TestGoogleDriveOAuthServiceStartAndComplete(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open returned error: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
sessions := repository.NewOAuthSessionRepository(db)
|
||||
service := NewGoogleDriveOAuthService(sessions, codec.New("encryption-secret"))
|
||||
service.now = func() time.Time { return time.Date(2026, 3, 7, 0, 0, 0, 0, time.UTC) }
|
||||
|
||||
@@ -18,14 +18,19 @@ import (
|
||||
|
||||
// InstallTokenService 负责一次性安装令牌的创建/消费/校验。
|
||||
type InstallTokenService struct {
|
||||
repo repository.AgentInstallTokenRepository
|
||||
nodeRepo repository.NodeRepository
|
||||
repo repository.AgentInstallTokenRepository
|
||||
nodeRepo repository.NodeRepository
|
||||
background BackgroundRunner
|
||||
}
|
||||
|
||||
func NewInstallTokenService(repo repository.AgentInstallTokenRepository, nodeRepo repository.NodeRepository) *InstallTokenService {
|
||||
return &InstallTokenService{repo: repo, nodeRepo: nodeRepo}
|
||||
}
|
||||
|
||||
func (s *InstallTokenService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
s.background = runner
|
||||
}
|
||||
|
||||
// InstallTokenInput 生成一次性安装令牌的输入。
|
||||
type InstallTokenInput struct {
|
||||
NodeID uint
|
||||
@@ -247,18 +252,18 @@ func (s *InstallTokenService) StartGC(ctx context.Context, interval time.Duratio
|
||||
if interval <= 0 {
|
||||
interval = time.Hour
|
||||
}
|
||||
go func() {
|
||||
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
_, _ = s.repo.DeleteExpiredBefore(ctx, time.Now().UTC().Add(-7*24*time.Hour))
|
||||
_, _ = s.repo.DeleteExpiredBefore(runCtx, time.Now().UTC().Add(-7*24*time.Hour))
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *InstallTokenService) validate(in InstallTokenInput) error {
|
||||
|
||||
@@ -22,6 +22,7 @@ func openInstallTokenTestDB(t *testing.T) *gorm.DB {
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
if err := db.AutoMigrate(&model.AgentInstallToken{}, &model.Node{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
@@ -67,11 +67,12 @@ type NodeUpdateInput struct {
|
||||
|
||||
// NodeService manages the cluster nodes.
|
||||
type NodeService struct {
|
||||
repo repository.NodeRepository
|
||||
taskRepo repository.BackupTaskRepository
|
||||
agentRPC NodeAgentRPC
|
||||
cmdRepo repository.AgentCommandRepository
|
||||
version string
|
||||
repo repository.NodeRepository
|
||||
taskRepo repository.BackupTaskRepository
|
||||
agentRPC NodeAgentRPC
|
||||
cmdRepo repository.AgentCommandRepository
|
||||
version string
|
||||
background BackgroundRunner
|
||||
}
|
||||
|
||||
// NodeAgentRPC 抽象 Agent 远程调用能力(避免 service 内循环依赖)。
|
||||
@@ -85,6 +86,10 @@ func NewNodeService(repo repository.NodeRepository, version string) *NodeService
|
||||
return &NodeService{repo: repo, version: version}
|
||||
}
|
||||
|
||||
func (s *NodeService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
s.background = runner
|
||||
}
|
||||
|
||||
// SetTaskRepository 注入任务仓储以支持删除前引用检查。可选注入,便于测试。
|
||||
func (s *NodeService) SetTaskRepository(taskRepo repository.BackupTaskRepository) {
|
||||
s.taskRepo = taskRepo
|
||||
@@ -315,19 +320,19 @@ func (s *NodeService) StartOfflineMonitor(ctx context.Context, interval time.Dur
|
||||
if interval <= 0 {
|
||||
interval = 15 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
threshold := time.Now().UTC().Add(-OfflineThreshold)
|
||||
_, _ = s.repo.MarkStaleOffline(ctx, threshold)
|
||||
_, _ = s.repo.MarkStaleOffline(runCtx, threshold)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// Heartbeat updates the node status when an agent reports in.
|
||||
|
||||
@@ -20,6 +20,7 @@ func openNodeServiceDB(t *testing.T) *gorm.DB {
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
if err := db.AutoMigrate(&model.Node{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ func (s *NotificationService) NotifyBackupResult(ctx context.Context, event Back
|
||||
if success {
|
||||
eventType = model.NotificationEventBackupSuccess
|
||||
}
|
||||
items, err := s.collectSubscribers(ctx, eventType, success)
|
||||
items, err := s.collectSubscribers(ctx, eventType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -194,9 +194,7 @@ func (s *NotificationService) DispatchEvent(ctx context.Context, eventType strin
|
||||
if s.broadcaster != nil {
|
||||
_ = s.broadcaster.Publish(ctx, eventType, title, body, fields)
|
||||
}
|
||||
// 将 fallback 布尔用于旧语义场景(backup_success / backup_failed)。
|
||||
fallbackSuccess := eventType == model.NotificationEventBackupSuccess
|
||||
items, err := s.collectSubscribers(ctx, eventType, fallbackSuccess)
|
||||
items, err := s.collectSubscribers(ctx, eventType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -254,7 +252,7 @@ func (s *NotificationService) sendFirstByType(ctx context.Context, notificationT
|
||||
|
||||
// collectSubscribers 按事件类型收集启用的订阅者。
|
||||
// 列出启用通知后按事件类型再过滤(避免引入新 repository 方法)。
|
||||
func (s *NotificationService) collectSubscribers(ctx context.Context, eventType string, fallbackSuccess bool) ([]model.Notification, error) {
|
||||
func (s *NotificationService) collectSubscribers(ctx context.Context, eventType string) ([]model.Notification, error) {
|
||||
all, err := s.notifications.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -284,8 +282,6 @@ func (s *NotificationService) collectSubscribers(ctx context.Context, eventType
|
||||
// 其他事件类型必须显式订阅才推送
|
||||
continue
|
||||
}
|
||||
// 额外校验 fallbackSuccess 参数,保持历史行为一致
|
||||
_ = fallbackSuccess
|
||||
}
|
||||
matched = append(matched, item)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"backupx/server/internal/repository"
|
||||
"backupx/server/internal/storage"
|
||||
"backupx/server/internal/storage/codec"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// ReplicationService 实现备份复制(3-2-1 规则核心)。
|
||||
@@ -36,9 +37,10 @@ type ReplicationService struct {
|
||||
eventDispatcher EventDispatcher
|
||||
tempDir string
|
||||
semaphore chan struct{}
|
||||
async func(func())
|
||||
async func(func(context.Context)) bool
|
||||
now func() time.Time
|
||||
metrics *metrics.Metrics
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// SetMetrics 注入 Prometheus 采集器。
|
||||
@@ -46,6 +48,19 @@ func (s *ReplicationService) SetMetrics(m *metrics.Metrics) {
|
||||
s.metrics = m
|
||||
}
|
||||
|
||||
func (s *ReplicationService) SetLogger(logger *zap.Logger) {
|
||||
if logger != nil {
|
||||
s.logger = logger
|
||||
}
|
||||
}
|
||||
|
||||
// SetBackgroundRunner binds replication work to the application lifecycle.
|
||||
func (s *ReplicationService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
if runner != nil {
|
||||
s.async = runner.Go
|
||||
}
|
||||
}
|
||||
|
||||
func NewReplicationService(
|
||||
replications repository.ReplicationRecordRepository,
|
||||
records repository.BackupRecordRepository,
|
||||
@@ -71,8 +86,9 @@ func NewReplicationService(
|
||||
cipher: cipher,
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
async: func(job func()) { go job() },
|
||||
async: runDetached,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
logger: zap.NewNop(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,13 +139,16 @@ func (s *ReplicationService) TriggerAutoReplication(ctx context.Context, task *m
|
||||
}
|
||||
// 跨节点 local_disk 场景保护:Master 无法访问远程节点本地文件
|
||||
if err := s.validateClusterAccessible(ctx, record); err != nil {
|
||||
s.logger.Warn("automatic replication skipped: source is not accessible", zap.Uint("backup_record_id", record.ID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, destID := range destIDs {
|
||||
if destID == record.StorageTargetID {
|
||||
continue // 源与目标相同,跳过
|
||||
}
|
||||
_, _ = s.Start(ctx, record.ID, destID, "system")
|
||||
if _, err := s.Start(ctx, record.ID, destID, "system"); err != nil {
|
||||
s.logger.Warn("automatic replication start failed", zap.Uint("backup_record_id", record.ID), zap.Uint("dest_target_id", destID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,20 +191,24 @@ func (s *ReplicationService) Start(ctx context.Context, backupRecordID, destTarg
|
||||
if err := s.replications.Create(ctx, rep); err != nil {
|
||||
return nil, apperror.Internal("REPLICATION_CREATE_FAILED", "无法创建复制记录", err)
|
||||
}
|
||||
s.async(func() {
|
||||
s.executeReplication(context.Background(), rep.ID)
|
||||
})
|
||||
repForRun := *rep
|
||||
if !s.async(func(runCtx context.Context) {
|
||||
s.executeReplication(runCtx, &repForRun)
|
||||
}) {
|
||||
message := "服务正在关闭,复制任务未启动"
|
||||
if finalizeErr := s.finalizeReplication(ctx, rep, model.ReplicationStatusFailed, message, 0); finalizeErr != nil {
|
||||
return nil, apperror.Internal("REPLICATION_FINALIZE_FAILED", "无法写回复制失败状态", finalizeErr)
|
||||
}
|
||||
return nil, backgroundTaskUnavailable("REPLICATION_SERVICE_SHUTTING_DOWN")
|
||||
}
|
||||
summary := s.toSummary(rep, "", dest.Name)
|
||||
return &summary, nil
|
||||
}
|
||||
|
||||
// executeReplication 实际执行:下载源对象到本地临时文件 → 上传到目标存储。
|
||||
func (s *ReplicationService) executeReplication(ctx context.Context, repID uint) {
|
||||
s.semaphore <- struct{}{}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
rep, err := s.replications.FindByID(ctx, repID)
|
||||
if err != nil || rep == nil {
|
||||
func (s *ReplicationService) executeReplication(ctx context.Context, rep *model.ReplicationRecord) {
|
||||
if rep == nil {
|
||||
s.logger.Error("replication record is nil")
|
||||
return
|
||||
}
|
||||
status := model.ReplicationStatusFailed
|
||||
@@ -193,19 +216,25 @@ func (s *ReplicationService) executeReplication(ctx context.Context, repID uint)
|
||||
fileSize := int64(0)
|
||||
|
||||
defer func() {
|
||||
completedAt := s.now()
|
||||
rep.Status = status
|
||||
rep.FileSize = fileSize
|
||||
rep.ErrorMessage = strings.TrimSpace(errMessage)
|
||||
rep.DurationSeconds = int(completedAt.Sub(rep.StartedAt).Seconds())
|
||||
rep.CompletedAt = &completedAt
|
||||
_ = s.replications.Update(ctx, rep)
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
defer cancel()
|
||||
if finalizeErr := s.finalizeReplication(persistCtx, rep, status, errMessage, fileSize); finalizeErr != nil {
|
||||
s.logger.Error("finalize replication record failed", zap.Uint("replication_id", rep.ID), zap.Error(finalizeErr))
|
||||
}
|
||||
s.metrics.ObserveReplication(status)
|
||||
if status == model.ReplicationStatusFailed {
|
||||
s.dispatchFailed(ctx, rep, errMessage)
|
||||
if dispatchErr := s.dispatchFailed(persistCtx, rep, errMessage); dispatchErr != nil {
|
||||
s.logger.Warn("dispatch replication failure event failed", zap.Uint("replication_id", rep.ID), zap.Error(dispatchErr))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if !acquireBackgroundSlot(ctx, s.semaphore) {
|
||||
errMessage = ctx.Err().Error()
|
||||
return
|
||||
}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
sourceProvider, err := s.resolveProvider(ctx, rep.SourceTargetID)
|
||||
if err != nil {
|
||||
errMessage = err.Error()
|
||||
@@ -271,9 +300,19 @@ func (s *ReplicationService) validateClusterAccessible(ctx context.Context, reco
|
||||
"REPLICATION_CROSS_NODE_LOCAL_DISK", "复制。请改用云存储作为主备份")
|
||||
}
|
||||
|
||||
func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.ReplicationRecord, message string) {
|
||||
func (s *ReplicationService) finalizeReplication(ctx context.Context, rep *model.ReplicationRecord, status, message string, fileSize int64) error {
|
||||
completedAt := s.now()
|
||||
rep.Status = status
|
||||
rep.FileSize = fileSize
|
||||
rep.ErrorMessage = strings.TrimSpace(message)
|
||||
rep.DurationSeconds = int(completedAt.Sub(rep.StartedAt).Seconds())
|
||||
rep.CompletedAt = &completedAt
|
||||
return s.replications.Update(ctx, rep)
|
||||
}
|
||||
|
||||
func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.ReplicationRecord, message string) error {
|
||||
if s.eventDispatcher == nil || rep == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
title := "BackupX 备份复制失败"
|
||||
body := fmt.Sprintf("备份记录:#%d\n源 → 目标:#%d → #%d\n错误:%s", rep.BackupRecordID, rep.SourceTargetID, rep.DestTargetID, message)
|
||||
@@ -285,7 +324,7 @@ func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.Repl
|
||||
"destTargetId": rep.DestTargetID,
|
||||
"error": message,
|
||||
}
|
||||
_ = s.eventDispatcher.DispatchEvent(ctx, model.NotificationEventReplicationFailed, title, body, fields)
|
||||
return s.eventDispatcher.DispatchEvent(ctx, model.NotificationEventReplicationFailed, title, body, fields)
|
||||
}
|
||||
|
||||
// List / Get / toSummary
|
||||
|
||||
@@ -47,6 +47,11 @@ func newReplicationTestHarness(t *testing.T) *replicationTestHarness {
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("db.DB: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
cipher := codec.NewConfigCipher("replicate-secret")
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
@@ -109,8 +114,9 @@ func TestReplicationService_MirrorsToDestTarget(t *testing.T) {
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
h.repl.async = func(job func()) {
|
||||
go func() { job(); close(done) }()
|
||||
h.repl.async = func(job func(context.Context)) bool {
|
||||
go func() { job(context.Background()); close(done) }()
|
||||
return true
|
||||
}
|
||||
summary, err := h.repl.Start(ctx, backupDetail.ID, 2, "tester")
|
||||
if err != nil {
|
||||
|
||||
@@ -36,6 +36,7 @@ func newReportTestHarness(t *testing.T) (*ReportService, *BackupExecutionService
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closeTestDatabase(t, db)
|
||||
cipher := codec.NewConfigCipher("report-secret")
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -39,7 +40,7 @@ type RestoreService struct {
|
||||
eventDispatcher EventDispatcher
|
||||
tempDir string
|
||||
semaphore chan struct{}
|
||||
async func(func())
|
||||
async func(func(context.Context)) bool
|
||||
now func() time.Time
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
@@ -49,6 +50,13 @@ func (s *RestoreService) SetMetrics(m *metrics.Metrics) {
|
||||
s.metrics = m
|
||||
}
|
||||
|
||||
// SetBackgroundRunner binds local restore work to the application lifecycle.
|
||||
func (s *RestoreService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
if runner != nil {
|
||||
s.async = runner.Go
|
||||
}
|
||||
}
|
||||
|
||||
// NewRestoreService 构造恢复服务。maxConcurrent 控制本地并发恢复数。
|
||||
func NewRestoreService(
|
||||
restores repository.RestoreRecordRepository,
|
||||
@@ -83,7 +91,7 @@ func NewRestoreService(
|
||||
dispatcher: dispatcher,
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
async: func(job func()) { go job() },
|
||||
async: runDetached,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
@@ -187,12 +195,18 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
|
||||
// 远程节点路由
|
||||
if remoteNode := s.resolveRemoteNode(ctx, restoreNodeID); remoteNode != nil {
|
||||
if s.dispatcher == nil {
|
||||
message := "Agent 下发通道未就绪"
|
||||
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, message); finalizeErr != nil {
|
||||
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
|
||||
}
|
||||
return nil, apperror.Internal("RESTORE_DISPATCH_UNAVAILABLE", "Agent 下发通道未就绪", nil)
|
||||
}
|
||||
// 节点离线 → 立即标记 failed,避免记录永远卡在 running
|
||||
if remoteNode.Status != model.NodeStatusOnline {
|
||||
offlineMsg := fmt.Sprintf("节点 %s 当前离线,无法执行恢复", remoteNode.Name)
|
||||
_ = s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, offlineMsg)
|
||||
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, offlineMsg); finalizeErr != nil {
|
||||
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
|
||||
}
|
||||
s.logHub.Append(restore.ID, "error", offlineMsg)
|
||||
s.logHub.Complete(restore.ID, model.RestoreRecordStatusFailed)
|
||||
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
|
||||
@@ -200,8 +214,10 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
|
||||
if _, dispatchErr := s.dispatcher.EnqueueCommand(ctx, restoreNodeID, model.AgentCommandTypeRestoreRecord, map[string]any{
|
||||
"restoreRecordId": restore.ID,
|
||||
}); dispatchErr != nil {
|
||||
_ = s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed,
|
||||
"下发恢复任务到远程节点失败: "+dispatchErr.Error())
|
||||
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed,
|
||||
"下发恢复任务到远程节点失败: "+dispatchErr.Error()); finalizeErr != nil {
|
||||
dispatchErr = errors.Join(dispatchErr, finalizeErr)
|
||||
}
|
||||
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发恢复任务到远程节点", dispatchErr)
|
||||
}
|
||||
s.logHub.Append(restore.ID, "info", fmt.Sprintf("已下发恢复任务到节点 %s(#%d),等待 Agent 执行", remoteNode.Name, restoreNodeID))
|
||||
@@ -209,10 +225,16 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
|
||||
}
|
||||
|
||||
// 本地节点:异步执行
|
||||
run := func() {
|
||||
s.executeLocally(context.Background(), restore.ID, task, record, selectedPaths, targetPath)
|
||||
run := func(runCtx context.Context) {
|
||||
s.executeLocally(runCtx, restore.ID, task, record, selectedPaths, targetPath)
|
||||
}
|
||||
if !s.async(run) {
|
||||
message := "服务正在关闭,恢复任务未启动"
|
||||
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, message); finalizeErr != nil {
|
||||
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
|
||||
}
|
||||
return nil, backgroundTaskUnavailable("RESTORE_SERVICE_SHUTTING_DOWN")
|
||||
}
|
||||
s.async(run)
|
||||
return s.getDetail(ctx, restore.ID)
|
||||
}
|
||||
|
||||
@@ -238,22 +260,30 @@ func (s *RestoreService) resolveRemoteNode(ctx context.Context, nodeID uint) *mo
|
||||
|
||||
// executeLocally 在 Master 本地执行恢复。
|
||||
func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, task *model.BackupTask, backupRecord *model.BackupRecord, selectedPaths []string, targetPath string) {
|
||||
s.semaphore <- struct{}{}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
logger := backup.NewExecutionLogger(restoreID, s.logHub)
|
||||
status := model.RestoreRecordStatusFailed
|
||||
errMessage := ""
|
||||
|
||||
defer func() {
|
||||
finalizeErr := s.finalizeWithLog(ctx, restoreID, status, errMessage, logger.String())
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
defer cancel()
|
||||
finalizeErr := s.finalizeWithLog(persistCtx, restoreID, status, errMessage, logger.String())
|
||||
if finalizeErr != nil {
|
||||
logger.Errorf("写回恢复记录失败:%v", finalizeErr)
|
||||
}
|
||||
s.logHub.Complete(restoreID, status)
|
||||
s.dispatchRestoreEvent(ctx, restoreID, status, errMessage, task)
|
||||
if dispatchErr := s.dispatchRestoreEvent(persistCtx, restoreID, status, errMessage, task); dispatchErr != nil {
|
||||
logger.Warnf("派发恢复结果事件失败:%v", dispatchErr)
|
||||
}
|
||||
}()
|
||||
|
||||
if !acquireBackgroundSlot(ctx, s.semaphore) {
|
||||
errMessage = ctx.Err().Error()
|
||||
logger.Warnf("等待恢复执行槽时任务被取消:%v", ctx.Err())
|
||||
return
|
||||
}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
logger.Infof("开始在本地执行恢复(备份记录 #%d)", backupRecord.ID)
|
||||
|
||||
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
|
||||
@@ -387,9 +417,9 @@ func backupKindLabel(kind string) string {
|
||||
|
||||
// dispatchRestoreEvent 按终态向事件总线派发 restore_success 或 restore_failed。
|
||||
// eventDispatcher 未注入时静默忽略,保持向后兼容。
|
||||
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) {
|
||||
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) error {
|
||||
if s.eventDispatcher == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
var eventType, title string
|
||||
switch status {
|
||||
@@ -400,7 +430,7 @@ func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uin
|
||||
eventType = model.NotificationEventRestoreFailed
|
||||
title = "BackupX 恢复失败"
|
||||
default:
|
||||
return
|
||||
return nil
|
||||
}
|
||||
taskName := "未知任务"
|
||||
if task != nil {
|
||||
@@ -419,7 +449,7 @@ func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uin
|
||||
if task != nil {
|
||||
fields["taskId"] = task.ID
|
||||
}
|
||||
_ = s.eventDispatcher.DispatchEvent(ctx, eventType, title, body, fields)
|
||||
return s.eventDispatcher.DispatchEvent(ctx, eventType, title, body, fields)
|
||||
}
|
||||
|
||||
// resolveProvider 解密存储目标配置并创建 provider(共享实现)。
|
||||
|
||||
@@ -84,6 +84,11 @@ func newRestoreTestHarness(t *testing.T, remoteNode bool) *restoreTestHarness {
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("db.DB: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
cipher := codec.NewConfigCipher("restore-secret")
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
@@ -159,11 +164,12 @@ func TestRestoreServiceStart_LocalNodeExecutesInline(t *testing.T) {
|
||||
|
||||
// 用同步 async 让测试可等待
|
||||
done := make(chan struct{})
|
||||
h.service.async = func(job func()) {
|
||||
h.service.async = func(job func(context.Context)) bool {
|
||||
go func() {
|
||||
job()
|
||||
job(context.Background())
|
||||
close(done)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
detail, err := h.service.Start(ctx, backupDetail.ID, "tester")
|
||||
if err != nil {
|
||||
@@ -200,6 +206,62 @@ func TestRestoreServiceStart_LocalNodeExecutesInline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreServiceStart_RepositoryRecord(t *testing.T) {
|
||||
h := newRestoreTestHarness(t, false)
|
||||
ctx := context.Background()
|
||||
task, err := h.tasks.FindByID(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("FindByID task: %v", err)
|
||||
}
|
||||
task.BackupMode = model.BackupModeRepository
|
||||
if err := h.tasks.Update(ctx, task); err != nil {
|
||||
t.Fatalf("Update repository task: %v", err)
|
||||
}
|
||||
|
||||
backupDetail, err := h.execution.RunTaskByIDSync(ctx, task.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTaskByIDSync repository: %v", err)
|
||||
}
|
||||
if backupDetail.BackupKind != model.BackupKindRepository {
|
||||
t.Fatalf("expected repository backup, got %#v", backupDetail)
|
||||
}
|
||||
if err := os.RemoveAll(h.sourceDir); err != nil {
|
||||
t.Fatalf("remove source: %v", err)
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
h.service.async = func(job func(context.Context)) bool {
|
||||
go func() {
|
||||
job(context.Background())
|
||||
close(done)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
detail, err := h.service.Start(ctx, backupDetail.ID, "repository-test")
|
||||
if err != nil {
|
||||
t.Fatalf("Start repository restore: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("repository restore did not complete in time")
|
||||
}
|
||||
final, err := h.service.Get(ctx, detail.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get repository restore: %v", err)
|
||||
}
|
||||
if final.Status != model.RestoreRecordStatusSuccess {
|
||||
t.Fatalf("expected repository restore success, got %s (err=%s)", final.Status, final.ErrorMessage)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(h.sourceDir, "index.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("read repository-restored file: %v", err)
|
||||
}
|
||||
if string(content) != "hello-restore" {
|
||||
t.Fatalf("unexpected repository-restored content: %q", content)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestoreServiceStart_RejectsCorruptedBackup 验证恢复在还原前做 SHA-256 完整性
|
||||
// 校验:若已存储的备份对象被损坏/篡改,恢复必须失败且不触碰源数据。
|
||||
func TestRestoreServiceStart_RejectsCorruptedBackup(t *testing.T) {
|
||||
@@ -242,8 +304,9 @@ func TestRestoreServiceStart_RejectsCorruptedBackup(t *testing.T) {
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
h.service.async = func(job func()) {
|
||||
go func() { job(); close(done) }()
|
||||
h.service.async = func(job func(context.Context)) bool {
|
||||
go func() { job(context.Background()); close(done) }()
|
||||
return true
|
||||
}
|
||||
detail, err := h.service.Start(ctx, backupDetail.ID, "tester")
|
||||
if err != nil {
|
||||
@@ -293,11 +356,12 @@ func TestRestoreServiceStart_RestoresToAlternatePath(t *testing.T) {
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
h.service.async = func(job func()) {
|
||||
h.service.async = func(job func(context.Context)) bool {
|
||||
go func() {
|
||||
job()
|
||||
job(context.Background())
|
||||
close(done)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
detail, err := h.service.StartSelective(ctx, backupDetail.ID, nil, altDir, "tester")
|
||||
if err != nil {
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/model"
|
||||
"backupx/server/internal/repository"
|
||||
"backupx/server/internal/storage"
|
||||
"backupx/server/internal/storage/codec"
|
||||
)
|
||||
|
||||
type RetentionService struct {
|
||||
records repository.BackupRecordRepository
|
||||
storageTargets repository.StorageTargetRepository
|
||||
storageRegistry *storage.Registry
|
||||
cipher *codec.ConfigCipher
|
||||
}
|
||||
|
||||
func NewRetentionService(records repository.BackupRecordRepository, storageTargets repository.StorageTargetRepository, storageRegistry *storage.Registry, cipher *codec.ConfigCipher) *RetentionService {
|
||||
return &RetentionService{records: records, storageTargets: storageTargets, storageRegistry: storageRegistry, cipher: cipher}
|
||||
}
|
||||
|
||||
func (s *RetentionService) Apply(ctx context.Context, task *model.BackupTask) error {
|
||||
if task == nil || (task.RetentionDays <= 0 && task.MaxBackups <= 0) {
|
||||
return nil
|
||||
}
|
||||
items, err := s.records.ListSuccessfulByTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removeSet := make(map[uint]model.BackupRecord)
|
||||
if task.RetentionDays > 0 {
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -task.RetentionDays)
|
||||
for _, item := range items {
|
||||
if item.CompletedAt != nil && item.CompletedAt.Before(cutoff) {
|
||||
removeSet[item.ID] = item
|
||||
}
|
||||
}
|
||||
}
|
||||
if task.MaxBackups > 0 {
|
||||
kept := 0
|
||||
for _, item := range items {
|
||||
if _, marked := removeSet[item.ID]; marked {
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
if kept > task.MaxBackups {
|
||||
removeSet[item.ID] = item
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(removeSet) == 0 {
|
||||
return nil
|
||||
}
|
||||
provider, _, err := buildStorageProviderFromRepos(ctx, task.StorageTargetID, s.storageTargets, s.storageRegistry, s.cipher)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range removeSet {
|
||||
if item.StoragePath != "" {
|
||||
if err := provider.Delete(ctx, item.StoragePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.records.Delete(ctx, item.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -95,6 +96,7 @@ type StorageTargetService struct {
|
||||
records repository.BackupRecordRepository
|
||||
registry *storage.Registry
|
||||
cipher *codec.ConfigCipher
|
||||
background BackgroundRunner
|
||||
}
|
||||
|
||||
func NewStorageTargetService(
|
||||
@@ -114,6 +116,10 @@ func (s *StorageTargetService) SetBackupRecordRepository(records repository.Back
|
||||
s.records = records
|
||||
}
|
||||
|
||||
func (s *StorageTargetService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
s.background = runner
|
||||
}
|
||||
|
||||
func (s *StorageTargetService) List(ctx context.Context) ([]StorageTargetSummary, error) {
|
||||
items, err := s.targets.List(ctx)
|
||||
if err != nil {
|
||||
@@ -254,7 +260,12 @@ func (s *StorageTargetService) TestConnection(ctx context.Context, input Storage
|
||||
item.LastTestMessage = "连接成功"
|
||||
}
|
||||
if item.ID != 0 {
|
||||
_ = s.targets.Update(ctx, item)
|
||||
if updateErr := s.targets.Update(ctx, item); updateErr != nil {
|
||||
if testErr != nil {
|
||||
return apperror.BadRequest("STORAGE_TARGET_TEST_FAILED", sanitizeMessage(testErr.Error()), errors.Join(testErr, fmt.Errorf("save connection test result: %w", updateErr)))
|
||||
}
|
||||
return apperror.Internal("STORAGE_TARGET_TEST_RESULT_SAVE_FAILED", "连接成功,但无法保存测试结果", updateErr)
|
||||
}
|
||||
}
|
||||
if testErr != nil {
|
||||
return apperror.BadRequest("STORAGE_TARGET_TEST_FAILED", sanitizeMessage(testErr.Error()), testErr)
|
||||
@@ -269,23 +280,23 @@ func (s *StorageTargetService) StartHealthMonitor(ctx context.Context, dispatche
|
||||
if interval <= 0 {
|
||||
interval = 5 * time.Minute
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
// notified 跟踪已告警的目标,避免每轮重复
|
||||
notified := map[uint]bool{}
|
||||
capacityNotified := map[uint]bool{}
|
||||
var mu sync.Mutex
|
||||
go func() {
|
||||
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-runCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.runHealthCheckOnce(ctx, dispatcher, &mu, notified)
|
||||
s.runCapacityCheckOnce(ctx, dispatcher, &mu, capacityNotified)
|
||||
s.runHealthCheckOnce(runCtx, dispatcher, &mu, notified)
|
||||
s.runCapacityCheckOnce(runCtx, dispatcher, &mu, capacityNotified)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// StorageCapacityWarningThreshold 存储使用率告警阈值(85%)。
|
||||
@@ -371,7 +382,6 @@ func (s *StorageTargetService) runHealthCheckOnce(ctx context.Context, dispatche
|
||||
if !target.Enabled {
|
||||
continue
|
||||
}
|
||||
previousStatus := target.LastTestStatus
|
||||
configMap := map[string]any{}
|
||||
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
|
||||
continue
|
||||
@@ -380,13 +390,13 @@ func (s *StorageTargetService) runHealthCheckOnce(ctx context.Context, dispatche
|
||||
now := time.Now().UTC()
|
||||
if err != nil {
|
||||
s.applyHealthResult(ctx, &target, now, false, err.Error())
|
||||
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, previousStatus, err.Error())
|
||||
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, err.Error())
|
||||
continue
|
||||
}
|
||||
testErr := provider.TestConnection(ctx)
|
||||
if testErr != nil {
|
||||
s.applyHealthResult(ctx, &target, now, false, testErr.Error())
|
||||
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, previousStatus, testErr.Error())
|
||||
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, testErr.Error())
|
||||
continue
|
||||
}
|
||||
s.applyHealthResult(ctx, &target, now, true, "连接成功")
|
||||
@@ -408,7 +418,7 @@ func (s *StorageTargetService) applyHealthResult(ctx context.Context, target *mo
|
||||
_ = s.targets.Update(ctx, target)
|
||||
}
|
||||
|
||||
func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, dispatcher EventDispatcher, mu *sync.Mutex, notified map[uint]bool, target *model.StorageTarget, previousStatus string, message string) {
|
||||
func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, dispatcher EventDispatcher, mu *sync.Mutex, notified map[uint]bool, target *model.StorageTarget, message string) {
|
||||
if dispatcher == nil {
|
||||
return
|
||||
}
|
||||
@@ -423,7 +433,6 @@ func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, di
|
||||
if already {
|
||||
return
|
||||
}
|
||||
_ = previousStatus // 保留参数便于未来扩展:区分"从未测试"与"从 success 掉线"
|
||||
title := "BackupX 存储目标连接失败"
|
||||
body := fmt.Sprintf("存储目标:%s (类型: %s)\n错误:%s", target.Name, target.Type, message)
|
||||
fields := map[string]any{
|
||||
@@ -473,7 +482,9 @@ func (s *StorageTargetService) CompleteGoogleDriveOAuth(ctx context.Context, inp
|
||||
// Mark used immediately to prevent duplicate requests (e.g. React StrictMode double invocation)
|
||||
now := time.Now().UTC()
|
||||
session.UsedAt = &now
|
||||
_ = s.oauthSessions.Update(ctx, session)
|
||||
if err := s.oauthSessions.Update(ctx, session); err != nil {
|
||||
return nil, apperror.Internal("STORAGE_GOOGLE_OAUTH_SESSION_FAILED", "无法锁定 Google Drive 授权会话", err)
|
||||
}
|
||||
|
||||
var draft googleDriveOAuthDraft
|
||||
if err := s.cipher.DecryptJSON(session.PayloadCiphertext, &draft); err != nil {
|
||||
|
||||
@@ -180,7 +180,7 @@ func (s *TaskExportService) Import(ctx context.Context, payload ExportPayload) (
|
||||
results = append(results, ImportResult{Name: t.Name, TaskID: detail.ID, Success: true})
|
||||
}
|
||||
// 第二阶段:依赖链接(上游任务名 → 新 ID)
|
||||
for i, t := range payload.Tasks {
|
||||
for _, t := range payload.Tasks {
|
||||
if len(t.DependsOnTaskNames) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -206,7 +206,6 @@ func (s *TaskExportService) Import(ctx context.Context, payload ExportPayload) (
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = i
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
|
||||
@@ -3,7 +3,6 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"backupx/server/internal/apperror"
|
||||
@@ -235,6 +234,3 @@ func toTemplateSummary(item *model.TaskTemplate) TaskTemplateSummary {
|
||||
UpdatedAt: item.UpdatedAt.Format("2006-01-02T15:04:05Z07:00"),
|
||||
}
|
||||
}
|
||||
|
||||
// 确保未使用告警
|
||||
var _ = fmt.Sprintf
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// closeTestDatabase releases SQLite file handles before testing.TempDir cleanup.
|
||||
// Windows does not permit removal of an open database file.
|
||||
func closeTestDatabase(t *testing.T, db *gorm.DB) {
|
||||
t.Helper()
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("get test database handle: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Errorf("close test database: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -39,7 +39,7 @@ type VerificationService struct {
|
||||
notifier VerificationNotifier
|
||||
tempDir string
|
||||
semaphore chan struct{}
|
||||
async func(func())
|
||||
async func(func(context.Context)) bool
|
||||
now func() time.Time
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
@@ -49,6 +49,13 @@ func (s *VerificationService) SetMetrics(m *metrics.Metrics) {
|
||||
s.metrics = m
|
||||
}
|
||||
|
||||
// SetBackgroundRunner binds local verification work to the application lifecycle.
|
||||
func (s *VerificationService) SetBackgroundRunner(runner BackgroundRunner) {
|
||||
if runner != nil {
|
||||
s.async = runner.Go
|
||||
}
|
||||
}
|
||||
|
||||
// VerificationNotifier 给用户推送验证完成/失败通知。
|
||||
// 可选注入:未注入时仅写记录。
|
||||
type VerificationNotifier interface {
|
||||
@@ -129,7 +136,7 @@ func NewVerificationService(
|
||||
notifier: noopVerificationNotifier{},
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
async: func(job func()) { go job() },
|
||||
async: runDetached,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
@@ -231,10 +238,16 @@ func (s *VerificationService) Start(ctx context.Context, backupRecordID uint, mo
|
||||
if err := s.verifications.Create(ctx, verification); err != nil {
|
||||
return nil, apperror.Internal("VERIFY_RECORD_CREATE_FAILED", "无法创建验证记录", err)
|
||||
}
|
||||
run := func() {
|
||||
s.executeLocally(context.Background(), verification.ID, task, record)
|
||||
run := func(runCtx context.Context) {
|
||||
s.executeLocally(runCtx, verification.ID, task, record)
|
||||
}
|
||||
if !s.async(run) {
|
||||
message := "服务正在关闭,验证任务未启动"
|
||||
if finalizeErr := s.finalize(ctx, verification.ID, model.VerificationRecordStatusFailed, message, "", ""); finalizeErr != nil {
|
||||
return nil, apperror.Internal("VERIFY_FINALIZE_FAILED", "无法写回验证失败状态", finalizeErr)
|
||||
}
|
||||
return nil, backgroundTaskUnavailable("VERIFY_SERVICE_SHUTTING_DOWN")
|
||||
}
|
||||
s.async(run)
|
||||
return s.getDetail(ctx, verification.ID)
|
||||
}
|
||||
|
||||
@@ -247,25 +260,37 @@ func (s *VerificationService) validateClusterAccessible(ctx context.Context, rec
|
||||
|
||||
// executeLocally 异步执行验证:下载 → 解密 → 解压 → 按类型校验。
|
||||
func (s *VerificationService) executeLocally(ctx context.Context, verID uint, task *model.BackupTask, backupRecord *model.BackupRecord) {
|
||||
s.semaphore <- struct{}{}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
logger := backup.NewExecutionLogger(verID, s.logHub)
|
||||
status := model.VerificationRecordStatusFailed
|
||||
errMessage := ""
|
||||
summary := ""
|
||||
|
||||
defer func() {
|
||||
_ = s.finalize(ctx, verID, status, errMessage, summary, logger.String())
|
||||
persistCtx, cancel := finalizationContext(ctx)
|
||||
defer cancel()
|
||||
if finalizeErr := s.finalize(persistCtx, verID, status, errMessage, summary, logger.String()); finalizeErr != nil {
|
||||
logger.Errorf("写回验证记录失败:%v", finalizeErr)
|
||||
}
|
||||
s.logHub.Complete(verID, status)
|
||||
// 失败时推送通知(best-effort)
|
||||
if status == model.VerificationRecordStatusFailed && s.notifier != nil {
|
||||
if record, err := s.verifications.FindByID(ctx, verID); err == nil && record != nil {
|
||||
_ = s.notifier.NotifyVerificationResult(ctx, task, record)
|
||||
if record, findErr := s.verifications.FindByID(persistCtx, verID); findErr != nil {
|
||||
logger.Warnf("读取验证记录以发送通知失败:%v", findErr)
|
||||
} else if record != nil {
|
||||
if notifyErr := s.notifier.NotifyVerificationResult(persistCtx, task, record); notifyErr != nil {
|
||||
logger.Warnf("发送验证失败通知失败:%v", notifyErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if !acquireBackgroundSlot(ctx, s.semaphore) {
|
||||
errMessage = ctx.Err().Error()
|
||||
logger.Warnf("等待验证执行槽时任务被取消:%v", ctx.Err())
|
||||
return
|
||||
}
|
||||
defer func() { <-s.semaphore }()
|
||||
|
||||
logger.Infof("开始验证备份记录 #%d(模式:%s)", backupRecord.ID, model.VerificationModeQuick)
|
||||
|
||||
if err := os.MkdirAll(s.tempDir, 0o755); err != nil {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"backupx/server/internal/backup"
|
||||
"backupx/server/internal/config"
|
||||
"backupx/server/internal/database"
|
||||
"backupx/server/internal/lifecycle"
|
||||
"backupx/server/internal/logger"
|
||||
"backupx/server/internal/model"
|
||||
"backupx/server/internal/repository"
|
||||
@@ -45,6 +46,11 @@ func newVerifyTestHarness(t *testing.T) *verifyTestHarness {
|
||||
if err != nil {
|
||||
t.Fatalf("database.Open: %v", err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("db.DB: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
cipher := codec.NewConfigCipher("verify-secret")
|
||||
targets := repository.NewStorageTargetRepository(db)
|
||||
tasks := repository.NewBackupTaskRepository(db)
|
||||
@@ -77,8 +83,9 @@ func (h *verifyTestHarness) runVerify(t *testing.T, backupRecordID uint) *Verifi
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
done := make(chan struct{})
|
||||
h.verify.async = func(job func()) {
|
||||
go func() { job(); close(done) }()
|
||||
h.verify.async = func(job func(context.Context)) bool {
|
||||
go func() { job(context.Background()); close(done) }()
|
||||
return true
|
||||
}
|
||||
detail, err := h.verify.Start(ctx, backupRecordID, "quick", "tester")
|
||||
if err != nil {
|
||||
@@ -96,6 +103,48 @@ func (h *verifyTestHarness) runVerify(t *testing.T, backupRecordID uint) *Verifi
|
||||
return final
|
||||
}
|
||||
|
||||
func TestVerificationServiceSupervisorCancellationFinalizesRecord(t *testing.T) {
|
||||
h := newVerifyTestHarness(t)
|
||||
backupDetail, err := h.execution.RunTaskByIDSync(context.Background(), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RunTaskByIDSync: %v", err)
|
||||
}
|
||||
|
||||
supervisor := lifecycle.NewSupervisor(context.Background())
|
||||
h.verify.SetBackgroundRunner(supervisor)
|
||||
// Occupy the only available execution path before cancellation so the test
|
||||
// deterministically exercises cancellation while queued.
|
||||
for i := 0; i < cap(h.verify.semaphore); i++ {
|
||||
h.verify.semaphore <- struct{}{}
|
||||
}
|
||||
detail, err := h.verify.Start(context.Background(), backupDetail.ID, model.VerificationModeQuick, "tester")
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
waitCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := supervisor.Shutdown(waitCtx); err != nil {
|
||||
t.Fatalf("Shutdown: %v", err)
|
||||
}
|
||||
for i := 0; i < cap(h.verify.semaphore); i++ {
|
||||
<-h.verify.semaphore
|
||||
}
|
||||
|
||||
final, err := h.verify.Get(context.Background(), detail.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if final.Status != model.VerificationRecordStatusFailed {
|
||||
t.Fatalf("status = %q, want failed", final.Status)
|
||||
}
|
||||
if final.CompletedAt == nil {
|
||||
t.Fatal("canceled verification was not finalized")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(final.ErrorMessage), "canceled") {
|
||||
t.Fatalf("error message = %q, want cancellation", final.ErrorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerificationService_Success 覆盖正常路径:对一个有效(gzip 压缩)的备份做验证应通过。
|
||||
// 同时回归保护 #77——新增的 SHA-256 校验不得误伤合法的压缩备份。
|
||||
func TestVerificationService_Success(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user