refactor: simplify architecture and harden lifecycle

Remove obsolete implementations, centralize background task ownership and terminal-state recovery, consolidate frontend routing and log streaming, and enforce project-wide verification in CI.
This commit is contained in:
Awuqing
2026-08-26 11:32:00 +08:00
parent 95b27af600
commit c06bbec383
189 changed files with 8983 additions and 4264 deletions
+153 -44
View File
@@ -19,6 +19,7 @@ import (
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
"go.uber.org/zap"
)
// AgentService 实现 Master 端 Agent 协议,提供给远程 Agent 通过 HTTP 调用。
@@ -32,6 +33,8 @@ type AgentService struct {
restoreRepo repository.RestoreRecordRepository
registry *storage.Registry
cipher *codec.ConfigCipher
logger *zap.Logger
background BackgroundRunner
}
func NewAgentService(
@@ -51,9 +54,24 @@ func NewAgentService(
cmdRepo: cmdRepo,
registry: registry,
cipher: cipher,
logger: zap.NewNop(),
}
}
// SetLogger attaches the application logger used by background command
// reconciliation. The no-op default keeps the service safe in tests.
func (s *AgentService) SetLogger(logger *zap.Logger) {
if logger != nil {
s.logger = logger
}
}
// SetBackgroundRunner makes the command timeout monitor part of the
// application lifecycle so shutdown waits for an in-flight reconciliation.
func (s *AgentService) SetBackgroundRunner(background BackgroundRunner) {
s.background = background
}
// SetRestoreRepository 注入恢复记录仓储,用于命令超时时联动 restore_record 状态。
// 可选注入:未注入时恢复命令超时仅标记命令 timeout,记录需另行查验。
func (s *AgentService) SetRestoreRepository(repo repository.RestoreRecordRepository) {
@@ -117,6 +135,16 @@ func (s *AgentService) SubmitCommandResult(ctx context.Context, node *model.Node
if cmd.NodeID != node.ID {
return apperror.Unauthorized("AGENT_COMMAND_FORBIDDEN", "命令不属于当前节点", nil)
}
// A failed terminal report may be retried after the command row was already
// completed but before its linked business record was updated. Re-run that
// idempotent convergence step so a transient database error cannot leave a
// backup or restore record stuck in running forever.
if cmd.Status == model.AgentCommandStatusFailed {
if result.Success {
return nil
}
return s.failLinkedRecord(ctx, cmd, agentCommandFailureMessage(result.ErrorMessage))
}
now := time.Now().UTC()
if result.Success {
cmd.Status = model.AgentCommandStatusSucceeded
@@ -128,8 +156,21 @@ func (s *AgentService) SubmitCommandResult(ctx context.Context, node *model.Node
cmd.Result = string(result.Result)
}
cmd.CompletedAt = &now
_, err = s.cmdRepo.CompleteDispatched(ctx, cmd)
return err
completed, err := s.cmdRepo.CompleteDispatched(ctx, cmd)
if err != nil || !completed || result.Success {
return err
}
persistCtx, cancel := finalizationContext(ctx)
defer cancel()
return s.failLinkedRecord(persistCtx, cmd, agentCommandFailureMessage(result.ErrorMessage))
}
func agentCommandFailureMessage(message string) string {
message = strings.TrimSpace(message)
if message == "" {
return "Agent 命令执行失败"
}
return "Agent 命令执行失败:" + message
}
// AgentTaskSpec 给 Agent 返回的任务规格,包含解密后的存储配置,供 Agent 直接执行。
@@ -616,19 +657,29 @@ func (s *AgentService) StartCommandTimeoutMonitor(ctx context.Context, interval
if timeout <= 0 {
timeout = 10 * time.Minute
}
ticker := time.NewTicker(interval)
go func() {
monitor := func(runCtx context.Context) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
threshold := time.Now().UTC().Add(-timeout)
s.processStaleCommands(ctx, threshold)
s.processStaleCommands(runCtx, threshold)
}
}
}()
}
if s.background != nil {
if !s.background.Go(monitor) {
s.logger.Warn("agent command timeout monitor not started: application is shutting down")
}
return
}
if ctx == nil {
ctx = context.Background()
}
go monitor(ctx)
}
// processStaleCommands 扫描已超时的 pending/dispatched 命令并联动关联记录。
@@ -636,12 +687,24 @@ func (s *AgentService) StartCommandTimeoutMonitor(ctx context.Context, interval
// 单条失败不影响后续处理。
func (s *AgentService) processStaleCommands(ctx context.Context, threshold time.Time) {
commands, err := s.cmdRepo.ListStaleActive(ctx, threshold)
if err != nil || len(commands) == 0 {
if err != nil {
s.logger.Error("list stale agent commands failed", zap.Error(err))
return
}
if len(commands) == 0 {
return
}
for i := range commands {
cmd := commands[i]
if s.commandStillActive(ctx, &cmd, threshold) {
stillActive, activeErr := s.commandStillActive(ctx, &cmd, threshold)
if activeErr != nil {
s.logger.Warn("check stale agent command activity failed",
zap.Uint("command_id", cmd.ID),
zap.String("command_type", cmd.Type),
zap.Error(activeErr))
continue
}
if stillActive {
continue
}
now := time.Now().UTC()
@@ -649,18 +712,33 @@ func (s *AgentService) processStaleCommands(ctx context.Context, threshold time.
cmd.ErrorMessage = "agent did not report result before timeout"
cmd.CompletedAt = &now
timedOut, err := s.cmdRepo.TimeoutActive(ctx, &cmd)
if err != nil || !timedOut {
if err != nil {
s.logger.Error("mark agent command timed out failed",
zap.Uint("command_id", cmd.ID),
zap.String("command_type", cmd.Type),
zap.Error(err))
continue
}
s.failLinkedRecord(ctx, &cmd)
if !timedOut {
continue
}
persistCtx, cancel := finalizationContext(ctx)
failErr := s.failLinkedRecord(persistCtx, &cmd)
cancel()
if failErr != nil {
s.logger.Error("mark timed-out agent command record failed",
zap.Uint("command_id", cmd.ID),
zap.String("command_type", cmd.Type),
zap.Error(failErr))
}
}
}
// commandStillActive 用关联记录状态、记录更新时间和节点心跳作为长任务续租信号。
// 仅 run_task / restore_record 允许续租,避免短 RPC 命令被在线节点长期保留。
func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentCommand, threshold time.Time) bool {
func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentCommand, threshold time.Time) (bool, error) {
if cmd.Status != model.AgentCommandStatusDispatched {
return false
return false, nil
}
switch cmd.Type {
case model.AgentCommandTypeRunTask:
@@ -668,90 +746,121 @@ func (s *AgentService) commandStillActive(ctx context.Context, cmd *model.AgentC
RecordID uint `json:"recordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RecordID == 0 {
return false
return false, nil
}
record, err := s.recordRepo.FindByID(ctx, payload.RecordID)
if err != nil || record == nil || record.Status != model.BackupRecordStatusRunning {
return false
if err != nil {
return false, fmt.Errorf("find backup record %d: %w", payload.RecordID, err)
}
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
return true
if record == nil || record.Status != model.BackupRecordStatusRunning {
return false, nil
}
return record.UpdatedAt.After(threshold)
nodeActive, err := s.nodeRecentlySeen(ctx, cmd.NodeID, threshold)
if err != nil {
return false, err
}
return nodeActive || record.UpdatedAt.After(threshold), nil
case model.AgentCommandTypeRestoreRecord:
if s.restoreRepo == nil {
return false
return false, nil
}
var payload struct {
RestoreRecordID uint `json:"restoreRecordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RestoreRecordID == 0 {
return false
return false, nil
}
restore, err := s.restoreRepo.FindByID(ctx, payload.RestoreRecordID)
if err != nil || restore == nil || restore.Status != model.RestoreRecordStatusRunning {
return false
if err != nil {
return false, fmt.Errorf("find restore record %d: %w", payload.RestoreRecordID, err)
}
if s.nodeRecentlySeen(ctx, cmd.NodeID, threshold) {
return true
if restore == nil || restore.Status != model.RestoreRecordStatusRunning {
return false, nil
}
return restore.UpdatedAt.After(threshold)
nodeActive, err := s.nodeRecentlySeen(ctx, cmd.NodeID, threshold)
if err != nil {
return false, err
}
return nodeActive || restore.UpdatedAt.After(threshold), nil
default:
return false
return false, nil
}
}
func (s *AgentService) nodeRecentlySeen(ctx context.Context, nodeID uint, threshold time.Time) bool {
func (s *AgentService) nodeRecentlySeen(ctx context.Context, nodeID uint, threshold time.Time) (bool, error) {
node, err := s.nodeRepo.FindByID(ctx, nodeID)
if err != nil || node == nil {
return false
if err != nil {
return false, fmt.Errorf("find agent node %d: %w", nodeID, err)
}
return node.Status == model.NodeStatusOnline && node.LastSeen.After(threshold)
if node == nil {
return false, nil
}
return node.Status == model.NodeStatusOnline && node.LastSeen.After(threshold), nil
}
// failLinkedRecord 根据命令类型把关联记录标记为 failed。
// 只对仍然处于 running 状态的记录生效,避免覆盖已完成的结果。
func (s *AgentService) failLinkedRecord(ctx context.Context, cmd *model.AgentCommand) {
const failureMessage = "Agent 未在超时前回传状态(节点可能已离线或崩溃)"
func (s *AgentService) failLinkedRecord(ctx context.Context, cmd *model.AgentCommand, messages ...string) error {
failureMessage := "Agent 未在超时前回传状态(节点可能已离线或崩溃)"
if len(messages) > 0 && strings.TrimSpace(messages[0]) != "" {
failureMessage = strings.TrimSpace(messages[0])
}
switch cmd.Type {
case model.AgentCommandTypeRunTask:
var payload struct {
RecordID uint `json:"recordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RecordID == 0 {
return
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil {
return fmt.Errorf("decode run_task payload: %w", err)
}
if payload.RecordID == 0 {
return errors.New("run_task payload has no recordId")
}
record, err := s.recordRepo.FindByID(ctx, payload.RecordID)
if err != nil || record == nil || record.Status != model.BackupRecordStatusRunning {
return
if err != nil {
return fmt.Errorf("find backup record %d: %w", payload.RecordID, err)
}
if record == nil || record.Status != model.BackupRecordStatusRunning {
return nil
}
completedAt := time.Now().UTC()
record.Status = model.BackupRecordStatusFailed
record.ErrorMessage = failureMessage
record.CompletedAt = &completedAt
record.DurationSeconds = int(completedAt.Sub(record.StartedAt).Seconds())
_ = s.recordRepo.Update(ctx, record)
if err := s.recordRepo.Update(ctx, record); err != nil {
return fmt.Errorf("update backup record %d: %w", record.ID, err)
}
case model.AgentCommandTypeRestoreRecord:
if s.restoreRepo == nil {
return
return errors.New("restore record repository is not configured")
}
var payload struct {
RestoreRecordID uint `json:"restoreRecordId"`
}
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil || payload.RestoreRecordID == 0 {
return
if err := json.Unmarshal([]byte(cmd.Payload), &payload); err != nil {
return fmt.Errorf("decode restore_record payload: %w", err)
}
if payload.RestoreRecordID == 0 {
return errors.New("restore_record payload has no restoreRecordId")
}
restore, err := s.restoreRepo.FindByID(ctx, payload.RestoreRecordID)
if err != nil || restore == nil || restore.Status != model.RestoreRecordStatusRunning {
return
if err != nil {
return fmt.Errorf("find restore record %d: %w", payload.RestoreRecordID, err)
}
if restore == nil || restore.Status != model.RestoreRecordStatusRunning {
return nil
}
completedAt := time.Now().UTC()
restore.Status = model.RestoreRecordStatusFailed
restore.ErrorMessage = failureMessage
restore.CompletedAt = &completedAt
restore.DurationSeconds = int(completedAt.Sub(restore.StartedAt).Seconds())
_ = s.restoreRepo.Update(ctx, restore)
if err := s.restoreRepo.Update(ctx, restore); err != nil {
return fmt.Errorf("update restore record %d: %w", restore.ID, err)
}
}
return nil
}
// AgentSelfStatus 是 /api/v1/agent/self 端点返回给 Agent 的轻量状态摘要。
@@ -24,6 +24,15 @@ import (
"gorm.io/gorm"
)
type failingUpdateBackupRecordRepository struct {
repository.BackupRecordRepository
updateErr error
}
func (r *failingUpdateBackupRecordRepository) Update(context.Context, *model.BackupRecord) error {
return r.updateErr
}
func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repository.BackupRecordRepository, repository.AgentCommandRepository, *model.Node, *model.Node) {
t.Helper()
log, err := logger.New(config.LogConfig{Level: "error"})
@@ -34,6 +43,7 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
closeTestDatabase(t, db)
cipher := codec.NewConfigCipher("agent-service-secret")
nodeRepo := repository.NewNodeRepository(db)
taskRepo := repository.NewBackupTaskRepository(db)
@@ -87,6 +97,23 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher, storageRegistry), db, recordRepo, cmdRepo, owner, other
}
func TestAgentServiceFailLinkedRecordPropagatesTerminalUpdateError(t *testing.T) {
svc, _, records, _, _, _ := newAgentServicePoolTestHarness(t)
wantErr := errors.New("record update failed")
svc.recordRepo = &failingUpdateBackupRecordRepository{
BackupRecordRepository: records,
updateErr: wantErr,
}
err := svc.failLinkedRecord(context.Background(), &model.AgentCommand{
Type: model.AgentCommandTypeRunTask,
Payload: `{"recordId":1}`,
})
if err == nil || !errors.Is(err, wantErr) {
t.Fatalf("failLinkedRecord error = %v, want wrapped update error", err)
}
}
func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) {
svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t)
ctx := context.Background()
@@ -752,6 +779,44 @@ func TestAgentServiceSubmitCommandResultDoesNotOverwriteTerminalCommand(t *testi
}
}
func TestAgentServiceSubmitFailedCommandConvergesLinkedRecord(t *testing.T) {
svc, _, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
dispatchedAt := time.Now().UTC()
command := &model.AgentCommand{
NodeID: owner.ID,
Type: model.AgentCommandTypeRunTask,
Status: model.AgentCommandStatusDispatched,
Payload: `{"recordId":1}`,
DispatchedAt: &dispatchedAt,
}
if err := commands.Create(ctx, command); err != nil {
t.Fatalf("Create command returned error: %v", err)
}
if err := svc.SubmitCommandResult(ctx, owner, command.ID, AgentCommandResult{
Success: false,
ErrorMessage: "terminal update could not reach Master",
}); err != nil {
t.Fatalf("SubmitCommandResult returned error: %v", err)
}
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
if record.Status != model.BackupRecordStatusFailed || !strings.Contains(record.ErrorMessage, "terminal update") {
t.Fatalf("linked record did not converge: %#v", record)
}
updatedCommand, err := commands.FindByID(ctx, command.ID)
if err != nil {
t.Fatalf("FindByID command returned error: %v", err)
}
if updatedCommand.Status != model.AgentCommandStatusFailed {
t.Fatalf("command status = %q, want failed", updatedCommand.Status)
}
}
func TestAgentServiceUpdateRecordDoesNotOverwriteTerminalRecord(t *testing.T) {
svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
@@ -24,6 +24,7 @@ func newApiKeyTestService(t *testing.T) *ApiKeyService {
if err != nil {
t.Fatalf("database.Open: %v", err)
}
closeTestDatabase(t, db)
return NewApiKeyService(repository.NewApiKeyRepository(db))
}
@@ -23,6 +23,11 @@ func TestAuditRetention(t *testing.T) {
if err != nil {
t.Fatal(err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
auditRepo := repository.NewAuditLogRepository(db)
configRepo := repository.NewSystemConfigRepository(db)
svc := NewAuditService(auditRepo)
+83 -28
View File
@@ -8,7 +8,6 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"log"
"net/http"
"strconv"
"strings"
@@ -18,6 +17,7 @@ import (
"backupx/server/internal/apperror"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"go.uber.org/zap"
)
// AuditEntry 是记录审计日志的输入结构
@@ -41,14 +41,35 @@ type AuditService struct {
webhookURL string
webhookSecret string
httpClient *http.Client
async func(func(context.Context)) bool
logger *zap.Logger
inFlight chan struct{}
}
const maxAuditInFlight = 64
func NewAuditService(repo repository.AuditLogRepository) *AuditService {
return &AuditService{
repo: repo,
httpClient: &http.Client{
Timeout: 3 * time.Second, // 短超时:审计 webhook 不应拖慢业务
},
async: runDetached,
logger: zap.NewNop(),
inFlight: make(chan struct{}, maxAuditInFlight),
}
}
func (s *AuditService) SetLogger(logger *zap.Logger) {
if logger != nil {
s.logger = logger
}
}
// SetBackgroundRunner binds audit persistence and webhook delivery to the application lifecycle.
func (s *AuditService) SetBackgroundRunner(runner BackgroundRunner) {
if runner != nil {
s.async = runner.Go
}
}
@@ -71,37 +92,55 @@ func (s *AuditService) StartRetentionMonitor(ctx context.Context, configs reposi
if interval <= 0 {
interval = 6 * time.Hour
}
go func() {
if ctx == nil {
ctx = context.Background()
}
accepted := s.async(func(workerCtx context.Context) {
monitorCtx, cancel := context.WithCancel(workerCtx)
defer cancel()
stopLink := context.AfterFunc(ctx, cancel)
defer stopLink()
ticker := time.NewTicker(interval)
defer ticker.Stop()
s.runRetentionOnce(ctx, configs) // 启动后立即跑一次
s.runRetentionOnce(monitorCtx, configs) // 启动后立即跑一次
for {
select {
case <-ctx.Done():
case <-monitorCtx.Done():
return
case <-ticker.C:
s.runRetentionOnce(ctx, configs)
s.runRetentionOnce(monitorCtx, configs)
}
}
}()
})
if !accepted {
s.logger.Warn("audit retention monitor not started: application is shutting down")
}
}
func (s *AuditService) runRetentionOnce(ctx context.Context, configs repository.SystemConfigRepository) {
cfg, err := configs.GetByKey(ctx, SettingKeyAuditRetentionDays)
if err != nil || cfg == nil {
if err != nil {
s.logger.Warn("read audit retention setting failed", zap.Error(err))
return
}
if cfg == nil {
return
}
days, err := strconv.Atoi(strings.TrimSpace(cfg.Value))
if err != nil || days <= 0 {
if err != nil {
s.logger.Warn("invalid audit retention setting", zap.String("value", cfg.Value), zap.Error(err))
return
}
if days <= 0 {
return
}
deleted, err := s.PurgeOlderThan(ctx, days)
if err != nil {
log.Printf("[audit] retention purge failed: %v", err)
s.logger.Warn("audit retention purge failed", zap.Error(err))
return
}
if deleted > 0 {
log.Printf("[audit] retention purge: deleted %d logs older than %d days", deleted, days)
s.logger.Info("audit retention purge completed", zap.Int64("deleted", deleted), zap.Int("retention_days", days))
}
}
@@ -123,12 +162,24 @@ func (s *AuditService) SetWebhook(url, secret string) {
s.webhookSecret = strings.TrimSpace(secret)
}
// Record 异步 fire-and-forget 写入审计日志,不阻塞业务逻辑
// Record asynchronously persists an audit event without blocking the request.
func (s *AuditService) Record(entry AuditEntry) {
if s == nil || s.repo == nil {
return
}
go func() {
select {
case s.inFlight <- struct{}{}:
default:
s.logger.Error("audit event rejected: in-flight limit reached",
zap.Int("limit", cap(s.inFlight)),
zap.String("category", entry.Category),
zap.String("action", entry.Action))
return
}
accepted := s.async(func(workerCtx context.Context) {
defer func() { <-s.inFlight }()
persistCtx, cancel := finalizationContext(workerCtx)
defer cancel()
record := &model.AuditLog{
UserID: entry.UserID,
Username: entry.Username,
@@ -140,24 +191,30 @@ func (s *AuditService) Record(entry AuditEntry) {
Detail: entry.Detail,
ClientIP: entry.ClientIP,
}
if err := s.repo.Create(context.Background(), record); err != nil {
log.Printf("[audit] failed to write audit log: %v", err)
if err := s.repo.Create(persistCtx, record); err != nil {
s.logger.Error("failed to write audit log", zap.String("category", entry.Category), zap.String("action", entry.Action), zap.Error(err))
}
s.fireWebhook(record)
}()
if err := s.fireWebhook(persistCtx, record); err != nil {
s.logger.Warn("audit webhook delivery failed", zap.String("category", entry.Category), zap.String("action", entry.Action), zap.Error(err))
}
})
if !accepted {
<-s.inFlight
s.logger.Warn("audit event rejected: application is shutting down", zap.String("category", entry.Category), zap.String("action", entry.Action))
}
}
// fireWebhook 异步向外部系统转发审计事件。失败降级到本地日志,永不影响主流程。
func (s *AuditService) fireWebhook(record *model.AuditLog) {
// fireWebhook forwards an audit event. The caller owns asynchronous execution.
func (s *AuditService) fireWebhook(ctx context.Context, record *model.AuditLog) error {
if s == nil {
return
return nil
}
s.webhookMu.RLock()
url := s.webhookURL
secret := s.webhookSecret
s.webhookMu.RUnlock()
if url == "" {
return
return nil
}
payload := map[string]any{
"eventType": "audit.log",
@@ -176,13 +233,11 @@ func (s *AuditService) fireWebhook(record *model.AuditLog) {
}
body, err := json.Marshal(payload)
if err != nil {
log.Printf("[audit] webhook marshal failed: %v", err)
return
return fmt.Errorf("marshal audit webhook: %w", err)
}
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, bytes.NewReader(body))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
log.Printf("[audit] webhook build request failed: %v", err)
return
return fmt.Errorf("build audit webhook request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "BackupX-Audit/1.0")
@@ -193,13 +248,13 @@ func (s *AuditService) fireWebhook(record *model.AuditLog) {
}
resp, err := s.httpClient.Do(req)
if err != nil {
log.Printf("[audit] webhook POST failed: %v", err)
return
return fmt.Errorf("post audit webhook: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
log.Printf("[audit] webhook returned status %d", resp.StatusCode)
return fmt.Errorf("audit webhook returned status %d", resp.StatusCode)
}
return nil
}
// List 分页查询审计日志
@@ -14,6 +14,7 @@ import (
"testing"
"time"
"backupx/server/internal/lifecycle"
"backupx/server/internal/model"
"backupx/server/internal/repository"
)
@@ -131,3 +132,40 @@ func TestAuditService_WebhookDisabledWhenURLEmpty(t *testing.T) {
time.Sleep(100 * time.Millisecond)
// 无显式断言:能不 panic 即算通过
}
func TestAuditServiceSupervisorShutdownFlushesAcceptedRecord(t *testing.T) {
repo := newFakeAuditRepo()
supervisor := lifecycle.NewSupervisor(context.Background())
svc := NewAuditService(repo)
svc.SetBackgroundRunner(supervisor)
svc.Record(AuditEntry{Username: "alice", Category: "auth", Action: "logout"})
waitCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := supervisor.Shutdown(waitCtx); err != nil {
t.Fatalf("Shutdown: %v", err)
}
select {
case <-repo.created:
default:
t.Fatal("accepted audit record was not flushed during shutdown")
}
}
func TestAuditServiceBoundsInFlightWork(t *testing.T) {
repo := newFakeAuditRepo()
svc := NewAuditService(repo)
svc.inFlight = make(chan struct{}, 1)
accepted := 0
svc.async = func(func(context.Context)) bool {
accepted++
return true
}
svc.Record(AuditEntry{Category: "auth", Action: "first"})
svc.Record(AuditEntry{Category: "auth", Action: "second"})
if accepted != 1 {
t.Fatalf("accepted tasks = %d, want 1", accepted)
}
}
@@ -0,0 +1,84 @@
package service
import (
"context"
"testing"
"time"
)
type capturingMonitorRunner struct {
tasks []func(context.Context)
}
func (r *capturingMonitorRunner) Go(task func(context.Context)) bool {
r.tasks = append(r.tasks, task)
return true
}
func TestLongRunningMonitorsUseConfiguredBackgroundRunner(t *testing.T) {
runner := &capturingMonitorRunner{}
nodes := NewNodeService(nil, "test")
nodes.SetBackgroundRunner(runner)
nodes.StartOfflineMonitor(context.Background(), time.Hour)
installTokens := NewInstallTokenService(nil, nil)
installTokens.SetBackgroundRunner(runner)
installTokens.StartGC(context.Background(), time.Hour)
dashboard := NewDashboardService(nil, nil, nil)
dashboard.SetBackgroundRunner(runner)
dashboard.StartSLAMonitor(context.Background(), nil, time.Hour, time.Hour)
versions := NewClusterVersionMonitor(nil, "test")
versions.SetBackgroundRunner(runner)
versions.Start(context.Background(), time.Hour, time.Hour)
storageTargets := NewStorageTargetService(nil, nil, nil, nil)
storageTargets.SetBackgroundRunner(runner)
storageTargets.StartHealthMonitor(context.Background(), nil, time.Hour)
if len(runner.tasks) != 5 {
t.Fatalf("background runner received %d tasks, want 5", len(runner.tasks))
}
// The monitor must listen to the supervisor-provided context, not retain
// the context passed to Start. Running one captured task is sufficient to
// lock this ownership contract for the shared helper.
runCtx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
runner.tasks[0](runCtx)
close(done)
}()
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("monitor did not stop when background runner context was canceled")
}
}
func TestBackgroundMonitorFallbackUsesCallerContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
started := make(chan struct{})
done := make(chan struct{})
if !startBackgroundMonitor(nil, ctx, func(runCtx context.Context) {
close(started)
<-runCtx.Done()
close(done)
}) {
t.Fatal("fallback monitor was rejected")
}
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("fallback monitor did not start")
}
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("fallback monitor did not stop when caller context was canceled")
}
}
@@ -0,0 +1,62 @@
package service
import (
"context"
"net/http"
"time"
"backupx/server/internal/apperror"
)
// BackgroundRunner is the narrow lifecycle dependency used by asynchronous
// services. lifecycle.Supervisor implements it at the application boundary.
type BackgroundRunner interface {
Go(func(context.Context)) bool
}
func runDetached(task func(context.Context)) bool {
if task == nil {
return false
}
go task(context.Background())
return true
}
// startBackgroundMonitor keeps the legacy caller-owned context when no
// lifecycle runner is configured, while allowing the application supervisor
// to own and wait for long-running monitors in production.
func startBackgroundMonitor(runner BackgroundRunner, fallbackCtx context.Context, task func(context.Context)) bool {
if task == nil {
return false
}
if runner != nil {
return runner.Go(task)
}
if fallbackCtx == nil {
fallbackCtx = context.Background()
}
go task(fallbackCtx)
return true
}
func backgroundTaskUnavailable(code string) *apperror.AppError {
return apperror.New(http.StatusServiceUnavailable, code, "服务正在关闭,无法启动新的后台任务", context.Canceled)
}
// finalizationContext lets a canceled task persist its terminal state. It is
// intentionally short-lived so shutdown cannot wait forever on cleanup I/O.
func finalizationContext(ctx context.Context) (context.Context, context.CancelFunc) {
if ctx == nil {
ctx = context.Background()
}
return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
}
func acquireBackgroundSlot(ctx context.Context, semaphore chan struct{}) bool {
select {
case semaphore <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
@@ -105,7 +105,7 @@ type BackupExecutionService struct {
agentDispatcher AgentDispatcher
replicationHook ReplicationTrigger
dependentsResolver DependentsResolver
async func(func())
async func(func(context.Context)) bool
now func() time.Time
tempDir string
semaphore chan struct{}
@@ -127,6 +127,13 @@ func (s *BackupExecutionService) SetMetrics(m *metrics.Metrics) {
s.metrics = m
}
// SetBackgroundRunner binds local asynchronous executions to the application lifecycle.
func (s *BackupExecutionService) SetBackgroundRunner(runner BackgroundRunner) {
if runner != nil {
s.async = runner.Go
}
}
// ReplicationTrigger 抽象备份成功后的副本派发(实现者:ReplicationService)。
type ReplicationTrigger interface {
TriggerAutoReplication(ctx context.Context, task *model.BackupTask, record *model.BackupRecord)
@@ -194,14 +201,12 @@ func NewBackupExecutionService(
retention: retention,
cipher: cipher,
notifier: notifier,
async: func(job func()) {
go job()
},
now: func() time.Time { return time.Now().UTC() },
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
retries: retries,
bandwidthLimit: bandwidthLimit,
async: runDetached,
now: func() time.Time { return time.Now().UTC() },
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
retries: retries,
bandwidthLimit: bandwidthLimit,
}
}
@@ -259,62 +264,6 @@ func (s *BackupExecutionService) DownloadRecord(ctx context.Context, recordID ui
return &DownloadedArtifact{FileName: fileName, Reader: reader}, nil
}
func (s *BackupExecutionService) RestoreRecord(ctx context.Context, recordID uint) error {
record, provider, err := s.loadRecordProvider(ctx, recordID)
if err != nil {
return err
}
task, err := s.tasks.FindByID(ctx, record.TaskID)
if err != nil {
return apperror.Internal("BACKUP_TASK_GET_FAILED", "无法获取关联备份任务", err)
}
if task == nil {
return apperror.New(404, "BACKUP_TASK_NOT_FOUND", "关联的备份任务不存在,无法执行恢复", fmt.Errorf("backup task %d not found", record.TaskID))
}
if record.BackupKind == model.BackupKindRepository {
spec, specErr := s.buildTaskSpec(task, record.StartedAt)
if specErr != nil {
return specErr
}
if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, backup.NopLogWriter{}); err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "从 CDC 仓库恢复备份失败", err)
}
return nil
}
tempDir, err := os.MkdirTemp("", "backupx-restore-*")
if err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法创建恢复目录", err)
}
defer os.RemoveAll(tempDir)
artifactPath := filepath.Join(tempDir, filepath.Base(record.FileName))
if strings.TrimSpace(filepath.Base(record.FileName)) == "" {
artifactPath = filepath.Join(tempDir, filepath.Base(record.StoragePath))
}
reader, err := provider.Download(ctx, record.StoragePath)
if err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法下载备份文件", err)
}
if err := writeReaderToFile(artifactPath, reader); err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法写入恢复文件", err)
}
preparedPath, err := s.prepareArtifactForRestore(artifactPath)
if err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法准备恢复文件", err)
}
spec, err := s.buildTaskSpec(task, record.StartedAt)
if err != nil {
return err
}
runner, err := s.runnerRegistry.Runner(spec.Type)
if err != nil {
return apperror.BadRequest("BACKUP_TASK_INVALID", "不支持的备份任务类型", err)
}
if err := runner.Restore(ctx, spec, preparedPath, backup.NopLogWriter{}); err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "恢复备份失败", err)
}
return nil
}
func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint) error {
record, err := s.records.FindByID(ctx, recordID)
if err != nil {
@@ -504,6 +453,11 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
task.LastRunAt = &startedAt
task.LastStatus = "running"
if err := s.tasks.Update(ctx, task); err != nil {
finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
"无法更新任务状态: "+err.Error(), "", "", 0, "", "", primaryTargetID)
if finalizeErr != nil {
err = errors.Join(err, finalizeErr)
}
return nil, apperror.Internal("BACKUP_TASK_UPDATE_FAILED", "无法更新任务状态", err)
}
// 多节点路由:task.NodeID 指向远程节点时,把执行任务入队给 Agent;
@@ -512,8 +466,10 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
// 节点离线 → 立即把刚创建的 running 记录标记 failed,返回明确错误
if remoteNode.Status != model.NodeStatusOnline {
offlineMsg := fmt.Sprintf("节点 %s 当前离线,无法执行备份任务", remoteNode.Name)
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
offlineMsg, "", "", 0, "", "", primaryTargetID)
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
offlineMsg, "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
return nil, apperror.Internal("BACKUP_RECORD_FINALIZE_FAILED", "无法写回备份失败状态", finalizeErr)
}
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
}
if _, enqueueErr := s.agentDispatcher.EnqueueCommand(ctx, resolvedNodeID, model.AgentCommandTypeRunTask, map[string]any{
@@ -521,19 +477,28 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
"recordId": record.ID,
}); enqueueErr != nil {
// 入队失败 → 在记录中标记失败,继续返回详情
_ = s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "", primaryTargetID)
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
"无法下发任务到远程节点: "+enqueueErr.Error(), "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
enqueueErr = errors.Join(enqueueErr, finalizeErr)
}
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发任务到远程节点", enqueueErr)
}
return s.getRecordDetail(ctx, record.ID)
}
run := func() {
s.executeTask(context.Background(), &runTask, record.ID, startedAt)
run := func(runCtx context.Context) {
s.executeTask(runCtx, &runTask, record.ID, startedAt)
}
if async {
s.async(run)
if !s.async(run) {
message := "服务正在关闭,备份任务未启动"
if finalizeErr := s.finalizeRecord(ctx, &runTask, record.ID, startedAt, model.BackupRecordStatusFailed,
message, "", "", 0, "", "", primaryTargetID); finalizeErr != nil {
return nil, apperror.Internal("BACKUP_RECORD_FINALIZE_FAILED", "无法写回备份失败状态", finalizeErr)
}
return nil, backgroundTaskUnavailable("BACKUP_SERVICE_SHUTTING_DOWN")
}
} else {
run()
run(ctx)
}
return s.getRecordDetail(ctx, record.ID)
}
@@ -844,20 +809,23 @@ func (s *BackupExecutionService) executeRepositoryTask(ctx context.Context, task
logger.Warnf("部分存储目标 CDC 仓库同步失败:%s", strings.Join(failures, "; "))
}
if s.dependentsResolver != nil {
go func(upstreamID uint, upstreamName string) {
dependents, resolveErr := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID)
accepted := s.async(func(runCtx context.Context) {
dependents, resolveErr := s.dependentsResolver.TriggerDependents(runCtx, task.ID)
if resolveErr != nil {
logger.Warnf("解析任务 %s 的下游依赖失败:%v", upstreamName, resolveErr)
logger.Warnf("解析任务 %s 的下游依赖失败:%v", task.Name, resolveErr)
return
}
for _, dependentID := range dependents {
if _, runErr := s.RunTaskByID(context.Background(), dependentID); runErr != nil {
logger.Warnf("触发下游任务 #%d 失败(上游: %s):%v", dependentID, upstreamName, runErr)
if _, runErr := s.RunTaskByID(runCtx, dependentID); runErr != nil {
logger.Warnf("触发下游任务 #%d 失败(上游: %s):%v", dependentID, task.Name, runErr)
} else {
logger.Infof("已触发下游任务 #%d(上游: %s", dependentID, upstreamName)
logger.Infof("已触发下游任务 #%d(上游: %s", dependentID, task.Name)
}
}
}(task.ID, task.Name)
})
if !accepted {
logger.Warnf("服务正在关闭,跳过触发任务 %s 的下游依赖", task.Name)
}
}
return result, nil
}
@@ -871,20 +839,6 @@ func (s *BackupExecutionService) acquireRepositoryLock(targetID uint) func() {
}
func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) {
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0
// 该节点上所有任务共享一个节点专属 semaphore,互相排队
nodeSem := s.acquireNodeSemaphore(ctx, task.NodeID)
if nodeSem != nil {
nodeSem <- struct{}{}
defer func() { <-nodeSem }()
}
s.semaphore <- struct{}{}
defer func() { <-s.semaphore }()
// Prometheus: running gauge + 完成时 observe 耗时/字节/状态
s.metrics.IncTaskRunning()
defer s.metrics.DecTaskRunning()
logger := backup.NewExecutionLogger(recordID, s.logHub)
status := "failed"
errMessage := ""
@@ -900,8 +854,10 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
var manifestJSON string
var repositoryProviders map[uint]storage.StorageProvider
completeRecord := func() {
persistCtx, cancel := finalizationContext(ctx)
defer cancel()
readyForRepositoryRetention := status == model.BackupRecordStatusSuccess
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
if finalizeErr := s.finalizeRecord(persistCtx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
logger.Errorf("写回备份记录失败:%v", finalizeErr)
readyForRepositoryRetention = false
}
@@ -913,7 +869,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
if marshalErr != nil {
logger.Warnf("序列化多目标上传结果失败:%v", marshalErr)
readyForRepositoryRetention = false
} else if record, findErr := s.records.FindByID(ctx, recordID); findErr != nil || record == nil {
} else if record, findErr := s.records.FindByID(persistCtx, recordID); findErr != nil || record == nil {
if findErr != nil {
logger.Warnf("读取备份记录以写回多目标结果失败:%v", findErr)
} else {
@@ -922,7 +878,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
readyForRepositoryRetention = false
} else {
record.StorageUploadResults = string(resultsJSON)
if updateErr := s.records.Update(ctx, record); updateErr != nil {
if updateErr := s.records.Update(persistCtx, record); updateErr != nil {
logger.Warnf("写回多目标上传结果失败:%v", updateErr)
readyForRepositoryRetention = false
}
@@ -930,11 +886,11 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
}
// 持久化差异链信息:全量记录其清单(供后续差异比对),差异记录其基线全量 ID。
if status == model.BackupRecordStatusSuccess && (backupKind != model.BackupKindFull || baseRecordID != 0 || manifestJSON != "") {
if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil {
if record, findErr := s.records.FindByID(persistCtx, recordID); findErr == nil && record != nil {
record.BackupKind = backupKind
record.BaseRecordID = baseRecordID
record.Manifest = manifestJSON
if updErr := s.records.Update(ctx, record); updErr != nil {
if updErr := s.records.Update(persistCtx, record); updErr != nil {
logger.Warnf("写回差异链信息失败:%v", updErr)
readyForRepositoryRetention = false
}
@@ -947,7 +903,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
readyForRepositoryRetention = false
}
}
if readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 {
if ctx.Err() == nil && readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 {
targetIDs := make([]uint, 0, len(repositoryProviders))
for targetID := range repositoryProviders {
targetIDs = append(targetIDs, targetID)
@@ -969,8 +925,8 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
}
}
}
if s.shouldNotify(ctx, task, status) {
if err := s.notifier.NotifyBackupResult(ctx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
if s.shouldNotify(persistCtx, task, status) {
if err := s.notifier.NotifyBackupResult(persistCtx, BackupExecutionNotification{Task: task, Record: &model.BackupRecord{ID: recordID, TaskID: task.ID, Status: status, FileName: fileName, FileSize: fileSize, StoragePath: storagePath, ErrorMessage: errMessage, StartedAt: startedAt}, Error: buildOptionalError(errMessage)}); err != nil {
logger.Warnf("发送备份通知失败:%v", err)
}
} else {
@@ -980,6 +936,28 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
}
defer completeRecord()
// 节点级并发限流:当任务绑定节点且节点配置了 MaxConcurrent>0
// 该节点上所有任务共享一个节点专属 semaphore,互相排队。
nodeSem := s.acquireNodeSemaphore(ctx, task.NodeID)
if nodeSem != nil {
if !acquireBackgroundSlot(ctx, nodeSem) {
errMessage = ctx.Err().Error()
logger.Warnf("等待节点执行槽时任务被取消:%v", ctx.Err())
return
}
defer func() { <-nodeSem }()
}
if !acquireBackgroundSlot(ctx, s.semaphore) {
errMessage = ctx.Err().Error()
logger.Warnf("等待全局执行槽时任务被取消:%v", ctx.Err())
return
}
defer func() { <-s.semaphore }()
// Prometheus: running gauge + 完成时 observe 耗时/字节/状态
s.metrics.IncTaskRunning()
defer s.metrics.DecTaskRunning()
spec, err := s.buildTaskSpec(task, startedAt)
if err != nil {
errMessage = err.Error()
@@ -1217,20 +1195,24 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
// 自动派发复制(3-2-1):任务配置 ReplicationTargetIDs 且本次有任意目标成功时生效
// 触发下游依赖任务(best-effort,失败仅 warn
if s.dependentsResolver != nil {
go func(upstreamID uint, upstreamName string) {
dependents, err := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID)
accepted := s.async(func(runCtx context.Context) {
dependents, err := s.dependentsResolver.TriggerDependents(runCtx, task.ID)
if err != nil {
logger.Warnf("解析任务 %s 的下游依赖失败:%v", task.Name, err)
return
}
for _, depID := range dependents {
_, runErr := s.RunTaskByID(context.Background(), depID)
_, runErr := s.RunTaskByID(runCtx, depID)
if runErr != nil {
logger.Warnf("触发下游任务 #%d 失败(上游: %s: %v", depID, upstreamName, runErr)
logger.Warnf("触发下游任务 #%d 失败(上游: %s: %v", depID, task.Name, runErr)
} else {
logger.Infof("已触发下游任务 #%d(上游: %s", depID, upstreamName)
logger.Infof("已触发下游任务 #%d(上游: %s", depID, task.Name)
}
}
}(task.ID, task.Name)
})
if !accepted {
logger.Warnf("服务正在关闭,跳过触发任务 %s 的下游依赖", task.Name)
}
}
if s.replicationHook != nil && strings.TrimSpace(task.ReplicationTargetIDs) != "" {
record := &model.BackupRecord{
@@ -1253,7 +1235,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
}
}
logger.Infof("触发自动复制(3-2-1 规则):%s", task.ReplicationTargetIDs)
s.replicationHook.TriggerAutoReplication(context.Background(), task, record)
s.replicationHook.TriggerAutoReplication(ctx, task, record)
}
} else {
errMessage = strings.Join(failedMessages, "; ")
@@ -1356,28 +1338,6 @@ func applyHANAExtraConfig(spec *backup.DatabaseSpec, extra map[string]any) {
}
}
func (s *BackupExecutionService) loadRecordProvider(ctx context.Context, recordID uint) (*model.BackupRecord, storage.StorageProvider, error) {
record, err := s.records.FindByID(ctx, recordID)
if err != nil {
return nil, nil, apperror.Internal("BACKUP_RECORD_GET_FAILED", "无法获取备份记录详情", err)
}
if record == nil {
return nil, nil, apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "备份记录不存在", fmt.Errorf("backup record %d not found", recordID))
}
if err := s.validateClusterAccessible(ctx, record); err != nil {
return nil, nil, err
}
provider, err := s.resolveProvider(ctx, record.StorageTargetID)
if err != nil {
return nil, nil, err
}
return record, provider, nil
}
func (s *BackupExecutionService) prepareArtifactForRestore(artifactPath string) (string, error) {
return prepareBackupArtifact(s.cipher, artifactPath, nil)
}
func (s *BackupExecutionService) getRecordDetail(ctx context.Context, recordID uint) (*BackupRecordDetail, error) {
record, err := s.records.FindByID(ctx, recordID)
if err != nil {
@@ -1410,25 +1370,6 @@ func buildOptionalError(message string) error {
return fmt.Errorf("%s", message)
}
func buildStorageProviderFromRepos(ctx context.Context, storageTargetID uint, storageTargets repository.StorageTargetRepository, storageRegistry *storage.Registry, cipher *codec.ConfigCipher) (storage.StorageProvider, *model.StorageTarget, error) {
target, err := storageTargets.FindByID(ctx, storageTargetID)
if err != nil {
return nil, nil, apperror.Internal("BACKUP_STORAGE_TARGET_LOOKUP_FAILED", "无法读取存储目标", err)
}
if target == nil {
return nil, nil, apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil)
}
var configMap map[string]any
if err := cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
return nil, nil, apperror.Internal("BACKUP_STORAGE_TARGET_DECRYPT_FAILED", "无法解密存储目标配置", err)
}
provider, err := storageRegistry.Create(ctx, storage.ParseProviderType(target.Type), configMap)
if err != nil {
return nil, nil, err
}
return provider, target, nil
}
// hashingReader 在上传过程中同步计算字节数和 SHA-256,零额外 I/O
type hashingReader struct {
reader io.Reader
@@ -33,6 +33,8 @@ func (f *testStorageFactory) Type() storage.ProviderType {
return "test_storage"
}
func (f *testStorageFactory) SensitiveFields() []string { return nil }
func (f *testStorageFactory) New(_ context.Context, config map[string]any) (storage.StorageProvider, error) {
name, _ := config["name"].(string)
provider := f.providers[name]
@@ -250,20 +252,6 @@ func TestBackupExecutionServiceRepositoryModeRoundTrip(t *testing.T) {
t.Fatalf("unexpected repository export: name=%s size=%d", download.FileName, len(exported))
}
if err := os.WriteFile(largePath, bytes.Repeat([]byte{0}, len(large)), 0o640); err != nil {
t.Fatalf("damage source before restore: %v", err)
}
if err := executionService.RestoreRecord(ctx, second.ID); err != nil {
t.Fatalf("restore repository record returned error: %v", err)
}
restored, err := os.ReadFile(largePath)
if err != nil {
t.Fatalf("read restored source: %v", err)
}
if !bytes.Equal(restored, large) {
t.Fatalf("repository restore did not reproduce the source")
}
if err := recordService.Delete(ctx, first.ID); err != nil {
t.Fatalf("delete first repository record: %v", err)
}
@@ -428,40 +416,6 @@ func TestBackupExecutionServiceDeleteRecordDispatchesRemoteLocalDiskCleanup(t *t
}
}
func TestBackupExecutionServiceRestoreRecordRejectsRemoteLocalDisk(t *testing.T) {
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
executionService.SetClusterDependencies(&nodeRepoStub{nodes: []model.Node{
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline},
}}, &fakeDispatcher{})
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
completedAt := time.Now().UTC()
record := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: 10,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: "file/2026/05/09/remote.tar.gz",
StartedAt: completedAt.Add(-time.Second),
CompletedAt: &completedAt,
}
if err := records.Create(ctx, record); err != nil {
t.Fatalf("Create record returned error: %v", err)
}
err = executionService.RestoreRecord(ctx, record.ID)
if err == nil {
t.Fatal("expected remote local_disk restore to be rejected")
}
if !strings.Contains(err.Error(), "Master 无法跨节点访问") {
t.Fatalf("expected cross-node local_disk error, got %v", err)
}
}
func TestBackupExecutionServiceDownloadsMasterRelayedLocalDiskRecord(t *testing.T) {
executionService, _, tasks, _, records, _, storageDir := newExecutionTestServices(t)
ctx := context.Background()
@@ -712,27 +666,6 @@ func TestBackupExecutionServiceContinuesWhenStorageUsageSnapshotFails(t *testing
}
}
func TestBackupRecordServiceRestore(t *testing.T) {
executionService, recordService, _, _, _, sourceDir, _ := newExecutionTestServices(t)
detail, err := executionService.RunTaskByIDSync(context.Background(), 1)
if err != nil {
t.Fatalf("RunTaskByIDSync returned error: %v", err)
}
if err := os.RemoveAll(sourceDir); err != nil {
t.Fatalf("RemoveAll returned error: %v", err)
}
if err := recordService.Restore(context.Background(), detail.ID); err != nil {
t.Fatalf("Restore returned error: %v", err)
}
content, err := os.ReadFile(filepath.Join(sourceDir, "index.html"))
if err != nil {
t.Fatalf("ReadFile returned error: %v", err)
}
if string(content) != "hello" {
t.Fatalf("unexpected restored content: %s", string(content))
}
}
type storageUsageCountingRecordRepo struct {
repository.BackupRecordRepository
mu sync.Mutex
@@ -37,6 +37,7 @@ func newLockTestHarness(t *testing.T) (*BackupRecordService, *BackupExecutionSer
if err != nil {
t.Fatal(err)
}
closeTestDatabase(t, db)
cipher := codec.NewConfigCipher("lock-secret")
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
@@ -156,10 +156,6 @@ func (s *BackupRecordService) Download(ctx context.Context, id uint) (*Downloade
return s.execution.DownloadRecord(ctx, id)
}
func (s *BackupRecordService) Restore(ctx context.Context, id uint) error {
return s.execution.RestoreRecord(ctx, id)
}
func (s *BackupRecordService) Delete(ctx context.Context, id uint) error {
return s.execution.DeleteRecord(ctx, id)
}
+18 -1
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
@@ -389,7 +390,15 @@ func (s *BackupTaskService) Delete(ctx context.Context, id uint) (*DeleteResult,
return nil, apperror.New(http.StatusNotFound, "BACKUP_TASK_NOT_FOUND", "备份任务不存在", fmt.Errorf("backup task %d not found", id))
}
if s.scheduler != nil {
_ = s.scheduler.RemoveTask(ctx, id)
if err := s.scheduler.RemoveTask(ctx, id); err != nil {
rollbackCtx, cancel := finalizationContext(ctx)
rollbackErr := s.scheduler.SyncTask(rollbackCtx, existing)
cancel()
if rollbackErr != nil {
err = errors.Join(err, fmt.Errorf("restore task schedule: %w", rollbackErr))
}
return nil, apperror.Internal("BACKUP_TASK_UNSCHEDULE_FAILED", "无法移除备份任务调度", err)
}
}
// 清理远端存储文件(尽力而为,不阻止删除)
@@ -397,6 +406,14 @@ func (s *BackupTaskService) Delete(ctx context.Context, id uint) (*DeleteResult,
result.RecordCount, result.CleanedFiles = s.cleanupRemoteFiles(ctx, id)
if err := s.tasks.Delete(ctx, id); err != nil {
if s.scheduler != nil {
rollbackCtx, cancel := finalizationContext(ctx)
rollbackErr := s.scheduler.SyncTask(rollbackCtx, existing)
cancel()
if rollbackErr != nil {
err = errors.Join(err, fmt.Errorf("restore task schedule: %w", rollbackErr))
}
}
return nil, apperror.Internal("BACKUP_TASK_DELETE_FAILED", "无法删除备份任务", err)
}
return result, nil
@@ -2,10 +2,12 @@ package service
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
"backupx/server/internal/apperror"
"backupx/server/internal/config"
"backupx/server/internal/database"
"backupx/server/internal/logger"
@@ -14,6 +16,34 @@ import (
"backupx/server/internal/storage/codec"
)
type backupTaskSchedulerStub struct {
removeErr error
syncErr error
removedIDs []uint
syncedTasks []model.BackupTask
}
func (s *backupTaskSchedulerStub) SyncTask(_ context.Context, task *model.BackupTask) error {
if task != nil {
s.syncedTasks = append(s.syncedTasks, *task)
}
return s.syncErr
}
func (s *backupTaskSchedulerStub) RemoveTask(_ context.Context, taskID uint) error {
s.removedIDs = append(s.removedIDs, taskID)
return s.removeErr
}
type failingDeleteBackupTaskRepository struct {
repository.BackupTaskRepository
deleteErr error
}
func (r *failingDeleteBackupTaskRepository) Delete(context.Context, uint) error {
return r.deleteErr
}
func newBackupTaskServiceForTest(t *testing.T) (*BackupTaskService, repository.StorageTargetRepository, repository.BackupTaskRepository) {
t.Helper()
log, err := logger.New(config.LogConfig{Level: "error"})
@@ -24,6 +54,7 @@ func newBackupTaskServiceForTest(t *testing.T) (*BackupTaskService, repository.S
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
closeTestDatabase(t, db)
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
service := NewBackupTaskService(tasks, targets, codec.NewConfigCipher("task-service-secret"))
@@ -138,6 +169,76 @@ func TestBackupTaskServiceCreateAndGet(t *testing.T) {
}
}
func TestBackupTaskServiceDeleteKeepsTaskWhenUnscheduleFails(t *testing.T) {
ctx := context.Background()
service, targets, tasks := newBackupTaskServiceForTest(t)
if err := targets.Create(ctx, &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "ciphertext", ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
t.Fatal(err)
}
created, err := service.Create(ctx, BackupTaskUpsertInput{
Name: "unschedule-failure", Type: "file", Enabled: true, SourcePath: "/srv/data",
StorageTargetID: 1, RetentionDays: 7, Compression: "gzip", MaxBackups: 3,
})
if err != nil {
t.Fatal(err)
}
scheduler := &backupTaskSchedulerStub{removeErr: errors.New("remove failed")}
service.SetScheduler(scheduler)
if _, err := service.Delete(ctx, created.ID); err == nil {
t.Fatal("Delete should fail when the scheduler cannot remove the task")
} else {
var appErr *apperror.AppError
if !errors.As(err, &appErr) || appErr.Code != "BACKUP_TASK_UNSCHEDULE_FAILED" {
t.Fatalf("Delete error = %#v", err)
}
}
stored, err := tasks.FindByID(ctx, created.ID)
if err != nil || stored == nil {
t.Fatalf("task should remain after unschedule failure: task=%#v err=%v", stored, err)
}
if len(scheduler.removedIDs) != 1 || len(scheduler.syncedTasks) != 1 {
t.Fatalf("scheduler calls = remove:%v sync:%d", scheduler.removedIDs, len(scheduler.syncedTasks))
}
}
func TestBackupTaskServiceDeleteRestoresScheduleWhenPersistenceFails(t *testing.T) {
ctx := context.Background()
service, targets, tasks := newBackupTaskServiceForTest(t)
if err := targets.Create(ctx, &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "ciphertext", ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
t.Fatal(err)
}
created, err := service.Create(ctx, BackupTaskUpsertInput{
Name: "delete-failure", Type: "file", Enabled: true, SourcePath: "/srv/data",
StorageTargetID: 1, RetentionDays: 7, Compression: "gzip", MaxBackups: 3,
})
if err != nil {
t.Fatal(err)
}
service.tasks = &failingDeleteBackupTaskRepository{
BackupTaskRepository: tasks,
deleteErr: errors.New("database delete failed"),
}
scheduler := &backupTaskSchedulerStub{}
service.SetScheduler(scheduler)
if _, err := service.Delete(ctx, created.ID); err == nil {
t.Fatal("Delete should fail when persistence fails")
} else {
var appErr *apperror.AppError
if !errors.As(err, &appErr) || appErr.Code != "BACKUP_TASK_DELETE_FAILED" {
t.Fatalf("Delete error = %#v", err)
}
}
if len(scheduler.removedIDs) != 1 || len(scheduler.syncedTasks) != 1 || scheduler.syncedTasks[0].ID != created.ID {
t.Fatalf("scheduler rollback calls = remove:%v sync:%#v", scheduler.removedIDs, scheduler.syncedTasks)
}
stored, err := tasks.FindByID(ctx, created.ID)
if err != nil || stored == nil {
t.Fatalf("task should remain after persistence failure: task=%#v err=%v", stored, err)
}
}
func TestBackupTaskServiceKeepsMaskedPasswordOnUpdate(t *testing.T) {
ctx := context.Background()
service, targets, tasks := newBackupTaskServiceForTest(t)
+10 -5
View File
@@ -21,6 +21,7 @@ type ClusterVersionMonitor struct {
nodeRepo repository.NodeRepository
eventDispatcher EventDispatcher
masterVersion string
background BackgroundRunner
mu sync.Mutex
notified map[uint]time.Time
}
@@ -37,6 +38,10 @@ func (m *ClusterVersionMonitor) SetEventDispatcher(dispatcher EventDispatcher) {
m.eventDispatcher = dispatcher
}
func (m *ClusterVersionMonitor) SetBackgroundRunner(runner BackgroundRunner) {
m.background = runner
}
// Start 启动后台扫描。ctx 取消时退出。
// scanInterval 建议 30 分钟;resetInterval 建议 24 小时。
func (m *ClusterVersionMonitor) Start(ctx context.Context, scanInterval, resetInterval time.Duration) {
@@ -47,19 +52,19 @@ func (m *ClusterVersionMonitor) Start(ctx context.Context, scanInterval, resetIn
resetInterval = 24 * time.Hour
}
// 启动立即跑一次,让控制台尽快看到
go func() {
m.scan(ctx, resetInterval)
startBackgroundMonitor(m.background, ctx, func(runCtx context.Context) {
m.scan(runCtx, resetInterval)
ticker := time.NewTicker(scanInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
m.scan(ctx, resetInterval)
m.scan(runCtx, resetInterval)
}
}
}()
})
}
func (m *ClusterVersionMonitor) scan(ctx context.Context, resetInterval time.Duration) {
@@ -45,6 +45,7 @@ func newDashboardNotificationTestDeps(t *testing.T) (*DashboardService, *Notific
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
closeTestDatabase(t, db)
tasks := repository.NewBackupTaskRepository(db)
records := repository.NewBackupRecordRepository(db)
targets := repository.NewStorageTargetRepository(db)
+10 -5
View File
@@ -34,6 +34,7 @@ type DashboardService struct {
targets repository.StorageTargetRepository
nodes repository.NodeRepository
masterVersion string
background BackgroundRunner
// slaMonitor 内部跟踪已告警的违约任务,避免每次扫描重复派发事件
slaNotified map[uint]time.Time
slaMu sync.Mutex
@@ -43,6 +44,10 @@ func NewDashboardService(tasks repository.BackupTaskRepository, records reposito
return &DashboardService{tasks: tasks, records: records, targets: targets, slaNotified: map[uint]time.Time{}}
}
func (s *DashboardService) SetBackgroundRunner(runner BackgroundRunner) {
s.background = runner
}
// SetClusterDependencies 注入节点仓储与 Master 版本,启用集群概览。
func (s *DashboardService) SetClusterDependencies(nodes repository.NodeRepository, masterVersion string) {
s.nodes = nodes
@@ -561,18 +566,18 @@ func (s *DashboardService) StartSLAMonitor(ctx context.Context, dispatcher Event
if resetInterval <= 0 {
resetInterval = 6 * time.Hour
}
ticker := time.NewTicker(scanInterval)
go func() {
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
ticker := time.NewTicker(scanInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
s.scanAndDispatchSLA(ctx, dispatcher, resetInterval)
s.scanAndDispatchSLA(runCtx, dispatcher, resetInterval)
}
}
}()
})
}
// scanAndDispatchSLA 执行一次 SLA 违约扫描并按需派发事件。
@@ -27,6 +27,7 @@ func TestGoogleDriveOAuthServiceStartAndComplete(t *testing.T) {
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
closeTestDatabase(t, db)
sessions := repository.NewOAuthSessionRepository(db)
service := NewGoogleDriveOAuthService(sessions, codec.New("encryption-secret"))
service.now = func() time.Time { return time.Date(2026, 3, 7, 0, 0, 0, 0, time.UTC) }
@@ -18,14 +18,19 @@ import (
// InstallTokenService 负责一次性安装令牌的创建/消费/校验。
type InstallTokenService struct {
repo repository.AgentInstallTokenRepository
nodeRepo repository.NodeRepository
repo repository.AgentInstallTokenRepository
nodeRepo repository.NodeRepository
background BackgroundRunner
}
func NewInstallTokenService(repo repository.AgentInstallTokenRepository, nodeRepo repository.NodeRepository) *InstallTokenService {
return &InstallTokenService{repo: repo, nodeRepo: nodeRepo}
}
func (s *InstallTokenService) SetBackgroundRunner(runner BackgroundRunner) {
s.background = runner
}
// InstallTokenInput 生成一次性安装令牌的输入。
type InstallTokenInput struct {
NodeID uint
@@ -247,18 +252,18 @@ func (s *InstallTokenService) StartGC(ctx context.Context, interval time.Duratio
if interval <= 0 {
interval = time.Hour
}
go func() {
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
_, _ = s.repo.DeleteExpiredBefore(ctx, time.Now().UTC().Add(-7*24*time.Hour))
_, _ = s.repo.DeleteExpiredBefore(runCtx, time.Now().UTC().Add(-7*24*time.Hour))
}
}
}()
})
}
func (s *InstallTokenService) validate(in InstallTokenInput) error {
@@ -22,6 +22,7 @@ func openInstallTokenTestDB(t *testing.T) *gorm.DB {
if err != nil {
t.Fatalf("open: %v", err)
}
closeTestDatabase(t, db)
if err := db.AutoMigrate(&model.AgentInstallToken{}, &model.Node{}); err != nil {
t.Fatalf("migrate: %v", err)
}
+15 -10
View File
@@ -67,11 +67,12 @@ type NodeUpdateInput struct {
// NodeService manages the cluster nodes.
type NodeService struct {
repo repository.NodeRepository
taskRepo repository.BackupTaskRepository
agentRPC NodeAgentRPC
cmdRepo repository.AgentCommandRepository
version string
repo repository.NodeRepository
taskRepo repository.BackupTaskRepository
agentRPC NodeAgentRPC
cmdRepo repository.AgentCommandRepository
version string
background BackgroundRunner
}
// NodeAgentRPC 抽象 Agent 远程调用能力(避免 service 内循环依赖)。
@@ -85,6 +86,10 @@ func NewNodeService(repo repository.NodeRepository, version string) *NodeService
return &NodeService{repo: repo, version: version}
}
func (s *NodeService) SetBackgroundRunner(runner BackgroundRunner) {
s.background = runner
}
// SetTaskRepository 注入任务仓储以支持删除前引用检查。可选注入,便于测试。
func (s *NodeService) SetTaskRepository(taskRepo repository.BackupTaskRepository) {
s.taskRepo = taskRepo
@@ -315,19 +320,19 @@ func (s *NodeService) StartOfflineMonitor(ctx context.Context, interval time.Dur
if interval <= 0 {
interval = 15 * time.Second
}
ticker := time.NewTicker(interval)
go func() {
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
threshold := time.Now().UTC().Add(-OfflineThreshold)
_, _ = s.repo.MarkStaleOffline(ctx, threshold)
_, _ = s.repo.MarkStaleOffline(runCtx, threshold)
}
}
}()
})
}
// Heartbeat updates the node status when an agent reports in.
@@ -20,6 +20,7 @@ func openNodeServiceDB(t *testing.T) *gorm.DB {
if err != nil {
t.Fatalf("open: %v", err)
}
closeTestDatabase(t, db)
if err := db.AutoMigrate(&model.Node{}); err != nil {
t.Fatalf("migrate: %v", err)
}
@@ -172,7 +172,7 @@ func (s *NotificationService) NotifyBackupResult(ctx context.Context, event Back
if success {
eventType = model.NotificationEventBackupSuccess
}
items, err := s.collectSubscribers(ctx, eventType, success)
items, err := s.collectSubscribers(ctx, eventType)
if err != nil {
return err
}
@@ -194,9 +194,7 @@ func (s *NotificationService) DispatchEvent(ctx context.Context, eventType strin
if s.broadcaster != nil {
_ = s.broadcaster.Publish(ctx, eventType, title, body, fields)
}
// 将 fallback 布尔用于旧语义场景(backup_success / backup_failed)。
fallbackSuccess := eventType == model.NotificationEventBackupSuccess
items, err := s.collectSubscribers(ctx, eventType, fallbackSuccess)
items, err := s.collectSubscribers(ctx, eventType)
if err != nil {
return err
}
@@ -254,7 +252,7 @@ func (s *NotificationService) sendFirstByType(ctx context.Context, notificationT
// collectSubscribers 按事件类型收集启用的订阅者。
// 列出启用通知后按事件类型再过滤(避免引入新 repository 方法)。
func (s *NotificationService) collectSubscribers(ctx context.Context, eventType string, fallbackSuccess bool) ([]model.Notification, error) {
func (s *NotificationService) collectSubscribers(ctx context.Context, eventType string) ([]model.Notification, error) {
all, err := s.notifications.List(ctx)
if err != nil {
return nil, err
@@ -284,8 +282,6 @@ func (s *NotificationService) collectSubscribers(ctx context.Context, eventType
// 其他事件类型必须显式订阅才推送
continue
}
// 额外校验 fallbackSuccess 参数,保持历史行为一致
_ = fallbackSuccess
}
matched = append(matched, item)
}
+62 -23
View File
@@ -15,6 +15,7 @@ import (
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
"go.uber.org/zap"
)
// ReplicationService 实现备份复制(3-2-1 规则核心)。
@@ -36,9 +37,10 @@ type ReplicationService struct {
eventDispatcher EventDispatcher
tempDir string
semaphore chan struct{}
async func(func())
async func(func(context.Context)) bool
now func() time.Time
metrics *metrics.Metrics
logger *zap.Logger
}
// SetMetrics 注入 Prometheus 采集器。
@@ -46,6 +48,19 @@ func (s *ReplicationService) SetMetrics(m *metrics.Metrics) {
s.metrics = m
}
func (s *ReplicationService) SetLogger(logger *zap.Logger) {
if logger != nil {
s.logger = logger
}
}
// SetBackgroundRunner binds replication work to the application lifecycle.
func (s *ReplicationService) SetBackgroundRunner(runner BackgroundRunner) {
if runner != nil {
s.async = runner.Go
}
}
func NewReplicationService(
replications repository.ReplicationRecordRepository,
records repository.BackupRecordRepository,
@@ -71,8 +86,9 @@ func NewReplicationService(
cipher: cipher,
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
async: func(job func()) { go job() },
async: runDetached,
now: func() time.Time { return time.Now().UTC() },
logger: zap.NewNop(),
}
}
@@ -123,13 +139,16 @@ func (s *ReplicationService) TriggerAutoReplication(ctx context.Context, task *m
}
// 跨节点 local_disk 场景保护:Master 无法访问远程节点本地文件
if err := s.validateClusterAccessible(ctx, record); err != nil {
s.logger.Warn("automatic replication skipped: source is not accessible", zap.Uint("backup_record_id", record.ID), zap.Error(err))
return
}
for _, destID := range destIDs {
if destID == record.StorageTargetID {
continue // 源与目标相同,跳过
}
_, _ = s.Start(ctx, record.ID, destID, "system")
if _, err := s.Start(ctx, record.ID, destID, "system"); err != nil {
s.logger.Warn("automatic replication start failed", zap.Uint("backup_record_id", record.ID), zap.Uint("dest_target_id", destID), zap.Error(err))
}
}
}
@@ -172,20 +191,24 @@ func (s *ReplicationService) Start(ctx context.Context, backupRecordID, destTarg
if err := s.replications.Create(ctx, rep); err != nil {
return nil, apperror.Internal("REPLICATION_CREATE_FAILED", "无法创建复制记录", err)
}
s.async(func() {
s.executeReplication(context.Background(), rep.ID)
})
repForRun := *rep
if !s.async(func(runCtx context.Context) {
s.executeReplication(runCtx, &repForRun)
}) {
message := "服务正在关闭,复制任务未启动"
if finalizeErr := s.finalizeReplication(ctx, rep, model.ReplicationStatusFailed, message, 0); finalizeErr != nil {
return nil, apperror.Internal("REPLICATION_FINALIZE_FAILED", "无法写回复制失败状态", finalizeErr)
}
return nil, backgroundTaskUnavailable("REPLICATION_SERVICE_SHUTTING_DOWN")
}
summary := s.toSummary(rep, "", dest.Name)
return &summary, nil
}
// executeReplication 实际执行:下载源对象到本地临时文件 → 上传到目标存储。
func (s *ReplicationService) executeReplication(ctx context.Context, repID uint) {
s.semaphore <- struct{}{}
defer func() { <-s.semaphore }()
rep, err := s.replications.FindByID(ctx, repID)
if err != nil || rep == nil {
func (s *ReplicationService) executeReplication(ctx context.Context, rep *model.ReplicationRecord) {
if rep == nil {
s.logger.Error("replication record is nil")
return
}
status := model.ReplicationStatusFailed
@@ -193,19 +216,25 @@ func (s *ReplicationService) executeReplication(ctx context.Context, repID uint)
fileSize := int64(0)
defer func() {
completedAt := s.now()
rep.Status = status
rep.FileSize = fileSize
rep.ErrorMessage = strings.TrimSpace(errMessage)
rep.DurationSeconds = int(completedAt.Sub(rep.StartedAt).Seconds())
rep.CompletedAt = &completedAt
_ = s.replications.Update(ctx, rep)
persistCtx, cancel := finalizationContext(ctx)
defer cancel()
if finalizeErr := s.finalizeReplication(persistCtx, rep, status, errMessage, fileSize); finalizeErr != nil {
s.logger.Error("finalize replication record failed", zap.Uint("replication_id", rep.ID), zap.Error(finalizeErr))
}
s.metrics.ObserveReplication(status)
if status == model.ReplicationStatusFailed {
s.dispatchFailed(ctx, rep, errMessage)
if dispatchErr := s.dispatchFailed(persistCtx, rep, errMessage); dispatchErr != nil {
s.logger.Warn("dispatch replication failure event failed", zap.Uint("replication_id", rep.ID), zap.Error(dispatchErr))
}
}
}()
if !acquireBackgroundSlot(ctx, s.semaphore) {
errMessage = ctx.Err().Error()
return
}
defer func() { <-s.semaphore }()
sourceProvider, err := s.resolveProvider(ctx, rep.SourceTargetID)
if err != nil {
errMessage = err.Error()
@@ -271,9 +300,19 @@ func (s *ReplicationService) validateClusterAccessible(ctx context.Context, reco
"REPLICATION_CROSS_NODE_LOCAL_DISK", "复制。请改用云存储作为主备份")
}
func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.ReplicationRecord, message string) {
func (s *ReplicationService) finalizeReplication(ctx context.Context, rep *model.ReplicationRecord, status, message string, fileSize int64) error {
completedAt := s.now()
rep.Status = status
rep.FileSize = fileSize
rep.ErrorMessage = strings.TrimSpace(message)
rep.DurationSeconds = int(completedAt.Sub(rep.StartedAt).Seconds())
rep.CompletedAt = &completedAt
return s.replications.Update(ctx, rep)
}
func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.ReplicationRecord, message string) error {
if s.eventDispatcher == nil || rep == nil {
return
return nil
}
title := "BackupX 备份复制失败"
body := fmt.Sprintf("备份记录:#%d\n源 → 目标:#%d → #%d\n错误:%s", rep.BackupRecordID, rep.SourceTargetID, rep.DestTargetID, message)
@@ -285,7 +324,7 @@ func (s *ReplicationService) dispatchFailed(ctx context.Context, rep *model.Repl
"destTargetId": rep.DestTargetID,
"error": message,
}
_ = s.eventDispatcher.DispatchEvent(ctx, model.NotificationEventReplicationFailed, title, body, fields)
return s.eventDispatcher.DispatchEvent(ctx, model.NotificationEventReplicationFailed, title, body, fields)
}
// List / Get / toSummary
@@ -47,6 +47,11 @@ func newReplicationTestHarness(t *testing.T) *replicationTestHarness {
if err != nil {
t.Fatalf("database.Open: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
cipher := codec.NewConfigCipher("replicate-secret")
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
@@ -109,8 +114,9 @@ func TestReplicationService_MirrorsToDestTarget(t *testing.T) {
}
done := make(chan struct{})
h.repl.async = func(job func()) {
go func() { job(); close(done) }()
h.repl.async = func(job func(context.Context)) bool {
go func() { job(context.Background()); close(done) }()
return true
}
summary, err := h.repl.Start(ctx, backupDetail.ID, 2, "tester")
if err != nil {
@@ -36,6 +36,7 @@ func newReportTestHarness(t *testing.T) (*ReportService, *BackupExecutionService
if err != nil {
t.Fatal(err)
}
closeTestDatabase(t, db)
cipher := codec.NewConfigCipher("report-secret")
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
+47 -17
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
@@ -39,7 +40,7 @@ type RestoreService struct {
eventDispatcher EventDispatcher
tempDir string
semaphore chan struct{}
async func(func())
async func(func(context.Context)) bool
now func() time.Time
metrics *metrics.Metrics
}
@@ -49,6 +50,13 @@ func (s *RestoreService) SetMetrics(m *metrics.Metrics) {
s.metrics = m
}
// SetBackgroundRunner binds local restore work to the application lifecycle.
func (s *RestoreService) SetBackgroundRunner(runner BackgroundRunner) {
if runner != nil {
s.async = runner.Go
}
}
// NewRestoreService 构造恢复服务。maxConcurrent 控制本地并发恢复数。
func NewRestoreService(
restores repository.RestoreRecordRepository,
@@ -83,7 +91,7 @@ func NewRestoreService(
dispatcher: dispatcher,
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
async: func(job func()) { go job() },
async: runDetached,
now: func() time.Time { return time.Now().UTC() },
}
}
@@ -187,12 +195,18 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
// 远程节点路由
if remoteNode := s.resolveRemoteNode(ctx, restoreNodeID); remoteNode != nil {
if s.dispatcher == nil {
message := "Agent 下发通道未就绪"
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, message); finalizeErr != nil {
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
}
return nil, apperror.Internal("RESTORE_DISPATCH_UNAVAILABLE", "Agent 下发通道未就绪", nil)
}
// 节点离线 → 立即标记 failed,避免记录永远卡在 running
if remoteNode.Status != model.NodeStatusOnline {
offlineMsg := fmt.Sprintf("节点 %s 当前离线,无法执行恢复", remoteNode.Name)
_ = s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, offlineMsg)
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, offlineMsg); finalizeErr != nil {
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
}
s.logHub.Append(restore.ID, "error", offlineMsg)
s.logHub.Complete(restore.ID, model.RestoreRecordStatusFailed)
return nil, apperror.BadRequest("NODE_OFFLINE", offlineMsg, nil)
@@ -200,8 +214,10 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
if _, dispatchErr := s.dispatcher.EnqueueCommand(ctx, restoreNodeID, model.AgentCommandTypeRestoreRecord, map[string]any{
"restoreRecordId": restore.ID,
}); dispatchErr != nil {
_ = s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed,
"下发恢复任务到远程节点失败: "+dispatchErr.Error())
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed,
"下发恢复任务到远程节点失败: "+dispatchErr.Error()); finalizeErr != nil {
dispatchErr = errors.Join(dispatchErr, finalizeErr)
}
return nil, apperror.Internal("AGENT_COMMAND_ENQUEUE_FAILED", "无法下发恢复任务到远程节点", dispatchErr)
}
s.logHub.Append(restore.ID, "info", fmt.Sprintf("已下发恢复任务到节点 %s#%d),等待 Agent 执行", remoteNode.Name, restoreNodeID))
@@ -209,10 +225,16 @@ func (s *RestoreService) StartSelective(ctx context.Context, backupRecordID uint
}
// 本地节点:异步执行
run := func() {
s.executeLocally(context.Background(), restore.ID, task, record, selectedPaths, targetPath)
run := func(runCtx context.Context) {
s.executeLocally(runCtx, restore.ID, task, record, selectedPaths, targetPath)
}
if !s.async(run) {
message := "服务正在关闭,恢复任务未启动"
if finalizeErr := s.finalize(ctx, restore.ID, model.RestoreRecordStatusFailed, message); finalizeErr != nil {
return nil, apperror.Internal("RESTORE_FINALIZE_FAILED", "无法写回恢复失败状态", finalizeErr)
}
return nil, backgroundTaskUnavailable("RESTORE_SERVICE_SHUTTING_DOWN")
}
s.async(run)
return s.getDetail(ctx, restore.ID)
}
@@ -238,22 +260,30 @@ func (s *RestoreService) resolveRemoteNode(ctx context.Context, nodeID uint) *mo
// executeLocally 在 Master 本地执行恢复。
func (s *RestoreService) executeLocally(ctx context.Context, restoreID uint, task *model.BackupTask, backupRecord *model.BackupRecord, selectedPaths []string, targetPath string) {
s.semaphore <- struct{}{}
defer func() { <-s.semaphore }()
logger := backup.NewExecutionLogger(restoreID, s.logHub)
status := model.RestoreRecordStatusFailed
errMessage := ""
defer func() {
finalizeErr := s.finalizeWithLog(ctx, restoreID, status, errMessage, logger.String())
persistCtx, cancel := finalizationContext(ctx)
defer cancel()
finalizeErr := s.finalizeWithLog(persistCtx, restoreID, status, errMessage, logger.String())
if finalizeErr != nil {
logger.Errorf("写回恢复记录失败:%v", finalizeErr)
}
s.logHub.Complete(restoreID, status)
s.dispatchRestoreEvent(ctx, restoreID, status, errMessage, task)
if dispatchErr := s.dispatchRestoreEvent(persistCtx, restoreID, status, errMessage, task); dispatchErr != nil {
logger.Warnf("派发恢复结果事件失败:%v", dispatchErr)
}
}()
if !acquireBackgroundSlot(ctx, s.semaphore) {
errMessage = ctx.Err().Error()
logger.Warnf("等待恢复执行槽时任务被取消:%v", ctx.Err())
return
}
defer func() { <-s.semaphore }()
logger.Infof("开始在本地执行恢复(备份记录 #%d)", backupRecord.ID)
spec, specErr := s.buildTaskSpec(task, backupRecord.StartedAt)
@@ -387,9 +417,9 @@ func backupKindLabel(kind string) string {
// dispatchRestoreEvent 按终态向事件总线派发 restore_success 或 restore_failed。
// eventDispatcher 未注入时静默忽略,保持向后兼容。
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) {
func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uint, status, errMessage string, task *model.BackupTask) error {
if s.eventDispatcher == nil {
return
return nil
}
var eventType, title string
switch status {
@@ -400,7 +430,7 @@ func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uin
eventType = model.NotificationEventRestoreFailed
title = "BackupX 恢复失败"
default:
return
return nil
}
taskName := "未知任务"
if task != nil {
@@ -419,7 +449,7 @@ func (s *RestoreService) dispatchRestoreEvent(ctx context.Context, restoreID uin
if task != nil {
fields["taskId"] = task.ID
}
_ = s.eventDispatcher.DispatchEvent(ctx, eventType, title, body, fields)
return s.eventDispatcher.DispatchEvent(ctx, eventType, title, body, fields)
}
// resolveProvider 解密存储目标配置并创建 provider(共享实现)。
@@ -84,6 +84,11 @@ func newRestoreTestHarness(t *testing.T, remoteNode bool) *restoreTestHarness {
if err != nil {
t.Fatalf("database.Open: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
cipher := codec.NewConfigCipher("restore-secret")
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
@@ -159,11 +164,12 @@ func TestRestoreServiceStart_LocalNodeExecutesInline(t *testing.T) {
// 用同步 async 让测试可等待
done := make(chan struct{})
h.service.async = func(job func()) {
h.service.async = func(job func(context.Context)) bool {
go func() {
job()
job(context.Background())
close(done)
}()
return true
}
detail, err := h.service.Start(ctx, backupDetail.ID, "tester")
if err != nil {
@@ -200,6 +206,62 @@ func TestRestoreServiceStart_LocalNodeExecutesInline(t *testing.T) {
}
}
func TestRestoreServiceStart_RepositoryRecord(t *testing.T) {
h := newRestoreTestHarness(t, false)
ctx := context.Background()
task, err := h.tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task: %v", err)
}
task.BackupMode = model.BackupModeRepository
if err := h.tasks.Update(ctx, task); err != nil {
t.Fatalf("Update repository task: %v", err)
}
backupDetail, err := h.execution.RunTaskByIDSync(ctx, task.ID)
if err != nil {
t.Fatalf("RunTaskByIDSync repository: %v", err)
}
if backupDetail.BackupKind != model.BackupKindRepository {
t.Fatalf("expected repository backup, got %#v", backupDetail)
}
if err := os.RemoveAll(h.sourceDir); err != nil {
t.Fatalf("remove source: %v", err)
}
done := make(chan struct{})
h.service.async = func(job func(context.Context)) bool {
go func() {
job(context.Background())
close(done)
}()
return true
}
detail, err := h.service.Start(ctx, backupDetail.ID, "repository-test")
if err != nil {
t.Fatalf("Start repository restore: %v", err)
}
select {
case <-done:
case <-time.After(15 * time.Second):
t.Fatal("repository restore did not complete in time")
}
final, err := h.service.Get(ctx, detail.ID)
if err != nil {
t.Fatalf("Get repository restore: %v", err)
}
if final.Status != model.RestoreRecordStatusSuccess {
t.Fatalf("expected repository restore success, got %s (err=%s)", final.Status, final.ErrorMessage)
}
content, err := os.ReadFile(filepath.Join(h.sourceDir, "index.html"))
if err != nil {
t.Fatalf("read repository-restored file: %v", err)
}
if string(content) != "hello-restore" {
t.Fatalf("unexpected repository-restored content: %q", content)
}
}
// TestRestoreServiceStart_RejectsCorruptedBackup 验证恢复在还原前做 SHA-256 完整性
// 校验:若已存储的备份对象被损坏/篡改,恢复必须失败且不触碰源数据。
func TestRestoreServiceStart_RejectsCorruptedBackup(t *testing.T) {
@@ -242,8 +304,9 @@ func TestRestoreServiceStart_RejectsCorruptedBackup(t *testing.T) {
}
done := make(chan struct{})
h.service.async = func(job func()) {
go func() { job(); close(done) }()
h.service.async = func(job func(context.Context)) bool {
go func() { job(context.Background()); close(done) }()
return true
}
detail, err := h.service.Start(ctx, backupDetail.ID, "tester")
if err != nil {
@@ -293,11 +356,12 @@ func TestRestoreServiceStart_RestoresToAlternatePath(t *testing.T) {
}
done := make(chan struct{})
h.service.async = func(job func()) {
h.service.async = func(job func(context.Context)) bool {
go func() {
job()
job(context.Background())
close(done)
}()
return true
}
detail, err := h.service.StartSelective(ctx, backupDetail.ID, nil, altDir, "tester")
if err != nil {
@@ -1,71 +0,0 @@
package service
import (
"context"
"time"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
)
type RetentionService struct {
records repository.BackupRecordRepository
storageTargets repository.StorageTargetRepository
storageRegistry *storage.Registry
cipher *codec.ConfigCipher
}
func NewRetentionService(records repository.BackupRecordRepository, storageTargets repository.StorageTargetRepository, storageRegistry *storage.Registry, cipher *codec.ConfigCipher) *RetentionService {
return &RetentionService{records: records, storageTargets: storageTargets, storageRegistry: storageRegistry, cipher: cipher}
}
func (s *RetentionService) Apply(ctx context.Context, task *model.BackupTask) error {
if task == nil || (task.RetentionDays <= 0 && task.MaxBackups <= 0) {
return nil
}
items, err := s.records.ListSuccessfulByTask(ctx, task.ID)
if err != nil {
return err
}
removeSet := make(map[uint]model.BackupRecord)
if task.RetentionDays > 0 {
cutoff := time.Now().UTC().AddDate(0, 0, -task.RetentionDays)
for _, item := range items {
if item.CompletedAt != nil && item.CompletedAt.Before(cutoff) {
removeSet[item.ID] = item
}
}
}
if task.MaxBackups > 0 {
kept := 0
for _, item := range items {
if _, marked := removeSet[item.ID]; marked {
continue
}
kept++
if kept > task.MaxBackups {
removeSet[item.ID] = item
}
}
}
if len(removeSet) == 0 {
return nil
}
provider, _, err := buildStorageProviderFromRepos(ctx, task.StorageTargetID, s.storageTargets, s.storageRegistry, s.cipher)
if err != nil {
return err
}
for _, item := range removeSet {
if item.StoragePath != "" {
if err := provider.Delete(ctx, item.StoragePath); err != nil {
return err
}
}
if err := s.records.Delete(ctx, item.ID); err != nil {
return err
}
}
return nil
}
@@ -2,6 +2,7 @@ package service
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
@@ -95,6 +96,7 @@ type StorageTargetService struct {
records repository.BackupRecordRepository
registry *storage.Registry
cipher *codec.ConfigCipher
background BackgroundRunner
}
func NewStorageTargetService(
@@ -114,6 +116,10 @@ func (s *StorageTargetService) SetBackupRecordRepository(records repository.Back
s.records = records
}
func (s *StorageTargetService) SetBackgroundRunner(runner BackgroundRunner) {
s.background = runner
}
func (s *StorageTargetService) List(ctx context.Context) ([]StorageTargetSummary, error) {
items, err := s.targets.List(ctx)
if err != nil {
@@ -254,7 +260,12 @@ func (s *StorageTargetService) TestConnection(ctx context.Context, input Storage
item.LastTestMessage = "连接成功"
}
if item.ID != 0 {
_ = s.targets.Update(ctx, item)
if updateErr := s.targets.Update(ctx, item); updateErr != nil {
if testErr != nil {
return apperror.BadRequest("STORAGE_TARGET_TEST_FAILED", sanitizeMessage(testErr.Error()), errors.Join(testErr, fmt.Errorf("save connection test result: %w", updateErr)))
}
return apperror.Internal("STORAGE_TARGET_TEST_RESULT_SAVE_FAILED", "连接成功,但无法保存测试结果", updateErr)
}
}
if testErr != nil {
return apperror.BadRequest("STORAGE_TARGET_TEST_FAILED", sanitizeMessage(testErr.Error()), testErr)
@@ -269,23 +280,23 @@ func (s *StorageTargetService) StartHealthMonitor(ctx context.Context, dispatche
if interval <= 0 {
interval = 5 * time.Minute
}
ticker := time.NewTicker(interval)
// notified 跟踪已告警的目标,避免每轮重复
notified := map[uint]bool{}
capacityNotified := map[uint]bool{}
var mu sync.Mutex
go func() {
startBackgroundMonitor(s.background, ctx, func(runCtx context.Context) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
case <-runCtx.Done():
return
case <-ticker.C:
s.runHealthCheckOnce(ctx, dispatcher, &mu, notified)
s.runCapacityCheckOnce(ctx, dispatcher, &mu, capacityNotified)
s.runHealthCheckOnce(runCtx, dispatcher, &mu, notified)
s.runCapacityCheckOnce(runCtx, dispatcher, &mu, capacityNotified)
}
}
}()
})
}
// StorageCapacityWarningThreshold 存储使用率告警阈值(85%)。
@@ -371,7 +382,6 @@ func (s *StorageTargetService) runHealthCheckOnce(ctx context.Context, dispatche
if !target.Enabled {
continue
}
previousStatus := target.LastTestStatus
configMap := map[string]any{}
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
continue
@@ -380,13 +390,13 @@ func (s *StorageTargetService) runHealthCheckOnce(ctx context.Context, dispatche
now := time.Now().UTC()
if err != nil {
s.applyHealthResult(ctx, &target, now, false, err.Error())
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, previousStatus, err.Error())
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, err.Error())
continue
}
testErr := provider.TestConnection(ctx)
if testErr != nil {
s.applyHealthResult(ctx, &target, now, false, testErr.Error())
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, previousStatus, testErr.Error())
s.notifyUnhealthyTransition(ctx, dispatcher, mu, notified, &target, testErr.Error())
continue
}
s.applyHealthResult(ctx, &target, now, true, "连接成功")
@@ -408,7 +418,7 @@ func (s *StorageTargetService) applyHealthResult(ctx context.Context, target *mo
_ = s.targets.Update(ctx, target)
}
func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, dispatcher EventDispatcher, mu *sync.Mutex, notified map[uint]bool, target *model.StorageTarget, previousStatus string, message string) {
func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, dispatcher EventDispatcher, mu *sync.Mutex, notified map[uint]bool, target *model.StorageTarget, message string) {
if dispatcher == nil {
return
}
@@ -423,7 +433,6 @@ func (s *StorageTargetService) notifyUnhealthyTransition(ctx context.Context, di
if already {
return
}
_ = previousStatus // 保留参数便于未来扩展:区分"从未测试"与"从 success 掉线"
title := "BackupX 存储目标连接失败"
body := fmt.Sprintf("存储目标:%s (类型: %s)\n错误:%s", target.Name, target.Type, message)
fields := map[string]any{
@@ -473,7 +482,9 @@ func (s *StorageTargetService) CompleteGoogleDriveOAuth(ctx context.Context, inp
// Mark used immediately to prevent duplicate requests (e.g. React StrictMode double invocation)
now := time.Now().UTC()
session.UsedAt = &now
_ = s.oauthSessions.Update(ctx, session)
if err := s.oauthSessions.Update(ctx, session); err != nil {
return nil, apperror.Internal("STORAGE_GOOGLE_OAUTH_SESSION_FAILED", "无法锁定 Google Drive 授权会话", err)
}
var draft googleDriveOAuthDraft
if err := s.cipher.DecryptJSON(session.PayloadCiphertext, &draft); err != nil {
@@ -180,7 +180,7 @@ func (s *TaskExportService) Import(ctx context.Context, payload ExportPayload) (
results = append(results, ImportResult{Name: t.Name, TaskID: detail.ID, Success: true})
}
// 第二阶段:依赖链接(上游任务名 → 新 ID)
for i, t := range payload.Tasks {
for _, t := range payload.Tasks {
if len(t.DependsOnTaskNames) == 0 {
continue
}
@@ -206,7 +206,6 @@ func (s *TaskExportService) Import(ctx context.Context, payload ExportPayload) (
break
}
}
_ = i
}
}
return results, nil
@@ -3,7 +3,6 @@ package service
import (
"context"
"encoding/json"
"fmt"
"strings"
"backupx/server/internal/apperror"
@@ -235,6 +234,3 @@ func toTemplateSummary(item *model.TaskTemplate) TaskTemplateSummary {
UpdatedAt: item.UpdatedAt.Format("2006-01-02T15:04:05Z07:00"),
}
}
// 确保未使用告警
var _ = fmt.Sprintf
@@ -0,0 +1,22 @@
package service
import (
"testing"
"gorm.io/gorm"
)
// closeTestDatabase releases SQLite file handles before testing.TempDir cleanup.
// Windows does not permit removal of an open database file.
func closeTestDatabase(t *testing.T, db *gorm.DB) {
t.Helper()
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("get test database handle: %v", err)
}
t.Cleanup(func() {
if err := sqlDB.Close(); err != nil {
t.Errorf("close test database: %v", err)
}
})
}
+36 -11
View File
@@ -39,7 +39,7 @@ type VerificationService struct {
notifier VerificationNotifier
tempDir string
semaphore chan struct{}
async func(func())
async func(func(context.Context)) bool
now func() time.Time
metrics *metrics.Metrics
}
@@ -49,6 +49,13 @@ func (s *VerificationService) SetMetrics(m *metrics.Metrics) {
s.metrics = m
}
// SetBackgroundRunner binds local verification work to the application lifecycle.
func (s *VerificationService) SetBackgroundRunner(runner BackgroundRunner) {
if runner != nil {
s.async = runner.Go
}
}
// VerificationNotifier 给用户推送验证完成/失败通知。
// 可选注入:未注入时仅写记录。
type VerificationNotifier interface {
@@ -129,7 +136,7 @@ func NewVerificationService(
notifier: noopVerificationNotifier{},
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
async: func(job func()) { go job() },
async: runDetached,
now: func() time.Time { return time.Now().UTC() },
}
}
@@ -231,10 +238,16 @@ func (s *VerificationService) Start(ctx context.Context, backupRecordID uint, mo
if err := s.verifications.Create(ctx, verification); err != nil {
return nil, apperror.Internal("VERIFY_RECORD_CREATE_FAILED", "无法创建验证记录", err)
}
run := func() {
s.executeLocally(context.Background(), verification.ID, task, record)
run := func(runCtx context.Context) {
s.executeLocally(runCtx, verification.ID, task, record)
}
if !s.async(run) {
message := "服务正在关闭,验证任务未启动"
if finalizeErr := s.finalize(ctx, verification.ID, model.VerificationRecordStatusFailed, message, "", ""); finalizeErr != nil {
return nil, apperror.Internal("VERIFY_FINALIZE_FAILED", "无法写回验证失败状态", finalizeErr)
}
return nil, backgroundTaskUnavailable("VERIFY_SERVICE_SHUTTING_DOWN")
}
s.async(run)
return s.getDetail(ctx, verification.ID)
}
@@ -247,25 +260,37 @@ func (s *VerificationService) validateClusterAccessible(ctx context.Context, rec
// executeLocally 异步执行验证:下载 → 解密 → 解压 → 按类型校验。
func (s *VerificationService) executeLocally(ctx context.Context, verID uint, task *model.BackupTask, backupRecord *model.BackupRecord) {
s.semaphore <- struct{}{}
defer func() { <-s.semaphore }()
logger := backup.NewExecutionLogger(verID, s.logHub)
status := model.VerificationRecordStatusFailed
errMessage := ""
summary := ""
defer func() {
_ = s.finalize(ctx, verID, status, errMessage, summary, logger.String())
persistCtx, cancel := finalizationContext(ctx)
defer cancel()
if finalizeErr := s.finalize(persistCtx, verID, status, errMessage, summary, logger.String()); finalizeErr != nil {
logger.Errorf("写回验证记录失败:%v", finalizeErr)
}
s.logHub.Complete(verID, status)
// 失败时推送通知(best-effort
if status == model.VerificationRecordStatusFailed && s.notifier != nil {
if record, err := s.verifications.FindByID(ctx, verID); err == nil && record != nil {
_ = s.notifier.NotifyVerificationResult(ctx, task, record)
if record, findErr := s.verifications.FindByID(persistCtx, verID); findErr != nil {
logger.Warnf("读取验证记录以发送通知失败:%v", findErr)
} else if record != nil {
if notifyErr := s.notifier.NotifyVerificationResult(persistCtx, task, record); notifyErr != nil {
logger.Warnf("发送验证失败通知失败:%v", notifyErr)
}
}
}
}()
if !acquireBackgroundSlot(ctx, s.semaphore) {
errMessage = ctx.Err().Error()
logger.Warnf("等待验证执行槽时任务被取消:%v", ctx.Err())
return
}
defer func() { <-s.semaphore }()
logger.Infof("开始验证备份记录 #%d(模式:%s)", backupRecord.ID, model.VerificationModeQuick)
if err := os.MkdirAll(s.tempDir, 0o755); err != nil {
@@ -11,6 +11,7 @@ import (
"backupx/server/internal/backup"
"backupx/server/internal/config"
"backupx/server/internal/database"
"backupx/server/internal/lifecycle"
"backupx/server/internal/logger"
"backupx/server/internal/model"
"backupx/server/internal/repository"
@@ -45,6 +46,11 @@ func newVerifyTestHarness(t *testing.T) *verifyTestHarness {
if err != nil {
t.Fatalf("database.Open: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
cipher := codec.NewConfigCipher("verify-secret")
targets := repository.NewStorageTargetRepository(db)
tasks := repository.NewBackupTaskRepository(db)
@@ -77,8 +83,9 @@ func (h *verifyTestHarness) runVerify(t *testing.T, backupRecordID uint) *Verifi
t.Helper()
ctx := context.Background()
done := make(chan struct{})
h.verify.async = func(job func()) {
go func() { job(); close(done) }()
h.verify.async = func(job func(context.Context)) bool {
go func() { job(context.Background()); close(done) }()
return true
}
detail, err := h.verify.Start(ctx, backupRecordID, "quick", "tester")
if err != nil {
@@ -96,6 +103,48 @@ func (h *verifyTestHarness) runVerify(t *testing.T, backupRecordID uint) *Verifi
return final
}
func TestVerificationServiceSupervisorCancellationFinalizesRecord(t *testing.T) {
h := newVerifyTestHarness(t)
backupDetail, err := h.execution.RunTaskByIDSync(context.Background(), 1)
if err != nil {
t.Fatalf("RunTaskByIDSync: %v", err)
}
supervisor := lifecycle.NewSupervisor(context.Background())
h.verify.SetBackgroundRunner(supervisor)
// Occupy the only available execution path before cancellation so the test
// deterministically exercises cancellation while queued.
for i := 0; i < cap(h.verify.semaphore); i++ {
h.verify.semaphore <- struct{}{}
}
detail, err := h.verify.Start(context.Background(), backupDetail.ID, model.VerificationModeQuick, "tester")
if err != nil {
t.Fatalf("Start: %v", err)
}
waitCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := supervisor.Shutdown(waitCtx); err != nil {
t.Fatalf("Shutdown: %v", err)
}
for i := 0; i < cap(h.verify.semaphore); i++ {
<-h.verify.semaphore
}
final, err := h.verify.Get(context.Background(), detail.ID)
if err != nil {
t.Fatalf("Get: %v", err)
}
if final.Status != model.VerificationRecordStatusFailed {
t.Fatalf("status = %q, want failed", final.Status)
}
if final.CompletedAt == nil {
t.Fatal("canceled verification was not finalized")
}
if !strings.Contains(strings.ToLower(final.ErrorMessage), "canceled") {
t.Fatalf("error message = %q, want cancellation", final.ErrorMessage)
}
}
// TestVerificationService_Success 覆盖正常路径:对一个有效(gzip 压缩)的备份做验证应通过。
// 同时回归保护 #77——新增的 SHA-256 校验不得误伤合法的压缩备份。
func TestVerificationService_Success(t *testing.T) {