refactor: simplify architecture and harden lifecycle

Remove obsolete implementations, centralize background task ownership and terminal-state recovery, consolidate frontend routing and log streaming, and enforce project-wide verification in CI.
This commit is contained in:
Awuqing
2026-08-26 11:32:00 +08:00
parent 95b27af600
commit c06bbec383
189 changed files with 8983 additions and 4264 deletions
+71 -27
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net"
"os"
"runtime"
@@ -13,6 +12,7 @@ import (
"time"
"backupx/server/internal/backup"
"go.uber.org/zap"
)
// Agent 是 Agent 进程的主控制器。
@@ -21,6 +21,7 @@ type Agent struct {
client *MasterClient
executor *Executor
version string
logger *zap.Logger
mu sync.Mutex
started bool
@@ -44,9 +45,20 @@ func New(cfg *Config, version string) (*Agent, error) {
client: client,
executor: executor,
version: version,
logger: zap.NewNop(),
}, nil
}
// SetLogger attaches the process logger used by the Agent runtime loop.
func (a *Agent) SetLogger(logger *zap.Logger) {
if logger != nil {
a.logger = logger
if a.executor != nil {
a.executor.SetLogger(logger)
}
}
}
// Run 启动 Agent 主循环,阻塞直到 ctx 被取消。
func (a *Agent) Run(ctx context.Context) error {
a.mu.Lock()
@@ -64,7 +76,7 @@ func (a *Agent) Run(ctx context.Context) error {
if err := a.heartbeatOnce(ctx); err != nil {
return fmt.Errorf("initial heartbeat failed: %w", err)
}
log.Printf("[agent] connected to master %s", a.cfg.Master)
a.logger.Info("agent connected to master", zap.String("master", a.cfg.Master))
var wg sync.WaitGroup
wg.Add(2)
@@ -90,14 +102,17 @@ func (a *Agent) heartbeatLoop(ctx context.Context, interval time.Duration) {
return
case <-ticker.C:
if err := a.heartbeatOnce(ctx); err != nil {
log.Printf("[agent] heartbeat failed: %v", err)
a.logger.Warn("agent heartbeat failed", zap.Error(err))
}
}
}
}
func (a *Agent) heartbeatOnce(ctx context.Context) error {
hostname, _ := os.Hostname()
hostname, err := os.Hostname()
if err != nil {
a.logger.Warn("resolve agent hostname failed", zap.Error(err))
}
req := HeartbeatRequest{
Hostname: hostname,
IPAddress: detectLocalIP(),
@@ -105,7 +120,7 @@ func (a *Agent) heartbeatOnce(ctx context.Context) error {
OS: runtime.GOOS,
Arch: runtime.GOARCH,
}
_, err := a.client.Heartbeat(ctx, req)
_, err = a.client.Heartbeat(ctx, req)
return err
}
@@ -126,13 +141,13 @@ func (a *Agent) pollLoop(ctx context.Context, interval time.Duration) {
func (a *Agent) pollAndHandleOnce(ctx context.Context) {
cmd, err := a.client.PollCommand(ctx)
if err != nil {
log.Printf("[agent] poll command failed: %v", err)
a.logger.Warn("poll agent command failed", zap.Error(err))
return
}
if cmd == nil {
return
}
log.Printf("[agent] received command #%d type=%s", cmd.ID, cmd.Type)
a.logger.Info("agent command received", zap.Uint("command_id", cmd.ID), zap.String("command_type", cmd.Type))
switch cmd.Type {
case "run_task":
a.handleRunTask(ctx, cmd)
@@ -146,8 +161,8 @@ func (a *Agent) pollAndHandleOnce(ctx context.Context) {
a.handleDeleteStorageObject(ctx, cmd)
default:
msg := fmt.Sprintf("unknown command type: %s", cmd.Type)
log.Printf("[agent] %s", msg)
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, msg, nil)
a.logger.Warn("unknown agent command", zap.Uint("command_id", cmd.ID), zap.String("command_type", cmd.Type))
a.submitCommandResult(ctx, cmd.ID, false, msg, nil)
}
}
@@ -158,14 +173,14 @@ func (a *Agent) handleRunTask(ctx context.Context, cmd *CommandPayload) {
RecordID uint `json:"recordId"`
}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
return
}
if err := a.executor.ExecuteRunTask(ctx, payload.TaskID, payload.RecordID); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
return
}
_ = a.client.SubmitCommandResult(ctx, cmd.ID, true, "", map[string]any{
a.submitCommandResult(ctx, cmd.ID, true, "", map[string]any{
"taskId": payload.TaskID,
"recordId": payload.RecordID,
})
@@ -177,18 +192,18 @@ func (a *Agent) handleRestoreRecord(ctx context.Context, cmd *CommandPayload) {
RestoreRecordID uint `json:"restoreRecordId"`
}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
return
}
if payload.RestoreRecordID == 0 {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "restoreRecordId is required", nil)
a.submitCommandResult(ctx, cmd.ID, false, "restoreRecordId is required", nil)
return
}
if err := a.executor.ExecuteRestore(ctx, payload.RestoreRecordID); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
return
}
_ = a.client.SubmitCommandResult(ctx, cmd.ID, true, "", map[string]any{
a.submitCommandResult(ctx, cmd.ID, true, "", map[string]any{
"restoreRecordId": payload.RestoreRecordID,
})
}
@@ -202,23 +217,23 @@ func (a *Agent) handleDeleteStorageObject(ctx context.Context, cmd *CommandPaylo
StoragePath string `json:"storagePath"`
}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
return
}
if strings.TrimSpace(payload.StoragePath) == "" {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "storagePath is required", nil)
a.submitCommandResult(ctx, cmd.ID, false, "storagePath is required", nil)
return
}
provider, err := a.executor.storageRegistry.Create(ctx, payload.TargetType, payload.TargetConfig)
if err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "create provider: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "create provider: "+err.Error(), nil)
return
}
if err := provider.Delete(ctx, payload.StoragePath); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "delete object: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "delete object: "+err.Error(), nil)
return
}
_ = a.client.SubmitCommandResult(ctx, cmd.ID, true, "", map[string]any{"deleted": true})
a.submitCommandResult(ctx, cmd.ID, true, "", map[string]any{"deleted": true})
}
// handleDiscoverDB 处理 discover_db 命令:在 Agent 本机执行 mysql/psql 列出数据库。
@@ -231,7 +246,7 @@ func (a *Agent) handleDiscoverDB(ctx context.Context, cmd *CommandPayload) {
Password string `json:"password"`
}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
return
}
databases, err := backup.DiscoverDatabases(ctx, backup.NewOSCommandExecutor(), backup.DiscoverRequest{
@@ -242,10 +257,10 @@ func (a *Agent) handleDiscoverDB(ctx context.Context, cmd *CommandPayload) {
Password: payload.Password,
})
if err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
return
}
_ = a.client.SubmitCommandResult(ctx, cmd.ID, true, "", map[string]any{"databases": databases})
a.submitCommandResult(ctx, cmd.ID, true, "", map[string]any{"databases": databases})
}
// handleListDir 处理 list_dir 命令(阶段四实现)
@@ -254,15 +269,44 @@ func (a *Agent) handleListDir(ctx context.Context, cmd *CommandPayload) {
Path string `json:"path"`
}
if err := json.Unmarshal(cmd.Payload, &payload); err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, "invalid payload: "+err.Error(), nil)
return
}
entries, err := listLocalDir(payload.Path)
if err != nil {
_ = a.client.SubmitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
a.submitCommandResult(ctx, cmd.ID, false, err.Error(), nil)
return
}
_ = a.client.SubmitCommandResult(ctx, cmd.ID, true, "", map[string]any{"entries": entries})
a.submitCommandResult(ctx, cmd.ID, true, "", map[string]any{"entries": entries})
}
func (a *Agent) submitCommandResult(ctx context.Context, commandID uint, success bool, message string, data any) {
reportCtx, cancel := agentFinalizationContext(ctx)
defer cancel()
var err error
attempts := 0
retryLoop:
for attempts < 3 {
attempts++
err = a.client.SubmitCommandResult(reportCtx, commandID, success, message, data)
if err == nil {
return
}
if attempts < 3 {
timer := time.NewTimer(time.Duration(attempts) * 100 * time.Millisecond)
select {
case <-reportCtx.Done():
timer.Stop()
break retryLoop
case <-timer.C:
}
}
}
a.logger.Error("submit agent command result failed",
zap.Uint("command_id", commandID),
zap.Bool("success", success),
zap.Int("attempts", attempts),
zap.Error(err))
}
// 辅助函数
+36
View File
@@ -0,0 +1,36 @@
package agent
import (
"context"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"go.uber.org/zap"
)
func TestSubmitCommandResultRetriesWithCanceledCommandContext(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if requests.Add(1) < 3 {
http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
agent := &Agent{
client: NewMasterClient(server.URL, "token", false),
logger: zap.NewNop(),
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
agent.submitCommandResult(ctx, 17, false, "backup failed", nil)
if got := requests.Load(); got != 3 {
t.Fatalf("submit requests = %d, want 3", got)
}
}
+79 -38
View File
@@ -17,6 +17,7 @@ import (
"backupx/server/internal/storage"
storageRclone "backupx/server/internal/storage/rclone"
"backupx/server/pkg/compress"
"go.uber.org/zap"
)
// Executor 负责在 Agent 本地执行命令。
@@ -25,35 +26,26 @@ type Executor struct {
tempDir string
backupRegistry *backup.Registry
storageRegistry *storage.Registry
logger *zap.Logger
}
// NewExecutor 构造执行器。预先初始化 backup runner 与 storage registry。
func NewExecutor(client *MasterClient, tempDir string) *Executor {
backupRegistry := backup.NewRegistry(
backup.NewFileRunner(),
backup.NewSQLiteRunner(),
backup.NewMySQLRunner(nil),
backup.NewPostgreSQLRunner(nil),
backup.NewSAPHANARunner(nil),
backup.NewMongoDBRunner(nil),
)
storageRegistry := storage.NewRegistry(
storageRclone.NewLocalDiskFactory(),
storageRclone.NewS3Factory(),
storageRclone.NewWebDAVFactory(),
storageRclone.NewGoogleDriveFactory(),
storageRclone.NewAliyunOSSFactory(),
storageRclone.NewTencentCOSFactory(),
storageRclone.NewQiniuKodoFactory(),
storageRclone.NewFTPFactory(),
storageRclone.NewRcloneFactory(),
)
storageRclone.RegisterAllBackends(storageRegistry)
backupRegistry := backup.NewDefaultRegistry()
storageRegistry := storageRclone.NewDefaultRegistry()
return &Executor{
client: client,
tempDir: tempDir,
backupRegistry: backupRegistry,
storageRegistry: storageRegistry,
logger: zap.NewNop(),
}
}
// SetLogger attaches the Agent process logger to execution and reporting paths.
func (e *Executor) SetLogger(logger *zap.Logger) {
if logger != nil {
e.logger = logger
}
}
@@ -90,7 +82,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
}
// 3) 运行 runner
logger := newRecordLogger(ctx, e.client, recordID)
logger := newRecordLogger(ctx, e.client, e.logger, recordID)
result, err := runner.Run(ctx, backupSpec, logger)
if err != nil {
e.reportRecordFailure(ctx, recordID, err.Error())
@@ -177,7 +169,9 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
}
// 6) 上报最终成功
return e.client.UpdateRecord(ctx, recordID, RecordUpdate{
reportCtx, cancel := agentFinalizationContext(ctx)
defer cancel()
if err := e.client.UpdateRecord(reportCtx, recordID, RecordUpdate{
Status: "success",
FileName: fileName,
FileSize: fileSize,
@@ -187,7 +181,10 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
StorageTransferMode: selectedStorageTransferMode,
StorageUploadResults: uploadResults,
LogAppend: fmt.Sprintf("[agent] 任务完成,总计 %d 字节\n", fileSize),
})
}); err != nil {
return fmt.Errorf("report backup success to master: %w", err)
}
return nil
}
// uploadToTarget 上传单个目标。为保持简化不做上传级重试(rclone 本身已有 low-level 重试)。
@@ -222,7 +219,11 @@ func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target Sto
// appendLog 追加日志到 Master 记录(尽力而为,失败不中断主流程)
func (e *Executor) appendLog(ctx context.Context, recordID uint, line string) {
_ = e.client.UpdateRecord(ctx, recordID, RecordUpdate{LogAppend: line})
if err := e.client.UpdateRecord(ctx, recordID, RecordUpdate{LogAppend: line}); err != nil {
e.logger.Warn("append backup record log to master failed",
zap.Uint("record_id", recordID),
zap.Error(err))
}
}
// reportRecordFailure 上报失败状态
@@ -231,12 +232,27 @@ func (e *Executor) reportRecordFailure(ctx context.Context, recordID uint, msg s
}
func (e *Executor) reportRecordFailureWithUploadResults(ctx context.Context, recordID uint, msg string, uploadResults []StorageResultItem) {
_ = e.client.UpdateRecord(ctx, recordID, RecordUpdate{
reportCtx, cancel := agentFinalizationContext(ctx)
defer cancel()
if err := e.client.UpdateRecord(reportCtx, recordID, RecordUpdate{
Status: "failed",
ErrorMessage: msg,
StorageUploadResults: uploadResults,
LogAppend: fmt.Sprintf("[agent] 错误: %s\n", msg),
})
}); err != nil {
e.logger.Error("report backup failure to master failed",
zap.Uint("record_id", recordID),
zap.Error(err))
}
}
// agentFinalizationContext lets terminal state reach the Master even when the
// command context was canceled, while bounding shutdown/network delays.
func agentFinalizationContext(ctx context.Context) (context.Context, context.CancelFunc) {
if ctx == nil {
ctx = context.Background()
}
return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
}
// buildBackupTaskSpec 把 AgentTaskSpec 转换为 backup.TaskSpec。
@@ -301,30 +317,40 @@ func compactStringList(items []string) []string {
type recordLogger struct {
ctx context.Context
client *MasterClient
logger *zap.Logger
recordID uint
}
func newRecordLogger(ctx context.Context, client *MasterClient, recordID uint) *recordLogger {
return &recordLogger{ctx: ctx, client: client, recordID: recordID}
func newRecordLogger(ctx context.Context, client *MasterClient, logger *zap.Logger, recordID uint) *recordLogger {
return &recordLogger{ctx: ctx, client: client, logger: logger, recordID: recordID}
}
func (l *recordLogger) WriteLine(message string) {
_ = l.client.UpdateRecord(l.ctx, l.recordID, RecordUpdate{LogAppend: message + "\n"})
if err := l.client.UpdateRecord(l.ctx, l.recordID, RecordUpdate{LogAppend: message + "\n"}); err != nil {
l.logger.Warn("append backup runner log to master failed",
zap.Uint("record_id", l.recordID),
zap.Error(err))
}
}
// restoreLogger 把 runner 日志回传到 Master 恢复记录。
type restoreLogger struct {
ctx context.Context
client *MasterClient
logger *zap.Logger
restoreID uint
}
func newRestoreLogger(ctx context.Context, client *MasterClient, restoreID uint) *restoreLogger {
return &restoreLogger{ctx: ctx, client: client, restoreID: restoreID}
func newRestoreLogger(ctx context.Context, client *MasterClient, logger *zap.Logger, restoreID uint) *restoreLogger {
return &restoreLogger{ctx: ctx, client: client, logger: logger, restoreID: restoreID}
}
func (l *restoreLogger) WriteLine(message string) {
_ = l.client.UpdateRestore(l.ctx, l.restoreID, RestoreUpdate{LogAppend: message + "\n"})
if err := l.client.UpdateRestore(l.ctx, l.restoreID, RestoreUpdate{LogAppend: message + "\n"}); err != nil {
l.logger.Warn("append restore runner log to master failed",
zap.Uint("restore_record_id", l.restoreID),
zap.Error(err))
}
}
// DeleteStorageObject 在 Agent 本机上删除指定存储对象(供跨节点清理调用)。
@@ -453,29 +479,44 @@ func (e *Executor) ExecuteRestore(ctx context.Context, restoreRecordID uint) err
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("不支持的备份类型: %v", err))
return err
}
logger := newRestoreLogger(ctx, e.client, restoreRecordID)
logger := newRestoreLogger(ctx, e.client, e.logger, restoreRecordID)
if err := runner.Restore(ctx, taskSpec, preparedPath, logger); err != nil {
e.reportRestoreFailure(ctx, restoreRecordID, err.Error())
return err
}
// 5) 上报成功
return e.client.UpdateRestore(ctx, restoreRecordID, RestoreUpdate{
reportCtx, cancel := agentFinalizationContext(ctx)
defer cancel()
if err := e.client.UpdateRestore(reportCtx, restoreRecordID, RestoreUpdate{
Status: "success",
LogAppend: "[agent] 恢复执行完成\n",
})
}); err != nil {
return fmt.Errorf("report restore success to master: %w", err)
}
return nil
}
func (e *Executor) appendRestoreLog(ctx context.Context, restoreID uint, line string) {
_ = e.client.UpdateRestore(ctx, restoreID, RestoreUpdate{LogAppend: line})
if err := e.client.UpdateRestore(ctx, restoreID, RestoreUpdate{LogAppend: line}); err != nil {
e.logger.Warn("append restore log to master failed",
zap.Uint("restore_record_id", restoreID),
zap.Error(err))
}
}
func (e *Executor) reportRestoreFailure(ctx context.Context, restoreID uint, msg string) {
_ = e.client.UpdateRestore(ctx, restoreID, RestoreUpdate{
reportCtx, cancel := agentFinalizationContext(ctx)
defer cancel()
if err := e.client.UpdateRestore(reportCtx, restoreID, RestoreUpdate{
Status: "failed",
ErrorMessage: msg,
LogAppend: fmt.Sprintf("[agent] 错误: %s\n", msg),
})
}); err != nil {
e.logger.Error("report restore failure to master failed",
zap.Uint("restore_record_id", restoreID),
zap.Error(err))
}
}
// buildRestoreBackupTaskSpec 把 RestoreSpec 转成 backup.TaskSpec。
+29
View File
@@ -18,8 +18,35 @@ import (
"time"
"backupx/server/internal/storage"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
)
func TestReportRecordFailureUsesFinalizationContextAndLogsUpdateError(t *testing.T) {
requestCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestCount++
http.Error(w, "master unavailable", http.StatusServiceUnavailable)
}))
defer server.Close()
core, observed := observer.New(zapcore.ErrorLevel)
executor := NewExecutor(NewMasterClient(server.URL, "token", false), t.TempDir())
executor.SetLogger(zap.New(core))
ctx, cancel := context.WithCancel(context.Background())
cancel()
executor.reportRecordFailure(ctx, 42, "backup failed")
if requestCount != 1 {
t.Fatalf("terminal update requests = %d, want 1 despite canceled command context", requestCount)
}
if observed.Len() != 1 || observed.All()[0].Message != "report backup failure to master failed" {
t.Fatalf("observed logs = %#v", observed.All())
}
}
func TestBuildBackupTaskSpecParsesJSONSourcePaths(t *testing.T) {
spec := &TaskSpec{
TaskID: 7,
@@ -323,6 +350,8 @@ func (f *agentTestStorageFactory) Type() storage.ProviderType {
return "agent_test_storage"
}
func (f *agentTestStorageFactory) SensitiveFields() []string { return nil }
func (f *agentTestStorageFactory) New(_ context.Context, config map[string]any) (storage.StorageProvider, error) {
name, _ := config["name"].(string)
provider := f.providers[name]
+13 -1
View File
@@ -20,7 +20,7 @@ type DirEntry struct {
func listLocalDir(path string) ([]DirEntry, error) {
cleaned := filepath.Clean(strings.TrimSpace(path))
if strings.TrimSpace(path) == "" || cleaned == "." {
cleaned = "/"
cleaned = localFilesystemRoot()
}
entries, err := os.ReadDir(cleaned)
if err != nil {
@@ -48,3 +48,15 @@ func listLocalDir(path string) ([]DirEntry, error) {
})
return result, nil
}
func localFilesystemRoot() string {
root := string(os.PathSeparator)
workingDir, err := os.Getwd()
if err != nil {
return root
}
if volume := filepath.VolumeName(workingDir); volume != "" {
return volume + root
}
return root
}