mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-09-04 15:11:23 +08:00
refactor: simplify architecture and harden lifecycle
Remove obsolete implementations, centralize background task ownership and terminal-state recovery, consolidate frontend routing and log streaming, and enforce project-wide verification in CI.
This commit is contained in:
Awuqing
@@ -1,10 +1,12 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/config"
|
||||
"backupx/server/internal/model"
|
||||
@@ -30,6 +32,22 @@ func Open(cfg config.DatabaseConfig, logger *zap.Logger) (*gorm.DB, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite: %w", err)
|
||||
}
|
||||
initialized := false
|
||||
defer func() {
|
||||
if initialized {
|
||||
return
|
||||
}
|
||||
sqlDB, dbErr := db.DB()
|
||||
if dbErr != nil {
|
||||
if logger != nil {
|
||||
logger.Warn("get database handle after initialization failure", zap.Error(dbErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
if closeErr := sqlDB.Close(); closeErr != nil && logger != nil {
|
||||
logger.Warn("close database after initialization failure", zap.Error(closeErr))
|
||||
}
|
||||
}()
|
||||
|
||||
if err := db.AutoMigrate(&model.User{}, &model.SystemConfig{}, &model.StorageTarget{}, &model.OAuthSession{}, &model.BackupTask{}, &model.BackupRecord{}, &model.Notification{}, &model.Node{}, &model.BackupTaskStorageTarget{}, &model.AuditLog{}, &model.AgentCommand{}, &model.AgentInstallToken{}, &model.RestoreRecord{}, &model.VerificationRecord{}, &model.ApiKey{}, &model.ReplicationRecord{}, &model.TaskTemplate{}); err != nil {
|
||||
return nil, fmt.Errorf("migrate schema: %w", err)
|
||||
@@ -37,11 +55,104 @@ func Open(cfg config.DatabaseConfig, logger *zap.Logger) (*gorm.DB, error) {
|
||||
|
||||
// 一次性数据迁移：从 backup_tasks.storage_target_id 回填到多对多中间表
|
||||
var count int64
|
||||
db.Model(&model.BackupTaskStorageTarget{}).Count(&count)
|
||||
if err := db.Model(&model.BackupTaskStorageTarget{}).Count(&count).Error; err != nil {
|
||||
return nil, fmt.Errorf("count backup task storage target mappings: %w", err)
|
||||
}
|
||||
if count == 0 {
|
||||
db.Exec("INSERT INTO backup_task_storage_targets (backup_task_id, storage_target_id) SELECT id, storage_target_id FROM backup_tasks WHERE storage_target_id > 0")
|
||||
if err := db.Exec("INSERT INTO backup_task_storage_targets (backup_task_id, storage_target_id) SELECT id, storage_target_id FROM backup_tasks WHERE storage_target_id > 0").Error; err != nil {
|
||||
return nil, fmt.Errorf("backfill backup task storage target mappings: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
reconciled, err := reconcileInterruptedOperations(db, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reconcile interrupted operations: %w", err)
|
||||
}
|
||||
if reconciled > 0 {
|
||||
logger.Warn("interrupted operations marked as failed", zap.Int64("records", reconciled))
|
||||
}
|
||||
|
||||
logger.Info("database initialized", zap.String("path", cfg.Path))
|
||||
initialized = true
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func reconcileInterruptedOperations(db *gorm.DB, completedAt time.Time) (int64, error) {
|
||||
const message = "应用在任务完成前重启，执行状态已自动收敛为失败"
|
||||
var reconciled int64
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
// Pending/dispatched Agent commands survive a Master restart. Their Agent
|
||||
// may still be executing (or may claim the pending command after startup),
|
||||
// so their linked records must not be mistaken for orphaned local work.
|
||||
var activeCommands []model.AgentCommand
|
||||
if err := tx.Where("status IN ? AND type IN ?",
|
||||
[]string{model.AgentCommandStatusPending, model.AgentCommandStatusDispatched},
|
||||
[]string{model.AgentCommandTypeRunTask, model.AgentCommandTypeRestoreRecord}).
|
||||
Find(&activeCommands).Error; err != nil {
|
||||
return fmt.Errorf("active agent commands: %w", err)
|
||||
}
|
||||
activeBackupRecordIDs := make([]uint, 0, len(activeCommands))
|
||||
activeRestoreRecordIDs := make([]uint, 0, len(activeCommands))
|
||||
for i := range activeCommands {
|
||||
cmd := &activeCommands[i]
|
||||
switch cmd.Type {
|
||||
case model.AgentCommandTypeRunTask:
|
||||
var payload struct {
|
||||
RecordID uint `json:"recordId"`
|
||||
}
|
||||
if json.Unmarshal([]byte(cmd.Payload), &payload) == nil && payload.RecordID > 0 {
|
||||
activeBackupRecordIDs = append(activeBackupRecordIDs, payload.RecordID)
|
||||
}
|
||||
case model.AgentCommandTypeRestoreRecord:
|
||||
var payload struct {
|
||||
RestoreRecordID uint `json:"restoreRecordId"`
|
||||
}
|
||||
if json.Unmarshal([]byte(cmd.Payload), &payload) == nil && payload.RestoreRecordID > 0 {
|
||||
activeRestoreRecordIDs = append(activeRestoreRecordIDs, payload.RestoreRecordID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
markFailed := func(entity any, runningStatus, failedStatus string, activeAgentRecordIDs []uint) error {
|
||||
query := tx.Model(entity).Where("status = ?", runningStatus)
|
||||
if len(activeAgentRecordIDs) > 0 {
|
||||
query = query.Where("id NOT IN ?", activeAgentRecordIDs)
|
||||
}
|
||||
result := query.
|
||||
Updates(map[string]any{
|
||||
"status": failedStatus,
|
||||
"error_message": message,
|
||||
"completed_at": completedAt,
|
||||
"duration_seconds": gorm.Expr("CAST(MAX(0, (julianday(?) - julianday(started_at)) * 86400) AS INTEGER)", completedAt),
|
||||
})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
reconciled += result.RowsAffected
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := markFailed(&model.BackupRecord{}, model.BackupRecordStatusRunning, model.BackupRecordStatusFailed, activeBackupRecordIDs); err != nil {
|
||||
return fmt.Errorf("backup records: %w", err)
|
||||
}
|
||||
if err := markFailed(&model.RestoreRecord{}, model.RestoreRecordStatusRunning, model.RestoreRecordStatusFailed, activeRestoreRecordIDs); err != nil {
|
||||
return fmt.Errorf("restore records: %w", err)
|
||||
}
|
||||
if err := markFailed(&model.VerificationRecord{}, model.VerificationRecordStatusRunning, model.VerificationRecordStatusFailed, nil); err != nil {
|
||||
return fmt.Errorf("verification records: %w", err)
|
||||
}
|
||||
if err := markFailed(&model.ReplicationRecord{}, model.ReplicationStatusRunning, model.ReplicationStatusFailed, nil); err != nil {
|
||||
return fmt.Errorf("replication records: %w", err)
|
||||
}
|
||||
|
||||
result := tx.Model(&model.BackupTask{}).
|
||||
Where("last_status = ? AND NOT EXISTS (SELECT 1 FROM backup_records WHERE backup_records.task_id = backup_tasks.id AND backup_records.status = ?)", model.BackupTaskStatusRunning, model.BackupRecordStatusRunning).
|
||||
Update("last_status", model.BackupTaskStatusFailed)
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("backup tasks: %w", result.Error)
|
||||
}
|
||||
reconciled += result.RowsAffected
|
||||
return nil
|
||||
})
|
||||
return reconciled, err
|
||||
}
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"backupx/server/internal/config"
|
||||
"backupx/server/internal/logger"
|
||||
"backupx/server/internal/model"
|
||||
)
|
||||
|
||||
func TestOpenConfiguresSQLiteForSingleMasterConcurrency(t *testing.T) {
|
||||
@@ -38,3 +41,178 @@ func TestOpenConfiguresSQLiteForSingleMasterConcurrency(t *testing.T) {
|
||||
t.Fatalf("busy_timeout = %d, want 5000", busyTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileInterruptedOperations(t *testing.T) {
|
||||
log, err := logger.New(config.LogConfig{Level: "error"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := Open(config.DatabaseConfig{Path: filepath.Join(t.TempDir(), "reconcile.db")}, log)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Errorf("close database: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
startedAt := time.Now().UTC().Add(-time.Minute)
|
||||
task := model.BackupTask{Name: "interrupted", Type: model.BackupTaskTypeFile, LastStatus: model.BackupTaskStatusRunning}
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items := []any{
|
||||
&model.BackupRecord{TaskID: task.ID, Status: model.BackupRecordStatusRunning, StartedAt: startedAt},
|
||||
&model.RestoreRecord{TaskID: task.ID, Status: model.RestoreRecordStatusRunning, StartedAt: startedAt},
|
||||
&model.VerificationRecord{TaskID: task.ID, Status: model.VerificationRecordStatusRunning, StartedAt: startedAt},
|
||||
&model.ReplicationRecord{TaskID: task.ID, Status: model.ReplicationStatusRunning, StartedAt: startedAt},
|
||||
}
|
||||
for _, item := range items {
|
||||
if err := db.Create(item).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
completedAt := time.Now().UTC()
|
||||
count, err := reconcileInterruptedOperations(db, completedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 5 {
|
||||
t.Fatalf("reconciled records = %d, want 5", count)
|
||||
}
|
||||
|
||||
var runningRecords int64
|
||||
for _, entity := range []any{&model.BackupRecord{}, &model.RestoreRecord{}, &model.VerificationRecord{}, &model.ReplicationRecord{}} {
|
||||
if err := db.Model(entity).Where("status = ?", "running").Count(&runningRecords).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runningRecords != 0 {
|
||||
t.Fatalf("%T still has %d running records", entity, runningRecords)
|
||||
}
|
||||
}
|
||||
if err := db.First(&task, task.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if task.LastStatus != model.BackupTaskStatusFailed {
|
||||
t.Fatalf("task last status = %q, want failed", task.LastStatus)
|
||||
}
|
||||
var backupRecord model.BackupRecord
|
||||
if err := db.Where("task_id = ?", task.ID).First(&backupRecord).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if backupRecord.DurationSeconds <= 0 {
|
||||
t.Fatalf("backup duration = %d, want positive duration", backupRecord.DurationSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileInterruptedOperationsPreservesRemoteAgentWork(t *testing.T) {
|
||||
log, err := logger.New(config.LogConfig{Level: "error"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db, err := Open(config.DatabaseConfig{Path: filepath.Join(t.TempDir(), "remote-reconcile.db")}, log)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Errorf("close database: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
localNode := model.Node{Name: "local", Token: "local-token", Status: model.NodeStatusOnline, IsLocal: true}
|
||||
remoteNode := model.Node{Name: "remote", Token: "remote-token", Status: model.NodeStatusOnline, IsLocal: false}
|
||||
if err := db.Create(&localNode).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&remoteNode).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
localTask := model.BackupTask{Name: "local-interrupted", Type: model.BackupTaskTypeFile, LastStatus: model.BackupTaskStatusRunning}
|
||||
remoteTask := model.BackupTask{Name: "remote-still-running", Type: model.BackupTaskTypeFile, LastStatus: model.BackupTaskStatusRunning}
|
||||
orphanRemoteTask := model.BackupTask{Name: "remote-without-command", Type: model.BackupTaskTypeFile, LastStatus: model.BackupTaskStatusRunning}
|
||||
if err := db.Create(&localTask).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&remoteTask).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&orphanRemoteTask).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startedAt := time.Now().UTC().Add(-time.Minute)
|
||||
localBackup := &model.BackupRecord{TaskID: localTask.ID, NodeID: localNode.ID, Status: model.BackupRecordStatusRunning, StartedAt: startedAt}
|
||||
localRestore := &model.RestoreRecord{TaskID: localTask.ID, NodeID: localNode.ID, Status: model.RestoreRecordStatusRunning, StartedAt: startedAt}
|
||||
remoteBackup := &model.BackupRecord{TaskID: remoteTask.ID, NodeID: remoteNode.ID, Status: model.BackupRecordStatusRunning, StartedAt: startedAt}
|
||||
remoteRestore := &model.RestoreRecord{TaskID: remoteTask.ID, NodeID: remoteNode.ID, Status: model.RestoreRecordStatusRunning, StartedAt: startedAt}
|
||||
orphanRemoteBackup := &model.BackupRecord{TaskID: orphanRemoteTask.ID, NodeID: remoteNode.ID, Status: model.BackupRecordStatusRunning, StartedAt: startedAt}
|
||||
items := []any{localBackup, localRestore, remoteBackup, remoteRestore, orphanRemoteBackup}
|
||||
for _, item := range items {
|
||||
if err := db.Create(item).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
activeCommands := []model.AgentCommand{
|
||||
{NodeID: remoteNode.ID, Type: model.AgentCommandTypeRunTask, Status: model.AgentCommandStatusDispatched, Payload: fmt.Sprintf(`{"recordId":%d}`, remoteBackup.ID)},
|
||||
{NodeID: remoteNode.ID, Type: model.AgentCommandTypeRestoreRecord, Status: model.AgentCommandStatusPending, Payload: fmt.Sprintf(`{"restoreRecordId":%d}`, remoteRestore.ID)},
|
||||
}
|
||||
if err := db.Create(&activeCommands).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count, err := reconcileInterruptedOperations(db, time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 5 {
|
||||
t.Fatalf("reconciled rows = %d, want local and unlinked remote records plus their tasks", count)
|
||||
}
|
||||
|
||||
for _, entity := range []any{&model.BackupRecord{}, &model.RestoreRecord{}} {
|
||||
var remoteRunning int64
|
||||
if err := db.Model(entity).Where("task_id = ? AND status = ?", remoteTask.ID, "running").Count(&remoteRunning).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if remoteRunning != 1 {
|
||||
t.Fatalf("%T remote running records = %d, want 1", entity, remoteRunning)
|
||||
}
|
||||
var localRunning int64
|
||||
if err := db.Model(entity).Where("task_id = ? AND status = ?", localTask.ID, "running").Count(&localRunning).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if localRunning != 0 {
|
||||
t.Fatalf("%T local running records = %d, want 0", entity, localRunning)
|
||||
}
|
||||
}
|
||||
if err := db.First(&localTask, localTask.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.First(&remoteTask, remoteTask.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.First(&orphanRemoteTask, orphanRemoteTask.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if localTask.LastStatus != model.BackupTaskStatusFailed || remoteTask.LastStatus != model.BackupTaskStatusRunning {
|
||||
t.Fatalf("task statuses = local:%q remote:%q", localTask.LastStatus, remoteTask.LastStatus)
|
||||
}
|
||||
if orphanRemoteTask.LastStatus != model.BackupTaskStatusFailed {
|
||||
t.Fatalf("unlinked remote task status = %q, want failed", orphanRemoteTask.LastStatus)
|
||||
}
|
||||
if err := db.First(orphanRemoteBackup, orphanRemoteBackup.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if orphanRemoteBackup.Status != model.BackupRecordStatusFailed {
|
||||
t.Fatalf("unlinked remote backup status = %q, want failed", orphanRemoteBackup.Status)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user