feat: 解决 CDC 去重、集中备份与首次初始化问题 (#105)

实现 CDC 内容寻址仓库、远程 Agent 中央中转备份与首次初始化体验，并补充安全校验、测试及双语文档。 Closes #94 Closes #101 Closes #104
2026-09-07 08:26:42 +08:00 · 2026-08-07 23:01:29 +08:00
parent 50ce6587d8
commit 00151e466c
74 changed files with 4507 additions and 400 deletions
@@ -1,19 +1,26 @@
 package service

 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
 	"errors"
+	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"

+	"backupx/server/internal/backup"
 	"backupx/server/internal/config"
 	"backupx/server/internal/database"
 	"backupx/server/internal/logger"
 	"backupx/server/internal/model"
 	"backupx/server/internal/repository"
+	"backupx/server/internal/storage"
 	"backupx/server/internal/storage/codec"
+	storageRclone "backupx/server/internal/storage/rclone"
 	"gorm.io/gorm"
 )

@@ -42,7 +49,7 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
 	if err := nodeRepo.Create(context.Background(), other); err != nil {
 		t.Fatalf("create other node: %v", err)
 	}
-	targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()})
+	targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir(), "masterRelay": true})
 	if err != nil {
 		t.Fatalf("EncryptJSON returned error: %v", err)
 	}
@@ -76,7 +83,8 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
 	if err := recordRepo.Create(context.Background(), record); err != nil {
 		t.Fatalf("create record: %v", err)
 	}
-	return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher), db, recordRepo, cmdRepo, owner, other
+	storageRegistry := storage.NewRegistry(storageRclone.NewLocalDiskFactory())
+	return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher, storageRegistry), db, recordRepo, cmdRepo, owner, other
 }

 func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) {
@@ -90,19 +98,27 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
 	if spec.TaskID != 1 || len(spec.StorageTargets) != 1 {
 		t.Fatalf("unexpected spec: %#v", spec)
 	}
+	if spec.StorageTargets[0].TransferMode != storage.TransferModeMasterRelay {
+		t.Fatalf("expected local disk to use Master relay, got %#v", spec.StorageTargets[0])
+	}
 	if _, err := svc.GetTaskSpec(ctx, other, 1); err == nil {
 		t.Fatal("expected non-owner node to be forbidden from pooled task spec")
 	}
+	record, err := records.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID record returned error: %v", err)
+	}
+	storagePath := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "backup.tar.gz")

 	if err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{
-		Status:          model.BackupRecordStatusSuccess,
-		FileName:        "backup.tar.gz",
-		FileSize:        123,
-		StoragePath:     "tasks/1/backup.tar.gz",
-		StorageTargetID: 2,
+		Status:              model.BackupRecordStatusSuccess,
+		FileName:            "backup.tar.gz",
+		FileSize:            123,
+		StoragePath:         storagePath,
+		StorageTargetID:     1,
+		StorageTransferMode: storage.TransferModeMasterRelay,
 		StorageUploadResults: []StorageUploadResultItem{
-			{StorageTargetID: 1, StorageTargetName: "first", Status: "failed", Error: "boom"},
-			{StorageTargetID: 2, StorageTargetName: "second", Status: "success", StoragePath: "tasks/1/backup.tar.gz", FileSize: 123},
+			{StorageTargetID: 1, StorageTargetName: "local", Status: "success", StoragePath: storagePath, FileSize: 123, TransferMode: storage.TransferModeMasterRelay},
 		},
 	}); err != nil {
 		t.Fatalf("owner UpdateRecord returned error: %v", err)
@@ -114,10 +130,13 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
 	if updated.Status != model.BackupRecordStatusSuccess || updated.NodeID != owner.ID {
 		t.Fatalf("unexpected updated record: %#v", updated)
 	}
-	if updated.StorageTargetID != 2 {
-		t.Fatalf("expected successful storage target id 2, got %d", updated.StorageTargetID)
+	if updated.StorageTargetID != 1 {
+		t.Fatalf("expected successful storage target id 1, got %d", updated.StorageTargetID)
 	}
-	if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"second"`) {
+	if updated.StorageTransferMode != storage.TransferModeMasterRelay {
+		t.Fatalf("expected Master relay transfer mode, got %q", updated.StorageTransferMode)
+	}
+	if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"local"`) {
 		t.Fatalf("expected upload results to be persisted, got %q", updated.StorageUploadResults)
 	}
 	if err := svc.UpdateRecord(ctx, other, 1, AgentRecordUpdate{LogAppend: "bad"}); err == nil {
@@ -125,6 +144,157 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
 	}
 }

+func TestAgentServiceRelaysRemoteArtifactToMasterLocalDisk(t *testing.T) {
+	svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t)
+	ctx := context.Background()
+	payload := []byte("artifact from remote source server")
+	digest := sha256.Sum256(payload)
+	checksum := fmt.Sprintf("%x", digest[:])
+	record, err := records.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID record returned error: %v", err)
+	}
+	objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "remote-source.tar")
+
+	if err := svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err != nil {
+		t.Fatalf("UploadArtifact returned error: %v", err)
+	}
+	target, err := svc.storageRepo.FindByID(ctx, 1)
+	if err != nil || target == nil {
+		t.Fatalf("FindByID target: target=%#v err=%v", target, err)
+	}
+	config := map[string]any{}
+	if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil {
+		t.Fatalf("DecryptJSON target config: %v", err)
+	}
+	basePath, _ := config["basePath"].(string)
+	stored, err := os.ReadFile(filepath.Join(basePath, filepath.FromSlash(objectKey)))
+	if err != nil {
+		t.Fatalf("read relayed artifact: %v", err)
+	}
+	if !bytes.Equal(stored, payload) {
+		t.Fatalf("relayed artifact differs: got %q", stored)
+	}
+	if err := svc.UploadArtifact(ctx, other, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err == nil {
+		t.Fatal("expected a different node to be forbidden from relaying the artifact")
+	}
+
+	legacyPayload := []byte("artifact from an older Agent")
+	legacyDigest := sha256.Sum256(legacyPayload)
+	legacyKey := backup.BuildStorageKey("file", record.StartedAt, "legacy-agent.tar")
+	canonicalKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy-agent.tar")
+	if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, legacyKey, int64(len(legacyPayload)), fmt.Sprintf("%x", legacyDigest[:]), bytes.NewReader(legacyPayload)); err != nil {
+		t.Fatalf("UploadArtifact legacy key returned error: %v", err)
+	}
+	stored, err = os.ReadFile(filepath.Join(basePath, filepath.FromSlash(canonicalKey)))
+	if err != nil || !bytes.Equal(stored, legacyPayload) {
+		t.Fatalf("legacy Agent artifact was not normalized: data=%q err=%v", stored, err)
+	}
+	if _, err := os.Stat(filepath.Join(basePath, filepath.FromSlash(legacyKey))); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("legacy object key should not be written directly: %v", err)
+	}
+	if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{
+		Status:          model.BackupRecordStatusSuccess,
+		FileName:        "legacy-agent.tar",
+		FileSize:        int64(len(legacyPayload)),
+		Checksum:        fmt.Sprintf("%x", legacyDigest[:]),
+		StoragePath:     legacyKey,
+		StorageTargetID: target.ID,
+		StorageUploadResults: []StorageUploadResultItem{{
+			StorageTargetID: target.ID,
+			Status:          "success",
+			StoragePath:     legacyKey,
+			FileSize:        int64(len(legacyPayload)),
+		}},
+	}); err != nil {
+		t.Fatalf("UpdateRecord legacy key returned error: %v", err)
+	}
+	updated, err := records.FindByID(ctx, record.ID)
+	if err != nil {
+		t.Fatalf("FindByID updated record returned error: %v", err)
+	}
+	if updated.StoragePath != canonicalKey || updated.StorageTransferMode != storage.TransferModeMasterRelay || !strings.Contains(updated.StorageUploadResults, canonicalKey) {
+		t.Fatalf("legacy Agent record was not normalized: %#v", updated)
+	}
+}
+
+func TestAgentServiceRejectsArtifactOutsideRecordNamespace(t *testing.T) {
+	svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
+	ctx := context.Background()
+	record, err := records.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID record returned error: %v", err)
+	}
+	target, err := svc.storageRepo.FindByID(ctx, 1)
+	if err != nil || target == nil {
+		t.Fatalf("FindByID target: target=%#v err=%v", target, err)
+	}
+	config := map[string]any{}
+	if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil {
+		t.Fatalf("DecryptJSON target config: %v", err)
+	}
+	basePath, _ := config["basePath"].(string)
+	victimKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID+1, "victim.tar")
+	victimPath := filepath.Join(basePath, filepath.FromSlash(victimKey))
+	if err := os.MkdirAll(filepath.Dir(victimPath), 0o755); err != nil {
+		t.Fatalf("MkdirAll victim parent: %v", err)
+	}
+	if err := os.WriteFile(victimPath, []byte("keep me"), 0o600); err != nil {
+		t.Fatalf("WriteFile victim: %v", err)
+	}
+	payload := []byte("overwrite")
+	digest := sha256.Sum256(payload)
+	if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, victimKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload)); err == nil {
+		t.Fatal("expected another record namespace to be rejected")
+	}
+	stored, err := os.ReadFile(victimPath)
+	if err != nil {
+		t.Fatalf("ReadFile victim: %v", err)
+	}
+	if string(stored) != "keep me" {
+		t.Fatalf("victim object changed: %q", stored)
+	}
+	if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{StoragePath: victimKey, StorageTargetID: target.ID, StorageTransferMode: storage.TransferModeMasterRelay}); err == nil {
+		t.Fatal("expected another record namespace in status update to be rejected")
+	}
+}
+
+func TestAgentServiceKeepsExistingLocalDiskTargetsAgentLocal(t *testing.T) {
+	svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
+	ctx := context.Background()
+	target, err := svc.storageRepo.FindByID(ctx, 1)
+	if err != nil || target == nil {
+		t.Fatalf("FindByID target: target=%#v err=%v", target, err)
+	}
+	legacyConfig, err := svc.cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()})
+	if err != nil {
+		t.Fatalf("EncryptJSON legacy target: %v", err)
+	}
+	target.ConfigCiphertext = legacyConfig
+	if err := svc.storageRepo.Update(ctx, target); err != nil {
+		t.Fatalf("Update legacy target: %v", err)
+	}
+
+	spec, err := svc.GetTaskSpec(ctx, owner, 1)
+	if err != nil {
+		t.Fatalf("GetTaskSpec returned error: %v", err)
+	}
+	if len(spec.StorageTargets) != 1 || spec.StorageTargets[0].TransferMode != storage.TransferModeDirect {
+		t.Fatalf("expected legacy local disk to stay Agent-local, got %#v", spec.StorageTargets)
+	}
+	payload := []byte("must not be relayed")
+	digest := sha256.Sum256(payload)
+	record, findErr := records.FindByID(ctx, 1)
+	if findErr != nil {
+		t.Fatalf("FindByID record returned error: %v", findErr)
+	}
+	objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy.tar")
+	err = svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload))
+	if err == nil {
+		t.Fatal("expected relay upload to be rejected for an Agent-local target")
+	}
+}
+
 func TestAgentServiceUpdateRecordRefreshesTaskSummaryOnTerminalStatus(t *testing.T) {
 	for _, status := range []string{model.BackupRecordStatusSuccess, model.BackupRecordStatusFailed} {
 		t.Run(status, func(t *testing.T) {