feat(cluster): 支持远程源服务器集中备份

为 Master 本地磁盘增加可选流式中转，远程 Agent 可直接把产物写入中央存储并通过反向通道恢复。持久化并校验传输模式，兼容既有 Agent 本机磁盘目标，补齐鉴权、完整性、配额、访问保护、前端配置及双向链路测试。 Closes #101
2026-09-05 07:26:43 +08:00 · 2026-08-07 05:54:34 +08:00
parent 05dd1baa61
commit cc50637b4b
28 changed files with 1012 additions and 180 deletions
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -125,10 +126,11 @@ type TaskSpec struct {

 // StorageTargetConfig 与 service.AgentStorageTargetConfig 对齐
 type StorageTargetConfig struct {
-	ID     uint            `json:"id"`
-	Type   string          `json:"type"`
-	Name   string          `json:"name"`
-	Config json.RawMessage `json:"config"`
+	ID           uint            `json:"id"`
+	Type         string          `json:"type"`
+	Name         string          `json:"name"`
+	Config       json.RawMessage `json:"config"`
+	TransferMode string          `json:"transferMode"`
 }

 // GetTaskSpec 拉取任务规格
@@ -149,6 +151,7 @@ type RecordUpdate struct {
 	Checksum             string              `json:"checksum,omitempty"`
 	StoragePath          string              `json:"storagePath,omitempty"`
 	StorageTargetID      uint                `json:"storageTargetId,omitempty"`
+	StorageTransferMode  string              `json:"storageTransferMode,omitempty"`
 	StorageUploadResults []StorageResultItem `json:"storageUploadResults,omitempty"`
 	ErrorMessage         string              `json:"errorMessage,omitempty"`
 	LogAppend            string              `json:"logAppend,omitempty"`
@@ -160,6 +163,7 @@ type StorageResultItem struct {
 	Status            string `json:"status"`
 	StoragePath       string `json:"storagePath,omitempty"`
 	FileSize          int64  `json:"fileSize,omitempty"`
+	TransferMode      string `json:"transferMode,omitempty"`
 	Error             string `json:"error,omitempty"`
 }

@@ -169,6 +173,39 @@ func (c *MasterClient) UpdateRecord(ctx context.Context, recordID uint, update R
 	return c.do(ctx, http.MethodPost, path, update, nil)
 }

+// UploadArtifact streams an artifact through the Master for storage targets
+// that are not directly reachable from the Agent.
+func (c *MasterClient) UploadArtifact(ctx context.Context, recordID, targetID uint, objectKey string, size int64, checksum string, reader io.Reader) error {
+	path := fmt.Sprintf("/api/agent/records/%d/artifacts/%d", recordID, targetID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.baseURL+path, reader)
+	if err != nil {
+		return err
+	}
+	// The executor owns and closes the artifact file. Prevent net/http from
+	// closing that underlying reader when it finishes the request body.
+	req.Body = io.NopCloser(reader)
+	req.ContentLength = size
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("X-Agent-Token", c.token)
+	req.Header.Set("X-BackupX-Object-Key", objectKey)
+	req.Header.Set("X-BackupX-SHA256", checksum)
+	client := *c.httpClient
+	client.Timeout = 0
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("relay artifact to Master: %w", err)
+	}
+	data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	closeErr := resp.Body.Close()
+	if readErr != nil || closeErr != nil {
+		return errors.Join(readErr, closeErr)
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("relay artifact to Master: http %d: %s", resp.StatusCode, string(data))
+	}
+	return nil
+}
+
 // RestoreSpec 与 service.AgentRestoreSpec 对齐
 type RestoreSpec struct {
 	RestoreRecordID uint                `json:"restoreRecordId"`
@@ -210,6 +247,27 @@ func (c *MasterClient) GetRestoreSpec(ctx context.Context, restoreRecordID uint)
 	return &spec, nil
 }

+func (c *MasterClient) DownloadRestoreArtifact(ctx context.Context, restoreRecordID uint) (io.ReadCloser, error) {
+	path := fmt.Sprintf("/api/agent/restores/%d/artifact", restoreRecordID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("X-Agent-Token", c.token)
+	client := *c.httpClient
+	client.Timeout = 0
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("download relayed artifact from Master: %w", err)
+	}
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return resp.Body, nil
+	}
+	data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	closeErr := resp.Body.Close()
+	return nil, errors.Join(fmt.Errorf("download relayed artifact from Master: http %d: %s", resp.StatusCode, string(data)), readErr, closeErr)
+}
+
 // UpdateRestore 上报恢复记录的状态/日志
 func (c *MasterClient) UpdateRestore(ctx context.Context, restoreRecordID uint, update RestoreUpdate) error {
 	path := fmt.Sprintf("/api/agent/restores/%d", restoreRecordID)
@@ -5,6 +5,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -137,13 +138,15 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
 	}
 	uploadResults := make([]StorageResultItem, 0, len(spec.StorageTargets))
 	selectedStorageTargetID := uint(0)
+	selectedStorageTransferMode := ""
 	var uploadErrors []string
 	for _, target := range spec.StorageTargets {
-		if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, spec.TaskID); err != nil {
+		if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, checksum, spec.TaskID); err != nil {
 			uploadResults = append(uploadResults, StorageResultItem{
 				StorageTargetID:   target.ID,
 				StorageTargetName: target.Name,
 				Status:            "failed",
+				TransferMode:      target.TransferMode,
 				Error:             err.Error(),
 			})
 			uploadErrors = append(uploadErrors, fmt.Sprintf("%s: %v", target.Name, err))
@@ -152,6 +155,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
 		}
 		if selectedStorageTargetID == 0 {
 			selectedStorageTargetID = target.ID
+			selectedStorageTransferMode = target.TransferMode
 		}
 		uploadResults = append(uploadResults, StorageResultItem{
 			StorageTargetID:   target.ID,
@@ -159,6 +163,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
 			Status:            "success",
 			StoragePath:       storagePath,
 			FileSize:          fileSize,
+			TransferMode:      target.TransferMode,
 		})
 		e.appendLog(ctx, recordID, fmt.Sprintf("[agent] 已上传到存储目标 %s\n", target.Name))
 	}
@@ -179,34 +184,40 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
 		Checksum:             checksum,
 		StoragePath:          storagePath,
 		StorageTargetID:      selectedStorageTargetID,
+		StorageTransferMode:  selectedStorageTransferMode,
 		StorageUploadResults: uploadResults,
 		LogAppend:            fmt.Sprintf("[agent] 任务完成，总计 %d 字节\n", fileSize),
 	})
 }

 // uploadToTarget 上传单个目标。为保持简化不做上传级重试（rclone 本身已有 low-level 重试）。
-func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, taskID uint) error {
-	var rawConfig map[string]any
-	if len(target.Config) > 0 {
-		// DecodeRawConfig 通过 json 解析
-		if err := jsonUnmarshalMap(target.Config, &rawConfig); err != nil {
-			return fmt.Errorf("parse storage config: %w", err)
-		}
-	}
-	provider, err := e.storageRegistry.Create(ctx, target.Type, rawConfig)
-	if err != nil {
-		return fmt.Errorf("create provider: %w", err)
-	}
+func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, checksum string, taskID uint) error {
 	f, err := os.Open(filePath)
 	if err != nil {
 		return fmt.Errorf("open artifact: %w", err)
 	}
-	defer f.Close()
+	if target.TransferMode == storage.TransferModeMasterRelay {
+		uploadErr := e.client.UploadArtifact(ctx, recordID, target.ID, objectKey, fileSize, checksum, f)
+		return errors.Join(uploadErr, f.Close())
+	}
+	var rawConfig map[string]any
+	if len(target.Config) > 0 {
+		// DecodeRawConfig 通过 json 解析
+		if err := jsonUnmarshalMap(target.Config, &rawConfig); err != nil {
+			return errors.Join(fmt.Errorf("parse storage config: %w", err), f.Close())
+		}
+	}
+	provider, err := e.storageRegistry.Create(ctx, target.Type, rawConfig)
+	if err != nil {
+		closeErr := f.Close()
+		return errors.Join(fmt.Errorf("create provider: %w", err), closeErr)
+	}
 	meta := map[string]string{
 		"taskId":   fmt.Sprintf("%d", taskID),
 		"recordId": fmt.Sprintf("%d", recordID),
 	}
-	return provider.Upload(ctx, objectKey, f, fileSize, meta)
+	uploadErr := provider.Upload(ctx, objectKey, f, fileSize, meta)
+	return errors.Join(uploadErr, f.Close())
 }

 // appendLog 追加日志到 Master 记录（尽力而为，失败不中断主流程）
@@ -328,7 +339,7 @@ func (e *Executor) DeleteStorageObject(ctx context.Context, targetType string, t
 // ExecuteRestore 处理 restore_record 命令：拉规格 → 下载 → 解压 → 执行 runner.Restore → 上报结果。
 //
 // 与 ExecuteRunTask 对称，但方向相反：
-//   - 下载：通过 spec.Storage 创建 provider → Download(spec.StoragePath)
+//   - 下载：直连共享存储，或通过 Master 中转其本地磁盘对象
 //   - 解密：当前 Agent 不支持加密恢复（密钥未下发），spec.Encrypt=true 会直接失败
 //   - 执行：backup.Registry.Runner(spec.Type).Restore
 //   - 上报：通过 UpdateRestore（status/logAppend）
@@ -357,28 +368,31 @@ func (e *Executor) ExecuteRestore(ctx context.Context, restoreRecordID uint) err
 	}
 	defer os.RemoveAll(tmpDir)

-	// 1) 创建 storage provider
-	var rawConfig map[string]any
-	if len(spec.Storage.Config) > 0 {
-		if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil {
-			e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err))
-			return err
-		}
-	}
-	provider, err := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig)
-	if err != nil {
-		e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", err))
-		return err
-	}
-
-	// 2) 下载
+	// 1) 下载
 	fileName := spec.FileName
 	if strings.TrimSpace(fileName) == "" {
 		fileName = filepath.Base(spec.StoragePath)
 	}
 	artifactPath := filepath.Join(tmpDir, filepath.Base(fileName))
 	e.appendRestoreLog(ctx, restoreRecordID, fmt.Sprintf("[agent] 下载备份文件 %s\n", spec.StoragePath))
-	reader, err := provider.Download(ctx, spec.StoragePath)
+	var reader io.ReadCloser
+	if spec.Storage.TransferMode == storage.TransferModeMasterRelay {
+		reader, err = e.client.DownloadRestoreArtifact(ctx, restoreRecordID)
+	} else {
+		var rawConfig map[string]any
+		if len(spec.Storage.Config) > 0 {
+			if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil {
+				e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err))
+				return err
+			}
+		}
+		provider, providerErr := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig)
+		if providerErr != nil {
+			e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", providerErr))
+			return providerErr
+		}
+		reader, err = provider.Download(ctx, spec.StoragePath)
+	}
 	if err != nil {
 		e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("下载备份失败: %v", err))
 		return err
@@ -489,8 +503,10 @@ func buildRestoreBackupTaskSpec(spec *RestoreSpec, startedAt time.Time, tempDir
 }

 // writeReaderToLocal 把 reader 写到本地文件（Agent 侧工具函数）。
-func writeReaderToLocal(targetPath string, reader io.ReadCloser) error {
-	defer reader.Close()
+func writeReaderToLocal(targetPath string, reader io.ReadCloser) (err error) {
+	defer func() {
+		err = errors.Join(err, reader.Close())
+	}()
 	if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
 		return err
 	}
@@ -498,9 +514,8 @@ func writeReaderToLocal(targetPath string, reader io.ReadCloser) error {
 	if err != nil {
 		return err
 	}
-	defer file.Close()
-	_, err = io.Copy(file, reader)
-	return err
+	_, copyErr := io.Copy(file, reader)
+	return errors.Join(copyErr, file.Close())
 }

 // 辅助函数
@@ -1,7 +1,10 @@
 package agent

 import (
+	"archive/tar"
+	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -109,6 +112,149 @@ func TestExecuteRunTaskRecordsPerTargetUploadResults(t *testing.T) {
 	}
 }

+func TestExecuteRunTaskRelaysMasterLocalDiskTarget(t *testing.T) {
+	sourceDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("centralize me"), 0o644); err != nil {
+		t.Fatalf("WriteFile returned error: %v", err)
+	}
+	var relayed []byte
+	var finalUpdate RecordUpdate
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/agent/tasks/1":
+			writeAgentEnvelope(t, w, TaskSpec{
+				TaskID:      1,
+				Name:        "remote-source",
+				Type:        "file",
+				SourcePath:  sourceDir,
+				Compression: "gzip",
+				StorageTargets: []StorageTargetConfig{{
+					ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay,
+				}},
+			})
+		case r.Method == http.MethodPut && r.URL.Path == "/api/agent/records/99/artifacts/11":
+			body, err := io.ReadAll(r.Body)
+			if err != nil {
+				t.Fatalf("ReadAll relayed body: %v", err)
+			}
+			digest := sha256.Sum256(body)
+			if got := r.Header.Get("X-BackupX-SHA256"); got != fmt.Sprintf("%x", digest[:]) {
+				t.Fatalf("relay checksum header = %q", got)
+			}
+			if r.Header.Get("X-BackupX-Object-Key") == "" || r.ContentLength != int64(len(body)) {
+				t.Fatalf("invalid relay metadata: key=%q length=%d body=%d", r.Header.Get("X-BackupX-Object-Key"), r.ContentLength, len(body))
+			}
+			relayed = append([]byte(nil), body...)
+			writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
+		case r.Method == http.MethodPost && r.URL.Path == "/api/agent/records/99":
+			var update RecordUpdate
+			if err := json.NewDecoder(r.Body).Decode(&update); err != nil {
+				t.Fatalf("Decode update returned error: %v", err)
+			}
+			if update.Status != "" {
+				finalUpdate = update
+			}
+			writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp"))
+	if err := executor.ExecuteRunTask(context.Background(), 1, 99); err != nil {
+		t.Fatalf("ExecuteRunTask returned error: %v", err)
+	}
+	if len(relayed) == 0 {
+		t.Fatal("expected artifact bytes to be streamed through Master")
+	}
+	if finalUpdate.Status != "success" || finalUpdate.StorageTransferMode != storage.TransferModeMasterRelay {
+		t.Fatalf("unexpected final relay update: %#v", finalUpdate)
+	}
+	if len(finalUpdate.StorageUploadResults) != 1 || finalUpdate.StorageUploadResults[0].TransferMode != storage.TransferModeMasterRelay {
+		t.Fatalf("unexpected relay target result: %#v", finalUpdate.StorageUploadResults)
+	}
+}
+
+func TestExecuteRestoreDownloadsMasterRelayedArtifact(t *testing.T) {
+	var archive bytes.Buffer
+	tarWriter := tar.NewWriter(&archive)
+	content := []byte("restored through Master")
+	header := &tar.Header{Name: "site/index.html", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg}
+	if err := tarWriter.WriteHeader(header); err != nil {
+		t.Fatalf("WriteHeader returned error: %v", err)
+	}
+	if _, err := tarWriter.Write(content); err != nil {
+		t.Fatalf("Write returned error: %v", err)
+	}
+	if err := tarWriter.Close(); err != nil {
+		t.Fatalf("Close returned error: %v", err)
+	}
+	artifact := append([]byte(nil), archive.Bytes()...)
+	digest := sha256.Sum256(artifact)
+	restoreRoot := t.TempDir()
+	restoreSource := filepath.Join(restoreRoot, "site")
+	artifactRequests := 0
+	var finalUpdate RestoreUpdate
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/spec":
+			writeAgentEnvelope(t, w, RestoreSpec{
+				RestoreRecordID: 77,
+				BackupRecordID:  99,
+				TaskID:          1,
+				TaskName:        "remote-source",
+				Type:            "file",
+				SourcePath:      restoreSource,
+				Storage: StorageTargetConfig{
+					ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay,
+				},
+				StoragePath: "BackupX/file/site.tar",
+				FileName:    "site.tar",
+				Checksum:    fmt.Sprintf("%x", digest[:]),
+			})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/artifact":
+			artifactRequests++
+			if r.Header.Get("X-Agent-Token") != "token" {
+				t.Fatalf("missing Agent token on relay download")
+			}
+			w.Header().Set("Content-Length", fmt.Sprintf("%d", len(artifact)))
+			_, _ = w.Write(artifact)
+		case r.Method == http.MethodPost && r.URL.Path == "/api/agent/restores/77":
+			var update RestoreUpdate
+			if err := json.NewDecoder(r.Body).Decode(&update); err != nil {
+				t.Fatalf("Decode update returned error: %v", err)
+			}
+			if update.Status != "" {
+				finalUpdate = update
+			}
+			writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp"))
+	if err := executor.ExecuteRestore(context.Background(), 77); err != nil {
+		t.Fatalf("ExecuteRestore returned error: %v", err)
+	}
+	if artifactRequests != 1 {
+		t.Fatalf("expected one relay artifact request, got %d", artifactRequests)
+	}
+	restored, err := os.ReadFile(filepath.Join(restoreRoot, "site", "index.html"))
+	if err != nil {
+		t.Fatalf("ReadFile returned error: %v", err)
+	}
+	if string(restored) != string(content) {
+		t.Fatalf("restored content = %q, want %q", restored, content)
+	}
+	if finalUpdate.Status != "success" {
+		t.Fatalf("unexpected final restore update: %#v", finalUpdate)
+	}
+}
+
 func TestExecuteRunTaskReportsPerTargetUploadResultsWhenAllTargetsFail(t *testing.T) {
 	sourceDir := t.TempDir()
 	if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("hello"), 0o644); err != nil {