feat(BackupX): harden agent cluster backup workflow

Squash merge PR #61
2026-09-05 15:37:03 +08:00 · 2026-05-13 14:24:45 +08:00
parent 7a6ffd4ddd
commit 7084d47c4b
30 changed files with 1360 additions and 155 deletions
@@ -230,13 +230,15 @@ func (s *AgentService) ensureTaskSpecAccess(ctx context.Context, node *model.Nod

 // AgentRecordUpdate Agent 上报备份记录的最终状态。
 type AgentRecordUpdate struct {
-	Status       string `json:"status"` // running | success | failed
-	FileName     string `json:"fileName,omitempty"`
-	FileSize     int64  `json:"fileSize,omitempty"`
-	Checksum     string `json:"checksum,omitempty"`
-	StoragePath  string `json:"storagePath,omitempty"`
-	ErrorMessage string `json:"errorMessage,omitempty"`
-	LogAppend    string `json:"logAppend,omitempty"` // 增量日志，追加到 record.log_content
+	Status               string                    `json:"status"` // running | success | failed
+	FileName             string                    `json:"fileName,omitempty"`
+	FileSize             int64                     `json:"fileSize,omitempty"`
+	Checksum             string                    `json:"checksum,omitempty"`
+	StoragePath          string                    `json:"storagePath,omitempty"`
+	StorageTargetID      uint                      `json:"storageTargetId,omitempty"`
+	StorageUploadResults []StorageUploadResultItem `json:"storageUploadResults,omitempty"`
+	ErrorMessage         string                    `json:"errorMessage,omitempty"`
+	LogAppend            string                    `json:"logAppend,omitempty"` // 增量日志，追加到 record.log_content
 }

 // UpdateRecord 更新备份记录的状态/日志。Agent 在执行过程中可多次调用。
@@ -273,6 +275,14 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
 	if update.StoragePath != "" {
 		record.StoragePath = update.StoragePath
 	}
+	if update.StorageTargetID > 0 {
+		record.StorageTargetID = update.StorageTargetID
+	}
+	if len(update.StorageUploadResults) > 0 {
+		if resultsJSON, marshalErr := json.Marshal(update.StorageUploadResults); marshalErr == nil {
+			record.StorageUploadResults = string(resultsJSON)
+		}
+	}
 	if update.ErrorMessage != "" {
 		record.ErrorMessage = update.ErrorMessage
 	}
@@ -294,7 +304,10 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
 	// 同步更新任务的 last_status
 	if update.Status == model.BackupRecordStatusSuccess || update.Status == model.BackupRecordStatusFailed {
 		task.LastStatus = update.Status
-		_ = s.taskRepo.Update(ctx, task)
+		task.LastRunAt = &record.StartedAt
+		if err := s.taskRepo.Update(ctx, task); err != nil {
+			return fmt.Errorf("update backup task summary: %w", err)
+		}
 	}
 	return nil
 }
@@ -2,7 +2,9 @@ package service

 import (
 	"context"
+	"errors"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"

@@ -93,10 +95,15 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
 	}

 	if err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{
-		Status:      model.BackupRecordStatusSuccess,
-		FileName:    "backup.tar.gz",
-		FileSize:    123,
-		StoragePath: "tasks/1/backup.tar.gz",
+		Status:          model.BackupRecordStatusSuccess,
+		FileName:        "backup.tar.gz",
+		FileSize:        123,
+		StoragePath:     "tasks/1/backup.tar.gz",
+		StorageTargetID: 2,
+		StorageUploadResults: []StorageUploadResultItem{
+			{StorageTargetID: 1, StorageTargetName: "first", Status: "failed", Error: "boom"},
+			{StorageTargetID: 2, StorageTargetName: "second", Status: "success", StoragePath: "tasks/1/backup.tar.gz", FileSize: 123},
+		},
 	}); err != nil {
 		t.Fatalf("owner UpdateRecord returned error: %v", err)
 	}
@@ -107,11 +114,60 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
 	if updated.Status != model.BackupRecordStatusSuccess || updated.NodeID != owner.ID {
 		t.Fatalf("unexpected updated record: %#v", updated)
 	}
+	if updated.StorageTargetID != 2 {
+		t.Fatalf("expected successful storage target id 2, got %d", updated.StorageTargetID)
+	}
+	if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"second"`) {
+		t.Fatalf("expected upload results to be persisted, got %q", updated.StorageUploadResults)
+	}
 	if err := svc.UpdateRecord(ctx, other, 1, AgentRecordUpdate{LogAppend: "bad"}); err == nil {
 		t.Fatal("expected non-owner node to be forbidden from record update")
 	}
 }

+func TestAgentServiceUpdateRecordRefreshesTaskSummaryOnTerminalStatus(t *testing.T) {
+	for _, status := range []string{model.BackupRecordStatusSuccess, model.BackupRecordStatusFailed} {
+		t.Run(status, func(t *testing.T) {
+			svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
+			ctx := context.Background()
+			record, err := records.FindByID(ctx, 1)
+			if err != nil {
+				t.Fatalf("FindByID record returned error: %v", err)
+			}
+
+			if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{Status: status}); err != nil {
+				t.Fatalf("UpdateRecord returned error: %v", err)
+			}
+
+			task, err := svc.taskRepo.FindByID(ctx, record.TaskID)
+			if err != nil {
+				t.Fatalf("FindByID task returned error: %v", err)
+			}
+			if task.LastStatus != status {
+				t.Fatalf("expected task LastStatus %q, got %q", status, task.LastStatus)
+			}
+			if task.LastRunAt == nil || !task.LastRunAt.Equal(record.StartedAt) {
+				t.Fatalf("expected task LastRunAt to match record startedAt %s, got %#v", record.StartedAt, task.LastRunAt)
+			}
+		})
+	}
+}
+
+func TestAgentServiceUpdateRecordReturnsTaskSummaryUpdateError(t *testing.T) {
+	svc, _, _, _, owner, _ := newAgentServicePoolTestHarness(t)
+	ctx := context.Background()
+	expectedErr := errors.New("task update failed")
+	svc.taskRepo = &failingUpdateTaskRepo{
+		BackupTaskRepository: svc.taskRepo,
+		err:                  expectedErr,
+	}
+
+	err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{Status: model.BackupRecordStatusSuccess})
+	if !errors.Is(err, expectedErr) {
+		t.Fatalf("expected task update error %v, got %v", expectedErr, err)
+	}
+}
+
 func TestAgentServiceProcessStaleCommandsFailsPendingRunTaskRecord(t *testing.T) {
 	svc, _, records, commands, owner, _ := newAgentServicePoolTestHarness(t)
 	ctx := context.Background()
@@ -587,3 +643,12 @@ func setBackupRecordUpdatedAt(db *gorm.DB, id uint, updatedAt time.Time) error {
 func setRestoreRecordUpdatedAt(db *gorm.DB, id uint, updatedAt time.Time) error {
 	return db.Model(&model.RestoreRecord{}).Where("id = ?", id).UpdateColumn("updated_at", updatedAt).Error
 }
+
+type failingUpdateTaskRepo struct {
+	repository.BackupTaskRepository
+	err error
+}
+
+func (r *failingUpdateTaskRepo) Update(context.Context, *model.BackupTask) error {
+	return r.err
+}
@@ -52,6 +52,11 @@ type StorageUploadResultItem struct {
 	Error             string `json:"error,omitempty"`
 }

+const (
+	uploadMaxAttempts  = 3
+	uploadRetryBackoff = 10 * time.Second
+)
+
 type DownloadedArtifact struct {
 	FileName string
 	Reader   io.ReadCloser
@@ -96,6 +101,7 @@ type BackupExecutionService struct {
 	retries        int    // rclone 底层重试次数
 	bandwidthLimit string // rclone 带宽限制（全局默认，节点配置可覆盖）
 	metrics        *metrics.Metrics
+	taskLocks      sync.Map
 }

 // SetMetrics 注入 Prometheus 采集器。nil 时所有埋点退化为 no-op。
@@ -358,6 +364,11 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
 	if task == nil {
 		return nil, apperror.New(404, "BACKUP_TASK_NOT_FOUND", "备份任务不存在", fmt.Errorf("backup task %d not found", id))
 	}
+	unlock := s.acquireTaskStartLock(task.ID)
+	defer unlock()
+	if err := s.ensureTaskNotRunning(ctx, task); err != nil {
+		return nil, err
+	}
 	// 维护窗口校验：手动执行同样尊重窗口，避免业务高峰期误触发。
 	if strings.TrimSpace(task.MaintenanceWindows) != "" {
 		windows := backup.ParseMaintenanceWindows(task.MaintenanceWindows)
@@ -427,6 +438,27 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
 	return s.getRecordDetail(ctx, record.ID)
 }

+func (s *BackupExecutionService) acquireTaskStartLock(taskID uint) func() {
+	value, _ := s.taskLocks.LoadOrStore(taskID, &sync.Mutex{})
+	mu := value.(*sync.Mutex)
+	mu.Lock()
+	return mu.Unlock
+}
+
+func (s *BackupExecutionService) ensureTaskNotRunning(ctx context.Context, task *model.BackupTask) error {
+	taskID := task.ID
+	items, err := s.records.List(ctx, repository.BackupRecordListOptions{TaskID: &taskID, Status: model.BackupRecordStatusRunning})
+	if err != nil {
+		return apperror.Internal("BACKUP_RECORD_LIST_FAILED", "无法检查任务运行状态", err)
+	}
+	if len(items) == 0 {
+		return nil
+	}
+	return apperror.BadRequest("BACKUP_TASK_ALREADY_RUNNING",
+		fmt.Sprintf("任务「%s」正在运行（记录 #%d），请等待完成后再触发。", task.Name, items[0].ID),
+		nil)
+}
+
 // shouldNotify 按任务的告警策略决定是否发送本次通知。
 // 成功结果：始终发送（方便用户确认备份状态）。
 // 失败结果：仅当"最近 N 条记录（含本次）均为 failed"时发送，N = AlertOnConsecutiveFails。
@@ -678,6 +710,11 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
 		logger.Errorf("没有关联的存储目标")
 		return
 	}
+	storageUsage, err := s.storageUsageSnapshot(ctx)
+	if err != nil {
+		logger.Warnf("读取存储目标用量失败，跳过本次软配额校验：%v", err)
+		storageUsage = map[uint]int64{}
+	}

 	// 并行上传到所有目标
 	uploadResults = make([]StorageUploadResultItem, len(targetIDs))
@@ -701,15 +738,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
 			}
 			// 软限额校验：QuotaBytes > 0 时，已累计 + 本次 > 配额 → 拒绝上传
 			if target != nil && target.QuotaBytes > 0 {
-				currentUsed := int64(0)
-				if items, err := s.records.StorageUsage(ctx); err == nil {
-					for _, it := range items {
-						if it.StorageTargetID == targetID {
-							currentUsed = it.TotalSize
-							break
-						}
-					}
-				}
+				currentUsed := storageUsage[targetID]
 				if currentUsed+fileSize > target.QuotaBytes {
 					quotaMsg := fmt.Sprintf("超出存储目标 %s 的配额（%d + %d > %d）", targetName, currentUsed, fileSize, target.QuotaBytes)
 					uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: quotaMsg}
@@ -718,15 +747,18 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
 				}
 			}
 			logger.Infof("开始上传备份到存储目标：%s", targetName)
-			// 上传级重试：最多 3 次，指数退避（10s, 30s, 90s）
-			maxAttempts := 3
+			// 上传级重试：最多 3 次，等待时间随 context 取消及时退出。
 			var lastUploadErr error
 			var hr *hashingReader
-			for attempt := 1; attempt <= maxAttempts; attempt++ {
+			for attempt := 1; attempt <= uploadMaxAttempts; attempt++ {
 				if attempt > 1 {
-					backoff := time.Duration(attempt*attempt) * 10 * time.Second
+					backoff := time.Duration(attempt-1) * uploadRetryBackoff
 					logger.Warnf("存储目标 %s 第 %d 次重试（等待 %v）：%v", targetName, attempt, backoff, lastUploadErr)
-					time.Sleep(backoff)
+					if waitErr := waitForUploadRetry(ctx, backoff); waitErr != nil {
+						uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: waitErr.Error()}
+						logger.Warnf("存储目标 %s 上传重试已取消：%v", targetName, waitErr)
+						return
+					}
 				}
 				artifact, openErr := os.Open(finalPath)
 				if openErr != nil {
@@ -756,7 +788,7 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
 			}
 			if lastUploadErr != nil {
 				uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: lastUploadErr.Error()}
-				logger.Warnf("存储目标 %s 上传失败（已重试 %d 次）：%v", targetName, maxAttempts, lastUploadErr)
+				logger.Warnf("存储目标 %s 上传失败（已重试 %d 次）：%v", targetName, uploadMaxAttempts, lastUploadErr)
 				return
 			}
 			// 完整性校验：对比实际传输字节数
@@ -881,6 +913,32 @@ func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model
 	return s.tasks.Update(ctx, task)
 }

+func (s *BackupExecutionService) storageUsageSnapshot(ctx context.Context) (map[uint]int64, error) {
+	items, err := s.records.StorageUsage(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("storage usage snapshot: %w", err)
+	}
+	usage := make(map[uint]int64, len(items))
+	for _, item := range items {
+		usage[item.StorageTargetID] = item.TotalSize
+	}
+	return usage, nil
+}
+
+func waitForUploadRetry(ctx context.Context, delay time.Duration) error {
+	if delay <= 0 {
+		return nil
+	}
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
 func (s *BackupExecutionService) resolveProvider(ctx context.Context, targetID uint) (storage.StorageProvider, error) {
 	return s.resolveProviderForNode(ctx, targetID, 0)
 }
@@ -2,11 +2,13 @@ package service

 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"

@@ -40,9 +42,11 @@ func (f *testStorageFactory) New(_ context.Context, config map[string]any) (stor
 }

 type testStorageProvider struct {
-	name       string
-	failUpload bool
-	objects    map[string][]byte
+	name        string
+	failUpload  bool
+	blockUpload <-chan struct{}
+	onUpload    func()
+	objects     map[string][]byte
 }

 func (p *testStorageProvider) Type() storage.ProviderType { return "test_storage" }
@@ -50,6 +54,12 @@ func (p *testStorageProvider) TestConnection(context.Context) error {
 	return nil
 }
 func (p *testStorageProvider) Upload(_ context.Context, objectKey string, reader io.Reader, _ int64, _ map[string]string) error {
+	if p.blockUpload != nil {
+		<-p.blockUpload
+	}
+	if p.onUpload != nil {
+		p.onUpload()
+	}
 	if p.failUpload {
 		return fmt.Errorf("upload failed for %s", p.name)
 	}
@@ -193,6 +203,39 @@ func TestBackupExecutionServiceNodePoolSelectionDoesNotPersistTaskNodeID(t *test
 	}
 }

+func TestBackupExecutionServiceRejectsDuplicateRunningTask(t *testing.T) {
+	executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
+	ctx := context.Background()
+
+	task, err := tasks.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID task returned error: %v", err)
+	}
+	startedAt := time.Now().UTC()
+	running := &model.BackupRecord{
+		TaskID:          task.ID,
+		StorageTargetID: task.StorageTargetID,
+		NodeID:          0,
+		Status:          model.BackupRecordStatusRunning,
+		StartedAt:       startedAt,
+	}
+	if err := records.Create(ctx, running); err != nil {
+		t.Fatalf("Create running record returned error: %v", err)
+	}
+
+	_, err = executionService.RunTaskByIDSync(ctx, task.ID)
+	if err == nil || !strings.Contains(err.Error(), "正在运行") {
+		t.Fatalf("expected duplicate running task to be rejected, got %v", err)
+	}
+	items, err := records.List(ctx, repository.BackupRecordListOptions{Status: model.BackupRecordStatusRunning})
+	if err != nil {
+		t.Fatalf("List running records returned error: %v", err)
+	}
+	if len(items) != 1 || items[0].ID != running.ID {
+		t.Fatalf("expected only the original running record, got %#v", items)
+	}
+}
+
 func TestBackupExecutionServiceDeleteRecordDispatchesRemoteLocalDiskCleanup(t *testing.T) {
 	executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
 	ctx := context.Background()
@@ -334,6 +377,155 @@ func TestBackupExecutionServiceRecordsFirstSuccessfulStorageTarget(t *testing.T)
 	}
 }

+func TestBackupExecutionServiceUploadRetryStopsWhenContextCancelled(t *testing.T) {
+	executionService, _, tasks, targets, records, _, _ := newExecutionTestServices(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	var cancelOnce sync.Once
+	failing := &testStorageProvider{
+		name:       "failing",
+		failUpload: true,
+		onUpload: func() {
+			cancelOnce.Do(cancel)
+		},
+	}
+	executionService.storageRegistry = storage.NewRegistry(&testStorageFactory{providers: map[string]*testStorageProvider{
+		"failing": failing,
+	}})
+	cipher := codec.NewConfigCipher("execution-secret")
+	failingConfig, err := cipher.EncryptJSON(map[string]any{"name": "failing"})
+	if err != nil {
+		t.Fatalf("EncryptJSON returned error: %v", err)
+	}
+	if err := targets.Update(ctx, &model.StorageTarget{
+		ID:               1,
+		Name:             "local",
+		Type:             "test_storage",
+		Enabled:          true,
+		ConfigCiphertext: failingConfig,
+		ConfigVersion:    1,
+		LastTestStatus:   "unknown",
+	}); err != nil {
+		t.Fatalf("Update target returned error: %v", err)
+	}
+	task, err := tasks.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID task returned error: %v", err)
+	}
+	startedAt := time.Now().UTC()
+	record := &model.BackupRecord{
+		TaskID:          task.ID,
+		StorageTargetID: task.StorageTargetID,
+		Status:          model.BackupRecordStatusRunning,
+		StartedAt:       startedAt,
+	}
+	if err := records.Create(ctx, record); err != nil {
+		t.Fatalf("Create record returned error: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		executionService.executeTask(ctx, task, record.ID, startedAt)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("expected cancelled upload retry to stop without waiting for backoff sleep")
+	}
+}
+
+func TestBackupExecutionServiceReadsStorageUsageOnceForMultiTargetQuotaChecks(t *testing.T) {
+	executionService, _, tasks, targets, records, _, _ := newExecutionTestServices(t)
+	ctx := context.Background()
+	first := &testStorageProvider{name: "first", objects: map[string][]byte{}}
+	second := &testStorageProvider{name: "second", objects: map[string][]byte{}}
+	executionService.storageRegistry = storage.NewRegistry(&testStorageFactory{providers: map[string]*testStorageProvider{
+		"first":  first,
+		"second": second,
+	}})
+	cipher := codec.NewConfigCipher("execution-secret")
+	firstConfig, err := cipher.EncryptJSON(map[string]any{"name": "first"})
+	if err != nil {
+		t.Fatalf("EncryptJSON first returned error: %v", err)
+	}
+	secondConfig, err := cipher.EncryptJSON(map[string]any{"name": "second"})
+	if err != nil {
+		t.Fatalf("EncryptJSON second returned error: %v", err)
+	}
+	if err := targets.Update(ctx, &model.StorageTarget{ID: 1, Name: "local", Type: "test_storage", Enabled: true, ConfigCiphertext: firstConfig, ConfigVersion: 1, LastTestStatus: "unknown", QuotaBytes: 1 << 30}); err != nil {
+		t.Fatalf("Update first target returned error: %v", err)
+	}
+	if err := targets.Create(ctx, &model.StorageTarget{Name: "second", Type: "test_storage", Enabled: true, ConfigCiphertext: secondConfig, ConfigVersion: 1, LastTestStatus: "unknown", QuotaBytes: 1 << 30}); err != nil {
+		t.Fatalf("Create second target returned error: %v", err)
+	}
+	task, err := tasks.FindByID(ctx, 1)
+	if err != nil {
+		t.Fatalf("FindByID task returned error: %v", err)
+	}
+	task.StorageTargets = []model.StorageTarget{{ID: 1}, {ID: 2}}
+	if err := tasks.Update(ctx, task); err != nil {
+		t.Fatalf("Update task returned error: %v", err)
+	}
+	executionService.records = &storageUsageCountingRecordRepo{BackupRecordRepository: records}
+
+	detail, err := executionService.RunTaskByIDSync(ctx, task.ID)
+	if err != nil {
+		t.Fatalf("RunTaskByIDSync returned error: %v", err)
+	}
+	if detail.Status != model.BackupRecordStatusSuccess {
+		t.Fatalf("expected success, got %#v", detail)
+	}
+	countingRepo := executionService.records.(*storageUsageCountingRecordRepo)
+	if countingRepo.usageCalls != 1 {
+		t.Fatalf("expected StorageUsage to be called once for quota snapshot, got %d", countingRepo.usageCalls)
+	}
+	if len(first.objects) != 1 || len(second.objects) != 1 {
+		t.Fatalf("expected both targets to receive upload, got first=%d second=%d", len(first.objects), len(second.objects))
+	}
+}
+
+func TestBackupExecutionServiceContinuesWhenStorageUsageSnapshotFails(t *testing.T) {
+	executionService, _, _, targets, records, _, _ := newExecutionTestServices(t)
+	ctx := context.Background()
+	provider := &testStorageProvider{name: "primary", objects: map[string][]byte{}}
+	executionService.storageRegistry = storage.NewRegistry(&testStorageFactory{providers: map[string]*testStorageProvider{
+		"primary": provider,
+	}})
+	cipher := codec.NewConfigCipher("execution-secret")
+	configCiphertext, err := cipher.EncryptJSON(map[string]any{"name": "primary"})
+	if err != nil {
+		t.Fatalf("EncryptJSON returned error: %v", err)
+	}
+	if err := targets.Update(ctx, &model.StorageTarget{
+		ID:               1,
+		Name:             "local",
+		Type:             "test_storage",
+		Enabled:          true,
+		ConfigCiphertext: configCiphertext,
+		ConfigVersion:    1,
+		LastTestStatus:   "unknown",
+		QuotaBytes:       1 << 30,
+	}); err != nil {
+		t.Fatalf("Update target returned error: %v", err)
+	}
+	executionService.records = &storageUsageFailingRecordRepo{
+		BackupRecordRepository: records,
+		err:                    errStorageUsageFailed,
+	}
+
+	detail, err := executionService.RunTaskByIDSync(ctx, 1)
+	if err != nil {
+		t.Fatalf("RunTaskByIDSync returned error: %v", err)
+	}
+	if detail.Status != model.BackupRecordStatusSuccess {
+		t.Fatalf("expected success despite soft quota usage snapshot error, got %#v", detail)
+	}
+	if len(provider.objects) != 1 {
+		t.Fatalf("expected upload to proceed, got %d uploaded objects", len(provider.objects))
+	}
+}
+
 func TestBackupRecordServiceRestore(t *testing.T) {
 	executionService, recordService, _, _, _, sourceDir, _ := newExecutionTestServices(t)
 	detail, err := executionService.RunTaskByIDSync(context.Background(), 1)
@@ -354,3 +546,27 @@ func TestBackupRecordServiceRestore(t *testing.T) {
 		t.Fatalf("unexpected restored content: %s", string(content))
 	}
 }
+
+type storageUsageCountingRecordRepo struct {
+	repository.BackupRecordRepository
+	mu         sync.Mutex
+	usageCalls int
+}
+
+func (r *storageUsageCountingRecordRepo) StorageUsage(ctx context.Context) ([]repository.BackupStorageUsageItem, error) {
+	r.mu.Lock()
+	r.usageCalls++
+	r.mu.Unlock()
+	return r.BackupRecordRepository.StorageUsage(ctx)
+}
+
+type storageUsageFailingRecordRepo struct {
+	repository.BackupRecordRepository
+	err error
+}
+
+func (r *storageUsageFailingRecordRepo) StorageUsage(context.Context) ([]repository.BackupStorageUsageItem, error) {
+	return nil, r.err
+}
+
+var errStorageUsageFailed = errors.New("storage usage failed")
@@ -33,16 +33,16 @@ type BackupTaskUpsertInput struct {
 	DBPassword       string   `json:"dbPassword" binding:"max=255"`
 	DBName           string   `json:"dbName" binding:"max=255"`
 	DBPath           string   `json:"dbPath" binding:"max=500"`
-	StorageTargetID  uint     `json:"storageTargetId"`                       // deprecated: 向后兼容
-	StorageTargetIDs []uint   `json:"storageTargetIds"`                      // 新增：多存储目标
-	NodeID           uint     `json:"nodeId"`                                // 执行节点（0 = 本机 Master 或节点池）
+	StorageTargetID  uint     `json:"storageTargetId"`  // deprecated: 向后兼容
+	StorageTargetIDs []uint   `json:"storageTargetIds"` // 新增：多存储目标
+	NodeID           uint     `json:"nodeId"`           // 执行节点（0 = 本机 Master 或节点池）
 	// NodePoolTag 节点池标签。NodeID=0 且本字段非空时，调度器动态从 Labels 命中的在线节点中选负载最低者。
-	NodePoolTag      string   `json:"nodePoolTag" binding:"max=64"`
-	Tags             string   `json:"tags" binding:"max=500"`                // 逗号分隔标签
-	RetentionDays    int      `json:"retentionDays"`
-	Compression      string   `json:"compression" binding:"omitempty,oneof=gzip none"`
-	Encrypt          bool     `json:"encrypt"`
-	MaxBackups       int      `json:"maxBackups"`
+	NodePoolTag   string `json:"nodePoolTag" binding:"max=64"`
+	Tags          string `json:"tags" binding:"max=500"` // 逗号分隔标签
+	RetentionDays int    `json:"retentionDays"`
+	Compression   string `json:"compression" binding:"omitempty,oneof=gzip none"`
+	Encrypt       bool   `json:"encrypt"`
+	MaxBackups    int    `json:"maxBackups"`
 	// ExtraConfig 类型特有扩展配置（如 SAP HANA 的 backupLevel/backupChannels）
 	ExtraConfig map[string]any `json:"extraConfig"`
 	// 验证（恢复演练）配置
@@ -70,8 +70,8 @@ type BackupTaskSummary struct {
 	Type               string     `json:"type"`
 	Enabled            bool       `json:"enabled"`
 	CronExpr           string     `json:"cronExpr"`
-	StorageTargetID    uint       `json:"storageTargetId"`              // deprecated: 取第一个
-	StorageTargetName  string     `json:"storageTargetName"`            // deprecated: 取第一个
+	StorageTargetID    uint       `json:"storageTargetId"`   // deprecated: 取第一个
+	StorageTargetName  string     `json:"storageTargetName"` // deprecated: 取第一个
 	StorageTargetIDs   []uint     `json:"storageTargetIds"`
 	StorageTargetNames []string   `json:"storageTargetNames"`
 	NodeID             uint       `json:"nodeId"`
@@ -91,10 +91,10 @@ type BackupTaskSummary struct {
 	SLAHoursRPO             int    `json:"slaHoursRpo"`
 	AlertOnConsecutiveFails int    `json:"alertOnConsecutiveFails"`
 	// 备份复制目标（3-2-1）
-	ReplicationTargetIDs []uint `json:"replicationTargetIds"`
-	MaintenanceWindows   string `json:"maintenanceWindows"`
-	DependsOnTaskIDs     []uint `json:"dependsOnTaskIds"`
-	UpdatedAt          time.Time  `json:"updatedAt"`
+	ReplicationTargetIDs []uint    `json:"replicationTargetIds"`
+	MaintenanceWindows   string    `json:"maintenanceWindows"`
+	DependsOnTaskIDs     []uint    `json:"dependsOnTaskIds"`
+	UpdatedAt            time.Time `json:"updatedAt"`
 }

 type BackupTaskDetail struct {
@@ -488,6 +488,7 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B
 			return apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", fmt.Sprintf("关联的存储目标 %d 不存在", tid), nil)
 		}
 	}
+	var fixedNode *model.Node
 	if input.NodeID > 0 && s.nodes != nil {
 		node, err := s.nodes.FindByID(ctx, input.NodeID)
 		if err != nil {
@@ -496,12 +497,17 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B
 		if node == nil {
 			return apperror.BadRequest("BACKUP_TASK_INVALID", "所选执行节点不存在", nil)
 		}
+		fixedNode = node
 	}
 	// 节点池与固定节点互斥：固定节点已确定执行位置，不再动态调度
 	if input.NodeID > 0 && strings.TrimSpace(input.NodePoolTag) != "" {
 		return apperror.BadRequest("BACKUP_TASK_INVALID",
 			"固定执行节点与节点池标签只能选其一", nil)
 	}
+	if input.Encrypt && (strings.TrimSpace(input.NodePoolTag) != "" || (fixedNode != nil && !fixedNode.IsLocal)) {
+		return apperror.BadRequest("BACKUP_TASK_REMOTE_ENCRYPT_UNSUPPORTED",
+			"远程节点暂不支持加密备份。请关闭加密，或将任务固定在 Master 本机执行。", nil)
+	}
 	if input.RetentionDays < 0 {
 		return apperror.BadRequest("BACKUP_TASK_INVALID", "保留天数不能小于 0", nil)
 	}
@@ -639,38 +645,38 @@ func (s *BackupTaskService) buildTask(existing *model.BackupTask, input BackupTa
 		return nil, apperror.BadRequest("BACKUP_TASK_INVALID", "扩展配置格式不合法", err)
 	}
 	item := &model.BackupTask{
-		Name:                 strings.TrimSpace(input.Name),
-		Type:                 normalizeBackupTaskType(input.Type),
-		Enabled:              input.Enabled,
-		CronExpr:             strings.TrimSpace(input.CronExpr),
-		SourcePath:           primarySourcePath,
-		SourcePaths:          sourcePathsJSON,
-		ExcludePatterns:      excludePatterns,
-		DBHost:               strings.TrimSpace(input.DBHost),
-		DBPort:               input.DBPort,
-		DBUser:               strings.TrimSpace(input.DBUser),
-		DBPasswordCiphertext: passwordCiphertext,
-		DBName:               strings.TrimSpace(input.DBName),
-		DBPath:               strings.TrimSpace(input.DBPath),
-		ExtraConfig:          extraConfigJSON,
-		StorageTargetID:      primaryTargetID,
-		StorageTargets:       storageTargets,
-		NodeID:               input.NodeID,
-		NodePoolTag:          strings.TrimSpace(input.NodePoolTag),
-		Tags:                 strings.TrimSpace(input.Tags),
-		RetentionDays:        input.RetentionDays,
-		Compression:          compression,
-		Encrypt:              input.Encrypt,
-		MaxBackups:           maxBackups,
-		LastStatus:           "idle",
-		VerifyEnabled:        input.VerifyEnabled,
-		VerifyCronExpr:       strings.TrimSpace(input.VerifyCronExpr),
-		VerifyMode:           normalizeVerifyMode(input.VerifyMode),
-		SLAHoursRPO:          maxInt(0, input.SLAHoursRPO),
+		Name:                    strings.TrimSpace(input.Name),
+		Type:                    normalizeBackupTaskType(input.Type),
+		Enabled:                 input.Enabled,
+		CronExpr:                strings.TrimSpace(input.CronExpr),
+		SourcePath:              primarySourcePath,
+		SourcePaths:             sourcePathsJSON,
+		ExcludePatterns:         excludePatterns,
+		DBHost:                  strings.TrimSpace(input.DBHost),
+		DBPort:                  input.DBPort,
+		DBUser:                  strings.TrimSpace(input.DBUser),
+		DBPasswordCiphertext:    passwordCiphertext,
+		DBName:                  strings.TrimSpace(input.DBName),
+		DBPath:                  strings.TrimSpace(input.DBPath),
+		ExtraConfig:             extraConfigJSON,
+		StorageTargetID:         primaryTargetID,
+		StorageTargets:          storageTargets,
+		NodeID:                  input.NodeID,
+		NodePoolTag:             strings.TrimSpace(input.NodePoolTag),
+		Tags:                    strings.TrimSpace(input.Tags),
+		RetentionDays:           input.RetentionDays,
+		Compression:             compression,
+		Encrypt:                 input.Encrypt,
+		MaxBackups:              maxBackups,
+		LastStatus:              "idle",
+		VerifyEnabled:           input.VerifyEnabled,
+		VerifyCronExpr:          strings.TrimSpace(input.VerifyCronExpr),
+		VerifyMode:              normalizeVerifyMode(input.VerifyMode),
+		SLAHoursRPO:             maxInt(0, input.SLAHoursRPO),
 		AlertOnConsecutiveFails: alertThreshold(input.AlertOnConsecutiveFails),
-		ReplicationTargetIDs: encodeUintCSV(input.ReplicationTargetIDs),
-		MaintenanceWindows:   strings.TrimSpace(input.MaintenanceWindows),
-		DependsOnTaskIDs:     encodeUintCSV(input.DependsOnTaskIDs),
+		ReplicationTargetIDs:    encodeUintCSV(input.ReplicationTargetIDs),
+		MaintenanceWindows:      strings.TrimSpace(input.MaintenanceWindows),
+		DependsOnTaskIDs:        encodeUintCSV(input.DependsOnTaskIDs),
 	}
 	if existing != nil {
 		item.LastRunAt = existing.LastRunAt
@@ -736,25 +742,25 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary {
 		primaryName = targetNames[0]
 	}
 	return BackupTaskSummary{
-		ID:                 item.ID,
-		Name:               item.Name,
-		Type:               normalizeBackupTaskType(item.Type),
-		Enabled:            item.Enabled,
-		CronExpr:           item.CronExpr,
-		StorageTargetID:    primaryID,
-		StorageTargetName:  primaryName,
-		StorageTargetIDs:   targetIDs,
-		StorageTargetNames: targetNames,
-		NodeID:             item.NodeID,
-		NodeName:           item.Node.Name,
-		NodePoolTag:        item.NodePoolTag,
-		Tags:               item.Tags,
-		RetentionDays:      item.RetentionDays,
-		Compression:        item.Compression,
-		Encrypt:            item.Encrypt,
-		MaxBackups:         item.MaxBackups,
-		LastRunAt:          item.LastRunAt,
-		LastStatus:         item.LastStatus,
+		ID:                      item.ID,
+		Name:                    item.Name,
+		Type:                    normalizeBackupTaskType(item.Type),
+		Enabled:                 item.Enabled,
+		CronExpr:                item.CronExpr,
+		StorageTargetID:         primaryID,
+		StorageTargetName:       primaryName,
+		StorageTargetIDs:        targetIDs,
+		StorageTargetNames:      targetNames,
+		NodeID:                  item.NodeID,
+		NodeName:                item.Node.Name,
+		NodePoolTag:             item.NodePoolTag,
+		Tags:                    item.Tags,
+		RetentionDays:           item.RetentionDays,
+		Compression:             item.Compression,
+		Encrypt:                 item.Encrypt,
+		MaxBackups:              item.MaxBackups,
+		LastRunAt:               item.LastRunAt,
+		LastStatus:              item.LastStatus,
 		VerifyEnabled:           item.VerifyEnabled,
 		VerifyCronExpr:          item.VerifyCronExpr,
 		VerifyMode:              item.VerifyMode,
@@ -763,7 +769,7 @@ func toBackupTaskSummary(item *model.BackupTask) BackupTaskSummary {
 		ReplicationTargetIDs:    parseUintCSV(item.ReplicationTargetIDs),
 		MaintenanceWindows:      item.MaintenanceWindows,
 		DependsOnTaskIDs:        parseUintCSV(item.DependsOnTaskIDs),
-		UpdatedAt:          item.UpdatedAt,
+		UpdatedAt:               item.UpdatedAt,
 	}
 }

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"testing"

 	"backupx/server/internal/config"
@@ -29,6 +30,82 @@ func newBackupTaskServiceForTest(t *testing.T) (*BackupTaskService, repository.S
 	return service, targets, tasks
 }

+func TestBackupTaskServiceRejectsEncryptedRemoteTasks(t *testing.T) {
+	ctx := context.Background()
+	service, targets, _ := newBackupTaskServiceForTest(t)
+	service.SetNodeRepository(&nodeRepoStub{nodes: []model.Node{
+		{ID: 41, Name: "master", Token: "master-token", Status: model.NodeStatusOnline, IsLocal: true},
+		{ID: 42, Name: "edge", Token: "edge-token", Status: model.NodeStatusOnline, IsLocal: false},
+	}})
+	if err := targets.Create(ctx, &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "ciphertext", ConfigVersion: 1, LastTestStatus: "unknown"}); err != nil {
+		t.Fatalf("seed storage target error: %v", err)
+	}
+
+	_, err := service.Create(ctx, BackupTaskUpsertInput{
+		Name:            "encrypted-node-pool",
+		Type:            "file",
+		Enabled:         true,
+		SourcePath:      "/srv/site",
+		StorageTargetID: 1,
+		NodePoolTag:     "db",
+		RetentionDays:   30,
+		Compression:     "gzip",
+		MaxBackups:      10,
+		Encrypt:         true,
+	})
+	if err == nil || !strings.Contains(err.Error(), "远程节点暂不支持加密备份") {
+		t.Fatalf("expected encrypted node-pool task to be rejected, got %v", err)
+	}
+
+	created, err := service.Create(ctx, BackupTaskUpsertInput{
+		Name:            "local-encrypted",
+		Type:            "file",
+		Enabled:         true,
+		SourcePath:      "/srv/site",
+		StorageTargetID: 1,
+		RetentionDays:   30,
+		Compression:     "gzip",
+		MaxBackups:      10,
+		Encrypt:         true,
+	})
+	if err != nil {
+		t.Fatalf("Create local encrypted task returned error: %v", err)
+	}
+	localNodeTask, err := service.Create(ctx, BackupTaskUpsertInput{
+		Name:            "local-node-encrypted",
+		Type:            "file",
+		Enabled:         true,
+		SourcePath:      "/srv/site",
+		StorageTargetID: 1,
+		NodeID:          41,
+		RetentionDays:   30,
+		Compression:     "gzip",
+		MaxBackups:      10,
+		Encrypt:         true,
+	})
+	if err != nil {
+		t.Fatalf("Create encrypted task pinned to local node returned error: %v", err)
+	}
+	if localNodeTask.NodeID != 41 || !localNodeTask.Encrypt {
+		t.Fatalf("expected encrypted task to keep local node, got %#v", localNodeTask)
+	}
+	_, err = service.Update(ctx, created.ID, BackupTaskUpsertInput{
+		Name:            created.Name,
+		Type:            created.Type,
+		Enabled:         true,
+		SourcePath:      "/srv/site",
+		StorageTargetID: 1,
+		NodeID:          42,
+		RetentionDays:   30,
+		Compression:     "gzip",
+		MaxBackups:      10,
+		Encrypt:         true,
+	})
+	if err == nil || !strings.Contains(err.Error(), "远程节点暂不支持加密备份") {
+		t.Fatalf("expected encrypted fixed-node update to be rejected, got %v", err)
+	}
+}
+
 func TestBackupTaskServiceCreateAndGet(t *testing.T) {
 	ctx := context.Background()
 	service, targets, _ := newBackupTaskServiceForTest(t)
@@ -36,6 +36,19 @@ type NodeSummary struct {
 	BandwidthLimit string    `json:"bandwidthLimit"`
 	Labels         string    `json:"labels"`
 	CreatedAt      time.Time `json:"createdAt"`
+	Queue          NodeQueue `json:"queue"`
+	RunningTasks   int       `json:"runningTasks"`
+	LastError      string    `json:"lastError,omitempty"`
+	Health         string    `json:"health"`
+}
+
+type NodeQueue struct {
+	Pending          int        `json:"pending"`
+	Dispatched       int        `json:"dispatched"`
+	Depth            int        `json:"depth"`
+	Timeouts         int        `json:"timeouts"`
+	OldestActiveAt   *time.Time `json:"oldestActiveAt,omitempty"`
+	OldestActiveAgeS int        `json:"oldestActiveAgeSeconds"`
 }

 // NodeCreateInput is the input for creating a new remote node.
@@ -54,10 +67,11 @@ type NodeUpdateInput struct {

 // NodeService manages the cluster nodes.
 type NodeService struct {
-	repo      repository.NodeRepository
-	taskRepo  repository.BackupTaskRepository
-	agentRPC  NodeAgentRPC
-	version   string
+	repo     repository.NodeRepository
+	taskRepo repository.BackupTaskRepository
+	agentRPC NodeAgentRPC
+	cmdRepo  repository.AgentCommandRepository
+	version  string
 }

 // NodeAgentRPC 抽象 Agent 远程调用能力（避免 service 内循环依赖）。
@@ -81,6 +95,10 @@ func (s *NodeService) SetAgentRPC(rpc NodeAgentRPC) {
 	s.agentRPC = rpc
 }

+func (s *NodeService) SetAgentCommandRepository(cmdRepo repository.AgentCommandRepository) {
+	s.cmdRepo = cmdRepo
+}
+
 // EnsureLocalNode creates the default "local" node if it does not exist.
 func (s *NodeService) EnsureLocalNode(ctx context.Context) error {
 	existing, err := s.repo.FindLocal(ctx)
@@ -120,24 +138,10 @@ func (s *NodeService) List(ctx context.Context) ([]NodeSummary, error) {
 	if err != nil {
 		return nil, err
 	}
+	queueByNode := s.loadQueueSummaries(ctx)
 	result := make([]NodeSummary, len(nodes))
 	for i, n := range nodes {
-		result[i] = NodeSummary{
-			ID:             n.ID,
-			Name:           n.Name,
-			Hostname:       n.Hostname,
-			IPAddress:      n.IPAddress,
-			Status:         n.Status,
-			IsLocal:        n.IsLocal,
-			OS:             n.OS,
-			Arch:           n.Arch,
-			AgentVersion:   n.AgentVer,
-			LastSeen:       n.LastSeen,
-			MaxConcurrent:  n.MaxConcurrent,
-			BandwidthLimit: n.BandwidthLimit,
-			Labels:         n.Labels,
-			CreatedAt:      n.CreatedAt,
-		}
+		result[i] = s.toNodeSummary(&n, queueByNode[n.ID])
 	}
 	return result, nil
 }
@@ -150,7 +154,24 @@ func (s *NodeService) Get(ctx context.Context, id uint) (*NodeSummary, error) {
 	if node == nil {
 		return nil, apperror.New(http.StatusNotFound, "NODE_NOT_FOUND", "节点不存在", nil)
 	}
-	return &NodeSummary{
+	queueByNode := s.loadQueueSummaries(ctx)
+	summary := s.toNodeSummary(node, queueByNode[node.ID])
+	return &summary, nil
+}
+
+func (s *NodeService) loadQueueSummaries(ctx context.Context) map[uint]repository.AgentCommandQueueSummary {
+	if s.cmdRepo == nil {
+		return nil
+	}
+	summaries, err := s.cmdRepo.NodeQueueSummaries(ctx)
+	if err != nil {
+		return nil
+	}
+	return summaries
+}
+
+func (s *NodeService) toNodeSummary(node *model.Node, queue repository.AgentCommandQueueSummary) NodeSummary {
+	summary := NodeSummary{
 		ID:             node.ID,
 		Name:           node.Name,
 		Hostname:       node.Hostname,
@@ -165,7 +186,31 @@ func (s *NodeService) Get(ctx context.Context, id uint) (*NodeSummary, error) {
 		BandwidthLimit: node.BandwidthLimit,
 		Labels:         node.Labels,
 		CreatedAt:      node.CreatedAt,
-	}, nil
+		Queue: NodeQueue{
+			Pending:        queue.Pending,
+			Dispatched:     queue.Dispatched,
+			Depth:          queue.Depth,
+			Timeouts:       queue.Timeouts,
+			OldestActiveAt: queue.OldestActiveAt,
+		},
+		RunningTasks: queue.Running,
+		LastError:    queue.LastError,
+		Health:       nodeHealth(node, queue),
+	}
+	if queue.OldestActiveAt != nil {
+		summary.Queue.OldestActiveAgeS = int(time.Since(*queue.OldestActiveAt).Seconds())
+	}
+	return summary
+}
+
+func nodeHealth(node *model.Node, queue repository.AgentCommandQueueSummary) string {
+	if node.Status != model.NodeStatusOnline {
+		return "offline"
+	}
+	if queue.Timeouts > 0 || strings.TrimSpace(queue.LastError) != "" {
+		return "degraded"
+	}
+	return "healthy"
 }

 // Create registers a new remote node and returns its authentication token.
@@ -23,6 +23,9 @@ func openNodeServiceDB(t *testing.T) *gorm.DB {
 	if err := db.AutoMigrate(&model.Node{}); err != nil {
 		t.Fatalf("migrate: %v", err)
 	}
+	if err := db.AutoMigrate(&model.AgentCommand{}); err != nil {
+		t.Fatalf("migrate agent commands: %v", err)
+	}
 	return db
 }

@@ -157,3 +160,48 @@ func TestRotateTokenNotFound(t *testing.T) {
 		t.Fatalf("expected not found error")
 	}
 }
+
+func TestNodeServiceListIncludesQueueHealthSummary(t *testing.T) {
+	db := openNodeServiceDB(t)
+	nodeRepo := repository.NewNodeRepository(db)
+	cmdRepo := repository.NewAgentCommandRepository(db)
+	svc := NewNodeService(nodeRepo, "test")
+	svc.SetAgentCommandRepository(cmdRepo)
+	ctx := context.Background()
+	node := &model.Node{
+		Name:     "edge-a",
+		Token:    "edge-token",
+		Status:   model.NodeStatusOnline,
+		IsLocal:  false,
+		LastSeen: time.Now().UTC(),
+	}
+	if err := nodeRepo.Create(ctx, node); err != nil {
+		t.Fatalf("Create node returned error: %v", err)
+	}
+	old := time.Now().UTC().Add(-time.Minute)
+	if err := cmdRepo.Create(ctx, &model.AgentCommand{NodeID: node.ID, Type: model.AgentCommandTypeRunTask, Status: model.AgentCommandStatusPending, CreatedAt: old}); err != nil {
+		t.Fatalf("Create pending command returned error: %v", err)
+	}
+	completedAt := time.Now().UTC()
+	if err := cmdRepo.Create(ctx, &model.AgentCommand{NodeID: node.ID, Type: model.AgentCommandTypeRunTask, Status: model.AgentCommandStatusTimeout, ErrorMessage: "agent timeout", CompletedAt: &completedAt}); err != nil {
+		t.Fatalf("Create timeout command returned error: %v", err)
+	}
+
+	items, err := svc.List(ctx)
+	if err != nil {
+		t.Fatalf("List returned error: %v", err)
+	}
+	if len(items) != 1 {
+		t.Fatalf("expected one node, got %#v", items)
+	}
+	got := items[0]
+	if got.Queue.Pending != 1 || got.Queue.Depth != 1 || got.Queue.Timeouts != 1 {
+		t.Fatalf("unexpected queue summary: %#v", got.Queue)
+	}
+	if got.Health != "degraded" || got.LastError != "agent timeout" {
+		t.Fatalf("expected terminal command errors to degrade healthy node, got %#v", got)
+	}
+	if got.Queue.OldestActiveAt == nil || got.Queue.OldestActiveAgeS <= 0 {
+		t.Fatalf("expected oldest active metadata, got %#v", got.Queue)
+	}
+}