GithubBackup/BackupX
1
0
Fork 0
mirror of https://github.com/Awuqing/BackupX.git synced 2026-05-31 23:39:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

功能: 集成 rclone 高级传输特性 + 全 70+ 后端支持

Browse Source 
1. 失败自动重试：rclone Pacer 指数退避，默认 10 次底层 HTTP 重试
2. 带宽限制：配置 bandwidth_limit + Settings 运行时可调
3. 上传实时进度：progressReader + LogHub SSE 推送字节级进度/速率
4. 存储空间查询：StorageAbout 可选接口，GetUsage 返回远端真实空间
5. 全 rclone 后端：backend/all 引入 70+ 后端，新增 rclone 存储类型，
   API 驱动的可搜索后端选择器 + 动态配置表单
This commit is contained in:
Awuqing
2026-03-31 23:37:59 +08:00
parent cf5740b573
commit 1003302bdd
23 changed files with 1220 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
server/internal/service/backup_execution_service.go
View File

@@ -21,6 +21,7 @@ import (
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
"backupx/server/internal/storage/rclone"
"backupx/server/pkg/compress"
backupcrypto "backupx/server/pkg/crypto"
)
@@ -84,6 +85,8 @@ type BackupExecutionService struct {
now func() time.Time
tempDir string
semaphore chan struct{}
retries int // rclone 底层重试次数
bandwidthLimit string // rclone 带宽限制
}
func NewBackupExecutionService(
@@ -98,6 +101,8 @@ func NewBackupExecutionService(
notifier BackupResultNotifier,
tempDir string,
maxConcurrent int,
retries int,
bandwidthLimit string,
) *BackupExecutionService {
if notifier == nil {
notifier = noopBackupNotifier{}
@@ -121,9 +126,11 @@ func NewBackupExecutionService(
async: func(job func()) {
go job()
},
now: func() time.Time { return time.Now().UTC() },
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
now: func() time.Time { return time.Now().UTC() },
tempDir: tempDir,
semaphore: make(chan struct{}, maxConcurrent),
retries: retries,
bandwidthLimit: bandwidthLimit,
}
}
@@ -366,7 +373,21 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
logger.Infof("开始上传备份到存储目标：%s", targetName)
// hashingReader: 上传过程中同步计算字节数 + SHA-256，单次读取零额外 I/O
hr := newHashingReader(artifact)
if uploadErr := provider.Upload(ctx, storagePath, hr, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil {
// progressReader: 包装 hashingReader，通过 LogHub 推送实时上传进度
pr := newProgressReader(hr, fileSize, func(bytesRead int64, speedBps float64) {
percent := float64(0)
if fileSize > 0 {
percent = float64(bytesRead) / float64(fileSize) * 100
}
s.logHub.AppendProgress(recordID, backup.ProgressInfo{
BytesSent: bytesRead,
TotalBytes: fileSize,
Percent: percent,
SpeedBps: speedBps,
TargetName: targetName,
})
})
if uploadErr := provider.Upload(ctx, storagePath, pr, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil {
uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()}
logger.Warnf("存储目标 %s 上传失败：%v", targetName, uploadErr)
return
@@ -447,6 +468,11 @@ func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model
}
func (s *BackupExecutionService) resolveProvider(ctx context.Context, targetID uint) (storage.StorageProvider, error) {
// 注入 rclone 传输配置（重试、带宽限制）
ctx = rclone.ConfiguredContext(ctx, rclone.TransferConfig{
LowLevelRetries: s.retries,
BandwidthLimit: s.bandwidthLimit,
})
target, err := s.targets.FindByID(ctx, targetID)
if err != nil {
return nil, apperror.Internal("BACKUP_STORAGE_TARGET_GET_FAILED", "无法获取存储目标详情", err)

2
server/internal/service/backup_execution_service_test.go
View File

@@ -59,7 +59,7 @@ func newExecutionTestServices(t *testing.T) (*BackupExecutionService, *BackupRec
if err := os.MkdirAll(tempDir, 0o755); err != nil {
t.Fatalf("MkdirAll tempDir returned error: %v", err)
}
executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, tempDir, 2)
executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, tempDir, 2, 10, "")
recordService := NewBackupRecordService(records, executionService, logHub)
return executionService, recordService, tasks, targets, records, sourceDir, storageDir
}

52
server/internal/service/progress_reader.go Normal file
View File

@@ -0,0 +1,52 @@
package service
import (
"io"
"sync/atomic"
"time"
)
// progressCallback 在每次读取时被调用，报告已读字节数和估算速率。
type progressCallback func(bytesRead int64, speedBps float64)
// progressReader 包装 io.Reader，定期通过回调报告传输进度。
type progressReader struct {
reader io.Reader
total int64
read atomic.Int64
callback progressCallback
startTime time.Time
lastCall time.Time
interval time.Duration
}
func newProgressReader(reader io.Reader, total int64, callback progressCallback) *progressReader {
now := time.Now()
return &progressReader{
reader: reader,
total: total,
callback: callback,
startTime: now,
lastCall: now,
interval: 500 * time.Millisecond,
}
}
func (r *progressReader) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
if n > 0 {
current := r.read.Add(int64(n))
now := time.Now()
isFinal := err == io.EOF || (r.total > 0 && current >= r.total)
if isFinal || now.Sub(r.lastCall) >= r.interval {
r.lastCall = now
elapsed := now.Sub(r.startTime).Seconds()
speed := float64(0)
if elapsed > 0 {
speed = float64(current) / elapsed
}
r.callback(current, speed)
}
}
return n, err
}

1
server/internal/service/settings_service.go
View File

@@ -22,6 +22,7 @@ var settingsKeys = []string{
"language",
"timezone",
"backup_notification_enabled",
"bandwidth_limit",
}
func (s *SettingsService) GetAll(ctx context.Context) (map[string]string, error) {

20
server/internal/service/storage_target_service.go
View File

@@ -544,10 +544,11 @@ func cloneMap(source map[string]any) map[string]any {
}
type StorageTargetUsage struct {
TargetID uint `json:"targetId"`
TargetName string `json:"targetName"`
RecordCount int64 `json:"recordCount"`
TotalSize int64 `json:"totalSize"`
TargetID uint `json:"targetId"`
TargetName string `json:"targetName"`
RecordCount int64 `json:"recordCount"`
TotalSize int64 `json:"totalSize"`
DiskUsage *storage.StorageUsageInfo `json:"diskUsage,omitempty"`
}
func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageTargetUsage, error) {
@@ -570,5 +571,16 @@ func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageT
}
}
}
// 尝试查询远端真实存储空间（部分后端如 local/Google Drive/WebDAV 支持）
configMap := map[string]any{}
if decryptErr := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); decryptErr == nil {
if provider, createErr := s.registry.Create(ctx, target.Type, configMap); createErr == nil {
if abouter, ok := provider.(storage.StorageAbout); ok {
if diskUsage, aboutErr := abouter.About(ctx); aboutErr == nil {
result.DiskUsage = diskUsage
}
}
}
}
return result, nil
}