mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-06-28 21:11:23 +08:00
功能: 集成 rclone 高级传输特性 + 全 70+ 后端支持
1. 失败自动重试:rclone Pacer 指数退避,默认 10 次底层 HTTP 重试 2. 带宽限制:配置 bandwidth_limit + Settings 运行时可调 3. 上传实时进度:progressReader + LogHub SSE 推送字节级进度/速率 4. 存储空间查询:StorageAbout 可选接口,GetUsage 返回远端真实空间 5. 全 rclone 后端:backend/all 引入 70+ 后端,新增 rclone 存储类型, API 驱动的可搜索后端选择器 + 动态配置表单
This commit is contained in:
@@ -71,6 +71,7 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application,
|
||||
storageRclone.NewTencentCOSFactory(),
|
||||
storageRclone.NewQiniuKodoFactory(),
|
||||
storageRclone.NewFTPFactory(),
|
||||
storageRclone.NewRcloneFactory(),
|
||||
)
|
||||
storageTargetService := service.NewStorageTargetService(storageTargetRepo, oauthSessionRepo, storageRegistry, configCipher)
|
||||
storageTargetService.SetBackupTaskRepository(backupTaskRepo)
|
||||
@@ -81,7 +82,14 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application,
|
||||
retentionService := backupretention.NewService(backupRecordRepo)
|
||||
notifyRegistry := notify.NewRegistry(notify.NewEmailNotifier(), notify.NewWebhookNotifier(), notify.NewTelegramNotifier())
|
||||
notificationService := service.NewNotificationService(notificationRepo, notifyRegistry, configCipher)
|
||||
backupExecutionService := service.NewBackupExecutionService(backupTaskRepo, backupRecordRepo, storageTargetRepo, storageRegistry, backupRunnerRegistry, logHub, retentionService, configCipher, notificationService, cfg.Backup.TempDir, cfg.Backup.MaxConcurrent)
|
||||
// 初始化 rclone 传输配置(重试 + 带宽限制)
|
||||
rcloneCtx := storageRclone.ConfiguredContext(ctx, storageRclone.TransferConfig{
|
||||
LowLevelRetries: cfg.Backup.Retries,
|
||||
BandwidthLimit: cfg.Backup.BandwidthLimit,
|
||||
})
|
||||
storageRclone.StartAccounting(rcloneCtx)
|
||||
|
||||
backupExecutionService := service.NewBackupExecutionService(backupTaskRepo, backupRecordRepo, storageTargetRepo, storageRegistry, backupRunnerRegistry, logHub, retentionService, configCipher, notificationService, cfg.Backup.TempDir, cfg.Backup.MaxConcurrent, cfg.Backup.Retries, cfg.Backup.BandwidthLimit)
|
||||
schedulerService := scheduler.NewService(backupTaskRepo, backupExecutionService, appLogger)
|
||||
backupTaskService.SetScheduler(schedulerService)
|
||||
backupRecordService := service.NewBackupRecordService(backupRecordRepo, backupExecutionService, logHub)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -99,6 +100,41 @@ func (h *LogHub) Complete(recordID uint, status string) {
|
||||
}
|
||||
}
|
||||
|
||||
// AppendProgress 推送上传进度事件(节流:每个 recordID 每 500ms 最多一次,最终值始终推送)。
|
||||
func (h *LogHub) AppendProgress(recordID uint, progress ProgressInfo) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
state := h.ensureState(recordID)
|
||||
|
||||
// 节流:距上次 progress 事件不足 500ms 且未完成则跳过(100% 始终推送)
|
||||
now := time.Now().UTC()
|
||||
isFinal := progress.Percent >= 100
|
||||
if !isFinal && len(state.events) > 0 {
|
||||
last := state.events[len(state.events)-1]
|
||||
if last.Progress != nil && now.Sub(last.Timestamp) < 500*time.Millisecond {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
state.nextSequence++
|
||||
event := LogEvent{
|
||||
RecordID: recordID,
|
||||
Sequence: state.nextSequence,
|
||||
Level: "progress",
|
||||
Message: fmt.Sprintf("上传进度: %.1f%%", progress.Percent),
|
||||
Timestamp: now,
|
||||
Status: state.status,
|
||||
Progress: &progress,
|
||||
}
|
||||
state.events = append(state.events, event)
|
||||
for _, subscriber := range state.subscribers {
|
||||
select {
|
||||
case subscriber <- event:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LogHub) ensureState(recordID uint) *logStreamState {
|
||||
state, ok := h.streams[recordID]
|
||||
if ok {
|
||||
|
||||
@@ -41,13 +41,23 @@ type RunResult struct {
|
||||
}
|
||||
|
||||
type LogEvent struct {
|
||||
RecordID uint `json:"recordId"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
Level string `json:"level"`
|
||||
Message string `json:"message"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Completed bool `json:"completed"`
|
||||
Status string `json:"status"`
|
||||
RecordID uint `json:"recordId"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
Level string `json:"level"`
|
||||
Message string `json:"message"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Completed bool `json:"completed"`
|
||||
Status string `json:"status"`
|
||||
Progress *ProgressInfo `json:"progress,omitempty"`
|
||||
}
|
||||
|
||||
// ProgressInfo 描述上传进度,通过 SSE 实时推送给前端。
|
||||
type ProgressInfo struct {
|
||||
BytesSent int64 `json:"bytesSent"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
Percent float64 `json:"percent"`
|
||||
SpeedBps float64 `json:"speedBps"` // bytes/sec
|
||||
TargetName string `json:"targetName"`
|
||||
}
|
||||
|
||||
type LogWriter interface {
|
||||
|
||||
@@ -33,8 +33,10 @@ type SecurityConfig struct {
|
||||
}
|
||||
|
||||
type BackupConfig struct {
|
||||
TempDir string `mapstructure:"temp_dir"`
|
||||
MaxConcurrent int `mapstructure:"max_concurrent"`
|
||||
TempDir string `mapstructure:"temp_dir"`
|
||||
MaxConcurrent int `mapstructure:"max_concurrent"`
|
||||
Retries int `mapstructure:"retries"` // 底层 HTTP 请求重试次数,默认 10
|
||||
BandwidthLimit string `mapstructure:"bandwidth_limit"` // 带宽限制,如 "10M",空不限
|
||||
}
|
||||
|
||||
type LogConfig struct {
|
||||
@@ -96,6 +98,9 @@ func Load(configPath string) (Config, error) {
|
||||
if cfg.Backup.MaxConcurrent <= 0 {
|
||||
cfg.Backup.MaxConcurrent = 2
|
||||
}
|
||||
if cfg.Backup.Retries <= 0 {
|
||||
cfg.Backup.Retries = 10
|
||||
}
|
||||
if cfg.Log.Level == "" {
|
||||
cfg.Log.Level = "info"
|
||||
}
|
||||
@@ -135,6 +140,8 @@ func applyDefaults(v *viper.Viper) {
|
||||
v.SetDefault("security.jwt_expire", "24h")
|
||||
v.SetDefault("backup.temp_dir", "/tmp/backupx")
|
||||
v.SetDefault("backup.max_concurrent", 2)
|
||||
v.SetDefault("backup.retries", 10)
|
||||
v.SetDefault("backup.bandwidth_limit", "")
|
||||
v.SetDefault("log.level", "info")
|
||||
v.SetDefault("log.file", "./data/backupx.log")
|
||||
v.SetDefault("log.max_size", 100)
|
||||
|
||||
21
server/internal/http/rclone_handler.go
Normal file
21
server/internal/http/rclone_handler.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
storageRclone "backupx/server/internal/storage/rclone"
|
||||
"backupx/server/pkg/response"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// RcloneHandler 处理 rclone 后端元数据查询。
|
||||
type RcloneHandler struct{}
|
||||
|
||||
func NewRcloneHandler() *RcloneHandler {
|
||||
return &RcloneHandler{}
|
||||
}
|
||||
|
||||
// ListBackends 返回所有可用的 rclone 后端及其配置选项。
|
||||
func (h *RcloneHandler) ListBackends(c *gin.Context) {
|
||||
backends := storageRclone.ListBackends()
|
||||
response.Success(c, backends)
|
||||
}
|
||||
@@ -71,18 +71,22 @@ func NewRouter(deps RouterDependencies) *gin.Engine {
|
||||
|
||||
storageTargets := api.Group("/storage-targets")
|
||||
storageTargets.Use(AuthMiddleware(deps.JWTManager))
|
||||
// 静态路由必须在参数路由 /:id 之前注册,避免 Gin 路由冲突
|
||||
storageTargets.GET("", storageTargetHandler.List)
|
||||
storageTargets.GET("/:id", storageTargetHandler.Get)
|
||||
storageTargets.POST("", storageTargetHandler.Create)
|
||||
storageTargets.PUT("/:id", storageTargetHandler.Update)
|
||||
storageTargets.DELETE("/:id", storageTargetHandler.Delete)
|
||||
storageTargets.PUT("/:id/star", storageTargetHandler.ToggleStar)
|
||||
storageTargets.POST("/test", storageTargetHandler.TestConnection)
|
||||
storageTargets.POST("/:id/test", storageTargetHandler.TestSavedConnection)
|
||||
storageTargets.GET("/:id/usage", storageTargetHandler.GetUsage)
|
||||
storageTargets.POST("/google-drive/auth-url", storageTargetHandler.StartGoogleDriveOAuth)
|
||||
storageTargets.POST("/google-drive/complete", storageTargetHandler.CompleteGoogleDriveOAuth)
|
||||
storageTargets.GET("/google-drive/callback", storageTargetHandler.HandleGoogleDriveCallback)
|
||||
rcloneHandler := NewRcloneHandler()
|
||||
storageTargets.GET("/rclone/backends", rcloneHandler.ListBackends)
|
||||
// 参数路由
|
||||
storageTargets.GET("/:id", storageTargetHandler.Get)
|
||||
storageTargets.PUT("/:id", storageTargetHandler.Update)
|
||||
storageTargets.DELETE("/:id", storageTargetHandler.Delete)
|
||||
storageTargets.PUT("/:id/star", storageTargetHandler.ToggleStar)
|
||||
storageTargets.POST("/:id/test", storageTargetHandler.TestSavedConnection)
|
||||
storageTargets.GET("/:id/usage", storageTargetHandler.GetUsage)
|
||||
storageTargets.GET("/:id/google-drive/profile", storageTargetHandler.GoogleDriveProfile)
|
||||
|
||||
backupTasks := api.Group("/backup/tasks")
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"backupx/server/internal/repository"
|
||||
"backupx/server/internal/storage"
|
||||
"backupx/server/internal/storage/codec"
|
||||
"backupx/server/internal/storage/rclone"
|
||||
"backupx/server/pkg/compress"
|
||||
backupcrypto "backupx/server/pkg/crypto"
|
||||
)
|
||||
@@ -84,6 +85,8 @@ type BackupExecutionService struct {
|
||||
now func() time.Time
|
||||
tempDir string
|
||||
semaphore chan struct{}
|
||||
retries int // rclone 底层重试次数
|
||||
bandwidthLimit string // rclone 带宽限制
|
||||
}
|
||||
|
||||
func NewBackupExecutionService(
|
||||
@@ -98,6 +101,8 @@ func NewBackupExecutionService(
|
||||
notifier BackupResultNotifier,
|
||||
tempDir string,
|
||||
maxConcurrent int,
|
||||
retries int,
|
||||
bandwidthLimit string,
|
||||
) *BackupExecutionService {
|
||||
if notifier == nil {
|
||||
notifier = noopBackupNotifier{}
|
||||
@@ -121,9 +126,11 @@ func NewBackupExecutionService(
|
||||
async: func(job func()) {
|
||||
go job()
|
||||
},
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tempDir: tempDir,
|
||||
semaphore: make(chan struct{}, maxConcurrent),
|
||||
retries: retries,
|
||||
bandwidthLimit: bandwidthLimit,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,7 +373,21 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
|
||||
logger.Infof("开始上传备份到存储目标:%s", targetName)
|
||||
// hashingReader: 上传过程中同步计算字节数 + SHA-256,单次读取零额外 I/O
|
||||
hr := newHashingReader(artifact)
|
||||
if uploadErr := provider.Upload(ctx, storagePath, hr, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil {
|
||||
// progressReader: 包装 hashingReader,通过 LogHub 推送实时上传进度
|
||||
pr := newProgressReader(hr, fileSize, func(bytesRead int64, speedBps float64) {
|
||||
percent := float64(0)
|
||||
if fileSize > 0 {
|
||||
percent = float64(bytesRead) / float64(fileSize) * 100
|
||||
}
|
||||
s.logHub.AppendProgress(recordID, backup.ProgressInfo{
|
||||
BytesSent: bytesRead,
|
||||
TotalBytes: fileSize,
|
||||
Percent: percent,
|
||||
SpeedBps: speedBps,
|
||||
TargetName: targetName,
|
||||
})
|
||||
})
|
||||
if uploadErr := provider.Upload(ctx, storagePath, pr, fileSize, map[string]string{"taskId": fmt.Sprintf("%d", task.ID), "recordId": fmt.Sprintf("%d", recordID)}); uploadErr != nil {
|
||||
uploadResults[index] = StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()}
|
||||
logger.Warnf("存储目标 %s 上传失败:%v", targetName, uploadErr)
|
||||
return
|
||||
@@ -447,6 +468,11 @@ func (s *BackupExecutionService) finalizeRecord(ctx context.Context, task *model
|
||||
}
|
||||
|
||||
func (s *BackupExecutionService) resolveProvider(ctx context.Context, targetID uint) (storage.StorageProvider, error) {
|
||||
// 注入 rclone 传输配置(重试、带宽限制)
|
||||
ctx = rclone.ConfiguredContext(ctx, rclone.TransferConfig{
|
||||
LowLevelRetries: s.retries,
|
||||
BandwidthLimit: s.bandwidthLimit,
|
||||
})
|
||||
target, err := s.targets.FindByID(ctx, targetID)
|
||||
if err != nil {
|
||||
return nil, apperror.Internal("BACKUP_STORAGE_TARGET_GET_FAILED", "无法获取存储目标详情", err)
|
||||
|
||||
@@ -59,7 +59,7 @@ func newExecutionTestServices(t *testing.T) (*BackupExecutionService, *BackupRec
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll tempDir returned error: %v", err)
|
||||
}
|
||||
executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, tempDir, 2)
|
||||
executionService := NewBackupExecutionService(tasks, records, targets, storageRegistry, runnerRegistry, logHub, retentionService, cipher, nil, tempDir, 2, 10, "")
|
||||
recordService := NewBackupRecordService(records, executionService, logHub)
|
||||
return executionService, recordService, tasks, targets, records, sourceDir, storageDir
|
||||
}
|
||||
|
||||
52
server/internal/service/progress_reader.go
Normal file
52
server/internal/service/progress_reader.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// progressCallback 在每次读取时被调用,报告已读字节数和估算速率。
|
||||
type progressCallback func(bytesRead int64, speedBps float64)
|
||||
|
||||
// progressReader 包装 io.Reader,定期通过回调报告传输进度。
|
||||
type progressReader struct {
|
||||
reader io.Reader
|
||||
total int64
|
||||
read atomic.Int64
|
||||
callback progressCallback
|
||||
startTime time.Time
|
||||
lastCall time.Time
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func newProgressReader(reader io.Reader, total int64, callback progressCallback) *progressReader {
|
||||
now := time.Now()
|
||||
return &progressReader{
|
||||
reader: reader,
|
||||
total: total,
|
||||
callback: callback,
|
||||
startTime: now,
|
||||
lastCall: now,
|
||||
interval: 500 * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *progressReader) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if n > 0 {
|
||||
current := r.read.Add(int64(n))
|
||||
now := time.Now()
|
||||
isFinal := err == io.EOF || (r.total > 0 && current >= r.total)
|
||||
if isFinal || now.Sub(r.lastCall) >= r.interval {
|
||||
r.lastCall = now
|
||||
elapsed := now.Sub(r.startTime).Seconds()
|
||||
speed := float64(0)
|
||||
if elapsed > 0 {
|
||||
speed = float64(current) / elapsed
|
||||
}
|
||||
r.callback(current, speed)
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -22,6 +22,7 @@ var settingsKeys = []string{
|
||||
"language",
|
||||
"timezone",
|
||||
"backup_notification_enabled",
|
||||
"bandwidth_limit",
|
||||
}
|
||||
|
||||
func (s *SettingsService) GetAll(ctx context.Context) (map[string]string, error) {
|
||||
|
||||
@@ -544,10 +544,11 @@ func cloneMap(source map[string]any) map[string]any {
|
||||
}
|
||||
|
||||
type StorageTargetUsage struct {
|
||||
TargetID uint `json:"targetId"`
|
||||
TargetName string `json:"targetName"`
|
||||
RecordCount int64 `json:"recordCount"`
|
||||
TotalSize int64 `json:"totalSize"`
|
||||
TargetID uint `json:"targetId"`
|
||||
TargetName string `json:"targetName"`
|
||||
RecordCount int64 `json:"recordCount"`
|
||||
TotalSize int64 `json:"totalSize"`
|
||||
DiskUsage *storage.StorageUsageInfo `json:"diskUsage,omitempty"`
|
||||
}
|
||||
|
||||
func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageTargetUsage, error) {
|
||||
@@ -570,5 +571,16 @@ func (s *StorageTargetService) GetUsage(ctx context.Context, id uint) (*StorageT
|
||||
}
|
||||
}
|
||||
}
|
||||
// 尝试查询远端真实存储空间(部分后端如 local/Google Drive/WebDAV 支持)
|
||||
configMap := map[string]any{}
|
||||
if decryptErr := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); decryptErr == nil {
|
||||
if provider, createErr := s.registry.Create(ctx, target.Type, configMap); createErr == nil {
|
||||
if abouter, ok := provider.(storage.StorageAbout); ok {
|
||||
if diskUsage, aboutErr := abouter.About(ctx); aboutErr == nil {
|
||||
result.DiskUsage = diskUsage
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
// Package rclone 提供基于 rclone 的统一存储后端实现。
|
||||
// 按需引入 rclone backend,避免 backend/all 导致二进制膨胀。
|
||||
// 引入全部 rclone backend,支持 70+ 存储后端。
|
||||
package rclone
|
||||
|
||||
import (
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/ftp"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
_ "github.com/rclone/rclone/backend/webdav"
|
||||
)
|
||||
import _ "github.com/rclone/rclone/backend/all"
|
||||
|
||||
36
server/internal/storage/rclone/config.go
Normal file
36
server/internal/storage/rclone/config.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
)
|
||||
|
||||
// TransferConfig 控制 rclone 传输层行为。
|
||||
type TransferConfig struct {
|
||||
LowLevelRetries int // 底层 HTTP 请求重试次数,0 保持 rclone 默认(10)
|
||||
BandwidthLimit string // 带宽限制,如 "10M"、"1M:500k"(上传:下载),空或 "0" 不限
|
||||
}
|
||||
|
||||
// ConfiguredContext 返回注入了 rclone 传输配置的 context。
|
||||
// 各 rclone 后端在 fs.NewFs 时读取 context 中的配置,自动应用重试和限速。
|
||||
func ConfiguredContext(ctx context.Context, cfg TransferConfig) context.Context {
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
if cfg.LowLevelRetries > 0 {
|
||||
ci.LowLevelRetries = cfg.LowLevelRetries
|
||||
}
|
||||
if cfg.BandwidthLimit != "" && cfg.BandwidthLimit != "0" {
|
||||
var bwTable fs.BwTimetable
|
||||
if err := bwTable.Set(cfg.BandwidthLimit); err == nil {
|
||||
ci.BwLimit = bwTable
|
||||
}
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// StartAccounting 初始化 rclone 的传输统计和令牌桶限速系统。
|
||||
// 应在应用启动时调用一次。
|
||||
func StartAccounting(ctx context.Context) {
|
||||
accounting.Start(ctx)
|
||||
}
|
||||
@@ -345,3 +345,92 @@ func (QiniuKodoFactory) New(ctx context.Context, rawConfig map[string]any) (stor
|
||||
}
|
||||
return newFs(ctx, storage.ProviderTypeQiniuKodo, buildS3Remote("Qiniu", cfg.AccessKey, cfg.SecretKey, endpoint, cfg.Region, cfg.Bucket, true))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 通用 Rclone 后端(支持全部 70+ 后端)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type RcloneFactory struct{}
|
||||
|
||||
func NewRcloneFactory() RcloneFactory { return RcloneFactory{} }
|
||||
|
||||
func (RcloneFactory) Type() storage.ProviderType { return storage.ProviderTypeRclone }
|
||||
func (RcloneFactory) SensitiveFields() []string { return []string{"pass", "password", "secret_access_key", "client_secret", "token"} }
|
||||
|
||||
func (RcloneFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||||
backend, _ := rawConfig["backend"].(string)
|
||||
backend = strings.TrimSpace(backend)
|
||||
if backend == "" {
|
||||
return nil, fmt.Errorf("rclone backend type is required")
|
||||
}
|
||||
root, _ := rawConfig["root"].(string)
|
||||
root = strings.TrimSpace(root)
|
||||
|
||||
// 构建连接字符串::backend,key1=val1,key2=val2:root
|
||||
var b strings.Builder
|
||||
b.WriteString(":")
|
||||
b.WriteString(backend)
|
||||
for key, val := range rawConfig {
|
||||
if key == "backend" || key == "root" {
|
||||
continue
|
||||
}
|
||||
strVal := fmt.Sprintf("%v", val)
|
||||
if strings.TrimSpace(strVal) == "" {
|
||||
continue
|
||||
}
|
||||
b.WriteString(",")
|
||||
b.WriteString(key)
|
||||
b.WriteString("=")
|
||||
b.WriteString(quoteParam(strVal))
|
||||
}
|
||||
b.WriteString(":")
|
||||
b.WriteString(root)
|
||||
|
||||
return newFs(ctx, storage.ProviderTypeRclone, b.String())
|
||||
}
|
||||
|
||||
// ListBackends 返回所有可用的 rclone 后端及其配置选项。
|
||||
func ListBackends() []BackendInfo {
|
||||
var backends []BackendInfo
|
||||
for _, ri := range fs.Registry {
|
||||
if ri.Name == "union" || ri.Name == "crypt" || ri.Name == "chunker" || ri.Name == "compress" || ri.Name == "hasher" || ri.Name == "combine" {
|
||||
continue // 跳过组合/加密类后端
|
||||
}
|
||||
info := BackendInfo{
|
||||
Name: ri.Name,
|
||||
Description: ri.Description,
|
||||
}
|
||||
for _, opt := range ri.Options {
|
||||
if opt.Hide != 0 {
|
||||
continue
|
||||
}
|
||||
// 跳过 rclone 为每个后端自动添加的通用选项
|
||||
if opt.Name == "description" {
|
||||
continue
|
||||
}
|
||||
info.Options = append(info.Options, BackendOption{
|
||||
Key: opt.Name,
|
||||
Label: opt.Help,
|
||||
Required: opt.Required,
|
||||
IsPassword: opt.IsPassword,
|
||||
})
|
||||
}
|
||||
backends = append(backends, info)
|
||||
}
|
||||
return backends
|
||||
}
|
||||
|
||||
// BackendInfo 描述一个 rclone 后端。
|
||||
type BackendInfo struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Options []BackendOption `json:"options"`
|
||||
}
|
||||
|
||||
// BackendOption 描述一个后端配置选项。
|
||||
type BackendOption struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Required bool `json:"required"`
|
||||
IsPassword bool `json:"isPassword"`
|
||||
}
|
||||
|
||||
@@ -102,6 +102,24 @@ func (p *Provider) List(ctx context.Context, prefix string) ([]storage.ObjectInf
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// About 查询远端存储空间。并非所有 rclone 后端都支持。
|
||||
func (p *Provider) About(ctx context.Context) (*storage.StorageUsageInfo, error) {
|
||||
about := p.rfs.Features().About
|
||||
if about == nil {
|
||||
return nil, fmt.Errorf("rclone about: backend %s does not support About", p.providerType)
|
||||
}
|
||||
usage, err := about(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rclone about: %w", err)
|
||||
}
|
||||
return &storage.StorageUsageInfo{
|
||||
Total: usage.Total,
|
||||
Used: usage.Used,
|
||||
Free: usage.Free,
|
||||
Objects: usage.Objects,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// pathDir 返回 objectKey 的目录部分(正斜杠分隔)。
|
||||
func pathDir(objectKey string) string {
|
||||
idx := strings.LastIndex(objectKey, "/")
|
||||
|
||||
@@ -110,6 +110,79 @@ func TestBuildS3Remote(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRcloneFactoryCRUD(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
factory := NewRcloneFactory()
|
||||
// 使用 rclone 的 local 后端
|
||||
provider, err := factory.New(context.Background(), map[string]any{
|
||||
"backend": "local",
|
||||
"root": dir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RcloneFactory.New returned error: %v", err)
|
||||
}
|
||||
if err := provider.Upload(context.Background(), "test.txt", strings.NewReader("rclone"), 6, nil); err != nil {
|
||||
t.Fatalf("Upload via rclone factory returned error: %v", err)
|
||||
}
|
||||
reader, err := provider.Download(context.Background(), "test.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Download returned error: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
content, _ := io.ReadAll(reader)
|
||||
if string(content) != "rclone" {
|
||||
t.Fatalf("expected 'rclone', got %q", string(content))
|
||||
}
|
||||
if err := provider.Delete(context.Background(), "test.txt"); err != nil {
|
||||
t.Fatalf("Delete returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRcloneFactoryRequiresBackend(t *testing.T) {
|
||||
_, err := NewRcloneFactory().New(context.Background(), map[string]any{"root": "/tmp"})
|
||||
if err == nil || !strings.Contains(err.Error(), "backend") {
|
||||
t.Fatalf("expected backend required error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListBackends(t *testing.T) {
|
||||
backends := ListBackends()
|
||||
if len(backends) < 30 {
|
||||
t.Fatalf("expected at least 30 backends, got %d", len(backends))
|
||||
}
|
||||
// 确认 sftp 在列表中
|
||||
found := false
|
||||
for _, b := range backends {
|
||||
if b.Name == "sftp" {
|
||||
found = true
|
||||
if len(b.Options) == 0 {
|
||||
t.Fatal("sftp backend should have options")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("sftp backend not found in ListBackends()")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderAbout(t *testing.T) {
|
||||
factory := NewLocalDiskFactory()
|
||||
provider, err := factory.New(context.Background(), map[string]any{"basePath": t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("Factory.New returned error: %v", err)
|
||||
}
|
||||
// local 后端支持 About
|
||||
rcloneProvider := provider.(*Provider)
|
||||
usage, err := rcloneProvider.About(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("About returned error: %v", err)
|
||||
}
|
||||
if usage.Total == nil || *usage.Total <= 0 {
|
||||
t.Fatalf("expected non-zero total disk space, got %v", usage.Total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathDir(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
|
||||
@@ -20,6 +20,7 @@ const (
|
||||
ProviderTypeTencentCOS ProviderType = "tencent_cos"
|
||||
ProviderTypeQiniuKodo ProviderType = "qiniu_kodo"
|
||||
ProviderTypeFTP ProviderType = "ftp"
|
||||
ProviderTypeRclone ProviderType = "rclone"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -52,6 +53,20 @@ type ProviderFactory interface {
|
||||
Type() ProviderType
|
||||
}
|
||||
|
||||
// StorageAbout 是可选能力接口,支持查询远端存储空间。
|
||||
// 并非所有后端都支持(如 S3/FTP 不支持),通过 type assertion 检测。
|
||||
type StorageAbout interface {
|
||||
About(ctx context.Context) (*StorageUsageInfo, error)
|
||||
}
|
||||
|
||||
// StorageUsageInfo 描述远端存储的空间使用情况。
|
||||
type StorageUsageInfo struct {
|
||||
Total *int64 `json:"total,omitempty"` // 总空间(字节)
|
||||
Used *int64 `json:"used,omitempty"` // 已用空间
|
||||
Free *int64 `json:"free,omitempty"` // 可用空间
|
||||
Objects *int64 `json:"objects,omitempty"` // 对象数量
|
||||
}
|
||||
|
||||
func DecodeConfig[T any](raw map[string]any) (T, error) {
|
||||
var cfg T
|
||||
encoded, err := json.Marshal(raw)
|
||||
|
||||
Reference in New Issue
Block a user