功能: 集成 rclone 高级传输特性 + 全 70+ 后端支持

1. 失败自动重试:rclone Pacer 指数退避,默认 10 次底层 HTTP 重试
2. 带宽限制:配置 bandwidth_limit + Settings 运行时可调
3. 上传实时进度:progressReader + LogHub SSE 推送字节级进度/速率
4. 存储空间查询:StorageAbout 可选接口,GetUsage 返回远端真实空间
5. 全 rclone 后端:backend/all 引入 70+ 后端,新增 rclone 存储类型,
   API 驱动的可搜索后端选择器 + 动态配置表单
This commit is contained in:
Awuqing
2026-03-31 23:37:59 +08:00
parent cf5740b573
commit 1003302bdd
23 changed files with 1220 additions and 74 deletions

View File

@@ -1,11 +1,5 @@
// Package rclone 提供基于 rclone 的统一存储后端实现。
// 按需引入 rclone backend避免 backend/all 导致二进制膨胀
// 引入全部 rclone backend支持 70+ 存储后端
package rclone
import (
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/ftp"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/s3"
_ "github.com/rclone/rclone/backend/webdav"
)
import _ "github.com/rclone/rclone/backend/all"

View File

@@ -0,0 +1,36 @@
package rclone
import (
"context"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
)
// TransferConfig 控制 rclone 传输层行为。
type TransferConfig struct {
LowLevelRetries int // 底层 HTTP 请求重试次数0 保持 rclone 默认10
BandwidthLimit string // 带宽限制,如 "10M"、"1M:500k"(上传:下载),空或 "0" 不限
}
// ConfiguredContext 返回注入了 rclone 传输配置的 context。
// 各 rclone 后端在 fs.NewFs 时读取 context 中的配置,自动应用重试和限速。
func ConfiguredContext(ctx context.Context, cfg TransferConfig) context.Context {
ctx, ci := fs.AddConfig(ctx)
if cfg.LowLevelRetries > 0 {
ci.LowLevelRetries = cfg.LowLevelRetries
}
if cfg.BandwidthLimit != "" && cfg.BandwidthLimit != "0" {
var bwTable fs.BwTimetable
if err := bwTable.Set(cfg.BandwidthLimit); err == nil {
ci.BwLimit = bwTable
}
}
return ctx
}
// StartAccounting 初始化 rclone 的传输统计和令牌桶限速系统。
// 应在应用启动时调用一次。
func StartAccounting(ctx context.Context) {
accounting.Start(ctx)
}

View File

@@ -345,3 +345,92 @@ func (QiniuKodoFactory) New(ctx context.Context, rawConfig map[string]any) (stor
}
return newFs(ctx, storage.ProviderTypeQiniuKodo, buildS3Remote("Qiniu", cfg.AccessKey, cfg.SecretKey, endpoint, cfg.Region, cfg.Bucket, true))
}
// ---------------------------------------------------------------------------
// 通用 Rclone 后端(支持全部 70+ 后端)
// ---------------------------------------------------------------------------
type RcloneFactory struct{}
func NewRcloneFactory() RcloneFactory { return RcloneFactory{} }
func (RcloneFactory) Type() storage.ProviderType { return storage.ProviderTypeRclone }
func (RcloneFactory) SensitiveFields() []string { return []string{"pass", "password", "secret_access_key", "client_secret", "token"} }
func (RcloneFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
backend, _ := rawConfig["backend"].(string)
backend = strings.TrimSpace(backend)
if backend == "" {
return nil, fmt.Errorf("rclone backend type is required")
}
root, _ := rawConfig["root"].(string)
root = strings.TrimSpace(root)
// 构建连接字符串::backend,key1=val1,key2=val2:root
var b strings.Builder
b.WriteString(":")
b.WriteString(backend)
for key, val := range rawConfig {
if key == "backend" || key == "root" {
continue
}
strVal := fmt.Sprintf("%v", val)
if strings.TrimSpace(strVal) == "" {
continue
}
b.WriteString(",")
b.WriteString(key)
b.WriteString("=")
b.WriteString(quoteParam(strVal))
}
b.WriteString(":")
b.WriteString(root)
return newFs(ctx, storage.ProviderTypeRclone, b.String())
}
// ListBackends 返回所有可用的 rclone 后端及其配置选项。
func ListBackends() []BackendInfo {
var backends []BackendInfo
for _, ri := range fs.Registry {
if ri.Name == "union" || ri.Name == "crypt" || ri.Name == "chunker" || ri.Name == "compress" || ri.Name == "hasher" || ri.Name == "combine" {
continue // 跳过组合/加密类后端
}
info := BackendInfo{
Name: ri.Name,
Description: ri.Description,
}
for _, opt := range ri.Options {
if opt.Hide != 0 {
continue
}
// 跳过 rclone 为每个后端自动添加的通用选项
if opt.Name == "description" {
continue
}
info.Options = append(info.Options, BackendOption{
Key: opt.Name,
Label: opt.Help,
Required: opt.Required,
IsPassword: opt.IsPassword,
})
}
backends = append(backends, info)
}
return backends
}
// BackendInfo 描述一个 rclone 后端。
type BackendInfo struct {
Name string `json:"name"`
Description string `json:"description"`
Options []BackendOption `json:"options"`
}
// BackendOption 描述一个后端配置选项。
type BackendOption struct {
Key string `json:"key"`
Label string `json:"label"`
Required bool `json:"required"`
IsPassword bool `json:"isPassword"`
}

View File

@@ -102,6 +102,24 @@ func (p *Provider) List(ctx context.Context, prefix string) ([]storage.ObjectInf
return items, nil
}
// About 查询远端存储空间。并非所有 rclone 后端都支持。
func (p *Provider) About(ctx context.Context) (*storage.StorageUsageInfo, error) {
about := p.rfs.Features().About
if about == nil {
return nil, fmt.Errorf("rclone about: backend %s does not support About", p.providerType)
}
usage, err := about(ctx)
if err != nil {
return nil, fmt.Errorf("rclone about: %w", err)
}
return &storage.StorageUsageInfo{
Total: usage.Total,
Used: usage.Used,
Free: usage.Free,
Objects: usage.Objects,
}, nil
}
// pathDir 返回 objectKey 的目录部分(正斜杠分隔)。
func pathDir(objectKey string) string {
idx := strings.LastIndex(objectKey, "/")

View File

@@ -110,6 +110,79 @@ func TestBuildS3Remote(t *testing.T) {
}
}
func TestRcloneFactoryCRUD(t *testing.T) {
dir := t.TempDir()
factory := NewRcloneFactory()
// 使用 rclone 的 local 后端
provider, err := factory.New(context.Background(), map[string]any{
"backend": "local",
"root": dir,
})
if err != nil {
t.Fatalf("RcloneFactory.New returned error: %v", err)
}
if err := provider.Upload(context.Background(), "test.txt", strings.NewReader("rclone"), 6, nil); err != nil {
t.Fatalf("Upload via rclone factory returned error: %v", err)
}
reader, err := provider.Download(context.Background(), "test.txt")
if err != nil {
t.Fatalf("Download returned error: %v", err)
}
defer reader.Close()
content, _ := io.ReadAll(reader)
if string(content) != "rclone" {
t.Fatalf("expected 'rclone', got %q", string(content))
}
if err := provider.Delete(context.Background(), "test.txt"); err != nil {
t.Fatalf("Delete returned error: %v", err)
}
}
func TestRcloneFactoryRequiresBackend(t *testing.T) {
_, err := NewRcloneFactory().New(context.Background(), map[string]any{"root": "/tmp"})
if err == nil || !strings.Contains(err.Error(), "backend") {
t.Fatalf("expected backend required error, got %v", err)
}
}
func TestListBackends(t *testing.T) {
backends := ListBackends()
if len(backends) < 30 {
t.Fatalf("expected at least 30 backends, got %d", len(backends))
}
// 确认 sftp 在列表中
found := false
for _, b := range backends {
if b.Name == "sftp" {
found = true
if len(b.Options) == 0 {
t.Fatal("sftp backend should have options")
}
break
}
}
if !found {
t.Fatal("sftp backend not found in ListBackends()")
}
}
func TestProviderAbout(t *testing.T) {
factory := NewLocalDiskFactory()
provider, err := factory.New(context.Background(), map[string]any{"basePath": t.TempDir()})
if err != nil {
t.Fatalf("Factory.New returned error: %v", err)
}
// local 后端支持 About
rcloneProvider := provider.(*Provider)
usage, err := rcloneProvider.About(context.Background())
if err != nil {
t.Fatalf("About returned error: %v", err)
}
if usage.Total == nil || *usage.Total <= 0 {
t.Fatalf("expected non-zero total disk space, got %v", usage.Total)
}
}
func TestPathDir(t *testing.T) {
tests := []struct {
input string

View File

@@ -20,6 +20,7 @@ const (
ProviderTypeTencentCOS ProviderType = "tencent_cos"
ProviderTypeQiniuKodo ProviderType = "qiniu_kodo"
ProviderTypeFTP ProviderType = "ftp"
ProviderTypeRclone ProviderType = "rclone"
)
const (
@@ -52,6 +53,20 @@ type ProviderFactory interface {
Type() ProviderType
}
// StorageAbout 是可选能力接口,支持查询远端存储空间。
// 并非所有后端都支持(如 S3/FTP 不支持),通过 type assertion 检测。
type StorageAbout interface {
About(ctx context.Context) (*StorageUsageInfo, error)
}
// StorageUsageInfo 描述远端存储的空间使用情况。
type StorageUsageInfo struct {
Total *int64 `json:"total,omitempty"` // 总空间(字节)
Used *int64 `json:"used,omitempty"` // 已用空间
Free *int64 `json:"free,omitempty"` // 可用空间
Objects *int64 `json:"objects,omitempty"` // 对象数量
}
func DecodeConfig[T any](raw map[string]any) (T, error) {
var cfg T
encoded, err := json.Marshal(raw)