mirror of
https://github.com/Awuqing/BackupX.git
synced 2026-05-17 08:57:36 +08:00
66b4714b71
将全部 70+ rclone 后端（SFTP、Azure Blob、Dropbox、OneDrive、B2、SMB 等） 自动注册为独立 Factory，与 S3、FTP 等内置类型完全平级。 - 新增 GenericBackendFactory + RegisterAllBackends 自动注册全部后端 - 移除 oneof 硬编码白名单，type 字段接受任意已注册存储类型 - 前端类型选择器合并内置类型和全部 rclone 后端为统一可搜索下拉框 - 选择 SFTP 直接存储 type="sftp"，非内置类型自动从 API 获取配置字段
509 lines
17 KiB
Go
509 lines
17 KiB
Go
package rclone
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"backupx/server/internal/storage"
|
||
|
||
"github.com/rclone/rclone/fs"
|
||
)
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 辅助函数
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// quoteParam 对 rclone 连接字符串中含特殊字符的值加单引号保护。
|
||
func quoteParam(s string) string {
|
||
if s == "" {
|
||
return s
|
||
}
|
||
if !strings.ContainsAny(s, ",:='") {
|
||
return s
|
||
}
|
||
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
|
||
}
|
||
|
||
// newFs 创建 rclone fs.Fs 实例并包装为 Provider。
|
||
func newFs(ctx context.Context, providerType storage.ProviderType, remote string) (*Provider, error) {
|
||
rfs, err := fs.NewFs(ctx, remote)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("create rclone fs for %s: %w", providerType, err)
|
||
}
|
||
return newProvider(providerType, rfs), nil
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// LocalDisk
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type LocalDiskFactory struct{}
|
||
|
||
func NewLocalDiskFactory() LocalDiskFactory { return LocalDiskFactory{} }
|
||
|
||
func (LocalDiskFactory) Type() storage.ProviderType { return storage.ProviderTypeLocalDisk }
|
||
func (LocalDiskFactory) SensitiveFields() []string { return nil }
|
||
|
||
func (LocalDiskFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[storage.LocalDiskConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
basePath := strings.TrimSpace(cfg.BasePath)
|
||
if basePath == "" {
|
||
return nil, fmt.Errorf("local disk basePath is required")
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeLocalDisk, basePath)
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// S3
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type S3Factory struct{}
|
||
|
||
func NewS3Factory() S3Factory { return S3Factory{} }
|
||
|
||
func (S3Factory) Type() storage.ProviderType { return storage.ProviderTypeS3 }
|
||
func (S3Factory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
|
||
|
||
func (S3Factory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[storage.S3Config](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if strings.TrimSpace(cfg.Bucket) == "" {
|
||
return nil, fmt.Errorf("s3 bucket is required")
|
||
}
|
||
if strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
|
||
return nil, fmt.Errorf("s3 credentials are required")
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeS3, buildS3Remote("Other", cfg.AccessKeyID, cfg.SecretAccessKey, cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.ForcePathStyle))
|
||
}
|
||
|
||
// buildS3Remote 构建 S3 兼容存储的 rclone 连接字符串。
|
||
func buildS3Remote(provider, keyID, secret, endpoint, region, bucket string, pathStyle bool) string {
|
||
var b strings.Builder
|
||
b.WriteString(":s3,provider=")
|
||
b.WriteString(quoteParam(provider))
|
||
b.WriteString(",access_key_id=")
|
||
b.WriteString(quoteParam(keyID))
|
||
b.WriteString(",secret_access_key=")
|
||
b.WriteString(quoteParam(secret))
|
||
if strings.TrimSpace(endpoint) != "" {
|
||
b.WriteString(",endpoint=")
|
||
b.WriteString(quoteParam(strings.TrimRight(endpoint, "/")))
|
||
}
|
||
if strings.TrimSpace(region) != "" {
|
||
b.WriteString(",region=")
|
||
b.WriteString(quoteParam(region))
|
||
}
|
||
if pathStyle {
|
||
b.WriteString(",force_path_style=true")
|
||
}
|
||
b.WriteString(",env_auth=false,no_check_bucket=true:")
|
||
b.WriteString(bucket)
|
||
return b.String()
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// WebDAV
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type WebDAVFactory struct{}
|
||
|
||
func NewWebDAVFactory() WebDAVFactory { return WebDAVFactory{} }
|
||
|
||
func (WebDAVFactory) Type() storage.ProviderType { return storage.ProviderTypeWebDAV }
|
||
func (WebDAVFactory) SensitiveFields() []string { return []string{"username", "password"} }
|
||
|
||
func (WebDAVFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[storage.WebDAVConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if strings.TrimSpace(cfg.Endpoint) == "" {
|
||
return nil, fmt.Errorf("webdav endpoint is required")
|
||
}
|
||
remote := fmt.Sprintf(":webdav,url=%s,user=%s,pass=%s:%s",
|
||
quoteParam(strings.TrimRight(cfg.Endpoint, "/")),
|
||
quoteParam(cfg.Username),
|
||
quoteParam(cfg.Password),
|
||
strings.TrimSpace(cfg.BasePath))
|
||
return newFs(ctx, storage.ProviderTypeWebDAV, remote)
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Google Drive
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type GoogleDriveFactory struct{}
|
||
|
||
func NewGoogleDriveFactory() GoogleDriveFactory { return GoogleDriveFactory{} }
|
||
|
||
func (GoogleDriveFactory) Type() storage.ProviderType { return storage.ProviderTypeGoogleDrive }
|
||
func (GoogleDriveFactory) SensitiveFields() []string {
|
||
return []string{"clientId", "clientSecret", "refreshToken"}
|
||
}
|
||
|
||
func (GoogleDriveFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[storage.GoogleDriveConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
cfg = cfg.Normalize()
|
||
if strings.TrimSpace(cfg.ClientID) == "" || strings.TrimSpace(cfg.ClientSecret) == "" {
|
||
return nil, fmt.Errorf("google drive client credentials are required")
|
||
}
|
||
if strings.TrimSpace(cfg.RefreshToken) == "" {
|
||
return nil, fmt.Errorf("google drive refresh token is required")
|
||
}
|
||
// 构造 rclone 所需的 OAuth2 token JSON
|
||
tokenJSON := fmt.Sprintf(`{"access_token":"","token_type":"Bearer","refresh_token":"%s","expiry":"0001-01-01T00:00:00Z"}`,
|
||
strings.ReplaceAll(cfg.RefreshToken, `"`, `\"`))
|
||
|
||
var b strings.Builder
|
||
b.WriteString(":drive,client_id=")
|
||
b.WriteString(quoteParam(cfg.ClientID))
|
||
b.WriteString(",client_secret=")
|
||
b.WriteString(quoteParam(cfg.ClientSecret))
|
||
b.WriteString(",token=")
|
||
b.WriteString(quoteParam(tokenJSON))
|
||
if strings.TrimSpace(cfg.FolderID) != "" {
|
||
b.WriteString(",root_folder_id=")
|
||
b.WriteString(quoteParam(cfg.FolderID))
|
||
}
|
||
b.WriteString(":")
|
||
return newFs(ctx, storage.ProviderTypeGoogleDrive, b.String())
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// FTP
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type FTPFactory struct{}
|
||
|
||
func NewFTPFactory() FTPFactory { return FTPFactory{} }
|
||
|
||
func (FTPFactory) Type() storage.ProviderType { return storage.ProviderTypeFTP }
|
||
func (FTPFactory) SensitiveFields() []string { return []string{"username", "password"} }
|
||
|
||
func (FTPFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[storage.FTPConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if strings.TrimSpace(cfg.Host) == "" {
|
||
return nil, fmt.Errorf("FTP host is required")
|
||
}
|
||
port := cfg.Port
|
||
if port == 0 {
|
||
port = 21
|
||
}
|
||
username := strings.TrimSpace(cfg.Username)
|
||
if username == "" {
|
||
username = "anonymous"
|
||
}
|
||
var b strings.Builder
|
||
b.WriteString(fmt.Sprintf(":ftp,host=%s,port=%d,user=%s,pass=%s",
|
||
quoteParam(cfg.Host), port, quoteParam(username), quoteParam(cfg.Password)))
|
||
if cfg.UseTLS {
|
||
b.WriteString(",tls=true,explicit_tls=true")
|
||
}
|
||
b.WriteString(":")
|
||
basePath := strings.TrimSpace(cfg.BasePath)
|
||
if basePath != "" {
|
||
b.WriteString(basePath)
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeFTP, b.String())
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 阿里云 OSS（委托 S3 引擎）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type AliyunOSSFactory struct{}
|
||
|
||
func NewAliyunOSSFactory() AliyunOSSFactory { return AliyunOSSFactory{} }
|
||
|
||
func (AliyunOSSFactory) Type() storage.ProviderType { return storage.ProviderTypeAliyunOSS }
|
||
func (AliyunOSSFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
|
||
|
||
// AliyunConfig 是阿里云 OSS 的用户配置。
|
||
type AliyunConfig struct {
|
||
Region string `json:"region"`
|
||
Bucket string `json:"bucket"`
|
||
AccessKeyID string `json:"accessKeyId"`
|
||
SecretAccessKey string `json:"secretAccessKey"`
|
||
Endpoint string `json:"endpoint"`
|
||
InternalNetwork bool `json:"internalNetwork"`
|
||
}
|
||
|
||
func (AliyunOSSFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[AliyunConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
endpoint := strings.TrimSpace(cfg.Endpoint)
|
||
if endpoint == "" {
|
||
region := strings.TrimSpace(cfg.Region)
|
||
if region == "" {
|
||
return nil, fmt.Errorf("aliyun oss region is required")
|
||
}
|
||
if cfg.InternalNetwork {
|
||
endpoint = fmt.Sprintf("https://oss-%s-internal.aliyuncs.com", region)
|
||
} else {
|
||
endpoint = fmt.Sprintf("https://oss-%s.aliyuncs.com", region)
|
||
}
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeAliyunOSS, buildS3Remote("Alibaba", cfg.AccessKeyID, cfg.SecretAccessKey, endpoint, cfg.Region, cfg.Bucket, false))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 腾讯云 COS（委托 S3 引擎）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type TencentCOSFactory struct{}
|
||
|
||
func NewTencentCOSFactory() TencentCOSFactory { return TencentCOSFactory{} }
|
||
|
||
func (TencentCOSFactory) Type() storage.ProviderType { return storage.ProviderTypeTencentCOS }
|
||
func (TencentCOSFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
|
||
|
||
// TencentConfig 是腾讯云 COS 的用户配置。
|
||
type TencentConfig struct {
|
||
Region string `json:"region"`
|
||
Bucket string `json:"bucket"`
|
||
SecretID string `json:"accessKeyId"`
|
||
SecretKey string `json:"secretAccessKey"`
|
||
Endpoint string `json:"endpoint"`
|
||
}
|
||
|
||
func (TencentCOSFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[TencentConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
endpoint := strings.TrimSpace(cfg.Endpoint)
|
||
if endpoint == "" {
|
||
region := strings.TrimSpace(cfg.Region)
|
||
if region == "" {
|
||
return nil, fmt.Errorf("tencent cos region is required")
|
||
}
|
||
endpoint = fmt.Sprintf("https://cos.%s.myqcloud.com", region)
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeTencentCOS, buildS3Remote("TencentCOS", cfg.SecretID, cfg.SecretKey, endpoint, cfg.Region, cfg.Bucket, false))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 七牛云 Kodo（委托 S3 引擎）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type QiniuKodoFactory struct{}
|
||
|
||
func NewQiniuKodoFactory() QiniuKodoFactory { return QiniuKodoFactory{} }
|
||
|
||
func (QiniuKodoFactory) Type() storage.ProviderType { return storage.ProviderTypeQiniuKodo }
|
||
func (QiniuKodoFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
|
||
|
||
// QiniuConfig 是七牛云 Kodo 的用户配置。
|
||
type QiniuConfig struct {
|
||
Region string `json:"region"`
|
||
Bucket string `json:"bucket"`
|
||
AccessKey string `json:"accessKeyId"`
|
||
SecretKey string `json:"secretAccessKey"`
|
||
Endpoint string `json:"endpoint"`
|
||
}
|
||
|
||
// regionEndpoints 映射七牛区域代码到 S3 兼容 endpoint。
|
||
var regionEndpoints = map[string]string{
|
||
"z0": "https://s3-cn-east-1.qiniucs.com",
|
||
"cn-east-2": "https://s3-cn-east-2.qiniucs.com",
|
||
"z1": "https://s3-cn-north-1.qiniucs.com",
|
||
"z2": "https://s3-cn-south-1.qiniucs.com",
|
||
"na0": "https://s3-us-north-1.qiniucs.com",
|
||
"as0": "https://s3-ap-southeast-1.qiniucs.com",
|
||
}
|
||
|
||
func (QiniuKodoFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
cfg, err := storage.DecodeConfig[QiniuConfig](rawConfig)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
endpoint := strings.TrimSpace(cfg.Endpoint)
|
||
if endpoint == "" {
|
||
region := strings.TrimSpace(cfg.Region)
|
||
if region == "" {
|
||
return nil, fmt.Errorf("qiniu kodo region is required")
|
||
}
|
||
var ok bool
|
||
endpoint, ok = regionEndpoints[region]
|
||
if !ok {
|
||
return nil, fmt.Errorf("unsupported qiniu region: %s (supported: z0, cn-east-2, z1, z2, na0, as0)", region)
|
||
}
|
||
}
|
||
return newFs(ctx, storage.ProviderTypeQiniuKodo, buildS3Remote("Qiniu", cfg.AccessKey, cfg.SecretKey, endpoint, cfg.Region, cfg.Bucket, true))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 通用 Rclone 后端（支持全部 70+ 后端）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type RcloneFactory struct{}
|
||
|
||
func NewRcloneFactory() RcloneFactory { return RcloneFactory{} }
|
||
|
||
func (RcloneFactory) Type() storage.ProviderType { return storage.ProviderTypeRclone }
|
||
func (RcloneFactory) SensitiveFields() []string { return []string{"pass", "password", "secret_access_key", "client_secret", "token"} }
|
||
|
||
func (RcloneFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
backend, _ := rawConfig["backend"].(string)
|
||
backend = strings.TrimSpace(backend)
|
||
if backend == "" {
|
||
return nil, fmt.Errorf("rclone backend type is required")
|
||
}
|
||
root, _ := rawConfig["root"].(string)
|
||
root = strings.TrimSpace(root)
|
||
|
||
// 构建连接字符串：:backend,key1=val1,key2=val2:root
|
||
var b strings.Builder
|
||
b.WriteString(":")
|
||
b.WriteString(backend)
|
||
for key, val := range rawConfig {
|
||
if key == "backend" || key == "root" {
|
||
continue
|
||
}
|
||
strVal := fmt.Sprintf("%v", val)
|
||
if strings.TrimSpace(strVal) == "" {
|
||
continue
|
||
}
|
||
b.WriteString(",")
|
||
b.WriteString(key)
|
||
b.WriteString("=")
|
||
b.WriteString(quoteParam(strVal))
|
||
}
|
||
b.WriteString(":")
|
||
b.WriteString(root)
|
||
|
||
return newFs(ctx, storage.ProviderTypeRclone, b.String())
|
||
}
|
||
|
||
// ListBackends 返回所有可用的 rclone 后端及其配置选项。
|
||
func ListBackends() []BackendInfo {
|
||
var backends []BackendInfo
|
||
for _, ri := range fs.Registry {
|
||
if ri.Name == "union" || ri.Name == "crypt" || ri.Name == "chunker" || ri.Name == "compress" || ri.Name == "hasher" || ri.Name == "combine" {
|
||
continue // 跳过组合/加密类后端
|
||
}
|
||
info := BackendInfo{
|
||
Name: ri.Name,
|
||
Description: ri.Description,
|
||
}
|
||
for _, opt := range ri.Options {
|
||
if opt.Hide != 0 {
|
||
continue
|
||
}
|
||
// 跳过 rclone 为每个后端自动添加的通用选项
|
||
if opt.Name == "description" {
|
||
continue
|
||
}
|
||
info.Options = append(info.Options, BackendOption{
|
||
Key: opt.Name,
|
||
Label: opt.Help,
|
||
Required: opt.Required,
|
||
IsPassword: opt.IsPassword,
|
||
})
|
||
}
|
||
backends = append(backends, info)
|
||
}
|
||
return backends
|
||
}
|
||
|
||
// BackendInfo 描述一个 rclone 后端。
|
||
type BackendInfo struct {
|
||
Name string `json:"name"`
|
||
Description string `json:"description"`
|
||
Options []BackendOption `json:"options"`
|
||
}
|
||
|
||
// BackendOption 描述一个后端配置选项。
|
||
type BackendOption struct {
|
||
Key string `json:"key"`
|
||
Label string `json:"label"`
|
||
Required bool `json:"required"`
|
||
IsPassword bool `json:"isPassword"`
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 通用 BackendFactory — 为任意 rclone 后端自动生成独立 Factory
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// GenericBackendFactory 为单个 rclone 后端创建独立的 ProviderFactory。
|
||
// 用户存储目标的 type 直接是后端名（如 "sftp"），与 "s3"、"ftp" 完全平级。
|
||
type GenericBackendFactory struct {
|
||
backendType string
|
||
sensitive []string
|
||
}
|
||
|
||
// NewBackendFactory 为指定 rclone 后端创建一个 Factory。
|
||
func NewBackendFactory(backendType string) GenericBackendFactory {
|
||
var sensitive []string
|
||
for _, ri := range fs.Registry {
|
||
if ri.Name == backendType {
|
||
for _, opt := range ri.Options {
|
||
if opt.IsPassword {
|
||
sensitive = append(sensitive, opt.Name)
|
||
}
|
||
}
|
||
break
|
||
}
|
||
}
|
||
return GenericBackendFactory{backendType: backendType, sensitive: sensitive}
|
||
}
|
||
|
||
func (f GenericBackendFactory) Type() storage.ProviderType { return storage.ProviderType(f.backendType) }
|
||
func (f GenericBackendFactory) SensitiveFields() []string { return f.sensitive }
|
||
|
||
func (f GenericBackendFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
|
||
root, _ := rawConfig["root"].(string)
|
||
root = strings.TrimSpace(root)
|
||
|
||
var b strings.Builder
|
||
b.WriteString(":")
|
||
b.WriteString(f.backendType)
|
||
for key, val := range rawConfig {
|
||
if key == "root" {
|
||
continue
|
||
}
|
||
strVal := fmt.Sprintf("%v", val)
|
||
if strings.TrimSpace(strVal) == "" {
|
||
continue
|
||
}
|
||
b.WriteString(",")
|
||
b.WriteString(key)
|
||
b.WriteString("=")
|
||
b.WriteString(quoteParam(strVal))
|
||
}
|
||
b.WriteString(":")
|
||
b.WriteString(root)
|
||
|
||
return newFs(ctx, storage.ProviderType(f.backendType), b.String())
|
||
}
|
||
|
||
// RegisterAllBackends 将所有 rclone 后端注册为独立 Factory 到 Registry。
|
||
// 已存在的内置类型（s3, ftp 等）不会被覆盖。
|
||
func RegisterAllBackends(registry *storage.Registry) {
|
||
builtinTypes := map[string]bool{
|
||
"local_disk": true, "s3": true, "webdav": true, "google_drive": true,
|
||
"ftp": true, "aliyun_oss": true, "tencent_cos": true, "qiniu_kodo": true,
|
||
"rclone": true, "local": true,
|
||
}
|
||
for _, info := range ListBackends() {
|
||
if builtinTypes[info.Name] {
|
||
continue
|
||
}
|
||
registry.Register(NewBackendFactory(info.Name))
|
||
}
|
||
}
|