Compare commits

..

5 Commits

19 changed files with 542 additions and 79 deletions

View File

@@ -35,6 +35,7 @@
- S3
- WebDAV
- Local filesystem
- Rclone (via command line)
- Telegram (re-upload to specified chats)
## 📦 Quick Start

View File

@@ -33,6 +33,7 @@
- S3
- WebDAV
- 本地磁盘
- Rclone
- Telegram (重传回指定聊天)
## 快速开始

View File

@@ -16,6 +16,7 @@ var storageFactories = map[storenum.StorageType]func(cfg *BaseConfig) (StorageCo
storenum.Minio: createStorageConfig(&MinioStorageConfig{}),
storenum.S3: createStorageConfig(&S3StorageConfig{}),
storenum.Telegram: createStorageConfig(&TelegramStorageConfig{}),
storenum.Rclone: createStorageConfig(&RcloneStorageConfig{}),
}
func createStorageConfig(configType StorageConfig) func(cfg *BaseConfig) (StorageConfig, error) {

33
config/storage/rclone.go Normal file
View File

@@ -0,0 +1,33 @@
package storage
import (
"fmt"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
)
type RcloneStorageConfig struct {
BaseConfig
// The name of the remote as defined in rclone config
Remote string `toml:"remote" mapstructure:"remote" json:"remote"`
BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"`
// The path to the rclone config file, if not using the default
ConfigPath string `toml:"config_path" mapstructure:"config_path" json:"config_path"`
// Additional flags to pass to rclone commands
Flags []string `toml:"flags" mapstructure:"flags" json:"flags"`
}
func (r *RcloneStorageConfig) Validate() error {
if r.Remote == "" {
return fmt.Errorf("remote is required for rclone storage")
}
return nil
}
func (r *RcloneStorageConfig) GetType() storenum.StorageType {
return storenum.Rclone
}
func (r *RcloneStorageConfig) GetName() string {
return r.Name
}

View File

@@ -45,16 +45,17 @@ func (t *Task) Execute(ctx context.Context) error {
fetchedTotalBytes.Add(resp.ContentLength)
file.Size = resp.ContentLength
if name := resp.Header.Get("Content-Disposition"); name != "" {
// Set file name from Content-Disposition header
filename := parseFilename(name)
file.Name = filename
if filename != "" {
file.Name = filename
}
}
// Fallback: extract filename from URL if no filename was determined from Content-Disposition
// extract filename from URL if Content-Disposition is empty or invalid
if file.Name == "" {
file.Name = filenameFromURL(file.URL)
file.Name = parseFilenameFromURL(file.URL)
}
if file.Name == "" {
return fmt.Errorf("could not determine filename for %s", file.URL)
return fmt.Errorf("failed to determine filename for %s: Content-Disposition header is empty and URL does not contain a valid filename", file.URL)
}
return nil

View File

@@ -76,6 +76,9 @@ func (t *Task) StorageName() string {
// StoragePath implements TaskInfo.
func (t *Task) StoragePath() string {
if len(t.files) == 1 {
return t.StorPath + "/" + t.files[0].Name
}
return t.StorPath
}

View File

@@ -144,6 +144,41 @@ func tryDecodeGBK(s string) string {
return ""
}
// parseFilenameFromURL extracts filename from URL path
// This is used as a fallback when Content-Disposition is not available
func parseFilenameFromURL(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return ""
}
// Get the path part and extract the last segment
path := parsed.Path
if path == "" {
return ""
}
// URL decode the path first
decodedPath, err := url.PathUnescape(path)
if err != nil {
decodedPath = path
}
// Get the last segment of the path
lastSlash := strings.LastIndex(decodedPath, "/")
if lastSlash == -1 {
return decodedPath
}
filename := decodedPath[lastSlash+1:]
// Remove query string if somehow still present
if idx := strings.Index(filename, "?"); idx != -1 {
filename = filename[:idx]
}
return filename
}
// parseFilenameFallback manually parses filename= when mime.ParseMediaType fails
func parseFilenameFallback(cd string) string {
// Look for filename= (case-insensitive)
@@ -173,35 +208,6 @@ func parseFilenameFallback(cd string) string {
return decodeFilenameParam(value)
}
// filenameFromURL extracts filename from a URL path.
// It uses the last path segment and removes any query parameters.
// Returns empty string if the URL cannot be parsed or has no valid path.
func filenameFromURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return ""
}
// Get the path and extract the base name
path := u.Path
if path == "" || path == "/" {
return ""
}
// Find the last path segment
idx := strings.LastIndex(path, "/")
if idx >= 0 && idx < len(path)-1 {
filename := path[idx+1:]
// URL decode the filename
if decoded, err := url.QueryUnescape(filename); err == nil {
return decoded
}
return filename
}
return ""
}
var progressUpdatesLevels = []struct {
size int64 // 文件大小阈值
stepPercent int // 每多少 % 更新一次

View File

@@ -4,41 +4,41 @@ import (
"testing"
)
func TestFilenameFromURL(t *testing.T) {
func TestParseFilenameFromURL(t *testing.T) {
tests := []struct {
name string
url string
expected string
}{
{
name: "simple file",
url: "https://example.com/file.zip",
expected: "file.zip",
},
{
name: "file with path",
url: "https://example.com/path/to/document.pdf",
name: "simple filename",
url: "https://example.com/files/document.pdf",
expected: "document.pdf",
},
{
name: "url with query params",
url: "https://example.com/file.mp4?token=abc123",
expected: "file.mp4",
},
{
name: "url with fragment",
url: "https://example.com/file.txt#section",
expected: "file.txt",
},
{
name: "url encoded filename",
url: "https://example.com/%E6%B5%8B%E8%AF%95.zip",
name: "filename with encoded characters",
url: "https://example.com/files/%E6%B5%8B%E8%AF%95.zip",
expected: "测试.zip",
},
{
name: "url encoded Chinese filename",
url: "https://example.com/10%E6%9C%8817%E6%97%A5(6).mp4",
expected: "10月17日(6).mp4",
name: "filename with query string in URL",
url: "https://example.com/files/image.png?token=abc123",
expected: "image.png",
},
{
name: "nested path",
url: "https://example.com/a/b/c/file.txt",
expected: "file.txt",
},
{
name: "URL with port",
url: "https://example.com:8080/downloads/archive.tar.gz",
expected: "archive.tar.gz",
},
{
name: "empty path",
url: "https://example.com",
expected: "",
},
{
name: "root path only",
@@ -46,37 +46,27 @@ func TestFilenameFromURL(t *testing.T) {
expected: "",
},
{
name: "no path",
url: "https://example.com",
expected: "",
name: "filename with spaces encoded",
url: "https://example.com/my%20file%20name.pdf",
expected: "my file name.pdf",
},
{
name: "empty url",
url: "",
expected: "",
name: "complex encoded filename",
url: "https://example.com/downloads/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6.docx",
expected: "中文文件.docx",
},
{
name: "file with spaces encoded",
url: "https://example.com/my%20file.txt",
expected: "my file.txt",
},
{
name: "complex path with multiple slashes",
url: "https://cdn.example.com/a/b/c/d/e/video.mkv",
expected: "video.mkv",
},
{
name: "malformed url with invalid characters",
url: "://invalid url",
name: "invalid URL",
url: "://invalid-url",
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := filenameFromURL(tt.url)
result := parseFilenameFromURL(tt.url)
if result != tt.expected {
t.Errorf("filenameFromURL(%q) = %q, expected %q", tt.url, result, tt.expected)
t.Errorf("parseFilenameFromURL(%q) = %q, want %q", tt.url, result, tt.expected)
}
})
}

View File

@@ -80,4 +80,60 @@ chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat
force_file = false # Force sending as file, default is false
skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded.
spilt_size_mb = 2000 # Split size in MB, default is 2000 MB (2 GB). Files larger than this will be split into multiple parts (zip format). Ignored when skip_large is true.
```
## Rclone
`type=rclone`
Supports multiple cloud storage services through the [rclone](https://rclone.org/) command-line tool. You need to install rclone and configure remote storage first.
```toml
# Remote name configured in rclone, can be any remote defined in rclone.conf
remote = "mydrive"
# Base path in the remote storage, all files will be stored under this path
base_path = "/telegram"
# Path to rclone config file, optional, leave empty to use default path (~/.config/rclone/rclone.conf)
config_path = ""
# Additional flags to pass to rclone commands, optional
flags = ["--transfers", "4", "--checkers", "8"]
```
### Configuring rclone Remote
First, you need to configure an rclone remote. Run `rclone config` for interactive configuration, or directly edit the `rclone.conf` file.
rclone supports many cloud storage services, including but not limited to:
- Google Drive
- Dropbox
- OneDrive
- Amazon S3 and compatible services
- SFTP
- FTP
- For more services, please refer to the [rclone official documentation](https://rclone.org/overview/)
### Usage Examples
After configuring Google Drive, you can configure the storage like this:
```toml
[[storages]]
name = "GoogleDrive"
type = "rclone"
enable = true
remote = "gdrive"
base_path = "/SaveAnyBot"
```
If using a custom rclone config file:
```toml
[[storages]]
name = "MyRemote"
type = "rclone"
enable = true
remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```

View File

@@ -29,6 +29,7 @@ title: 介绍
- S3
- WebDAV
- 本地磁盘
- Rclone (通过命令行调用)
- Telegram (重传回指定聊天)
## [贡献者](https://github.com/krau/SaveAny-Bot/graphs/contributors)

View File

@@ -86,4 +86,60 @@ skip_large = false
# 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式)
# 当 skip_large 启用时, 该选项无效.
spilt_size_mb = 2000
```
## Rclone
`type=rclone`
通过 [rclone](https://rclone.org/) 命令行工具支持多种云存储服务. 需要先安装 rclone 并配置好远程存储.
```toml
# rclone 配置的远程名称, 可以是任何在 rclone.conf 中配置的远程
remote = "mydrive"
# 在远程存储中的基础路径, 所有文件将存储在此路径下
base_path = "/telegram"
# rclone 配置文件的路径, 可选, 留空使用默认路径 (~/.config/rclone/rclone.conf)
config_path = ""
# 传递给 rclone 命令的额外参数, 可选
flags = ["--transfers", "4", "--checkers", "8"]
```
### 配置 rclone 远程
首先需要配置 rclone 远程, 运行 `rclone config` 命令进行交互式配置, 或直接编辑 `rclone.conf` 文件.
rclone 支持多种云存储服务, 包括但不限于:
- Google Drive
- Dropbox
- OneDrive
- Amazon S3 及兼容服务
- SFTP
- FTP
- 更多服务请参考 [rclone 官方文档](https://rclone.org/overview/)
### 使用示例
配置 Google Drive 后, 可以这样配置存储:
```toml
[[storages]]
name = "GoogleDrive"
type = "rclone"
enable = true
remote = "gdrive"
base_path = "/SaveAnyBot"
```
如果使用自定义的 rclone 配置文件:
```toml
[[storages]]
name = "MyRemote"
type = "rclone"
enable = true
remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```

View File

@@ -1,3 +1,5 @@
module github.com/krau/SaveAny-Bot/docs
go 1.24.4
require github.com/alex-shpak/hugo-book v0.0.0-20250530233833-f2c703e15588 // indirect

View File

@@ -0,0 +1,2 @@
github.com/alex-shpak/hugo-book v0.0.0-20250530233833-f2c703e15588 h1:pwxkzpzw/iJSxMBgQLWjYMQubhIemLG3UrNjeWoCkSM=
github.com/alex-shpak/hugo-book v0.0.0-20250530233833-f2c703e15588/go.mod h1:L4NMyzbn15fpLIpmmtDg9ZFFyTZzw87/lk7M2bMQ7ds=

View File

@@ -4,6 +4,6 @@ package storage
// StorageType
/* ENUM(
local, webdav, alist, minio, telegram, s3
local, webdav, alist, minio, telegram, s3, rclone
) */
type StorageType string

View File

@@ -24,6 +24,8 @@ const (
Telegram StorageType = "telegram"
// S3 is a StorageType of type s3.
S3 StorageType = "s3"
// Rclone is a StorageType of type rclone.
Rclone StorageType = "rclone"
)
var ErrInvalidStorageType = fmt.Errorf("not a valid StorageType, try [%s]", strings.Join(_StorageTypeNames, ", "))
@@ -35,6 +37,7 @@ var _StorageTypeNames = []string{
string(Minio),
string(Telegram),
string(S3),
string(Rclone),
}
// StorageTypeNames returns a list of possible string values of StorageType.
@@ -53,6 +56,7 @@ func StorageTypeValues() []StorageType {
Minio,
Telegram,
S3,
Rclone,
}
}
@@ -75,6 +79,7 @@ var _StorageTypeValue = map[string]StorageType{
"minio": Minio,
"telegram": Telegram,
"s3": S3,
"rclone": Rclone,
}
// ParseStorageType attempts to convert a string to a StorageType.

View File

@@ -104,7 +104,7 @@ func (a *Alist) Name() string {
func (a *Alist) Save(ctx context.Context, reader io.Reader, storagePath string) error {
a.logger.Infof("Saving file to %s", storagePath)
storagePath = a.JoinStoragePath(storagePath)
ext := path.Ext(storagePath)
base := strings.TrimSuffix(storagePath, ext)
candidate := storagePath

14
storage/rclone/errs.go Normal file
View File

@@ -0,0 +1,14 @@
package rclone
import "errors"
var (
ErrRcloneNotFound = errors.New("rclone: rclone command not found in PATH")
ErrRemoteNotFound = errors.New("rclone: remote not found")
ErrFailedToSaveFile = errors.New("rclone: failed to save file")
ErrFailedToListFiles = errors.New("rclone: failed to list files")
ErrFailedToOpenFile = errors.New("rclone: failed to open file")
ErrFailedToCheckFile = errors.New("rclone: failed to check file exists")
ErrFailedToCreateDir = errors.New("rclone: failed to create directory")
ErrCommandFailed = errors.New("rclone: command execution failed")
)

289
storage/rclone/rclone.go Normal file
View File

@@ -0,0 +1,289 @@
package rclone
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os/exec"
"path"
"strings"
"time"
"github.com/charmbracelet/log"
config "github.com/krau/SaveAny-Bot/config/storage"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
"github.com/rs/xid"
)
type Rclone struct {
config config.RcloneStorageConfig
logger *log.Logger
}
func (r *Rclone) Init(ctx context.Context, cfg config.StorageConfig) error {
rcloneConfig, ok := cfg.(*config.RcloneStorageConfig)
if !ok {
return fmt.Errorf("failed to cast rclone config")
}
if err := rcloneConfig.Validate(); err != nil {
return err
}
r.config = *rcloneConfig
r.logger = log.FromContext(ctx).WithPrefix(fmt.Sprintf("rclone[%s]", r.config.Name))
// 检查 rclone 是否安装
if _, err := exec.LookPath("rclone"); err != nil {
return ErrRcloneNotFound
}
args := r.buildBaseArgs()
args = append(args, "listremotes")
cmd := exec.CommandContext(ctx, "rclone", args...)
output, err := cmd.Output()
if err != nil {
r.logger.Errorf("Failed to list remotes: %v", err)
return fmt.Errorf("failed to verify rclone: %w", err)
}
remoteName := strings.TrimSuffix(r.config.Remote, ":")
if !strings.HasSuffix(r.config.Remote, ":") {
remoteName = r.config.Remote
}
found := false
scanner := bufio.NewScanner(bytes.NewReader(output))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
line = strings.TrimSuffix(line, ":")
if line == remoteName {
found = true
break
}
}
if !found {
r.logger.Errorf("Remote %s not found in rclone config", r.config.Remote)
return ErrRemoteNotFound
}
r.logger.Infof("Initialized rclone storage with remote: %s", r.config.Remote)
return nil
}
func (r *Rclone) Type() storenum.StorageType {
return storenum.Rclone
}
func (r *Rclone) Name() string {
return r.config.Name
}
func (r *Rclone) buildBaseArgs() []string {
var args []string
if r.config.ConfigPath != "" {
args = append(args, "--config", r.config.ConfigPath)
}
args = append(args, r.config.Flags...)
return args
}
func (r *Rclone) getRemotePath(storagePath string) string {
remote := r.config.Remote
if !strings.HasSuffix(remote, ":") {
remote += ":"
}
basePath := strings.TrimPrefix(r.config.BasePath, "/")
fullPath := path.Join(basePath, storagePath)
return remote + fullPath
}
func (r *Rclone) Save(ctx context.Context, reader io.Reader, storagePath string) error {
r.logger.Infof("Saving file to %s", storagePath)
ext := path.Ext(storagePath)
base := strings.TrimSuffix(storagePath, ext)
candidate := storagePath
for i := 1; r.Exists(ctx, candidate); i++ {
candidate = fmt.Sprintf("%s_%d%s", base, i, ext)
if i > 100 {
r.logger.Errorf("Too many attempts to find a unique filename for %s", storagePath)
candidate = fmt.Sprintf("%s_%s%s", base, xid.New().String(), ext)
break
}
}
remotePath := r.getRemotePath(candidate)
r.logger.Debugf("Remote path: %s", remotePath)
// Use rclone rcat to read from stdin and upload
args := r.buildBaseArgs()
args = append(args, "rcat", remotePath)
cmd := exec.CommandContext(ctx, "rclone", args...)
cmd.Stdin = reader
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
r.logger.Errorf("Failed to save file: %v, stderr: %s", err, stderr.String())
return fmt.Errorf("%w: %s", ErrFailedToSaveFile, stderr.String())
}
r.logger.Infof("Successfully saved file to %s", candidate)
return nil
}
func (r *Rclone) Exists(ctx context.Context, storagePath string) bool {
remotePath := r.getRemotePath(storagePath)
args := r.buildBaseArgs()
args = append(args, "lsf", remotePath)
cmd := exec.CommandContext(ctx, "rclone", args...)
err := cmd.Run()
return err == nil
}
// lsjsonItem represents a single entry in the output of `rclone lsjson`
type lsjsonItem struct {
Path string `json:"Path"`
Name string `json:"Name"`
Size int64 `json:"Size"`
MimeType string `json:"MimeType"`
ModTime string `json:"ModTime"`
IsDir bool `json:"IsDir"`
}
// ListFiles implements storage.StorageListable
func (r *Rclone) ListFiles(ctx context.Context, dirPath string) ([]storagetypes.FileInfo, error) {
r.logger.Infof("Listing files in %s", dirPath)
remotePath := r.getRemotePath(dirPath)
args := r.buildBaseArgs()
args = append(args, "lsjson", remotePath)
cmd := exec.CommandContext(ctx, "rclone", args...)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
r.logger.Errorf("Failed to list files: %v, stderr: %s", err, stderr.String())
return nil, fmt.Errorf("%w: %s", ErrFailedToListFiles, stderr.String())
}
var items []lsjsonItem
if err := json.Unmarshal(stdout.Bytes(), &items); err != nil {
r.logger.Errorf("Failed to parse lsjson output: %v", err)
return nil, fmt.Errorf("failed to parse lsjson output: %w", err)
}
files := make([]storagetypes.FileInfo, 0, len(items))
for _, item := range items {
var modTime time.Time
if item.ModTime != "" {
parsedTime, err := time.Parse(time.RFC3339Nano, item.ModTime)
if err != nil {
r.logger.Warnf("Failed to parse mod time %q for %s: %v", item.ModTime, item.Name, err)
} else {
modTime = parsedTime
}
}
files = append(files, storagetypes.FileInfo{
Name: item.Name,
Path: path.Join(dirPath, item.Name),
Size: item.Size,
IsDir: item.IsDir,
ModTime: modTime,
})
}
r.logger.Debugf("Found %d files/directories in %s", len(files), dirPath)
return files, nil
}
// OpenFile implements storage.StorageReadable
func (r *Rclone) OpenFile(ctx context.Context, filePath string) (io.ReadCloser, int64, error) {
r.logger.Infof("Opening file %s", filePath)
remotePath := r.getRemotePath(filePath)
size, err := r.getFileSize(ctx, remotePath)
if err != nil {
r.logger.Errorf("Failed to get file size: %v", err)
return nil, 0, fmt.Errorf("%w: %v", ErrFailedToOpenFile, err)
}
args := r.buildBaseArgs()
args = append(args, "cat", remotePath)
cmd := exec.CommandContext(ctx, "rclone", args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, 0, fmt.Errorf("failed to create stdout pipe: %w", err)
}
if err := cmd.Start(); err != nil {
return nil, 0, fmt.Errorf("failed to start rclone cat: %w", err)
}
reader := &rcloneCatReader{
reader: stdout,
cmd: cmd,
logger: r.logger,
}
r.logger.Debugf("Opened file %s (size: %d bytes)", filePath, size)
return reader, size, nil
}
func (r *Rclone) getFileSize(ctx context.Context, remotePath string) (int64, error) {
args := r.buildBaseArgs()
args = append(args, "lsjson", remotePath)
cmd := exec.CommandContext(ctx, "rclone", args...)
var stdout bytes.Buffer
cmd.Stdout = &stdout
if err := cmd.Run(); err != nil {
return 0, err
}
var items []lsjsonItem
if err := json.Unmarshal(stdout.Bytes(), &items); err != nil {
return 0, err
}
if len(items) > 0 {
return items[0].Size, nil
}
return 0, nil
}
type rcloneCatReader struct {
reader io.ReadCloser
cmd *exec.Cmd
logger *log.Logger
}
func (r *rcloneCatReader) Read(p []byte) (n int, err error) {
return r.reader.Read(p)
}
func (r *rcloneCatReader) Close() error {
if err := r.reader.Close(); err != nil {
r.logger.Warnf("Failed to close reader: %v", err)
}
if err := r.cmd.Wait(); err != nil {
r.logger.Warnf("rclone cat process exited with error: %v", err)
}
return nil
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/krau/SaveAny-Bot/storage/alist"
"github.com/krau/SaveAny-Bot/storage/local"
"github.com/krau/SaveAny-Bot/storage/minio"
"github.com/krau/SaveAny-Bot/storage/rclone"
"github.com/krau/SaveAny-Bot/storage/s3"
"github.com/krau/SaveAny-Bot/storage/telegram"
"github.com/krau/SaveAny-Bot/storage/webdav"
@@ -53,6 +54,7 @@ var storageConstructors = map[storenum.StorageType]StorageConstructor{
storenum.Minio: func() Storage { return new(minio.Minio) },
storenum.S3: func() Storage { return new(s3.S3) },
storenum.Telegram: func() Storage { return new(telegram.Telegram) },
storenum.Rclone: func() Storage { return new(rclone.Rclone) },
}
// NewStorage creates a new storage instance based on the provided config and initializes it