fix: deprecate minio and introduce s3 storage backend

This commit is contained in:
krau
2025-12-04 22:59:23 +08:00
parent 685047e463
commit 91814a83c7
17 changed files with 269 additions and 22 deletions

View File

@@ -29,7 +29,7 @@
- 使用 js 编写解析器插件以转存任意网站的文件
- 存储端支持:
- Alist
- S3 (MinioSDK)
- S3
- WebDAV
- 本地磁盘
- Telegram (重传回指定聊天)

View File

@@ -22,7 +22,7 @@ url = "socks5://127.0.0.1:7890"
[[storages]]
# 标识名, 需要唯一
name = "本机1"
# 存储类型, 目前可用: local, alist, webdav, minio, telegram
# 存储类型, 目前可用: local, alist, webdav, s3, telegram
type = "local"
# 启用存储
enable = true

View File

@@ -14,6 +14,7 @@ var storageFactories = map[storenum.StorageType]func(cfg *BaseConfig) (StorageCo
storenum.Alist: createStorageConfig(&AlistStorageConfig{}),
storenum.Webdav: createStorageConfig(&WebdavStorageConfig{}),
storenum.Minio: createStorageConfig(&MinioStorageConfig{}),
storenum.S3: createStorageConfig(&S3StorageConfig{}),
storenum.Telegram: createStorageConfig(&TelegramStorageConfig{}),
}

42
config/storage/s3.go Normal file
View File

@@ -0,0 +1,42 @@
package storage
import (
"fmt"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
)
type S3StorageConfig struct {
BaseConfig
Endpoint string `toml:"endpoint" mapstructure:"endpoint" json:"endpoint"`
AccessKeyID string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"`
SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"`
BucketName string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"`
UseSSL bool `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"`
BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"`
Region string `toml:"region" mapstructure:"region" json:"region"`
}
func (m *S3StorageConfig) Validate() error {
if m.Endpoint == "" {
return fmt.Errorf("endpoint is required for s3 storage")
}
if m.AccessKeyID == "" || m.SecretAccessKey == "" {
return fmt.Errorf("access_key_id and secret_access_key are required for s3 storage")
}
if m.BucketName == "" {
return fmt.Errorf("bucket_name is required for s3 storage")
}
if m.BasePath == "" {
return fmt.Errorf("base_path is required for s3 storage")
}
return nil
}
func (m *S3StorageConfig) GetType() storenum.StorageType {
return storenum.S3
}
func (m *S3StorageConfig) GetName() string {
return m.Name
}

View File

@@ -21,7 +21,7 @@ Save Any Bot is a tool that allows you to save files from Telegram to various st
- Automatic organization based on storage rules
- Supports multiple storage backends:
- Alist
- Minio (S3 compatible)
- S3
- WebDAV
- Telegram (re-upload to specified chat)
- Local disk

View File

@@ -79,7 +79,7 @@ Each storage endpoint requires at least the following fields:
- `local`: Local disk
- `alist`: Alist
- `webdav`: WebDAV
- `minio`: MinIO (compatible with S3 API)
- `s3`: aws S3 and other S3 compatible services
- `telegram`: Upload to Telegram
Example, this is a configuration that includes local storage and webdav storage:

View File

@@ -41,17 +41,18 @@ password = "your_password" # Password for WebDAV
base_path = "/path/to/webdav" # Base path in WebDAV, all files will be stored under this path
```
## MinIO (S3)
## S3
`type=minio`
`type=s3`
```toml
endpoint = "minio.example.com" # Endpoint for MinIO or S3
access_key_id = "your_access_key_id" # Access key ID for MinIO or S3
secret_access_key = "your_secret_access_key" # Secret access key for MinIO or S3
bucket_name = "your_bucket_name" # Bucket name for MinIO or S3
endpoint = "s3.example.com" # Endpoint for S3
region = "us-east-1" # Region for S3
access_key_id = "your_access_key_id" # Access key ID for S3
secret_access_key = "your_secret_access_key" # Secret access key for S3
bucket_name = "your_bucket_name" # Bucket name for S3
use_ssl = true # Whether to use SSL, default is true
base_path = "/path/to/minio" # Base path in MinIO, all files will be stored under this path
base_path = "/path/to/s3" # Base path in S3, all files will be stored under this path
```
## Telegram

View File

@@ -23,7 +23,7 @@ title: 介绍
- 使用 js 编写解析器插件以转存任意网站的文件
- 存储端支持:
- Alist
- S3 (MinioSDK)
- S3
- WebDAV
- 本地磁盘
- Telegram (重传回指定聊天)

View File

@@ -93,7 +93,7 @@ session = "data/usersession.db"
- `local`: 本地磁盘
- `alist`: Alist
- `webdav`: WebDAV
- `minio`: MinIO (兼容 S3 API)
- `s3`: aws S3 及其他兼容 S3 的服务
- `telegram`: 上传到 Telegram
示例, 这是一个包含本地存储和 webdav 存储的配置:

View File

@@ -41,17 +41,18 @@ password = "your_password" # WebDAV 的密码
base_path = "/path/to/webdav" # WebDAV 中的基础路径, 所有文件将存储在此路径下
```
## MinIO (S3)
## S3
`type=minio`
`type=s3`
```toml
endpoint = "minio.example.com" # MinIO 或 S3 的端点
access_key_id = "your_access_key_id" # MinIO 或 S3 的访问密钥 ID
secret_access_key = "your_secret_access_key" # MinIO 或 S3 的秘密访问密钥
bucket_name = "your_bucket_name" # MinIO 或 S3 的存储桶名称
endpoint = "s3.example.com" # S3 的端点
region = "us-east-1" # S3 的区域
access_key_id = "your_access_key_id" # S3 的访问密钥 ID
secret_access_key = "your_secret_access_key" # S3 的秘密访问密钥
bucket_name = "your_bucket_name" # S3 的存储桶名称
use_ssl = true # 是否使用 SSL, 默认为 true
base_path = "/path/to/minio" # MinIO 中的基础路径, 所有文件将存储在此路径下
base_path = "/path/to/s3" # S3 中的基础路径, 所有文件将存储在此路径下
```
## Telegram

19
go.mod
View File

@@ -3,6 +3,10 @@ module github.com/krau/SaveAny-Bot
go 1.24.0
require (
github.com/aws/aws-sdk-go-v2 v1.40.1
github.com/aws/aws-sdk-go-v2/config v1.32.3
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
github.com/blang/semver v3.5.1+incompatible
github.com/celestix/gotgproto v1.0.0-beta22
github.com/cenkalti/backoff/v4 v4.3.0
@@ -26,6 +30,21 @@ require (
require (
github.com/AnimeKaizoku/cacher v1.0.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/colorprofile v0.3.2 // indirect

38
go.sum
View File

@@ -4,6 +4,44 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.3 h1:cpz7H2uMNTDa0h/5CYL5dLUEzPSLo2g0NkbxTRJtSSU=
github.com/aws/aws-sdk-go-v2/config v1.32.3/go.mod h1:srtPKaJJe3McW6T/+GMBZyIPc+SeqJsNPJsd4mOYZ6s=
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 h1:d/6xOGIllc/XW1lzG9a4AUBMmpLA9PXcQnVPTuHHcik=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3/go.mod h1:fQ7E7Qj9GiW8y0ClD7cUJk3Bz5Iw8wZkWDHsTe8vDKs=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=

View File

@@ -4,6 +4,6 @@ package storage
// StorageType
/* ENUM(
local, webdav, alist, minio, telegram
local, webdav, alist, minio, telegram, s3
) */
type StorageType string

View File

@@ -22,6 +22,8 @@ const (
Minio StorageType = "minio"
// Telegram is a StorageType of type telegram.
Telegram StorageType = "telegram"
// S3 is a StorageType of type s3.
S3 StorageType = "s3"
)
var ErrInvalidStorageType = fmt.Errorf("not a valid StorageType, try [%s]", strings.Join(_StorageTypeNames, ", "))
@@ -32,6 +34,7 @@ var _StorageTypeNames = []string{
string(Alist),
string(Minio),
string(Telegram),
string(S3),
}
// StorageTypeNames returns a list of possible string values of StorageType.
@@ -49,6 +52,7 @@ func StorageTypeValues() []StorageType {
Alist,
Minio,
Telegram,
S3,
}
}
@@ -70,6 +74,7 @@ var _StorageTypeValue = map[string]StorageType{
"alist": Alist,
"minio": Minio,
"telegram": Telegram,
"s3": S3,
}
// ParseStorageType attempts to convert a string to a StorageType.

View File

@@ -6,6 +6,7 @@ import (
"io"
"path"
"strings"
"sync"
"github.com/charmbracelet/log"
config "github.com/krau/SaveAny-Bot/config/storage"
@@ -16,6 +17,10 @@ import (
"github.com/rs/xid"
)
var (
deprecatedOnce sync.Once
)
type Minio struct {
config config.MinioStorageConfig
client *minio.Client
@@ -23,6 +28,9 @@ type Minio struct {
}
func (m *Minio) Init(ctx context.Context, cfg config.StorageConfig) error {
deprecatedOnce.Do(func() {
log.FromContext(ctx).Warn("Minio storage is deprecated, please use S3 storage type instead.")
})
minioConfig, ok := cfg.(*config.MinioStorageConfig)
if !ok {
return fmt.Errorf("failed to cast minio config")
@@ -73,7 +81,7 @@ func (m *Minio) Save(ctx context.Context, r io.Reader, storagePath string) error
candidate := storagePath
for i := 1; m.Exists(ctx, candidate); i++ {
candidate = fmt.Sprintf("%s_%d%s", base, i, ext)
if i > 1000 {
if i > 100 {
m.logger.Errorf("Too many attempts to find a unique filename for %s", storagePath)
candidate = fmt.Sprintf("%s_%s%s", base, xid.New().String(), ext)
break

130
storage/s3/s3.go Normal file
View File

@@ -0,0 +1,130 @@
package s3
import (
"context"
"fmt"
"io"
"path"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/charmbracelet/log"
storconfig "github.com/krau/SaveAny-Bot/config/storage"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
"github.com/rs/xid"
)
type S3 struct {
config storconfig.S3StorageConfig
client *s3.Client
logger *log.Logger
}
func (m *S3) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
s3Config, ok := cfg.(*storconfig.S3StorageConfig)
if !ok {
return fmt.Errorf("failed to cast s3 config")
}
if err := s3Config.Validate(); err != nil {
return err
}
m.config = *s3Config
m.logger = log.FromContext(ctx).WithPrefix(fmt.Sprintf("s3[%s]", m.config.Name))
awsCfg, err := config.LoadDefaultConfig(
ctx,
config.WithRegion(m.config.Region),
config.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
m.config.AccessKeyID,
m.config.SecretAccessKey,
"",
),
),
)
if err != nil {
return fmt.Errorf("failed to load AWS config: %w", err)
}
m.client = s3.NewFromConfig(awsCfg)
// Check if bucket exists
_, err = m.client.HeadBucket(ctx, &s3.HeadBucketInput{
Bucket: aws.String(m.config.BucketName),
})
if err != nil {
return fmt.Errorf("bucket %s not accessible: %w", m.config.BucketName, err)
}
return nil
}
func (m *S3) Type() storenum.StorageType {
return storenum.S3
}
func (m *S3) Name() string {
return m.config.Name
}
func (m *S3) JoinStoragePath(p string) string {
return strings.TrimPrefix(path.Join(m.config.BasePath, p), "/")
}
func (m *S3) Save(ctx context.Context, r io.Reader, storagePath string) error {
m.logger.Infof("Saving file from reader to %s", storagePath)
ext := path.Ext(storagePath)
base := strings.TrimSuffix(storagePath, ext)
candidate := storagePath
// Unique filename
for i := 1; m.Exists(ctx, candidate); i++ {
candidate = fmt.Sprintf("%s_%d%s", base, i, ext)
if i > 100 {
m.logger.Errorf("Too many attempts for unique filename: %s", storagePath)
candidate = fmt.Sprintf("%s_%s%s", base, xid.New().String(), ext)
break
}
}
// Determine content length
size := int64(-1)
if length := ctx.Value(ctxkey.ContentLength); length != nil {
if l, ok := length.(int64); ok && l > 0 {
size = l
}
}
// S3 PutObject needs either size or StreamingBody
input := &s3.PutObjectInput{
Bucket: aws.String(m.config.BucketName),
Key: aws.String(candidate),
Body: r,
}
if size >= 0 {
input.ContentLength = &size
}
_, err := m.client.PutObject(ctx, input)
if err != nil {
return fmt.Errorf("failed to upload file to S3: %w", err)
}
return nil
}
func (m *S3) Exists(ctx context.Context, storagePath string) bool {
m.logger.Debugf("Checking if file exists at %s", storagePath)
_, err := m.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(m.config.BucketName),
Key: aws.String(storagePath),
})
return err == nil
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/krau/SaveAny-Bot/storage/alist"
"github.com/krau/SaveAny-Bot/storage/local"
"github.com/krau/SaveAny-Bot/storage/minio"
"github.com/krau/SaveAny-Bot/storage/s3"
"github.com/krau/SaveAny-Bot/storage/telegram"
"github.com/krau/SaveAny-Bot/storage/webdav"
)
@@ -37,6 +38,7 @@ var storageConstructors = map[storenum.StorageType]StorageConstructor{
storenum.Local: func() Storage { return new(local.Local) },
storenum.Webdav: func() Storage { return new(webdav.Webdav) },
storenum.Minio: func() Storage { return new(minio.Minio) },
storenum.S3: func() Storage { return new(s3.S3) },
storenum.Telegram: func() Storage { return new(telegram.Telegram) },
}