feat: add Minio storage support

Signed-off-by: ysicing <i@ysicing.me>
2025-03-11 21:29:35 +08:00
parent 3def9df4b4
commit 495ad3ea5c
9 changed files with 244 additions and 3 deletions
--- a/config.example.toml
+++ b/config.example.toml
@@ -23,7 +23,7 @@ url = "socks5://127.0.0.1:7890"
 [[storages]]
 # 标识名, 需要唯一
 name = "本机1"
-# 存储类型, 目前可用: local , alist , webdav
+# 存储类型, 目前可用: local, alist, webdav, minio
 type = "local"
 # 启用存储
 enable = true
@@ -59,6 +59,16 @@ url = 'https://example.com/dav'
 username = 'username'
 password = 'password'

+[[storages]]
+name = "MyMinio"
+type = "minio"
+enable = true
+endpoint = 'play.min.io'
+use_ssl = true
+access_key_id = 'Q3AM3UQ867SPQQA43P2F'
+secret_access_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
+bucket_name = 'saveanybot'
+base_path = '/path/telegram'

 # 用户列表
 [[users]]
--- a/config/storage_factory.go
+++ b/config/storage_factory.go
@@ -36,6 +36,7 @@ func init() {
 	RegisterStorageFactory(string(types.StorageTypeLocal), newLocalStorageConfig)
 	RegisterStorageFactory(string(types.StorageTypeAlist), newAlistStorageConfig)
 	RegisterStorageFactory(string(types.StorageTypeWebdav), newWebdavStorageConfig)
+	RegisterStorageFactory(string(types.StorageTypeMinio), newMinioStorageConfig)
 }

 func newLocalStorageConfig(cfg *NewStorageConfig) (StorageConfig, error) {
@@ -102,3 +103,12 @@ func LoadStorageConfigs(v *viper.Viper) ([]StorageConfig, error) {

 	return configs, nil
 }
+
+func newMinioStorageConfig(cfg *NewStorageConfig) (StorageConfig, error) {
+	var minioCfg MinioStorageConfig
+	minioCfg.NewStorageConfig = *cfg
+	if err := mapstructure.Decode(cfg.RawConfig, &minioCfg); err != nil {
+		return nil, fmt.Errorf("failed to decode minio storage config: %w", err)
+	}
+	return &minioCfg, nil
+}
--- a/config/storages.go
+++ b/config/storages.go
@@ -104,3 +104,37 @@ func (w *WebdavStorageConfig) GetType() types.StorageType {
 func (w *WebdavStorageConfig) GetName() string {
 	return w.Name
 }
+
+type MinioStorageConfig struct {
+	NewStorageConfig
+	Endpoint        string `toml:"endpoint" mapstructure:"endpoint" json:"endpoint"`
+	AccessKeyID     string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"`
+	SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"`
+	BucketName      string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"`
+	UseSSL          bool   `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"`
+	BasePath        string `toml:"base_path" mapstructure:"base_path" json:"base_path"`
+}
+
+func (m *MinioStorageConfig) Validate() error {
+	if m.Endpoint == "" {
+		return fmt.Errorf("endpoint is required for minio storage")
+	}
+	if m.AccessKeyID == "" || m.SecretAccessKey == "" {
+		return fmt.Errorf("access_key_id and secret_access_key are required for minio storage")
+	}
+	if m.BucketName == "" {
+		return fmt.Errorf("bucket_name is required for minio storage")
+	}
+	if m.BasePath == "" {
+		return fmt.Errorf("base_path is required for minio storage")
+	}
+	return nil
+}
+
+func (m *MinioStorageConfig) GetType() types.StorageType {
+	return types.StorageTypeMinio
+}
+
+func (m *MinioStorageConfig) GetName() string {
+	return m.Name
+}
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/gookit/slog v0.5.7
 	github.com/gotd/contrib v0.21.0
 	github.com/gotd/td v0.120.0
+	github.com/minio/minio-go/v7 v7.0.81
 	github.com/rhysd/go-github-selfupdate v1.2.3
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
@@ -31,7 +32,9 @@ require (
 	github.com/go-faster/jx v1.1.0 // indirect
 	github.com/go-faster/xor v1.0.0 // indirect
 	github.com/go-faster/yaml v0.4.6 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf // indirect
@@ -41,13 +44,16 @@ require (
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/ogen-go/ogen v1.10.0 // indirect
 	github.com/onsi/gomega v1.36.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/tcnksm/go-gitconfig v0.1.2 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
--- a/go.sum
+++ b/go.sum
@@ -49,6 +49,8 @@ github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
 github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
 github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
 github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -56,6 +58,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
@@ -111,6 +115,9 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -128,6 +135,10 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
 github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA=
+github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
@@ -150,6 +161,8 @@ github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7
 github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzxazpPAODuqarmPDe2Rg=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
 github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
@@ -224,6 +237,7 @@ golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
--- a/storage/minio/client.go
+++ b/storage/minio/client.go
@@ -0,0 +1,71 @@
+package minio
+
+import (
+	"context"
+	"fmt"
+	"path"
+
+	"github.com/krau/SaveAny-Bot/config"
+	"github.com/krau/SaveAny-Bot/logger"
+	"github.com/krau/SaveAny-Bot/types"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type Minio struct {
+	config config.MinioStorageConfig
+	client *minio.Client
+}
+
+func (m *Minio) Init(cfg config.StorageConfig) error {
+	minioConfig, ok := cfg.(*config.MinioStorageConfig)
+	if !ok {
+		return fmt.Errorf("failed to cast minio config")
+	}
+	if err := minioConfig.Validate(); err != nil {
+		return err
+	}
+	m.config = *minioConfig
+
+	client, err := minio.New(m.config.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(m.config.AccessKeyID, m.config.SecretAccessKey, ""),
+		Secure: m.config.UseSSL,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create minio client: %w", err)
+	}
+
+	exists, err := client.BucketExists(context.Background(), m.config.BucketName)
+	if err != nil {
+		return fmt.Errorf("failed to check bucket existence: %w", err)
+	}
+	if !exists {
+		return fmt.Errorf("bucket %s does not exist", m.config.BucketName)
+	}
+
+	m.client = client
+	return nil
+}
+
+func (m *Minio) Type() types.StorageType {
+	return types.StorageTypeMinio
+}
+
+func (m *Minio) Name() string {
+	return m.config.Name
+}
+
+func (m *Minio) JoinStoragePath(task types.Task) string {
+	return path.Join(m.config.BasePath, task.StoragePath)
+}
+
+func (m *Minio) Save(ctx context.Context, localFilePath, storagePath string) error {
+	logger.L.Infof("Saving file %s to %s", localFilePath, storagePath)
+
+	_, err := m.client.FPutObject(ctx, m.config.BucketName, storagePath, localFilePath, minio.PutObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to upload file to minio: %w", err)
+	}
+
+	return nil
+}
--- a/storage/minio/stream.go
+++ b/storage/minio/stream.go
@@ -0,0 +1,92 @@
+package minio
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/krau/SaveAny-Bot/logger"
+	"github.com/minio/minio-go/v7"
+)
+
+type MinioWriter struct {
+	pipeWriter *io.PipeWriter
+	done       chan error
+	path       string
+	ctx        context.Context
+	closed     bool
+}
+
+func (w *MinioWriter) Write(p []byte) (n int, err error) {
+	select {
+	case <-w.ctx.Done():
+		return 0, w.ctx.Err()
+	default:
+		return w.pipeWriter.Write(p)
+	}
+}
+
+func (w *MinioWriter) Close() error {
+	if w.closed {
+		return nil
+	}
+	w.closed = true
+
+	if err := w.pipeWriter.Close(); err != nil {
+		return fmt.Errorf("failed to close pipe writer: %w", err)
+	}
+
+	select {
+	case err := <-w.done:
+		if err != nil {
+			return fmt.Errorf("upload failed: %w", err)
+		}
+		return nil
+	case <-w.ctx.Done():
+		return fmt.Errorf("upload cancelled: %w", w.ctx.Err())
+	}
+}
+
+func (m *Minio) NewUploadStream(ctx context.Context, storagePath string) (io.WriteCloser, error) {
+	logger.L.Infof("Creating upload stream for %s", storagePath)
+
+	uploadCtx, cancel := context.WithCancel(ctx)
+	pipeReader, pipeWriter := io.Pipe()
+	done := make(chan error, 1)
+
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				done <- fmt.Errorf("panic during upload: %v", r)
+			}
+			pipeReader.Close()
+			cancel()
+		}()
+
+		info, err := m.client.PutObject(
+			uploadCtx,
+			m.config.BucketName,
+			storagePath,
+			pipeReader,
+			-1,
+			minio.PutObjectOptions{},
+		)
+
+		if err != nil {
+			logger.L.Errorf("Failed to upload to %s: %v", storagePath, err)
+			done <- err
+			return
+		}
+
+		logger.L.Infof("uploaded %d bytes to %s", info.Size, storagePath)
+		done <- nil
+	}()
+
+	return &MinioWriter{
+		pipeWriter: pipeWriter,
+		done:       done,
+		path:       storagePath,
+		ctx:        uploadCtx,
+		closed:     false,
+	}, nil
+}
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -10,6 +10,7 @@ import (
 	"github.com/krau/SaveAny-Bot/storage/alist"
 	"github.com/krau/SaveAny-Bot/storage/local"
 	"github.com/krau/SaveAny-Bot/storage/webdav"
+	"github.com/krau/SaveAny-Bot/storage/minio"
 	"github.com/krau/SaveAny-Bot/types"
 )

@@ -90,6 +91,7 @@ var storageConstructors = map[string]StorageConstructor{
 	string(types.StorageTypeAlist):  func() Storage { return new(alist.Alist) },
 	string(types.StorageTypeLocal):  func() Storage { return new(local.Local) },
 	string(types.StorageTypeWebdav): func() Storage { return new(webdav.Webdav) },
+	string(types.StorageTypeMinio):  func() Storage { return new(minio.Minio) },
 }

 func NewStorage(cfg config.StorageConfig) (Storage, error) {
--- a/types/types.go
+++ b/types/types.go
@@ -25,13 +25,15 @@ var (
 	StorageTypeLocal  StorageType = "local"
 	StorageTypeWebdav StorageType = "webdav"
 	StorageTypeAlist  StorageType = "alist"
+	StorageTypeMinio  StorageType = "minio"
 )

-var StorageTypes = []StorageType{StorageTypeLocal, StorageTypeAlist, StorageTypeWebdav}
+var StorageTypes = []StorageType{StorageTypeLocal, StorageTypeAlist, StorageTypeWebdav, StorageTypeMinio}
 var StorageTypeDisplay = map[StorageType]string{
 	StorageTypeLocal:  "本地磁盘",
 	StorageTypeWebdav: "WebDAV",
 	StorageTypeAlist:  "Alist",
+	StorageTypeMinio:  "Minio",
 }

 type Task struct {