From 495ad3ea5c8575ccf5df5b4b02ddac0b9bcd75d2 Mon Sep 17 00:00:00 2001 From: ysicing Date: Tue, 11 Mar 2025 21:29:35 +0800 Subject: [PATCH] feat: add Minio storage support Signed-off-by: ysicing --- config.example.toml | 14 +++++- config/storage_factory.go | 10 +++++ config/storages.go | 34 +++++++++++++++ go.mod | 6 +++ go.sum | 14 ++++++ storage/minio/client.go | 71 ++++++++++++++++++++++++++++++ storage/minio/stream.go | 92 +++++++++++++++++++++++++++++++++++++++ storage/storage.go | 2 + types/types.go | 4 +- 9 files changed, 244 insertions(+), 3 deletions(-) create mode 100644 storage/minio/client.go create mode 100644 storage/minio/stream.go diff --git a/config.example.toml b/config.example.toml index b3ae958..6dc8651 100644 --- a/config.example.toml +++ b/config.example.toml @@ -23,7 +23,7 @@ url = "socks5://127.0.0.1:7890" [[storages]] # 标识名, 需要唯一 name = "本机1" -# 存储类型, 目前可用: local , alist , webdav +# 存储类型, 目前可用: local, alist, webdav, minio type = "local" # 启用存储 enable = true @@ -59,6 +59,16 @@ url = 'https://example.com/dav' username = 'username' password = 'password' +[[storages]] +name = "MyMinio" +type = "minio" +enable = true +endpoint = 'play.min.io' +use_ssl = true +access_key_id = 'Q3AM3UQ867SPQQA43P2F' +secret_access_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' +bucket_name = 'saveanybot' +base_path = '/path/telegram' # 用户列表 [[users]] @@ -91,4 +101,4 @@ storages = ["本机1"] # cache_ttl = 30 # [db] -# path = "data/data.db" # 数据库文件路径 \ No newline at end of file +# path = "data/data.db" # 数据库文件路径 diff --git a/config/storage_factory.go b/config/storage_factory.go index 120dc0e..83508ec 100644 --- a/config/storage_factory.go +++ b/config/storage_factory.go @@ -36,6 +36,7 @@ func init() { RegisterStorageFactory(string(types.StorageTypeLocal), newLocalStorageConfig) RegisterStorageFactory(string(types.StorageTypeAlist), newAlistStorageConfig) RegisterStorageFactory(string(types.StorageTypeWebdav), newWebdavStorageConfig) + RegisterStorageFactory(string(types.StorageTypeMinio), newMinioStorageConfig) } func newLocalStorageConfig(cfg *NewStorageConfig) (StorageConfig, error) { @@ -102,3 +103,12 @@ func LoadStorageConfigs(v *viper.Viper) ([]StorageConfig, error) { return configs, nil } + +func newMinioStorageConfig(cfg *NewStorageConfig) (StorageConfig, error) { + var minioCfg MinioStorageConfig + minioCfg.NewStorageConfig = *cfg + if err := mapstructure.Decode(cfg.RawConfig, &minioCfg); err != nil { + return nil, fmt.Errorf("failed to decode minio storage config: %w", err) + } + return &minioCfg, nil +} diff --git a/config/storages.go b/config/storages.go index a11fd5d..d49230f 100644 --- a/config/storages.go +++ b/config/storages.go @@ -104,3 +104,37 @@ func (w *WebdavStorageConfig) GetType() types.StorageType { func (w *WebdavStorageConfig) GetName() string { return w.Name } + +type MinioStorageConfig struct { + NewStorageConfig + Endpoint string `toml:"endpoint" mapstructure:"endpoint" json:"endpoint"` + AccessKeyID string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"` + SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"` + BucketName string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"` + UseSSL bool `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"` + BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"` +} + +func (m *MinioStorageConfig) Validate() error { + if m.Endpoint == "" { + return fmt.Errorf("endpoint is required for minio storage") + } + if m.AccessKeyID == "" || m.SecretAccessKey == "" { + return fmt.Errorf("access_key_id and secret_access_key are required for minio storage") + } + if m.BucketName == "" { + return fmt.Errorf("bucket_name is required for minio storage") + } + if m.BasePath == "" { + return fmt.Errorf("base_path is required for minio storage") + } + return nil +} + +func (m *MinioStorageConfig) GetType() types.StorageType { + return types.StorageTypeMinio +} + +func (m *MinioStorageConfig) GetName() string { + return m.Name +} diff --git a/go.mod b/go.mod index 2667698..f0942ef 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/gookit/slog v0.5.7 github.com/gotd/contrib v0.21.0 github.com/gotd/td v0.120.0 + github.com/minio/minio-go/v7 v7.0.81 github.com/rhysd/go-github-selfupdate v1.2.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -31,7 +32,9 @@ require ( github.com/go-faster/jx v1.1.0 // indirect github.com/go-faster/xor v1.0.0 // indirect github.com/go-faster/yaml v0.4.6 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf // indirect @@ -41,13 +44,16 @@ require ( github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/ogen-go/ogen v1.10.0 // indirect github.com/onsi/gomega v1.36.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.6.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/tcnksm/go-gitconfig v0.1.2 // indirect github.com/ulikunitz/xz v0.5.12 // indirect diff --git a/go.sum b/go.sum index f9b8233..0488280 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38= github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ= github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I= github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -56,6 +58,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -111,6 +115,9 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -128,6 +135,10 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA= github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA= +github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -150,6 +161,8 @@ github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7 github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzxazpPAODuqarmPDe2Rg= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= @@ -224,6 +237,7 @@ golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/storage/minio/client.go b/storage/minio/client.go new file mode 100644 index 0000000..c596f2a --- /dev/null +++ b/storage/minio/client.go @@ -0,0 +1,71 @@ +package minio + +import ( + "context" + "fmt" + "path" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/logger" + "github.com/krau/SaveAny-Bot/types" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Minio struct { + config config.MinioStorageConfig + client *minio.Client +} + +func (m *Minio) Init(cfg config.StorageConfig) error { + minioConfig, ok := cfg.(*config.MinioStorageConfig) + if !ok { + return fmt.Errorf("failed to cast minio config") + } + if err := minioConfig.Validate(); err != nil { + return err + } + m.config = *minioConfig + + client, err := minio.New(m.config.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(m.config.AccessKeyID, m.config.SecretAccessKey, ""), + Secure: m.config.UseSSL, + }) + if err != nil { + return fmt.Errorf("failed to create minio client: %w", err) + } + + exists, err := client.BucketExists(context.Background(), m.config.BucketName) + if err != nil { + return fmt.Errorf("failed to check bucket existence: %w", err) + } + if !exists { + return fmt.Errorf("bucket %s does not exist", m.config.BucketName) + } + + m.client = client + return nil +} + +func (m *Minio) Type() types.StorageType { + return types.StorageTypeMinio +} + +func (m *Minio) Name() string { + return m.config.Name +} + +func (m *Minio) JoinStoragePath(task types.Task) string { + return path.Join(m.config.BasePath, task.StoragePath) +} + +func (m *Minio) Save(ctx context.Context, localFilePath, storagePath string) error { + logger.L.Infof("Saving file %s to %s", localFilePath, storagePath) + + _, err := m.client.FPutObject(ctx, m.config.BucketName, storagePath, localFilePath, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("failed to upload file to minio: %w", err) + } + + return nil +} diff --git a/storage/minio/stream.go b/storage/minio/stream.go new file mode 100644 index 0000000..dbf7ff2 --- /dev/null +++ b/storage/minio/stream.go @@ -0,0 +1,92 @@ +package minio + +import ( + "context" + "fmt" + "io" + + "github.com/krau/SaveAny-Bot/logger" + "github.com/minio/minio-go/v7" +) + +type MinioWriter struct { + pipeWriter *io.PipeWriter + done chan error + path string + ctx context.Context + closed bool +} + +func (w *MinioWriter) Write(p []byte) (n int, err error) { + select { + case <-w.ctx.Done(): + return 0, w.ctx.Err() + default: + return w.pipeWriter.Write(p) + } +} + +func (w *MinioWriter) Close() error { + if w.closed { + return nil + } + w.closed = true + + if err := w.pipeWriter.Close(); err != nil { + return fmt.Errorf("failed to close pipe writer: %w", err) + } + + select { + case err := <-w.done: + if err != nil { + return fmt.Errorf("upload failed: %w", err) + } + return nil + case <-w.ctx.Done(): + return fmt.Errorf("upload cancelled: %w", w.ctx.Err()) + } +} + +func (m *Minio) NewUploadStream(ctx context.Context, storagePath string) (io.WriteCloser, error) { + logger.L.Infof("Creating upload stream for %s", storagePath) + + uploadCtx, cancel := context.WithCancel(ctx) + pipeReader, pipeWriter := io.Pipe() + done := make(chan error, 1) + + go func() { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic during upload: %v", r) + } + pipeReader.Close() + cancel() + }() + + info, err := m.client.PutObject( + uploadCtx, + m.config.BucketName, + storagePath, + pipeReader, + -1, + minio.PutObjectOptions{}, + ) + + if err != nil { + logger.L.Errorf("Failed to upload to %s: %v", storagePath, err) + done <- err + return + } + + logger.L.Infof("uploaded %d bytes to %s", info.Size, storagePath) + done <- nil + }() + + return &MinioWriter{ + pipeWriter: pipeWriter, + done: done, + path: storagePath, + ctx: uploadCtx, + closed: false, + }, nil +} diff --git a/storage/storage.go b/storage/storage.go index ee6cd2b..12426f2 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -10,6 +10,7 @@ import ( "github.com/krau/SaveAny-Bot/storage/alist" "github.com/krau/SaveAny-Bot/storage/local" "github.com/krau/SaveAny-Bot/storage/webdav" + "github.com/krau/SaveAny-Bot/storage/minio" "github.com/krau/SaveAny-Bot/types" ) @@ -90,6 +91,7 @@ var storageConstructors = map[string]StorageConstructor{ string(types.StorageTypeAlist): func() Storage { return new(alist.Alist) }, string(types.StorageTypeLocal): func() Storage { return new(local.Local) }, string(types.StorageTypeWebdav): func() Storage { return new(webdav.Webdav) }, + string(types.StorageTypeMinio): func() Storage { return new(minio.Minio) }, } func NewStorage(cfg config.StorageConfig) (Storage, error) { diff --git a/types/types.go b/types/types.go index c2e0ca4..1c32eaa 100644 --- a/types/types.go +++ b/types/types.go @@ -25,13 +25,15 @@ var ( StorageTypeLocal StorageType = "local" StorageTypeWebdav StorageType = "webdav" StorageTypeAlist StorageType = "alist" + StorageTypeMinio StorageType = "minio" ) -var StorageTypes = []StorageType{StorageTypeLocal, StorageTypeAlist, StorageTypeWebdav} +var StorageTypes = []StorageType{StorageTypeLocal, StorageTypeAlist, StorageTypeWebdav, StorageTypeMinio} var StorageTypeDisplay = map[StorageType]string{ StorageTypeLocal: "本地磁盘", StorageTypeWebdav: "WebDAV", StorageTypeAlist: "Alist", + StorageTypeMinio: "Minio", } type Task struct {