Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32cc1e4b5a | ||
|
|
c974791dc0 | ||
|
|
91814a83c7 | ||
|
|
685047e463 |
@@ -29,7 +29,7 @@
|
||||
- 使用 js 编写解析器插件以转存任意网站的文件
|
||||
- 存储端支持:
|
||||
- Alist
|
||||
- S3 (MinioSDK)
|
||||
- S3
|
||||
- WebDAV
|
||||
- 本地磁盘
|
||||
- Telegram (重传回指定聊天)
|
||||
|
||||
@@ -34,7 +34,7 @@ func (m matchedStorName) String() string {
|
||||
}
|
||||
|
||||
// can we use this storage name directly?
|
||||
func (m matchedStorName) IsUsable() bool {
|
||||
func (m matchedStorName) Usable() bool {
|
||||
return m != "" && m != rule.RuleStorNameChosen
|
||||
}
|
||||
|
||||
|
||||
@@ -126,7 +126,7 @@ func GetFilesFromUpdateLinkMessageWithReplyEdit(ctx *ext.Context, update *ext.Up
|
||||
}
|
||||
msg, err := tgutil.GetMessageByID(tctx, chatId, msgId)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get message by ID: %s", err)
|
||||
logger.Error(err)
|
||||
continue
|
||||
}
|
||||
groupID, isGroup := msg.GetGroupedID()
|
||||
|
||||
@@ -41,7 +41,7 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage
|
||||
if matchedDirPath != "" {
|
||||
dirPath = matchedDirPath.String()
|
||||
}
|
||||
if matchedStorageName.IsUsable() {
|
||||
if matchedStorageName.Usable() {
|
||||
stor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, matchedStorageName.String())
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to get storage by user ID and name: %s", err)
|
||||
@@ -111,7 +111,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
|
||||
return stor.Name(), ruleutil.MatchedDirPath(dirPath)
|
||||
}
|
||||
storname := storName.String()
|
||||
if !storName.IsUsable() {
|
||||
if !storName.Usable() {
|
||||
storname = stor.Name()
|
||||
}
|
||||
return storname, dirP
|
||||
|
||||
@@ -228,7 +228,7 @@ func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent) {
|
||||
goto startCreateTask
|
||||
}
|
||||
dirPath = matchedDirPath.String()
|
||||
if matchedStorageName.IsUsable() {
|
||||
if matchedStorageName.Usable() {
|
||||
stor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, matchedStorageName.String())
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to get storage by user ID and name: %s", err)
|
||||
|
||||
@@ -32,6 +32,10 @@ bot:
|
||||
/save [自定义文件名] - 保存文件
|
||||
/dir - 管理存储目录
|
||||
/rule - 管理规则
|
||||
/config - 修改配置
|
||||
/fnametmpl - 设置文件自定义命名模板
|
||||
/parser - 管理解析器插件
|
||||
/watch - 监听聊天并自动保存 (UserBot)
|
||||
/update - 检查更新并升级
|
||||
|
||||
使用帮助: https://sabot.unv.app/usage
|
||||
@@ -57,6 +61,6 @@ bot:
|
||||
- [filter]: 可选, 格式为 过滤器类型:表达式 , 所有支持类型的过滤器请查看文档
|
||||
|
||||
命令示例:
|
||||
/watch 2229835658 msgre:.*plana.*
|
||||
/watch -1002229835658 msgre:.*plana.*
|
||||
|
||||
这将监听 ID 为 2229835658 的聊天, 并转存所有包含 "plana" 的媒体消息
|
||||
这将监听 ID 为 -1002229835658 的聊天, 并转存所有包含 "plana" 的媒体消息
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package tfile
|
||||
package tdler
|
||||
|
||||
import (
|
||||
"github.com/gotd/td/telegram/downloader"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/consts/tglimit"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
)
|
||||
|
||||
func NewDownloader(file TGFile) *downloader.Builder {
|
||||
func NewDownloader(file tfile.TGFile) *downloader.Builder {
|
||||
return downloader.NewDownloader().WithPartSize(tglimit.MaxPartSize).
|
||||
Download(file.Dler(), file.Location()).WithThreads(dlutil.BestThreads(file.Size(), config.C().Threads))
|
||||
}
|
||||
@@ -9,12 +9,12 @@ import (
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/duke-git/lancet/v2/maputil"
|
||||
|
||||
"github.com/duke-git/lancet/v2/mathutil"
|
||||
"github.com/duke-git/lancet/v2/slice"
|
||||
lcstrutil "github.com/duke-git/lancet/v2/strutil"
|
||||
"github.com/duke-git/lancet/v2/validator"
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
"github.com/gotd/td/constant"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/cache"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/strutil"
|
||||
@@ -112,6 +112,31 @@ func InputMessageClassSliceFromInt(ids []int) []tg.InputMessageClass {
|
||||
}
|
||||
|
||||
func GetMessagesRange(ctx *ext.Context, chatID int64, minId, maxId int) ([]*tg.Message, error) {
|
||||
if msg, err := getMessagesRange(ctx, chatID, minId, maxId); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
in := constant.TDLibPeerID(chatID)
|
||||
plain := in.ToPlain()
|
||||
|
||||
var channel constant.TDLibPeerID
|
||||
channel.Channel(plain)
|
||||
if msg, err := getMessagesRange(ctx, int64(channel), minId, maxId); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
var userID constant.TDLibPeerID
|
||||
userID.User(plain)
|
||||
if msg, err := getMessagesRange(ctx, int64(userID), minId, maxId); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
var chat constant.TDLibPeerID
|
||||
chat.Chat(plain)
|
||||
if msg, err := getMessagesRange(ctx, int64(chat), minId, maxId); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get messages range for chatID %d", chatID)
|
||||
}
|
||||
|
||||
func getMessagesRange(ctx *ext.Context, chatID int64, minId, maxId int) ([]*tg.Message, error) {
|
||||
if minId > maxId {
|
||||
return nil, fmt.Errorf("minId (%d) cannot be greater than maxId (%d)", minId, maxId)
|
||||
}
|
||||
@@ -167,97 +192,98 @@ func GetMessagesRange(ctx *ext.Context, chatID int64, minId, maxId int) ([]*tg.M
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type MessageItem struct {
|
||||
Message *tg.Message
|
||||
Error error
|
||||
}
|
||||
// [TODO]
|
||||
// type MessageItem struct {
|
||||
// Message *tg.Message
|
||||
// Error error
|
||||
// }
|
||||
|
||||
func IterMessages(ctx *ext.Context, chatID int64, minId, maxId int) (<-chan MessageItem, error) {
|
||||
total := maxId - minId + 1
|
||||
ch := make(chan MessageItem, 100)
|
||||
// func IterMessages(ctx *ext.Context, chatID int64, minId, maxId int) (<-chan MessageItem, error) {
|
||||
// total := maxId - minId + 1
|
||||
// ch := make(chan MessageItem, 100)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
if !ctx.Self.Bot {
|
||||
perr := ctx.PeerStorage.GetInputPeerById(chatID)
|
||||
if perr == nil || perr.(*tg.InputPeerEmpty) != nil {
|
||||
ch <- MessageItem{
|
||||
Error: fmt.Errorf("peer not found: %d", chatID),
|
||||
}
|
||||
return
|
||||
}
|
||||
// go func() {
|
||||
// defer close(ch)
|
||||
// if !ctx.Self.Bot {
|
||||
// perr := ctx.PeerStorage.GetInputPeerById(chatID)
|
||||
// if perr == nil || perr.(*tg.InputPeerEmpty) != nil {
|
||||
// ch <- MessageItem{
|
||||
// Error: fmt.Errorf("peer not found: %d", chatID),
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
|
||||
for i := 0; i < total; i += 100 {
|
||||
start := minId + i
|
||||
end := min(start+100, maxId)
|
||||
msgs, err := ctx.Raw.MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
|
||||
Peer: perr,
|
||||
OffsetID: start,
|
||||
AddOffset: start - end,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
ch <- MessageItem{
|
||||
Error: fmt.Errorf("failed to get messages: %w", err),
|
||||
}
|
||||
return
|
||||
}
|
||||
var msgClass []tg.MessageClass
|
||||
switch msgsv := msgs.(type) {
|
||||
case *tg.MessagesMessages:
|
||||
msgClass = msgsv.GetMessages()
|
||||
case *tg.MessagesMessagesSlice:
|
||||
msgClass = msgsv.GetMessages()
|
||||
case *tg.MessagesChannelMessages:
|
||||
msgClass = msgsv.GetMessages()
|
||||
default:
|
||||
ch <- MessageItem{
|
||||
Error: fmt.Errorf("unsupported message type: %T", msgsv),
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, msg := range msgClass {
|
||||
msg, ok := msg.AsNotEmpty()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch msg := msg.(type) {
|
||||
case *tg.Message:
|
||||
key := fmt.Sprintf("tgmsg:%d:%d:%d", ctx.Self.ID, chatID, msg.GetID())
|
||||
cache.Set(key, msg)
|
||||
ch <- MessageItem{
|
||||
Message: msg,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i := 0; i < total; i += 100 {
|
||||
start := minId + i
|
||||
end := min(start+100, maxId)
|
||||
msgs, err := GetMessagesRange(ctx, chatID, start, end)
|
||||
if err != nil {
|
||||
ch <- MessageItem{
|
||||
Error: fmt.Errorf("failed to get messages: %w", err),
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
ch <- MessageItem{
|
||||
Message: msg,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
// for i := 0; i < total; i += 100 {
|
||||
// start := minId + i
|
||||
// end := min(start+100, maxId)
|
||||
// msgs, err := ctx.Raw.MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{
|
||||
// Peer: perr,
|
||||
// OffsetID: start,
|
||||
// AddOffset: start - end,
|
||||
// Limit: 100,
|
||||
// })
|
||||
// if err != nil {
|
||||
// ch <- MessageItem{
|
||||
// Error: fmt.Errorf("failed to get messages: %w", err),
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
// var msgClass []tg.MessageClass
|
||||
// switch msgsv := msgs.(type) {
|
||||
// case *tg.MessagesMessages:
|
||||
// msgClass = msgsv.GetMessages()
|
||||
// case *tg.MessagesMessagesSlice:
|
||||
// msgClass = msgsv.GetMessages()
|
||||
// case *tg.MessagesChannelMessages:
|
||||
// msgClass = msgsv.GetMessages()
|
||||
// default:
|
||||
// ch <- MessageItem{
|
||||
// Error: fmt.Errorf("unsupported message type: %T", msgsv),
|
||||
// }
|
||||
// continue
|
||||
// }
|
||||
// for _, msg := range msgClass {
|
||||
// msg, ok := msg.AsNotEmpty()
|
||||
// if !ok {
|
||||
// continue
|
||||
// }
|
||||
// switch msg := msg.(type) {
|
||||
// case *tg.Message:
|
||||
// key := fmt.Sprintf("tgmsg:%d:%d:%d", ctx.Self.ID, chatID, msg.GetID())
|
||||
// cache.Set(key, msg)
|
||||
// ch <- MessageItem{
|
||||
// Message: msg,
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// for i := 0; i < total; i += 100 {
|
||||
// start := minId + i
|
||||
// end := min(start+100, maxId)
|
||||
// msgs, err := GetMessagesRange(ctx, chatID, start, end)
|
||||
// if err != nil {
|
||||
// ch <- MessageItem{
|
||||
// Error: fmt.Errorf("failed to get messages: %w", err),
|
||||
// }
|
||||
// return
|
||||
// }
|
||||
// for _, msg := range msgs {
|
||||
// if msg == nil {
|
||||
// continue
|
||||
// }
|
||||
// ch <- MessageItem{
|
||||
// Message: msg,
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
// return ch, nil
|
||||
// }
|
||||
|
||||
func GetMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) {
|
||||
func getMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) {
|
||||
key := fmt.Sprintf("tgmsg:%d:%d:%d", ctx.Self.ID, chatID, msgID)
|
||||
if msg, ok := cache.Get[*tg.Message](key); ok {
|
||||
return msg, nil
|
||||
@@ -280,6 +306,33 @@ func GetMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, err
|
||||
return tgm, nil
|
||||
}
|
||||
|
||||
// f**k gotgproto's breaking changes
|
||||
func GetMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) {
|
||||
// we don't know what the input chatID is bot api style(e.g. channel with -100 prefix) or plain tdlib style(no any prefix and every id is positive)
|
||||
if msg, err := getMessageByID(ctx, chatID, msgID); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
in := constant.TDLibPeerID(chatID)
|
||||
plain := in.ToPlain()
|
||||
var channel constant.TDLibPeerID
|
||||
channel.Channel(plain)
|
||||
if msg, err := getMessageByID(ctx, int64(channel), msgID); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
var chat constant.TDLibPeerID
|
||||
chat.Chat(plain)
|
||||
if msg, err := getMessageByID(ctx, int64(chat), msgID); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
var userID constant.TDLibPeerID
|
||||
userID.User(plain)
|
||||
if msg, err := getMessageByID(ctx, int64(userID), msgID); err == nil {
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to get message by ID: chatID=%d, msgID=%d", chatID, msgID)
|
||||
}
|
||||
|
||||
func GetGroupedMessages(ctx *ext.Context, chatID int64, msg *tg.Message) ([]*tg.Message, error) {
|
||||
groupID, isGroup := msg.GetGroupedID()
|
||||
if !isGroup || groupID == 0 {
|
||||
|
||||
@@ -22,7 +22,7 @@ url = "socks5://127.0.0.1:7890"
|
||||
[[storages]]
|
||||
# 标识名, 需要唯一
|
||||
name = "本机1"
|
||||
# 存储类型, 目前可用: local, alist, webdav, minio, telegram
|
||||
# 存储类型, 目前可用: local, alist, webdav, s3, telegram
|
||||
type = "local"
|
||||
# 启用存储
|
||||
enable = true
|
||||
|
||||
@@ -14,6 +14,7 @@ var storageFactories = map[storenum.StorageType]func(cfg *BaseConfig) (StorageCo
|
||||
storenum.Alist: createStorageConfig(&AlistStorageConfig{}),
|
||||
storenum.Webdav: createStorageConfig(&WebdavStorageConfig{}),
|
||||
storenum.Minio: createStorageConfig(&MinioStorageConfig{}),
|
||||
storenum.S3: createStorageConfig(&S3StorageConfig{}),
|
||||
storenum.Telegram: createStorageConfig(&TelegramStorageConfig{}),
|
||||
}
|
||||
|
||||
|
||||
43
config/storage/s3.go
Normal file
43
config/storage/s3.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
)
|
||||
|
||||
type S3StorageConfig struct {
|
||||
BaseConfig
|
||||
Endpoint string `toml:"endpoint" mapstructure:"endpoint" json:"endpoint"`
|
||||
AccessKeyID string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"`
|
||||
SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"`
|
||||
BucketName string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"`
|
||||
UseSSL bool `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"`
|
||||
BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"`
|
||||
Region string `toml:"region" mapstructure:"region" json:"region"`
|
||||
VirtualHost bool `toml:"virtual_host" mapstructure:"virtual_host" json:"virtual_host"`
|
||||
}
|
||||
|
||||
func (m *S3StorageConfig) Validate() error {
|
||||
if m.Endpoint == "" {
|
||||
return fmt.Errorf("endpoint is required for s3 storage")
|
||||
}
|
||||
if m.AccessKeyID == "" || m.SecretAccessKey == "" {
|
||||
return fmt.Errorf("access_key_id and secret_access_key are required for s3 storage")
|
||||
}
|
||||
if m.BucketName == "" {
|
||||
return fmt.Errorf("bucket_name is required for s3 storage")
|
||||
}
|
||||
if m.BasePath == "" {
|
||||
return fmt.Errorf("base_path is required for s3 storage")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *S3StorageConfig) GetType() storenum.StorageType {
|
||||
return storenum.S3
|
||||
}
|
||||
|
||||
func (m *S3StorageConfig) GetName() string {
|
||||
return m.Name
|
||||
}
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/krau/SaveAny-Bot/common/tdler"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -68,7 +68,7 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
errg.Go(func() error {
|
||||
defer pw.Close()
|
||||
logger.Info("Starting file download in stream mode")
|
||||
_, err := tfile.NewDownloader(elem.File).Stream(uploadCtx, wr)
|
||||
_, err := tdler.NewDownloader(elem.File).Stream(uploadCtx, wr)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to download file: %v", err)
|
||||
pw.CloseWithError(err)
|
||||
@@ -95,7 +95,7 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
t.downloaded.Add(int64(n))
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
})
|
||||
_, err = tfile.NewDownloader(elem.File).Parallel(ctx, wrAt)
|
||||
_, err = tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file: %w", err)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"go.uber.org/multierr"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -48,13 +47,10 @@ func (t *Task) processPic(ctx context.Context, picUrl string, index int) error {
|
||||
retry.Context(ctx),
|
||||
retry.RetryTimes(uint(config.C().Retry)),
|
||||
}
|
||||
var lastErr error
|
||||
err := retry.Retry(func() error {
|
||||
var body io.ReadCloser
|
||||
body, lastErr = t.client.Download(ctx, picUrl)
|
||||
if lastErr != nil {
|
||||
lastErr = fmt.Errorf("failed to download picture %s: %w", picUrl, lastErr)
|
||||
return lastErr
|
||||
body, err := t.client.Download(ctx, picUrl)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download picture %s: %w", picUrl, err)
|
||||
}
|
||||
defer body.Close()
|
||||
filename := fmt.Sprintf("%d%s", index+1, path.Ext(picUrl))
|
||||
@@ -63,8 +59,7 @@ func (t *Task) processPic(ctx context.Context, picUrl string, index int) error {
|
||||
fmt.Sprintf("tph_%s_%s", t.TaskID(), filename),
|
||||
))
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("failed to create cache file for picture %s: %w", filename, err)
|
||||
return lastErr
|
||||
return fmt.Errorf("failed to create cache file for picture %s: %w", filename, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := cacheFile.CloseAndRemove(); err != nil {
|
||||
@@ -72,26 +67,26 @@ func (t *Task) processPic(ctx context.Context, picUrl string, index int) error {
|
||||
logger.Errorf("Failed to close and remove cache file for picture %s: %v", filename, err)
|
||||
}
|
||||
}()
|
||||
_, lastErr = io.Copy(cacheFile, body)
|
||||
if lastErr != nil {
|
||||
lastErr = fmt.Errorf("failed to copy picture %s to cache file: %w", filename, lastErr)
|
||||
return lastErr
|
||||
_, err = io.Copy(cacheFile, body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy picture %s to cache file: %w", filename, err)
|
||||
}
|
||||
_, err = cacheFile.Seek(0, 0)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("failed to seek cache file for picture %s: %w", filename, err)
|
||||
return lastErr
|
||||
return fmt.Errorf("failed to seek cache file for picture %s: %w", filename, err)
|
||||
}
|
||||
err = t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, filename))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save picture %s: %w", filename, err)
|
||||
}
|
||||
lastErr = t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, filename))
|
||||
} else {
|
||||
lastErr = t.Stor.Save(ctx, body, path.Join(t.StorPath, filename))
|
||||
err = t.Stor.Save(ctx, body, path.Join(t.StorPath, filename))
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
lastErr = fmt.Errorf("failed to save picture %s: %w", filename, lastErr)
|
||||
return lastErr
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save picture %s: %w", filename, err)
|
||||
}
|
||||
return nil
|
||||
}, retryOpts...)
|
||||
return multierr.Combine(err, lastErr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5,13 +5,13 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/krau/SaveAny-Bot/common/tdler"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
)
|
||||
|
||||
func (t *Task) Execute(ctx context.Context) error {
|
||||
@@ -40,7 +40,7 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
t.Progress.OnDone(ctx, t, err)
|
||||
}
|
||||
}()
|
||||
_, err = tfile.NewDownloader(t.File).Parallel(ctx, wrAt)
|
||||
_, err = tdler.NewDownloader(t.File).Parallel(ctx, wrAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file: %w", err)
|
||||
}
|
||||
@@ -57,30 +57,19 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
return fmt.Errorf("failed to get file stat: %w", err)
|
||||
}
|
||||
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
|
||||
for i := range config.C().Retry + 1 {
|
||||
if err = vctx.Err(); err != nil {
|
||||
return fmt.Errorf("context canceled while saving file: %w", err)
|
||||
}
|
||||
var file *os.File
|
||||
file, err = os.Open(t.localPath)
|
||||
err = retry.Retry(func() error {
|
||||
file, err := os.Open(t.localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open cache file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
|
||||
if i == config.C().Retry {
|
||||
return fmt.Errorf("failed to save file: %w", err)
|
||||
}
|
||||
logger.Errorf("Failed to save file: %s, retrying...", err)
|
||||
select {
|
||||
case <-vctx.Done():
|
||||
return fmt.Errorf("context canceled during retry delay: %w", vctx.Err())
|
||||
case <-time.After(time.Duration(i*500) * time.Millisecond):
|
||||
}
|
||||
continue
|
||||
return fmt.Errorf("failed to save file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}, retry.RetryTimes(uint(config.C().Retry)), retry.Context(vctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save file after retries: %w", err)
|
||||
}
|
||||
return fmt.Errorf("failed to save file after retries")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
"github.com/krau/SaveAny-Bot/common/tdler"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ func executeStream(ctx context.Context, task *Task) error {
|
||||
errg.Go(func() error {
|
||||
defer pw.Close()
|
||||
logger.Info("Starting file download in stream mode")
|
||||
_, err := tfile.NewDownloader(task.File).Stream(uploadCtx, wr)
|
||||
_, err := tdler.NewDownloader(task.File).Stream(uploadCtx, wr)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to download file: %v", err)
|
||||
pw.CloseWithError(err)
|
||||
|
||||
@@ -21,7 +21,7 @@ Save Any Bot is a tool that allows you to save files from Telegram to various st
|
||||
- Automatic organization based on storage rules
|
||||
- Supports multiple storage backends:
|
||||
- Alist
|
||||
- Minio (S3 compatible)
|
||||
- S3
|
||||
- WebDAV
|
||||
- Telegram (re-upload to specified chat)
|
||||
- Local disk
|
||||
|
||||
@@ -79,7 +79,7 @@ Each storage endpoint requires at least the following fields:
|
||||
- `local`: Local disk
|
||||
- `alist`: Alist
|
||||
- `webdav`: WebDAV
|
||||
- `minio`: MinIO (compatible with S3 API)
|
||||
- `s3`: aws S3 and other S3 compatible services
|
||||
- `telegram`: Upload to Telegram
|
||||
|
||||
Example, this is a configuration that includes local storage and webdav storage:
|
||||
|
||||
@@ -41,17 +41,18 @@ password = "your_password" # Password for WebDAV
|
||||
base_path = "/path/to/webdav" # Base path in WebDAV, all files will be stored under this path
|
||||
```
|
||||
|
||||
## MinIO (S3)
|
||||
## S3
|
||||
|
||||
`type=minio`
|
||||
`type=s3`
|
||||
|
||||
```toml
|
||||
endpoint = "minio.example.com" # Endpoint for MinIO or S3
|
||||
access_key_id = "your_access_key_id" # Access key ID for MinIO or S3
|
||||
secret_access_key = "your_secret_access_key" # Secret access key for MinIO or S3
|
||||
bucket_name = "your_bucket_name" # Bucket name for MinIO or S3
|
||||
endpoint = "s3.example.com" # Endpoint for S3
|
||||
region = "us-east-1" # Region for S3
|
||||
access_key_id = "your_access_key_id" # Access key ID for S3
|
||||
secret_access_key = "your_secret_access_key" # Secret access key for S3
|
||||
bucket_name = "your_bucket_name" # Bucket name for S3
|
||||
use_ssl = true # Whether to use SSL, default is true
|
||||
base_path = "/path/to/minio" # Base path in MinIO, all files will be stored under this path
|
||||
base_path = "/path/to/s3" # Base path in S3, all files will be stored under this path
|
||||
```
|
||||
|
||||
## Telegram
|
||||
|
||||
@@ -23,7 +23,7 @@ title: 介绍
|
||||
- 使用 js 编写解析器插件以转存任意网站的文件
|
||||
- 存储端支持:
|
||||
- Alist
|
||||
- S3 (MinioSDK)
|
||||
- S3
|
||||
- WebDAV
|
||||
- 本地磁盘
|
||||
- Telegram (重传回指定聊天)
|
||||
|
||||
@@ -93,7 +93,7 @@ session = "data/usersession.db"
|
||||
- `local`: 本地磁盘
|
||||
- `alist`: Alist
|
||||
- `webdav`: WebDAV
|
||||
- `minio`: MinIO (兼容 S3 API)
|
||||
- `s3`: aws S3 及其他兼容 S3 的服务
|
||||
- `telegram`: 上传到 Telegram
|
||||
|
||||
示例, 这是一个包含本地存储和 webdav 存储的配置:
|
||||
|
||||
@@ -41,17 +41,18 @@ password = "your_password" # WebDAV 的密码
|
||||
base_path = "/path/to/webdav" # WebDAV 中的基础路径, 所有文件将存储在此路径下
|
||||
```
|
||||
|
||||
## MinIO (S3)
|
||||
## S3
|
||||
|
||||
`type=minio`
|
||||
`type=s3`
|
||||
|
||||
```toml
|
||||
endpoint = "minio.example.com" # MinIO 或 S3 的端点
|
||||
access_key_id = "your_access_key_id" # MinIO 或 S3 的访问密钥 ID
|
||||
secret_access_key = "your_secret_access_key" # MinIO 或 S3 的秘密访问密钥
|
||||
bucket_name = "your_bucket_name" # MinIO 或 S3 的存储桶名称
|
||||
endpoint = "s3.example.com" # S3 的端点
|
||||
region = "us-east-1" # S3 的区域
|
||||
access_key_id = "your_access_key_id" # S3 的访问密钥 ID
|
||||
secret_access_key = "your_secret_access_key" # S3 的秘密访问密钥
|
||||
bucket_name = "your_bucket_name" # S3 的存储桶名称
|
||||
use_ssl = true # 是否使用 SSL, 默认为 true
|
||||
base_path = "/path/to/minio" # MinIO 中的基础路径, 所有文件将存储在此路径下
|
||||
base_path = "/path/to/s3" # S3 中的基础路径, 所有文件将存储在此路径下
|
||||
```
|
||||
|
||||
## Telegram
|
||||
|
||||
36
go.mod
36
go.mod
@@ -3,6 +3,10 @@ module github.com/krau/SaveAny-Bot
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
|
||||
github.com/blang/semver v3.5.1+incompatible
|
||||
github.com/celestix/gotgproto v1.0.0-beta22
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
@@ -10,7 +14,8 @@ require (
|
||||
github.com/gabriel-vasile/mimetype v1.4.10
|
||||
github.com/goccy/go-yaml v1.18.0
|
||||
github.com/gotd/contrib v0.21.1
|
||||
github.com/gotd/td v0.132.0
|
||||
github.com/gotd/td v0.136.0
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3
|
||||
github.com/krau/ffmpeg-go v0.6.0
|
||||
github.com/minio/minio-go/v7 v7.0.95
|
||||
github.com/playwright-community/playwright-go v0.5200.1
|
||||
@@ -26,6 +31,21 @@ require (
|
||||
|
||||
require (
|
||||
github.com/AnimeKaizoku/cacher v1.0.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.3.2 // indirect
|
||||
@@ -42,7 +62,7 @@ require (
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/glebarez/go-sqlite v1.22.0 // indirect
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/go-faster/jx v1.1.0 // indirect
|
||||
github.com/go-faster/jx v1.2.0 // indirect
|
||||
github.com/go-faster/xor v1.0.0 // indirect
|
||||
github.com/go-faster/yaml v0.4.6 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
@@ -76,6 +96,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
|
||||
github.com/segmentio/asm v1.2.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/tetratelabs/wazero v1.10.1 // indirect
|
||||
@@ -84,12 +105,13 @@ require (
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
modernc.org/libc v1.66.10 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
@@ -105,7 +127,7 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/glebarez/sqlite v1.11.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.1 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/ncruces/go-sqlite3 v0.30.1
|
||||
github.com/ncruces/go-sqlite3/gormlite v0.30.1
|
||||
@@ -117,7 +139,7 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
go.uber.org/multierr v1.11.0
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/sync v0.18.0
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
|
||||
64
go.sum
64
go.sum
@@ -4,6 +4,46 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg
|
||||
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3 h1:cpz7H2uMNTDa0h/5CYL5dLUEzPSLo2g0NkbxTRJtSSU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3/go.mod h1:srtPKaJJe3McW6T/+GMBZyIPc+SeqJsNPJsd4mOYZ6s=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 h1:d/6xOGIllc/XW1lzG9a4AUBMmpLA9PXcQnVPTuHHcik=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3/go.mod h1:fQ7E7Qj9GiW8y0ClD7cUJk3Bz5Iw8wZkWDHsTe8vDKs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
@@ -14,6 +54,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
|
||||
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
||||
github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI=
|
||||
github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
@@ -66,6 +108,8 @@ github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AY
|
||||
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
|
||||
github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
|
||||
github.com/go-faster/jx v1.2.0 h1:T2YHJPrFaYu21fJtUxC9GzmluKu8rVIFDwwGBKTDseI=
|
||||
github.com/go-faster/jx v1.2.0/go.mod h1:UWLOVDmMG597a5tBFPLIWJdUxz5/2emOpfsj9Neg0PE=
|
||||
github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
|
||||
github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
|
||||
github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
|
||||
@@ -113,6 +157,8 @@ github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
|
||||
github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ=
|
||||
github.com/gotd/td v0.132.0 h1:Iqm3S2b+8kDgA9237IDXRxj7sryUpvy+4Cr50/0tpx4=
|
||||
github.com/gotd/td v0.132.0/go.mod h1:4CDGYS+rDtOqotRheGaF9MS5g6jaUewvSXqBNJnx8SQ=
|
||||
github.com/gotd/td v0.136.0 h1:f7vx/1rlvP59L5EKR820XpMRO2k267wW8/F0rAWbepc=
|
||||
github.com/gotd/td v0.136.0/go.mod h1:mStcqs/9FXhNhWnPTguptSwqkQbRIwXLw3SCSpzPJxM=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
@@ -121,8 +167,12 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3 h1:2713fQZ560HxoNVgfJH41GKzjMjIG+DW4hH6nYXfXW8=
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE=
|
||||
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
|
||||
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -183,6 +233,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
||||
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
|
||||
@@ -220,6 +272,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu
|
||||
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c h1:xA2TJS9Hu/ivzaZIrDcwvpJ3Fnpsk5fDOJ4iSnL6J0w=
|
||||
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c/go.mod h1:WSZ59bidJOO40JSJmLqlkBJrjZCtjbKKkygEMfzY/kc=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
@@ -228,6 +282,8 @@ go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgf
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8=
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
@@ -236,6 +292,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
@@ -249,6 +307,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
@@ -298,12 +358,16 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
||||
@@ -4,6 +4,6 @@ package storage
|
||||
|
||||
// StorageType
|
||||
/* ENUM(
|
||||
local, webdav, alist, minio, telegram
|
||||
local, webdav, alist, minio, telegram, s3
|
||||
) */
|
||||
type StorageType string
|
||||
|
||||
@@ -22,6 +22,8 @@ const (
|
||||
Minio StorageType = "minio"
|
||||
// Telegram is a StorageType of type telegram.
|
||||
Telegram StorageType = "telegram"
|
||||
// S3 is a StorageType of type s3.
|
||||
S3 StorageType = "s3"
|
||||
)
|
||||
|
||||
var ErrInvalidStorageType = fmt.Errorf("not a valid StorageType, try [%s]", strings.Join(_StorageTypeNames, ", "))
|
||||
@@ -32,6 +34,7 @@ var _StorageTypeNames = []string{
|
||||
string(Alist),
|
||||
string(Minio),
|
||||
string(Telegram),
|
||||
string(S3),
|
||||
}
|
||||
|
||||
// StorageTypeNames returns a list of possible string values of StorageType.
|
||||
@@ -49,6 +52,7 @@ func StorageTypeValues() []StorageType {
|
||||
Alist,
|
||||
Minio,
|
||||
Telegram,
|
||||
S3,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +74,7 @@ var _StorageTypeValue = map[string]StorageType{
|
||||
"alist": Alist,
|
||||
"minio": Minio,
|
||||
"telegram": Telegram,
|
||||
"s3": S3,
|
||||
}
|
||||
|
||||
// ParseStorageType attempts to convert a string to a StorageType.
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
config "github.com/krau/SaveAny-Bot/config/storage"
|
||||
@@ -16,6 +17,10 @@ import (
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
|
||||
var (
|
||||
deprecatedOnce sync.Once
|
||||
)
|
||||
|
||||
type Minio struct {
|
||||
config config.MinioStorageConfig
|
||||
client *minio.Client
|
||||
@@ -23,6 +28,9 @@ type Minio struct {
|
||||
}
|
||||
|
||||
func (m *Minio) Init(ctx context.Context, cfg config.StorageConfig) error {
|
||||
deprecatedOnce.Do(func() {
|
||||
log.FromContext(ctx).Warn("Minio storage is deprecated, please use S3 storage type instead.")
|
||||
})
|
||||
minioConfig, ok := cfg.(*config.MinioStorageConfig)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to cast minio config")
|
||||
@@ -73,7 +81,7 @@ func (m *Minio) Save(ctx context.Context, r io.Reader, storagePath string) error
|
||||
candidate := storagePath
|
||||
for i := 1; m.Exists(ctx, candidate); i++ {
|
||||
candidate = fmt.Sprintf("%s_%d%s", base, i, ext)
|
||||
if i > 1000 {
|
||||
if i > 100 {
|
||||
m.logger.Errorf("Too many attempts to find a unique filename for %s", storagePath)
|
||||
candidate = fmt.Sprintf("%s_%s%s", base, xid.New().String(), ext)
|
||||
break
|
||||
|
||||
162
storage/s3/s3.go
Normal file
162
storage/s3/s3.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/charmbracelet/log"
|
||||
storconfig "github.com/krau/SaveAny-Bot/config/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
config storconfig.S3StorageConfig
|
||||
client *s3.Client
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func (m *S3) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
|
||||
s3Config, ok := cfg.(*storconfig.S3StorageConfig)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to cast s3 config")
|
||||
}
|
||||
if err := s3Config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.config = *s3Config
|
||||
m.logger = log.FromContext(ctx).WithPrefix(fmt.Sprintf("s3[%s]", m.config.Name))
|
||||
loadOpts := make([]config.LoadOptionsFunc, 0)
|
||||
if m.config.Region != "" {
|
||||
loadOpts = append(loadOpts, config.WithRegion(m.config.Region))
|
||||
}
|
||||
if endpoint := m.config.Endpoint; endpoint != "" {
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
if m.config.UseSSL {
|
||||
endpoint = "https://" + endpoint
|
||||
} else {
|
||||
endpoint = "http://" + endpoint
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := url.Parse(endpoint); err != nil {
|
||||
return fmt.Errorf("invalid s3 endpoint %q: %w", m.config.Endpoint, err)
|
||||
}
|
||||
loadOpts = append(loadOpts, config.WithBaseEndpoint(endpoint))
|
||||
}
|
||||
loadOpts = append(loadOpts, config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
m.config.AccessKeyID,
|
||||
m.config.SecretAccessKey,
|
||||
"",
|
||||
),
|
||||
))
|
||||
awsCfg, err := config.LoadDefaultConfig(
|
||||
ctx,
|
||||
func() []func(*config.LoadOptions) error {
|
||||
// wtf aws sdk
|
||||
// https://github.com/aws/aws-sdk-go-v2/issues/2193
|
||||
funcs := make([]func(*config.LoadOptions) error, 0)
|
||||
for _, fn := range loadOpts {
|
||||
funcs = append(funcs, fn)
|
||||
}
|
||||
return funcs
|
||||
}()...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
m.client = s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
// Path style: https://s3.amazonaws.com/mybucket/path/to/file.jpg
|
||||
// virtual hosted style: https://mybucket.s3.amazonaws.com/path/to/file.jpg
|
||||
o.UsePathStyle = !m.config.VirtualHost
|
||||
})
|
||||
|
||||
// Check if bucket exists
|
||||
_, err = m.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("bucket %s not accessible: %w", m.config.BucketName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *S3) Type() storenum.StorageType {
|
||||
return storenum.S3
|
||||
}
|
||||
|
||||
func (m *S3) Name() string {
|
||||
return m.config.Name
|
||||
}
|
||||
|
||||
func (m *S3) JoinStoragePath(p string) string {
|
||||
return strings.TrimPrefix(path.Join(m.config.BasePath, p), "/")
|
||||
}
|
||||
|
||||
func (m *S3) Save(ctx context.Context, r io.Reader, storagePath string) error {
|
||||
m.logger.Infof("Saving file from reader to %s", storagePath)
|
||||
|
||||
ext := path.Ext(storagePath)
|
||||
base := strings.TrimSuffix(storagePath, ext)
|
||||
candidate := storagePath
|
||||
|
||||
// Unique filename
|
||||
for i := 1; m.Exists(ctx, candidate); i++ {
|
||||
candidate = fmt.Sprintf("%s_%d%s", base, i, ext)
|
||||
if i > 100 {
|
||||
m.logger.Errorf("Too many attempts for unique filename: %s", storagePath)
|
||||
candidate = fmt.Sprintf("%s_%s%s", base, xid.New().String(), ext)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Determine content length
|
||||
size := int64(-1)
|
||||
if length := ctx.Value(ctxkey.ContentLength); length != nil {
|
||||
if l, ok := length.(int64); ok && l > 0 {
|
||||
size = l
|
||||
}
|
||||
}
|
||||
|
||||
// S3 PutObject needs either size or StreamingBody
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
Key: aws.String(candidate),
|
||||
Body: r,
|
||||
}
|
||||
|
||||
if size >= 0 {
|
||||
input.ContentLength = &size
|
||||
}
|
||||
|
||||
_, err := m.client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file to S3: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *S3) Exists(ctx context.Context, storagePath string) bool {
|
||||
m.logger.Debugf("Checking if file exists at %s", storagePath)
|
||||
|
||||
_, err := m.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
Key: aws.String(storagePath),
|
||||
})
|
||||
|
||||
return err == nil
|
||||
}
|
||||
72
storage/s3/s3_test.go
Normal file
72
storage/s3/s3_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/johannesboyne/gofakes3"
|
||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
||||
storconfig "github.com/krau/SaveAny-Bot/config/storage"
|
||||
)
|
||||
|
||||
func newTestContext(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
logger := log.NewWithOptions(nil, log.Options{ReportTimestamp: false})
|
||||
ctx := context.Background()
|
||||
return log.WithContext(ctx, logger)
|
||||
}
|
||||
|
||||
func newFakeS3(t *testing.T) (*S3, *storconfig.S3StorageConfig) {
|
||||
t.Helper()
|
||||
|
||||
backend := s3mem.New()
|
||||
fakeSrv := gofakes3.New(backend)
|
||||
ts := httptest.NewServer(fakeSrv.Server())
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
cfg := &storconfig.S3StorageConfig{
|
||||
BaseConfig: storconfig.BaseConfig{
|
||||
Name: "test-s3",
|
||||
Type: "s3",
|
||||
Enable: true,
|
||||
},
|
||||
Endpoint: ts.URL,
|
||||
AccessKeyID: "test-access-key",
|
||||
SecretAccessKey: "test-secret",
|
||||
BucketName: "test-bucket",
|
||||
BasePath: "base",
|
||||
Region: "us-east-1",
|
||||
}
|
||||
|
||||
if err := backend.CreateBucket("test-bucket"); err != nil {
|
||||
t.Fatalf("failed to create fake bucket: %v", err)
|
||||
}
|
||||
|
||||
s := &S3{}
|
||||
ctx := newTestContext(t)
|
||||
if err := s.Init(ctx, cfg); err != nil {
|
||||
t.Fatalf("init s3 failed: %v", err)
|
||||
}
|
||||
|
||||
return s, cfg
|
||||
}
|
||||
|
||||
func TestS3_SaveAndExists(t *testing.T) {
|
||||
s, _ := newFakeS3(t)
|
||||
ctx := context.Background()
|
||||
|
||||
content := []byte("hello world")
|
||||
reader := bytes.NewReader(content)
|
||||
key := "foo/bar.txt"
|
||||
|
||||
if err := s.Save(ctx, reader, key); err != nil {
|
||||
t.Fatalf("Save failed: %v", err)
|
||||
}
|
||||
|
||||
if !s.Exists(ctx, key) {
|
||||
t.Fatalf("Exists should return true for saved key")
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/storage/alist"
|
||||
"github.com/krau/SaveAny-Bot/storage/local"
|
||||
"github.com/krau/SaveAny-Bot/storage/minio"
|
||||
"github.com/krau/SaveAny-Bot/storage/s3"
|
||||
"github.com/krau/SaveAny-Bot/storage/telegram"
|
||||
"github.com/krau/SaveAny-Bot/storage/webdav"
|
||||
)
|
||||
@@ -37,6 +38,7 @@ var storageConstructors = map[storenum.StorageType]StorageConstructor{
|
||||
storenum.Local: func() Storage { return new(local.Local) },
|
||||
storenum.Webdav: func() Storage { return new(webdav.Webdav) },
|
||||
storenum.Minio: func() Storage { return new(minio.Minio) },
|
||||
storenum.S3: func() Storage { return new(s3.S3) },
|
||||
storenum.Telegram: func() Storage { return new(telegram.Telegram) },
|
||||
}
|
||||
|
||||
|
||||
@@ -103,8 +103,8 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
if filename == "" {
|
||||
filename = xid.New().String() + mtype.Extension()
|
||||
}
|
||||
peer := tctx.PeerStorage.GetInputPeerById(chatID)
|
||||
if peer == nil {
|
||||
peer := tryGetInputPeer(tctx, chatID)
|
||||
if peer == nil || peer.Zero() {
|
||||
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,9 @@ import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/gotd/td/constant"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/ffmpeg-go"
|
||||
"github.com/yapingcat/gomedia/go-mp4"
|
||||
)
|
||||
@@ -133,3 +136,28 @@ func extractFrameAt(rs io.ReadSeeker, timestamp float64) ([]byte, error) {
|
||||
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
func tryGetInputPeer(ctx *ext.Context, chatID int64) tg.InputPeerClass {
|
||||
peer := ctx.PeerStorage.GetInputPeerById(chatID)
|
||||
if peer != nil && !peer.Zero() {
|
||||
return peer
|
||||
}
|
||||
id := constant.TDLibPeerID(chatID)
|
||||
plain := id.ToPlain()
|
||||
var channel constant.TDLibPeerID
|
||||
channel.Channel(plain)
|
||||
peer = ctx.PeerStorage.GetInputPeerById(int64(channel))
|
||||
if peer != nil && !peer.Zero() {
|
||||
return peer
|
||||
}
|
||||
var chat constant.TDLibPeerID
|
||||
chat.Chat(plain)
|
||||
peer = ctx.PeerStorage.GetInputPeerById(int64(chat))
|
||||
if peer != nil && !peer.Zero() {
|
||||
return peer
|
||||
}
|
||||
var user constant.TDLibPeerID
|
||||
user.User(plain)
|
||||
peer = ctx.PeerStorage.GetInputPeerById(int64(user))
|
||||
return peer
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user