mirror of
https://github.com/krau/SaveAny-Bot.git
synced 2026-08-07 05:23:22 +08:00
Compare commits
7 Commits
refactor/p
...
v0.59.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d794a7b9c | ||
|
|
52f880f0f2 | ||
|
|
fc11ca775f | ||
|
|
056e2fd546 | ||
|
|
c9bb6c9e3c | ||
|
|
2bc460c609 | ||
|
|
f02860ff3f |
@@ -7,6 +7,8 @@ ARG BuildTime="Unknown"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
@@ -31,5 +33,9 @@ FROM scratch
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/saveany-bot .
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
|
||||
ENV SSL_CERT_DIR=/etc/ssl/certs
|
||||
|
||||
ENTRYPOINT ["/app/saveany-bot"]
|
||||
|
||||
@@ -60,6 +60,7 @@ func resolveChatID(_ context.Context, idOrUsername string) (int64, error) {
|
||||
}
|
||||
|
||||
// ParseMessageLink 解析 Telegram 消息链接
|
||||
// 支持的域名: t.me, telegram.me
|
||||
// 支持格式:
|
||||
// - https://t.me/username/123
|
||||
// - https://t.me/c/123456789/123
|
||||
@@ -268,5 +269,15 @@ func ExtractFilesFromLinks(ctx context.Context, links []string) ([]tfile.TGFileM
|
||||
|
||||
// isValidMessageLink 检查是否是有效的 Telegram 消息链接
|
||||
func isValidMessageLink(link string) bool {
|
||||
return strings.HasPrefix(link, "https://t.me/") || strings.HasPrefix(link, "http://t.me/")
|
||||
for _, prefix := range []string{
|
||||
"https://t.me/",
|
||||
"http://t.me/",
|
||||
"https://telegram.me/",
|
||||
"http://telegram.me/",
|
||||
} {
|
||||
if strings.HasPrefix(link, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/strutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/database"
|
||||
"github.com/krau/SaveAny-Bot/pkg/rule"
|
||||
)
|
||||
@@ -84,6 +85,46 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error {
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleInfoCreateRuleSuccess, nil)), nil)
|
||||
case "preset":
|
||||
// /rule preset <storage> [base_path]
|
||||
if len(args) < 3 {
|
||||
ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildRuleHelpStyling(user.ApplyRule, user.Rules)), nil)
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
storageName := args[2]
|
||||
if !config.C().HasStorage(user.ChatID, storageName) {
|
||||
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorStorageNotFound, map[string]any{
|
||||
"Storage": storageName,
|
||||
})), nil)
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
basePath := ""
|
||||
if len(args) >= 4 {
|
||||
basePath = args[3]
|
||||
}
|
||||
presets := rule.PresetCategories(basePath)
|
||||
imported := 0
|
||||
for _, p := range presets {
|
||||
rd := &database.Rule{
|
||||
Type: rule.FileNameRegex.String(),
|
||||
Data: p.Regex,
|
||||
StorageName: storageName,
|
||||
DirPath: p.Dir,
|
||||
UserID: user.ID,
|
||||
}
|
||||
if err := database.CreateRule(ctx, rd); err != nil {
|
||||
logger.Errorf("failed to create preset rule %s: %s", p.Name, err)
|
||||
continue
|
||||
}
|
||||
imported++
|
||||
}
|
||||
if imported == 0 {
|
||||
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorCreateRuleFailed, nil)), nil)
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleInfoPresetImported, map[string]any{
|
||||
"Count": imported,
|
||||
})), nil)
|
||||
case "del":
|
||||
// /rule del <id>
|
||||
if len(args) < 3 {
|
||||
|
||||
@@ -24,6 +24,8 @@ func BuildRuleHelpStyling(enabled bool, rules []database.Rule) []styling.StyledT
|
||||
styling.Plain(i18n.T(i18nk.BotMsgRuleHelpSwitchSuffix, nil)),
|
||||
styling.Code("add"),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgRuleHelpAddSuffix, nil)),
|
||||
styling.Code("preset"),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgRuleHelpPresetSuffix, nil)),
|
||||
styling.Code("del"),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgRuleHelpDelSuffix, nil)),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgRuleHelpExistingRulesPrefix, nil)),
|
||||
|
||||
@@ -3,7 +3,7 @@ package re
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
TgMessageLinkRegexString = `https?://t\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
|
||||
TgMessageLinkRegexString = `https?://(?:t|telegram)\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
|
||||
TgMessageLinkRegexp = regexp.MustCompile(TgMessageLinkRegexString)
|
||||
TelegraphUrlRegexString = `https://telegra.ph/.*`
|
||||
TelegraphUrlRegexp = regexp.MustCompile(TelegraphUrlRegexString)
|
||||
|
||||
@@ -84,8 +84,8 @@ const (
|
||||
BotMsgCommonPromptSelectDefaultDir Key = "bot.msg.common.prompt_select_default_dir"
|
||||
BotMsgCommonPromptSelectDefaultStorage Key = "bot.msg.common.prompt_select_default_storage"
|
||||
BotMsgCommonPromptSelectDir Key = "bot.msg.common.prompt_select_dir"
|
||||
BotMsgConfigButtonFilenameStrategy Key = "bot.msg.config.button_filename_strategy"
|
||||
BotMsgConfigButtonConflictStrategy Key = "bot.msg.config.button_conflict_strategy"
|
||||
BotMsgConfigButtonFilenameStrategy Key = "bot.msg.config.button_filename_strategy"
|
||||
BotMsgConfigConflictStrategyAsk Key = "bot.msg.config.conflict_strategy_ask"
|
||||
BotMsgConfigConflictStrategyOverwrite Key = "bot.msg.config.conflict_strategy_overwrite"
|
||||
BotMsgConfigConflictStrategyRename Key = "bot.msg.config.conflict_strategy_rename"
|
||||
@@ -93,8 +93,8 @@ const (
|
||||
BotMsgConfigErrorInvalidCallbackData Key = "bot.msg.config.error_invalid_callback_data"
|
||||
BotMsgConfigErrorInvalidTemplate Key = "bot.msg.config.error_invalid_template"
|
||||
BotMsgConfigFnametmplHelp Key = "bot.msg.config.fnametmpl_help"
|
||||
BotMsgConfigInfoCurrentTemplatePrefix Key = "bot.msg.config.info_current_template_prefix"
|
||||
BotMsgConfigInfoConflictStrategySet Key = "bot.msg.config.info_conflict_strategy_set"
|
||||
BotMsgConfigInfoCurrentTemplatePrefix Key = "bot.msg.config.info_current_template_prefix"
|
||||
BotMsgConfigInfoFilenameStrategySet Key = "bot.msg.config.info_filename_strategy_set"
|
||||
BotMsgConfigInfoTemplateUpdated Key = "bot.msg.config.info_template_updated"
|
||||
BotMsgConfigPromptSelectConflictStrategy Key = "bot.msg.config.prompt_select_conflict_strategy"
|
||||
@@ -200,6 +200,7 @@ const (
|
||||
BotMsgRuleErrorGetUserRulesFailed Key = "bot.msg.rule.error_get_user_rules_failed"
|
||||
BotMsgRuleErrorInvalidRuleId Key = "bot.msg.rule.error_invalid_rule_id"
|
||||
BotMsgRuleErrorInvalidRuleType Key = "bot.msg.rule.error_invalid_rule_type"
|
||||
BotMsgRuleErrorStorageNotFound Key = "bot.msg.rule.error_storage_not_found"
|
||||
BotMsgRuleErrorUpdateUserFailed Key = "bot.msg.rule.error_update_user_failed"
|
||||
BotMsgRuleHelpAddSuffix Key = "bot.msg.rule.help_add_suffix"
|
||||
BotMsgRuleHelpAvailableOps Key = "bot.msg.rule.help_available_ops"
|
||||
@@ -207,13 +208,16 @@ const (
|
||||
BotMsgRuleHelpCurrentModeEnabled Key = "bot.msg.rule.help_current_mode_enabled"
|
||||
BotMsgRuleHelpDelSuffix Key = "bot.msg.rule.help_del_suffix"
|
||||
BotMsgRuleHelpExistingRulesPrefix Key = "bot.msg.rule.help_existing_rules_prefix"
|
||||
BotMsgRuleHelpPresetSuffix Key = "bot.msg.rule.help_preset_suffix"
|
||||
BotMsgRuleHelpSwitchSuffix Key = "bot.msg.rule.help_switch_suffix"
|
||||
BotMsgRuleHelpUsage Key = "bot.msg.rule.help_usage"
|
||||
BotMsgRuleInfoCreateRuleSuccess Key = "bot.msg.rule.info_create_rule_success"
|
||||
BotMsgRuleInfoDeleteRuleSuccess Key = "bot.msg.rule.info_delete_rule_success"
|
||||
BotMsgRuleInfoPresetImported Key = "bot.msg.rule.info_preset_imported"
|
||||
BotMsgRuleInfoRuleModeDisabled Key = "bot.msg.rule.info_rule_mode_disabled"
|
||||
BotMsgRuleInfoRuleModeEnabled Key = "bot.msg.rule.info_rule_mode_enabled"
|
||||
BotMsgRulePromptProvideRuleId Key = "bot.msg.rule.prompt_provide_rule_id"
|
||||
BotMsgRulePromptProvideStorageName Key = "bot.msg.rule.prompt_provide_storage_name"
|
||||
BotMsgSaveErrorInvalidIdOrUsername Key = "bot.msg.save.error_invalid_id_or_username"
|
||||
BotMsgSaveHelpText Key = "bot.msg.save_help_text"
|
||||
BotMsgStorageInfoFilenamePrefix Key = "bot.msg.storage.info_filename_prefix"
|
||||
|
||||
@@ -196,7 +196,11 @@ bot:
|
||||
help_switch_suffix: " - Toggle rule mode\n"
|
||||
help_add_suffix: " <type> <data> <storage_name> <path> - Add rule\n"
|
||||
help_del_suffix: " <rule_id> - Delete rule\n"
|
||||
help_preset_suffix: " <storage_name> [base_path] - Import built-in filetype rules (video/image/audio/document/archive)\n"
|
||||
help_existing_rules_prefix: "\nCurrent rules:\n"
|
||||
prompt_provide_storage_name: "Please provide a storage name"
|
||||
error_storage_not_found: "Storage not found: {{.Storage}}"
|
||||
info_preset_imported: "Imported {{.Count}} built-in classification rules into storage {{.Storage}}"
|
||||
dir:
|
||||
error_get_user_dirs_failed: "Failed to get user directories"
|
||||
error_get_user_failed: "Failed to get user"
|
||||
|
||||
@@ -197,7 +197,11 @@ bot:
|
||||
help_switch_suffix: " - 开关规则模式\n"
|
||||
help_add_suffix: " <类型> <数据> <存储名> <路径> - 添加规则\n"
|
||||
help_del_suffix: " <规则ID> - 删除规则\n"
|
||||
help_preset_suffix: " <存储名> [基础路径] - 导入内置文件类型分类规则(视频/图片/音频/文档/压缩包)\n"
|
||||
help_existing_rules_prefix: "\n当前已添加的规则:\n"
|
||||
prompt_provide_storage_name: "请提供存储名称"
|
||||
error_storage_not_found: "未找到存储: {{.Storage}}"
|
||||
info_preset_imported: "已导入 {{.Count}} 条内置分类规则到存储 {{.Storage}}"
|
||||
dir:
|
||||
error_get_user_dirs_failed: "获取用户文件夹失败"
|
||||
error_get_user_failed: "获取用户失败"
|
||||
|
||||
@@ -2,6 +2,7 @@ package tgutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
@@ -359,9 +360,16 @@ func GetGroupedMessages(ctx *ext.Context, chatID int64, msg *tg.Message) ([]*tg.
|
||||
groupedMessages = append(groupedMessages, m)
|
||||
}
|
||||
}
|
||||
sortMessagesByID(groupedMessages)
|
||||
return groupedMessages, nil
|
||||
}
|
||||
|
||||
func sortMessagesByID(messages []*tg.Message) {
|
||||
sort.Slice(messages, func(i, j int) bool {
|
||||
return messages[i].GetID() < messages[j].GetID()
|
||||
})
|
||||
}
|
||||
|
||||
func ExtractMessageEntityUrls(msg *tg.Message) []string {
|
||||
if len(msg.Entities) == 0 {
|
||||
return nil
|
||||
|
||||
18
common/utils/tgutil/message_test.go
Normal file
18
common/utils/tgutil/message_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package tgutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
func TestSortMessagesByID(t *testing.T) {
|
||||
messages := []*tg.Message{{ID: 9}, {ID: 3}, {ID: 7}}
|
||||
sortMessagesByID(messages)
|
||||
want := []int{3, 7, 9}
|
||||
for i := range messages {
|
||||
if messages[i].GetID() != want[i] {
|
||||
t.Fatalf("message %d has ID %d, want %d", i, messages[i].GetID(), want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,36 +14,47 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
"github.com/krau/SaveAny-Bot/pkg/taskevent"
|
||||
"github.com/krau/SaveAny-Bot/storage"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type executionGroup struct {
|
||||
elems []*TaskElement
|
||||
batchSaver storage.StorageBatchSaver
|
||||
}
|
||||
|
||||
func (g executionGroup) usesBatchSaver() bool {
|
||||
return g.batchSaver != nil
|
||||
}
|
||||
|
||||
func (t *Task) Execute(ctx context.Context) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("batch_file[%s]", t.ID))
|
||||
logger.Info("Starting batch file task")
|
||||
t.Progress.OnStart(ctx, t)
|
||||
workers := config.C().Workers
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(workers)
|
||||
for _, elem := range t.elems {
|
||||
eg.Go(func() error {
|
||||
t.processingMu.RLock()
|
||||
if t.processing[elem.ID] != nil {
|
||||
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
|
||||
groups := t.executionGroups()
|
||||
var err error
|
||||
for i := 0; i < len(groups); {
|
||||
if groups[i].usesBatchSaver() {
|
||||
err = t.processBatch(ctx, groups[i])
|
||||
i++
|
||||
} else {
|
||||
end := i + 1
|
||||
for end < len(groups) && !groups[end].usesBatchSaver() {
|
||||
end++
|
||||
}
|
||||
t.processingMu.RUnlock()
|
||||
t.processingMu.Lock()
|
||||
t.processing[elem.ID] = &elem
|
||||
t.processingMu.Unlock()
|
||||
defer func() {
|
||||
t.processingMu.Lock()
|
||||
delete(t.processing, elem.ID)
|
||||
t.processingMu.Unlock()
|
||||
}()
|
||||
return t.processElement(gctx, elem)
|
||||
})
|
||||
elems := make([]*TaskElement, 0, end-i)
|
||||
for _, group := range groups[i:end] {
|
||||
elems = append(elems, group.elems...)
|
||||
}
|
||||
err = t.processElements(ctx, elems)
|
||||
i = end
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
err := eg.Wait()
|
||||
if err != nil {
|
||||
logger.Errorf("Error during batch file processing: %v", err)
|
||||
} else {
|
||||
@@ -53,6 +64,159 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *Task) executionGroups() []executionGroup {
|
||||
groups := make([]executionGroup, 0, len(t.elems))
|
||||
for i := 0; i < len(t.elems); {
|
||||
elem := &t.elems[i]
|
||||
batchSaver, batchCapable := elem.Storage.(storage.StorageBatchSaver)
|
||||
if !batchCapable || elem.sourceGroupKey == "" {
|
||||
groups = append(groups, executionGroup{elems: []*TaskElement{elem}})
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
end := i + 1
|
||||
for end < len(t.elems) {
|
||||
next := &t.elems[end]
|
||||
if next.Storage != elem.Storage || next.sourceGroupKey != elem.sourceGroupKey {
|
||||
break
|
||||
}
|
||||
end++
|
||||
}
|
||||
elems := make([]*TaskElement, 0, end-i)
|
||||
for j := i; j < end; j++ {
|
||||
elems = append(elems, &t.elems[j])
|
||||
}
|
||||
groups = append(groups, executionGroup{elems: elems, batchSaver: batchSaver})
|
||||
i = end
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error {
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
for _, elem := range elems {
|
||||
eg.Go(func() error {
|
||||
if err := t.markProcessing(elem); err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.unmarkProcessing(elem.ID)
|
||||
return t.processElement(gctx, *elem)
|
||||
})
|
||||
}
|
||||
return eg.Wait()
|
||||
}
|
||||
|
||||
func (t *Task) processBatch(ctx context.Context, group executionGroup) error {
|
||||
defer func() {
|
||||
for _, elem := range group.elems {
|
||||
if err := os.Remove(elem.localPath); err != nil && !os.IsNotExist(err) {
|
||||
log.FromContext(ctx).Warnf("Failed to cleanup batch cache file %s: %v", elem.localPath, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
for _, elem := range group.elems {
|
||||
eg.Go(func() error {
|
||||
if err := t.markProcessing(elem); err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.unmarkProcessing(elem.ID)
|
||||
return t.downloadElement(gctx, elem)
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
items := make([]storagetypes.BatchItem, 0, len(group.elems))
|
||||
openFiles := make([]*os.File, 0, len(group.elems))
|
||||
defer func() {
|
||||
for _, file := range openFiles {
|
||||
if err := file.Close(); err != nil {
|
||||
log.FromContext(ctx).Warnf("Failed to close batch cache file %s: %v", file.Name(), err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, elem := range group.elems {
|
||||
file, err := os.Open(elem.localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open cache file: %w", err)
|
||||
}
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("failed to get cache file stat: %w", err)
|
||||
}
|
||||
openFiles = append(openFiles, file)
|
||||
items = append(items, storagetypes.BatchItem{
|
||||
Reader: file,
|
||||
StoragePath: elem.Path,
|
||||
Size: stat.Size(),
|
||||
SourceGroupKey: elem.sourceGroupKey,
|
||||
Caption: elem.sourceCaption,
|
||||
PreserveCaption: elem.preserveCaption,
|
||||
})
|
||||
}
|
||||
if err := group.batchSaver.SaveBatch(ctx, items); err != nil {
|
||||
return fmt.Errorf("failed to save batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) markProcessing(elem *TaskElement) error {
|
||||
t.processingMu.Lock()
|
||||
defer t.processingMu.Unlock()
|
||||
if t.processing[elem.ID] != nil {
|
||||
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
|
||||
}
|
||||
t.processing[elem.ID] = elem
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) unmarkProcessing(id string) {
|
||||
t.processingMu.Lock()
|
||||
delete(t.processing, id)
|
||||
t.processingMu.Unlock()
|
||||
}
|
||||
|
||||
func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
|
||||
logger.Info("Starting file download")
|
||||
localFile, err := fsutil.CreateFile(elem.localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create local file: %w", err)
|
||||
}
|
||||
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
|
||||
downloaded := t.downloaded.Add(int64(n))
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
taskevent.Emit(ctx, taskevent.Event{
|
||||
TaskID: t.ID,
|
||||
Phase: taskevent.PhaseProgress,
|
||||
TotalBytes: t.totalSize,
|
||||
DownloadedBytes: downloaded,
|
||||
})
|
||||
})
|
||||
_, downloadErr := tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
|
||||
closeErr := localFile.Close()
|
||||
if downloadErr != nil {
|
||||
return fmt.Errorf("failed to download file: %w", downloadErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("failed to close cache file: %w", closeErr)
|
||||
}
|
||||
logger.Info("File downloaded successfully")
|
||||
if path.Ext(elem.FileName()) == "" {
|
||||
if ext := fsutil.DetectFileExt(elem.localPath); ext != "" {
|
||||
elem.Path += ext
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
|
||||
if elem.stream {
|
||||
|
||||
57
core/tasks/batchtfile/execute_group_test.go
Normal file
57
core/tasks/batchtfile/execute_group_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package batchtfile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
tgstorage "github.com/krau/SaveAny-Bot/storage/telegram"
|
||||
)
|
||||
|
||||
func TestExecutionGroupsPreserveSourceAlbums(t *testing.T) {
|
||||
stor := new(tgstorage.Telegram)
|
||||
otherStor := new(tgstorage.Telegram)
|
||||
task := Task{elems: []TaskElement{
|
||||
{Storage: stor, sourceGroupKey: "album-1"},
|
||||
{Storage: stor, sourceGroupKey: "album-1"},
|
||||
{Storage: stor},
|
||||
{Storage: stor, sourceGroupKey: "album-2"},
|
||||
{Storage: stor, sourceGroupKey: "album-2"},
|
||||
{Storage: otherStor, sourceGroupKey: "album-2"},
|
||||
}}
|
||||
|
||||
groups := task.executionGroups()
|
||||
wantSizes := []int{2, 1, 2, 1}
|
||||
wantBatch := []bool{true, false, true, true}
|
||||
if len(groups) != len(wantSizes) {
|
||||
t.Fatalf("got %d groups, want %d", len(groups), len(wantSizes))
|
||||
}
|
||||
for i := range groups {
|
||||
if got := len(groups[i].elems); got != wantSizes[i] {
|
||||
t.Errorf("group %d has %d elements, want %d", i, got, wantSizes[i])
|
||||
}
|
||||
if got := groups[i].usesBatchSaver(); got != wantBatch[i] {
|
||||
t.Errorf("group %d batch=%v, want %v", i, got, wantBatch[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceMetadataPreservesAlbumIdentityAndCaption(t *testing.T) {
|
||||
msg := &tg.Message{
|
||||
PeerID: &tg.PeerChannel{ChannelID: 77},
|
||||
Message: "original caption",
|
||||
}
|
||||
msg.SetGroupedID(42)
|
||||
file := tfile.NewTGFile(nil, nil, 0, "photo.jpg", tfile.WithMessage(msg))
|
||||
|
||||
groupKey, caption, preserveCaption := sourceMetadata(file)
|
||||
if groupKey != "*tg.PeerChannel:77:42" {
|
||||
t.Fatalf("group key = %q, want %q", groupKey, "*tg.PeerChannel:77:42")
|
||||
}
|
||||
if caption != "original caption" {
|
||||
t.Fatalf("caption = %q, want original caption", caption)
|
||||
}
|
||||
if !preserveCaption {
|
||||
t.Fatal("preserveCaption = false, want true")
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/core"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
|
||||
@@ -18,12 +19,15 @@ import (
|
||||
var _ core.Executable = (*Task)(nil)
|
||||
|
||||
type TaskElement struct {
|
||||
ID string
|
||||
Storage storage.Storage
|
||||
Path string
|
||||
File tfile.TGFile
|
||||
localPath string
|
||||
stream bool
|
||||
ID string
|
||||
Storage storage.Storage
|
||||
Path string
|
||||
File tfile.TGFile
|
||||
localPath string
|
||||
stream bool
|
||||
sourceGroupKey string
|
||||
sourceCaption string
|
||||
preserveCaption bool
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
@@ -54,6 +58,7 @@ func NewTaskElement(
|
||||
file tfile.TGFile,
|
||||
) (*TaskElement, error) {
|
||||
id := xid.New().String()
|
||||
groupKey, caption, preserveCaption := sourceMetadata(file)
|
||||
_, ok := stor.(storage.StorageCannotStream)
|
||||
if !config.C().Stream || ok {
|
||||
cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name())))
|
||||
@@ -61,22 +66,42 @@ func NewTaskElement(
|
||||
return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
|
||||
}
|
||||
return &TaskElement{
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
localPath: cachePath,
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
localPath: cachePath,
|
||||
sourceGroupKey: groupKey,
|
||||
sourceCaption: caption,
|
||||
preserveCaption: preserveCaption,
|
||||
}, nil
|
||||
}
|
||||
return &TaskElement{
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
stream: true,
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
stream: true,
|
||||
sourceGroupKey: groupKey,
|
||||
sourceCaption: caption,
|
||||
preserveCaption: preserveCaption,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sourceMetadata(file tfile.TGFile) (groupKey, caption string, preserveCaption bool) {
|
||||
messageFile, ok := file.(tfile.TGFileMessage)
|
||||
if !ok || messageFile.Message() == nil {
|
||||
return "", "", false
|
||||
}
|
||||
msg := messageFile.Message()
|
||||
groupID, grouped := msg.GetGroupedID()
|
||||
if !grouped || groupID == 0 {
|
||||
return "", "", false
|
||||
}
|
||||
chatID := tgutil.ChatIdFromPeer(msg.GetPeerID())
|
||||
return fmt.Sprintf("%T:%d:%d", msg.GetPeerID(), chatID, groupID), msg.GetMessage(), true
|
||||
}
|
||||
|
||||
func NewBatchTGFileTask(
|
||||
id string,
|
||||
ctx context.Context,
|
||||
|
||||
@@ -113,6 +113,51 @@ secret = "your-rpc-secret"
|
||||
remove_after_transfer = true
|
||||
```
|
||||
|
||||
### yt-dlp Configuration
|
||||
|
||||
Configures the behavior of the `/ytdlp` command and the `ytdlp` HTTP-API task type when no custom flags are passed.
|
||||
|
||||
- `max_height`: Default maximum video resolution by height in pixels (e.g. `1080`, `720`). `0` means no limit (best available). Ignored when `format` is set.
|
||||
- `format`: A raw yt-dlp format selector (`-f`). When set, it takes precedence over `max_height` and gives you full control, e.g. `bv*[height<=720]+ba/b`.
|
||||
- `recode`: The target video container yt-dlp recodes into after download (e.g. `mp4`). Leave empty to disable recoding.
|
||||
|
||||
{{< hint info >}}
|
||||
These defaults only apply when using the `/ytdlp` command without passing any custom flags. Passing custom flags on the command (or `flags` in the API) overrides them.
|
||||
{{< /hint >}}
|
||||
|
||||
```toml
|
||||
[ytdlp]
|
||||
max_height = 1080
|
||||
format = "" # e.g. "bv*[height<=720]+ba/b"
|
||||
recode = "mp4" # empty disables recoding
|
||||
```
|
||||
|
||||
### HTTP API Configuration
|
||||
|
||||
When enabled, SaveAny-Bot exposes an HTTP API for creating/querying/canceling tasks programmatically. See [HTTP API](../../usage/api) for the full endpoint reference.
|
||||
|
||||
- `enable`: Whether to enable the HTTP API server, default is `false`.
|
||||
- `host`: Bind address, default `0.0.0.0`.
|
||||
- `port`: Listen port, default `8080`.
|
||||
- `token`: Authentication token. **Strongly recommended** — if empty, the API is exposed without any authentication.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
enable = false
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
token = "your-token"
|
||||
```
|
||||
|
||||
### Log Configuration
|
||||
|
||||
- `level`: Log level. One of `debug`, `info`, `warn`, `error`, `fatal`. Default is `info`.
|
||||
|
||||
```toml
|
||||
[log]
|
||||
level = "info"
|
||||
```
|
||||
|
||||
### Storage Endpoints List
|
||||
|
||||
The storage endpoints list is used to define the storage locations supported by the Bot. Each storage endpoint needs to specify a name, type, and related configuration, using the double bracket syntax `[[storages]]`.
|
||||
|
||||
90
docs/content/en/usage/cli.md
Normal file
90
docs/content/en/usage/cli.md
Normal file
@@ -0,0 +1,90 @@
|
||||
---
|
||||
title: "CLI Subcommands"
|
||||
weight: 21
|
||||
---
|
||||
|
||||
# CLI Subcommands
|
||||
|
||||
Besides running the Telegram bot with `./saveany-bot` (no subcommand), the binary exposes two helper subcommands for moving local files into a storage backend: `upload` (one-shot) and `watch` (continuous).
|
||||
|
||||
These subcommands load the same `config.toml` as the bot, initialize the database and caches, then perform their task. They do **not** start the Telegram bot itself, although storages of type `telegram` will spin up the bot client just for the upload.
|
||||
|
||||
## `upload` — Upload a Single File
|
||||
|
||||
```
|
||||
saveany-bot upload -f <file> -s <storage> [-d <dir>] [--no-progress]
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Required | Description |
|
||||
|---|---|---|
|
||||
| `-f, --file` | Yes | Path to the local file to upload |
|
||||
| `-s, --storage` | Yes | Target storage name (must exist in `config.toml`) |
|
||||
| `-d, --dir` | No | Destination directory within the storage. Defaults to the storage's `base_path` |
|
||||
| `--no-progress` | No | Disable the terminal progress bar |
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
# Upload a file to the default dir of storage "MyAlist"
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist
|
||||
|
||||
# Upload into a specific subdirectory
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
|
||||
|
||||
# Upload via Telegram storage without a progress bar
|
||||
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
|
||||
```
|
||||
|
||||
## `watch` — Watch a Directory and Auto-Upload
|
||||
|
||||
The `watch` subcommand continuously monitors a local directory and uploads created or modified files to a storage backend, preserving the relative directory structure from the watch root.
|
||||
|
||||
```
|
||||
saveany-bot watch -p <path> -s <storage> [-d <dir>] [options]
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Default | Description |
|
||||
|---|---|---|
|
||||
| `-p, --path` | *(required)* | Local directory to watch |
|
||||
| `-s, --storage` | *(required)* | Target storage name |
|
||||
| `-d, --dir` | storage's `base_path` | Destination directory within the storage |
|
||||
| `-r, --recursive` | `false` | Watch subdirectories recursively |
|
||||
| `--overwrite` | `false` | Overwrite existing files on the storage instead of skipping them |
|
||||
| `--initial-scan` | `false` | Upload files already present in the directory on startup |
|
||||
| `--debounce` | `2s` | How long to wait after the last write before uploading a file |
|
||||
| `--upload-workers` | `config.workers` | Number of concurrent uploads |
|
||||
| `--retry-delay` | `3s` | Delay between upload retries |
|
||||
|
||||
{{< hint info >}}
|
||||
Write-completion detection: the watcher debounces per file and only uploads once the file size stays unchanged across the debounce window, so partial/write-in-progress files are not uploaded.
|
||||
<br />
|
||||
If a file changes while being uploaded, it is re-uploaded once after the current upload finishes (instead of being queued multiple times).
|
||||
{{< /hint >}}
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
# Watch ./inbox and upload new files to "MyAlist" recursively
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -r
|
||||
|
||||
# Watch with a custom destination dir and overwrite
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
|
||||
|
||||
# On startup, also upload everything already in ./inbox
|
||||
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
|
||||
```
|
||||
|
||||
### Behavior notes
|
||||
|
||||
- Relative directory structure is preserved under the destination directory. A file written to `./inbox/sub/file.txt` with `--path ./inbox` is uploaded to `<dest_dir>/sub/file.txt`.
|
||||
- `watch` runs until interrupted (e.g. `Ctrl-C` / `SIGINT`); in-flight uploads are drained before exit.
|
||||
- Retries follow the global `retry` value from `config.toml`, with `--retry-delay` between attempts.
|
||||
- Telegram-type storages will start the bot client automatically to perform uploads.
|
||||
|
||||
{{< hint warning >}}
|
||||
`watch` is unrelated to the in-bot `/watch` command (which watches Telegram chats). This subcommand watches a **local filesystem directory** and uploads to a storage backend, independent of Telegram.
|
||||
{{< /hint >}}
|
||||
75
docs/content/en/usage/config.md
Normal file
75
docs/content/en/usage/config.md
Normal file
@@ -0,0 +1,75 @@
|
||||
---
|
||||
title: "File Naming & Conflict Strategies"
|
||||
weight: 11
|
||||
---
|
||||
|
||||
# File Naming & Conflict Strategies
|
||||
|
||||
SaveAny-Bot lets you customize how saved files are named and how collisions with existing files are resolved, directly in Telegram via the `/config` and `/fnametmpl` commands.
|
||||
|
||||
## `/config` — User Configuration
|
||||
|
||||
The `/config` command opens an inline menu where you can change two per-user settings:
|
||||
|
||||
- **Filename strategy** — how the saved file is named
|
||||
- **Duplicate file strategy** — what happens when a file with the same name already exists in the target storage
|
||||
|
||||
Settings are stored per user and apply to all of that user's subsequent save/transfer tasks.
|
||||
|
||||
### Filename strategy
|
||||
|
||||
| Option | Behavior |
|
||||
|---|---|
|
||||
| `Default` | Use the original media filename, or a generated name when no original filename is available |
|
||||
| `Gen From Msg First` | Generate the filename from the message content (e.g. caption, text) and prefer that over the original filename |
|
||||
| `Template` | Render the filename from a custom template you define with `/fnametmpl` |
|
||||
|
||||
### Duplicate file strategy
|
||||
|
||||
| Option | Behavior |
|
||||
|---|---|
|
||||
| `Always rename` (default) | Keep the existing file and save the new one with an alternate name |
|
||||
| `Ask every time` | Prompt you with inline buttons each time a collision occurs |
|
||||
| `Always overwrite` | Replace the existing file with the new one |
|
||||
| `Always skip` | Do nothing for conflicting files |
|
||||
|
||||
{{< hint info >}}
|
||||
The conflict strategy only kicks in for storage backends that can detect the existence of a file. Backends that do not support existence checks will fall back to overwriting.
|
||||
{{< /hint >}}
|
||||
|
||||
## `/fnametmpl` — Custom Filename Template
|
||||
|
||||
When the filename strategy is set to `Template`, SaveAny-Bot renders each saved file's name using the template configured via `/fnametmpl`.
|
||||
|
||||
```
|
||||
/fnametmpl [template]
|
||||
```
|
||||
|
||||
- Running `/fnametmpl` without arguments shows your current template and the help text.
|
||||
- Running it with a template string sets that template as your filename template.
|
||||
|
||||
The template uses Go [`text/template`](https://pkg.go.dev/text/template) syntax. The available variables are:
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `{{.msgid}}` | Telegram message ID |
|
||||
| `{{.msgtags}}` | Hashtags found in the message, joined with `_` |
|
||||
| `{{.msggen}}` | Filename generated from the message |
|
||||
| `{{.msgdate}}` | Message date, formatted `YYYY-MM-DD_HH-MM-SS` |
|
||||
| `{{.msgraw}}` | Raw, unprocessed message text |
|
||||
| `{{.origname}}` | The media's original filename (if any) |
|
||||
| `{{.chatid}}` | Chat ID of the message |
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
# Fixed prefix + message id + date
|
||||
/fnametmpl Image_{{.msgid}}_{{.msgdate}}.jpg
|
||||
|
||||
# Use original name if available, otherwise a generated name
|
||||
/fnametmpl {{.origname}}
|
||||
```
|
||||
|
||||
{{< hint warning >}}
|
||||
The template only takes effect when the filename strategy is set to `Template`. If template parsing fails, SaveAny-Bot falls back to the default filename naming logic.
|
||||
{{< /hint >}}
|
||||
@@ -27,6 +27,45 @@ Pay attention to spaces; the bot can only parse correctly formatted syntax. Belo
|
||||
|
||||
In addition, if `CHOSEN` is used as the storage name in the rule, it means files will be stored under the path of the storage you selected by clicking the inline button.
|
||||
|
||||
You can also toggle whether rules are applied with `/rule switch`. When rule mode is off, all files go to the default storage.
|
||||
|
||||
## Preset Rules
|
||||
|
||||
Manually writing regex rules for common file types is tedious, so the bot ships a built-in set of preset categories (video, image, audio, document, archive) that you can import in one command:
|
||||
|
||||
```
|
||||
/rule preset <storage> [base_path]
|
||||
```
|
||||
|
||||
Parameters:
|
||||
|
||||
- `storage`: Target storage name (must exist and be accessible to you)
|
||||
- `base_path`: Optional. Each preset category's subdirectory is created under this path. If omitted, the default category directory names are used directly.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
# Import preset rules into "MyAlist" with the default directory layout
|
||||
/rule preset MyAlist
|
||||
|
||||
# Import preset rules with a custom base path "downloads/sorted"
|
||||
/rule preset MyAlist downloads/sorted
|
||||
```
|
||||
|
||||
This will create `FILENAME-REGEX` rules for each category, routing matched files to the corresponding subdirectory under `base_path`:
|
||||
|
||||
| Category | Matched extensions | Default directory |
|
||||
|---|---|---|
|
||||
| video | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
|
||||
| image | jpg, jpeg, png, gif, webp, bmp | `图片` |
|
||||
| audio | mp3, flac, wav, aac, m4a, ogg | `音频` |
|
||||
| document | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
|
||||
| archive | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
|
||||
|
||||
{{< hint info >}}
|
||||
Preset rules are regular `FILENAME-REGEX` rules once imported. You can view, edit, or delete them individually with `/rule` and `/rule del <id>` like any other rule.
|
||||
{{< /hint >}}
|
||||
|
||||
Rule types:
|
||||
|
||||
## FILENAME-REGEX
|
||||
|
||||
@@ -30,6 +30,7 @@ base_path = "./downloads"
|
||||
|
||||
### 全局配置
|
||||
|
||||
- `lang`: Bot 使用的语言, 默认为 `zh-CN` (简体中文), 设为 `en` 则使用英语.
|
||||
- `stream`: 是否启用 Stream 模式, 默认为 `false`. 启用后 Bot 将直接将文件流式传输到存储端(若存储端支持), 不需要下载到本地
|
||||
{{< hint warning >}}
|
||||
Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一些弊端:
|
||||
@@ -47,6 +48,7 @@ Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一
|
||||
- `proxy`: 全局代理配置, 配置后程序内一切网络连接将会尝试使用该代理, 可选.
|
||||
|
||||
```toml
|
||||
lang = "zh-CN"
|
||||
stream = false
|
||||
workers = 3
|
||||
threads = 4
|
||||
@@ -111,6 +113,51 @@ secret = "your-rpc-secret"
|
||||
remove_after_transfer = true
|
||||
```
|
||||
|
||||
### yt-dlp 配置
|
||||
|
||||
用于配置 `/ytdlp` 命令以及 HTTP API 中 `ytdlp` 任务类型在未传自定义参数时的默认行为.
|
||||
|
||||
- `max_height`: 默认下载的最高视频清晰度 (按高度限制), 如 `1080`, `720`, `480`; `0` 表示不限制 (下载最佳画质). 当设置了 `format` 时此项被忽略.
|
||||
- `format`: 直接指定 yt-dlp format 选择表达式, 设置后优先级高于 `max_height`, 例如 `bv*[height<=720]+ba/b`.
|
||||
- `recode`: 下载后转封装的视频容器格式 (如 `mp4`), 留空则不转封装.
|
||||
|
||||
{{< hint info >}}
|
||||
这些默认值仅在使用 `/ytdlp` 命令且未传任何自定义参数时生效. 在命令上传递自定义参数 (或在 API 中传 `flags`) 会覆盖这些默认值.
|
||||
{{< /hint >}}
|
||||
|
||||
```toml
|
||||
[ytdlp]
|
||||
max_height = 1080
|
||||
format = "" # 例如 "bv*[height<=720]+ba/b"
|
||||
recode = "mp4" # 留空则不转封装
|
||||
```
|
||||
|
||||
### HTTP API 配置
|
||||
|
||||
启用后, SaveAny-Bot 会暴露一套 HTTP API, 用于以编程方式创建/查询/取消任务. 完整的接口说明见 [HTTP API](../../usage/api).
|
||||
|
||||
- `enable`: 是否启用 HTTP API 服务, 默认为 `false`.
|
||||
- `host`: 监听地址, 默认 `0.0.0.0`.
|
||||
- `port`: 监听端口, 默认 `8080`.
|
||||
- `token`: 鉴权 Token, **强烈建议设置** — 若为空, API 将在无任何鉴权的情况下暴露.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
enable = false
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
token = "your-token"
|
||||
```
|
||||
|
||||
### 日志配置
|
||||
|
||||
- `level`: 日志级别, 可选 `debug`, `info`, `warn`, `error`, `fatal`. 默认为 `info`.
|
||||
|
||||
```toml
|
||||
[log]
|
||||
level = "info"
|
||||
```
|
||||
|
||||
### 存储端列表
|
||||
|
||||
存储端列表用于定义 Bot 支持的存储位置, 每个存储端需要指定名称、类型和相关配置, 使用双中括号语法 `[[storages]]` 定义.
|
||||
|
||||
90
docs/content/zh/usage/cli.md
Normal file
90
docs/content/zh/usage/cli.md
Normal file
@@ -0,0 +1,90 @@
|
||||
---
|
||||
title: "命令行子命令"
|
||||
weight: 21
|
||||
---
|
||||
|
||||
# 命令行子命令
|
||||
|
||||
除了直接运行 `./saveany-bot` (不带子命令) 启动 Telegram Bot 外, 这个二进制文件还提供两个把本地文件上传到存储后端的辅助子命令: `upload` (一次性) 和 `watch` (持续监听).
|
||||
|
||||
这些子命令会读取与 Bot 相同的 `config.toml`, 初始化数据库和缓存, 然后执行任务. 它们**不会**启动 Telegram Bot 本身, 但 `telegram` 类型的存储会在需要上传时临时启动 Bot 客户端来执行上传.
|
||||
|
||||
## `upload` — 上传单个文件
|
||||
|
||||
```
|
||||
saveany-bot upload -f <文件> -s <存储名> [-d <目录>] [--no-progress]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
| 参数 | 必填 | 说明 |
|
||||
|---|---|---|
|
||||
| `-f, --file` | 是 | 待上传的本地文件路径 |
|
||||
| `-s, --storage` | 是 | 目标存储名 (必须存在于 `config.toml`) |
|
||||
| `-d, --dir` | 否 | 存储中的目标目录, 默认使用存储的 `base_path` |
|
||||
| `--no-progress` | 否 | 关闭终端进度条 |
|
||||
|
||||
示例:
|
||||
|
||||
```bash
|
||||
# 上传文件到 "MyAlist" 的默认目录
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist
|
||||
|
||||
# 上传到指定子目录
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
|
||||
|
||||
# 通过 Telegram 存储上传并关闭进度条
|
||||
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
|
||||
```
|
||||
|
||||
## `watch` — 监听目录并自动上传
|
||||
|
||||
`watch` 子命令持续监听一个本地目录, 将新建或修改的文件上传到存储后端, 并保留相对监听根目录的子目录结构.
|
||||
|
||||
```
|
||||
saveany-bot watch -p <路径> -s <存储名> [-d <目录>] [选项]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `-p, --path` | *(必填)* | 要监听的本地目录 |
|
||||
| `-s, --storage` | *(必填)* | 目标存储名 |
|
||||
| `-d, --dir` | 存储的 `base_path` | 存储中的目标目录 |
|
||||
| `-r, --recursive` | `false` | 是否递归监听子目录 |
|
||||
| `--overwrite` | `false` | 覆盖存储上已有的文件, 而非跳过 |
|
||||
| `--initial-scan` | `false` | 启动时将目录中已存在的文件也上传 |
|
||||
| `--debounce` | `2s` | 文件最后一次写入后, 等待多久再上传 |
|
||||
| `--upload-workers` | `config.workers` | 并发上传数 |
|
||||
| `--retry-delay` | `3s` | 上传重试之间的延迟 |
|
||||
|
||||
{{< hint info >}}
|
||||
写入完成检测: 监听器会按文件做防抖处理, 仅当文件大小在一个 debounce 窗口内保持不变时才上传, 因此不会上传未写完的半成品文件.
|
||||
<br />
|
||||
若某文件在上传过程中又被修改, 它会在当前上传完成后再上传一次, 而不是被重复排队.
|
||||
{{< /hint >}}
|
||||
|
||||
示例:
|
||||
|
||||
```bash
|
||||
# 递归监听 ./inbox 并且把新文件上传到 "MyAlist"
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -r
|
||||
|
||||
# 自定义目标目录并覆盖已有文件
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
|
||||
|
||||
# 启动时把 ./inbox 中已有的内容也一并上传
|
||||
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
|
||||
```
|
||||
|
||||
### 行为说明
|
||||
|
||||
- 相对子目录结构会被保留: 以 `--path ./inbox` 为例, 写入 `./inbox/sub/file.txt` 的文件会被上传到 `<目标目录>/sub/file.txt`.
|
||||
- `watch` 会一直运行直到被中断 (如 `Ctrl-C` / `SIGINT`), 退出前会等待所有进行中的上传完成.
|
||||
- 重试次数遵循 `config.toml` 中的全局 `retry` 值, 各次重试之间间隔 `--retry-delay`.
|
||||
- `telegram` 类型的存储会自动启动 Bot 客户端来执行上传.
|
||||
|
||||
{{< hint warning >}}
|
||||
`watch` 子命令与 Bot 内的 `/watch` 命令 (监听 Telegram 聊天) 无关. 本子命令监听的是**本地文件系统目录**, 不依赖 Telegram.
|
||||
{{< /hint >}}
|
||||
75
docs/content/zh/usage/config.md
Normal file
75
docs/content/zh/usage/config.md
Normal file
@@ -0,0 +1,75 @@
|
||||
---
|
||||
title: "文件命名与重名策略"
|
||||
weight: 11
|
||||
---
|
||||
|
||||
# 文件命名与重名策略
|
||||
|
||||
SaveAny-Bot 支持在 Telegram 中通过 `/config` 和 `/fnametmpl` 命令自定义保存文件的命名方式, 以及处理与已存在文件重名时的冲突策略.
|
||||
|
||||
## `/config` — 用户配置
|
||||
|
||||
`/config` 命令会弹出一个内联菜单, 你可以在其中修改以下两项用户级设置:
|
||||
|
||||
- **文件名策略** — 保存文件的命名方式
|
||||
- **重名文件保存策略** — 目标存储中已存在同名文件时的处理方式
|
||||
|
||||
设置按用户分别保存, 对该用户后续所有的保存/转存任务生效.
|
||||
|
||||
### 文件名策略
|
||||
|
||||
| 选项 | 行为 |
|
||||
|---|---|
|
||||
| `默认` | 使用媒体原始文件名, 没有原始文件名时使用生成名 |
|
||||
| `优先从消息生成` | 优先根据消息内容 (如 caption、文本) 生成文件名, 而非原始文件名 |
|
||||
| `自定义模板` | 使用 `/fnametmpl` 设置的自定义模板渲染文件名 |
|
||||
|
||||
### 重名文件保存策略
|
||||
|
||||
| 选项 | 行为 |
|
||||
|---|---|
|
||||
| `始终重命名` (默认) | 保留已有文件, 将新文件以另一个名字保存 |
|
||||
| `每次询问` | 每次遇到重名时通过内联按钮提示你选择 |
|
||||
| `始终覆盖` | 用新文件替换已有文件 |
|
||||
| `始终跳过` | 对重名文件不做处理 |
|
||||
|
||||
{{< hint info >}}
|
||||
重名策略仅在能够检测文件是否已存在的存储后端生效. 不支持检测文件是否存在的存储后端会退化为覆盖行为.
|
||||
{{< /hint >}}
|
||||
|
||||
## `/fnametmpl` — 自定义文件名模板
|
||||
|
||||
当文件名策略设置为 `自定义模板` 时, SaveAny-Bot 会用 `/fnametmpl` 配置的模板来渲染所保存文件的文件名.
|
||||
|
||||
```
|
||||
/fnametmpl [模板]
|
||||
```
|
||||
|
||||
- 不带参数运行 `/fnametmpl` 会显示当前模板以及帮助说明
|
||||
- 带模板字符串运行则会把它设为你的文件名模板
|
||||
|
||||
模板使用 Go [`text/template`](https://pkg.go.dev/text/template) 语法. 可用变量如下:
|
||||
|
||||
| 变量 | 说明 |
|
||||
|---|---|
|
||||
| `{{.msgid}}` | Telegram 消息 ID |
|
||||
| `{{.msgtags}}` | 消息中的标签, 以 `_` 连接输出 |
|
||||
| `{{.msggen}}` | 根据消息生成的文件名 |
|
||||
| `{{.msgdate}}` | 消息日期, 格式 `YYYY-MM-DD_HH-MM-SS` |
|
||||
| `{{.msgraw}}` | 消息的原始文本内容 (不做处理) |
|
||||
| `{{.origname}}` | 媒体的原始文件名 (如有) |
|
||||
| `{{.chatid}}` | 消息所在聊天的 ID |
|
||||
|
||||
示例:
|
||||
|
||||
```
|
||||
# 固定前缀 + 消息 ID + 日期
|
||||
/fnametmpl 图片_{{.msgid}}_{{.msgdate}}.jpg
|
||||
|
||||
# 优先使用原始文件名, 没有则用生成名
|
||||
/fnametmpl {{.origname}}
|
||||
```
|
||||
|
||||
{{< hint warning >}}
|
||||
模板仅在文件名策略设置为 `自定义模板` 时生效. 如果模板解析失败, SaveAny-Bot 会回退到默认的文件名生成逻辑.
|
||||
{{< /hint >}}
|
||||
@@ -27,6 +27,45 @@ weight: 3
|
||||
|
||||
此外, 规则中的存储名若使用 "CHOSEN" , 则表示存储到点击按钮选择的存储端的路径下
|
||||
|
||||
你也可以使用 `/rule switch` 来开关规则模式. 关闭规则模式时, 所有文件都将保存到默认存储.
|
||||
|
||||
## 预设规则
|
||||
|
||||
为常见文件类型手动编写正则规则比较繁琐, 因此 Bot 内置了一组预设分类 (视频、图片、音频、文档、压缩包), 可以通过一条命令批量导入:
|
||||
|
||||
```
|
||||
/rule preset <存储名> [基础路径]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
- `存储名`: 目标存储名 (必须存在且你有权访问)
|
||||
- `基础路径`: 可选. 各预设分类的子目录会创建在此路径下; 若不填则直接使用默认分类目录名
|
||||
|
||||
示例:
|
||||
|
||||
```
|
||||
# 导入预设规则到 "MyAlist", 使用默认目录布局
|
||||
/rule preset MyAlist
|
||||
|
||||
# 在自定义基础路径 "downloads/sorted" 下导入预设规则
|
||||
/rule preset MyAlist downloads/sorted
|
||||
```
|
||||
|
||||
此命令会为每个分类创建 `FILENAME-REGEX` 规则, 将匹配的文件路由到 `基础路径` 下对应的子目录:
|
||||
|
||||
| 分类 | 匹配的扩展名 | 默认目录 |
|
||||
|---|---|---|
|
||||
| 视频 | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
|
||||
| 图片 | jpg, jpeg, png, gif, webp, bmp | `图片` |
|
||||
| 音频 | mp3, flac, wav, aac, m4a, ogg | `音频` |
|
||||
| 文档 | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
|
||||
| 压缩包 | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
|
||||
|
||||
{{< hint info >}}
|
||||
导入后的预设规则就是普通的 `FILENAME-REGEX` 规则. 你可以像其他规则一样通过 `/rule` 查看或用 `/rule del <id>` 单独删除/编辑它们.
|
||||
{{< /hint >}}
|
||||
|
||||
规则类型:
|
||||
|
||||
## FILENAME-REGEX
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
{"Target":"book.min.a22f4c7d8c2bdc5e3d6e34ba11cb59ab50ea5772594e71305bfd5a595dc78b7e.css","MediaType":"text/css","Data":{"Integrity":"sha256-oi9MfYwr3F49bjS6EctZq1DqV3JZTnEwW/1aWV3Hi34="}}
|
||||
{"Target":"book.min.a643d39733d3a0ac48d6369128a52703207c5e11a74c3a70cfebfe0c15838ab8.css","MediaType":"text/css","Data":{"Integrity":"sha256-pkPTlzPToKxI1jaRKKUnAyB8XhGnTDpwz+v+DBWDirg="}}
|
||||
55
pkg/rule/preset.go
Normal file
55
pkg/rule/preset.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package rule
|
||||
|
||||
import "path"
|
||||
|
||||
// PresetCategory describes a built-in filetype classification: files whose name
|
||||
// matches Regex are routed into the Dir subdirectory (joined with a user base path).
|
||||
type PresetCategory struct {
|
||||
// Name is a stable identifier for the category (used in logs/messages).
|
||||
Name string
|
||||
// Regex is a FILENAME-REGEX rule data string matching this category's extensions.
|
||||
Regex string
|
||||
// Dir is the default subdirectory name for this category.
|
||||
Dir string
|
||||
}
|
||||
|
||||
// presetCategories holds the default filetype classification rules.
|
||||
// Regexes are case-insensitive and match common file extensions.
|
||||
var presetCategories = []PresetCategory{
|
||||
{
|
||||
Name: "video",
|
||||
Regex: `(?i)\.(mp4|mkv|ts|avi|flv|mov|webm|wmv|rmvb|m2ts)$`,
|
||||
Dir: "视频",
|
||||
},
|
||||
{
|
||||
Name: "image",
|
||||
Regex: `(?i)\.(jpg|jpeg|png|gif|webp|bmp)$`,
|
||||
Dir: "图片",
|
||||
},
|
||||
{
|
||||
Name: "audio",
|
||||
Regex: `(?i)\.(mp3|flac|wav|aac|m4a|ogg)$`,
|
||||
Dir: "音频",
|
||||
},
|
||||
{
|
||||
Name: "document",
|
||||
Regex: `(?i)\.(pdf|doc|docx|xls|xlsx|ppt|pptx|txt|md|csv|epub|mobi|azw3|chm)$`,
|
||||
Dir: "文档",
|
||||
},
|
||||
{
|
||||
Name: "archive",
|
||||
Regex: `(?i)\.(zip|rar|7z|tar|gz|bz2|xz|r\d{1,3}|z\d{1,3}|\d{3}|part\d+\.rar|7z\.\d{3})$`,
|
||||
Dir: "压缩包",
|
||||
},
|
||||
}
|
||||
|
||||
// PresetCategories returns the built-in filetype classification rules with each
|
||||
// category's directory joined under basePath. basePath may be empty.
|
||||
func PresetCategories(basePath string) []PresetCategory {
|
||||
out := make([]PresetCategory, len(presetCategories))
|
||||
for i, c := range presetCategories {
|
||||
c.Dir = path.Join(basePath, c.Dir)
|
||||
out[i] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
55
pkg/rule/preset_test.go
Normal file
55
pkg/rule/preset_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package rule
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPresetCategoriesCompile(t *testing.T) {
|
||||
for _, c := range PresetCategories("") {
|
||||
if _, err := regexp.Compile(c.Regex); err != nil {
|
||||
t.Errorf("preset %q has invalid regex %q: %v", c.Name, c.Regex, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresetCategoriesMatch(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"video": "movie.MP4",
|
||||
"image": "photo.jpg",
|
||||
"audio": "song.flac",
|
||||
"document": "report.pdf",
|
||||
"archive": "backup.zip",
|
||||
}
|
||||
|
||||
byName := make(map[string]*regexp.Regexp)
|
||||
for _, c := range PresetCategories("") {
|
||||
byName[c.Name] = regexp.MustCompile(c.Regex)
|
||||
}
|
||||
|
||||
for name, filename := range cases {
|
||||
re, ok := byName[name]
|
||||
if !ok {
|
||||
t.Errorf("missing preset category %q", name)
|
||||
continue
|
||||
}
|
||||
if !re.MatchString(filename) {
|
||||
t.Errorf("preset %q did not match %q", name, filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresetCategoriesBasePath(t *testing.T) {
|
||||
presets := PresetCategories("/media")
|
||||
for _, c := range presets {
|
||||
if c.Dir == "" || c.Dir[0] != '/' {
|
||||
t.Errorf("preset %q dir %q not joined under base path", c.Name, c.Dir)
|
||||
}
|
||||
}
|
||||
// Empty base path must not prefix a separator.
|
||||
for _, c := range PresetCategories("") {
|
||||
if c.Dir == "" || c.Dir[0] == '/' {
|
||||
t.Errorf("preset %q dir %q should be relative when base path empty", c.Name, c.Dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
17
pkg/storagetypes/batch.go
Normal file
17
pkg/storagetypes/batch.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package storagetypes
|
||||
|
||||
import "io"
|
||||
|
||||
// BatchItem describes one seekable file in a logical batch storage operation.
|
||||
type BatchItem struct {
|
||||
Reader io.ReadSeeker
|
||||
StoragePath string
|
||||
Size int64
|
||||
|
||||
// SourceGroupKey is empty for standalone source messages.
|
||||
SourceGroupKey string
|
||||
Caption string
|
||||
// PreserveCaption distinguishes an intentionally empty source caption from
|
||||
// the storage backend's default caption.
|
||||
PreserveCaption bool
|
||||
}
|
||||
@@ -31,6 +31,13 @@ type StorageCannotStream interface {
|
||||
CannotStream() string
|
||||
}
|
||||
|
||||
// StorageBatchSaver can preserve relationships between files when saving a
|
||||
// logical batch, such as a Telegram media album.
|
||||
type StorageBatchSaver interface {
|
||||
Storage
|
||||
SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error
|
||||
}
|
||||
|
||||
// StorageListable 表示支持列举目录内容的存储
|
||||
type StorageListable interface {
|
||||
Storage
|
||||
|
||||
140
storage/telegram/media_group_test.go
Normal file
140
storage/telegram/media_group_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
)
|
||||
|
||||
func TestPlanMediaGroups(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
items []batchMediaItem
|
||||
wantSizes []int
|
||||
}{
|
||||
{
|
||||
name: "same source album",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 1, true),
|
||||
},
|
||||
wantSizes: []int{2},
|
||||
},
|
||||
{
|
||||
name: "different source albums",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("b", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "ungrouped messages",
|
||||
items: []batchMediaItem{
|
||||
albumItem("", 1, true),
|
||||
albumItem("", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "different target chats",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 2, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "ineligible media does not bridge albums",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 1, false),
|
||||
albumItem("a", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1, 1},
|
||||
},
|
||||
{
|
||||
name: "maximum album size",
|
||||
items: repeatedAlbumItems(11),
|
||||
wantSizes: []int{10, 1},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
groups := planMediaGroups(tt.items)
|
||||
if len(groups) != len(tt.wantSizes) {
|
||||
t.Fatalf("got %d groups, want %d", len(groups), len(tt.wantSizes))
|
||||
}
|
||||
for i, want := range tt.wantSizes {
|
||||
if got := len(groups[i]); got != want {
|
||||
t.Errorf("group %d has %d items, want %d", i, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaCaption(t *testing.T) {
|
||||
empty := ""
|
||||
original := "original caption"
|
||||
tests := []struct {
|
||||
name string
|
||||
override *string
|
||||
wantLen int
|
||||
}{
|
||||
{name: "filename fallback", wantLen: 1},
|
||||
{name: "preserve empty source caption", override: &empty, wantLen: 0},
|
||||
{name: "preserve source caption", override: &original, wantLen: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := len(mediaCaption("file.jpg", tt.override)); got != tt.wantLen {
|
||||
t.Fatalf("got %d caption options, want %d", got, tt.wantLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectBatchItemRewindsBeforeMimetypeDetection(t *testing.T) {
|
||||
data := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
|
||||
reader := bytes.NewReader(data)
|
||||
if _, err := reader.Seek(4, io.SeekStart); err != nil {
|
||||
t.Fatalf("failed to set initial reader offset: %v", err)
|
||||
}
|
||||
|
||||
mediaItem, err := new(Telegram).inspectBatchItem(nil, storagetypes.BatchItem{
|
||||
Reader: reader,
|
||||
StoragePath: "photo.png",
|
||||
Size: int64(len(data)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspectBatchItem returned an error: %v", err)
|
||||
}
|
||||
if !mediaItem.albumEligible {
|
||||
t.Fatal("albumEligible = false, want true for PNG input")
|
||||
}
|
||||
if offset, err := reader.Seek(0, io.SeekCurrent); err != nil {
|
||||
t.Fatalf("failed to get final reader offset: %v", err)
|
||||
} else if offset != 0 {
|
||||
t.Fatalf("reader offset = %d, want 0", offset)
|
||||
}
|
||||
}
|
||||
|
||||
func albumItem(group string, chatID int64, eligible bool) batchMediaItem {
|
||||
return batchMediaItem{
|
||||
item: storagetypes.BatchItem{SourceGroupKey: group},
|
||||
chatID: chatID,
|
||||
albumEligible: eligible,
|
||||
}
|
||||
}
|
||||
|
||||
func repeatedAlbumItems(count int) []batchMediaItem {
|
||||
items := make([]batchMediaItem, count)
|
||||
for i := range items {
|
||||
items[i] = albumItem("a", 1, true)
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/duke-git/lancet/v2/slice"
|
||||
"github.com/duke-git/lancet/v2/validator"
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/pkg/consts/tglimit"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
"github.com/rs/xid"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
@@ -41,6 +43,19 @@ type Telegram struct {
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
type preparedMedia struct {
|
||||
peer tg.InputPeerClass
|
||||
uploader *uploader.Uploader
|
||||
media message.MultiMediaOption
|
||||
}
|
||||
|
||||
type batchMediaItem struct {
|
||||
item storagetypes.BatchItem
|
||||
chatID int64
|
||||
albumEligible bool
|
||||
useSingleSave bool
|
||||
}
|
||||
|
||||
func (t *Telegram) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
|
||||
telegramConfig, ok := cfg.(*storconfig.TelegramStorageConfig)
|
||||
if !ok {
|
||||
@@ -71,37 +86,76 @@ func (t *Telegram) Exists(ctx context.Context, storagePath string) bool {
|
||||
}
|
||||
|
||||
func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) error {
|
||||
storagePath = path.Clean(storagePath)
|
||||
tctx := tgutil.ExtFromContext(ctx)
|
||||
if tctx == nil {
|
||||
return fmt.Errorf("failed to get telegram context")
|
||||
}
|
||||
size := func() int64 {
|
||||
if length := ctx.Value(ctxkey.ContentLength); length != nil {
|
||||
if l, ok := length.(int64); ok {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return -1 // unknown size
|
||||
}()
|
||||
size := contentLength(ctx)
|
||||
if t.config.SkipLarge && size > MaxUploadFileSize {
|
||||
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", MaxUploadFileSize, size)
|
||||
return nil
|
||||
}
|
||||
rs, seekable := r.(io.ReadSeeker)
|
||||
splitSize := t.config.SplitSizeMB * 1024 * 1024
|
||||
if splitSize <= 0 {
|
||||
splitSize = DefaultSplitSize
|
||||
if size > t.splitSize() {
|
||||
filename, chatID := t.target(tctx, path.Clean(storagePath))
|
||||
if filename == "" {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
mtype, err := mimetype.DetectReader(rs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
}
|
||||
filename = xid.New().String() + mtype.Extension()
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek reader: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
upler := t.newUploader(tctx, size)
|
||||
peer := tryGetInputPeer(tctx, chatID)
|
||||
if peer == nil || peer.Zero() {
|
||||
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
}
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
return t.splitUpload(tctx, r, filename, upler, peer, size, t.splitSize())
|
||||
}
|
||||
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
prepared, err := t.prepareMedia(ctx, tctx, r, storagePath, size, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tctx.Sender.
|
||||
WithUploader(prepared.uploader).
|
||||
To(prepared.peer).
|
||||
Media(ctx, prepared.media)
|
||||
return err
|
||||
}
|
||||
|
||||
func contentLength(ctx context.Context) int64 {
|
||||
if length := ctx.Value(ctxkey.ContentLength); length != nil {
|
||||
if size, ok := length.(int64); ok {
|
||||
return size
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (t *Telegram) splitSize() int64 {
|
||||
splitSize := t.config.SplitSizeMB * 1024 * 1024
|
||||
if splitSize <= 0 {
|
||||
return DefaultSplitSize
|
||||
}
|
||||
return splitSize
|
||||
}
|
||||
|
||||
func (t *Telegram) target(tctx *ext.Context, storagePath string) (string, int64) {
|
||||
// 去除前导斜杠并分隔路径, 当 len(parts):
|
||||
// ==0, 存储到配置文件中的 chat_id, 随机文件名
|
||||
// ==1, 视作只有文件名, 存储到配置文件中的 chat_id
|
||||
// ==2, parts[0]: 视作要存储到的 chat_id, parts[1]: filename
|
||||
// >=2, parts[0]: 视作要存储到的 chat_id, 最后一项为 filename
|
||||
parts := slice.Compact(strings.Split(strings.TrimPrefix(storagePath, "/"), "/"))
|
||||
filename := ""
|
||||
chatID := t.config.ChatID
|
||||
@@ -111,38 +165,54 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
if len(parts) >= 2 && validator.IsAlphaNumeric(parts[0]) {
|
||||
cid, err := tgutil.ParseChatID(tctx, parts[0])
|
||||
if err != nil {
|
||||
// id不合法时使用配置文件中的 chat_id
|
||||
log.FromContext(ctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
|
||||
log.FromContext(tctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
|
||||
cid = chatID
|
||||
}
|
||||
chatID = cid
|
||||
}
|
||||
upler := uploader.NewUploader(tctx.Raw).
|
||||
return filename, chatID
|
||||
}
|
||||
|
||||
func (t *Telegram) newUploader(tctx *ext.Context, size int64) *uploader.Uploader {
|
||||
return uploader.NewUploader(tctx.Raw).
|
||||
WithPartSize(tglimit.MaxUploadPartSize).
|
||||
WithThreads(dlutil.BestThreads(size, config.C().Threads))
|
||||
}
|
||||
|
||||
func mediaCaption(filename string, override *string) []message.StyledTextOption {
|
||||
if override == nil {
|
||||
return []message.StyledTextOption{styling.Plain(filename)}
|
||||
}
|
||||
if *override == "" {
|
||||
return nil
|
||||
}
|
||||
return []message.StyledTextOption{styling.Plain(*override)}
|
||||
}
|
||||
|
||||
func (t *Telegram) prepareMedia(ctx context.Context, tctx *ext.Context, r io.Reader, storagePath string, size int64, captionOverride *string) (*preparedMedia, error) {
|
||||
storagePath = path.Clean(storagePath)
|
||||
filename, chatID := t.target(tctx, storagePath)
|
||||
upler := t.newUploader(tctx, size)
|
||||
peer := tryGetInputPeer(tctx, chatID)
|
||||
if peer == nil || peer.Zero() {
|
||||
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
return nil, fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
}
|
||||
|
||||
rs, seekable := r.(io.ReadSeeker)
|
||||
var mtype *mimetype.MIME
|
||||
if seekable {
|
||||
var err error
|
||||
mtype, err = mimetype.DetectReader(rs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
return nil, fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
}
|
||||
if filename == "" {
|
||||
filename = xid.New().String() + mtype.Extension()
|
||||
}
|
||||
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek reader: %w", err)
|
||||
return nil, fmt.Errorf("failed to seek reader: %w", err)
|
||||
}
|
||||
}
|
||||
if size > splitSize {
|
||||
// large file, use split uploader
|
||||
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize)
|
||||
}
|
||||
|
||||
var file tg.InputFileClass
|
||||
var err error
|
||||
@@ -152,21 +222,20 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
file, err = upler.Upload(ctx, uploader.NewUpload(filename, r, size))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file to telegram: %w", err)
|
||||
return nil, fmt.Errorf("failed to upload file to telegram: %w", err)
|
||||
}
|
||||
caption := styling.Plain(filename)
|
||||
caption := mediaCaption(filename, captionOverride)
|
||||
forceFile := t.config.ForceFile
|
||||
|
||||
if mtype != nil && strings.HasPrefix(mtype.String(), "image/") && size >= tglimit.MaxPhotoSize {
|
||||
forceFile = true
|
||||
}
|
||||
doc := message.UploadedDocument(file, caption).
|
||||
doc := message.UploadedDocument(file, caption...).
|
||||
Filename(filename).
|
||||
ForceFile(forceFile)
|
||||
if mtype != nil {
|
||||
doc = doc.MIME(mtype.String())
|
||||
}
|
||||
var media message.MediaOption = doc
|
||||
var media message.MultiMediaOption = doc
|
||||
if mtype != nil && rs != nil {
|
||||
switch mtypeStr := mtype.String(); {
|
||||
case strings.HasPrefix(mtypeStr, "video/"):
|
||||
@@ -205,12 +274,131 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
case strings.HasPrefix(mtypeStr, "audio/"):
|
||||
media = doc.Audio().Title(filename)
|
||||
case strings.HasPrefix(mtypeStr, "image/") && !strings.HasSuffix(mtypeStr, "webp"):
|
||||
media = message.UploadedPhoto(file, caption)
|
||||
media = message.UploadedPhoto(file, caption...)
|
||||
}
|
||||
}
|
||||
sender := tctx.Sender
|
||||
_, err = sender.WithUploader(upler).To(peer).Media(ctx, media)
|
||||
return err
|
||||
return &preparedMedia{
|
||||
peer: peer,
|
||||
uploader: upler,
|
||||
media: media,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SaveBatch preserves each source photo/video group as a Telegram album.
|
||||
func (t *Telegram) SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error {
|
||||
tctx := tgutil.ExtFromContext(ctx)
|
||||
if tctx == nil {
|
||||
return fmt.Errorf("failed to get telegram context")
|
||||
}
|
||||
|
||||
inspected := make([]batchMediaItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
mediaItem, err := t.inspectBatchItem(tctx, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inspected = append(inspected, mediaItem)
|
||||
}
|
||||
for _, group := range planMediaGroups(inspected) {
|
||||
if err := t.saveMediaGroup(ctx, tctx, group); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Telegram) inspectBatchItem(tctx *ext.Context, item storagetypes.BatchItem) (batchMediaItem, error) {
|
||||
_, chatID := t.target(tctx, path.Clean(item.StoragePath))
|
||||
result := batchMediaItem{item: item, chatID: chatID}
|
||||
if (t.config.SkipLarge && item.Size > MaxUploadFileSize) || item.Size > t.splitSize() {
|
||||
result.useSingleSave = true
|
||||
return result, nil
|
||||
}
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("failed to seek batch item before mimetype detection: %w", err)
|
||||
}
|
||||
mtype, err := mimetype.DetectReader(item.Reader)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to detect batch item mimetype: %w", err)
|
||||
}
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
mtypeStr := mtype.String()
|
||||
forceFile := t.config.ForceFile || strings.HasPrefix(mtypeStr, "image/") && item.Size >= tglimit.MaxPhotoSize
|
||||
result.albumEligible = !forceFile && (strings.HasPrefix(mtypeStr, "video/") ||
|
||||
strings.HasPrefix(mtypeStr, "image/") && mtypeStr != "image/webp" && mtypeStr != "image/gif")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func planMediaGroups(items []batchMediaItem) [][]batchMediaItem {
|
||||
groups := make([][]batchMediaItem, 0, len(items))
|
||||
for i := 0; i < len(items); {
|
||||
item := items[i]
|
||||
if item.useSingleSave || !item.albumEligible || item.item.SourceGroupKey == "" {
|
||||
groups = append(groups, items[i:i+1])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
end := i + 1
|
||||
for end < len(items) && end-i < 10 {
|
||||
next := items[end]
|
||||
if next.useSingleSave || !next.albumEligible || next.chatID != item.chatID || next.item.SourceGroupKey != item.item.SourceGroupKey {
|
||||
break
|
||||
}
|
||||
end++
|
||||
}
|
||||
groups = append(groups, items[i:end])
|
||||
i = end
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (t *Telegram) saveMediaGroup(ctx context.Context, tctx *ext.Context, group []batchMediaItem) error {
|
||||
return retry.Retry(func() error {
|
||||
if len(group) == 1 && group[0].useSingleSave {
|
||||
item := group[0].item
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
itemCtx := context.WithValue(ctx, ctxkey.ContentLength, item.Size)
|
||||
return t.Save(itemCtx, item.Reader, item.StoragePath)
|
||||
}
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
|
||||
prepared := make([]preparedMedia, 0, len(group))
|
||||
for _, mediaItem := range group {
|
||||
item := mediaItem.item
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
var captionOverride *string
|
||||
if item.PreserveCaption {
|
||||
captionOverride = &item.Caption
|
||||
}
|
||||
media, err := t.prepareMedia(ctx, tctx, item.Reader, item.StoragePath, item.Size, captionOverride)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prepared = append(prepared, *media)
|
||||
}
|
||||
|
||||
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
|
||||
if len(prepared) == 1 {
|
||||
_, err := builder.Media(ctx, prepared[0].media)
|
||||
return err
|
||||
}
|
||||
media := make([]message.MultiMediaOption, len(prepared))
|
||||
for i := range prepared {
|
||||
media[i] = prepared[i].media
|
||||
}
|
||||
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
|
||||
return fmt.Errorf("failed to send media album: %w", err)
|
||||
}
|
||||
return nil
|
||||
}, retry.Context(ctx), retry.RetryTimes(uint(config.C().Retry)))
|
||||
}
|
||||
|
||||
func (t *Telegram) CannotStream() string {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
@@ -14,6 +15,34 @@ import (
|
||||
"github.com/yapingcat/gomedia/go-mp4"
|
||||
)
|
||||
|
||||
// sourceFile returns a filesystem path to the media for ffmpeg/ffprobe. Those
|
||||
// tools need a SEEKABLE input: feeding them a pipe (pipe:0) fails for
|
||||
// non-faststart MP4s whose moov atom is at the END of the file, because ffmpeg
|
||||
// cannot seek backwards on a stream to read it. That silently broke thumbnail
|
||||
// and metadata extraction for a large share of downloaded videos. If rs is
|
||||
// already an *os.File we use it directly; otherwise we spool it to a temp file
|
||||
// and return a cleanup func.
|
||||
func sourceFile(rs io.ReadSeeker) (path string, cleanup func(), err error) {
|
||||
noop := func() {}
|
||||
if f, ok := rs.(*os.File); ok {
|
||||
return f.Name(), noop, nil
|
||||
}
|
||||
if _, err = rs.Seek(0, io.SeekStart); err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
tf, err := os.CreateTemp("", "saveany-media-*.tmp")
|
||||
if err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
if _, err = io.Copy(tf, rs); err != nil {
|
||||
tf.Close()
|
||||
os.Remove(tf.Name())
|
||||
return "", noop, err
|
||||
}
|
||||
tf.Close()
|
||||
return tf.Name(), func() { os.Remove(tf.Name()) }, nil
|
||||
}
|
||||
|
||||
type VideoMetadata struct {
|
||||
Duration int
|
||||
Width int
|
||||
@@ -52,16 +81,14 @@ func getMP4Meta(rs io.ReadSeeker) (metadata *VideoMetadata, err error) {
|
||||
|
||||
// getVideoMetadata uses ffprobe to get video metadata
|
||||
func getVideoMetadata(rs io.ReadSeeker) (*VideoMetadata, error) {
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
path, cleanup, err := sourceFile(rs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
rs.Seek(0, io.SeekStart)
|
||||
io.Copy(pipeWriter, rs)
|
||||
}()
|
||||
|
||||
result, err := ffmpeg.ProbeReaderWithTimeout(
|
||||
pipeReader,
|
||||
result, err := ffmpeg.ProbeWithTimeout(
|
||||
path,
|
||||
time.Second*10,
|
||||
ffmpeg.KwArgs{
|
||||
"select_streams": "v:0",
|
||||
@@ -114,25 +141,22 @@ func extractThumbFrame(rs io.ReadSeeker) ([]byte, error) {
|
||||
}
|
||||
|
||||
func extractFrameAt(rs io.ReadSeeker, timestamp float64) ([]byte, error) {
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
rs.Seek(0, io.SeekStart)
|
||||
io.Copy(pipeWriter, rs)
|
||||
}()
|
||||
path, cleanup, err := sourceFile(rs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
err := ffmpeg.
|
||||
Input("pipe:0", ffmpeg.KwArgs{
|
||||
err = ffmpeg.
|
||||
Input(path, ffmpeg.KwArgs{
|
||||
"ss": fmt.Sprintf("%.3f", timestamp),
|
||||
}).
|
||||
Output("pipe:1", ffmpeg.KwArgs{
|
||||
"vframes": 1,
|
||||
"f": "mjpeg",
|
||||
}).
|
||||
WithInput(pipeReader).
|
||||
WithOutput(&out).
|
||||
OverWriteOutput().
|
||||
Run()
|
||||
|
||||
Reference in New Issue
Block a user