Compare commits

...

11 Commits

Author SHA1 Message Date
krau
a1bd2ed762 fix: upgrade deps 2026-08-11 08:57:09 +08:00
krau
7e3e26fcf4 style: format codebase 2026-08-11 08:47:35 +08:00
Haopeng Huo
e49ebef977 fix(telegraph): parse page URLs from message entities (#232) 2026-08-11 08:43:46 +08:00
Haopeng Huo
da38f5bb88 fix: remove cache files after reader closes (#231) 2026-08-11 08:41:40 +08:00
Haopeng Huo
0e6ed66ef5 feat: show upload progress for batch file tasks (#228)
* feat: show upload progress for file tasks

* feat: show upload progress for batch file tasks

* style: distinguish download and upload phases

* style: mark successful task completion

* fix: preserve storage save error context

* fix: serialize single-file progress updates

* fix: stabilize batch upload progress reporting

Correct batch upload totals, completion state, actual file sizes, and synchronized progress snapshots. Add regression coverage for concurrent updates and phase transitions.

* feat: show per-file transfer progress

Format single-file and batch download and upload states with Telegram entities, blockquotes, speeds, transferred sizes, progress bars, concise counters, and accurate confirmation handling. Cover upload retries plus final, error, and cancellation messages with regression tests.

* chore: remove transfer progress tests

* test: restore critical transfer progress coverage

* test: cover interleaved batch transfers

* fix: declare progress styles in locale templates
2026-08-11 08:40:29 +08:00
Haopeng Huo
e4144e73e6 feat(telegram): split oversized videos into playable parts (#229)
* feat(telegram): split oversized videos losslessly

* fix(telegram): respect Premium upload limits

* fix(telegram): keep split videos in one album
2026-08-05 16:57:46 +08:00
Haopeng Huo
1d794a7b9c feat(telegram): preserve source media groups (#226)
* feat(telegram): preserve source media groups

* fix(telegram): rewind batch readers before inspection
2026-08-04 16:46:29 +08:00
irudisca
52f880f0f2 fix(telegram): use seekable file path for thumbnail & metadata extraction (#225)
extractFrameAt() and getVideoMetadata() piped the media into ffmpeg/ffprobe
via pipe:0. A pipe is not seekable, so ffmpeg cannot decode non-faststart MP4s
whose moov atom sits at the END of the file (very common for yt-dlp / HLS-merged
downloads): the frame grab and probe silently fail, and the video is uploaded
with no thumbnail (and, for non-mp4 containers, no dimensions).

Hand ffmpeg/ffprobe a seekable file path instead. When the reader already is an
*os.File we use it directly; otherwise we spool to a temp file and clean it up.

Verified with an A/B upload of a 56 MB non-faststart clip: stock build produced
no thumbnail, patched build produced a correct thumbnail.

Co-authored-by: pennyucloud <valentino@pennyu.co.id>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-23 18:28:03 +08:00
krau
fc11ca775f fix: update message link parsing to support both t.me and telegram.me domains 2026-07-14 12:53:09 +08:00
krau
056e2fd546 fix: add ca-certificates installation and environment variables for SSL support in Dockerfile, close #222 2026-07-06 11:34:28 +08:00
krau
c9bb6c9e3c feat(docs): add CLI subcommands documentation and file naming strategies
- Introduced new documentation for CLI subcommands: `upload` and `watch`, detailing their usage, flags, and examples.
- Added a section on file naming and conflict strategies, explaining how users can customize file names and handle duplicates via `/config` and `/fnametmpl` commands.
- Translated the new documentation into Chinese for broader accessibility.
2026-06-28 18:00:02 +08:00
59 changed files with 4845 additions and 573 deletions

View File

@@ -7,6 +7,8 @@ ARG BuildTime="Unknown"
WORKDIR /app
RUN apk add --no-cache ca-certificates
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/go/pkg/mod \
go mod download
@@ -31,5 +33,9 @@ FROM scratch
WORKDIR /app
COPY --from=builder /app/saveany-bot .
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
ENV SSL_CERT_DIR=/etc/ssl/certs
ENTRYPOINT ["/app/saveany-bot"]

View File

@@ -11,23 +11,23 @@ import (
// guarded by mu. It implements taskevent.Sink so the task layer can update it
// without knowing about the API.
type TaskProgressInfo struct {
mu sync.Mutex
TaskID string
Type string
Status TaskStatus
Title string
TotalBytes int64
DownloadedBytes int64
TotalFiles int
DownloadedFiles int
Storage string
Path string
Error string
CreatedAt time.Time
UpdatedAt time.Time
StartedAt time.Time
Webhook string
webhookNotified bool
mu sync.Mutex
TaskID string
Type string
Status TaskStatus
Title string
TotalBytes int64
DownloadedBytes int64
TotalFiles int
DownloadedFiles int
Storage string
Path string
Error string
CreatedAt time.Time
UpdatedAt time.Time
StartedAt time.Time
Webhook string
webhookNotified bool
}
// progressStore holds all API tasks. Entries are removed a fixed duration after
@@ -208,9 +208,9 @@ func NewProgressTracker(taskID, taskType, storage, path, title, webhook string)
return &ProgressTracker{}
}
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
func (p *ProgressTracker) OnProgress(downloadedBytes int64, downloadedFiles int) {}
func (p *ProgressTracker) OnDone(err error) {}
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
func (p *ProgressTracker) UpdateProgressFiles(files int) {}
func (p *ProgressTracker) OnDone(err error) {}
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
func (p *ProgressTracker) UpdateProgressFiles(files int) {}

View File

@@ -60,6 +60,7 @@ func resolveChatID(_ context.Context, idOrUsername string) (int64, error) {
}
// ParseMessageLink 解析 Telegram 消息链接
// 支持的域名: t.me, telegram.me
// 支持格式:
// - https://t.me/username/123
// - https://t.me/c/123456789/123
@@ -268,5 +269,15 @@ func ExtractFilesFromLinks(ctx context.Context, links []string) ([]tfile.TGFileM
// isValidMessageLink 检查是否是有效的 Telegram 消息链接
func isValidMessageLink(link string) bool {
return strings.HasPrefix(link, "https://t.me/") || strings.HasPrefix(link, "http://t.me/")
for _, prefix := range []string{
"https://t.me/",
"http://t.me/",
"https://telegram.me/",
"http://telegram.me/",
} {
if strings.HasPrefix(link, prefix) {
return true
}
}
return false
}

View File

@@ -13,8 +13,8 @@ import (
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
"github.com/krau/SaveAny-Bot/pkg/tfile"
"github.com/krau/SaveAny-Bot/storage"

View File

@@ -3,8 +3,8 @@ package re
import "regexp"
var (
TgMessageLinkRegexString = `https?://t\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
TgMessageLinkRegexString = `https?://(?:t|telegram)\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
TgMessageLinkRegexp = regexp.MustCompile(TgMessageLinkRegexString)
TelegraphUrlRegexString = `https://telegra.ph/.*`
TelegraphUrlRegexString = `https://telegra\.ph/[^\s]+`
TelegraphUrlRegexp = regexp.MustCompile(TelegraphUrlRegexString)
)

View File

@@ -3,6 +3,7 @@ package shortcut
import (
"encoding/json"
"fmt"
"net/url"
"strings"
@@ -179,21 +180,19 @@ type TelegraphResult struct {
// return replied message, image urls, telegraph path(unescaped), error
func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*types.Message, *TelegraphResult, error) {
logger := log.FromContext(ctx)
tphurl := re.TelegraphUrlRegexp.FindString(tgutil.ExtractMessageEntityUrlsText(update.EffectiveMessage.Message))
tphurl := findTelegraphURL(update.EffectiveMessage.Message)
if tphurl == "" {
logger.Warnf("No telegraph url found but called handleTelegraph")
return nil, nil, dispatcher.ContinueGroups
}
pagepath := strings.Split(tphurl, "/")[len(strings.Split(tphurl, "/"))-1]
tphdir, err := url.PathUnescape(pagepath)
pagepath, err := parseTelegraphPagePath(tphurl)
if err != nil {
logger.Errorf("Failed to unescape telegraph path: %s", err)
logger.Errorf("Failed to parse telegraph path: %s", err)
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorParseTelegraphPathFailed, map[string]any{
"Error": err.Error(),
})), nil)
return nil, nil, dispatcher.EndGroups
}
tphdir = strings.TrimSpace(tphdir)
msg, err := ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoFetchingTelegraphPage, nil)), nil)
if err != nil {
logger.Errorf("Failed to reply to update: %s", err)
@@ -244,7 +243,57 @@ func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*type
}
return msg, &TelegraphResult{
Pics: imgs,
TphDir: tphdir,
TphDir: pagepath,
Page: page,
}, nil
}
func findTelegraphURL(msg *tg.Message) string {
if msg == nil {
return ""
}
var firstMatch string
findValid := func(text string) string {
for _, tphurl := range re.TelegraphUrlRegexp.FindAllString(text, -1) {
if firstMatch == "" {
firstMatch = tphurl
}
if _, err := parseTelegraphPagePath(tphurl); err == nil {
return tphurl
}
}
return ""
}
for _, entityURL := range tgutil.ExtractMessageEntityUrls(msg) {
if tphurl := findValid(entityURL); tphurl != "" {
return tphurl
}
}
if tphurl := findValid(msg.GetMessage()); tphurl != "" {
return tphurl
}
return firstMatch
}
func parseTelegraphPagePath(pageURL string) (string, error) {
u, err := url.Parse(pageURL)
if err != nil {
return "", fmt.Errorf("invalid telegraph URL: %w", err)
}
if u.Scheme != "https" || !strings.EqualFold(u.Hostname(), "telegra.ph") {
return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host)
}
pagepath := strings.Trim(u.EscapedPath(), "/")
if pagepath == "" || strings.Contains(pagepath, "/") {
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
}
pagepath, err = url.PathUnescape(pagepath)
if err != nil {
return "", fmt.Errorf("failed to unescape telegraph path: %w", err)
}
pagepath = strings.TrimSpace(pagepath)
if pagepath == "" || strings.Contains(pagepath, "/") {
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
}
return pagepath, nil
}

View File

@@ -0,0 +1,163 @@
package shortcut
import (
"testing"
"github.com/gotd/td/tg"
)
func TestFindTelegraphURL(t *testing.T) {
tests := []struct {
name string
msg *tg.Message
want string
}{
{
name: "single URL entity",
msg: &tg.Message{
Message: "https://telegra.ph/Example-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 32},
},
},
want: "https://telegra.ph/Example-01-02",
},
{
name: "Telegraph URL before another URL",
msg: &tg.Message{
Message: "https://telegra.ph/Example-01-02 https://example.com/",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 32},
&tg.MessageEntityURL{Offset: 33, Length: 20},
},
},
want: "https://telegra.ph/Example-01-02",
},
{
name: "hidden Telegraph URL",
msg: &tg.Message{
Message: "article",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityTextURL{
Offset: 0,
Length: 7,
URL: "https://telegra.ph/Hidden-01-02",
},
},
},
want: "https://telegra.ph/Hidden-01-02",
},
{
name: "URL entity after non-BMP character",
msg: &tg.Message{
Message: "😀 https://telegra.ph/Emoji-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 3, Length: 30},
},
},
want: "https://telegra.ph/Emoji-01-02",
},
{
name: "valid Telegraph URL after invalid candidate",
msg: &tg.Message{
Message: "https://telegra.ph/nested/Bad https://telegra.ph/Valid-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 29},
&tg.MessageEntityURL{Offset: 30, Length: 30},
},
},
want: "https://telegra.ph/Valid-01-02",
},
{
name: "plain message fallback",
msg: &tg.Message{Message: "read https://telegra.ph/Plain-01-02 now"},
want: "https://telegra.ph/Plain-01-02",
},
{
name: "nil message",
msg: nil,
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := findTelegraphURL(tt.msg); got != tt.want {
t.Fatalf("findTelegraphURL() = %q, want %q", got, tt.want)
}
})
}
}
func TestParseTelegraphPagePath(t *testing.T) {
tests := []struct {
name string
pageURL string
want string
wantErr bool
}{
{
name: "plain path",
pageURL: "https://telegra.ph/Example-01-02",
want: "Example-01-02",
},
{
name: "escaped path with query and fragment",
pageURL: "https://telegra.ph/%E6%B5%8B%E8%AF%95-01-02?source=telegram#top",
want: "测试-01-02",
},
{
name: "trailing slash",
pageURL: "https://telegra.ph/Example-01-02/",
want: "Example-01-02",
},
{
name: "root URL",
pageURL: "https://telegra.ph/",
wantErr: true,
},
{
name: "wrong host",
pageURL: "https://example.com/Example-01-02",
wantErr: true,
},
{
name: "wrong scheme",
pageURL: "http://telegra.ph/Example-01-02",
wantErr: true,
},
{
name: "invalid percent escape",
pageURL: "https://telegra.ph/Invalid-%zz",
wantErr: true,
},
{
name: "nested path",
pageURL: "https://telegra.ph/nested/Example-01-02",
wantErr: true,
},
{
name: "encoded slash",
pageURL: "https://telegra.ph/nested%2FExample-01-02",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseTelegraphPagePath(tt.pageURL)
if tt.wantErr {
if err == nil {
t.Fatalf("parseTelegraphPagePath(%q) returned no error", tt.pageURL)
}
return
}
if err != nil {
t.Fatalf("parseTelegraphPagePath(%q) failed: %v", tt.pageURL, err)
}
if got != tt.want {
t.Fatalf("parseTelegraphPagePath(%q) = %q, want %q", tt.pageURL, got, tt.want)
}
})
}
}

View File

@@ -150,28 +150,50 @@ const (
BotMsgProgressAria2Downloading Key = "bot.msg.progress.aria2_downloading"
BotMsgProgressAria2Start Key = "bot.msg.progress.aria2_start"
BotMsgProgressAvgSpeedPrefix Key = "bot.msg.progress.avg_speed_prefix"
BotMsgProgressBatchDonePrefix Key = "bot.msg.progress.batch_done_prefix"
BotMsgProgressBatchProcessingPrefix Key = "bot.msg.progress.batch_processing_prefix"
BotMsgProgressBatchStartPrefix Key = "bot.msg.progress.batch_start_prefix"
BotMsgProgressBatchCanceled Key = "bot.msg.progress.batch_canceled"
BotMsgProgressBatchDone Key = "bot.msg.progress.batch_done"
BotMsgProgressBatchDoneWithSkipped Key = "bot.msg.progress.batch_done_with_skipped"
BotMsgProgressBatchFailedGroup Key = "bot.msg.progress.batch_failed_group"
BotMsgProgressBatchFailedItem Key = "bot.msg.progress.batch_failed_item"
BotMsgProgressBatchFailedTask Key = "bot.msg.progress.batch_failed_task"
BotMsgProgressBatchFailureStageBatchUpload Key = "bot.msg.progress.batch_failure_stage_batch_upload"
BotMsgProgressBatchFailureStageCache Key = "bot.msg.progress.batch_failure_stage_cache"
BotMsgProgressBatchFailureStageConfirm Key = "bot.msg.progress.batch_failure_stage_confirm"
BotMsgProgressBatchFailureStageDownload Key = "bot.msg.progress.batch_failure_stage_download"
BotMsgProgressBatchFailureStageInternal Key = "bot.msg.progress.batch_failure_stage_internal"
BotMsgProgressBatchFailureStageUpload Key = "bot.msg.progress.batch_failure_stage_upload"
BotMsgProgressBatchItemConfirming Key = "bot.msg.progress.batch_item_confirming"
BotMsgProgressBatchItemDownloading Key = "bot.msg.progress.batch_item_downloading"
BotMsgProgressBatchItemDownloadingUnknown Key = "bot.msg.progress.batch_item_downloading_unknown"
BotMsgProgressBatchItemRetrying Key = "bot.msg.progress.batch_item_retrying"
BotMsgProgressBatchItemTransferring Key = "bot.msg.progress.batch_item_transferring"
BotMsgProgressBatchItemTransferringUnknown Key = "bot.msg.progress.batch_item_transferring_unknown"
BotMsgProgressBatchItemUploading Key = "bot.msg.progress.batch_item_uploading"
BotMsgProgressBatchStatusHeader Key = "bot.msg.progress.batch_status_header"
BotMsgProgressBatchSummaryConfirming Key = "bot.msg.progress.batch_summary_confirming"
BotMsgProgressBatchSummaryFailed Key = "bot.msg.progress.batch_summary_failed"
BotMsgProgressBatchSummaryHiddenActive Key = "bot.msg.progress.batch_summary_hidden_active"
BotMsgProgressBatchSummarySkipped Key = "bot.msg.progress.batch_summary_skipped"
BotMsgProgressCurrentProgressPrefix Key = "bot.msg.progress.current_progress_prefix"
BotMsgProgressCurrentSpeedPrefix Key = "bot.msg.progress.current_speed_prefix"
BotMsgProgressDirectDonePrefix Key = "bot.msg.progress.direct_done_prefix"
BotMsgProgressDirectStart Key = "bot.msg.progress.direct_start"
BotMsgProgressDownloadDonePrefix Key = "bot.msg.progress.download_done_prefix"
BotMsgProgressDownloadFailedPrefix Key = "bot.msg.progress.download_failed_prefix"
BotMsgProgressDownloadedPrefix Key = "bot.msg.progress.downloaded_prefix"
BotMsgProgressDownloadingPrefix Key = "bot.msg.progress.downloading_prefix"
BotMsgProgressErrorPrefix Key = "bot.msg.progress.error_prefix"
BotMsgProgressFileNamePrefix Key = "bot.msg.progress.file_name_prefix"
BotMsgProgressFileProcessingPrefix Key = "bot.msg.progress.file_processing_prefix"
BotMsgProgressFileSizePrefix Key = "bot.msg.progress.file_size_prefix"
BotMsgProgressFileStartPrefix Key = "bot.msg.progress.file_start_prefix"
BotMsgProgressParsedDonePrefix Key = "bot.msg.progress.parsed_done_prefix"
BotMsgProgressParsedStartPrefix Key = "bot.msg.progress.parsed_start_prefix"
BotMsgProgressProcessingListPrefix Key = "bot.msg.progress.processing_list_prefix"
BotMsgProgressProcessingNone Key = "bot.msg.progress.processing_none"
BotMsgProgressSavePathPrefix Key = "bot.msg.progress.save_path_prefix"
BotMsgProgressTaskCanceled Key = "bot.msg.progress.task_canceled"
BotMsgProgressSingleCanceled Key = "bot.msg.progress.single_canceled"
BotMsgProgressSingleDone Key = "bot.msg.progress.single_done"
BotMsgProgressSingleDownloading Key = "bot.msg.progress.single_downloading"
BotMsgProgressSingleDownloadingUnknown Key = "bot.msg.progress.single_downloading_unknown"
BotMsgProgressSingleFailed Key = "bot.msg.progress.single_failed"
BotMsgProgressSingleStatusHeader Key = "bot.msg.progress.single_status_header"
BotMsgProgressSingleUploading Key = "bot.msg.progress.single_uploading"
BotMsgProgressSingleUploadRetrying Key = "bot.msg.progress.single_upload_retrying"
BotMsgProgressTaskCanceledWithId Key = "bot.msg.progress.task_canceled_with_id"
BotMsgProgressTaskFailedWithError Key = "bot.msg.progress.task_failed_with_error"
BotMsgProgressTelegraphDonePrefix Key = "bot.msg.progress.telegraph_done_prefix"

View File

@@ -351,32 +351,54 @@ bot:
info_filename_prefix: "Filename: "
info_prompt_select_storage: "\nPlease select storage"
progress:
batch_start_prefix: "Starting batch download task\nTotal size: "
batch_processing_prefix: "Processing batch download task\nTotal size: "
batch_status_header: "<b>📦 Processing</b>\n\nFiles: <code>{{.Total}}</code>\nStatus: ✅ <code>{{.Completed}}</code> | 📥 <code>{{.Downloaded}}</code> | ⏳ <code>{{.Waiting}}</code>\nTotal speed: ⬇️ <code>{{.DownloadSpeed}}</code> | ⬆️ <code>{{.UploadSpeed}}</code>"
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nRetry: <code>{{.Attempt}}/{{.Limit}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nReason: <code>{{.Reason}}</code></blockquote>"
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} Waiting</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>Uploaded, awaiting remote confirmation</i></blockquote>"
batch_summary_hidden_active: "🔄 <code>{{.Count}}</code> more files are active"
batch_summary_confirming: "☁️ Uploaded, awaiting group send: <code>{{.Count}}</code>"
batch_summary_failed: "❌ Failed: <code>{{.Count}}</code>"
batch_summary_skipped: "⏭️ Skipped: <code>{{.Count}}</code>"
batch_done: "<b>✅ Completed</b>\n\nFiles: <code>{{.Count}}</code>\nTotal size: <code>{{.Size}}</code>"
batch_done_with_skipped: "<b>⚠️ Completed</b>\n\nSucceeded: <code>{{.Success}}</code>\nSkipped: <code>{{.Skipped}}</code>\nTotal size: <code>{{.Size}}</code>"
batch_canceled: "<b>🚫 Task canceled</b>\n\nFiles: <code>{{.Total}}</code>\nCompleted: <code>{{.Completed}}</code>\nIncomplete: <code>{{.Incomplete}}</code>\nSkipped: <code>{{.Skipped}}</code>"
batch_failed_item: "<b>❌ Processing failed</b>\n\nFailed file: <code>{{.Index}}. {{.Name}}</code>\nStage: <code>{{.Stage}}</code>\nProgress: <code>{{.Progress}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Failed: <code>{{.Failed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failed_group: "<b>❌ Batch upload failed</b>\n\nAffected files: <code>{{.Affected}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Batch failed: <code>{{.Failed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failed_task: "<b>❌ Processing failed</b>\n\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failure_stage_download: "download"
batch_failure_stage_cache: "local cache"
batch_failure_stage_upload: "upload"
batch_failure_stage_confirm: "remote confirmation"
batch_failure_stage_batch_upload: "batch upload"
batch_failure_stage_internal: "internal task"
single_status_header: "<b>📦 Processing</b>"
single_downloading: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_downloading_unknown: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown\nSave to: <code>{{.Destination}}</code></blockquote>"
single_uploading: "<blockquote><b>⬆️ Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_upload_retrying: "<blockquote><b>🔁 Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nAttempt: <code>{{.Attempt}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_done: "<b>✅ Completed</b>\n\nFilename: <code>{{.Name}}</code>\nTotal size: <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code>"
single_canceled: "<b>🚫 Task canceled</b>\n\nFilename: <code>{{.Name}}</code>"
single_failed: "<b>❌ Processing failed</b>\n\nFilename: <code>{{.Name}}</code>\nReason: <code>{{.Reason}}</code>"
downloading_prefix: "Downloading\nTotal size: "
processing_list_prefix: "\nProcessing:\n"
processing_none: " - None"
avg_speed_prefix: "\nAverage speed: "
current_progress_prefix: "\nCurrent progress: "
task_canceled: "Task canceled"
task_canceled_with_id: "Processing canceled: {{.TaskID}}"
task_failed_with_error: "Processing failed: {{.Error}}"
batch_done_prefix: "Completed\nFile count: "
direct_done_prefix: "Completed, file count: "
parsed_start_prefix: "Starting download from {{.Site}}\nTotal size: "
parsed_done_prefix: "Completed, resource count: "
telegraph_start_prefix: "Starting Telegraph download\nImage count: "
telegraph_progress_prefix: "Downloading\nCurrent progress: "
telegraph_done_prefix: "Completed\nImage count: "
file_start_prefix: "Starting download\nFilename: "
file_processing_prefix: "Processing download task\nFilename: "
download_failed_prefix: "Download failed\nFilename: "
download_done_prefix: "Download completed\nFilename: "
file_size_prefix: "\nFile size: "
save_path_prefix: "\nSave path: "
total_size_prefix: "\nTotal size: "
direct_start: "Starting download, total size: {{.SizeMB}} MB ({{.Count}} files)"
file_name_prefix: "Filename: "
error_prefix: "\nError: "
aria2_start: "Waiting for Aria2 to complete download (GID: {{.GID}})..."
aria2_downloading: "Aria2 downloading (GID: {{.GID}})\n"

View File

@@ -352,32 +352,54 @@ bot:
info_filename_prefix: "文件名: "
info_prompt_select_storage: "\n请选择存储位置"
progress:
batch_start_prefix: "开始执行批量下载任务\n总大小: "
batch_processing_prefix: "正在处理批量下载任务\n总大小: "
batch_status_header: "<b>📦 正在处理</b>\n\n文件<code>{{.Total}}</code>\n状态✅ <code>{{.Completed}}</code> 📥 <code>{{.Downloaded}}</code> ⏳ <code>{{.Waiting}}</code>\n总速度 <code>{{.DownloadSpeed}}</code> ⬆️ <code>{{.UploadSpeed}}</code>"
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知</blockquote>"
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知</blockquote>"
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n重试次数<code>{{.Attempt}}/{{.Limit}}</code>\n失败前速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n原因<code>{{.Reason}}</code></blockquote>"
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} 等待中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>文件已上传,正在等待远端确认</i></blockquote>"
batch_summary_hidden_active: "🔄 另有 <code>{{.Count}}</code> 个文件正在处理"
batch_summary_confirming: "☁️ 已上传,等待整组发送:<code>{{.Count}}</code>"
batch_summary_failed: "❌ 失败:<code>{{.Count}}</code>"
batch_summary_skipped: "⏭️ 已跳过:<code>{{.Count}}</code>"
batch_done: "<b>✅ 处理完成</b>\n\n文件数: <code>{{.Count}}</code>\n总大小: <code>{{.Size}}</code>"
batch_done_with_skipped: "<b>⚠️ 处理完成</b>\n\n成功: <code>{{.Success}}</code>\n已跳过: <code>{{.Skipped}}</code>\n总大小: <code>{{.Size}}</code>"
batch_canceled: "<b>🚫 任务已取消</b>\n\n文件数: <code>{{.Total}}</code>\n已完成: <code>{{.Completed}}</code>\n未完成: <code>{{.Incomplete}}</code>\n已跳过: <code>{{.Skipped}}</code>"
batch_failed_item: "<b>❌ 处理失败</b>\n\n失败文件: <code>{{.Index}}. {{.Name}}</code>\n失败阶段: <code>{{.Stage}}</code>\n失败进度: <code>{{.Progress}}</code>\n失败前速度: <code>{{.Speed}}</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 失败: <code>{{.Failed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failed_group: "<b>❌ 批量上传失败</b>\n\n受影响文件: <code>{{.Affected}} 个</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 批次失败: <code>{{.Failed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failed_task: "<b>❌ 处理失败</b>\n\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failure_stage_download: "下载"
batch_failure_stage_cache: "本地缓存"
batch_failure_stage_upload: "上传"
batch_failure_stage_confirm: "云端确认"
batch_failure_stage_batch_upload: "批量上传"
batch_failure_stage_internal: "任务内部"
single_status_header: "<b>📦 正在处理</b>"
single_downloading: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_downloading_unknown: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知\n保存至<code>{{.Destination}}</code></blockquote>"
single_uploading: "<blockquote><b>⬆️ 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_upload_retrying: "<blockquote><b>🔁 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n尝试次数<code>{{.Attempt}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_done: "<b>✅ 处理完成</b>\n\n文件名<code>{{.Name}}</code>\n总大小<code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code>"
single_canceled: "<b>🚫 任务已取消</b>\n\n文件名<code>{{.Name}}</code>"
single_failed: "<b>❌ 处理失败</b>\n\n文件名<code>{{.Name}}</code>\n原因<code>{{.Reason}}</code>"
downloading_prefix: "正在下载\n总大小: "
processing_list_prefix: "\n正在处理:\n"
processing_none: " - 无"
avg_speed_prefix: "\n平均速度: "
current_progress_prefix: "\n当前进度: "
task_canceled: "任务已取消"
task_canceled_with_id: "处理已取消: {{.TaskID}}"
task_failed_with_error: "处理失败: {{.Error}}"
batch_done_prefix: "处理完成\n文件数: "
direct_done_prefix: "处理完成, 文件数量: "
parsed_start_prefix: "开始下载 {{.Site}} 的资源\n总大小: "
parsed_done_prefix: "处理完成, 资源数量: "
telegraph_start_prefix: "开始下载Telegraph\n图片数量: "
telegraph_progress_prefix: "正在下载\n当前进度: "
telegraph_done_prefix: "处理完成\n图片数量: "
file_start_prefix: "开始下载\n文件名: "
file_processing_prefix: "正在处理下载任务\n文件名: "
download_failed_prefix: "下载失败\n文件名: "
download_done_prefix: "下载完成\n文件名: "
file_size_prefix: "\n文件大小: "
save_path_prefix: "\n保存路径: "
total_size_prefix: "\n总大小: "
direct_start: "开始下载, 总大小: {{.SizeMB}} MB ({{.Count}} 个文件)"
file_name_prefix: "文件名: "
error_prefix: "\n错误: "
aria2_start: "等待 Aria2 下载完成 (GID: {{.GID}})..."
aria2_downloading: "Aria2 正在下载 (GID: {{.GID}})\n"

View File

@@ -0,0 +1,42 @@
package fsutil_test
import (
"errors"
"os"
"path/filepath"
"testing"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
)
func TestCloseAndRemove(t *testing.T) {
tests := []struct {
name string
preClose bool
}{
{name: "open file"},
{name: "already closed file", preClose: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "cache-file")
file, err := fsutil.CreateFile(filePath)
if err != nil {
t.Fatalf("CreateFile() failed: %v", err)
}
if tt.preClose {
if err := file.Close(); err != nil {
t.Fatalf("Close() failed: %v", err)
}
}
if err := file.CloseAndRemove(); err != nil {
t.Fatalf("CloseAndRemove() failed: %v", err)
}
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("cache file still exists after CloseAndRemove(): %v", err)
}
})
}
}

View File

@@ -1,6 +1,7 @@
package fsutil
import (
"errors"
"os"
"path/filepath"
"strings"
@@ -41,10 +42,11 @@ func (f *File) Remove() error {
}
func (f *File) CloseAndRemove() error {
if err := f.Close(); err != nil {
return err
closeErr := f.Close()
if errors.Is(closeErr, os.ErrClosed) {
closeErr = nil
}
return f.Remove()
return errors.Join(closeErr, f.Remove())
}
func CreateFile(fp string) (*File, error) {

View File

@@ -17,7 +17,11 @@ type ProgressReadSeeker struct {
// Seek implements io.ReadSeeker.
func (pr *ProgressReadSeeker) Seek(offset int64, whence int) (int64, error) {
return pr.reader.Seek(offset, whence)
position, err := pr.reader.Seek(offset, whence)
if err == nil {
pr.read.Store(position)
}
return position, err
}
// NewProgressReader creates a new ProgressReader
@@ -54,7 +58,7 @@ func (pr *ProgressReadSeeker) Progress() float64 {
return float64(pr.read.Load()) / float64(pr.total.Load())
}
// Read returns the number of bytes read so far
// BytesRead returns the current tracked reader position.
func (pr *ProgressReadSeeker) BytesRead() int64 {
return pr.read.Load()
}

View File

@@ -0,0 +1,50 @@
package ioutil
import (
"bytes"
"io"
"testing"
)
func TestProgressReadSeekerTracksReads(t *testing.T) {
var gotRead, gotTotal int64
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, func(read, total int64) {
gotRead = read
gotTotal = total
})
buffer := make([]byte, 4)
if _, err := io.ReadFull(reader, buffer); err != nil {
t.Fatalf("read failed: %v", err)
}
if gotRead != 4 || gotTotal != 6 {
t.Fatalf("progress = %d/%d, want 4/6", gotRead, gotTotal)
}
if reader.BytesRead() != 4 {
t.Fatalf("BytesRead() = %d, want 4", reader.BytesRead())
}
}
func TestProgressReadSeekerResetsPositionOnSeek(t *testing.T) {
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, nil)
buffer := make([]byte, 4)
if _, err := io.ReadFull(reader, buffer); err != nil {
t.Fatalf("read failed: %v", err)
}
position, err := reader.Seek(0, io.SeekStart)
if err != nil {
t.Fatalf("seek failed: %v", err)
}
if position != 0 || reader.BytesRead() != 0 {
t.Fatalf("position after seek = %d (tracked %d), want 0", position, reader.BytesRead())
}
if _, err := io.ReadFull(reader, buffer[:2]); err != nil {
t.Fatalf("read after seek failed: %v", err)
}
if reader.BytesRead() != 2 {
t.Fatalf("BytesRead() after seek and read = %d, want 2", reader.BytesRead())
}
}

View File

@@ -0,0 +1,35 @@
package tgutil
import (
stdhtml "html"
"github.com/gotd/td/telegram/message/entity"
messagehtml "github.com/gotd/td/telegram/message/html"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
)
// EscapeHTMLTemplateData returns a copy of data with string values escaped for
// interpolation into Telegram HTML templates.
func EscapeHTMLTemplateData(data map[string]any) map[string]any {
escaped := make(map[string]any, len(data))
for key, value := range data {
if text, ok := value.(string); ok {
escaped[key] = stdhtml.EscapeString(text)
continue
}
escaped[key] = value
}
return escaped
}
// RenderHTML renders Telegram-compatible HTML into plain text and message
// entities.
func RenderHTML(markup string) (string, []tg.MessageEntityClass, error) {
var builder entity.Builder
if err := styling.Perform(&builder, messagehtml.String(nil, markup)); err != nil {
return "", nil, err
}
text, entities := builder.Complete()
return text, entities, nil
}

View File

@@ -0,0 +1,54 @@
package tgutil
import (
"testing"
"github.com/gotd/td/tg"
)
func TestEscapeHTMLTemplateDataDoesNotMutateInput(t *testing.T) {
input := map[string]any{
"Text": `<b>A&B</b>`,
"Count": 2,
}
escaped := EscapeHTMLTemplateData(input)
if got, want := escaped["Text"], "&lt;b&gt;A&amp;B&lt;/b&gt;"; got != want {
t.Fatalf("escaped text = %q, want %q", got, want)
}
if got := input["Text"]; got != `<b>A&B</b>` {
t.Fatalf("input was mutated: %q", got)
}
if got := escaped["Count"]; got != 2 {
t.Fatalf("non-string value = %v, want 2", got)
}
}
func TestRenderHTMLUsesTemplateStylesAndDecodesValues(t *testing.T) {
data := EscapeHTMLTemplateData(map[string]any{"Name": `<b>A&B</b>.bin`})
markup := `<blockquote><b>Uploading</b>
<code>` + data["Name"].(string) + `</code></blockquote>`
text, entities, err := RenderHTML(markup)
if err != nil {
t.Fatalf("RenderHTML() failed: %v", err)
}
if want := "Uploading\n<b>A&B</b>.bin"; text != want {
t.Fatalf("rendered text = %q, want %q", text, want)
}
var bold, code, blockquote int
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
}
}
if bold != 1 || code != 1 || blockquote != 1 {
t.Fatalf("entity counts = bold:%d code:%d blockquote:%d", bold, code, blockquote)
}
}

View File

@@ -2,6 +2,7 @@ package tgutil
import (
"fmt"
"sort"
"strconv"
"strings"
"unicode"
@@ -359,9 +360,16 @@ func GetGroupedMessages(ctx *ext.Context, chatID int64, msg *tg.Message) ([]*tg.
groupedMessages = append(groupedMessages, m)
}
}
sortMessagesByID(groupedMessages)
return groupedMessages, nil
}
func sortMessagesByID(messages []*tg.Message) {
sort.Slice(messages, func(i, j int) bool {
return messages[i].GetID() < messages[j].GetID()
})
}
func ExtractMessageEntityUrls(msg *tg.Message) []string {
if len(msg.Entities) == 0 {
return nil

View File

@@ -0,0 +1,18 @@
package tgutil
import (
"testing"
"github.com/gotd/td/tg"
)
func TestSortMessagesByID(t *testing.T) {
messages := []*tg.Message{{ID: 9}, {ID: 3}, {ID: 7}}
sortMessagesByID(messages)
want := []int{3, 7, 9}
for i := range messages {
if messages[i].GetID() != want[i] {
t.Fatalf("message %d has ID %d, want %d", i, messages[i].GetID(), want[i])
}
}
}

View File

@@ -8,14 +8,16 @@ import (
type TelegramStorageConfig struct {
BaseConfig
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
// split files larger than Telegram limit(2GB) into parts of specified size, in MB, leave 0 to set default(2000MB)
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
SplitLargeVideo bool `toml:"split_large_video" mapstructure:"split_large_video" json:"split_large_video"`
// split files larger than the uploader account limit into parts of specified size, in MB
// leave 0 to use the account limit (2000MB for bots/regular users, 4000MB for Premium users)
// only effective when SkipLarge is false
// use zip when splitting
// use zip when splitting non-video files or when lossless video splitting is disabled/unavailable
SplitSizeMB int64 `toml:"split_size_mb" mapstructure:"split_size_mb" json:"split_size_mb"`
}

View File

@@ -6,6 +6,7 @@ import (
"io"
"os"
"path"
"time"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/retry"
@@ -14,45 +15,269 @@ import (
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
"github.com/krau/SaveAny-Bot/pkg/taskevent"
"github.com/krau/SaveAny-Bot/storage"
"golang.org/x/sync/errgroup"
)
type executionGroup struct {
elems []*TaskElement
batchSaver storage.StorageBatchSaver
}
func (g executionGroup) usesBatchSaver() bool {
return g.batchSaver != nil
}
func (t *Task) Execute(ctx context.Context) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("batch_file[%s]", t.ID))
logger.Info("Starting batch file task")
t.Progress.OnStart(ctx, t)
workers := config.C().Workers
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(workers)
for _, elem := range t.elems {
eg.Go(func() error {
t.processingMu.RLock()
if t.processing[elem.ID] != nil {
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
groups := t.executionGroups()
var err error
for i := 0; i < len(groups); {
if groups[i].usesBatchSaver() {
err = t.processBatch(ctx, groups[i])
i++
} else {
end := i + 1
for end < len(groups) && !groups[end].usesBatchSaver() {
end++
}
t.processingMu.RUnlock()
t.processingMu.Lock()
t.processing[elem.ID] = &elem
t.processingMu.Unlock()
defer func() {
t.processingMu.Lock()
delete(t.processing, elem.ID)
t.processingMu.Unlock()
}()
return t.processElement(gctx, elem)
})
elems := make([]*TaskElement, 0, end-i)
for _, group := range groups[i:end] {
elems = append(elems, group.elems...)
}
err = t.processElements(ctx, elems)
i = end
}
if err != nil {
break
}
}
err := eg.Wait()
if err != nil {
logger.Errorf("Error during batch file processing: %v", err)
} else {
logger.Info("Batch file task completed successfully")
}
t.finishItems(err)
t.Progress.OnDone(ctx, t, err)
return err
}
func (t *Task) executionGroups() []executionGroup {
groups := make([]executionGroup, 0, len(t.elems))
for i := 0; i < len(t.elems); {
elem := &t.elems[i]
batchSaver, batchCapable := elem.Storage.(storage.StorageBatchSaver)
if !batchCapable || elem.sourceGroupKey == "" {
groups = append(groups, executionGroup{elems: []*TaskElement{elem}})
i++
continue
}
end := i + 1
for end < len(t.elems) {
next := &t.elems[end]
if next.Storage != elem.Storage || next.sourceGroupKey != elem.sourceGroupKey {
break
}
end++
}
elems := make([]*TaskElement, 0, end-i)
for j := i; j < end; j++ {
elems = append(elems, &t.elems[j])
}
groups = append(groups, executionGroup{elems: elems, batchSaver: batchSaver})
i = end
}
return groups
}
func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error {
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(config.C().Workers)
for _, elem := range elems {
eg.Go(func() error {
if err := t.markProcessing(ctx, elem); err != nil {
return err
}
defer t.unmarkProcessing(elem.ID)
return t.processElement(gctx, *elem)
})
}
return eg.Wait()
}
func (t *Task) processBatch(ctx context.Context, group executionGroup) error {
defer func() {
for _, elem := range group.elems {
if err := os.Remove(elem.localPath); err != nil && !os.IsNotExist(err) {
log.FromContext(ctx).Warnf("Failed to cleanup batch cache file %s: %v", elem.localPath, err)
}
}
}()
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(config.C().Workers)
for _, elem := range group.elems {
eg.Go(func() error {
if err := t.markProcessing(ctx, elem); err != nil {
return err
}
defer t.unmarkProcessing(elem.ID)
return t.downloadElement(gctx, elem)
})
}
if err := eg.Wait(); err != nil {
return err
}
items := make([]storagetypes.BatchItem, 0, len(group.elems))
openFiles := make([]*os.File, 0, len(group.elems))
defer func() {
for _, file := range openFiles {
if err := file.Close(); err != nil {
log.FromContext(ctx).Warnf("Failed to close batch cache file %s: %v", file.Name(), err)
}
}
}()
for _, elem := range group.elems {
file, err := os.Open(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
return fmt.Errorf("failed to open cache file: %w", err)
}
stat, err := file.Stat()
if err != nil {
file.Close()
t.markItemFailed(elem.ID, FailureStageCache, err)
return fmt.Errorf("failed to get cache file stat: %w", err)
}
openFiles = append(openFiles, file)
items = append(items, storagetypes.BatchItem{
Reader: file,
StoragePath: elem.Path,
Size: stat.Size(),
SourceGroupKey: elem.sourceGroupKey,
Caption: elem.sourceCaption,
PreserveCaption: elem.preserveCaption,
})
}
for index, item := range items {
t.recordDownloadComplete(group.elems[index].ID, item.Size)
}
return t.saveBatchItems(ctx, group, items)
}
func (t *Task) saveBatchItems(ctx context.Context, group executionGroup, items []storagetypes.BatchItem) error {
t.startUpload(ctx)
if progressSaver, ok := group.batchSaver.(storage.StorageBatchProgressSaver); ok {
err := progressSaver.SaveBatchWithProgress(ctx, items, func(index int, uploaded, total int64) {
if index < 0 || index >= len(group.elems) {
return
}
t.uploadCallback(ctx, group.elems[index].ID)(uploaded, total)
})
if err != nil {
for _, elem := range group.elems {
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
}
t.notifyStateChange(ctx)
return fmt.Errorf("failed to save batch: %w", err)
}
for index, elem := range group.elems {
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
t.markItemCompleted(elem.ID)
}
t.notifyStateChange(ctx)
return nil
}
for i := range items {
items[i].Reader = ioutil.NewProgressReader(
items[i].Reader,
items[i].Size,
t.uploadCallback(ctx, group.elems[i].ID),
)
}
if err := group.batchSaver.SaveBatch(ctx, items); err != nil {
for _, elem := range group.elems {
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
}
t.notifyStateChange(ctx)
return fmt.Errorf("failed to save batch: %w", err)
}
for index, elem := range group.elems {
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
t.markItemCompleted(elem.ID)
}
t.notifyStateChange(ctx)
return nil
}
func (t *Task) markProcessing(ctx context.Context, elem *TaskElement) error {
t.processingMu.Lock()
if t.processing[elem.ID] != nil {
t.processingMu.Unlock()
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
}
t.processing[elem.ID] = elem
t.processingMu.Unlock()
t.markItemActive(elem.ID, elem.stream, time.Now())
t.Progress.OnProgress(ctx, t)
return nil
}
func (t *Task) unmarkProcessing(id string) {
t.processingMu.Lock()
delete(t.processing, id)
t.processingMu.Unlock()
}
func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
logger.Info("Starting file download")
localFile, err := fsutil.CreateFile(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to create local file: %w", err)
}
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
TaskID: t.ID,
Phase: taskevent.PhaseProgress,
TotalBytes: t.totalSize,
DownloadedBytes: downloaded,
})
})
_, downloadErr := tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
closeErr := localFile.Close()
if downloadErr != nil {
t.markItemFailed(elem.ID, FailureStageDownload, downloadErr)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to download file: %w", downloadErr)
}
if closeErr != nil {
t.markItemFailed(elem.ID, FailureStageCache, closeErr)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to close cache file: %w", closeErr)
}
logger.Info("File downloaded successfully")
if path.Ext(elem.FileName()) == "" {
if ext := fsutil.DetectFileExt(elem.localPath); ext != "" {
elem.Path += ext
}
}
t.markItemDownloaded(elem.ID)
t.Progress.OnProgress(ctx, t)
return nil
}
func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
if elem.stream {
@@ -60,9 +285,15 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
defer pr.Close()
errg, uploadCtx := errgroup.WithContext(ctx)
errg.Go(func() error {
return elem.Storage.Save(uploadCtx, pr, elem.Path)
err := elem.Storage.Save(uploadCtx, pr, elem.Path)
if err != nil {
t.markItemFailed(elem.ID, FailureStageUpload, err)
t.notifyStateChange(ctx)
}
return err
})
wr := ioutil.NewProgressWriter(pw, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
@@ -78,6 +309,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
_, err := tdler.NewDownloader(elem.File).Stream(uploadCtx, wr)
if err != nil {
logger.Errorf("Failed to download file: %v", err)
t.markItemFailed(elem.ID, FailureStageDownload, err)
t.notifyStateChange(ctx)
pw.CloseWithError(err)
}
return err
@@ -85,12 +318,17 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
if err := errg.Wait(); err != nil {
return fmt.Errorf("failed to download file in stream mode: %w", err)
}
t.recordDownloadComplete(elem.ID, 0)
t.markItemCompleted(elem.ID)
t.notifyStateChange(ctx)
logger.Info("File downloaded successfully in stream mode")
return nil
}
logger.Info("Starting file download")
localFile, err := fsutil.CreateFile(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to create local file: %w", err)
}
defer func() {
@@ -99,6 +337,7 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
}
}()
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
@@ -110,6 +349,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
})
_, err = tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
if err != nil {
t.markItemFailed(elem.ID, FailureStageDownload, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to download file: %w", err)
}
logger.Info("File downloaded successfully")
@@ -122,21 +363,50 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
var fileStat os.FileInfo
fileStat, err = os.Stat(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to get file stat: %w", err)
}
t.recordDownloadComplete(elem.ID, fileStat.Size())
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
t.startUpload(vctx)
onProgress := t.uploadCallback(vctx, elem.ID)
attempt := 0
retryLimit := int(config.C().Retry)
lastFailureStage := FailureStageUpload
err = retry.Retry(func() error {
attempt++
var file *os.File
file, err = os.Open(elem.localPath)
if err != nil {
lastFailureStage = FailureStageCache
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
t.notifyStateChange(vctx)
return fmt.Errorf("failed to open cache file: %w", err)
}
defer file.Close()
if err = elem.Storage.Save(vctx, file, elem.Path); err != nil {
onProgress(0, fileStat.Size())
if progressSaver, ok := elem.Storage.(storage.StorageProgressSaver); ok {
err = progressSaver.SaveWithProgress(vctx, file, elem.Path, onProgress)
} else {
err = elem.Storage.Save(vctx, ioutil.NewProgressReader(file, fileStat.Size(), onProgress), elem.Path)
}
if err != nil {
logger.Errorf("Failed to save file: %s, retrying...", err)
lastFailureStage = t.itemFailureStage(elem.ID)
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
t.notifyStateChange(vctx)
return err
}
return nil
}, retry.Context(vctx), retry.RetryTimes(uint(config.C().Retry)))
if err == nil {
onProgress(fileStat.Size(), fileStat.Size())
t.markItemCompleted(elem.ID)
t.notifyStateChange(vctx)
} else {
t.markItemFailed(elem.ID, lastFailureStage, err)
t.notifyStateChange(vctx)
}
return err
}

View File

@@ -0,0 +1,57 @@
package batchtfile
import (
"testing"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/pkg/tfile"
tgstorage "github.com/krau/SaveAny-Bot/storage/telegram"
)
func TestExecutionGroupsPreserveSourceAlbums(t *testing.T) {
stor := new(tgstorage.Telegram)
otherStor := new(tgstorage.Telegram)
task := Task{elems: []TaskElement{
{Storage: stor, sourceGroupKey: "album-1"},
{Storage: stor, sourceGroupKey: "album-1"},
{Storage: stor},
{Storage: stor, sourceGroupKey: "album-2"},
{Storage: stor, sourceGroupKey: "album-2"},
{Storage: otherStor, sourceGroupKey: "album-2"},
}}
groups := task.executionGroups()
wantSizes := []int{2, 1, 2, 1}
wantBatch := []bool{true, false, true, true}
if len(groups) != len(wantSizes) {
t.Fatalf("got %d groups, want %d", len(groups), len(wantSizes))
}
for i := range groups {
if got := len(groups[i].elems); got != wantSizes[i] {
t.Errorf("group %d has %d elements, want %d", i, got, wantSizes[i])
}
if got := groups[i].usesBatchSaver(); got != wantBatch[i] {
t.Errorf("group %d batch=%v, want %v", i, got, wantBatch[i])
}
}
}
func TestSourceMetadataPreservesAlbumIdentityAndCaption(t *testing.T) {
msg := &tg.Message{
PeerID: &tg.PeerChannel{ChannelID: 77},
Message: "original caption",
}
msg.SetGroupedID(42)
file := tfile.NewTGFile(nil, nil, 0, "photo.jpg", tfile.WithMessage(msg))
groupKey, caption, preserveCaption := sourceMetadata(file)
if groupKey != "*tg.PeerChannel:77:42" {
t.Fatalf("group key = %q, want %q", groupKey, "*tg.PeerChannel:77:42")
}
if caption != "original caption" {
t.Fatalf("caption = %q, want original caption", caption)
}
if !preserveCaption {
t.Fatal("preserveCaption = false, want true")
}
}

View File

@@ -0,0 +1,360 @@
package batchtfile
import (
"context"
"errors"
"strings"
"time"
)
const (
transferSpeedWindow = 5 * time.Second
transferSamplePeriod = 250 * time.Millisecond
)
// ItemPhase describes the current lifecycle stage of one batch item.
type ItemPhase uint8
const (
ItemPhaseWaiting ItemPhase = iota
ItemPhaseDownloading
ItemPhaseTransferring
ItemPhaseDownloaded
ItemPhaseUploading
ItemPhaseRetrying
ItemPhaseConfirming
ItemPhaseCompleted
ItemPhaseFailed
ItemPhaseStopped
)
// FailureStage identifies the operation that failed for one batch item.
type FailureStage uint8
const (
FailureStageNone FailureStage = iota
FailureStageDownload
FailureStageCache
FailureStageUpload
FailureStageConfirm
FailureStageBatchUpload
FailureStageInternal
)
// TaskItemProgress is an immutable progress snapshot for one batch item.
type TaskItemProgress struct {
Index int
ID string
Name string
Size int64
Downloaded int64
Uploaded int64
DownloadSpeed float64
UploadSpeed float64
Phase ItemPhase
FailureStage FailureStage
RetryAttempt int
RetryLimit int
Error string
}
type transferSample struct {
at time.Time
bytes int64
}
type transferMeter struct {
samples []transferSample
latest transferSample
hasData bool
}
func (m *transferMeter) record(now time.Time, transferred int64) {
if m.hasData && transferred < m.latest.bytes {
m.reset()
}
if m.hasData && now.Before(m.latest.at) {
now = m.latest.at
}
sample := transferSample{at: now, bytes: transferred}
m.latest = sample
m.hasData = true
if len(m.samples) == 0 {
m.samples = append(m.samples, sample)
return
}
if now.Sub(m.samples[len(m.samples)-1].at) >= transferSamplePeriod {
m.samples = append(m.samples, sample)
}
cutoff := now.Add(-transferSpeedWindow)
for len(m.samples) > 1 && m.samples[0].at.Before(cutoff) {
m.samples = m.samples[1:]
}
}
func (m *transferMeter) speed() float64 {
if !m.hasData || len(m.samples) == 0 {
return 0
}
first := m.samples[0]
last := m.latest
elapsed := last.at.Sub(first.at).Seconds()
if elapsed <= 0 || last.bytes <= first.bytes {
return 0
}
return float64(last.bytes-first.bytes) / elapsed
}
func (m *transferMeter) reset() {
m.samples = m.samples[:0]
m.latest = transferSample{}
m.hasData = false
}
type itemProgressState struct {
index int
id string
name string
expectedSize int64
actualSize int64
downloaded int64
uploaded int64
phase ItemPhase
failureStage FailureStage
retryAttempt int
retryLimit int
err string
downloadMeter transferMeter
uploadMeter transferMeter
}
func newItemProgressStates(elems []TaskElement) ([]itemProgressState, map[string]int) {
states := make([]itemProgressState, 0, len(elems))
index := make(map[string]int, len(elems))
for i, elem := range elems {
name := ""
size := int64(0)
if elem.File != nil {
name = elem.File.Name()
size = elem.File.Size()
}
states = append(states, itemProgressState{
index: i + 1,
id: elem.ID,
name: name,
expectedSize: size,
phase: ItemPhaseWaiting,
})
index[elem.ID] = i
}
return states, index
}
func (t *Task) updateItem(id string, update func(*itemProgressState)) bool {
t.itemMu.Lock()
defer t.itemMu.Unlock()
index, ok := t.itemIndex[id]
if !ok || index < 0 || index >= len(t.itemStates) {
return false
}
update(&t.itemStates[index])
return true
}
func (t *Task) markItemActive(id string, stream bool, now time.Time) {
t.updateItem(id, func(item *itemProgressState) {
if stream {
item.phase = ItemPhaseTransferring
} else {
item.phase = ItemPhaseDownloading
}
item.failureStage = FailureStageNone
item.err = ""
item.downloadMeter.record(now, item.downloaded)
if stream {
item.uploadMeter.record(now, item.uploaded)
}
})
}
func (t *Task) recordItemDownload(id string, n int64, now time.Time) {
if n <= 0 {
return
}
t.updateItem(id, func(item *itemProgressState) {
item.downloaded += n
item.downloadMeter.record(now, item.downloaded)
if item.phase == ItemPhaseTransferring {
item.uploaded += n
item.uploadMeter.record(now, item.uploaded)
}
})
}
func (t *Task) markItemDownloaded(id string) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseDownloaded
})
}
func (t *Task) recordItemDownloaded(id string, actualSize int64) {
t.updateItem(id, func(item *itemProgressState) {
if actualSize > 0 {
item.actualSize = actualSize
}
if item.actualSize == 0 {
item.actualSize = item.downloaded
}
if item.phase != ItemPhaseTransferring {
item.phase = ItemPhaseDownloaded
item.uploadMeter.reset()
}
})
}
func (t *Task) recordItemUpload(id string, uploaded, total int64, now time.Time) bool {
becameConfirming := false
t.updateItem(id, func(item *itemProgressState) {
if uploaded < item.uploaded {
if item.phase != ItemPhaseRetrying {
return
}
item.uploadMeter.reset()
}
if total > 0 {
item.actualSize = total
}
item.uploaded = uploaded
item.uploadMeter.record(now, uploaded)
if total > 0 && uploaded >= total {
becameConfirming = item.phase != ItemPhaseConfirming
item.phase = ItemPhaseConfirming
return
}
item.phase = ItemPhaseUploading
item.failureStage = FailureStageNone
item.err = ""
})
return becameConfirming
}
func (t *Task) markItemRetry(id string, stage FailureStage, attempt, limit int, err error) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseRetrying
item.failureStage = stage
item.retryAttempt = attempt
item.retryLimit = limit
item.err = compactError(err)
})
}
func (t *Task) markItemFailed(id string, stage FailureStage, err error) {
t.updateItem(id, func(item *itemProgressState) {
if item.phase == ItemPhaseFailed || item.phase == ItemPhaseCompleted {
return
}
if errors.Is(err, context.Canceled) {
item.phase = ItemPhaseStopped
return
}
item.phase = ItemPhaseFailed
item.failureStage = stage
item.err = compactError(err)
})
}
func (t *Task) markItemCompleted(id string) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseCompleted
item.failureStage = FailureStageNone
item.err = ""
item.retryAttempt = 0
item.retryLimit = 0
if item.actualSize == 0 {
item.actualSize = max(item.downloaded, item.uploaded)
}
})
}
func (t *Task) finishItems(err error) {
if err == nil {
return
}
t.itemMu.Lock()
defer t.itemMu.Unlock()
for i := range t.itemStates {
item := &t.itemStates[i]
if item.phase == ItemPhaseCompleted || item.phase == ItemPhaseFailed {
continue
}
item.phase = ItemPhaseStopped
}
}
func (t *Task) itemFailureStage(id string) FailureStage {
t.itemMu.RLock()
defer t.itemMu.RUnlock()
index, ok := t.itemIndex[id]
if !ok || index < 0 || index >= len(t.itemStates) {
return FailureStageUpload
}
if t.itemStates[index].phase == ItemPhaseConfirming {
return FailureStageConfirm
}
return FailureStageUpload
}
func (t *Task) Items() []TaskItemProgress {
t.itemMu.RLock()
defer t.itemMu.RUnlock()
items := make([]TaskItemProgress, 0, len(t.itemStates))
for i := range t.itemStates {
item := &t.itemStates[i]
size := item.actualSize
if size == 0 {
size = item.expectedSize
}
items = append(items, TaskItemProgress{
Index: item.index,
ID: item.id,
Name: item.name,
Size: size,
Downloaded: item.downloaded,
Uploaded: item.uploaded,
DownloadSpeed: item.downloadMeter.speed(),
UploadSpeed: item.uploadMeter.speed(),
Phase: item.phase,
FailureStage: item.failureStage,
RetryAttempt: item.retryAttempt,
RetryLimit: item.retryLimit,
Error: item.err,
})
}
return items
}
func (t *Task) ActualTotalSize() int64 {
items := t.Items()
var total int64
for _, item := range items {
total += item.Size
}
return total
}
type stateProgressTracker interface {
OnStateChange(ctx context.Context, info TaskInfo)
}
func (t *Task) notifyStateChange(ctx context.Context) {
if tracker, ok := t.Progress.(stateProgressTracker); ok {
tracker.OnStateChange(ctx, t)
}
}
func compactError(err error) string {
if err == nil {
return ""
}
return strings.Join(strings.Fields(err.Error()), " ")
}

View File

@@ -4,20 +4,19 @@ import (
"context"
"errors"
"fmt"
"strconv"
"path"
"strings"
"sync/atomic"
"sync"
"time"
"unicode/utf8"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/slice"
"github.com/gotd/td/telegram/message/entity"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/config"
)
type ProgressTracker interface {
@@ -27,159 +26,471 @@ type ProgressTracker interface {
}
type Progress struct {
MessageID int
ChatID int64
start time.Time
lastUpdatePercent atomic.Int32
skippedFiles []string
MessageID int
ChatID int64
updateMu sync.Mutex
lastUpdateAt time.Time
lastText string
done bool
skippedFiles []string
}
type renderedBatchMessage struct {
Text string
Entities []tg.MessageEntityClass
Err error
}
const (
progressRenderInterval = time.Second
maxVisibleActiveItems = 5
progressBarWidth = 10
maxDisplayNameRunes = 36
maxDisplayErrorRunes = 240
)
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
p.start = time.Now()
p.lastUpdatePercent.Store(0)
log.FromContext(ctx).Debugf("Batch task progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchStartPrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
return
}
p.render(ctx, info, true)
}
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) {
if !shouldUpdateProgress(info.TotalSize(), info.Downloaded(), int(p.lastUpdatePercent.Load())) {
p.render(ctx, info, false)
}
func (p *Progress) OnStateChange(ctx context.Context, info TaskInfo) {
p.render(ctx, info, true)
}
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, _ int64) {
p.render(ctx, info, true)
}
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, _, _ int64) {
p.render(ctx, info, false)
}
func (p *Progress) render(ctx context.Context, info TaskInfo, priority bool) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
if p.done {
return
}
percent := int((info.Downloaded() * 100) / info.TotalSize())
if p.lastUpdatePercent.Load() == int32(percent) {
now := time.Now()
if !priority && !p.lastUpdateAt.IsZero() && now.Sub(p.lastUpdateAt) < progressRenderInterval {
return
}
p.lastUpdatePercent.Store(int32(percent))
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.TaskID(), info.Downloaded(), info.TotalSize())
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchProcessingPrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
styling.Plain(i18n.T(i18nk.BotMsgProgressProcessingListPrefix, nil)),
func() styling.StyledTextOption {
var lines []string
for _, elem := range info.Processing() {
lines = append(lines, fmt.Sprintf(" - %s (%.2f MB)", elem.FileName(), float64(elem.FileSize())/(1024*1024)))
}
if len(lines) == 0 {
lines = append(lines, i18n.T(i18nk.BotMsgProgressProcessingNone, nil))
}
return styling.Plain(slice.Join(lines, "\n"))
}(),
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(info.Downloaded(), p.start)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f%%", float64(info.Downloaded())/float64(info.TotalSize())*100)),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
message := buildBatchProgressMessage(info, p.skippedFiles, visibleActiveItems())
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
if message.Text == p.lastText {
return
}
p.lastText = message.Text
p.lastUpdateAt = now
p.editMessage(ctx, info.TaskID(), message, true)
}
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
if err != nil {
log.FromContext(ctx).Errorf("Batch task %s failed: %s", info.TaskID(), err)
} else {
log.FromContext(ctx).Debugf("Batch task %s completed successfully", info.TaskID())
}
entityBuilder := entity.Builder{}
var stylingErr error
if err != nil {
if errors.Is(err, context.Canceled) {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
)
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskFailedWithError, map[string]any{
"Error": "",
})),
styling.Code(err.Error()),
)
}
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchDonePrefix, nil)),
styling.Code(strconv.Itoa(info.Count())),
styling.Plain(i18n.T(i18nk.BotMsgProgressTotalSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(info.TotalSize())/(1024*1024))),
func() styling.StyledTextOption {
if len(p.skippedFiles) == 0 {
return styling.Plain("")
}
return styling.Plain("\n\n" + i18n.T(i18nk.BotMsgCommonInfoConflictFilesSkipped, map[string]any{
"Skipped": strings.Join(p.skippedFiles, "\n"),
}))
}(),
)
}
if stylingErr != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
p.updateMu.Lock()
defer p.updateMu.Unlock()
if p.done {
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
p.done = true
message := buildBatchDoneMessage(info, p.skippedFiles, err)
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render final batch progress message: %v", message.Err)
return
}
req.SetMessage(text)
req.SetEntities(entities)
p.lastText = message.Text
p.editMessage(ctx, info.TaskID(), message, false)
}
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedBatchMessage, cancellable bool) {
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
return
}
req := buildBatchEditMessageRequest(p.MessageID, taskID, message, cancellable)
if ext := tgutil.ExtFromContext(ctx); ext != nil {
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
log.FromContext(ctx).Errorf("Failed to edit batch progress message: %v", err)
}
}
}
func buildBatchEditMessageRequest(messageID int, taskID string, message renderedBatchMessage, cancellable bool) *tg.MessagesEditMessageRequest {
req := &tg.MessagesEditMessageRequest{ID: messageID}
req.SetMessage(message.Text)
if len(message.Entities) > 0 {
req.SetEntities(message.Entities)
}
if cancellable {
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
}}})
}
return req
}
func buildBatchProgressText(info TaskInfo, skipped []string, activeLimit int) string {
return buildBatchProgressMessage(info, skipped, activeLimit).Text
}
func buildBatchProgressMessage(info TaskInfo, skipped []string, activeLimit int) renderedBatchMessage {
items := info.Items()
completed, waiting, downloaded, failed := itemCounts(items)
downloadSpeed, uploadSpeed := aggregateSpeeds(items)
if activeLimit < 1 {
activeLimit = 1
}
total := len(items) + len(skipped)
downloadSpeedText := formatSpeed(downloadSpeed)
uploadSpeedText := formatSpeed(uploadSpeed)
header := localizedProgressMarkup(i18nk.BotMsgProgressBatchStatusHeader, map[string]any{
"Total": total,
"Completed": completed,
"Downloaded": downloaded,
"Waiting": waiting,
"DownloadSpeed": downloadSpeedText,
"UploadSpeed": uploadSpeedText,
})
var markup strings.Builder
markup.WriteString(header)
visibleItems, hiddenTransfers, summarizedConfirming := visibleBatchItems(items, activeLimit)
for _, item := range visibleItems {
markup.WriteString("\n\n")
markup.WriteString(formatActiveItemMarkup(item, len(items)))
}
if hiddenTransfers > 0 {
markup.WriteString("\n\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryHiddenActive, map[string]any{"Count": hiddenTransfers}))
}
if summarizedConfirming > 0 {
markup.WriteString("\n\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryConfirming, map[string]any{"Count": summarizedConfirming}))
}
if failed > 0 {
markup.WriteString("\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryFailed, map[string]any{"Count": failed}))
}
if len(skipped) > 0 {
markup.WriteString("\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummarySkipped, map[string]any{"Count": len(skipped)}))
}
return completeBatchMessage(markup.String())
}
func buildBatchDoneMarkup(info TaskInfo, skipped []string, err error) string {
items := info.Items()
totalSize := info.ActualTotalSize()
if totalSize == 0 {
totalSize = info.TotalSize()
}
if err == nil {
if len(skipped) > 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDoneWithSkipped, map[string]any{
"Success": len(items),
"Skipped": len(skipped),
"Size": dlutil.FormatSize(totalSize),
})
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDone, map[string]any{
"Count": len(items),
"Size": dlutil.FormatSize(totalSize),
})
}
completed, _, _, failed := itemCounts(items)
incomplete := max(len(items)-completed-failed, 0)
if errors.Is(err, context.Canceled) {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchCanceled, map[string]any{
"Total": len(items) + len(skipped),
"Completed": completed,
"Incomplete": incomplete,
"Skipped": len(skipped),
})
}
failedItems := make([]TaskItemProgress, 0, failed)
for _, item := range items {
if item.Phase == ItemPhaseFailed {
failedItems = append(failedItems, item)
}
}
if len(failedItems) > 1 && failedItems[0].FailureStage == FailureStageBatchUpload {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedGroup, map[string]any{
"Affected": len(failedItems),
"Reason": displayError(firstError(failedItems), err),
"Completed": completed,
"Failed": len(failedItems),
"Incomplete": incomplete,
})
}
if len(failedItems) == 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedTask, map[string]any{
"Reason": displayError("", err),
"Completed": completed,
"Incomplete": incomplete,
})
}
item := failedItems[0]
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedItem, map[string]any{
"Index": item.Index,
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
"Stage": failureStageLabel(item.FailureStage),
"Progress": failureProgress(item),
"Speed": failureSpeed(item),
"Reason": displayError(item.Error, err),
"Completed": completed,
"Failed": failed,
"Incomplete": incomplete,
})
}
func buildBatchDoneMessage(info TaskInfo, skipped []string, err error) renderedBatchMessage {
return completeBatchMessage(buildBatchDoneMarkup(info, skipped, err))
}
func formatActiveItemMarkup(item TaskItemProgress, total int) string {
data := map[string]any{
"Index": item.Index,
"Total": total,
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
"Speed": formatSpeed(itemSpeed(item)),
"Progress": itemPercent(item),
"Bar": textProgressBar(itemPercent(item)),
"Current": dlutil.FormatSize(itemBytes(item)),
"Size": dlutil.FormatSize(item.Size),
"Attempt": min(max(item.RetryAttempt, 1), max(item.RetryLimit, 1)),
"Limit": max(item.RetryLimit, 1),
"Reason": truncateRunes(item.Error, maxDisplayErrorRunes),
}
switch item.Phase {
case ItemPhaseDownloading:
if item.Size <= 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloadingUnknown, data)
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloading, data)
case ItemPhaseTransferring:
if item.Size <= 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferringUnknown, data)
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferring, data)
case ItemPhaseUploading:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemUploading, data)
case ItemPhaseRetrying:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemRetrying, data)
case ItemPhaseConfirming:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemConfirming, data)
default:
return ""
}
}
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
}
func visibleBatchItems(items []TaskItemProgress, limit int) (visible []TaskItemProgress, hiddenTransfers, summarizedConfirming int) {
visible = make([]TaskItemProgress, 0, limit)
transferCount := 0
confirmingCount := 0
for _, item := range items {
switch {
case isTransferPhase(item.Phase):
transferCount++
if len(visible) < limit {
visible = append(visible, item)
}
case item.Phase == ItemPhaseConfirming:
confirmingCount++
}
}
hiddenTransfers = transferCount - len(visible)
if confirmingCount == 1 && len(visible) < limit {
for _, item := range items {
if item.Phase == ItemPhaseConfirming {
visible = append(visible, item)
return visible, hiddenTransfers, 0
}
}
}
return visible, hiddenTransfers, confirmingCount
}
func completeBatchMessage(markup string) renderedBatchMessage {
text, entities, err := tgutil.RenderHTML(markup)
return renderedBatchMessage{Text: text, Entities: entities, Err: err}
}
func itemCounts(items []TaskItemProgress) (completed, waiting, downloaded, failed int) {
for _, item := range items {
switch item.Phase {
case ItemPhaseCompleted:
completed++
case ItemPhaseWaiting:
waiting++
case ItemPhaseDownloaded:
downloaded++
case ItemPhaseFailed:
failed++
}
}
return
}
func aggregateSpeeds(items []TaskItemProgress) (download, upload float64) {
for _, item := range items {
switch item.Phase {
case ItemPhaseDownloading:
download += item.DownloadSpeed
case ItemPhaseTransferring:
download += item.DownloadSpeed
upload += item.UploadSpeed
case ItemPhaseUploading:
upload += item.UploadSpeed
}
}
return
}
func isTransferPhase(phase ItemPhase) bool {
switch phase {
case ItemPhaseDownloading, ItemPhaseTransferring, ItemPhaseUploading, ItemPhaseRetrying:
return true
default:
return false
}
}
func itemBytes(item TaskItemProgress) int64 {
switch item.Phase {
case ItemPhaseDownloading, ItemPhaseTransferring:
return item.Downloaded
default:
return item.Uploaded
}
}
func itemSpeed(item TaskItemProgress) float64 {
switch item.Phase {
case ItemPhaseDownloading, ItemPhaseTransferring:
return item.DownloadSpeed
case ItemPhaseUploading:
return item.UploadSpeed
case ItemPhaseRetrying:
return item.UploadSpeed
default:
return 0
}
}
func itemPercent(item TaskItemProgress) int {
if item.Size <= 0 {
return 0
}
return int(min(itemBytes(item), item.Size) * 100 / item.Size)
}
func textProgressBar(percent int) string {
percent = min(max(percent, 0), 100)
filled := percent * progressBarWidth / 100
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", progressBarWidth-filled)
}
func formatSpeed(speed float64) string {
if speed <= 0 {
return "0 B/s"
}
return dlutil.FormatSize(int64(speed)) + "/s"
}
func truncateFilename(name string, limit int) string {
if utf8.RuneCountInString(name) <= limit {
return name
}
ext := path.Ext(name)
if utf8.RuneCountInString(ext) >= limit-2 {
return truncateRunes(name, limit-1) + "…"
}
base := strings.TrimSuffix(name, ext)
baseLimit := limit - utf8.RuneCountInString(ext) - 1
return truncateRunes(base, baseLimit) + "…" + ext
}
func truncateRunes(value string, limit int) string {
if limit <= 0 {
return ""
}
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}
func displayError(itemError string, fallback error) string {
if itemError == "" && fallback != nil {
itemError = compactError(fallback)
}
return truncateRunes(itemError, maxDisplayErrorRunes)
}
func firstError(items []TaskItemProgress) string {
for _, item := range items {
if item.Error != "" {
return item.Error
}
}
return ""
}
func failureStageLabel(stage FailureStage) string {
switch stage {
case FailureStageDownload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageDownload, nil)
case FailureStageCache:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageCache, nil)
case FailureStageUpload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageUpload, nil)
case FailureStageConfirm:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageConfirm, nil)
case FailureStageBatchUpload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageBatchUpload, nil)
default:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageInternal, nil)
}
}
func failureProgress(item TaskItemProgress) string {
if item.Size <= 0 {
return dlutil.FormatSize(failureBytes(item))
}
return fmt.Sprintf("%d%%", min(failureBytes(item), item.Size)*100/item.Size)
}
func failureSpeed(item TaskItemProgress) string {
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
return formatSpeed(item.DownloadSpeed)
}
return formatSpeed(item.UploadSpeed)
}
func failureBytes(item TaskItemProgress) int64 {
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
return item.Downloaded
}
return item.Uploaded
}
func visibleActiveItems() int {
return min(max(config.C().Workers, 1), maxVisibleActiveItems)
}
func NewProgressTracker(messageID int, chatID int64) ProgressTracker {

View File

@@ -0,0 +1,311 @@
package batchtfile
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/pkg/tfile"
)
type progressRegressionRecorder struct {
mu sync.Mutex
startTotal int64
notifications []int64
}
func (*progressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
func (*progressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
func (*progressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
func (r *progressRegressionRecorder) OnUploadStart(_ context.Context, _ TaskInfo, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
r.startTotal = total
}
func (r *progressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
r.mu.Lock()
defer r.mu.Unlock()
r.notifications = append(r.notifications, uploaded)
}
type orderedProgressRegressionRecorder struct {
firstEntered chan struct{}
releaseFirst chan struct{}
secondEntered chan struct{}
mu sync.Mutex
notifications []int64
}
func (*orderedProgressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
func (*orderedProgressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
func (*orderedProgressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
func (*orderedProgressRegressionRecorder) OnUploadStart(context.Context, TaskInfo, int64) {
}
func (r *orderedProgressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
if uploaded == 100 {
close(r.firstEntered)
<-r.releaseFirst
}
if uploaded == 200 {
close(r.secondEntered)
}
r.mu.Lock()
r.notifications = append(r.notifications, uploaded)
r.mu.Unlock()
}
func TestBatchProgressShowsTransferSpeedAndSize(t *testing.T) {
useProgressRegressionLocale(t)
task := newProgressRegressionTask(nil,
progressRegressionFile{"downloading", 1000},
progressRegressionFile{"uploading", 1000},
progressRegressionFile{"waiting", 1000},
)
started := time.Unix(100, 0)
task.markItemActive("downloading", false, started)
task.recordItemDownload("downloading", 500, started.Add(time.Second))
task.recordItemDownloaded("uploading", 1000)
task.recordItemUpload("uploading", 0, 1000, started.Add(time.Second))
task.recordItemUpload("uploading", 250, 1000, started.Add(2*time.Second))
message := buildBatchProgressMessage(task, nil, 2)
if message.Err != nil {
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
}
assertProgressRegressionContains(t, message.Text,
"状态:✅ 0 📥 0 ⏳ 1",
"⬇️ 1/3 下载中",
"速度500 B/s",
"大小500 B / 1000 B",
"⬆️ 2/3 上传中",
"速度250 B/s",
"大小250 B / 1000 B",
)
bold, _, blockquote, _ := batchEntityCounts(message.Entities)
if bold != 3 || blockquote != 2 {
t.Fatalf("entity counts = bold:%d blockquote:%d, want bold:3 blockquote:2", bold, blockquote)
}
}
func TestBatchProgressLimitsRowsWithoutHidingActiveUpload(t *testing.T) {
useProgressRegressionLocale(t)
task := newProgressRegressionTask(nil,
progressRegressionFile{"confirm-01", 100},
progressRegressionFile{"confirm-02", 100},
progressRegressionFile{"uploading", 100},
progressRegressionFile{"downloading", 100},
)
started := time.Unix(100, 0)
task.recordItemUpload("confirm-01", 100, 100, started)
task.recordItemUpload("confirm-02", 100, 100, started)
task.recordItemUpload("uploading", 40, 100, started.Add(time.Second))
task.markItemActive("downloading", false, started)
message := buildBatchProgressText(task, nil, 2)
assertProgressRegressionContains(t, message,
"uploading.bin",
"downloading.bin",
"☁️ 已上传等待整组发送2",
)
if strings.Contains(message, "confirm-01.bin") || strings.Contains(message, "confirm-02.bin") {
t.Fatalf("confirmation rows displaced active transfers:\n%s", message)
}
}
func TestBatchProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
useProgressRegressionLocale(t)
fileID := `<b>A&B</b>`
task := newProgressRegressionTask(nil, progressRegressionFile{fileID, 100})
task.markItemRetry(fileID, FailureStageUpload, 1, 3, errors.New(`<i>remote & failed</i>`))
message := buildBatchProgressMessage(task, nil, 1)
if message.Err != nil {
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
}
assertProgressRegressionContains(t, message.Text,
`<b>A&B</b>.bin`,
`<i>remote & failed</i>`,
)
bold, _, blockquote, italic := batchEntityCounts(message.Entities)
if bold != 2 || blockquote != 1 || italic != 0 {
t.Fatalf("entity counts = bold:%d blockquote:%d italic:%d", bold, blockquote, italic)
}
i18n.Init("en")
english := buildBatchProgressMessage(task, nil, 1)
if english.Err != nil {
t.Fatalf("English batch template failed: %v", english.Err)
}
assertProgressRegressionContains(t, english.Text, "📦 Processing", "Retrying upload", `<b>A&B</b>.bin`)
}
func TestDownloadProgressContinuesAfterUploadStarts(t *testing.T) {
useProgressRegressionLocale(t)
progress := new(Progress)
task := newProgressRegressionTask(progress,
progressRegressionFile{"uploading", 100},
progressRegressionFile{"downloading", 100},
)
progress.OnStart(t.Context(), task)
task.recordDownloadComplete("uploading", 100)
task.uploadCallback(t.Context(), "uploading")(50, 100)
started := time.Unix(100, 0)
task.markItemActive("downloading", false, started)
task.recordItemDownload("downloading", 50, started.Add(time.Second))
progress.updateMu.Lock()
progress.lastUpdateAt = time.Now().Add(-progressRenderInterval)
progress.updateMu.Unlock()
progress.OnProgress(t.Context(), task)
progress.updateMu.Lock()
text := progress.lastText
progress.updateMu.Unlock()
assertProgressRegressionContains(t, text,
"uploading.bin",
"🟩🟩🟩🟩🟩⬜️⬜️⬜️⬜️⬜️ 50%",
"总速度:⬇️ 50 B/s ⬆️ 0 B/s",
"🔄 另有 1 个文件正在处理",
)
}
func TestBatchUploadIgnoresOutOfOrderBytesAndAllowsRetryReset(t *testing.T) {
recorder := new(progressRegressionRecorder)
task := newProgressRegressionTask(recorder, progressRegressionFile{"file", 100})
task.recordDownloadComplete("file", 100)
callback := task.uploadCallback(t.Context(), "file")
callback(80, 100)
callback(10, 100)
if got := task.Items()[0].Uploaded; got != 80 {
t.Fatalf("out-of-order callback regressed item to %d, want 80", got)
}
task.markItemRetry("file", FailureStageUpload, 1, 3, context.DeadlineExceeded)
callback(0, 100)
callback(10, 100)
if got := task.Items()[0].Uploaded; got != 10 {
t.Fatalf("retry did not reset item progress: got %d, want 10", got)
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for index := 1; index < len(recorder.notifications); index++ {
if recorder.notifications[index] < recorder.notifications[index-1] {
t.Fatalf("aggregate progress regressed: %v", recorder.notifications)
}
}
}
func TestUploadProgressNotificationsRemainOrdered(t *testing.T) {
recorder := &orderedProgressRegressionRecorder{
firstEntered: make(chan struct{}),
releaseFirst: make(chan struct{}),
secondEntered: make(chan struct{}),
}
task := newProgressRegressionTask(recorder,
progressRegressionFile{"first", 100},
progressRegressionFile{"second", 100},
)
task.recordDownloadComplete("first", 100)
task.recordDownloadComplete("second", 100)
first := task.uploadCallback(t.Context(), "first")
second := task.uploadCallback(t.Context(), "second")
var wait sync.WaitGroup
wait.Go(func() {
first(100, 100)
})
<-recorder.firstEntered
wait.Go(func() {
second(100, 100)
})
overtook := false
select {
case <-recorder.secondEntered:
overtook = true
case <-time.After(100 * time.Millisecond):
}
close(recorder.releaseFirst)
wait.Wait()
if overtook {
t.Fatal("later aggregate notification overtook the first callback")
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
if got := recorder.notifications; len(got) != 2 || got[0] != 100 || got[1] != 200 {
t.Fatalf("upload notifications = %v, want [100 200]", got)
}
}
func TestBatchUploadUsesActualSizeWhenMetadataIsUnknown(t *testing.T) {
recorder := new(progressRegressionRecorder)
task := newProgressRegressionTask(recorder, progressRegressionFile{"photo", 0})
task.recordDownloadComplete("photo", 25)
task.uploadCallback(t.Context(), "photo")(25, 25)
recorder.mu.Lock()
defer recorder.mu.Unlock()
if recorder.startTotal != 25 {
t.Fatalf("upload start total = %d, want actual size 25", recorder.startTotal)
}
if got := task.ActualTotalSize(); got != 25 {
t.Fatalf("actual total size = %d, want 25", got)
}
}
type progressRegressionFile struct {
id string
size int64
}
func newProgressRegressionTask(progress ProgressTracker, files ...progressRegressionFile) *Task {
elems := make([]TaskElement, 0, len(files))
for _, file := range files {
elems = append(elems, TaskElement{
ID: file.id,
File: tfile.NewTGFile(nil, nil, file.size, file.id+".bin"),
})
}
return NewBatchTGFileTask("progress-regression", context.Background(), elems, progress, true)
}
func useProgressRegressionLocale(t *testing.T) {
t.Helper()
i18n.Init("zh-Hans")
t.Cleanup(func() { i18n.Init("zh-Hans") })
}
func assertProgressRegressionContains(t *testing.T, value string, wants ...string) {
t.Helper()
for _, want := range wants {
if !strings.Contains(value, want) {
t.Fatalf("text does not contain %q:\n%s", want, value)
}
}
}
func batchEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
case *tg.MessageEntityItalic:
italic++
}
}
return
}

View File

@@ -7,6 +7,7 @@ import (
"sync"
"sync/atomic"
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/core"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
@@ -18,25 +19,35 @@ import (
var _ core.Executable = (*Task)(nil)
type TaskElement struct {
ID string
Storage storage.Storage
Path string
File tfile.TGFile
localPath string
stream bool
ID string
Storage storage.Storage
Path string
File tfile.TGFile
localPath string
stream bool
sourceGroupKey string
sourceCaption string
preserveCaption bool
}
type Task struct {
ID string
ctx context.Context
elems []TaskElement
Progress ProgressTracker
IgnoreErrors bool // if true, errors during processing will be ignored
downloaded atomic.Int64
totalSize int64
processing map[string]TaskElementInfo
processingMu sync.RWMutex
failed map[string]error // [TODO] errors for each element
ID string
ctx context.Context
elems []TaskElement
Progress ProgressTracker
IgnoreErrors bool // if true, errors during processing will be ignored
downloaded atomic.Int64
totalSize int64
uploadTotalSize atomic.Int64
processing map[string]TaskElementInfo
processingMu sync.RWMutex
itemStates []itemProgressState
itemIndex map[string]int
itemMu sync.RWMutex
uploadOnce sync.Once
uploadMu sync.Mutex
uploaded map[string]int64
failed map[string]error // [TODO] errors for each element
}
// Title implements core.Exectable.
@@ -54,6 +65,7 @@ func NewTaskElement(
file tfile.TGFile,
) (*TaskElement, error) {
id := xid.New().String()
groupKey, caption, preserveCaption := sourceMetadata(file)
_, ok := stor.(storage.StorageCannotStream)
if !config.C().Stream || ok {
cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name())))
@@ -61,22 +73,42 @@ func NewTaskElement(
return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
}
return &TaskElement{
ID: id,
Storage: stor,
Path: path,
File: file,
localPath: cachePath,
ID: id,
Storage: stor,
Path: path,
File: file,
localPath: cachePath,
sourceGroupKey: groupKey,
sourceCaption: caption,
preserveCaption: preserveCaption,
}, nil
}
return &TaskElement{
ID: id,
Storage: stor,
Path: path,
File: file,
stream: true,
ID: id,
Storage: stor,
Path: path,
File: file,
stream: true,
sourceGroupKey: groupKey,
sourceCaption: caption,
preserveCaption: preserveCaption,
}, nil
}
func sourceMetadata(file tfile.TGFile) (groupKey, caption string, preserveCaption bool) {
messageFile, ok := file.(tfile.TGFileMessage)
if !ok || messageFile.Message() == nil {
return "", "", false
}
msg := messageFile.Message()
groupID, grouped := msg.GetGroupedID()
if !grouped || groupID == 0 {
return "", "", false
}
chatID := tgutil.ChatIdFromPeer(msg.GetPeerID())
return fmt.Sprintf("%T:%d:%d", msg.GetPeerID(), chatID, groupID), msg.GetMessage(), true
}
func NewBatchTGFileTask(
id string,
ctx context.Context,
@@ -84,6 +116,7 @@ func NewBatchTGFileTask(
progress ProgressTracker,
ignoreErrors bool,
) *Task {
itemStates, itemIndex := newItemProgressStates(files)
task := &Task{
ID: id,
ctx: ctx,
@@ -98,6 +131,9 @@ func NewBatchTGFileTask(
return total
}(),
processing: make(map[string]TaskElementInfo),
itemStates: itemStates,
itemIndex: itemIndex,
uploaded: make(map[string]int64),
IgnoreErrors: ignoreErrors,
processingMu: sync.RWMutex{},
failed: make(map[string]error),

View File

@@ -27,8 +27,10 @@ type TaskInfo interface {
TaskID() string
TotalSize() int64
Downloaded() int64
ActualTotalSize() int64
Count() int
Processing() []TaskElementInfo
Items() []TaskItemProgress
}
func (t *Task) TaskID() string {

View File

@@ -0,0 +1,70 @@
package batchtfile
import (
"context"
"time"
)
// UploadProgressTracker optionally extends a batch progress tracker with a
// distinct aggregate upload phase.
type UploadProgressTracker interface {
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
}
func (t *Task) startUpload(ctx context.Context) {
tracker, ok := t.Progress.(UploadProgressTracker)
if !ok {
return
}
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
t.uploadOnce.Do(func() {
tracker.OnUploadStart(ctx, t, t.uploadTotalSize.Load())
})
}
func (t *Task) uploadCallback(ctx context.Context, id string) func(uploaded, total int64) {
return func(uploaded, total int64) {
tracker, ok := t.Progress.(UploadProgressTracker)
if !ok || uploaded < 0 {
return
}
t.startUpload(ctx)
if total > 0 && uploaded > total {
uploaded = total
}
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
becameConfirming := t.recordItemUpload(id, uploaded, total, time.Now())
if t.uploaded == nil {
t.uploaded = make(map[string]int64)
}
previous, tracked := t.uploaded[id]
if !tracked || uploaded > previous {
t.uploaded[id] = uploaded
}
var aggregate int64
for _, current := range t.uploaded {
aggregate += current
}
uploadTotal := t.uploadTotalSize.Load()
if aggregate > uploadTotal {
aggregate = uploadTotal
}
tracker.OnUploadProgress(ctx, t, aggregate, uploadTotal)
if becameConfirming {
t.notifyStateChange(ctx)
}
}
}
func (t *Task) recordDownloadComplete(id string, uploadSize int64) {
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
if uploadSize > 0 {
t.uploadTotalSize.Add(uploadSize)
}
t.recordItemDownloaded(id, uploadSize)
}

View File

@@ -0,0 +1,42 @@
package tfile
import (
"testing"
"github.com/gotd/td/tg"
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
)
func TestSourceCaption(t *testing.T) {
tests := []struct {
name string
file tfilepkg.TGFile
want string
ok bool
}{
{
name: "original caption",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{Message: "original caption"})),
want: "original caption",
ok: true,
},
{
name: "empty caption suppresses storage fallback",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{})),
ok: true,
},
{
name: "file without source message",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := sourceCaption(tt.file)
if ok != tt.ok || got != tt.want {
t.Fatalf("sourceCaption() = (%q, %v), want (%q, %v)", got, ok, tt.want, tt.ok)
}
})
}
}

View File

@@ -3,6 +3,7 @@ package tfile
import (
"context"
"fmt"
"io"
"os"
"path"
@@ -10,8 +11,12 @@ import (
"github.com/duke-git/lancet/v2/retry"
"github.com/krau/SaveAny-Bot/common/tdler"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
"github.com/krau/SaveAny-Bot/storage"
)
func (t *Task) Execute(ctx context.Context) error {
@@ -57,13 +62,34 @@ func (t *Task) Execute(ctx context.Context) error {
return fmt.Errorf("failed to get file stat: %w", err)
}
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
if caption, ok := sourceCaption(t.File); ok {
vctx = storagetypes.WithSourceCaption(vctx, caption)
}
err = retry.Retry(func() error {
file, err := os.Open(t.localPath)
if err != nil {
return fmt.Errorf("failed to open cache file: %w", err)
}
defer file.Close()
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
uploadProgress, tracksUpload := t.Progress.(UploadProgressTracker)
if !tracksUpload {
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
return fmt.Errorf("failed to save file: %w", err)
}
return nil
}
uploadProgress.OnUploadStart(vctx, t, fileStat.Size())
onProgress := func(uploaded, total int64) {
uploadProgress.OnUploadProgress(vctx, t, uploaded, total)
}
if progressSaver, ok := t.Storage.(storage.StorageProgressSaver); ok {
err = progressSaver.SaveWithProgress(vctx, file, t.Path, onProgress)
} else {
var reader io.Reader = ioutil.NewProgressReader(file, fileStat.Size(), onProgress)
err = t.Storage.Save(vctx, reader, t.Path)
}
if err != nil {
return fmt.Errorf("failed to save file: %w", err)
}
return nil
@@ -73,3 +99,11 @@ func (t *Task) Execute(ctx context.Context) error {
}
return nil
}
func sourceCaption(file tfilepkg.TGFile) (string, bool) {
messageFile, ok := file.(tfilepkg.TGFileMessage)
if !ok || messageFile.Message() == nil {
return "", false
}
return messageFile.Message().GetMessage(), true
}

View File

@@ -4,12 +4,12 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/charmbracelet/log"
"github.com/gotd/td/telegram/message/entity"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
@@ -23,153 +23,319 @@ type ProgressTracker interface {
OnDone(ctx context.Context, info TaskInfo, err error)
}
// UploadProgressTracker optionally extends a task progress tracker with a
// distinct upload phase. Keeping it separate preserves compatibility with
// custom download-only trackers.
type UploadProgressTracker interface {
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
}
type Progress struct {
MessageID int
ChatID int64
start time.Time
lastUpdatePercent atomic.Int32
lastUpdateAt atomic.Int64
updateMu sync.Mutex
uploadAttempt int
uploadedBytes int64
actualSize int64
hasActualSize bool
}
const (
uploadProgressMinInterval = time.Second
uploadProgressMaxInterval = 3 * time.Second
singleProgressBarWidth = 10
maxSingleErrorRunes = 240
)
type singleProgressPhase int
const (
singlePhaseDownloading singleProgressPhase = iota
singlePhaseUploading
singlePhaseRetrying
)
type renderedSingleMessage struct {
Text string
Entities []tg.MessageEntityClass
Err error
}
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
p.start = time.Now()
p.lastUpdatePercent.Store(0)
p.lastUpdateAt.Store(0)
p.uploadAttempt = 0
p.uploadedBytes = 0
p.actualSize = 0
p.hasActualSize = false
log.FromContext(ctx).Debugf("Progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressFileStartPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(info.FileSize())/(1024*1024))),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
return
}
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, singlePhaseDownloading, 0, info.FileSize(), 0, 0), true)
}
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo, downloaded, total int64) {
if !shouldUpdateProgress(total, downloaded, int(p.lastUpdatePercent.Load())) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
now := time.Now()
elapsed := uploadProgressMaxInterval
if lastUpdateAt := p.lastUpdateAt.Load(); lastUpdateAt > 0 {
elapsed = now.Sub(time.Unix(0, lastUpdateAt))
}
if !shouldUpdateSingleDownloadProgress(total, downloaded, int(p.lastUpdatePercent.Load()), elapsed) {
return
}
percent := int32((downloaded * 100) / total)
if p.lastUpdatePercent.Load() == percent {
return
if total > 0 {
percent := int32((downloaded * 100) / total)
if p.lastUpdatePercent.Load() == percent {
return
}
p.lastUpdatePercent.Store(percent)
}
p.lastUpdatePercent.Store(percent)
p.lastUpdateAt.Store(now.UnixNano())
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.FileName(), downloaded, total)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressFileProcessingPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(total)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(downloaded, p.start)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f%%", float64(downloaded)/float64(total)*100)),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
info,
singlePhaseDownloading,
downloaded,
total,
dlutil.GetSpeed(downloaded, p.start),
0,
), true)
}
func shouldUpdateSingleDownloadProgress(total, downloaded int64, lastPercent int, elapsed time.Duration) bool {
if total > 0 {
return shouldUpdateProgress(total, downloaded, lastPercent)
}
return downloaded > 0 && elapsed >= uploadProgressMaxInterval
}
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, total int64) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
p.start = time.Now()
p.lastUpdatePercent.Store(0)
p.lastUpdateAt.Store(p.start.UnixNano())
p.uploadAttempt++
p.uploadedBytes = 0
p.actualSize = max(total, 0)
p.hasActualSize = true
log.FromContext(ctx).Debugf("Upload progress tracking started: %s", info.FileName())
phase := singleUploadPhase(p.uploadAttempt)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, phase, 0, total, 0, p.uploadAttempt), true)
}
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64) {
if total <= 0 || uploaded <= 0 {
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
p.updateMu.Lock()
defer p.updateMu.Unlock()
if uploaded > total {
uploaded = total
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
if uploaded < p.uploadedBytes {
return
}
p.uploadedBytes = uploaded
now := time.Now()
lastUpdateAt := time.Unix(0, p.lastUpdateAt.Load())
lastPercent := int(p.lastUpdatePercent.Load())
if !shouldUpdateUploadProgress(total, uploaded, lastPercent, now.Sub(lastUpdateAt)) {
return
}
percent := int32((uploaded * 100) / total)
p.lastUpdatePercent.Store(percent)
p.lastUpdateAt.Store(now.UnixNano())
log.FromContext(ctx).Debugf("Upload progress update: %s, %d/%d", info.FileName(), uploaded, total)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
info,
singleUploadPhase(p.uploadAttempt),
uploaded,
total,
dlutil.GetSpeed(uploaded, p.start),
p.uploadAttempt,
), true)
}
func shouldUpdateUploadProgress(total, uploaded int64, lastPercent int, elapsed time.Duration) bool {
if total <= 0 || uploaded <= 0 {
return false
}
if uploaded >= total {
return lastPercent < 100 && elapsed >= uploadProgressMinInterval
}
percent := int((uploaded * 100) / total)
if percent < lastPercent {
return false
}
if elapsed < uploadProgressMinInterval {
return false
}
if percent == lastPercent {
return elapsed >= uploadProgressMaxInterval
}
return shouldUpdateProgress(total, uploaded, lastPercent) || elapsed >= uploadProgressMaxInterval
}
func singleUploadPhase(attempt int) singleProgressPhase {
if attempt > 1 {
return singlePhaseRetrying
}
return singlePhaseUploading
}
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
if err != nil {
log.FromContext(ctx).Errorf("Progress error for file [%s]: %v", info.FileName(), err)
} else {
log.FromContext(ctx).Debugf("Progress done for file [%s]", info.FileName())
}
entityBuilder := entity.Builder{}
var stylingErr error
p.editMessage(ctx, info.TaskID(), buildSingleDoneMessage(info, p.doneSize(info), err), false)
}
if err != nil {
if errors.Is(err, context.Canceled) {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
styling.Plain("\n"),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileNamePrefix, nil)),
styling.Code(info.FileName()),
)
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadFailedPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressErrorPrefix, nil)),
styling.Bold(err.Error()),
)
}
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadDonePrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
)
func (p *Progress) doneSize(info TaskInfo) int64 {
if p.hasActualSize {
return p.actualSize
}
return max(info.FileSize(), 0)
}
if stylingErr != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedSingleMessage, cancellable bool) {
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render file progress message: %v", message.Err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
req := buildSingleEditMessageRequest(p.MessageID, taskID, message, cancellable)
if ext := tgutil.ExtFromContext(ctx); ext != nil {
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
log.FromContext(ctx).Errorf("Failed to edit file progress message: %v", err)
}
}
req.SetMessage(text)
req.SetEntities(entities)
}
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
func buildSingleEditMessageRequest(messageID int, taskID string, message renderedSingleMessage, cancellable bool) *tg.MessagesEditMessageRequest {
req := &tg.MessagesEditMessageRequest{ID: messageID}
req.SetMessage(message.Text)
if len(message.Entities) > 0 {
req.SetEntities(message.Entities)
}
if cancellable {
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
}}})
}
return req
}
func buildSingleProgressMessage(
info TaskInfo,
phase singleProgressPhase,
current int64,
total int64,
speed float64,
attempt int,
) renderedSingleMessage {
if current < 0 {
current = 0
}
if total > 0 && current > total {
current = total
}
percent := singleProgressPercent(current, total)
destination := fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())
data := map[string]any{
"Name": info.FileName(),
"Bar": singleProgressBar(percent),
"Progress": percent,
"Speed": singleProgressSpeed(speed),
"Current": dlutil.FormatSize(current),
"Size": dlutil.FormatSize(total),
"Destination": destination,
"Attempt": max(attempt, 1),
}
var key i18nk.Key
switch phase {
case singlePhaseUploading:
key = i18nk.BotMsgProgressSingleUploading
case singlePhaseRetrying:
key = i18nk.BotMsgProgressSingleUploadRetrying
default:
key = i18nk.BotMsgProgressSingleDownloading
if total <= 0 {
key = i18nk.BotMsgProgressSingleDownloadingUnknown
}
}
markup := i18n.T(i18nk.BotMsgProgressSingleStatusHeader, nil) + "\n\n" + localizedProgressMarkup(key, data)
return completeSingleMessage(markup)
}
func buildSingleDoneMessage(info TaskInfo, size int64, err error) renderedSingleMessage {
data := map[string]any{
"Name": info.FileName(),
"Size": dlutil.FormatSize(max(size, 0)),
"Destination": fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath()),
}
var key i18nk.Key
switch {
case err == nil:
key = i18nk.BotMsgProgressSingleDone
case errors.Is(err, context.Canceled):
key = i18nk.BotMsgProgressSingleCanceled
default:
data["Reason"] = truncateSingleError(err.Error())
key = i18nk.BotMsgProgressSingleFailed
}
return completeSingleMessage(localizedProgressMarkup(key, data))
}
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
}
func completeSingleMessage(markup string) renderedSingleMessage {
text, entities, err := tgutil.RenderHTML(markup)
return renderedSingleMessage{Text: text, Entities: entities, Err: err}
}
func singleProgressPercent(current, total int64) int {
if total <= 0 {
return 0
}
return int(min(max(current, 0), total) * 100 / total)
}
func singleProgressBar(percent int) string {
percent = min(max(percent, 0), 100)
filled := percent * singleProgressBarWidth / 100
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", singleProgressBarWidth-filled)
}
func singleProgressSpeed(speed float64) string {
if speed <= 0 {
return "0 B/s"
}
return dlutil.FormatSize(int64(speed)) + "/s"
}
func truncateSingleError(value string) string {
runes := []rune(value)
if len(runes) <= maxSingleErrorRunes {
return value
}
return string(runes[:maxSingleErrorRunes])
}
type ProgressOption func(*Progress)

View File

@@ -0,0 +1,178 @@
package tfile
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
)
type progressTestTaskInfo struct{}
func (progressTestTaskInfo) TaskID() string { return "task" }
func (progressTestTaskInfo) FileName() string { return "file.bin" }
func (progressTestTaskInfo) FileSize() int64 { return 100 << 20 }
func (progressTestTaskInfo) StoragePath() string { return "file.bin" }
func (progressTestTaskInfo) StorageName() string { return "test" }
func TestShouldUpdateUploadProgress(t *testing.T) {
tests := []struct {
name string
total int64
uploaded int64
lastPercent int
elapsed time.Duration
want bool
}{
{name: "invalid total", total: 0, uploaded: 1, want: false},
{name: "no uploaded bytes", total: 100, uploaded: 0, want: false},
{name: "percentage threshold", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval, want: true},
{name: "percentage threshold rate limited", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
{name: "maximum time threshold", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval, want: true},
{name: "below thresholds", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval - time.Millisecond, want: false},
{name: "completion", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval, want: true},
{name: "completion rate limited", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
{name: "completion already reported", total: 100, uploaded: 100, lastPercent: 100, elapsed: uploadProgressMinInterval, want: false},
{name: "out of order callback", total: 100, uploaded: 40, lastPercent: 60, elapsed: uploadProgressMaxInterval, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := shouldUpdateUploadProgress(tt.total, tt.uploaded, tt.lastPercent, tt.elapsed)
if got != tt.want {
t.Fatalf("shouldUpdateUploadProgress() = %v, want %v", got, tt.want)
}
})
}
}
func TestUploadProgressConcurrentCallbacks(t *testing.T) {
progress := new(Progress)
ctx := context.Background()
info := progressTestTaskInfo{}
const total = int64(100 << 20)
progress.OnUploadStart(ctx, info, total)
progress.lastUpdateAt.Store(time.Now().Add(-uploadProgressMaxInterval).UnixNano())
var wg sync.WaitGroup
for uploaded := int64(1 << 20); uploaded <= total; uploaded += 1 << 20 {
uploaded := uploaded
wg.Go(func() {
progress.OnUploadProgress(ctx, info, uploaded, total)
})
}
wg.Wait()
percent := progress.lastUpdatePercent.Load()
if percent <= 0 || percent > 100 {
t.Fatalf("last upload percentage = %d, want a value in (0, 100]", percent)
}
if progress.uploadedBytes != total {
t.Fatalf("maximum uploaded bytes = %d, want %d", progress.uploadedBytes, total)
}
}
func TestSingleUploadRetryKeepsRichProgressLayout(t *testing.T) {
i18n.Init("zh-Hans")
t.Cleanup(func() { i18n.Init("zh-Hans") })
message := buildSingleProgressMessage(
progressTestTaskInfo{},
singleUploadPhase(2),
25<<20,
100<<20,
5<<20,
2,
)
if message.Err != nil {
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
}
for _, want := range []string{
"🔁 上传重试",
"🟩🟩⬜️⬜️⬜️⬜️⬜️⬜️⬜️⬜️ 25%",
"尝试次数2",
"速度5.00 MB/s",
"大小25.00 MB / 100.00 MB",
} {
if !strings.Contains(message.Text, want) {
t.Fatalf("retry progress does not contain %q:\n%s", want, message.Text)
}
}
}
func TestSingleProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
i18n.Init("en")
t.Cleanup(func() { i18n.Init("zh-Hans") })
info := htmlProgressTestTaskInfo{}
message := buildSingleProgressMessage(info, singlePhaseDownloading, 50, 100, 25, 0)
if message.Err != nil {
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
}
for _, want := range []string{
`<b>A&B</b>.bin`,
`[store<&>]:dir/<i>x</i>&`,
"Speed: 25 B/s",
} {
if !strings.Contains(message.Text, want) {
t.Fatalf("progress text does not contain %q:\n%s", want, message.Text)
}
}
bold, code, blockquote, italic := singleEntityCounts(message.Entities)
if bold != 2 || code != 6 || blockquote != 1 || italic != 0 {
t.Fatalf("progress entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
}
failure := buildSingleDoneMessage(info, 100, errors.New(`<i>remote & failed</i>`))
if failure.Err != nil {
t.Fatalf("buildSingleDoneMessage() failed: %v", failure.Err)
}
if !strings.Contains(failure.Text, `<i>remote & failed</i>`) {
t.Fatalf("failure reason was not preserved literally:\n%s", failure.Text)
}
bold, code, blockquote, italic = singleEntityCounts(failure.Entities)
if bold != 1 || code != 2 || blockquote != 0 || italic != 0 {
t.Fatalf("failure entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
}
}
func TestSingleDoneSizeUsesActualUploadSize(t *testing.T) {
progress := new(Progress)
info := progressTestTaskInfo{}
progress.OnStart(context.Background(), info)
progress.OnUploadStart(context.Background(), info, 2048)
if got := progress.doneSize(info); got != 2048 {
t.Fatalf("done size = %d, want actual upload size 2048", got)
}
}
type htmlProgressTestTaskInfo struct{}
func (htmlProgressTestTaskInfo) TaskID() string { return "html-task" }
func (htmlProgressTestTaskInfo) FileName() string { return `<b>A&B</b>.bin` }
func (htmlProgressTestTaskInfo) FileSize() int64 { return 100 }
func (htmlProgressTestTaskInfo) StoragePath() string { return `dir/<i>x</i>&` }
func (htmlProgressTestTaskInfo) StorageName() string { return `store<&>` }
func singleEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
case *tg.MessageEntityItalic:
italic++
}
}
return
}

View File

@@ -113,6 +113,51 @@ secret = "your-rpc-secret"
remove_after_transfer = true
```
### yt-dlp Configuration
Configures the behavior of the `/ytdlp` command and the `ytdlp` HTTP-API task type when no custom flags are passed.
- `max_height`: Default maximum video resolution by height in pixels (e.g. `1080`, `720`). `0` means no limit (best available). Ignored when `format` is set.
- `format`: A raw yt-dlp format selector (`-f`). When set, it takes precedence over `max_height` and gives you full control, e.g. `bv*[height<=720]+ba/b`.
- `recode`: The target video container yt-dlp recodes into after download (e.g. `mp4`). Leave empty to disable recoding.
{{< hint info >}}
These defaults only apply when using the `/ytdlp` command without passing any custom flags. Passing custom flags on the command (or `flags` in the API) overrides them.
{{< /hint >}}
```toml
[ytdlp]
max_height = 1080
format = "" # e.g. "bv*[height<=720]+ba/b"
recode = "mp4" # empty disables recoding
```
### HTTP API Configuration
When enabled, SaveAny-Bot exposes an HTTP API for creating/querying/canceling tasks programmatically. See [HTTP API](../../usage/api) for the full endpoint reference.
- `enable`: Whether to enable the HTTP API server, default is `false`.
- `host`: Bind address, default `0.0.0.0`.
- `port`: Listen port, default `8080`.
- `token`: Authentication token. **Strongly recommended** — if empty, the API is exposed without any authentication.
```toml
[api]
enable = false
host = "0.0.0.0"
port = 8080
token = "your-token"
```
### Log Configuration
- `level`: Log level. One of `debug`, `info`, `warn`, `error`, `fatal`. Default is `info`.
```toml
[log]
level = "info"
```
### Storage Endpoints List
The storage endpoints list is used to define the storage locations supported by the Bot. Each storage endpoint needs to specify a name, type, and related configuration, using the double bracket syntax `[[storages]]`.

View File

@@ -79,7 +79,8 @@ Stream mode is not supported.
chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat
force_file = false # Force sending as file, default is false
skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded.
spilt_size_mb = 2000 # Split size in MB, default is 2000 MB (2 GB). Files larger than this will be split into multiple parts (zip format). Ignored when skip_large is true.
split_large_video = false # Losslessly split oversized videos into one album of up to 10 independently playable parts. Falls back to ZIP parts on failure or when more than 10 parts are required.
split_size_mb = 0 # Split size in MB. 0 uses the uploader account limit: 2000 MB for bots/regular users and 4000 MB for Premium users. Oversized non-video files use ZIP parts. Ignored when skip_large is true.
```
## Rclone
@@ -136,4 +137,4 @@ remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```
```

View File

@@ -0,0 +1,90 @@
---
title: "CLI Subcommands"
weight: 21
---
# CLI Subcommands
Besides running the Telegram bot with `./saveany-bot` (no subcommand), the binary exposes two helper subcommands for moving local files into a storage backend: `upload` (one-shot) and `watch` (continuous).
These subcommands load the same `config.toml` as the bot, initialize the database and caches, then perform their task. They do **not** start the Telegram bot itself, although storages of type `telegram` will spin up the bot client just for the upload.
## `upload` — Upload a Single File
```
saveany-bot upload -f <file> -s <storage> [-d <dir>] [--no-progress]
```
Flags:
| Flag | Required | Description |
|---|---|---|
| `-f, --file` | Yes | Path to the local file to upload |
| `-s, --storage` | Yes | Target storage name (must exist in `config.toml`) |
| `-d, --dir` | No | Destination directory within the storage. Defaults to the storage's `base_path` |
| `--no-progress` | No | Disable the terminal progress bar |
Examples:
```bash
# Upload a file to the default dir of storage "MyAlist"
./saveany-bot upload -f ./movie.mp4 -s MyAlist
# Upload into a specific subdirectory
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
# Upload via Telegram storage without a progress bar
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
```
## `watch` — Watch a Directory and Auto-Upload
The `watch` subcommand continuously monitors a local directory and uploads created or modified files to a storage backend, preserving the relative directory structure from the watch root.
```
saveany-bot watch -p <path> -s <storage> [-d <dir>] [options]
```
Flags:
| Flag | Default | Description |
|---|---|---|
| `-p, --path` | *(required)* | Local directory to watch |
| `-s, --storage` | *(required)* | Target storage name |
| `-d, --dir` | storage's `base_path` | Destination directory within the storage |
| `-r, --recursive` | `false` | Watch subdirectories recursively |
| `--overwrite` | `false` | Overwrite existing files on the storage instead of skipping them |
| `--initial-scan` | `false` | Upload files already present in the directory on startup |
| `--debounce` | `2s` | How long to wait after the last write before uploading a file |
| `--upload-workers` | `config.workers` | Number of concurrent uploads |
| `--retry-delay` | `3s` | Delay between upload retries |
{{< hint info >}}
Write-completion detection: the watcher debounces per file and only uploads once the file size stays unchanged across the debounce window, so partial/write-in-progress files are not uploaded.
<br />
If a file changes while being uploaded, it is re-uploaded once after the current upload finishes (instead of being queued multiple times).
{{< /hint >}}
Examples:
```bash
# Watch ./inbox and upload new files to "MyAlist" recursively
./saveany-bot watch -p ./inbox -s MyAlist -r
# Watch with a custom destination dir and overwrite
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
# On startup, also upload everything already in ./inbox
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
```
### Behavior notes
- Relative directory structure is preserved under the destination directory. A file written to `./inbox/sub/file.txt` with `--path ./inbox` is uploaded to `<dest_dir>/sub/file.txt`.
- `watch` runs until interrupted (e.g. `Ctrl-C` / `SIGINT`); in-flight uploads are drained before exit.
- Retries follow the global `retry` value from `config.toml`, with `--retry-delay` between attempts.
- Telegram-type storages will start the bot client automatically to perform uploads.
{{< hint warning >}}
`watch` is unrelated to the in-bot `/watch` command (which watches Telegram chats). This subcommand watches a **local filesystem directory** and uploads to a storage backend, independent of Telegram.
{{< /hint >}}

View File

@@ -0,0 +1,75 @@
---
title: "File Naming & Conflict Strategies"
weight: 11
---
# File Naming & Conflict Strategies
SaveAny-Bot lets you customize how saved files are named and how collisions with existing files are resolved, directly in Telegram via the `/config` and `/fnametmpl` commands.
## `/config` — User Configuration
The `/config` command opens an inline menu where you can change two per-user settings:
- **Filename strategy** — how the saved file is named
- **Duplicate file strategy** — what happens when a file with the same name already exists in the target storage
Settings are stored per user and apply to all of that user's subsequent save/transfer tasks.
### Filename strategy
| Option | Behavior |
|---|---|
| `Default` | Use the original media filename, or a generated name when no original filename is available |
| `Gen From Msg First` | Generate the filename from the message content (e.g. caption, text) and prefer that over the original filename |
| `Template` | Render the filename from a custom template you define with `/fnametmpl` |
### Duplicate file strategy
| Option | Behavior |
|---|---|
| `Always rename` (default) | Keep the existing file and save the new one with an alternate name |
| `Ask every time` | Prompt you with inline buttons each time a collision occurs |
| `Always overwrite` | Replace the existing file with the new one |
| `Always skip` | Do nothing for conflicting files |
{{< hint info >}}
The conflict strategy only kicks in for storage backends that can detect the existence of a file. Backends that do not support existence checks will fall back to overwriting.
{{< /hint >}}
## `/fnametmpl` — Custom Filename Template
When the filename strategy is set to `Template`, SaveAny-Bot renders each saved file's name using the template configured via `/fnametmpl`.
```
/fnametmpl [template]
```
- Running `/fnametmpl` without arguments shows your current template and the help text.
- Running it with a template string sets that template as your filename template.
The template uses Go [`text/template`](https://pkg.go.dev/text/template) syntax. The available variables are:
| Variable | Description |
|---|---|
| `{{.msgid}}` | Telegram message ID |
| `{{.msgtags}}` | Hashtags found in the message, joined with `_` |
| `{{.msggen}}` | Filename generated from the message |
| `{{.msgdate}}` | Message date, formatted `YYYY-MM-DD_HH-MM-SS` |
| `{{.msgraw}}` | Raw, unprocessed message text |
| `{{.origname}}` | The media's original filename (if any) |
| `{{.chatid}}` | Chat ID of the message |
Examples:
```
# Fixed prefix + message id + date
/fnametmpl Image_{{.msgid}}_{{.msgdate}}.jpg
# Use original name if available, otherwise a generated name
/fnametmpl {{.origname}}
```
{{< hint warning >}}
The template only takes effect when the filename strategy is set to `Template`. If template parsing fails, SaveAny-Bot falls back to the default filename naming logic.
{{< /hint >}}

View File

@@ -27,6 +27,45 @@ Pay attention to spaces; the bot can only parse correctly formatted syntax. Belo
In addition, if `CHOSEN` is used as the storage name in the rule, it means files will be stored under the path of the storage you selected by clicking the inline button.
You can also toggle whether rules are applied with `/rule switch`. When rule mode is off, all files go to the default storage.
## Preset Rules
Manually writing regex rules for common file types is tedious, so the bot ships a built-in set of preset categories (video, image, audio, document, archive) that you can import in one command:
```
/rule preset <storage> [base_path]
```
Parameters:
- `storage`: Target storage name (must exist and be accessible to you)
- `base_path`: Optional. Each preset category's subdirectory is created under this path. If omitted, the default category directory names are used directly.
Examples:
```
# Import preset rules into "MyAlist" with the default directory layout
/rule preset MyAlist
# Import preset rules with a custom base path "downloads/sorted"
/rule preset MyAlist downloads/sorted
```
This will create `FILENAME-REGEX` rules for each category, routing matched files to the corresponding subdirectory under `base_path`:
| Category | Matched extensions | Default directory |
|---|---|---|
| video | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
| image | jpg, jpeg, png, gif, webp, bmp | `图片` |
| audio | mp3, flac, wav, aac, m4a, ogg | `音频` |
| document | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
| archive | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
{{< hint info >}}
Preset rules are regular `FILENAME-REGEX` rules once imported. You can view, edit, or delete them individually with `/rule` and `/rule del <id>` like any other rule.
{{< /hint >}}
Rule types:
## FILENAME-REGEX

View File

@@ -30,6 +30,7 @@ base_path = "./downloads"
### 全局配置
- `lang`: Bot 使用的语言, 默认为 `zh-CN` (简体中文), 设为 `en` 则使用英语.
- `stream`: 是否启用 Stream 模式, 默认为 `false`. 启用后 Bot 将直接将文件流式传输到存储端(若存储端支持), 不需要下载到本地
{{< hint warning >}}
Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一些弊端:
@@ -47,6 +48,7 @@ Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一
- `proxy`: 全局代理配置, 配置后程序内一切网络连接将会尝试使用该代理, 可选.
```toml
lang = "zh-CN"
stream = false
workers = 3
threads = 4
@@ -111,6 +113,51 @@ secret = "your-rpc-secret"
remove_after_transfer = true
```
### yt-dlp 配置
用于配置 `/ytdlp` 命令以及 HTTP API 中 `ytdlp` 任务类型在未传自定义参数时的默认行为.
- `max_height`: 默认下载的最高视频清晰度 (按高度限制), 如 `1080`, `720`, `480`; `0` 表示不限制 (下载最佳画质). 当设置了 `format` 时此项被忽略.
- `format`: 直接指定 yt-dlp format 选择表达式, 设置后优先级高于 `max_height`, 例如 `bv*[height<=720]+ba/b`.
- `recode`: 下载后转封装的视频容器格式 (如 `mp4`), 留空则不转封装.
{{< hint info >}}
这些默认值仅在使用 `/ytdlp` 命令且未传任何自定义参数时生效. 在命令上传递自定义参数 (或在 API 中传 `flags`) 会覆盖这些默认值.
{{< /hint >}}
```toml
[ytdlp]
max_height = 1080
format = "" # 例如 "bv*[height<=720]+ba/b"
recode = "mp4" # 留空则不转封装
```
### HTTP API 配置
启用后, SaveAny-Bot 会暴露一套 HTTP API, 用于以编程方式创建/查询/取消任务. 完整的接口说明见 [HTTP API](../../usage/api).
- `enable`: 是否启用 HTTP API 服务, 默认为 `false`.
- `host`: 监听地址, 默认 `0.0.0.0`.
- `port`: 监听端口, 默认 `8080`.
- `token`: 鉴权 Token, **强烈建议设置** — 若为空, API 将在无任何鉴权的情况下暴露.
```toml
[api]
enable = false
host = "0.0.0.0"
port = 8080
token = "your-token"
```
### 日志配置
- `level`: 日志级别, 可选 `debug`, `info`, `warn`, `error`, `fatal`. 默认为 `info`.
```toml
[log]
level = "info"
```
### 存储端列表
存储端列表用于定义 Bot 支持的存储位置, 每个存储端需要指定名称、类型和相关配置, 使用双中括号语法 `[[storages]]` 定义.

View File

@@ -82,10 +82,12 @@ chat_id = "123456789"
force_file = false
# 是否跳过大文件, 默认为 false. 如果启用, 超过 Telegram 限制的文件将不会上传.
skip_large = false
# 分卷大小, 单位 MB, 默认为 2000 MB (2 GB).
# 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式)
# 超限视频是否使用 FFmpeg 无损分割成一个媒体组(最多 10 个)内可独立播放的小视频;失败或需要超过 10 段时回退到 ZIP 分卷.
split_large_video = false
# 分卷大小, 单位 MB. 设为 0 时使用实际上传账号的限制Bot/普通用户为 2000 MBPremium 用户为 4000 MB.
# 超过该大小的文件将被分割成多个部分上传;非视频使用 ZIP 格式.
# 当 skip_large 启用时, 该选项无效.
spilt_size_mb = 2000
split_size_mb = 0
```
## Rclone
@@ -142,4 +144,4 @@ remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```
```

View File

@@ -0,0 +1,90 @@
---
title: "命令行子命令"
weight: 21
---
# 命令行子命令
除了直接运行 `./saveany-bot` (不带子命令) 启动 Telegram Bot 外, 这个二进制文件还提供两个把本地文件上传到存储后端的辅助子命令: `upload` (一次性) 和 `watch` (持续监听).
这些子命令会读取与 Bot 相同的 `config.toml`, 初始化数据库和缓存, 然后执行任务. 它们**不会**启动 Telegram Bot 本身, 但 `telegram` 类型的存储会在需要上传时临时启动 Bot 客户端来执行上传.
## `upload` — 上传单个文件
```
saveany-bot upload -f <文件> -s <存储名> [-d <目录>] [--no-progress]
```
参数:
| 参数 | 必填 | 说明 |
|---|---|---|
| `-f, --file` | 是 | 待上传的本地文件路径 |
| `-s, --storage` | 是 | 目标存储名 (必须存在于 `config.toml`) |
| `-d, --dir` | 否 | 存储中的目标目录, 默认使用存储的 `base_path` |
| `--no-progress` | 否 | 关闭终端进度条 |
示例:
```bash
# 上传文件到 "MyAlist" 的默认目录
./saveany-bot upload -f ./movie.mp4 -s MyAlist
# 上传到指定子目录
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
# 通过 Telegram 存储上传并关闭进度条
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
```
## `watch` — 监听目录并自动上传
`watch` 子命令持续监听一个本地目录, 将新建或修改的文件上传到存储后端, 并保留相对监听根目录的子目录结构.
```
saveany-bot watch -p <路径> -s <存储名> [-d <目录>] [选项]
```
参数:
| 参数 | 默认值 | 说明 |
|---|---|---|
| `-p, --path` | *(必填)* | 要监听的本地目录 |
| `-s, --storage` | *(必填)* | 目标存储名 |
| `-d, --dir` | 存储的 `base_path` | 存储中的目标目录 |
| `-r, --recursive` | `false` | 是否递归监听子目录 |
| `--overwrite` | `false` | 覆盖存储上已有的文件, 而非跳过 |
| `--initial-scan` | `false` | 启动时将目录中已存在的文件也上传 |
| `--debounce` | `2s` | 文件最后一次写入后, 等待多久再上传 |
| `--upload-workers` | `config.workers` | 并发上传数 |
| `--retry-delay` | `3s` | 上传重试之间的延迟 |
{{< hint info >}}
写入完成检测: 监听器会按文件做防抖处理, 仅当文件大小在一个 debounce 窗口内保持不变时才上传, 因此不会上传未写完的半成品文件.
<br />
若某文件在上传过程中又被修改, 它会在当前上传完成后再上传一次, 而不是被重复排队.
{{< /hint >}}
示例:
```bash
# 递归监听 ./inbox 并且把新文件上传到 "MyAlist"
./saveany-bot watch -p ./inbox -s MyAlist -r
# 自定义目标目录并覆盖已有文件
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
# 启动时把 ./inbox 中已有的内容也一并上传
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
```
### 行为说明
- 相对子目录结构会被保留: 以 `--path ./inbox` 为例, 写入 `./inbox/sub/file.txt` 的文件会被上传到 `<目标目录>/sub/file.txt`.
- `watch` 会一直运行直到被中断 (如 `Ctrl-C` / `SIGINT`), 退出前会等待所有进行中的上传完成.
- 重试次数遵循 `config.toml` 中的全局 `retry` 值, 各次重试之间间隔 `--retry-delay`.
- `telegram` 类型的存储会自动启动 Bot 客户端来执行上传.
{{< hint warning >}}
`watch` 子命令与 Bot 内的 `/watch` 命令 (监听 Telegram 聊天) 无关. 本子命令监听的是**本地文件系统目录**, 不依赖 Telegram.
{{< /hint >}}

View File

@@ -0,0 +1,75 @@
---
title: "文件命名与重名策略"
weight: 11
---
# 文件命名与重名策略
SaveAny-Bot 支持在 Telegram 中通过 `/config``/fnametmpl` 命令自定义保存文件的命名方式, 以及处理与已存在文件重名时的冲突策略.
## `/config` — 用户配置
`/config` 命令会弹出一个内联菜单, 你可以在其中修改以下两项用户级设置:
- **文件名策略** — 保存文件的命名方式
- **重名文件保存策略** — 目标存储中已存在同名文件时的处理方式
设置按用户分别保存, 对该用户后续所有的保存/转存任务生效.
### 文件名策略
| 选项 | 行为 |
|---|---|
| `默认` | 使用媒体原始文件名, 没有原始文件名时使用生成名 |
| `优先从消息生成` | 优先根据消息内容 (如 caption、文本) 生成文件名, 而非原始文件名 |
| `自定义模板` | 使用 `/fnametmpl` 设置的自定义模板渲染文件名 |
### 重名文件保存策略
| 选项 | 行为 |
|---|---|
| `始终重命名` (默认) | 保留已有文件, 将新文件以另一个名字保存 |
| `每次询问` | 每次遇到重名时通过内联按钮提示你选择 |
| `始终覆盖` | 用新文件替换已有文件 |
| `始终跳过` | 对重名文件不做处理 |
{{< hint info >}}
重名策略仅在能够检测文件是否已存在的存储后端生效. 不支持检测文件是否存在的存储后端会退化为覆盖行为.
{{< /hint >}}
## `/fnametmpl` — 自定义文件名模板
当文件名策略设置为 `自定义模板` 时, SaveAny-Bot 会用 `/fnametmpl` 配置的模板来渲染所保存文件的文件名.
```
/fnametmpl [模板]
```
- 不带参数运行 `/fnametmpl` 会显示当前模板以及帮助说明
- 带模板字符串运行则会把它设为你的文件名模板
模板使用 Go [`text/template`](https://pkg.go.dev/text/template) 语法. 可用变量如下:
| 变量 | 说明 |
|---|---|
| `{{.msgid}}` | Telegram 消息 ID |
| `{{.msgtags}}` | 消息中的标签, 以 `_` 连接输出 |
| `{{.msggen}}` | 根据消息生成的文件名 |
| `{{.msgdate}}` | 消息日期, 格式 `YYYY-MM-DD_HH-MM-SS` |
| `{{.msgraw}}` | 消息的原始文本内容 (不做处理) |
| `{{.origname}}` | 媒体的原始文件名 (如有) |
| `{{.chatid}}` | 消息所在聊天的 ID |
示例:
```
# 固定前缀 + 消息 ID + 日期
/fnametmpl 图片_{{.msgid}}_{{.msgdate}}.jpg
# 优先使用原始文件名, 没有则用生成名
/fnametmpl {{.origname}}
```
{{< hint warning >}}
模板仅在文件名策略设置为 `自定义模板` 时生效. 如果模板解析失败, SaveAny-Bot 会回退到默认的文件名生成逻辑.
{{< /hint >}}

View File

@@ -27,6 +27,45 @@ weight: 3
此外, 规则中的存储名若使用 "CHOSEN" , 则表示存储到点击按钮选择的存储端的路径下
你也可以使用 `/rule switch` 来开关规则模式. 关闭规则模式时, 所有文件都将保存到默认存储.
## 预设规则
为常见文件类型手动编写正则规则比较繁琐, 因此 Bot 内置了一组预设分类 (视频、图片、音频、文档、压缩包), 可以通过一条命令批量导入:
```
/rule preset <存储名> [基础路径]
```
参数:
- `存储名`: 目标存储名 (必须存在且你有权访问)
- `基础路径`: 可选. 各预设分类的子目录会创建在此路径下; 若不填则直接使用默认分类目录名
示例:
```
# 导入预设规则到 "MyAlist", 使用默认目录布局
/rule preset MyAlist
# 在自定义基础路径 "downloads/sorted" 下导入预设规则
/rule preset MyAlist downloads/sorted
```
此命令会为每个分类创建 `FILENAME-REGEX` 规则, 将匹配的文件路由到 `基础路径` 下对应的子目录:
| 分类 | 匹配的扩展名 | 默认目录 |
|---|---|---|
| 视频 | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
| 图片 | jpg, jpeg, png, gif, webp, bmp | `图片` |
| 音频 | mp3, flac, wav, aac, m4a, ogg | `音频` |
| 文档 | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
| 压缩包 | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
{{< hint info >}}
导入后的预设规则就是普通的 `FILENAME-REGEX` 规则. 你可以像其他规则一样通过 `/rule` 查看或用 `/rule del <id>` 单独删除/编辑它们.
{{< /hint >}}
规则类型:
## FILENAME-REGEX

File diff suppressed because one or more lines are too long

View File

@@ -1 +1 @@
{"Target":"book.min.a22f4c7d8c2bdc5e3d6e34ba11cb59ab50ea5772594e71305bfd5a595dc78b7e.css","MediaType":"text/css","Data":{"Integrity":"sha256-oi9MfYwr3F49bjS6EctZq1DqV3JZTnEwW/1aWV3Hi34="}}
{"Target":"book.min.a643d39733d3a0ac48d6369128a52703207c5e11a74c3a70cfebfe0c15838ab8.css","MediaType":"text/css","Data":{"Integrity":"sha256-pkPTlzPToKxI1jaRKKUnAyB8XhGnTDpwz+v+DBWDirg="}}

80
go.mod
View File

@@ -11,22 +11,22 @@ require (
github.com/charmbracelet/lipgloss v1.1.0
github.com/charmbracelet/log v1.0.0
github.com/dustin/go-humanize v1.0.1
github.com/gabriel-vasile/mimetype v1.4.13
github.com/gabriel-vasile/mimetype v1.4.15
github.com/goccy/go-yaml v1.19.2
github.com/gotd/contrib v0.21.1
github.com/gotd/td v0.143.0
github.com/gotd/td v0.149.0
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3
github.com/krau/ffmpeg-go v0.6.0
github.com/lrstanley/go-ytdlp v1.3.5
github.com/minio/minio-go/v7 v7.2.0
github.com/playwright-community/playwright-go v0.5700.1
github.com/minio/minio-go/v7 v7.2.1
github.com/playwright-community/playwright-go v0.6000.0
github.com/rs/xid v1.6.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/unvgo/ghselfupdate v1.0.1
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c
golang.org/x/net v0.56.0
golang.org/x/term v0.44.0
golang.org/x/net v0.57.0
golang.org/x/term v0.45.0
golang.org/x/time v0.15.0
)
@@ -43,16 +43,16 @@ require (
github.com/charmbracelet/x/term v0.2.2 // indirect
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/cloudflare/circl v1.6.4 // indirect
github.com/cloudflare/circl v1.6.5 // indirect
github.com/coder/websocket v1.8.15 // indirect
github.com/deckarep/golang-set/v2 v2.9.0 // indirect
github.com/dlclark/regexp2 v1.12.0 // indirect
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
github.com/dlclark/regexp2/v2 v2.6.0 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/glebarez/go-sqlite v1.22.0 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/glebarez/go-sqlite v1.23.0 // indirect
github.com/go-faster/errors v0.8.0 // indirect
github.com/go-faster/jx v1.2.0 // indirect
github.com/go-faster/xor v1.0.0 // indirect
github.com/go-faster/yaml v0.4.6 // indirect
@@ -63,29 +63,29 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 // indirect
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gotd/ige v0.2.2 // indirect
github.com/gotd/ige v0.3.0 // indirect
github.com/gotd/neo v0.1.5 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/mattn/go-runewidth v0.0.27 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 // indirect
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/ogen-go/ogen v1.22.0 // indirect
github.com/ogen-go/ogen v1.24.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -94,52 +94,52 @@ require (
github.com/segmentio/asm v1.2.1 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/yuin/goldmark v1.8.2 // indirect
github.com/ulikunitz/xz v0.5.16 // indirect
github.com/yuin/goldmark v1.8.5 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.mongodb.org/mongo-driver v1.17.9 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/mod v0.37.0 // indirect
golang.org/x/tools v0.46.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/mod v0.39.0 // indirect
golang.org/x/tools v0.48.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/libc v1.75.3 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.53.0 // indirect
modernc.org/memory v1.12.0 // indirect
modernc.org/sqlite v1.56.0 // indirect
rsc.io/qr v0.2.0 // indirect
)
require (
github.com/dgraph-io/ristretto/v2 v2.4.0
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59
github.com/dgraph-io/ristretto/v2 v2.4.2
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6
github.com/duke-git/lancet/v2 v2.3.9
github.com/fsnotify/fsnotify v1.10.1
github.com/glebarez/sqlite v1.11.0
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.35.1 // indirect
github.com/ncruces/go-sqlite3 v0.35.3 // indirect
github.com/ncruces/go-sqlite3/gormlite v0.34.0
github.com/nicksnyder/go-i18n/v2 v2.6.1
github.com/pelletier/go-toml/v2 v2.4.2 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/xo/terminfo v1.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
golang.org/x/sync v0.21.0
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.38.0
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da // indirect
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0
gorm.io/gorm v1.31.2
)

180
go.sum
View File

@@ -66,8 +66,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U=
github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY=
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c=
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -76,16 +76,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4M5yOCKI=
github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM=
github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgraph-io/ristretto/v2 v2.4.2 h1:x0cvjmUKxt764Yxdk2nr94we1AvPPAMh1rh5TQ+Jo80=
github.com/dgraph-io/ristretto/v2 v2.4.2/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59 h1:DjKLmvKK9u15djHZ88N8M0DhgnHVgJJ8bnEe0h7Lga8=
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59/go.mod h1:Sc+QOu1WruvaaeT/cxFez/pXHpI9ZDjg/E8QNfSVveI=
github.com/dlclark/regexp2/v2 v2.6.0 h1:KugbSrpXcRpziHIcqqFEXwKwi09LbPkTzvLRItf2mo8=
github.com/dlclark/regexp2/v2 v2.6.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6 h1:Oh2rRG1un7tLlC3/NJDzKppZ4CeZGkVFJCUOTRwLpfw=
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6/go.mod h1:LiIEzozrcvNXorsG/3+ypGqdTUAqZryhzSsqi0oU/Qg=
github.com/duke-git/lancet/v2 v2.3.9 h1:ZxUvfoEY7YbsGIeoXRxHWIkRCAt6VN7UBKWgCCqBB3U=
github.com/duke-git/lancet/v2 v2.3.9/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -98,19 +98,18 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
github.com/glebarez/go-sqlite v1.23.0 h1:FyhIq4jqmgphQAUlY79zPldYGwISEZikaDfhiGWkkaI=
github.com/glebarez/go-sqlite v1.23.0/go.mod h1:IIYrOH3L0rHY3jb4IXOHoWdklNajSGUN2eJcvK8WrnI=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
github.com/go-faster/errors v0.8.0 h1:9T9eJrM+72dFk7n4DfhuaDDe6cyuFCSW2oNUkN77Yqc=
github.com/go-faster/errors v0.8.0/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
github.com/go-faster/jx v1.2.0 h1:T2YHJPrFaYu21fJtUxC9GzmluKu8rVIFDwwGBKTDseI=
github.com/go-faster/jx v1.2.0/go.mod h1:UWLOVDmMG597a5tBFPLIWJdUxz5/2emOpfsj9Neg0PE=
github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
@@ -119,8 +118,8 @@ github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQ
github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE=
github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
@@ -141,18 +140,18 @@ github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQF
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE=
github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gotd/contrib v0.21.1 h1:NSF+0YEnosQ34QEo2o4s6MA5YFDAor1LVvLhN1L3H1M=
github.com/gotd/contrib v0.21.1/go.mod h1:trVJBP9Q/TJbjmJbVnLc0cnX/8T4N0RpQBULVa3BNnE=
github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk=
github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0=
github.com/gotd/ige v0.3.0 h1:4f6LEHWsVDLBG0bT9wWG2/9TZb5aWm265G8ZlTXmRRU=
github.com/gotd/ige v0.3.0/go.mod h1:FE9bTaQtvfArizAcZuI4sS6gXaEUBmixdUufVHoCKac=
github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ=
github.com/gotd/td v0.143.0 h1:p0U/Nn92zXmAsahDn5CIVzay2kQ36lBBENT/FlWR2nQ=
github.com/gotd/td v0.143.0/go.mod h1:8GA5ecTI5iswLwBAlqf0u6/+j+BqSWUARSrX2Xk1usQ=
github.com/gotd/td v0.149.0 h1:vXzNO99FFzWKKyI0vizvtgkpI7AqmbcXHLWjxkD+69o=
github.com/gotd/td v0.149.0/go.mod h1:+s0fRWlKL+RqoKJAMnCRoj0sWtxzRlWm886elNZEpJQ=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
@@ -165,11 +164,11 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3 h1:2713fQZ560HxoNVgfJH41GKzjMjIG+DW4hH6nYXfXW8=
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -180,24 +179,24 @@ github.com/krau/ffmpeg-go v0.6.0 h1:F4HWvOrKXQsfLsFTOnUfP0HY6WISJqOrsAFGSIzkKto=
github.com/krau/ffmpeg-go v0.6.0/go.mod h1:sa7/bWHB6fO9j4lhmxnWQ1U07o+dE1leFjhctotxU7A=
github.com/lrstanley/go-ytdlp v1.3.5 h1:eT+29mK3Lp+XPMQOH25+jVerrrjifYW1o3IkTYJ9SMs=
github.com/lrstanley/go-ytdlp v1.3.5/go.mod h1:VgjnTrvkTf+23JuySjyPq1iQ8ijSovBtTPpXH5XrLtI=
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss=
github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw=
github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
@@ -206,10 +205,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/ncruces/go-sqlite3 v0.35.1 h1:h/LaVyQwIvBBT0+2JmVe2tbYyWjUQ093/pYhpBqdxJo=
github.com/ncruces/go-sqlite3 v0.35.1/go.mod h1:fXOSIkWwN5NXgbJk+7Zls8QIW4xOflmgh11OFvcY+J0=
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 h1:Cew7/eNAMd1zhpXYBjofBua/63pFvbvB2h4PM/p6gKU=
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302/go.mod h1:xe0CfafDUxfh+fSVKjHHMiAxoG9KALt5nFtbGNb/jRs=
github.com/ncruces/go-sqlite3 v0.35.3 h1:Ei07Zv1qfV/vyXzelhFsyS5Oh9TArBZHsmFk14Xv3GY=
github.com/ncruces/go-sqlite3 v0.35.3/go.mod h1:i1rhym/NIiB5xeEfzbN+e24Y+i7NGUpf7C2xZ3Dpwks=
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 h1:5NoQAewtgKNK3G4bjNPxVoGXu6F6NzLXWCTdD5FFAEY=
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304/go.mod h1:o8gr9w/50fXA5TDskg6bNUjvqmFfw4KaXth4q+yDSjg=
github.com/ncruces/go-sqlite3/gormlite v0.34.0 h1:QLlOy/i7OabsFUQ+d5KyXmq2hw9sMh/CRW435+eQMRY=
github.com/ncruces/go-sqlite3/gormlite v0.34.0/go.mod h1:CMv+6YhqLmPBXYACiQtrWA0q/JLIMTKB4E65SUfLgF0=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -218,24 +217,24 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
github.com/ogen-go/ogen v1.22.0 h1:7wU+jcIKg/JBAhM95909ULLdAkGr43KQOuvNpJ7Mxb4=
github.com/ogen-go/ogen v1.22.0/go.mod h1:7BOh9a51QiPCC92RMrj1LlkLjejhBAyPhR+oMc6lR9g=
github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/ogen-go/ogen v1.24.0 h1:NehBG/8s0JeM6jYHPJeFJntBG3cE/xQgZd1q8BchPNM=
github.com/ogen-go/ogen v1.24.0/go.mod h1:hcg4aTzLcu/MS1SGGahUriPBEOZB3c82RGIC6dYcdFU=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/playwright-community/playwright-go v0.5700.1 h1:PNFb1byWqrTT720rEO0JL88C6Ju0EmUnR5deFLvtP/U=
github.com/playwright-community/playwright-go v0.5700.1/go.mod h1:MlSn1dZrx8rszbCxY6x3qK89ZesJUYVx21B2JnkoNF0=
github.com/playwright-community/playwright-go v0.6000.0 h1:R7sENcRI6n0Zd5ZoW8EKdVF1ZVJgvTubfJeqKHNGvsw=
github.com/playwright-community/playwright-go v0.6000.0/go.mod h1:z/YpFVdU4LAi+0f9VPOCkGvmdH6dCrtza9nxnXFXgiE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -273,17 +272,17 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
github.com/unvgo/ghselfupdate v1.0.1 h1:4clbOkfPbfEmRnnYxVXDSBs0JG12DO+0FfqplJckreU=
github.com/unvgo/ghselfupdate v1.0.1/go.mod h1:3snWV5vEHGXQqqhY7FwKjPOtH6e7cFdHYN7UMAihhxs=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY=
github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c h1:xA2TJS9Hu/ivzaZIrDcwvpJ3Fnpsk5fDOJ4iSnL6J0w=
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c/go.mod h1:WSZ59bidJOO40JSJmLqlkBJrjZCtjbKKkygEMfzY/kc=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA=
github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
@@ -294,12 +293,12 @@ go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8=
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@@ -310,34 +309,35 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da h1:YKw1FZDyWyXXZcBxalRHz3CHieUKnKxanrbcO280Zwc=
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -347,31 +347,31 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -391,30 +391,30 @@ gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g=
modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU=
modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

17
pkg/storagetypes/batch.go Normal file
View File

@@ -0,0 +1,17 @@
package storagetypes
import "io"
// BatchItem describes one seekable file in a logical batch storage operation.
type BatchItem struct {
Reader io.ReadSeeker
StoragePath string
Size int64
// SourceGroupKey is empty for standalone source messages.
SourceGroupKey string
Caption string
// PreserveCaption distinguishes an intentionally empty source caption from
// the storage backend's default caption.
PreserveCaption bool
}

View File

@@ -0,0 +1,19 @@
package storagetypes
import "context"
type sourceCaptionContextKey struct{}
// WithSourceCaption records the source Telegram message caption for storage
// backends that can preserve it. Calling this function with an empty caption
// intentionally suppresses a backend-generated fallback caption.
func WithSourceCaption(ctx context.Context, caption string) context.Context {
return context.WithValue(ctx, sourceCaptionContextKey{}, caption)
}
// SourceCaptionFromContext returns the source caption and whether the caller
// explicitly supplied one.
func SourceCaptionFromContext(ctx context.Context) (string, bool) {
caption, ok := ctx.Value(sourceCaptionContextKey{}).(string)
return caption, ok
}

View File

@@ -0,0 +1,24 @@
package storagetypes
import (
"context"
"testing"
)
func TestSourceCaptionContext(t *testing.T) {
if _, ok := SourceCaptionFromContext(context.Background()); ok {
t.Fatal("caption unexpectedly present on empty context")
}
tests := []string{"original caption", ""}
for _, want := range tests {
ctx := WithSourceCaption(context.Background(), want)
got, ok := SourceCaptionFromContext(ctx)
if !ok {
t.Fatalf("caption %q was not marked as present", want)
}
if got != want {
t.Fatalf("caption = %q, want %q", got, want)
}
}
}

View File

@@ -1,6 +1,8 @@
package tcbdata
import (
"slices"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
"github.com/krau/SaveAny-Bot/pkg/parser"
"github.com/krau/SaveAny-Bot/pkg/telegraph"
@@ -31,12 +33,7 @@ func ConflictStrategyValues() []string {
}
func IsConflictStrategy(strategy string) bool {
for _, value := range ConflictStrategyValues() {
if strategy == value {
return true
}
}
return false
return slices.Contains(ConflictStrategyValues(), strategy)
}
// type TaskDataTGFiles struct {

View File

@@ -31,6 +31,38 @@ type StorageCannotStream interface {
CannotStream() string
}
// StorageBatchSaver can preserve relationships between files when saving a
// logical batch, such as a Telegram media album.
type StorageBatchSaver interface {
Storage
SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error
}
// StorageBatchProgressSaver reports confirmed upload progress for each item in
// a logical batch. The item index matches the items slice passed to
// SaveBatchWithProgress.
type StorageBatchProgressSaver interface {
StorageBatchSaver
SaveBatchWithProgress(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error
}
// StorageProgressSaver reports bytes after the backend has accepted them for
// upload. Backends with native progress support should implement this instead
// of relying on progress inferred from reads of the input stream.
type StorageProgressSaver interface {
Storage
SaveWithProgress(
ctx context.Context,
reader io.Reader,
storagePath string,
onProgress func(uploaded, total int64),
) error
}
// StorageListable 表示支持列举目录内容的存储
type StorageListable interface {
Storage
@@ -45,6 +77,9 @@ type StorageReadable interface {
var Storages = make(map[string]Storage)
var _ StorageProgressSaver = (*telegram.Telegram)(nil)
var _ StorageBatchProgressSaver = (*telegram.Telegram)(nil)
type StorageConstructor func() Storage
var storageConstructors = map[storenum.StorageType]StorageConstructor{

View File

@@ -0,0 +1,140 @@
package telegram
import (
"bytes"
"io"
"testing"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestPlanMediaGroups(t *testing.T) {
tests := []struct {
name string
items []batchMediaItem
wantSizes []int
}{
{
name: "same source album",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 1, true),
},
wantSizes: []int{2},
},
{
name: "different source albums",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("b", 1, true),
},
wantSizes: []int{1, 1},
},
{
name: "ungrouped messages",
items: []batchMediaItem{
albumItem("", 1, true),
albumItem("", 1, true),
},
wantSizes: []int{1, 1},
},
{
name: "different target chats",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 2, true),
},
wantSizes: []int{1, 1},
},
{
name: "ineligible media does not bridge albums",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 1, false),
albumItem("a", 1, true),
},
wantSizes: []int{1, 1, 1},
},
{
name: "maximum album size",
items: repeatedAlbumItems(11),
wantSizes: []int{10, 1},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
groups := planMediaGroups(tt.items)
if len(groups) != len(tt.wantSizes) {
t.Fatalf("got %d groups, want %d", len(groups), len(tt.wantSizes))
}
for i, want := range tt.wantSizes {
if got := len(groups[i]); got != want {
t.Errorf("group %d has %d items, want %d", i, got, want)
}
}
})
}
}
func TestMediaCaption(t *testing.T) {
empty := ""
original := "original caption"
tests := []struct {
name string
override *string
wantLen int
}{
{name: "filename fallback", wantLen: 1},
{name: "preserve empty source caption", override: &empty, wantLen: 0},
{name: "preserve source caption", override: &original, wantLen: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := len(mediaCaption("file.jpg", tt.override)); got != tt.wantLen {
t.Fatalf("got %d caption options, want %d", got, tt.wantLen)
}
})
}
}
func TestInspectBatchItemRewindsBeforeMimetypeDetection(t *testing.T) {
data := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
reader := bytes.NewReader(data)
if _, err := reader.Seek(4, io.SeekStart); err != nil {
t.Fatalf("failed to set initial reader offset: %v", err)
}
mediaItem, err := new(Telegram).inspectBatchItem(nil, storagetypes.BatchItem{
Reader: reader,
StoragePath: "photo.png",
Size: int64(len(data)),
})
if err != nil {
t.Fatalf("inspectBatchItem returned an error: %v", err)
}
if !mediaItem.albumEligible {
t.Fatal("albumEligible = false, want true for PNG input")
}
if offset, err := reader.Seek(0, io.SeekCurrent); err != nil {
t.Fatalf("failed to get final reader offset: %v", err)
} else if offset != 0 {
t.Fatalf("reader offset = %d, want 0", offset)
}
}
func albumItem(group string, chatID int64, eligible bool) batchMediaItem {
return batchMediaItem{
item: storagetypes.BatchItem{SourceGroupKey: group},
chatID: chatID,
albumEligible: eligible,
}
}
func repeatedAlbumItems(count int) []batchMediaItem {
items := make([]batchMediaItem, count)
for i := range items {
items[i] = albumItem("a", 1, true)
}
return items
}

View File

@@ -0,0 +1,60 @@
package telegram
import (
"context"
"sync"
"github.com/gotd/td/telegram/uploader"
)
var _ uploader.Progress = (*uploadProgress)(nil)
type uploadProgress struct {
mu sync.Mutex
onProgress func(uploaded, total int64)
total int64
uploaded int64
byID map[int64]int64
}
func newUploadProgress(total int64, onProgress func(uploaded, total int64)) *uploadProgress {
return &uploadProgress{
onProgress: onProgress,
total: total,
byID: make(map[int64]int64),
}
}
func (p *uploadProgress) Chunk(ctx context.Context, state uploader.ProgressState) error {
if err := ctx.Err(); err != nil {
return err
}
p.mu.Lock()
previous := p.byID[state.ID]
if state.Uploaded <= previous {
p.mu.Unlock()
return nil
}
p.byID[state.ID] = state.Uploaded
p.uploaded += state.Uploaded - previous
uploaded := p.uploaded
total := p.total
if total <= 0 {
total = state.Total
}
p.mu.Unlock()
if p.onProgress != nil {
p.onProgress(uploaded, total)
}
return nil
}
func (p *uploadProgress) reset(total int64) {
p.mu.Lock()
p.total = total
p.uploaded = 0
p.byID = make(map[int64]int64)
p.mu.Unlock()
}

View File

@@ -0,0 +1,80 @@
package telegram
import (
"context"
"testing"
"github.com/gotd/td/telegram/uploader"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestUploadProgressAggregatesUploaderParts(t *testing.T) {
type update struct {
uploaded int64
total int64
}
var updates []update
progress := newUploadProgress(100, func(uploaded, total int64) {
updates = append(updates, update{uploaded: uploaded, total: total})
})
states := []uploader.ProgressState{
{ID: 1, Uploaded: 20, Total: 60},
{ID: 1, Uploaded: 20, Total: 60},
{ID: 1, Uploaded: 60, Total: 60},
{ID: 2, Uploaded: 10, Total: 40},
{ID: 2, Uploaded: 40, Total: 40},
}
for _, state := range states {
if err := progress.Chunk(context.Background(), state); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
}
want := []update{{20, 100}, {60, 100}, {70, 100}, {100, 100}}
if len(updates) != len(want) {
t.Fatalf("got %d updates, want %d", len(updates), len(want))
}
for i := range want {
if updates[i] != want[i] {
t.Fatalf("update %d = %+v, want %+v", i, updates[i], want[i])
}
}
}
func TestUploadProgressResetForSplitFiles(t *testing.T) {
var uploaded, total int64
progress := newUploadProgress(100, func(current, size int64) {
uploaded, total = current, size
})
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 100, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
progress.reset(120)
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 2, Uploaded: 30, Total: 60}); err != nil {
t.Fatalf("Chunk() after reset failed: %v", err)
}
if uploaded != 30 || total != 120 {
t.Fatalf("progress after reset = %d/%d, want 30/120", uploaded, total)
}
}
func TestBatchItemUploadProgressPreservesItemIndex(t *testing.T) {
var gotIndex int
var gotUploaded, gotTotal int64
progress := batchItemUploadProgress(batchMediaItem{
index: 4,
item: storagetypes.BatchItem{Size: 100},
}, func(index int, uploaded, total int64) {
gotIndex = index
gotUploaded = uploaded
gotTotal = total
})
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 25, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
if gotIndex != 4 || gotUploaded != 25 || gotTotal != 100 {
t.Fatalf("batch progress = index %d, %d/%d; want index 4, 25/100", gotIndex, gotUploaded, gotTotal)
}
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/celestix/gotgproto/ext"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/retry"
"github.com/duke-git/lancet/v2/slice"
"github.com/duke-git/lancet/v2/validator"
"github.com/gabriel-vasile/mimetype"
@@ -26,14 +27,16 @@ import (
"github.com/krau/SaveAny-Bot/pkg/consts/tglimit"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
"github.com/rs/xid"
"golang.org/x/time/rate"
)
const (
// https://core.telegram.org/api/config#upload-max-fileparts-default
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
PremiumMaxUploadFileSize = 8000 * 524288 // 8000 * 512 KB
)
type Telegram struct {
@@ -41,6 +44,20 @@ type Telegram struct {
limiter *rate.Limiter
}
type preparedMedia struct {
peer tg.InputPeerClass
uploader *uploader.Uploader
media message.MultiMediaOption
}
type batchMediaItem struct {
item storagetypes.BatchItem
index int
chatID int64
albumEligible bool
useSingleSave bool
}
func (t *Telegram) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
telegramConfig, ok := cfg.(*storconfig.TelegramStorageConfig)
if !ok {
@@ -71,37 +88,147 @@ func (t *Telegram) Exists(ctx context.Context, storagePath string) bool {
}
func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) error {
return t.save(ctx, r, storagePath, nil)
}
// SaveWithProgress saves a file while reporting Telegram-confirmed upload
// progress after each uploaded part.
func (t *Telegram) SaveWithProgress(
ctx context.Context,
r io.Reader,
storagePath string,
onProgress func(uploaded, total int64),
) error {
size := contentLength(ctx)
return t.save(ctx, r, storagePath, newUploadProgress(size, onProgress))
}
func (t *Telegram) save(ctx context.Context, r io.Reader, storagePath string, progress *uploadProgress) error {
storagePath = path.Clean(storagePath)
captionOverride := sourceCaptionOverride(ctx)
tctx := tgutil.ExtFromContext(ctx)
if tctx == nil {
return fmt.Errorf("failed to get telegram context")
}
size := func() int64 {
if length := ctx.Value(ctxkey.ContentLength); length != nil {
if l, ok := length.(int64); ok {
return l
}
}
return -1 // unknown size
}()
if t.config.SkipLarge && size > MaxUploadFileSize {
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", MaxUploadFileSize, size)
size := contentLength(ctx)
maxUploadSize := maxUploadFileSize(tctx)
if t.config.SkipLarge && size > maxUploadSize {
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", maxUploadSize, size)
return nil
}
rs, seekable := r.(io.ReadSeeker)
splitSize := t.config.SplitSizeMB * 1024 * 1024
if splitSize <= 0 {
splitSize = DefaultSplitSize
splitSize := t.splitSize(maxUploadSize)
if size > splitSize {
filename, chatID := t.target(tctx, storagePath)
if filename == "" {
if rs, ok := r.(io.ReadSeeker); ok {
mtype, err := mimetype.DetectReader(rs)
if err != nil {
return fmt.Errorf("failed to detect mimetype: %w", err)
}
filename = xid.New().String() + mtype.Extension()
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek reader: %w", err)
}
}
}
upler := t.newUploader(tctx, size, progress)
peer := tryGetInputPeer(tctx, chatID)
if peer == nil || peer.Zero() {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
if t.config.SplitLargeVideo {
rs, ok := r.(io.ReadSeeker)
if ok {
mtype, detectErr := mimetype.DetectReader(rs)
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek large file after mimetype detection: %w", seekErr)
}
if detectErr != nil {
log.FromContext(ctx).Warnf("Failed to detect large file type, falling back to ZIP split: %s", detectErr)
} else if strings.HasPrefix(mtype.String(), "video/") {
parts, cleanup, splitErr := createLosslessVideoParts(
ctx,
rs,
filename,
size,
splitSize,
)
if splitErr != nil {
if ctx.Err() != nil {
return ctx.Err()
}
log.FromContext(ctx).Warnf("Lossless video split failed, falling back to ZIP split: %s", splitErr)
} else {
defer cleanup()
log.FromContext(ctx).Infof("Uploading oversized video as %d lossless-playable parts", len(parts))
for _, part := range parts {
log.FromContext(ctx).Infof("Prepared lossless video part %s (%d bytes)", part.Name, part.Size)
}
return t.uploadLosslessVideoParts(ctx, tctx, storagePath, parts, captionOverride, progress)
}
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek large video before ZIP fallback: %w", seekErr)
}
}
}
}
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize, progress)
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
prepared, err := t.prepareMedia(ctx, tctx, r, storagePath, size, nil, progress)
if err != nil {
return err
}
_, err = tctx.Sender.
WithUploader(prepared.uploader).
To(prepared.peer).
Media(ctx, prepared.media)
return err
}
func contentLength(ctx context.Context) int64 {
if length := ctx.Value(ctxkey.ContentLength); length != nil {
if size, ok := length.(int64); ok {
return size
}
}
return -1
}
func sourceCaptionOverride(ctx context.Context) *string {
caption, ok := storagetypes.SourceCaptionFromContext(ctx)
if !ok {
return nil
}
return &caption
}
func maxUploadFileSize(tctx *ext.Context) int64 {
if tctx != nil && tctx.Self != nil && !tctx.Self.GetBot() && tctx.Self.GetPremium() {
return PremiumMaxUploadFileSize
}
return MaxUploadFileSize
}
func (t *Telegram) splitSize(maxUploadSize int64) int64 {
splitSize := t.config.SplitSizeMB * 1024 * 1024
if splitSize <= 0 {
return maxUploadSize
}
return min(splitSize, maxUploadSize)
}
func (t *Telegram) target(tctx *ext.Context, storagePath string) (string, int64) {
// 去除前导斜杠并分隔路径, 当 len(parts):
// ==0, 存储到配置文件中的 chat_id, 随机文件名
// ==1, 视作只有文件名, 存储到配置文件中的 chat_id
// ==2, parts[0]: 视作要存储到的 chat_id, parts[1]: filename
// >=2, parts[0]: 视作要存储到的 chat_id, 最后一项为 filename
parts := slice.Compact(strings.Split(strings.TrimPrefix(storagePath, "/"), "/"))
filename := ""
chatID := t.config.ChatID
@@ -111,38 +238,66 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
if len(parts) >= 2 && validator.IsAlphaNumeric(parts[0]) {
cid, err := tgutil.ParseChatID(tctx, parts[0])
if err != nil {
// id不合法时使用配置文件中的 chat_id
log.FromContext(ctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
log.FromContext(tctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
cid = chatID
}
chatID = cid
}
return filename, chatID
}
func (t *Telegram) newUploader(tctx *ext.Context, size int64, progress *uploadProgress) *uploader.Uploader {
upler := uploader.NewUploader(tctx.Raw).
WithPartSize(tglimit.MaxUploadPartSize).
WithThreads(dlutil.BestThreads(size, config.C().Threads))
if progress != nil {
upler = upler.WithProgress(progress)
}
return upler
}
func mediaCaption(filename string, override *string) []message.StyledTextOption {
if override == nil {
return []message.StyledTextOption{styling.Plain(filename)}
}
if *override == "" {
return nil
}
return []message.StyledTextOption{styling.Plain(*override)}
}
func (t *Telegram) prepareMedia(
ctx context.Context,
tctx *ext.Context,
r io.Reader,
storagePath string,
size int64,
captionOverride *string,
progress *uploadProgress,
) (*preparedMedia, error) {
storagePath = path.Clean(storagePath)
filename, chatID := t.target(tctx, storagePath)
upler := t.newUploader(tctx, size, progress)
peer := tryGetInputPeer(tctx, chatID)
if peer == nil || peer.Zero() {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
return nil, fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
rs, seekable := r.(io.ReadSeeker)
var mtype *mimetype.MIME
if seekable {
var err error
mtype, err = mimetype.DetectReader(rs)
if err != nil {
return fmt.Errorf("failed to detect mimetype: %w", err)
return nil, fmt.Errorf("failed to detect mimetype: %w", err)
}
if filename == "" {
filename = xid.New().String() + mtype.Extension()
}
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek reader: %w", err)
return nil, fmt.Errorf("failed to seek reader: %w", err)
}
}
if size > splitSize {
// large file, use split uploader
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize)
}
var file tg.InputFileClass
var err error
@@ -152,21 +307,20 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
file, err = upler.Upload(ctx, uploader.NewUpload(filename, r, size))
}
if err != nil {
return fmt.Errorf("failed to upload file to telegram: %w", err)
return nil, fmt.Errorf("failed to upload file to telegram: %w", err)
}
caption := styling.Plain(filename)
caption := mediaCaption(filename, captionOverride)
forceFile := t.config.ForceFile
if mtype != nil && strings.HasPrefix(mtype.String(), "image/") && size >= tglimit.MaxPhotoSize {
forceFile = true
}
doc := message.UploadedDocument(file, caption).
doc := message.UploadedDocument(file, caption...).
Filename(filename).
ForceFile(forceFile)
if mtype != nil {
doc = doc.MIME(mtype.String())
}
var media message.MediaOption = doc
var media message.MultiMediaOption = doc
if mtype != nil && rs != nil {
switch mtypeStr := mtype.String(); {
case strings.HasPrefix(mtypeStr, "video/"):
@@ -205,19 +359,194 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
case strings.HasPrefix(mtypeStr, "audio/"):
media = doc.Audio().Title(filename)
case strings.HasPrefix(mtypeStr, "image/") && !strings.HasSuffix(mtypeStr, "webp"):
media = message.UploadedPhoto(file, caption)
media = message.UploadedPhoto(file, caption...)
}
}
sender := tctx.Sender
_, err = sender.WithUploader(upler).To(peer).Media(ctx, media)
return err
return &preparedMedia{
peer: peer,
uploader: upler,
media: media,
}, nil
}
// SaveBatch preserves each source photo/video group as a Telegram album.
func (t *Telegram) SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error {
return t.saveBatch(ctx, items, nil)
}
// SaveBatchWithProgress preserves source media groups while reporting native
// Telegram upload progress for each input item.
func (t *Telegram) SaveBatchWithProgress(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error {
return t.saveBatch(ctx, items, onProgress)
}
func (t *Telegram) saveBatch(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error {
tctx := tgutil.ExtFromContext(ctx)
if tctx == nil {
return fmt.Errorf("failed to get telegram context")
}
inspected := make([]batchMediaItem, 0, len(items))
for index, item := range items {
mediaItem, err := t.inspectBatchItem(tctx, item)
if err != nil {
return err
}
mediaItem.index = index
inspected = append(inspected, mediaItem)
}
for _, group := range planMediaGroups(inspected) {
if err := t.saveMediaGroup(ctx, tctx, group, onProgress); err != nil {
return err
}
}
return nil
}
func (t *Telegram) inspectBatchItem(tctx *ext.Context, item storagetypes.BatchItem) (batchMediaItem, error) {
_, chatID := t.target(tctx, path.Clean(item.StoragePath))
result := batchMediaItem{item: item, chatID: chatID}
maxUploadSize := maxUploadFileSize(tctx)
if (t.config.SkipLarge && item.Size > maxUploadSize) ||
item.Size > t.splitSize(maxUploadSize) {
result.useSingleSave = true
return result, nil
}
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return result, fmt.Errorf("failed to seek batch item before mimetype detection: %w", err)
}
mtype, err := mimetype.DetectReader(item.Reader)
if err != nil {
return result, fmt.Errorf("failed to detect batch item mimetype: %w", err)
}
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return result, fmt.Errorf("failed to seek batch item: %w", err)
}
mtypeStr := mtype.String()
forceFile := t.config.ForceFile || strings.HasPrefix(mtypeStr, "image/") && item.Size >= tglimit.MaxPhotoSize
result.albumEligible = !forceFile && (strings.HasPrefix(mtypeStr, "video/") ||
strings.HasPrefix(mtypeStr, "image/") && mtypeStr != "image/webp" && mtypeStr != "image/gif")
return result, nil
}
func planMediaGroups(items []batchMediaItem) [][]batchMediaItem {
groups := make([][]batchMediaItem, 0, len(items))
for i := 0; i < len(items); {
item := items[i]
if item.useSingleSave || !item.albumEligible || item.item.SourceGroupKey == "" {
groups = append(groups, items[i:i+1])
i++
continue
}
end := i + 1
for end < len(items) && end-i < 10 {
next := items[end]
if next.useSingleSave || !next.albumEligible || next.chatID != item.chatID || next.item.SourceGroupKey != item.item.SourceGroupKey {
break
}
end++
}
groups = append(groups, items[i:end])
i = end
}
return groups
}
func batchItemUploadProgress(
mediaItem batchMediaItem,
onProgress func(index int, uploaded, total int64),
) *uploadProgress {
if onProgress == nil {
return nil
}
return newUploadProgress(mediaItem.item.Size, func(uploaded, total int64) {
onProgress(mediaItem.index, uploaded, total)
})
}
func (t *Telegram) saveMediaGroup(
ctx context.Context,
tctx *ext.Context,
group []batchMediaItem,
onProgress func(index int, uploaded, total int64),
) error {
return retry.Retry(func() error {
if len(group) == 1 && group[0].useSingleSave {
mediaItem := group[0]
item := mediaItem.item
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek batch item: %w", err)
}
itemCtx := context.WithValue(ctx, ctxkey.ContentLength, item.Size)
if item.PreserveCaption {
itemCtx = storagetypes.WithSourceCaption(itemCtx, item.Caption)
}
if onProgress == nil {
return t.Save(itemCtx, item.Reader, item.StoragePath)
}
return t.SaveWithProgress(itemCtx, item.Reader, item.StoragePath, func(uploaded, total int64) {
onProgress(mediaItem.index, uploaded, total)
})
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
prepared := make([]preparedMedia, 0, len(group))
for _, mediaItem := range group {
item := mediaItem.item
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek batch item: %w", err)
}
var captionOverride *string
if item.PreserveCaption {
captionOverride = &item.Caption
}
progress := batchItemUploadProgress(mediaItem, onProgress)
media, err := t.prepareMedia(ctx, tctx, item.Reader, item.StoragePath, item.Size, captionOverride, progress)
if err != nil {
return err
}
prepared = append(prepared, *media)
}
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
if len(prepared) == 1 {
_, err := builder.Media(ctx, prepared[0].media)
return err
}
media := make([]message.MultiMediaOption, len(prepared))
for i := range prepared {
media[i] = prepared[i].media
}
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
return fmt.Errorf("failed to send media album: %w", err)
}
return nil
}, retry.Context(ctx), retry.RetryTimes(uint(config.C().Retry)))
}
func (t *Telegram) CannotStream() string {
return "Telegram storage must use a ReaderSeeker"
}
func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, upler *uploader.Uploader, peer tg.InputPeerClass, fileSize, splitSize int64) error {
func (t *Telegram) splitUpload(
ctx *ext.Context,
r io.Reader,
filename string,
upler *uploader.Uploader,
peer tg.InputPeerClass,
fileSize, splitSize int64,
progress *uploadProgress,
) error {
tempId := xid.New().String()
outputBase := filepath.Join(config.C().Temp.BasePath, tempId, strings.Split(filename, ".")[0])
defer func() {
@@ -234,6 +563,17 @@ func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, u
return fmt.Errorf("failed to glob split files: %w", err)
}
inputFiles := make([]tg.InputFileClass, 0, len(matched))
if progress != nil {
var uploadSize int64
for _, partPath := range matched {
partInfo, err := os.Stat(partPath)
if err != nil {
return fmt.Errorf("failed to stat split part %s: %w", partPath, err)
}
uploadSize += partInfo.Size()
}
progress.reset(uploadSize)
}
for _, partPath := range matched {
// 串行上传, 不然容易被tg风控
err = func() error {

View File

@@ -0,0 +1,119 @@
package telegram
import (
"bytes"
"testing"
"github.com/celestix/gotgproto/ext"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestMaxUploadFileSize(t *testing.T) {
tests := []struct {
name string
ctx *ext.Context
want int64
}{
{name: "missing context", want: MaxUploadFileSize},
{name: "missing self", ctx: new(ext.Context), want: MaxUploadFileSize},
{
name: "bot",
ctx: uploadAccountContext(true, false),
want: MaxUploadFileSize,
},
{
name: "premium bot still uses bot limit",
ctx: uploadAccountContext(true, true),
want: MaxUploadFileSize,
},
{
name: "regular user",
ctx: uploadAccountContext(false, false),
want: MaxUploadFileSize,
},
{
name: "premium user",
ctx: uploadAccountContext(false, true),
want: PremiumMaxUploadFileSize,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := maxUploadFileSize(tt.ctx); got != tt.want {
t.Fatalf("maxUploadFileSize() = %d, want %d", got, tt.want)
}
})
}
}
func uploadAccountContext(bot, premium bool) *ext.Context {
self := new(tg.User)
self.SetBot(bot)
self.SetPremium(premium)
return &ext.Context{Self: self}
}
func TestSplitSizeUsesUploaderLimit(t *testing.T) {
tests := []struct {
name string
splitSizeMB int64
accountLimit int64
want int64
}{
{
name: "bot default",
accountLimit: MaxUploadFileSize,
want: MaxUploadFileSize,
},
{
name: "premium default",
accountLimit: PremiumMaxUploadFileSize,
want: PremiumMaxUploadFileSize,
},
{
name: "explicit lower limit",
splitSizeMB: 1500,
accountLimit: PremiumMaxUploadFileSize,
want: 1500 * 1024 * 1024,
},
{
name: "explicit limit is capped by account",
splitSizeMB: 5000,
accountLimit: PremiumMaxUploadFileSize,
want: PremiumMaxUploadFileSize,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
telegramStorage := Telegram{}
telegramStorage.config.SplitSizeMB = tt.splitSizeMB
if got := telegramStorage.splitSize(tt.accountLimit); got != tt.want {
t.Fatalf("splitSize() = %d, want %d", got, tt.want)
}
})
}
}
func TestBatchUploadLimitAppliesPerFile(t *testing.T) {
telegramStorage := Telegram{}
tctx := uploadAccountContext(true, false)
data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom")
itemSize := int64(MaxUploadFileSize/2 + 1)
for index := range 2 {
item, err := telegramStorage.inspectBatchItem(tctx, storagetypes.BatchItem{
Reader: bytes.NewReader(data),
StoragePath: "video.mp4",
Size: itemSize,
})
if err != nil {
t.Fatalf("inspectBatchItem() failed for item %d: %v", index, err)
}
if item.useSingleSave {
t.Fatalf("item %d was treated as oversized even though only the batch total exceeds the limit", index)
}
}
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"time"
"github.com/celestix/gotgproto/ext"
@@ -14,6 +15,34 @@ import (
"github.com/yapingcat/gomedia/go-mp4"
)
// sourceFile returns a filesystem path to the media for ffmpeg/ffprobe. Those
// tools need a SEEKABLE input: feeding them a pipe (pipe:0) fails for
// non-faststart MP4s whose moov atom is at the END of the file, because ffmpeg
// cannot seek backwards on a stream to read it. That silently broke thumbnail
// and metadata extraction for a large share of downloaded videos. If rs is
// already an *os.File we use it directly; otherwise we spool it to a temp file
// and return a cleanup func.
func sourceFile(rs io.ReadSeeker) (path string, cleanup func(), err error) {
noop := func() {}
if f, ok := rs.(*os.File); ok {
return f.Name(), noop, nil
}
if _, err = rs.Seek(0, io.SeekStart); err != nil {
return "", noop, err
}
tf, err := os.CreateTemp("", "saveany-media-*.tmp")
if err != nil {
return "", noop, err
}
if _, err = io.Copy(tf, rs); err != nil {
tf.Close()
os.Remove(tf.Name())
return "", noop, err
}
tf.Close()
return tf.Name(), func() { os.Remove(tf.Name()) }, nil
}
type VideoMetadata struct {
Duration int
Width int
@@ -52,16 +81,14 @@ func getMP4Meta(rs io.ReadSeeker) (metadata *VideoMetadata, err error) {
// getVideoMetadata uses ffprobe to get video metadata
func getVideoMetadata(rs io.ReadSeeker) (*VideoMetadata, error) {
pipeReader, pipeWriter := io.Pipe()
path, cleanup, err := sourceFile(rs)
if err != nil {
return nil, err
}
defer cleanup()
go func() {
defer pipeWriter.Close()
rs.Seek(0, io.SeekStart)
io.Copy(pipeWriter, rs)
}()
result, err := ffmpeg.ProbeReaderWithTimeout(
pipeReader,
result, err := ffmpeg.ProbeWithTimeout(
path,
time.Second*10,
ffmpeg.KwArgs{
"select_streams": "v:0",
@@ -114,25 +141,22 @@ func extractThumbFrame(rs io.ReadSeeker) ([]byte, error) {
}
func extractFrameAt(rs io.ReadSeeker, timestamp float64) ([]byte, error) {
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
rs.Seek(0, io.SeekStart)
io.Copy(pipeWriter, rs)
}()
path, cleanup, err := sourceFile(rs)
if err != nil {
return nil, err
}
defer cleanup()
var out bytes.Buffer
err := ffmpeg.
Input("pipe:0", ffmpeg.KwArgs{
err = ffmpeg.
Input(path, ffmpeg.KwArgs{
"ss": fmt.Sprintf("%.3f", timestamp),
}).
Output("pipe:1", ffmpeg.KwArgs{
"vframes": 1,
"f": "mjpeg",
}).
WithInput(pipeReader).
WithOutput(&out).
OverWriteOutput().
Run()

View File

@@ -0,0 +1,372 @@
package telegram
import (
"context"
"fmt"
"io"
"math"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/celestix/gotgproto/ext"
"github.com/charmbracelet/log"
"github.com/gotd/td/telegram/message"
"github.com/rs/xid"
"github.com/krau/SaveAny-Bot/config"
)
const (
videoPartTargetRatio = 0.95
videoSplitAttempts = 4
minSegmentDuration = 1.0
maxLosslessVideoParts = 10
)
type losslessVideoPart struct {
Path string
Name string
Size int64
}
type mediaToolRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
var runMediaTool mediaToolRunner = func(ctx context.Context, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
output, err := cmd.CombinedOutput()
if err != nil {
message := strings.TrimSpace(string(output))
if message == "" {
return nil, err
}
return nil, fmt.Errorf("%w: %s", err, message)
}
return output, nil
}
func createLosslessVideoParts(
ctx context.Context,
r io.ReadSeeker,
filename string,
fileSize, maxPartSize int64,
) ([]losslessVideoPart, func(), error) {
inputPath, sourceCleanup, err := sourceFile(r)
if err != nil {
return nil, func() {}, fmt.Errorf("failed to prepare seekable video source: %w", err)
}
tempBase := config.C().Temp.BasePath
if err := os.MkdirAll(tempBase, 0o755); err != nil {
sourceCleanup()
return nil, func() {}, fmt.Errorf("failed to create video split base directory: %w", err)
}
tempDir, err := os.MkdirTemp(tempBase, "telegram-video-split-*")
if err != nil {
sourceCleanup()
return nil, func() {}, fmt.Errorf("failed to create video split directory: %w", err)
}
cleanup := func() {
if err := os.RemoveAll(tempDir); err != nil {
log.FromContext(ctx).Warnf("Failed to clean lossless video parts: %s", err)
}
sourceCleanup()
}
parts, err := splitLosslessVideo(ctx, inputPath, tempDir, filename, fileSize, maxPartSize)
if err != nil {
cleanup()
return nil, func() {}, err
}
return parts, cleanup, nil
}
func splitLosslessVideo(
ctx context.Context,
inputPath, outputDir, filename string,
fileSize, maxPartSize int64,
) ([]losslessVideoPart, error) {
if fileSize <= maxPartSize {
return nil, fmt.Errorf("video size %d does not exceed part limit %d", fileSize, maxPartSize)
}
if maxPartSize <= 0 {
return nil, fmt.Errorf("invalid video part limit: %d", maxPartSize)
}
duration, err := probeMediaDuration(ctx, inputPath)
if err != nil {
return nil, fmt.Errorf("failed to probe source video duration: %w", err)
}
if duration < minSegmentDuration {
return nil, fmt.Errorf("invalid source video duration: %.3f", duration)
}
targetSize := int64(float64(maxPartSize) * videoPartTargetRatio)
segmentDuration := initialSegmentDuration(duration, fileSize, targetSize)
extension := videoPartExtension(filename)
outputPattern := filepath.Join(outputDir, "part-%03d"+extension)
var lastOversize int64
for range videoSplitAttempts {
if err := clearVideoParts(outputDir); err != nil {
return nil, err
}
if err := runFFmpegSegment(ctx, inputPath, outputPattern, extension, segmentDuration); err != nil {
return nil, fmt.Errorf("failed to losslessly split video: %w", err)
}
parts, largest, err := collectVideoParts(ctx, outputDir, filename)
if err != nil {
return nil, err
}
if len(parts) < 2 {
return nil, fmt.Errorf("video split produced %d part(s), expected at least 2", len(parts))
}
if len(parts) > maxLosslessVideoParts {
return nil, fmt.Errorf(
"video split produced %d parts, exceeding the single-album limit of %d",
len(parts),
maxLosslessVideoParts,
)
}
if largest <= maxPartSize {
return parts, nil
}
lastOversize = largest
segmentDuration *= float64(targetSize) / float64(largest)
if segmentDuration < minSegmentDuration {
break
}
}
return nil, fmt.Errorf(
"unable to keep lossless video parts below %d bytes; largest part was %d bytes",
maxPartSize,
lastOversize,
)
}
func initialSegmentDuration(duration float64, fileSize, targetSize int64) float64 {
partCount := math.Ceil(float64(fileSize) / float64(targetSize))
if partCount < 2 {
partCount = 2
}
segmentDuration := duration / partCount
if segmentDuration < minSegmentDuration {
return minSegmentDuration
}
return segmentDuration
}
func videoPartExtension(filename string) string {
extension := strings.ToLower(filepath.Ext(filepath.Base(filename)))
switch extension {
case ".avi", ".m4v", ".mkv", ".mov", ".mp4", ".ts", ".webm":
return extension
default:
return ".mp4"
}
}
func runFFmpegSegment(
ctx context.Context,
inputPath, outputPattern, extension string,
segmentDuration float64,
) error {
args := []string{
"-hide_banner",
"-loglevel", "error",
"-nostdin",
"-y",
"-i", inputPath,
"-map", "0",
"-map_metadata", "0",
"-c", "copy",
"-f", "segment",
"-segment_time", strconv.FormatFloat(segmentDuration, 'f', 3, 64),
"-segment_start_number", "1",
"-reset_timestamps", "1",
"-avoid_negative_ts", "make_zero",
}
if extension == ".mp4" || extension == ".m4v" || extension == ".mov" {
args = append(args, "-segment_format_options", "movflags=+faststart")
}
args = append(args, outputPattern)
_, err := runMediaTool(ctx, "ffmpeg", args...)
return err
}
func probeMediaDuration(ctx context.Context, filePath string) (float64, error) {
output, err := runMediaTool(
ctx,
"ffprobe",
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
filePath,
)
if err != nil {
return 0, err
}
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
if err != nil {
return 0, fmt.Errorf("invalid ffprobe duration %q: %w", strings.TrimSpace(string(output)), err)
}
return duration, nil
}
func clearVideoParts(outputDir string) error {
entries, err := os.ReadDir(outputDir)
if err != nil {
return fmt.Errorf("failed to read video split directory: %w", err)
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
if err := os.Remove(filepath.Join(outputDir, entry.Name())); err != nil {
return fmt.Errorf("failed to clear old video part %s: %w", entry.Name(), err)
}
}
return nil
}
func collectVideoParts(
ctx context.Context,
outputDir, filename string,
) ([]losslessVideoPart, int64, error) {
matches, err := filepath.Glob(filepath.Join(outputDir, "part-*"))
if err != nil {
return nil, 0, fmt.Errorf("failed to list video parts: %w", err)
}
sort.Strings(matches)
if len(matches) == 0 {
return nil, 0, fmt.Errorf("ffmpeg did not produce any video parts")
}
extension := videoPartExtension(filename)
base := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filepath.Base(filename)))
if base == "" || base == "." {
base = xid.New().String()
}
parts := make([]losslessVideoPart, 0, len(matches))
var largest int64
for index, match := range matches {
info, err := os.Stat(match)
if err != nil {
return nil, 0, fmt.Errorf("failed to stat video part %s: %w", match, err)
}
if info.Size() <= 0 {
return nil, 0, fmt.Errorf("video part %s is empty", match)
}
if _, err := probeMediaDuration(ctx, match); err != nil {
return nil, 0, fmt.Errorf("failed to validate video part %s: %w", match, err)
}
if info.Size() > largest {
largest = info.Size()
}
parts = append(parts, losslessVideoPart{
Path: match,
Name: fmt.Sprintf("%s.part%03d%s", base, index+1, extension),
Size: info.Size(),
})
}
return parts, largest, nil
}
func partStoragePath(storagePath, partName string) string {
directory := path.Dir(path.Clean(storagePath))
if directory == "." || directory == "/" {
return partName
}
return path.Join(directory, partName)
}
func (t *Telegram) uploadLosslessVideoParts(
ctx context.Context,
tctx *ext.Context,
storagePath string,
parts []losslessVideoPart,
sourceCaption *string,
progress *uploadProgress,
) error {
if len(parts) == 0 {
return fmt.Errorf("no lossless video parts to upload")
}
if len(parts) > maxLosslessVideoParts {
return fmt.Errorf(
"refusing to upload %d lossless video parts as multiple albums; maximum is %d",
len(parts),
maxLosslessVideoParts,
)
}
resetLosslessVideoUploadProgress(progress, parts)
prepared := make([]preparedMedia, 0, len(parts))
for index, part := range parts {
partFile, err := os.Open(part.Path)
if err != nil {
return fmt.Errorf("failed to open video part %s: %w", part.Name, err)
}
media, prepareErr := t.prepareMedia(
ctx,
tctx,
partFile,
partStoragePath(storagePath, part.Name),
part.Size,
videoPartCaption(sourceCaption, index),
progress,
)
closeErr := partFile.Close()
if prepareErr != nil {
return fmt.Errorf("failed to prepare video part %s: %w", part.Name, prepareErr)
}
if closeErr != nil {
return fmt.Errorf("failed to close video part %s: %w", part.Name, closeErr)
}
prepared = append(prepared, *media)
}
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
if len(prepared) == 1 {
if _, err := builder.Media(ctx, prepared[0].media); err != nil {
return fmt.Errorf("failed to send video part: %w", err)
}
return nil
}
media := make([]message.MultiMediaOption, len(prepared))
for index := range prepared {
media[index] = prepared[index].media
}
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
return fmt.Errorf("failed to send video parts as album: %w", err)
}
return nil
}
func resetLosslessVideoUploadProgress(progress *uploadProgress, parts []losslessVideoPart) {
if progress == nil {
return
}
var total int64
for _, part := range parts {
total += part.Size
}
progress.reset(total)
}
func videoPartCaption(sourceCaption *string, index int) *string {
if sourceCaption == nil {
return nil
}
if index == 0 {
return sourceCaption
}
empty := ""
return &empty
}

View File

@@ -0,0 +1,195 @@
package telegram
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gotd/td/telegram/uploader"
)
func TestInitialSegmentDuration(t *testing.T) {
tests := []struct {
name string
duration float64
fileSize int64
targetSize int64
want float64
}{
{name: "two balanced parts", duration: 120, fileSize: 2200, targetSize: 1900, want: 60},
{name: "three balanced parts", duration: 120, fileSize: 3900, targetSize: 1900, want: 40},
{name: "minimum one second", duration: 1.5, fileSize: 3900, targetSize: 1900, want: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := initialSegmentDuration(tt.duration, tt.fileSize, tt.targetSize)
if got != tt.want {
t.Fatalf("initialSegmentDuration() = %v, want %v", got, tt.want)
}
})
}
}
func TestVideoPartExtension(t *testing.T) {
tests := map[string]string{
"clip.MOV": ".mov",
"clip.mkv": ".mkv",
"clip.exe": ".mp4",
"clip": ".mp4",
}
for input, want := range tests {
if got := videoPartExtension(input); got != want {
t.Fatalf("videoPartExtension(%q) = %q, want %q", input, got, want)
}
}
}
func TestPartStoragePath(t *testing.T) {
tests := map[string]string{
"clip.mov": "clip.part001.mov",
"123456/clip.mov": "123456/clip.part001.mov",
"/123456/clip.mov": "/123456/clip.part001.mov",
"folder/sub/clip.mov": "folder/sub/clip.part001.mov",
}
for input, want := range tests {
if got := partStoragePath(input, "clip.part001.mov"); got != want {
t.Fatalf("partStoragePath(%q) = %q, want %q", input, got, want)
}
}
}
func TestVideoPartCaption(t *testing.T) {
if got := videoPartCaption(nil, 0); got != nil {
t.Fatalf("caption without source = %q, want nil", *got)
}
source := "original caption"
first := videoPartCaption(&source, 0)
if first == nil || *first != source {
t.Fatalf("first part caption = %v, want %q", first, source)
}
second := videoPartCaption(&source, 1)
if second == nil || *second != "" {
t.Fatalf("second part caption = %v, want an explicit empty caption", second)
}
}
func TestResetLosslessVideoUploadProgressUsesCombinedPartSize(t *testing.T) {
var uploaded, total int64
progress := newUploadProgress(999, func(current, size int64) {
uploaded = current
total = size
})
resetLosslessVideoUploadProgress(progress, []losslessVideoPart{
{Size: 100},
{Size: 250},
})
if err := progress.Chunk(t.Context(), uploader.ProgressState{ID: 1, Uploaded: 50, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
if uploaded != 50 || total != 350 {
t.Fatalf("lossless video progress = %d/%d, want 50/350", uploaded, total)
}
}
func TestSplitLosslessVideoRetriesOversizedPart(t *testing.T) {
tempDir := t.TempDir()
inputPath := filepath.Join(tempDir, "source.mov")
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
t.Fatal(err)
}
outputDir := filepath.Join(tempDir, "parts")
if err := os.Mkdir(outputDir, 0o755); err != nil {
t.Fatal(err)
}
originalRunner := runMediaTool
t.Cleanup(func() { runMediaTool = originalRunner })
ffmpegCalls := 0
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "ffprobe":
return []byte("120.0\n"), nil
case "ffmpeg":
ffmpegCalls++
pattern := args[len(args)-1]
sizes := []int{2001, 199}
if ffmpegCalls > 1 {
sizes = []int{1100, 1100}
}
for index, size := range sizes {
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
if err := os.WriteFile(partPath, make([]byte, size), 0o600); err != nil {
return nil, err
}
}
return nil, nil
default:
return nil, fmt.Errorf("unexpected media tool %q", name)
}
}
parts, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
if err != nil {
t.Fatalf("splitLosslessVideo() failed: %v", err)
}
if ffmpegCalls != 2 {
t.Fatalf("ffmpeg calls = %d, want 2", ffmpegCalls)
}
if len(parts) != 2 {
t.Fatalf("parts = %d, want 2", len(parts))
}
if parts[0].Name != "example.part001.mov" || parts[1].Name != "example.part002.mov" {
t.Fatalf("unexpected part names: %#v", parts)
}
for _, part := range parts {
if part.Size > 2000 {
t.Fatalf("part %s exceeds limit: %d", part.Name, part.Size)
}
}
}
func TestSplitLosslessVideoRejectsMultipleAlbums(t *testing.T) {
tempDir := t.TempDir()
inputPath := filepath.Join(tempDir, "source.mov")
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
t.Fatal(err)
}
outputDir := filepath.Join(tempDir, "parts")
if err := os.Mkdir(outputDir, 0o755); err != nil {
t.Fatal(err)
}
originalRunner := runMediaTool
t.Cleanup(func() { runMediaTool = originalRunner })
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "ffprobe":
return []byte("120.0\n"), nil
case "ffmpeg":
pattern := args[len(args)-1]
for index := range maxLosslessVideoParts + 1 {
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
if err := os.WriteFile(partPath, make([]byte, 100), 0o600); err != nil {
return nil, err
}
}
return nil, nil
default:
return nil, fmt.Errorf("unexpected media tool %q", name)
}
}
_, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
if err == nil {
t.Fatal("splitLosslessVideo() succeeded with more than one album of parts")
}
if !strings.Contains(err.Error(), "single-album limit") {
t.Fatalf("splitLosslessVideo() error = %q, want single-album limit", err)
}
}

View File

@@ -119,21 +119,21 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
return nil
}
parts := strings.Split(dirPath, "/")
currentPath := ""
var currentPath strings.Builder
for i, part := range parts {
if i > 0 {
currentPath += "/"
currentPath.WriteString("/")
}
currentPath += part
currentPath.WriteString(part)
exists, err := c.Exists(ctx, currentPath)
exists, err := c.Exists(ctx, currentPath.String())
if err != nil {
return err
}
if exists {
continue
}
url := c.BaseURL + currentPath
url := c.BaseURL + currentPath.String()
resp, err := c.doRequest(ctx, WebdavMethodMkcol, url, nil)
if err != nil {
return err
@@ -141,7 +141,7 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("MKCOL %s: %s", currentPath, resp.Status)
return fmt.Errorf("MKCOL %s: %s", currentPath.String(), resp.Status)
}
}
return nil