Compare commits

...

8 Commits

Author SHA1 Message Date
krau
a1bd2ed762 fix: upgrade deps 2026-08-11 08:57:09 +08:00
krau
7e3e26fcf4 style: format codebase 2026-08-11 08:47:35 +08:00
Haopeng Huo
e49ebef977 fix(telegraph): parse page URLs from message entities (#232) 2026-08-11 08:43:46 +08:00
Haopeng Huo
da38f5bb88 fix: remove cache files after reader closes (#231) 2026-08-11 08:41:40 +08:00
Haopeng Huo
0e6ed66ef5 feat: show upload progress for batch file tasks (#228)
* feat: show upload progress for file tasks

* feat: show upload progress for batch file tasks

* style: distinguish download and upload phases

* style: mark successful task completion

* fix: preserve storage save error context

* fix: serialize single-file progress updates

* fix: stabilize batch upload progress reporting

Correct batch upload totals, completion state, actual file sizes, and synchronized progress snapshots. Add regression coverage for concurrent updates and phase transitions.

* feat: show per-file transfer progress

Format single-file and batch download and upload states with Telegram entities, blockquotes, speeds, transferred sizes, progress bars, concise counters, and accurate confirmation handling. Cover upload retries plus final, error, and cancellation messages with regression tests.

* chore: remove transfer progress tests

* test: restore critical transfer progress coverage

* test: cover interleaved batch transfers

* fix: declare progress styles in locale templates
2026-08-11 08:40:29 +08:00
Haopeng Huo
e4144e73e6 feat(telegram): split oversized videos into playable parts (#229)
* feat(telegram): split oversized videos losslessly

* fix(telegram): respect Premium upload limits

* fix(telegram): keep split videos in one album
2026-08-05 16:57:46 +08:00
Haopeng Huo
1d794a7b9c feat(telegram): preserve source media groups (#226)
* feat(telegram): preserve source media groups

* fix(telegram): rewind batch readers before inspection
2026-08-04 16:46:29 +08:00
irudisca
52f880f0f2 fix(telegram): use seekable file path for thumbnail & metadata extraction (#225)
extractFrameAt() and getVideoMetadata() piped the media into ffmpeg/ffprobe
via pipe:0. A pipe is not seekable, so ffmpeg cannot decode non-faststart MP4s
whose moov atom sits at the END of the file (very common for yt-dlp / HLS-merged
downloads): the frame grab and probe silently fail, and the video is uploaded
with no thumbnail (and, for non-mp4 containers, no dimensions).

Hand ffmpeg/ffprobe a seekable file path instead. When the reader already is an
*os.File we use it directly; otherwise we spool to a temp file and clean it up.

Verified with an A/B upload of a 56 MB non-faststart clip: stock build produced
no thumbnail, patched build produced a correct thumbnail.

Co-authored-by: pennyucloud <valentino@pennyu.co.id>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-23 18:28:03 +08:00
47 changed files with 4324 additions and 569 deletions

View File

@@ -11,23 +11,23 @@ import (
// guarded by mu. It implements taskevent.Sink so the task layer can update it
// without knowing about the API.
type TaskProgressInfo struct {
mu sync.Mutex
TaskID string
Type string
Status TaskStatus
Title string
TotalBytes int64
DownloadedBytes int64
TotalFiles int
DownloadedFiles int
Storage string
Path string
Error string
CreatedAt time.Time
UpdatedAt time.Time
StartedAt time.Time
Webhook string
webhookNotified bool
mu sync.Mutex
TaskID string
Type string
Status TaskStatus
Title string
TotalBytes int64
DownloadedBytes int64
TotalFiles int
DownloadedFiles int
Storage string
Path string
Error string
CreatedAt time.Time
UpdatedAt time.Time
StartedAt time.Time
Webhook string
webhookNotified bool
}
// progressStore holds all API tasks. Entries are removed a fixed duration after
@@ -208,9 +208,9 @@ func NewProgressTracker(taskID, taskType, storage, path, title, webhook string)
return &ProgressTracker{}
}
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
func (p *ProgressTracker) OnProgress(downloadedBytes int64, downloadedFiles int) {}
func (p *ProgressTracker) OnDone(err error) {}
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
func (p *ProgressTracker) UpdateProgressFiles(files int) {}
func (p *ProgressTracker) OnDone(err error) {}
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
func (p *ProgressTracker) UpdateProgressFiles(files int) {}

View File

@@ -13,8 +13,8 @@ import (
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
"github.com/krau/SaveAny-Bot/pkg/tfile"
"github.com/krau/SaveAny-Bot/storage"

View File

@@ -5,6 +5,6 @@ import "regexp"
var (
TgMessageLinkRegexString = `https?://(?:t|telegram)\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
TgMessageLinkRegexp = regexp.MustCompile(TgMessageLinkRegexString)
TelegraphUrlRegexString = `https://telegra.ph/.*`
TelegraphUrlRegexString = `https://telegra\.ph/[^\s]+`
TelegraphUrlRegexp = regexp.MustCompile(TelegraphUrlRegexString)
)

View File

@@ -3,6 +3,7 @@ package shortcut
import (
"encoding/json"
"fmt"
"net/url"
"strings"
@@ -179,21 +180,19 @@ type TelegraphResult struct {
// return replied message, image urls, telegraph path(unescaped), error
func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*types.Message, *TelegraphResult, error) {
logger := log.FromContext(ctx)
tphurl := re.TelegraphUrlRegexp.FindString(tgutil.ExtractMessageEntityUrlsText(update.EffectiveMessage.Message))
tphurl := findTelegraphURL(update.EffectiveMessage.Message)
if tphurl == "" {
logger.Warnf("No telegraph url found but called handleTelegraph")
return nil, nil, dispatcher.ContinueGroups
}
pagepath := strings.Split(tphurl, "/")[len(strings.Split(tphurl, "/"))-1]
tphdir, err := url.PathUnescape(pagepath)
pagepath, err := parseTelegraphPagePath(tphurl)
if err != nil {
logger.Errorf("Failed to unescape telegraph path: %s", err)
logger.Errorf("Failed to parse telegraph path: %s", err)
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorParseTelegraphPathFailed, map[string]any{
"Error": err.Error(),
})), nil)
return nil, nil, dispatcher.EndGroups
}
tphdir = strings.TrimSpace(tphdir)
msg, err := ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoFetchingTelegraphPage, nil)), nil)
if err != nil {
logger.Errorf("Failed to reply to update: %s", err)
@@ -244,7 +243,57 @@ func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*type
}
return msg, &TelegraphResult{
Pics: imgs,
TphDir: tphdir,
TphDir: pagepath,
Page: page,
}, nil
}
func findTelegraphURL(msg *tg.Message) string {
if msg == nil {
return ""
}
var firstMatch string
findValid := func(text string) string {
for _, tphurl := range re.TelegraphUrlRegexp.FindAllString(text, -1) {
if firstMatch == "" {
firstMatch = tphurl
}
if _, err := parseTelegraphPagePath(tphurl); err == nil {
return tphurl
}
}
return ""
}
for _, entityURL := range tgutil.ExtractMessageEntityUrls(msg) {
if tphurl := findValid(entityURL); tphurl != "" {
return tphurl
}
}
if tphurl := findValid(msg.GetMessage()); tphurl != "" {
return tphurl
}
return firstMatch
}
func parseTelegraphPagePath(pageURL string) (string, error) {
u, err := url.Parse(pageURL)
if err != nil {
return "", fmt.Errorf("invalid telegraph URL: %w", err)
}
if u.Scheme != "https" || !strings.EqualFold(u.Hostname(), "telegra.ph") {
return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host)
}
pagepath := strings.Trim(u.EscapedPath(), "/")
if pagepath == "" || strings.Contains(pagepath, "/") {
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
}
pagepath, err = url.PathUnescape(pagepath)
if err != nil {
return "", fmt.Errorf("failed to unescape telegraph path: %w", err)
}
pagepath = strings.TrimSpace(pagepath)
if pagepath == "" || strings.Contains(pagepath, "/") {
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
}
return pagepath, nil
}

View File

@@ -0,0 +1,163 @@
package shortcut
import (
"testing"
"github.com/gotd/td/tg"
)
func TestFindTelegraphURL(t *testing.T) {
tests := []struct {
name string
msg *tg.Message
want string
}{
{
name: "single URL entity",
msg: &tg.Message{
Message: "https://telegra.ph/Example-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 32},
},
},
want: "https://telegra.ph/Example-01-02",
},
{
name: "Telegraph URL before another URL",
msg: &tg.Message{
Message: "https://telegra.ph/Example-01-02 https://example.com/",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 32},
&tg.MessageEntityURL{Offset: 33, Length: 20},
},
},
want: "https://telegra.ph/Example-01-02",
},
{
name: "hidden Telegraph URL",
msg: &tg.Message{
Message: "article",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityTextURL{
Offset: 0,
Length: 7,
URL: "https://telegra.ph/Hidden-01-02",
},
},
},
want: "https://telegra.ph/Hidden-01-02",
},
{
name: "URL entity after non-BMP character",
msg: &tg.Message{
Message: "😀 https://telegra.ph/Emoji-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 3, Length: 30},
},
},
want: "https://telegra.ph/Emoji-01-02",
},
{
name: "valid Telegraph URL after invalid candidate",
msg: &tg.Message{
Message: "https://telegra.ph/nested/Bad https://telegra.ph/Valid-01-02",
Entities: []tg.MessageEntityClass{
&tg.MessageEntityURL{Offset: 0, Length: 29},
&tg.MessageEntityURL{Offset: 30, Length: 30},
},
},
want: "https://telegra.ph/Valid-01-02",
},
{
name: "plain message fallback",
msg: &tg.Message{Message: "read https://telegra.ph/Plain-01-02 now"},
want: "https://telegra.ph/Plain-01-02",
},
{
name: "nil message",
msg: nil,
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := findTelegraphURL(tt.msg); got != tt.want {
t.Fatalf("findTelegraphURL() = %q, want %q", got, tt.want)
}
})
}
}
func TestParseTelegraphPagePath(t *testing.T) {
tests := []struct {
name string
pageURL string
want string
wantErr bool
}{
{
name: "plain path",
pageURL: "https://telegra.ph/Example-01-02",
want: "Example-01-02",
},
{
name: "escaped path with query and fragment",
pageURL: "https://telegra.ph/%E6%B5%8B%E8%AF%95-01-02?source=telegram#top",
want: "测试-01-02",
},
{
name: "trailing slash",
pageURL: "https://telegra.ph/Example-01-02/",
want: "Example-01-02",
},
{
name: "root URL",
pageURL: "https://telegra.ph/",
wantErr: true,
},
{
name: "wrong host",
pageURL: "https://example.com/Example-01-02",
wantErr: true,
},
{
name: "wrong scheme",
pageURL: "http://telegra.ph/Example-01-02",
wantErr: true,
},
{
name: "invalid percent escape",
pageURL: "https://telegra.ph/Invalid-%zz",
wantErr: true,
},
{
name: "nested path",
pageURL: "https://telegra.ph/nested/Example-01-02",
wantErr: true,
},
{
name: "encoded slash",
pageURL: "https://telegra.ph/nested%2FExample-01-02",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseTelegraphPagePath(tt.pageURL)
if tt.wantErr {
if err == nil {
t.Fatalf("parseTelegraphPagePath(%q) returned no error", tt.pageURL)
}
return
}
if err != nil {
t.Fatalf("parseTelegraphPagePath(%q) failed: %v", tt.pageURL, err)
}
if got != tt.want {
t.Fatalf("parseTelegraphPagePath(%q) = %q, want %q", tt.pageURL, got, tt.want)
}
})
}
}

View File

@@ -150,28 +150,50 @@ const (
BotMsgProgressAria2Downloading Key = "bot.msg.progress.aria2_downloading"
BotMsgProgressAria2Start Key = "bot.msg.progress.aria2_start"
BotMsgProgressAvgSpeedPrefix Key = "bot.msg.progress.avg_speed_prefix"
BotMsgProgressBatchDonePrefix Key = "bot.msg.progress.batch_done_prefix"
BotMsgProgressBatchProcessingPrefix Key = "bot.msg.progress.batch_processing_prefix"
BotMsgProgressBatchStartPrefix Key = "bot.msg.progress.batch_start_prefix"
BotMsgProgressBatchCanceled Key = "bot.msg.progress.batch_canceled"
BotMsgProgressBatchDone Key = "bot.msg.progress.batch_done"
BotMsgProgressBatchDoneWithSkipped Key = "bot.msg.progress.batch_done_with_skipped"
BotMsgProgressBatchFailedGroup Key = "bot.msg.progress.batch_failed_group"
BotMsgProgressBatchFailedItem Key = "bot.msg.progress.batch_failed_item"
BotMsgProgressBatchFailedTask Key = "bot.msg.progress.batch_failed_task"
BotMsgProgressBatchFailureStageBatchUpload Key = "bot.msg.progress.batch_failure_stage_batch_upload"
BotMsgProgressBatchFailureStageCache Key = "bot.msg.progress.batch_failure_stage_cache"
BotMsgProgressBatchFailureStageConfirm Key = "bot.msg.progress.batch_failure_stage_confirm"
BotMsgProgressBatchFailureStageDownload Key = "bot.msg.progress.batch_failure_stage_download"
BotMsgProgressBatchFailureStageInternal Key = "bot.msg.progress.batch_failure_stage_internal"
BotMsgProgressBatchFailureStageUpload Key = "bot.msg.progress.batch_failure_stage_upload"
BotMsgProgressBatchItemConfirming Key = "bot.msg.progress.batch_item_confirming"
BotMsgProgressBatchItemDownloading Key = "bot.msg.progress.batch_item_downloading"
BotMsgProgressBatchItemDownloadingUnknown Key = "bot.msg.progress.batch_item_downloading_unknown"
BotMsgProgressBatchItemRetrying Key = "bot.msg.progress.batch_item_retrying"
BotMsgProgressBatchItemTransferring Key = "bot.msg.progress.batch_item_transferring"
BotMsgProgressBatchItemTransferringUnknown Key = "bot.msg.progress.batch_item_transferring_unknown"
BotMsgProgressBatchItemUploading Key = "bot.msg.progress.batch_item_uploading"
BotMsgProgressBatchStatusHeader Key = "bot.msg.progress.batch_status_header"
BotMsgProgressBatchSummaryConfirming Key = "bot.msg.progress.batch_summary_confirming"
BotMsgProgressBatchSummaryFailed Key = "bot.msg.progress.batch_summary_failed"
BotMsgProgressBatchSummaryHiddenActive Key = "bot.msg.progress.batch_summary_hidden_active"
BotMsgProgressBatchSummarySkipped Key = "bot.msg.progress.batch_summary_skipped"
BotMsgProgressCurrentProgressPrefix Key = "bot.msg.progress.current_progress_prefix"
BotMsgProgressCurrentSpeedPrefix Key = "bot.msg.progress.current_speed_prefix"
BotMsgProgressDirectDonePrefix Key = "bot.msg.progress.direct_done_prefix"
BotMsgProgressDirectStart Key = "bot.msg.progress.direct_start"
BotMsgProgressDownloadDonePrefix Key = "bot.msg.progress.download_done_prefix"
BotMsgProgressDownloadFailedPrefix Key = "bot.msg.progress.download_failed_prefix"
BotMsgProgressDownloadedPrefix Key = "bot.msg.progress.downloaded_prefix"
BotMsgProgressDownloadingPrefix Key = "bot.msg.progress.downloading_prefix"
BotMsgProgressErrorPrefix Key = "bot.msg.progress.error_prefix"
BotMsgProgressFileNamePrefix Key = "bot.msg.progress.file_name_prefix"
BotMsgProgressFileProcessingPrefix Key = "bot.msg.progress.file_processing_prefix"
BotMsgProgressFileSizePrefix Key = "bot.msg.progress.file_size_prefix"
BotMsgProgressFileStartPrefix Key = "bot.msg.progress.file_start_prefix"
BotMsgProgressParsedDonePrefix Key = "bot.msg.progress.parsed_done_prefix"
BotMsgProgressParsedStartPrefix Key = "bot.msg.progress.parsed_start_prefix"
BotMsgProgressProcessingListPrefix Key = "bot.msg.progress.processing_list_prefix"
BotMsgProgressProcessingNone Key = "bot.msg.progress.processing_none"
BotMsgProgressSavePathPrefix Key = "bot.msg.progress.save_path_prefix"
BotMsgProgressTaskCanceled Key = "bot.msg.progress.task_canceled"
BotMsgProgressSingleCanceled Key = "bot.msg.progress.single_canceled"
BotMsgProgressSingleDone Key = "bot.msg.progress.single_done"
BotMsgProgressSingleDownloading Key = "bot.msg.progress.single_downloading"
BotMsgProgressSingleDownloadingUnknown Key = "bot.msg.progress.single_downloading_unknown"
BotMsgProgressSingleFailed Key = "bot.msg.progress.single_failed"
BotMsgProgressSingleStatusHeader Key = "bot.msg.progress.single_status_header"
BotMsgProgressSingleUploading Key = "bot.msg.progress.single_uploading"
BotMsgProgressSingleUploadRetrying Key = "bot.msg.progress.single_upload_retrying"
BotMsgProgressTaskCanceledWithId Key = "bot.msg.progress.task_canceled_with_id"
BotMsgProgressTaskFailedWithError Key = "bot.msg.progress.task_failed_with_error"
BotMsgProgressTelegraphDonePrefix Key = "bot.msg.progress.telegraph_done_prefix"

View File

@@ -351,32 +351,54 @@ bot:
info_filename_prefix: "Filename: "
info_prompt_select_storage: "\nPlease select storage"
progress:
batch_start_prefix: "Starting batch download task\nTotal size: "
batch_processing_prefix: "Processing batch download task\nTotal size: "
batch_status_header: "<b>📦 Processing</b>\n\nFiles: <code>{{.Total}}</code>\nStatus: ✅ <code>{{.Completed}}</code> | 📥 <code>{{.Downloaded}}</code> | ⏳ <code>{{.Waiting}}</code>\nTotal speed: ⬇️ <code>{{.DownloadSpeed}}</code> | ⬆️ <code>{{.UploadSpeed}}</code>"
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nRetry: <code>{{.Attempt}}/{{.Limit}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nReason: <code>{{.Reason}}</code></blockquote>"
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} Waiting</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>Uploaded, awaiting remote confirmation</i></blockquote>"
batch_summary_hidden_active: "🔄 <code>{{.Count}}</code> more files are active"
batch_summary_confirming: "☁️ Uploaded, awaiting group send: <code>{{.Count}}</code>"
batch_summary_failed: "❌ Failed: <code>{{.Count}}</code>"
batch_summary_skipped: "⏭️ Skipped: <code>{{.Count}}</code>"
batch_done: "<b>✅ Completed</b>\n\nFiles: <code>{{.Count}}</code>\nTotal size: <code>{{.Size}}</code>"
batch_done_with_skipped: "<b>⚠️ Completed</b>\n\nSucceeded: <code>{{.Success}}</code>\nSkipped: <code>{{.Skipped}}</code>\nTotal size: <code>{{.Size}}</code>"
batch_canceled: "<b>🚫 Task canceled</b>\n\nFiles: <code>{{.Total}}</code>\nCompleted: <code>{{.Completed}}</code>\nIncomplete: <code>{{.Incomplete}}</code>\nSkipped: <code>{{.Skipped}}</code>"
batch_failed_item: "<b>❌ Processing failed</b>\n\nFailed file: <code>{{.Index}}. {{.Name}}</code>\nStage: <code>{{.Stage}}</code>\nProgress: <code>{{.Progress}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Failed: <code>{{.Failed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failed_group: "<b>❌ Batch upload failed</b>\n\nAffected files: <code>{{.Affected}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Batch failed: <code>{{.Failed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failed_task: "<b>❌ Processing failed</b>\n\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n⏹ Incomplete: <code>{{.Incomplete}}</code>"
batch_failure_stage_download: "download"
batch_failure_stage_cache: "local cache"
batch_failure_stage_upload: "upload"
batch_failure_stage_confirm: "remote confirmation"
batch_failure_stage_batch_upload: "batch upload"
batch_failure_stage_internal: "internal task"
single_status_header: "<b>📦 Processing</b>"
single_downloading: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_downloading_unknown: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown\nSave to: <code>{{.Destination}}</code></blockquote>"
single_uploading: "<blockquote><b>⬆️ Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_upload_retrying: "<blockquote><b>🔁 Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nAttempt: <code>{{.Attempt}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
single_done: "<b>✅ Completed</b>\n\nFilename: <code>{{.Name}}</code>\nTotal size: <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code>"
single_canceled: "<b>🚫 Task canceled</b>\n\nFilename: <code>{{.Name}}</code>"
single_failed: "<b>❌ Processing failed</b>\n\nFilename: <code>{{.Name}}</code>\nReason: <code>{{.Reason}}</code>"
downloading_prefix: "Downloading\nTotal size: "
processing_list_prefix: "\nProcessing:\n"
processing_none: " - None"
avg_speed_prefix: "\nAverage speed: "
current_progress_prefix: "\nCurrent progress: "
task_canceled: "Task canceled"
task_canceled_with_id: "Processing canceled: {{.TaskID}}"
task_failed_with_error: "Processing failed: {{.Error}}"
batch_done_prefix: "Completed\nFile count: "
direct_done_prefix: "Completed, file count: "
parsed_start_prefix: "Starting download from {{.Site}}\nTotal size: "
parsed_done_prefix: "Completed, resource count: "
telegraph_start_prefix: "Starting Telegraph download\nImage count: "
telegraph_progress_prefix: "Downloading\nCurrent progress: "
telegraph_done_prefix: "Completed\nImage count: "
file_start_prefix: "Starting download\nFilename: "
file_processing_prefix: "Processing download task\nFilename: "
download_failed_prefix: "Download failed\nFilename: "
download_done_prefix: "Download completed\nFilename: "
file_size_prefix: "\nFile size: "
save_path_prefix: "\nSave path: "
total_size_prefix: "\nTotal size: "
direct_start: "Starting download, total size: {{.SizeMB}} MB ({{.Count}} files)"
file_name_prefix: "Filename: "
error_prefix: "\nError: "
aria2_start: "Waiting for Aria2 to complete download (GID: {{.GID}})..."
aria2_downloading: "Aria2 downloading (GID: {{.GID}})\n"

View File

@@ -352,32 +352,54 @@ bot:
info_filename_prefix: "文件名: "
info_prompt_select_storage: "\n请选择存储位置"
progress:
batch_start_prefix: "开始执行批量下载任务\n总大小: "
batch_processing_prefix: "正在处理批量下载任务\n总大小: "
batch_status_header: "<b>📦 正在处理</b>\n\n文件<code>{{.Total}}</code>\n状态✅ <code>{{.Completed}}</code> 📥 <code>{{.Downloaded}}</code> ⏳ <code>{{.Waiting}}</code>\n总速度 <code>{{.DownloadSpeed}}</code> ⬆️ <code>{{.UploadSpeed}}</code>"
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知</blockquote>"
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知</blockquote>"
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n重试次数<code>{{.Attempt}}/{{.Limit}}</code>\n失败前速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n原因<code>{{.Reason}}</code></blockquote>"
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} 等待中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>文件已上传,正在等待远端确认</i></blockquote>"
batch_summary_hidden_active: "🔄 另有 <code>{{.Count}}</code> 个文件正在处理"
batch_summary_confirming: "☁️ 已上传,等待整组发送:<code>{{.Count}}</code>"
batch_summary_failed: "❌ 失败:<code>{{.Count}}</code>"
batch_summary_skipped: "⏭️ 已跳过:<code>{{.Count}}</code>"
batch_done: "<b>✅ 处理完成</b>\n\n文件数: <code>{{.Count}}</code>\n总大小: <code>{{.Size}}</code>"
batch_done_with_skipped: "<b>⚠️ 处理完成</b>\n\n成功: <code>{{.Success}}</code>\n已跳过: <code>{{.Skipped}}</code>\n总大小: <code>{{.Size}}</code>"
batch_canceled: "<b>🚫 任务已取消</b>\n\n文件数: <code>{{.Total}}</code>\n已完成: <code>{{.Completed}}</code>\n未完成: <code>{{.Incomplete}}</code>\n已跳过: <code>{{.Skipped}}</code>"
batch_failed_item: "<b>❌ 处理失败</b>\n\n失败文件: <code>{{.Index}}. {{.Name}}</code>\n失败阶段: <code>{{.Stage}}</code>\n失败进度: <code>{{.Progress}}</code>\n失败前速度: <code>{{.Speed}}</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 失败: <code>{{.Failed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failed_group: "<b>❌ 批量上传失败</b>\n\n受影响文件: <code>{{.Affected}} 个</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 批次失败: <code>{{.Failed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failed_task: "<b>❌ 处理失败</b>\n\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n⏹ 未完成: <code>{{.Incomplete}}</code>"
batch_failure_stage_download: "下载"
batch_failure_stage_cache: "本地缓存"
batch_failure_stage_upload: "上传"
batch_failure_stage_confirm: "云端确认"
batch_failure_stage_batch_upload: "批量上传"
batch_failure_stage_internal: "任务内部"
single_status_header: "<b>📦 正在处理</b>"
single_downloading: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_downloading_unknown: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / 未知\n保存至<code>{{.Destination}}</code></blockquote>"
single_uploading: "<blockquote><b>⬆️ 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_upload_retrying: "<blockquote><b>🔁 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n尝试次数<code>{{.Attempt}}</code>\n速度<code>{{.Speed}}</code>\n大小<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code></blockquote>"
single_done: "<b>✅ 处理完成</b>\n\n文件名<code>{{.Name}}</code>\n总大小<code>{{.Size}}</code>\n保存至<code>{{.Destination}}</code>"
single_canceled: "<b>🚫 任务已取消</b>\n\n文件名<code>{{.Name}}</code>"
single_failed: "<b>❌ 处理失败</b>\n\n文件名<code>{{.Name}}</code>\n原因<code>{{.Reason}}</code>"
downloading_prefix: "正在下载\n总大小: "
processing_list_prefix: "\n正在处理:\n"
processing_none: " - 无"
avg_speed_prefix: "\n平均速度: "
current_progress_prefix: "\n当前进度: "
task_canceled: "任务已取消"
task_canceled_with_id: "处理已取消: {{.TaskID}}"
task_failed_with_error: "处理失败: {{.Error}}"
batch_done_prefix: "处理完成\n文件数: "
direct_done_prefix: "处理完成, 文件数量: "
parsed_start_prefix: "开始下载 {{.Site}} 的资源\n总大小: "
parsed_done_prefix: "处理完成, 资源数量: "
telegraph_start_prefix: "开始下载Telegraph\n图片数量: "
telegraph_progress_prefix: "正在下载\n当前进度: "
telegraph_done_prefix: "处理完成\n图片数量: "
file_start_prefix: "开始下载\n文件名: "
file_processing_prefix: "正在处理下载任务\n文件名: "
download_failed_prefix: "下载失败\n文件名: "
download_done_prefix: "下载完成\n文件名: "
file_size_prefix: "\n文件大小: "
save_path_prefix: "\n保存路径: "
total_size_prefix: "\n总大小: "
direct_start: "开始下载, 总大小: {{.SizeMB}} MB ({{.Count}} 个文件)"
file_name_prefix: "文件名: "
error_prefix: "\n错误: "
aria2_start: "等待 Aria2 下载完成 (GID: {{.GID}})..."
aria2_downloading: "Aria2 正在下载 (GID: {{.GID}})\n"

View File

@@ -0,0 +1,42 @@
package fsutil_test
import (
"errors"
"os"
"path/filepath"
"testing"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
)
func TestCloseAndRemove(t *testing.T) {
tests := []struct {
name string
preClose bool
}{
{name: "open file"},
{name: "already closed file", preClose: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "cache-file")
file, err := fsutil.CreateFile(filePath)
if err != nil {
t.Fatalf("CreateFile() failed: %v", err)
}
if tt.preClose {
if err := file.Close(); err != nil {
t.Fatalf("Close() failed: %v", err)
}
}
if err := file.CloseAndRemove(); err != nil {
t.Fatalf("CloseAndRemove() failed: %v", err)
}
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("cache file still exists after CloseAndRemove(): %v", err)
}
})
}
}

View File

@@ -1,6 +1,7 @@
package fsutil
import (
"errors"
"os"
"path/filepath"
"strings"
@@ -41,10 +42,11 @@ func (f *File) Remove() error {
}
func (f *File) CloseAndRemove() error {
if err := f.Close(); err != nil {
return err
closeErr := f.Close()
if errors.Is(closeErr, os.ErrClosed) {
closeErr = nil
}
return f.Remove()
return errors.Join(closeErr, f.Remove())
}
func CreateFile(fp string) (*File, error) {

View File

@@ -17,7 +17,11 @@ type ProgressReadSeeker struct {
// Seek implements io.ReadSeeker.
func (pr *ProgressReadSeeker) Seek(offset int64, whence int) (int64, error) {
return pr.reader.Seek(offset, whence)
position, err := pr.reader.Seek(offset, whence)
if err == nil {
pr.read.Store(position)
}
return position, err
}
// NewProgressReader creates a new ProgressReader
@@ -54,7 +58,7 @@ func (pr *ProgressReadSeeker) Progress() float64 {
return float64(pr.read.Load()) / float64(pr.total.Load())
}
// Read returns the number of bytes read so far
// BytesRead returns the current tracked reader position.
func (pr *ProgressReadSeeker) BytesRead() int64 {
return pr.read.Load()
}

View File

@@ -0,0 +1,50 @@
package ioutil
import (
"bytes"
"io"
"testing"
)
func TestProgressReadSeekerTracksReads(t *testing.T) {
var gotRead, gotTotal int64
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, func(read, total int64) {
gotRead = read
gotTotal = total
})
buffer := make([]byte, 4)
if _, err := io.ReadFull(reader, buffer); err != nil {
t.Fatalf("read failed: %v", err)
}
if gotRead != 4 || gotTotal != 6 {
t.Fatalf("progress = %d/%d, want 4/6", gotRead, gotTotal)
}
if reader.BytesRead() != 4 {
t.Fatalf("BytesRead() = %d, want 4", reader.BytesRead())
}
}
func TestProgressReadSeekerResetsPositionOnSeek(t *testing.T) {
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, nil)
buffer := make([]byte, 4)
if _, err := io.ReadFull(reader, buffer); err != nil {
t.Fatalf("read failed: %v", err)
}
position, err := reader.Seek(0, io.SeekStart)
if err != nil {
t.Fatalf("seek failed: %v", err)
}
if position != 0 || reader.BytesRead() != 0 {
t.Fatalf("position after seek = %d (tracked %d), want 0", position, reader.BytesRead())
}
if _, err := io.ReadFull(reader, buffer[:2]); err != nil {
t.Fatalf("read after seek failed: %v", err)
}
if reader.BytesRead() != 2 {
t.Fatalf("BytesRead() after seek and read = %d, want 2", reader.BytesRead())
}
}

View File

@@ -0,0 +1,35 @@
package tgutil
import (
stdhtml "html"
"github.com/gotd/td/telegram/message/entity"
messagehtml "github.com/gotd/td/telegram/message/html"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
)
// EscapeHTMLTemplateData returns a copy of data with string values escaped for
// interpolation into Telegram HTML templates.
func EscapeHTMLTemplateData(data map[string]any) map[string]any {
escaped := make(map[string]any, len(data))
for key, value := range data {
if text, ok := value.(string); ok {
escaped[key] = stdhtml.EscapeString(text)
continue
}
escaped[key] = value
}
return escaped
}
// RenderHTML renders Telegram-compatible HTML into plain text and message
// entities.
func RenderHTML(markup string) (string, []tg.MessageEntityClass, error) {
var builder entity.Builder
if err := styling.Perform(&builder, messagehtml.String(nil, markup)); err != nil {
return "", nil, err
}
text, entities := builder.Complete()
return text, entities, nil
}

View File

@@ -0,0 +1,54 @@
package tgutil
import (
"testing"
"github.com/gotd/td/tg"
)
func TestEscapeHTMLTemplateDataDoesNotMutateInput(t *testing.T) {
input := map[string]any{
"Text": `<b>A&B</b>`,
"Count": 2,
}
escaped := EscapeHTMLTemplateData(input)
if got, want := escaped["Text"], "&lt;b&gt;A&amp;B&lt;/b&gt;"; got != want {
t.Fatalf("escaped text = %q, want %q", got, want)
}
if got := input["Text"]; got != `<b>A&B</b>` {
t.Fatalf("input was mutated: %q", got)
}
if got := escaped["Count"]; got != 2 {
t.Fatalf("non-string value = %v, want 2", got)
}
}
func TestRenderHTMLUsesTemplateStylesAndDecodesValues(t *testing.T) {
data := EscapeHTMLTemplateData(map[string]any{"Name": `<b>A&B</b>.bin`})
markup := `<blockquote><b>Uploading</b>
<code>` + data["Name"].(string) + `</code></blockquote>`
text, entities, err := RenderHTML(markup)
if err != nil {
t.Fatalf("RenderHTML() failed: %v", err)
}
if want := "Uploading\n<b>A&B</b>.bin"; text != want {
t.Fatalf("rendered text = %q, want %q", text, want)
}
var bold, code, blockquote int
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
}
}
if bold != 1 || code != 1 || blockquote != 1 {
t.Fatalf("entity counts = bold:%d code:%d blockquote:%d", bold, code, blockquote)
}
}

View File

@@ -2,6 +2,7 @@ package tgutil
import (
"fmt"
"sort"
"strconv"
"strings"
"unicode"
@@ -359,9 +360,16 @@ func GetGroupedMessages(ctx *ext.Context, chatID int64, msg *tg.Message) ([]*tg.
groupedMessages = append(groupedMessages, m)
}
}
sortMessagesByID(groupedMessages)
return groupedMessages, nil
}
func sortMessagesByID(messages []*tg.Message) {
sort.Slice(messages, func(i, j int) bool {
return messages[i].GetID() < messages[j].GetID()
})
}
func ExtractMessageEntityUrls(msg *tg.Message) []string {
if len(msg.Entities) == 0 {
return nil

View File

@@ -0,0 +1,18 @@
package tgutil
import (
"testing"
"github.com/gotd/td/tg"
)
func TestSortMessagesByID(t *testing.T) {
messages := []*tg.Message{{ID: 9}, {ID: 3}, {ID: 7}}
sortMessagesByID(messages)
want := []int{3, 7, 9}
for i := range messages {
if messages[i].GetID() != want[i] {
t.Fatalf("message %d has ID %d, want %d", i, messages[i].GetID(), want[i])
}
}
}

View File

@@ -8,14 +8,16 @@ import (
type TelegramStorageConfig struct {
BaseConfig
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
// split files larger than Telegram limit(2GB) into parts of specified size, in MB, leave 0 to set default(2000MB)
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
SplitLargeVideo bool `toml:"split_large_video" mapstructure:"split_large_video" json:"split_large_video"`
// split files larger than the uploader account limit into parts of specified size, in MB
// leave 0 to use the account limit (2000MB for bots/regular users, 4000MB for Premium users)
// only effective when SkipLarge is false
// use zip when splitting
// use zip when splitting non-video files or when lossless video splitting is disabled/unavailable
SplitSizeMB int64 `toml:"split_size_mb" mapstructure:"split_size_mb" json:"split_size_mb"`
}

View File

@@ -6,6 +6,7 @@ import (
"io"
"os"
"path"
"time"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/retry"
@@ -14,45 +15,269 @@ import (
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
"github.com/krau/SaveAny-Bot/pkg/taskevent"
"github.com/krau/SaveAny-Bot/storage"
"golang.org/x/sync/errgroup"
)
type executionGroup struct {
elems []*TaskElement
batchSaver storage.StorageBatchSaver
}
func (g executionGroup) usesBatchSaver() bool {
return g.batchSaver != nil
}
func (t *Task) Execute(ctx context.Context) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("batch_file[%s]", t.ID))
logger.Info("Starting batch file task")
t.Progress.OnStart(ctx, t)
workers := config.C().Workers
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(workers)
for _, elem := range t.elems {
eg.Go(func() error {
t.processingMu.RLock()
if t.processing[elem.ID] != nil {
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
groups := t.executionGroups()
var err error
for i := 0; i < len(groups); {
if groups[i].usesBatchSaver() {
err = t.processBatch(ctx, groups[i])
i++
} else {
end := i + 1
for end < len(groups) && !groups[end].usesBatchSaver() {
end++
}
t.processingMu.RUnlock()
t.processingMu.Lock()
t.processing[elem.ID] = &elem
t.processingMu.Unlock()
defer func() {
t.processingMu.Lock()
delete(t.processing, elem.ID)
t.processingMu.Unlock()
}()
return t.processElement(gctx, elem)
})
elems := make([]*TaskElement, 0, end-i)
for _, group := range groups[i:end] {
elems = append(elems, group.elems...)
}
err = t.processElements(ctx, elems)
i = end
}
if err != nil {
break
}
}
err := eg.Wait()
if err != nil {
logger.Errorf("Error during batch file processing: %v", err)
} else {
logger.Info("Batch file task completed successfully")
}
t.finishItems(err)
t.Progress.OnDone(ctx, t, err)
return err
}
func (t *Task) executionGroups() []executionGroup {
groups := make([]executionGroup, 0, len(t.elems))
for i := 0; i < len(t.elems); {
elem := &t.elems[i]
batchSaver, batchCapable := elem.Storage.(storage.StorageBatchSaver)
if !batchCapable || elem.sourceGroupKey == "" {
groups = append(groups, executionGroup{elems: []*TaskElement{elem}})
i++
continue
}
end := i + 1
for end < len(t.elems) {
next := &t.elems[end]
if next.Storage != elem.Storage || next.sourceGroupKey != elem.sourceGroupKey {
break
}
end++
}
elems := make([]*TaskElement, 0, end-i)
for j := i; j < end; j++ {
elems = append(elems, &t.elems[j])
}
groups = append(groups, executionGroup{elems: elems, batchSaver: batchSaver})
i = end
}
return groups
}
func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error {
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(config.C().Workers)
for _, elem := range elems {
eg.Go(func() error {
if err := t.markProcessing(ctx, elem); err != nil {
return err
}
defer t.unmarkProcessing(elem.ID)
return t.processElement(gctx, *elem)
})
}
return eg.Wait()
}
func (t *Task) processBatch(ctx context.Context, group executionGroup) error {
defer func() {
for _, elem := range group.elems {
if err := os.Remove(elem.localPath); err != nil && !os.IsNotExist(err) {
log.FromContext(ctx).Warnf("Failed to cleanup batch cache file %s: %v", elem.localPath, err)
}
}
}()
eg, gctx := errgroup.WithContext(ctx)
eg.SetLimit(config.C().Workers)
for _, elem := range group.elems {
eg.Go(func() error {
if err := t.markProcessing(ctx, elem); err != nil {
return err
}
defer t.unmarkProcessing(elem.ID)
return t.downloadElement(gctx, elem)
})
}
if err := eg.Wait(); err != nil {
return err
}
items := make([]storagetypes.BatchItem, 0, len(group.elems))
openFiles := make([]*os.File, 0, len(group.elems))
defer func() {
for _, file := range openFiles {
if err := file.Close(); err != nil {
log.FromContext(ctx).Warnf("Failed to close batch cache file %s: %v", file.Name(), err)
}
}
}()
for _, elem := range group.elems {
file, err := os.Open(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
return fmt.Errorf("failed to open cache file: %w", err)
}
stat, err := file.Stat()
if err != nil {
file.Close()
t.markItemFailed(elem.ID, FailureStageCache, err)
return fmt.Errorf("failed to get cache file stat: %w", err)
}
openFiles = append(openFiles, file)
items = append(items, storagetypes.BatchItem{
Reader: file,
StoragePath: elem.Path,
Size: stat.Size(),
SourceGroupKey: elem.sourceGroupKey,
Caption: elem.sourceCaption,
PreserveCaption: elem.preserveCaption,
})
}
for index, item := range items {
t.recordDownloadComplete(group.elems[index].ID, item.Size)
}
return t.saveBatchItems(ctx, group, items)
}
func (t *Task) saveBatchItems(ctx context.Context, group executionGroup, items []storagetypes.BatchItem) error {
t.startUpload(ctx)
if progressSaver, ok := group.batchSaver.(storage.StorageBatchProgressSaver); ok {
err := progressSaver.SaveBatchWithProgress(ctx, items, func(index int, uploaded, total int64) {
if index < 0 || index >= len(group.elems) {
return
}
t.uploadCallback(ctx, group.elems[index].ID)(uploaded, total)
})
if err != nil {
for _, elem := range group.elems {
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
}
t.notifyStateChange(ctx)
return fmt.Errorf("failed to save batch: %w", err)
}
for index, elem := range group.elems {
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
t.markItemCompleted(elem.ID)
}
t.notifyStateChange(ctx)
return nil
}
for i := range items {
items[i].Reader = ioutil.NewProgressReader(
items[i].Reader,
items[i].Size,
t.uploadCallback(ctx, group.elems[i].ID),
)
}
if err := group.batchSaver.SaveBatch(ctx, items); err != nil {
for _, elem := range group.elems {
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
}
t.notifyStateChange(ctx)
return fmt.Errorf("failed to save batch: %w", err)
}
for index, elem := range group.elems {
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
t.markItemCompleted(elem.ID)
}
t.notifyStateChange(ctx)
return nil
}
func (t *Task) markProcessing(ctx context.Context, elem *TaskElement) error {
t.processingMu.Lock()
if t.processing[elem.ID] != nil {
t.processingMu.Unlock()
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
}
t.processing[elem.ID] = elem
t.processingMu.Unlock()
t.markItemActive(elem.ID, elem.stream, time.Now())
t.Progress.OnProgress(ctx, t)
return nil
}
func (t *Task) unmarkProcessing(id string) {
t.processingMu.Lock()
delete(t.processing, id)
t.processingMu.Unlock()
}
func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
logger.Info("Starting file download")
localFile, err := fsutil.CreateFile(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to create local file: %w", err)
}
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
TaskID: t.ID,
Phase: taskevent.PhaseProgress,
TotalBytes: t.totalSize,
DownloadedBytes: downloaded,
})
})
_, downloadErr := tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
closeErr := localFile.Close()
if downloadErr != nil {
t.markItemFailed(elem.ID, FailureStageDownload, downloadErr)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to download file: %w", downloadErr)
}
if closeErr != nil {
t.markItemFailed(elem.ID, FailureStageCache, closeErr)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to close cache file: %w", closeErr)
}
logger.Info("File downloaded successfully")
if path.Ext(elem.FileName()) == "" {
if ext := fsutil.DetectFileExt(elem.localPath); ext != "" {
elem.Path += ext
}
}
t.markItemDownloaded(elem.ID)
t.Progress.OnProgress(ctx, t)
return nil
}
func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
if elem.stream {
@@ -60,9 +285,15 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
defer pr.Close()
errg, uploadCtx := errgroup.WithContext(ctx)
errg.Go(func() error {
return elem.Storage.Save(uploadCtx, pr, elem.Path)
err := elem.Storage.Save(uploadCtx, pr, elem.Path)
if err != nil {
t.markItemFailed(elem.ID, FailureStageUpload, err)
t.notifyStateChange(ctx)
}
return err
})
wr := ioutil.NewProgressWriter(pw, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
@@ -78,6 +309,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
_, err := tdler.NewDownloader(elem.File).Stream(uploadCtx, wr)
if err != nil {
logger.Errorf("Failed to download file: %v", err)
t.markItemFailed(elem.ID, FailureStageDownload, err)
t.notifyStateChange(ctx)
pw.CloseWithError(err)
}
return err
@@ -85,12 +318,17 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
if err := errg.Wait(); err != nil {
return fmt.Errorf("failed to download file in stream mode: %w", err)
}
t.recordDownloadComplete(elem.ID, 0)
t.markItemCompleted(elem.ID)
t.notifyStateChange(ctx)
logger.Info("File downloaded successfully in stream mode")
return nil
}
logger.Info("Starting file download")
localFile, err := fsutil.CreateFile(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to create local file: %w", err)
}
defer func() {
@@ -99,6 +337,7 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
}
}()
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
t.recordItemDownload(elem.ID, int64(n), time.Now())
downloaded := t.downloaded.Add(int64(n))
t.Progress.OnProgress(ctx, t)
taskevent.Emit(ctx, taskevent.Event{
@@ -110,6 +349,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
})
_, err = tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
if err != nil {
t.markItemFailed(elem.ID, FailureStageDownload, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to download file: %w", err)
}
logger.Info("File downloaded successfully")
@@ -122,21 +363,50 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
var fileStat os.FileInfo
fileStat, err = os.Stat(elem.localPath)
if err != nil {
t.markItemFailed(elem.ID, FailureStageCache, err)
t.notifyStateChange(ctx)
return fmt.Errorf("failed to get file stat: %w", err)
}
t.recordDownloadComplete(elem.ID, fileStat.Size())
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
t.startUpload(vctx)
onProgress := t.uploadCallback(vctx, elem.ID)
attempt := 0
retryLimit := int(config.C().Retry)
lastFailureStage := FailureStageUpload
err = retry.Retry(func() error {
attempt++
var file *os.File
file, err = os.Open(elem.localPath)
if err != nil {
lastFailureStage = FailureStageCache
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
t.notifyStateChange(vctx)
return fmt.Errorf("failed to open cache file: %w", err)
}
defer file.Close()
if err = elem.Storage.Save(vctx, file, elem.Path); err != nil {
onProgress(0, fileStat.Size())
if progressSaver, ok := elem.Storage.(storage.StorageProgressSaver); ok {
err = progressSaver.SaveWithProgress(vctx, file, elem.Path, onProgress)
} else {
err = elem.Storage.Save(vctx, ioutil.NewProgressReader(file, fileStat.Size(), onProgress), elem.Path)
}
if err != nil {
logger.Errorf("Failed to save file: %s, retrying...", err)
lastFailureStage = t.itemFailureStage(elem.ID)
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
t.notifyStateChange(vctx)
return err
}
return nil
}, retry.Context(vctx), retry.RetryTimes(uint(config.C().Retry)))
if err == nil {
onProgress(fileStat.Size(), fileStat.Size())
t.markItemCompleted(elem.ID)
t.notifyStateChange(vctx)
} else {
t.markItemFailed(elem.ID, lastFailureStage, err)
t.notifyStateChange(vctx)
}
return err
}

View File

@@ -0,0 +1,57 @@
package batchtfile
import (
"testing"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/pkg/tfile"
tgstorage "github.com/krau/SaveAny-Bot/storage/telegram"
)
func TestExecutionGroupsPreserveSourceAlbums(t *testing.T) {
stor := new(tgstorage.Telegram)
otherStor := new(tgstorage.Telegram)
task := Task{elems: []TaskElement{
{Storage: stor, sourceGroupKey: "album-1"},
{Storage: stor, sourceGroupKey: "album-1"},
{Storage: stor},
{Storage: stor, sourceGroupKey: "album-2"},
{Storage: stor, sourceGroupKey: "album-2"},
{Storage: otherStor, sourceGroupKey: "album-2"},
}}
groups := task.executionGroups()
wantSizes := []int{2, 1, 2, 1}
wantBatch := []bool{true, false, true, true}
if len(groups) != len(wantSizes) {
t.Fatalf("got %d groups, want %d", len(groups), len(wantSizes))
}
for i := range groups {
if got := len(groups[i].elems); got != wantSizes[i] {
t.Errorf("group %d has %d elements, want %d", i, got, wantSizes[i])
}
if got := groups[i].usesBatchSaver(); got != wantBatch[i] {
t.Errorf("group %d batch=%v, want %v", i, got, wantBatch[i])
}
}
}
func TestSourceMetadataPreservesAlbumIdentityAndCaption(t *testing.T) {
msg := &tg.Message{
PeerID: &tg.PeerChannel{ChannelID: 77},
Message: "original caption",
}
msg.SetGroupedID(42)
file := tfile.NewTGFile(nil, nil, 0, "photo.jpg", tfile.WithMessage(msg))
groupKey, caption, preserveCaption := sourceMetadata(file)
if groupKey != "*tg.PeerChannel:77:42" {
t.Fatalf("group key = %q, want %q", groupKey, "*tg.PeerChannel:77:42")
}
if caption != "original caption" {
t.Fatalf("caption = %q, want original caption", caption)
}
if !preserveCaption {
t.Fatal("preserveCaption = false, want true")
}
}

View File

@@ -0,0 +1,360 @@
package batchtfile
import (
"context"
"errors"
"strings"
"time"
)
const (
transferSpeedWindow = 5 * time.Second
transferSamplePeriod = 250 * time.Millisecond
)
// ItemPhase describes the current lifecycle stage of one batch item.
type ItemPhase uint8
const (
ItemPhaseWaiting ItemPhase = iota
ItemPhaseDownloading
ItemPhaseTransferring
ItemPhaseDownloaded
ItemPhaseUploading
ItemPhaseRetrying
ItemPhaseConfirming
ItemPhaseCompleted
ItemPhaseFailed
ItemPhaseStopped
)
// FailureStage identifies the operation that failed for one batch item.
type FailureStage uint8
const (
FailureStageNone FailureStage = iota
FailureStageDownload
FailureStageCache
FailureStageUpload
FailureStageConfirm
FailureStageBatchUpload
FailureStageInternal
)
// TaskItemProgress is an immutable progress snapshot for one batch item.
type TaskItemProgress struct {
Index int
ID string
Name string
Size int64
Downloaded int64
Uploaded int64
DownloadSpeed float64
UploadSpeed float64
Phase ItemPhase
FailureStage FailureStage
RetryAttempt int
RetryLimit int
Error string
}
type transferSample struct {
at time.Time
bytes int64
}
type transferMeter struct {
samples []transferSample
latest transferSample
hasData bool
}
func (m *transferMeter) record(now time.Time, transferred int64) {
if m.hasData && transferred < m.latest.bytes {
m.reset()
}
if m.hasData && now.Before(m.latest.at) {
now = m.latest.at
}
sample := transferSample{at: now, bytes: transferred}
m.latest = sample
m.hasData = true
if len(m.samples) == 0 {
m.samples = append(m.samples, sample)
return
}
if now.Sub(m.samples[len(m.samples)-1].at) >= transferSamplePeriod {
m.samples = append(m.samples, sample)
}
cutoff := now.Add(-transferSpeedWindow)
for len(m.samples) > 1 && m.samples[0].at.Before(cutoff) {
m.samples = m.samples[1:]
}
}
func (m *transferMeter) speed() float64 {
if !m.hasData || len(m.samples) == 0 {
return 0
}
first := m.samples[0]
last := m.latest
elapsed := last.at.Sub(first.at).Seconds()
if elapsed <= 0 || last.bytes <= first.bytes {
return 0
}
return float64(last.bytes-first.bytes) / elapsed
}
func (m *transferMeter) reset() {
m.samples = m.samples[:0]
m.latest = transferSample{}
m.hasData = false
}
type itemProgressState struct {
index int
id string
name string
expectedSize int64
actualSize int64
downloaded int64
uploaded int64
phase ItemPhase
failureStage FailureStage
retryAttempt int
retryLimit int
err string
downloadMeter transferMeter
uploadMeter transferMeter
}
func newItemProgressStates(elems []TaskElement) ([]itemProgressState, map[string]int) {
states := make([]itemProgressState, 0, len(elems))
index := make(map[string]int, len(elems))
for i, elem := range elems {
name := ""
size := int64(0)
if elem.File != nil {
name = elem.File.Name()
size = elem.File.Size()
}
states = append(states, itemProgressState{
index: i + 1,
id: elem.ID,
name: name,
expectedSize: size,
phase: ItemPhaseWaiting,
})
index[elem.ID] = i
}
return states, index
}
func (t *Task) updateItem(id string, update func(*itemProgressState)) bool {
t.itemMu.Lock()
defer t.itemMu.Unlock()
index, ok := t.itemIndex[id]
if !ok || index < 0 || index >= len(t.itemStates) {
return false
}
update(&t.itemStates[index])
return true
}
func (t *Task) markItemActive(id string, stream bool, now time.Time) {
t.updateItem(id, func(item *itemProgressState) {
if stream {
item.phase = ItemPhaseTransferring
} else {
item.phase = ItemPhaseDownloading
}
item.failureStage = FailureStageNone
item.err = ""
item.downloadMeter.record(now, item.downloaded)
if stream {
item.uploadMeter.record(now, item.uploaded)
}
})
}
func (t *Task) recordItemDownload(id string, n int64, now time.Time) {
if n <= 0 {
return
}
t.updateItem(id, func(item *itemProgressState) {
item.downloaded += n
item.downloadMeter.record(now, item.downloaded)
if item.phase == ItemPhaseTransferring {
item.uploaded += n
item.uploadMeter.record(now, item.uploaded)
}
})
}
func (t *Task) markItemDownloaded(id string) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseDownloaded
})
}
func (t *Task) recordItemDownloaded(id string, actualSize int64) {
t.updateItem(id, func(item *itemProgressState) {
if actualSize > 0 {
item.actualSize = actualSize
}
if item.actualSize == 0 {
item.actualSize = item.downloaded
}
if item.phase != ItemPhaseTransferring {
item.phase = ItemPhaseDownloaded
item.uploadMeter.reset()
}
})
}
func (t *Task) recordItemUpload(id string, uploaded, total int64, now time.Time) bool {
becameConfirming := false
t.updateItem(id, func(item *itemProgressState) {
if uploaded < item.uploaded {
if item.phase != ItemPhaseRetrying {
return
}
item.uploadMeter.reset()
}
if total > 0 {
item.actualSize = total
}
item.uploaded = uploaded
item.uploadMeter.record(now, uploaded)
if total > 0 && uploaded >= total {
becameConfirming = item.phase != ItemPhaseConfirming
item.phase = ItemPhaseConfirming
return
}
item.phase = ItemPhaseUploading
item.failureStage = FailureStageNone
item.err = ""
})
return becameConfirming
}
func (t *Task) markItemRetry(id string, stage FailureStage, attempt, limit int, err error) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseRetrying
item.failureStage = stage
item.retryAttempt = attempt
item.retryLimit = limit
item.err = compactError(err)
})
}
func (t *Task) markItemFailed(id string, stage FailureStage, err error) {
t.updateItem(id, func(item *itemProgressState) {
if item.phase == ItemPhaseFailed || item.phase == ItemPhaseCompleted {
return
}
if errors.Is(err, context.Canceled) {
item.phase = ItemPhaseStopped
return
}
item.phase = ItemPhaseFailed
item.failureStage = stage
item.err = compactError(err)
})
}
func (t *Task) markItemCompleted(id string) {
t.updateItem(id, func(item *itemProgressState) {
item.phase = ItemPhaseCompleted
item.failureStage = FailureStageNone
item.err = ""
item.retryAttempt = 0
item.retryLimit = 0
if item.actualSize == 0 {
item.actualSize = max(item.downloaded, item.uploaded)
}
})
}
func (t *Task) finishItems(err error) {
if err == nil {
return
}
t.itemMu.Lock()
defer t.itemMu.Unlock()
for i := range t.itemStates {
item := &t.itemStates[i]
if item.phase == ItemPhaseCompleted || item.phase == ItemPhaseFailed {
continue
}
item.phase = ItemPhaseStopped
}
}
func (t *Task) itemFailureStage(id string) FailureStage {
t.itemMu.RLock()
defer t.itemMu.RUnlock()
index, ok := t.itemIndex[id]
if !ok || index < 0 || index >= len(t.itemStates) {
return FailureStageUpload
}
if t.itemStates[index].phase == ItemPhaseConfirming {
return FailureStageConfirm
}
return FailureStageUpload
}
func (t *Task) Items() []TaskItemProgress {
t.itemMu.RLock()
defer t.itemMu.RUnlock()
items := make([]TaskItemProgress, 0, len(t.itemStates))
for i := range t.itemStates {
item := &t.itemStates[i]
size := item.actualSize
if size == 0 {
size = item.expectedSize
}
items = append(items, TaskItemProgress{
Index: item.index,
ID: item.id,
Name: item.name,
Size: size,
Downloaded: item.downloaded,
Uploaded: item.uploaded,
DownloadSpeed: item.downloadMeter.speed(),
UploadSpeed: item.uploadMeter.speed(),
Phase: item.phase,
FailureStage: item.failureStage,
RetryAttempt: item.retryAttempt,
RetryLimit: item.retryLimit,
Error: item.err,
})
}
return items
}
func (t *Task) ActualTotalSize() int64 {
items := t.Items()
var total int64
for _, item := range items {
total += item.Size
}
return total
}
type stateProgressTracker interface {
OnStateChange(ctx context.Context, info TaskInfo)
}
func (t *Task) notifyStateChange(ctx context.Context) {
if tracker, ok := t.Progress.(stateProgressTracker); ok {
tracker.OnStateChange(ctx, t)
}
}
func compactError(err error) string {
if err == nil {
return ""
}
return strings.Join(strings.Fields(err.Error()), " ")
}

View File

@@ -4,20 +4,19 @@ import (
"context"
"errors"
"fmt"
"strconv"
"path"
"strings"
"sync/atomic"
"sync"
"time"
"unicode/utf8"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/slice"
"github.com/gotd/td/telegram/message/entity"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/config"
)
type ProgressTracker interface {
@@ -27,159 +26,471 @@ type ProgressTracker interface {
}
type Progress struct {
MessageID int
ChatID int64
start time.Time
lastUpdatePercent atomic.Int32
skippedFiles []string
MessageID int
ChatID int64
updateMu sync.Mutex
lastUpdateAt time.Time
lastText string
done bool
skippedFiles []string
}
type renderedBatchMessage struct {
Text string
Entities []tg.MessageEntityClass
Err error
}
const (
progressRenderInterval = time.Second
maxVisibleActiveItems = 5
progressBarWidth = 10
maxDisplayNameRunes = 36
maxDisplayErrorRunes = 240
)
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
p.start = time.Now()
p.lastUpdatePercent.Store(0)
log.FromContext(ctx).Debugf("Batch task progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchStartPrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
return
}
p.render(ctx, info, true)
}
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) {
if !shouldUpdateProgress(info.TotalSize(), info.Downloaded(), int(p.lastUpdatePercent.Load())) {
p.render(ctx, info, false)
}
func (p *Progress) OnStateChange(ctx context.Context, info TaskInfo) {
p.render(ctx, info, true)
}
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, _ int64) {
p.render(ctx, info, true)
}
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, _, _ int64) {
p.render(ctx, info, false)
}
func (p *Progress) render(ctx context.Context, info TaskInfo, priority bool) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
if p.done {
return
}
percent := int((info.Downloaded() * 100) / info.TotalSize())
if p.lastUpdatePercent.Load() == int32(percent) {
now := time.Now()
if !priority && !p.lastUpdateAt.IsZero() && now.Sub(p.lastUpdateAt) < progressRenderInterval {
return
}
p.lastUpdatePercent.Store(int32(percent))
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.TaskID(), info.Downloaded(), info.TotalSize())
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchProcessingPrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
styling.Plain(i18n.T(i18nk.BotMsgProgressProcessingListPrefix, nil)),
func() styling.StyledTextOption {
var lines []string
for _, elem := range info.Processing() {
lines = append(lines, fmt.Sprintf(" - %s (%.2f MB)", elem.FileName(), float64(elem.FileSize())/(1024*1024)))
}
if len(lines) == 0 {
lines = append(lines, i18n.T(i18nk.BotMsgProgressProcessingNone, nil))
}
return styling.Plain(slice.Join(lines, "\n"))
}(),
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(info.Downloaded(), p.start)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f%%", float64(info.Downloaded())/float64(info.TotalSize())*100)),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
message := buildBatchProgressMessage(info, p.skippedFiles, visibleActiveItems())
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
if message.Text == p.lastText {
return
}
p.lastText = message.Text
p.lastUpdateAt = now
p.editMessage(ctx, info.TaskID(), message, true)
}
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
if err != nil {
log.FromContext(ctx).Errorf("Batch task %s failed: %s", info.TaskID(), err)
} else {
log.FromContext(ctx).Debugf("Batch task %s completed successfully", info.TaskID())
}
entityBuilder := entity.Builder{}
var stylingErr error
if err != nil {
if errors.Is(err, context.Canceled) {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
)
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskFailedWithError, map[string]any{
"Error": "",
})),
styling.Code(err.Error()),
)
}
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchDonePrefix, nil)),
styling.Code(strconv.Itoa(info.Count())),
styling.Plain(i18n.T(i18nk.BotMsgProgressTotalSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(info.TotalSize())/(1024*1024))),
func() styling.StyledTextOption {
if len(p.skippedFiles) == 0 {
return styling.Plain("")
}
return styling.Plain("\n\n" + i18n.T(i18nk.BotMsgCommonInfoConflictFilesSkipped, map[string]any{
"Skipped": strings.Join(p.skippedFiles, "\n"),
}))
}(),
)
}
if stylingErr != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
p.updateMu.Lock()
defer p.updateMu.Unlock()
if p.done {
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
p.done = true
message := buildBatchDoneMessage(info, p.skippedFiles, err)
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render final batch progress message: %v", message.Err)
return
}
req.SetMessage(text)
req.SetEntities(entities)
p.lastText = message.Text
p.editMessage(ctx, info.TaskID(), message, false)
}
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedBatchMessage, cancellable bool) {
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
return
}
req := buildBatchEditMessageRequest(p.MessageID, taskID, message, cancellable)
if ext := tgutil.ExtFromContext(ctx); ext != nil {
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
log.FromContext(ctx).Errorf("Failed to edit batch progress message: %v", err)
}
}
}
func buildBatchEditMessageRequest(messageID int, taskID string, message renderedBatchMessage, cancellable bool) *tg.MessagesEditMessageRequest {
req := &tg.MessagesEditMessageRequest{ID: messageID}
req.SetMessage(message.Text)
if len(message.Entities) > 0 {
req.SetEntities(message.Entities)
}
if cancellable {
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
}}})
}
return req
}
func buildBatchProgressText(info TaskInfo, skipped []string, activeLimit int) string {
return buildBatchProgressMessage(info, skipped, activeLimit).Text
}
func buildBatchProgressMessage(info TaskInfo, skipped []string, activeLimit int) renderedBatchMessage {
items := info.Items()
completed, waiting, downloaded, failed := itemCounts(items)
downloadSpeed, uploadSpeed := aggregateSpeeds(items)
if activeLimit < 1 {
activeLimit = 1
}
total := len(items) + len(skipped)
downloadSpeedText := formatSpeed(downloadSpeed)
uploadSpeedText := formatSpeed(uploadSpeed)
header := localizedProgressMarkup(i18nk.BotMsgProgressBatchStatusHeader, map[string]any{
"Total": total,
"Completed": completed,
"Downloaded": downloaded,
"Waiting": waiting,
"DownloadSpeed": downloadSpeedText,
"UploadSpeed": uploadSpeedText,
})
var markup strings.Builder
markup.WriteString(header)
visibleItems, hiddenTransfers, summarizedConfirming := visibleBatchItems(items, activeLimit)
for _, item := range visibleItems {
markup.WriteString("\n\n")
markup.WriteString(formatActiveItemMarkup(item, len(items)))
}
if hiddenTransfers > 0 {
markup.WriteString("\n\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryHiddenActive, map[string]any{"Count": hiddenTransfers}))
}
if summarizedConfirming > 0 {
markup.WriteString("\n\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryConfirming, map[string]any{"Count": summarizedConfirming}))
}
if failed > 0 {
markup.WriteString("\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryFailed, map[string]any{"Count": failed}))
}
if len(skipped) > 0 {
markup.WriteString("\n")
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummarySkipped, map[string]any{"Count": len(skipped)}))
}
return completeBatchMessage(markup.String())
}
func buildBatchDoneMarkup(info TaskInfo, skipped []string, err error) string {
items := info.Items()
totalSize := info.ActualTotalSize()
if totalSize == 0 {
totalSize = info.TotalSize()
}
if err == nil {
if len(skipped) > 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDoneWithSkipped, map[string]any{
"Success": len(items),
"Skipped": len(skipped),
"Size": dlutil.FormatSize(totalSize),
})
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDone, map[string]any{
"Count": len(items),
"Size": dlutil.FormatSize(totalSize),
})
}
completed, _, _, failed := itemCounts(items)
incomplete := max(len(items)-completed-failed, 0)
if errors.Is(err, context.Canceled) {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchCanceled, map[string]any{
"Total": len(items) + len(skipped),
"Completed": completed,
"Incomplete": incomplete,
"Skipped": len(skipped),
})
}
failedItems := make([]TaskItemProgress, 0, failed)
for _, item := range items {
if item.Phase == ItemPhaseFailed {
failedItems = append(failedItems, item)
}
}
if len(failedItems) > 1 && failedItems[0].FailureStage == FailureStageBatchUpload {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedGroup, map[string]any{
"Affected": len(failedItems),
"Reason": displayError(firstError(failedItems), err),
"Completed": completed,
"Failed": len(failedItems),
"Incomplete": incomplete,
})
}
if len(failedItems) == 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedTask, map[string]any{
"Reason": displayError("", err),
"Completed": completed,
"Incomplete": incomplete,
})
}
item := failedItems[0]
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedItem, map[string]any{
"Index": item.Index,
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
"Stage": failureStageLabel(item.FailureStage),
"Progress": failureProgress(item),
"Speed": failureSpeed(item),
"Reason": displayError(item.Error, err),
"Completed": completed,
"Failed": failed,
"Incomplete": incomplete,
})
}
func buildBatchDoneMessage(info TaskInfo, skipped []string, err error) renderedBatchMessage {
return completeBatchMessage(buildBatchDoneMarkup(info, skipped, err))
}
func formatActiveItemMarkup(item TaskItemProgress, total int) string {
data := map[string]any{
"Index": item.Index,
"Total": total,
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
"Speed": formatSpeed(itemSpeed(item)),
"Progress": itemPercent(item),
"Bar": textProgressBar(itemPercent(item)),
"Current": dlutil.FormatSize(itemBytes(item)),
"Size": dlutil.FormatSize(item.Size),
"Attempt": min(max(item.RetryAttempt, 1), max(item.RetryLimit, 1)),
"Limit": max(item.RetryLimit, 1),
"Reason": truncateRunes(item.Error, maxDisplayErrorRunes),
}
switch item.Phase {
case ItemPhaseDownloading:
if item.Size <= 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloadingUnknown, data)
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloading, data)
case ItemPhaseTransferring:
if item.Size <= 0 {
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferringUnknown, data)
}
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferring, data)
case ItemPhaseUploading:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemUploading, data)
case ItemPhaseRetrying:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemRetrying, data)
case ItemPhaseConfirming:
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemConfirming, data)
default:
return ""
}
}
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
}
func visibleBatchItems(items []TaskItemProgress, limit int) (visible []TaskItemProgress, hiddenTransfers, summarizedConfirming int) {
visible = make([]TaskItemProgress, 0, limit)
transferCount := 0
confirmingCount := 0
for _, item := range items {
switch {
case isTransferPhase(item.Phase):
transferCount++
if len(visible) < limit {
visible = append(visible, item)
}
case item.Phase == ItemPhaseConfirming:
confirmingCount++
}
}
hiddenTransfers = transferCount - len(visible)
if confirmingCount == 1 && len(visible) < limit {
for _, item := range items {
if item.Phase == ItemPhaseConfirming {
visible = append(visible, item)
return visible, hiddenTransfers, 0
}
}
}
return visible, hiddenTransfers, confirmingCount
}
func completeBatchMessage(markup string) renderedBatchMessage {
text, entities, err := tgutil.RenderHTML(markup)
return renderedBatchMessage{Text: text, Entities: entities, Err: err}
}
func itemCounts(items []TaskItemProgress) (completed, waiting, downloaded, failed int) {
for _, item := range items {
switch item.Phase {
case ItemPhaseCompleted:
completed++
case ItemPhaseWaiting:
waiting++
case ItemPhaseDownloaded:
downloaded++
case ItemPhaseFailed:
failed++
}
}
return
}
func aggregateSpeeds(items []TaskItemProgress) (download, upload float64) {
for _, item := range items {
switch item.Phase {
case ItemPhaseDownloading:
download += item.DownloadSpeed
case ItemPhaseTransferring:
download += item.DownloadSpeed
upload += item.UploadSpeed
case ItemPhaseUploading:
upload += item.UploadSpeed
}
}
return
}
func isTransferPhase(phase ItemPhase) bool {
switch phase {
case ItemPhaseDownloading, ItemPhaseTransferring, ItemPhaseUploading, ItemPhaseRetrying:
return true
default:
return false
}
}
func itemBytes(item TaskItemProgress) int64 {
switch item.Phase {
case ItemPhaseDownloading, ItemPhaseTransferring:
return item.Downloaded
default:
return item.Uploaded
}
}
func itemSpeed(item TaskItemProgress) float64 {
switch item.Phase {
case ItemPhaseDownloading, ItemPhaseTransferring:
return item.DownloadSpeed
case ItemPhaseUploading:
return item.UploadSpeed
case ItemPhaseRetrying:
return item.UploadSpeed
default:
return 0
}
}
func itemPercent(item TaskItemProgress) int {
if item.Size <= 0 {
return 0
}
return int(min(itemBytes(item), item.Size) * 100 / item.Size)
}
func textProgressBar(percent int) string {
percent = min(max(percent, 0), 100)
filled := percent * progressBarWidth / 100
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", progressBarWidth-filled)
}
func formatSpeed(speed float64) string {
if speed <= 0 {
return "0 B/s"
}
return dlutil.FormatSize(int64(speed)) + "/s"
}
func truncateFilename(name string, limit int) string {
if utf8.RuneCountInString(name) <= limit {
return name
}
ext := path.Ext(name)
if utf8.RuneCountInString(ext) >= limit-2 {
return truncateRunes(name, limit-1) + "…"
}
base := strings.TrimSuffix(name, ext)
baseLimit := limit - utf8.RuneCountInString(ext) - 1
return truncateRunes(base, baseLimit) + "…" + ext
}
func truncateRunes(value string, limit int) string {
if limit <= 0 {
return ""
}
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}
func displayError(itemError string, fallback error) string {
if itemError == "" && fallback != nil {
itemError = compactError(fallback)
}
return truncateRunes(itemError, maxDisplayErrorRunes)
}
func firstError(items []TaskItemProgress) string {
for _, item := range items {
if item.Error != "" {
return item.Error
}
}
return ""
}
func failureStageLabel(stage FailureStage) string {
switch stage {
case FailureStageDownload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageDownload, nil)
case FailureStageCache:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageCache, nil)
case FailureStageUpload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageUpload, nil)
case FailureStageConfirm:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageConfirm, nil)
case FailureStageBatchUpload:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageBatchUpload, nil)
default:
return i18n.T(i18nk.BotMsgProgressBatchFailureStageInternal, nil)
}
}
func failureProgress(item TaskItemProgress) string {
if item.Size <= 0 {
return dlutil.FormatSize(failureBytes(item))
}
return fmt.Sprintf("%d%%", min(failureBytes(item), item.Size)*100/item.Size)
}
func failureSpeed(item TaskItemProgress) string {
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
return formatSpeed(item.DownloadSpeed)
}
return formatSpeed(item.UploadSpeed)
}
func failureBytes(item TaskItemProgress) int64 {
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
return item.Downloaded
}
return item.Uploaded
}
func visibleActiveItems() int {
return min(max(config.C().Workers, 1), maxVisibleActiveItems)
}
func NewProgressTracker(messageID int, chatID int64) ProgressTracker {

View File

@@ -0,0 +1,311 @@
package batchtfile
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/pkg/tfile"
)
type progressRegressionRecorder struct {
mu sync.Mutex
startTotal int64
notifications []int64
}
func (*progressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
func (*progressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
func (*progressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
func (r *progressRegressionRecorder) OnUploadStart(_ context.Context, _ TaskInfo, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
r.startTotal = total
}
func (r *progressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
r.mu.Lock()
defer r.mu.Unlock()
r.notifications = append(r.notifications, uploaded)
}
type orderedProgressRegressionRecorder struct {
firstEntered chan struct{}
releaseFirst chan struct{}
secondEntered chan struct{}
mu sync.Mutex
notifications []int64
}
func (*orderedProgressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
func (*orderedProgressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
func (*orderedProgressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
func (*orderedProgressRegressionRecorder) OnUploadStart(context.Context, TaskInfo, int64) {
}
func (r *orderedProgressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
if uploaded == 100 {
close(r.firstEntered)
<-r.releaseFirst
}
if uploaded == 200 {
close(r.secondEntered)
}
r.mu.Lock()
r.notifications = append(r.notifications, uploaded)
r.mu.Unlock()
}
func TestBatchProgressShowsTransferSpeedAndSize(t *testing.T) {
useProgressRegressionLocale(t)
task := newProgressRegressionTask(nil,
progressRegressionFile{"downloading", 1000},
progressRegressionFile{"uploading", 1000},
progressRegressionFile{"waiting", 1000},
)
started := time.Unix(100, 0)
task.markItemActive("downloading", false, started)
task.recordItemDownload("downloading", 500, started.Add(time.Second))
task.recordItemDownloaded("uploading", 1000)
task.recordItemUpload("uploading", 0, 1000, started.Add(time.Second))
task.recordItemUpload("uploading", 250, 1000, started.Add(2*time.Second))
message := buildBatchProgressMessage(task, nil, 2)
if message.Err != nil {
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
}
assertProgressRegressionContains(t, message.Text,
"状态:✅ 0 📥 0 ⏳ 1",
"⬇️ 1/3 下载中",
"速度500 B/s",
"大小500 B / 1000 B",
"⬆️ 2/3 上传中",
"速度250 B/s",
"大小250 B / 1000 B",
)
bold, _, blockquote, _ := batchEntityCounts(message.Entities)
if bold != 3 || blockquote != 2 {
t.Fatalf("entity counts = bold:%d blockquote:%d, want bold:3 blockquote:2", bold, blockquote)
}
}
func TestBatchProgressLimitsRowsWithoutHidingActiveUpload(t *testing.T) {
useProgressRegressionLocale(t)
task := newProgressRegressionTask(nil,
progressRegressionFile{"confirm-01", 100},
progressRegressionFile{"confirm-02", 100},
progressRegressionFile{"uploading", 100},
progressRegressionFile{"downloading", 100},
)
started := time.Unix(100, 0)
task.recordItemUpload("confirm-01", 100, 100, started)
task.recordItemUpload("confirm-02", 100, 100, started)
task.recordItemUpload("uploading", 40, 100, started.Add(time.Second))
task.markItemActive("downloading", false, started)
message := buildBatchProgressText(task, nil, 2)
assertProgressRegressionContains(t, message,
"uploading.bin",
"downloading.bin",
"☁️ 已上传等待整组发送2",
)
if strings.Contains(message, "confirm-01.bin") || strings.Contains(message, "confirm-02.bin") {
t.Fatalf("confirmation rows displaced active transfers:\n%s", message)
}
}
func TestBatchProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
useProgressRegressionLocale(t)
fileID := `<b>A&B</b>`
task := newProgressRegressionTask(nil, progressRegressionFile{fileID, 100})
task.markItemRetry(fileID, FailureStageUpload, 1, 3, errors.New(`<i>remote & failed</i>`))
message := buildBatchProgressMessage(task, nil, 1)
if message.Err != nil {
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
}
assertProgressRegressionContains(t, message.Text,
`<b>A&B</b>.bin`,
`<i>remote & failed</i>`,
)
bold, _, blockquote, italic := batchEntityCounts(message.Entities)
if bold != 2 || blockquote != 1 || italic != 0 {
t.Fatalf("entity counts = bold:%d blockquote:%d italic:%d", bold, blockquote, italic)
}
i18n.Init("en")
english := buildBatchProgressMessage(task, nil, 1)
if english.Err != nil {
t.Fatalf("English batch template failed: %v", english.Err)
}
assertProgressRegressionContains(t, english.Text, "📦 Processing", "Retrying upload", `<b>A&B</b>.bin`)
}
func TestDownloadProgressContinuesAfterUploadStarts(t *testing.T) {
useProgressRegressionLocale(t)
progress := new(Progress)
task := newProgressRegressionTask(progress,
progressRegressionFile{"uploading", 100},
progressRegressionFile{"downloading", 100},
)
progress.OnStart(t.Context(), task)
task.recordDownloadComplete("uploading", 100)
task.uploadCallback(t.Context(), "uploading")(50, 100)
started := time.Unix(100, 0)
task.markItemActive("downloading", false, started)
task.recordItemDownload("downloading", 50, started.Add(time.Second))
progress.updateMu.Lock()
progress.lastUpdateAt = time.Now().Add(-progressRenderInterval)
progress.updateMu.Unlock()
progress.OnProgress(t.Context(), task)
progress.updateMu.Lock()
text := progress.lastText
progress.updateMu.Unlock()
assertProgressRegressionContains(t, text,
"uploading.bin",
"🟩🟩🟩🟩🟩⬜️⬜️⬜️⬜️⬜️ 50%",
"总速度:⬇️ 50 B/s ⬆️ 0 B/s",
"🔄 另有 1 个文件正在处理",
)
}
func TestBatchUploadIgnoresOutOfOrderBytesAndAllowsRetryReset(t *testing.T) {
recorder := new(progressRegressionRecorder)
task := newProgressRegressionTask(recorder, progressRegressionFile{"file", 100})
task.recordDownloadComplete("file", 100)
callback := task.uploadCallback(t.Context(), "file")
callback(80, 100)
callback(10, 100)
if got := task.Items()[0].Uploaded; got != 80 {
t.Fatalf("out-of-order callback regressed item to %d, want 80", got)
}
task.markItemRetry("file", FailureStageUpload, 1, 3, context.DeadlineExceeded)
callback(0, 100)
callback(10, 100)
if got := task.Items()[0].Uploaded; got != 10 {
t.Fatalf("retry did not reset item progress: got %d, want 10", got)
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for index := 1; index < len(recorder.notifications); index++ {
if recorder.notifications[index] < recorder.notifications[index-1] {
t.Fatalf("aggregate progress regressed: %v", recorder.notifications)
}
}
}
func TestUploadProgressNotificationsRemainOrdered(t *testing.T) {
recorder := &orderedProgressRegressionRecorder{
firstEntered: make(chan struct{}),
releaseFirst: make(chan struct{}),
secondEntered: make(chan struct{}),
}
task := newProgressRegressionTask(recorder,
progressRegressionFile{"first", 100},
progressRegressionFile{"second", 100},
)
task.recordDownloadComplete("first", 100)
task.recordDownloadComplete("second", 100)
first := task.uploadCallback(t.Context(), "first")
second := task.uploadCallback(t.Context(), "second")
var wait sync.WaitGroup
wait.Go(func() {
first(100, 100)
})
<-recorder.firstEntered
wait.Go(func() {
second(100, 100)
})
overtook := false
select {
case <-recorder.secondEntered:
overtook = true
case <-time.After(100 * time.Millisecond):
}
close(recorder.releaseFirst)
wait.Wait()
if overtook {
t.Fatal("later aggregate notification overtook the first callback")
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
if got := recorder.notifications; len(got) != 2 || got[0] != 100 || got[1] != 200 {
t.Fatalf("upload notifications = %v, want [100 200]", got)
}
}
func TestBatchUploadUsesActualSizeWhenMetadataIsUnknown(t *testing.T) {
recorder := new(progressRegressionRecorder)
task := newProgressRegressionTask(recorder, progressRegressionFile{"photo", 0})
task.recordDownloadComplete("photo", 25)
task.uploadCallback(t.Context(), "photo")(25, 25)
recorder.mu.Lock()
defer recorder.mu.Unlock()
if recorder.startTotal != 25 {
t.Fatalf("upload start total = %d, want actual size 25", recorder.startTotal)
}
if got := task.ActualTotalSize(); got != 25 {
t.Fatalf("actual total size = %d, want 25", got)
}
}
type progressRegressionFile struct {
id string
size int64
}
func newProgressRegressionTask(progress ProgressTracker, files ...progressRegressionFile) *Task {
elems := make([]TaskElement, 0, len(files))
for _, file := range files {
elems = append(elems, TaskElement{
ID: file.id,
File: tfile.NewTGFile(nil, nil, file.size, file.id+".bin"),
})
}
return NewBatchTGFileTask("progress-regression", context.Background(), elems, progress, true)
}
func useProgressRegressionLocale(t *testing.T) {
t.Helper()
i18n.Init("zh-Hans")
t.Cleanup(func() { i18n.Init("zh-Hans") })
}
func assertProgressRegressionContains(t *testing.T, value string, wants ...string) {
t.Helper()
for _, want := range wants {
if !strings.Contains(value, want) {
t.Fatalf("text does not contain %q:\n%s", want, value)
}
}
}
func batchEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
case *tg.MessageEntityItalic:
italic++
}
}
return
}

View File

@@ -7,6 +7,7 @@ import (
"sync"
"sync/atomic"
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/core"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
@@ -18,25 +19,35 @@ import (
var _ core.Executable = (*Task)(nil)
type TaskElement struct {
ID string
Storage storage.Storage
Path string
File tfile.TGFile
localPath string
stream bool
ID string
Storage storage.Storage
Path string
File tfile.TGFile
localPath string
stream bool
sourceGroupKey string
sourceCaption string
preserveCaption bool
}
type Task struct {
ID string
ctx context.Context
elems []TaskElement
Progress ProgressTracker
IgnoreErrors bool // if true, errors during processing will be ignored
downloaded atomic.Int64
totalSize int64
processing map[string]TaskElementInfo
processingMu sync.RWMutex
failed map[string]error // [TODO] errors for each element
ID string
ctx context.Context
elems []TaskElement
Progress ProgressTracker
IgnoreErrors bool // if true, errors during processing will be ignored
downloaded atomic.Int64
totalSize int64
uploadTotalSize atomic.Int64
processing map[string]TaskElementInfo
processingMu sync.RWMutex
itemStates []itemProgressState
itemIndex map[string]int
itemMu sync.RWMutex
uploadOnce sync.Once
uploadMu sync.Mutex
uploaded map[string]int64
failed map[string]error // [TODO] errors for each element
}
// Title implements core.Exectable.
@@ -54,6 +65,7 @@ func NewTaskElement(
file tfile.TGFile,
) (*TaskElement, error) {
id := xid.New().String()
groupKey, caption, preserveCaption := sourceMetadata(file)
_, ok := stor.(storage.StorageCannotStream)
if !config.C().Stream || ok {
cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name())))
@@ -61,22 +73,42 @@ func NewTaskElement(
return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
}
return &TaskElement{
ID: id,
Storage: stor,
Path: path,
File: file,
localPath: cachePath,
ID: id,
Storage: stor,
Path: path,
File: file,
localPath: cachePath,
sourceGroupKey: groupKey,
sourceCaption: caption,
preserveCaption: preserveCaption,
}, nil
}
return &TaskElement{
ID: id,
Storage: stor,
Path: path,
File: file,
stream: true,
ID: id,
Storage: stor,
Path: path,
File: file,
stream: true,
sourceGroupKey: groupKey,
sourceCaption: caption,
preserveCaption: preserveCaption,
}, nil
}
func sourceMetadata(file tfile.TGFile) (groupKey, caption string, preserveCaption bool) {
messageFile, ok := file.(tfile.TGFileMessage)
if !ok || messageFile.Message() == nil {
return "", "", false
}
msg := messageFile.Message()
groupID, grouped := msg.GetGroupedID()
if !grouped || groupID == 0 {
return "", "", false
}
chatID := tgutil.ChatIdFromPeer(msg.GetPeerID())
return fmt.Sprintf("%T:%d:%d", msg.GetPeerID(), chatID, groupID), msg.GetMessage(), true
}
func NewBatchTGFileTask(
id string,
ctx context.Context,
@@ -84,6 +116,7 @@ func NewBatchTGFileTask(
progress ProgressTracker,
ignoreErrors bool,
) *Task {
itemStates, itemIndex := newItemProgressStates(files)
task := &Task{
ID: id,
ctx: ctx,
@@ -98,6 +131,9 @@ func NewBatchTGFileTask(
return total
}(),
processing: make(map[string]TaskElementInfo),
itemStates: itemStates,
itemIndex: itemIndex,
uploaded: make(map[string]int64),
IgnoreErrors: ignoreErrors,
processingMu: sync.RWMutex{},
failed: make(map[string]error),

View File

@@ -27,8 +27,10 @@ type TaskInfo interface {
TaskID() string
TotalSize() int64
Downloaded() int64
ActualTotalSize() int64
Count() int
Processing() []TaskElementInfo
Items() []TaskItemProgress
}
func (t *Task) TaskID() string {

View File

@@ -0,0 +1,70 @@
package batchtfile
import (
"context"
"time"
)
// UploadProgressTracker optionally extends a batch progress tracker with a
// distinct aggregate upload phase.
type UploadProgressTracker interface {
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
}
func (t *Task) startUpload(ctx context.Context) {
tracker, ok := t.Progress.(UploadProgressTracker)
if !ok {
return
}
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
t.uploadOnce.Do(func() {
tracker.OnUploadStart(ctx, t, t.uploadTotalSize.Load())
})
}
func (t *Task) uploadCallback(ctx context.Context, id string) func(uploaded, total int64) {
return func(uploaded, total int64) {
tracker, ok := t.Progress.(UploadProgressTracker)
if !ok || uploaded < 0 {
return
}
t.startUpload(ctx)
if total > 0 && uploaded > total {
uploaded = total
}
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
becameConfirming := t.recordItemUpload(id, uploaded, total, time.Now())
if t.uploaded == nil {
t.uploaded = make(map[string]int64)
}
previous, tracked := t.uploaded[id]
if !tracked || uploaded > previous {
t.uploaded[id] = uploaded
}
var aggregate int64
for _, current := range t.uploaded {
aggregate += current
}
uploadTotal := t.uploadTotalSize.Load()
if aggregate > uploadTotal {
aggregate = uploadTotal
}
tracker.OnUploadProgress(ctx, t, aggregate, uploadTotal)
if becameConfirming {
t.notifyStateChange(ctx)
}
}
}
func (t *Task) recordDownloadComplete(id string, uploadSize int64) {
t.uploadMu.Lock()
defer t.uploadMu.Unlock()
if uploadSize > 0 {
t.uploadTotalSize.Add(uploadSize)
}
t.recordItemDownloaded(id, uploadSize)
}

View File

@@ -0,0 +1,42 @@
package tfile
import (
"testing"
"github.com/gotd/td/tg"
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
)
func TestSourceCaption(t *testing.T) {
tests := []struct {
name string
file tfilepkg.TGFile
want string
ok bool
}{
{
name: "original caption",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{Message: "original caption"})),
want: "original caption",
ok: true,
},
{
name: "empty caption suppresses storage fallback",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{})),
ok: true,
},
{
name: "file without source message",
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := sourceCaption(tt.file)
if ok != tt.ok || got != tt.want {
t.Fatalf("sourceCaption() = (%q, %v), want (%q, %v)", got, ok, tt.want, tt.ok)
}
})
}
}

View File

@@ -3,6 +3,7 @@ package tfile
import (
"context"
"fmt"
"io"
"os"
"path"
@@ -10,8 +11,12 @@ import (
"github.com/duke-git/lancet/v2/retry"
"github.com/krau/SaveAny-Bot/common/tdler"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
"github.com/krau/SaveAny-Bot/storage"
)
func (t *Task) Execute(ctx context.Context) error {
@@ -57,13 +62,34 @@ func (t *Task) Execute(ctx context.Context) error {
return fmt.Errorf("failed to get file stat: %w", err)
}
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
if caption, ok := sourceCaption(t.File); ok {
vctx = storagetypes.WithSourceCaption(vctx, caption)
}
err = retry.Retry(func() error {
file, err := os.Open(t.localPath)
if err != nil {
return fmt.Errorf("failed to open cache file: %w", err)
}
defer file.Close()
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
uploadProgress, tracksUpload := t.Progress.(UploadProgressTracker)
if !tracksUpload {
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
return fmt.Errorf("failed to save file: %w", err)
}
return nil
}
uploadProgress.OnUploadStart(vctx, t, fileStat.Size())
onProgress := func(uploaded, total int64) {
uploadProgress.OnUploadProgress(vctx, t, uploaded, total)
}
if progressSaver, ok := t.Storage.(storage.StorageProgressSaver); ok {
err = progressSaver.SaveWithProgress(vctx, file, t.Path, onProgress)
} else {
var reader io.Reader = ioutil.NewProgressReader(file, fileStat.Size(), onProgress)
err = t.Storage.Save(vctx, reader, t.Path)
}
if err != nil {
return fmt.Errorf("failed to save file: %w", err)
}
return nil
@@ -73,3 +99,11 @@ func (t *Task) Execute(ctx context.Context) error {
}
return nil
}
func sourceCaption(file tfilepkg.TGFile) (string, bool) {
messageFile, ok := file.(tfilepkg.TGFileMessage)
if !ok || messageFile.Message() == nil {
return "", false
}
return messageFile.Message().GetMessage(), true
}

View File

@@ -4,12 +4,12 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/charmbracelet/log"
"github.com/gotd/td/telegram/message/entity"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
@@ -23,153 +23,319 @@ type ProgressTracker interface {
OnDone(ctx context.Context, info TaskInfo, err error)
}
// UploadProgressTracker optionally extends a task progress tracker with a
// distinct upload phase. Keeping it separate preserves compatibility with
// custom download-only trackers.
type UploadProgressTracker interface {
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
}
type Progress struct {
MessageID int
ChatID int64
start time.Time
lastUpdatePercent atomic.Int32
lastUpdateAt atomic.Int64
updateMu sync.Mutex
uploadAttempt int
uploadedBytes int64
actualSize int64
hasActualSize bool
}
const (
uploadProgressMinInterval = time.Second
uploadProgressMaxInterval = 3 * time.Second
singleProgressBarWidth = 10
maxSingleErrorRunes = 240
)
type singleProgressPhase int
const (
singlePhaseDownloading singleProgressPhase = iota
singlePhaseUploading
singlePhaseRetrying
)
type renderedSingleMessage struct {
Text string
Entities []tg.MessageEntityClass
Err error
}
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
p.start = time.Now()
p.lastUpdatePercent.Store(0)
p.lastUpdateAt.Store(0)
p.uploadAttempt = 0
p.uploadedBytes = 0
p.actualSize = 0
p.hasActualSize = false
log.FromContext(ctx).Debugf("Progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressFileStartPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(info.FileSize())/(1024*1024))),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
return
}
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, singlePhaseDownloading, 0, info.FileSize(), 0, 0), true)
}
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo, downloaded, total int64) {
if !shouldUpdateProgress(total, downloaded, int(p.lastUpdatePercent.Load())) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
now := time.Now()
elapsed := uploadProgressMaxInterval
if lastUpdateAt := p.lastUpdateAt.Load(); lastUpdateAt > 0 {
elapsed = now.Sub(time.Unix(0, lastUpdateAt))
}
if !shouldUpdateSingleDownloadProgress(total, downloaded, int(p.lastUpdatePercent.Load()), elapsed) {
return
}
percent := int32((downloaded * 100) / total)
if p.lastUpdatePercent.Load() == percent {
return
if total > 0 {
percent := int32((downloaded * 100) / total)
if p.lastUpdatePercent.Load() == percent {
return
}
p.lastUpdatePercent.Store(percent)
}
p.lastUpdatePercent.Store(percent)
p.lastUpdateAt.Store(now.UnixNano())
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.FileName(), downloaded, total)
entityBuilder := entity.Builder{}
var entities []tg.MessageEntityClass
if err := styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressFileProcessingPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
styling.Code(fmt.Sprintf("%.2f MB", float64(total)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(downloaded, p.start)/(1024*1024))),
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
styling.Bold(fmt.Sprintf("%.2f%%", float64(downloaded)/float64(total)*100)),
); err != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
info,
singlePhaseDownloading,
downloaded,
total,
dlutil.GetSpeed(downloaded, p.start),
0,
), true)
}
func shouldUpdateSingleDownloadProgress(total, downloaded int64, lastPercent int, elapsed time.Duration) bool {
if total > 0 {
return shouldUpdateProgress(total, downloaded, lastPercent)
}
return downloaded > 0 && elapsed >= uploadProgressMaxInterval
}
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, total int64) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
p.start = time.Now()
p.lastUpdatePercent.Store(0)
p.lastUpdateAt.Store(p.start.UnixNano())
p.uploadAttempt++
p.uploadedBytes = 0
p.actualSize = max(total, 0)
p.hasActualSize = true
log.FromContext(ctx).Debugf("Upload progress tracking started: %s", info.FileName())
phase := singleUploadPhase(p.uploadAttempt)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, phase, 0, total, 0, p.uploadAttempt), true)
}
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64) {
if total <= 0 || uploaded <= 0 {
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
p.updateMu.Lock()
defer p.updateMu.Unlock()
if uploaded > total {
uploaded = total
}
req.SetMessage(text)
req.SetEntities(entities)
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
tgutil.BuildCancelButton(info.TaskID()),
},
},
}},
)
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
if uploaded < p.uploadedBytes {
return
}
p.uploadedBytes = uploaded
now := time.Now()
lastUpdateAt := time.Unix(0, p.lastUpdateAt.Load())
lastPercent := int(p.lastUpdatePercent.Load())
if !shouldUpdateUploadProgress(total, uploaded, lastPercent, now.Sub(lastUpdateAt)) {
return
}
percent := int32((uploaded * 100) / total)
p.lastUpdatePercent.Store(percent)
p.lastUpdateAt.Store(now.UnixNano())
log.FromContext(ctx).Debugf("Upload progress update: %s, %d/%d", info.FileName(), uploaded, total)
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
info,
singleUploadPhase(p.uploadAttempt),
uploaded,
total,
dlutil.GetSpeed(uploaded, p.start),
p.uploadAttempt,
), true)
}
func shouldUpdateUploadProgress(total, uploaded int64, lastPercent int, elapsed time.Duration) bool {
if total <= 0 || uploaded <= 0 {
return false
}
if uploaded >= total {
return lastPercent < 100 && elapsed >= uploadProgressMinInterval
}
percent := int((uploaded * 100) / total)
if percent < lastPercent {
return false
}
if elapsed < uploadProgressMinInterval {
return false
}
if percent == lastPercent {
return elapsed >= uploadProgressMaxInterval
}
return shouldUpdateProgress(total, uploaded, lastPercent) || elapsed >= uploadProgressMaxInterval
}
func singleUploadPhase(attempt int) singleProgressPhase {
if attempt > 1 {
return singlePhaseRetrying
}
return singlePhaseUploading
}
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
p.updateMu.Lock()
defer p.updateMu.Unlock()
if err != nil {
log.FromContext(ctx).Errorf("Progress error for file [%s]: %v", info.FileName(), err)
} else {
log.FromContext(ctx).Debugf("Progress done for file [%s]", info.FileName())
}
entityBuilder := entity.Builder{}
var stylingErr error
p.editMessage(ctx, info.TaskID(), buildSingleDoneMessage(info, p.doneSize(info), err), false)
}
if err != nil {
if errors.Is(err, context.Canceled) {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
styling.Plain("\n"),
styling.Plain(i18n.T(i18nk.BotMsgProgressFileNamePrefix, nil)),
styling.Code(info.FileName()),
)
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadFailedPrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressErrorPrefix, nil)),
styling.Bold(err.Error()),
)
}
} else {
stylingErr = styling.Perform(&entityBuilder,
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadDonePrefix, nil)),
styling.Code(info.FileName()),
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
)
func (p *Progress) doneSize(info TaskInfo) int64 {
if p.hasActualSize {
return p.actualSize
}
return max(info.FileSize(), 0)
}
if stylingErr != nil {
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedSingleMessage, cancellable bool) {
if message.Err != nil {
log.FromContext(ctx).Errorf("Failed to render file progress message: %v", message.Err)
return
}
text, entities := entityBuilder.Complete()
req := &tg.MessagesEditMessageRequest{
ID: p.MessageID,
req := buildSingleEditMessageRequest(p.MessageID, taskID, message, cancellable)
if ext := tgutil.ExtFromContext(ctx); ext != nil {
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
log.FromContext(ctx).Errorf("Failed to edit file progress message: %v", err)
}
}
req.SetMessage(text)
req.SetEntities(entities)
}
ext := tgutil.ExtFromContext(ctx)
if ext != nil {
ext.EditMessage(p.ChatID, req)
func buildSingleEditMessageRequest(messageID int, taskID string, message renderedSingleMessage, cancellable bool) *tg.MessagesEditMessageRequest {
req := &tg.MessagesEditMessageRequest{ID: messageID}
req.SetMessage(message.Text)
if len(message.Entities) > 0 {
req.SetEntities(message.Entities)
}
if cancellable {
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
}}})
}
return req
}
func buildSingleProgressMessage(
info TaskInfo,
phase singleProgressPhase,
current int64,
total int64,
speed float64,
attempt int,
) renderedSingleMessage {
if current < 0 {
current = 0
}
if total > 0 && current > total {
current = total
}
percent := singleProgressPercent(current, total)
destination := fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())
data := map[string]any{
"Name": info.FileName(),
"Bar": singleProgressBar(percent),
"Progress": percent,
"Speed": singleProgressSpeed(speed),
"Current": dlutil.FormatSize(current),
"Size": dlutil.FormatSize(total),
"Destination": destination,
"Attempt": max(attempt, 1),
}
var key i18nk.Key
switch phase {
case singlePhaseUploading:
key = i18nk.BotMsgProgressSingleUploading
case singlePhaseRetrying:
key = i18nk.BotMsgProgressSingleUploadRetrying
default:
key = i18nk.BotMsgProgressSingleDownloading
if total <= 0 {
key = i18nk.BotMsgProgressSingleDownloadingUnknown
}
}
markup := i18n.T(i18nk.BotMsgProgressSingleStatusHeader, nil) + "\n\n" + localizedProgressMarkup(key, data)
return completeSingleMessage(markup)
}
func buildSingleDoneMessage(info TaskInfo, size int64, err error) renderedSingleMessage {
data := map[string]any{
"Name": info.FileName(),
"Size": dlutil.FormatSize(max(size, 0)),
"Destination": fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath()),
}
var key i18nk.Key
switch {
case err == nil:
key = i18nk.BotMsgProgressSingleDone
case errors.Is(err, context.Canceled):
key = i18nk.BotMsgProgressSingleCanceled
default:
data["Reason"] = truncateSingleError(err.Error())
key = i18nk.BotMsgProgressSingleFailed
}
return completeSingleMessage(localizedProgressMarkup(key, data))
}
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
}
func completeSingleMessage(markup string) renderedSingleMessage {
text, entities, err := tgutil.RenderHTML(markup)
return renderedSingleMessage{Text: text, Entities: entities, Err: err}
}
func singleProgressPercent(current, total int64) int {
if total <= 0 {
return 0
}
return int(min(max(current, 0), total) * 100 / total)
}
func singleProgressBar(percent int) string {
percent = min(max(percent, 0), 100)
filled := percent * singleProgressBarWidth / 100
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", singleProgressBarWidth-filled)
}
func singleProgressSpeed(speed float64) string {
if speed <= 0 {
return "0 B/s"
}
return dlutil.FormatSize(int64(speed)) + "/s"
}
func truncateSingleError(value string) string {
runes := []rune(value)
if len(runes) <= maxSingleErrorRunes {
return value
}
return string(runes[:maxSingleErrorRunes])
}
type ProgressOption func(*Progress)

View File

@@ -0,0 +1,178 @@
package tfile
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/common/i18n"
)
type progressTestTaskInfo struct{}
func (progressTestTaskInfo) TaskID() string { return "task" }
func (progressTestTaskInfo) FileName() string { return "file.bin" }
func (progressTestTaskInfo) FileSize() int64 { return 100 << 20 }
func (progressTestTaskInfo) StoragePath() string { return "file.bin" }
func (progressTestTaskInfo) StorageName() string { return "test" }
func TestShouldUpdateUploadProgress(t *testing.T) {
tests := []struct {
name string
total int64
uploaded int64
lastPercent int
elapsed time.Duration
want bool
}{
{name: "invalid total", total: 0, uploaded: 1, want: false},
{name: "no uploaded bytes", total: 100, uploaded: 0, want: false},
{name: "percentage threshold", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval, want: true},
{name: "percentage threshold rate limited", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
{name: "maximum time threshold", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval, want: true},
{name: "below thresholds", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval - time.Millisecond, want: false},
{name: "completion", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval, want: true},
{name: "completion rate limited", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
{name: "completion already reported", total: 100, uploaded: 100, lastPercent: 100, elapsed: uploadProgressMinInterval, want: false},
{name: "out of order callback", total: 100, uploaded: 40, lastPercent: 60, elapsed: uploadProgressMaxInterval, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := shouldUpdateUploadProgress(tt.total, tt.uploaded, tt.lastPercent, tt.elapsed)
if got != tt.want {
t.Fatalf("shouldUpdateUploadProgress() = %v, want %v", got, tt.want)
}
})
}
}
func TestUploadProgressConcurrentCallbacks(t *testing.T) {
progress := new(Progress)
ctx := context.Background()
info := progressTestTaskInfo{}
const total = int64(100 << 20)
progress.OnUploadStart(ctx, info, total)
progress.lastUpdateAt.Store(time.Now().Add(-uploadProgressMaxInterval).UnixNano())
var wg sync.WaitGroup
for uploaded := int64(1 << 20); uploaded <= total; uploaded += 1 << 20 {
uploaded := uploaded
wg.Go(func() {
progress.OnUploadProgress(ctx, info, uploaded, total)
})
}
wg.Wait()
percent := progress.lastUpdatePercent.Load()
if percent <= 0 || percent > 100 {
t.Fatalf("last upload percentage = %d, want a value in (0, 100]", percent)
}
if progress.uploadedBytes != total {
t.Fatalf("maximum uploaded bytes = %d, want %d", progress.uploadedBytes, total)
}
}
func TestSingleUploadRetryKeepsRichProgressLayout(t *testing.T) {
i18n.Init("zh-Hans")
t.Cleanup(func() { i18n.Init("zh-Hans") })
message := buildSingleProgressMessage(
progressTestTaskInfo{},
singleUploadPhase(2),
25<<20,
100<<20,
5<<20,
2,
)
if message.Err != nil {
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
}
for _, want := range []string{
"🔁 上传重试",
"🟩🟩⬜️⬜️⬜️⬜️⬜️⬜️⬜️⬜️ 25%",
"尝试次数2",
"速度5.00 MB/s",
"大小25.00 MB / 100.00 MB",
} {
if !strings.Contains(message.Text, want) {
t.Fatalf("retry progress does not contain %q:\n%s", want, message.Text)
}
}
}
func TestSingleProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
i18n.Init("en")
t.Cleanup(func() { i18n.Init("zh-Hans") })
info := htmlProgressTestTaskInfo{}
message := buildSingleProgressMessage(info, singlePhaseDownloading, 50, 100, 25, 0)
if message.Err != nil {
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
}
for _, want := range []string{
`<b>A&B</b>.bin`,
`[store<&>]:dir/<i>x</i>&`,
"Speed: 25 B/s",
} {
if !strings.Contains(message.Text, want) {
t.Fatalf("progress text does not contain %q:\n%s", want, message.Text)
}
}
bold, code, blockquote, italic := singleEntityCounts(message.Entities)
if bold != 2 || code != 6 || blockquote != 1 || italic != 0 {
t.Fatalf("progress entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
}
failure := buildSingleDoneMessage(info, 100, errors.New(`<i>remote & failed</i>`))
if failure.Err != nil {
t.Fatalf("buildSingleDoneMessage() failed: %v", failure.Err)
}
if !strings.Contains(failure.Text, `<i>remote & failed</i>`) {
t.Fatalf("failure reason was not preserved literally:\n%s", failure.Text)
}
bold, code, blockquote, italic = singleEntityCounts(failure.Entities)
if bold != 1 || code != 2 || blockquote != 0 || italic != 0 {
t.Fatalf("failure entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
}
}
func TestSingleDoneSizeUsesActualUploadSize(t *testing.T) {
progress := new(Progress)
info := progressTestTaskInfo{}
progress.OnStart(context.Background(), info)
progress.OnUploadStart(context.Background(), info, 2048)
if got := progress.doneSize(info); got != 2048 {
t.Fatalf("done size = %d, want actual upload size 2048", got)
}
}
type htmlProgressTestTaskInfo struct{}
func (htmlProgressTestTaskInfo) TaskID() string { return "html-task" }
func (htmlProgressTestTaskInfo) FileName() string { return `<b>A&B</b>.bin` }
func (htmlProgressTestTaskInfo) FileSize() int64 { return 100 }
func (htmlProgressTestTaskInfo) StoragePath() string { return `dir/<i>x</i>&` }
func (htmlProgressTestTaskInfo) StorageName() string { return `store<&>` }
func singleEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
for _, messageEntity := range entities {
switch messageEntity.(type) {
case *tg.MessageEntityBold:
bold++
case *tg.MessageEntityCode:
code++
case *tg.MessageEntityBlockquote:
blockquote++
case *tg.MessageEntityItalic:
italic++
}
}
return
}

View File

@@ -79,7 +79,8 @@ Stream mode is not supported.
chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat
force_file = false # Force sending as file, default is false
skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded.
spilt_size_mb = 2000 # Split size in MB, default is 2000 MB (2 GB). Files larger than this will be split into multiple parts (zip format). Ignored when skip_large is true.
split_large_video = false # Losslessly split oversized videos into one album of up to 10 independently playable parts. Falls back to ZIP parts on failure or when more than 10 parts are required.
split_size_mb = 0 # Split size in MB. 0 uses the uploader account limit: 2000 MB for bots/regular users and 4000 MB for Premium users. Oversized non-video files use ZIP parts. Ignored when skip_large is true.
```
## Rclone
@@ -136,4 +137,4 @@ remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```
```

View File

@@ -82,10 +82,12 @@ chat_id = "123456789"
force_file = false
# 是否跳过大文件, 默认为 false. 如果启用, 超过 Telegram 限制的文件将不会上传.
skip_large = false
# 分卷大小, 单位 MB, 默认为 2000 MB (2 GB).
# 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式)
# 超限视频是否使用 FFmpeg 无损分割成一个媒体组(最多 10 个)内可独立播放的小视频;失败或需要超过 10 段时回退到 ZIP 分卷.
split_large_video = false
# 分卷大小, 单位 MB. 设为 0 时使用实际上传账号的限制Bot/普通用户为 2000 MBPremium 用户为 4000 MB.
# 超过该大小的文件将被分割成多个部分上传;非视频使用 ZIP 格式.
# 当 skip_large 启用时, 该选项无效.
spilt_size_mb = 2000
split_size_mb = 0
```
## Rclone
@@ -142,4 +144,4 @@ remote = "myremote"
base_path = "/backup"
config_path = "/path/to/rclone.conf"
flags = ["--progress"]
```
```

80
go.mod
View File

@@ -11,22 +11,22 @@ require (
github.com/charmbracelet/lipgloss v1.1.0
github.com/charmbracelet/log v1.0.0
github.com/dustin/go-humanize v1.0.1
github.com/gabriel-vasile/mimetype v1.4.13
github.com/gabriel-vasile/mimetype v1.4.15
github.com/goccy/go-yaml v1.19.2
github.com/gotd/contrib v0.21.1
github.com/gotd/td v0.143.0
github.com/gotd/td v0.149.0
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3
github.com/krau/ffmpeg-go v0.6.0
github.com/lrstanley/go-ytdlp v1.3.5
github.com/minio/minio-go/v7 v7.2.0
github.com/playwright-community/playwright-go v0.5700.1
github.com/minio/minio-go/v7 v7.2.1
github.com/playwright-community/playwright-go v0.6000.0
github.com/rs/xid v1.6.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/unvgo/ghselfupdate v1.0.1
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c
golang.org/x/net v0.56.0
golang.org/x/term v0.44.0
golang.org/x/net v0.57.0
golang.org/x/term v0.45.0
golang.org/x/time v0.15.0
)
@@ -43,16 +43,16 @@ require (
github.com/charmbracelet/x/term v0.2.2 // indirect
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/cloudflare/circl v1.6.4 // indirect
github.com/cloudflare/circl v1.6.5 // indirect
github.com/coder/websocket v1.8.15 // indirect
github.com/deckarep/golang-set/v2 v2.9.0 // indirect
github.com/dlclark/regexp2 v1.12.0 // indirect
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
github.com/dlclark/regexp2/v2 v2.6.0 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/glebarez/go-sqlite v1.22.0 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/glebarez/go-sqlite v1.23.0 // indirect
github.com/go-faster/errors v0.8.0 // indirect
github.com/go-faster/jx v1.2.0 // indirect
github.com/go-faster/xor v1.0.0 // indirect
github.com/go-faster/yaml v0.4.6 // indirect
@@ -63,29 +63,29 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 // indirect
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gotd/ige v0.2.2 // indirect
github.com/gotd/ige v0.3.0 // indirect
github.com/gotd/neo v0.1.5 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/mattn/go-runewidth v0.0.27 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 // indirect
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/ogen-go/ogen v1.22.0 // indirect
github.com/ogen-go/ogen v1.24.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -94,52 +94,52 @@ require (
github.com/segmentio/asm v1.2.1 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/yuin/goldmark v1.8.2 // indirect
github.com/ulikunitz/xz v0.5.16 // indirect
github.com/yuin/goldmark v1.8.5 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.mongodb.org/mongo-driver v1.17.9 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/mod v0.37.0 // indirect
golang.org/x/tools v0.46.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/mod v0.39.0 // indirect
golang.org/x/tools v0.48.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/libc v1.75.3 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.53.0 // indirect
modernc.org/memory v1.12.0 // indirect
modernc.org/sqlite v1.56.0 // indirect
rsc.io/qr v0.2.0 // indirect
)
require (
github.com/dgraph-io/ristretto/v2 v2.4.0
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59
github.com/dgraph-io/ristretto/v2 v2.4.2
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6
github.com/duke-git/lancet/v2 v2.3.9
github.com/fsnotify/fsnotify v1.10.1
github.com/glebarez/sqlite v1.11.0
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/ncruces/go-sqlite3 v0.35.1 // indirect
github.com/ncruces/go-sqlite3 v0.35.3 // indirect
github.com/ncruces/go-sqlite3/gormlite v0.34.0
github.com/nicksnyder/go-i18n/v2 v2.6.1
github.com/pelletier/go-toml/v2 v2.4.2 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/xo/terminfo v1.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
golang.org/x/sync v0.21.0
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.38.0
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da // indirect
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0
gorm.io/gorm v1.31.2
)

180
go.sum
View File

@@ -66,8 +66,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U=
github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY=
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c=
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -76,16 +76,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4M5yOCKI=
github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM=
github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgraph-io/ristretto/v2 v2.4.2 h1:x0cvjmUKxt764Yxdk2nr94we1AvPPAMh1rh5TQ+Jo80=
github.com/dgraph-io/ristretto/v2 v2.4.2/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59 h1:DjKLmvKK9u15djHZ88N8M0DhgnHVgJJ8bnEe0h7Lga8=
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59/go.mod h1:Sc+QOu1WruvaaeT/cxFez/pXHpI9ZDjg/E8QNfSVveI=
github.com/dlclark/regexp2/v2 v2.6.0 h1:KugbSrpXcRpziHIcqqFEXwKwi09LbPkTzvLRItf2mo8=
github.com/dlclark/regexp2/v2 v2.6.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6 h1:Oh2rRG1un7tLlC3/NJDzKppZ4CeZGkVFJCUOTRwLpfw=
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6/go.mod h1:LiIEzozrcvNXorsG/3+ypGqdTUAqZryhzSsqi0oU/Qg=
github.com/duke-git/lancet/v2 v2.3.9 h1:ZxUvfoEY7YbsGIeoXRxHWIkRCAt6VN7UBKWgCCqBB3U=
github.com/duke-git/lancet/v2 v2.3.9/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -98,19 +98,18 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
github.com/glebarez/go-sqlite v1.23.0 h1:FyhIq4jqmgphQAUlY79zPldYGwISEZikaDfhiGWkkaI=
github.com/glebarez/go-sqlite v1.23.0/go.mod h1:IIYrOH3L0rHY3jb4IXOHoWdklNajSGUN2eJcvK8WrnI=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
github.com/go-faster/errors v0.8.0 h1:9T9eJrM+72dFk7n4DfhuaDDe6cyuFCSW2oNUkN77Yqc=
github.com/go-faster/errors v0.8.0/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
github.com/go-faster/jx v1.2.0 h1:T2YHJPrFaYu21fJtUxC9GzmluKu8rVIFDwwGBKTDseI=
github.com/go-faster/jx v1.2.0/go.mod h1:UWLOVDmMG597a5tBFPLIWJdUxz5/2emOpfsj9Neg0PE=
github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
@@ -119,8 +118,8 @@ github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQ
github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE=
github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
@@ -141,18 +140,18 @@ github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQF
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE=
github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gotd/contrib v0.21.1 h1:NSF+0YEnosQ34QEo2o4s6MA5YFDAor1LVvLhN1L3H1M=
github.com/gotd/contrib v0.21.1/go.mod h1:trVJBP9Q/TJbjmJbVnLc0cnX/8T4N0RpQBULVa3BNnE=
github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk=
github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0=
github.com/gotd/ige v0.3.0 h1:4f6LEHWsVDLBG0bT9wWG2/9TZb5aWm265G8ZlTXmRRU=
github.com/gotd/ige v0.3.0/go.mod h1:FE9bTaQtvfArizAcZuI4sS6gXaEUBmixdUufVHoCKac=
github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ=
github.com/gotd/td v0.143.0 h1:p0U/Nn92zXmAsahDn5CIVzay2kQ36lBBENT/FlWR2nQ=
github.com/gotd/td v0.143.0/go.mod h1:8GA5ecTI5iswLwBAlqf0u6/+j+BqSWUARSrX2Xk1usQ=
github.com/gotd/td v0.149.0 h1:vXzNO99FFzWKKyI0vizvtgkpI7AqmbcXHLWjxkD+69o=
github.com/gotd/td v0.149.0/go.mod h1:+s0fRWlKL+RqoKJAMnCRoj0sWtxzRlWm886elNZEpJQ=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
@@ -165,11 +164,11 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3 h1:2713fQZ560HxoNVgfJH41GKzjMjIG+DW4hH6nYXfXW8=
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -180,24 +179,24 @@ github.com/krau/ffmpeg-go v0.6.0 h1:F4HWvOrKXQsfLsFTOnUfP0HY6WISJqOrsAFGSIzkKto=
github.com/krau/ffmpeg-go v0.6.0/go.mod h1:sa7/bWHB6fO9j4lhmxnWQ1U07o+dE1leFjhctotxU7A=
github.com/lrstanley/go-ytdlp v1.3.5 h1:eT+29mK3Lp+XPMQOH25+jVerrrjifYW1o3IkTYJ9SMs=
github.com/lrstanley/go-ytdlp v1.3.5/go.mod h1:VgjnTrvkTf+23JuySjyPq1iQ8ijSovBtTPpXH5XrLtI=
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss=
github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw=
github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
@@ -206,10 +205,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/ncruces/go-sqlite3 v0.35.1 h1:h/LaVyQwIvBBT0+2JmVe2tbYyWjUQ093/pYhpBqdxJo=
github.com/ncruces/go-sqlite3 v0.35.1/go.mod h1:fXOSIkWwN5NXgbJk+7Zls8QIW4xOflmgh11OFvcY+J0=
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 h1:Cew7/eNAMd1zhpXYBjofBua/63pFvbvB2h4PM/p6gKU=
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302/go.mod h1:xe0CfafDUxfh+fSVKjHHMiAxoG9KALt5nFtbGNb/jRs=
github.com/ncruces/go-sqlite3 v0.35.3 h1:Ei07Zv1qfV/vyXzelhFsyS5Oh9TArBZHsmFk14Xv3GY=
github.com/ncruces/go-sqlite3 v0.35.3/go.mod h1:i1rhym/NIiB5xeEfzbN+e24Y+i7NGUpf7C2xZ3Dpwks=
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 h1:5NoQAewtgKNK3G4bjNPxVoGXu6F6NzLXWCTdD5FFAEY=
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304/go.mod h1:o8gr9w/50fXA5TDskg6bNUjvqmFfw4KaXth4q+yDSjg=
github.com/ncruces/go-sqlite3/gormlite v0.34.0 h1:QLlOy/i7OabsFUQ+d5KyXmq2hw9sMh/CRW435+eQMRY=
github.com/ncruces/go-sqlite3/gormlite v0.34.0/go.mod h1:CMv+6YhqLmPBXYACiQtrWA0q/JLIMTKB4E65SUfLgF0=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -218,24 +217,24 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
github.com/ogen-go/ogen v1.22.0 h1:7wU+jcIKg/JBAhM95909ULLdAkGr43KQOuvNpJ7Mxb4=
github.com/ogen-go/ogen v1.22.0/go.mod h1:7BOh9a51QiPCC92RMrj1LlkLjejhBAyPhR+oMc6lR9g=
github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/ogen-go/ogen v1.24.0 h1:NehBG/8s0JeM6jYHPJeFJntBG3cE/xQgZd1q8BchPNM=
github.com/ogen-go/ogen v1.24.0/go.mod h1:hcg4aTzLcu/MS1SGGahUriPBEOZB3c82RGIC6dYcdFU=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/playwright-community/playwright-go v0.5700.1 h1:PNFb1byWqrTT720rEO0JL88C6Ju0EmUnR5deFLvtP/U=
github.com/playwright-community/playwright-go v0.5700.1/go.mod h1:MlSn1dZrx8rszbCxY6x3qK89ZesJUYVx21B2JnkoNF0=
github.com/playwright-community/playwright-go v0.6000.0 h1:R7sENcRI6n0Zd5ZoW8EKdVF1ZVJgvTubfJeqKHNGvsw=
github.com/playwright-community/playwright-go v0.6000.0/go.mod h1:z/YpFVdU4LAi+0f9VPOCkGvmdH6dCrtza9nxnXFXgiE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -273,17 +272,17 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
github.com/unvgo/ghselfupdate v1.0.1 h1:4clbOkfPbfEmRnnYxVXDSBs0JG12DO+0FfqplJckreU=
github.com/unvgo/ghselfupdate v1.0.1/go.mod h1:3snWV5vEHGXQqqhY7FwKjPOtH6e7cFdHYN7UMAihhxs=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY=
github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c h1:xA2TJS9Hu/ivzaZIrDcwvpJ3Fnpsk5fDOJ4iSnL6J0w=
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c/go.mod h1:WSZ59bidJOO40JSJmLqlkBJrjZCtjbKKkygEMfzY/kc=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA=
github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
@@ -294,12 +293,12 @@ go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8=
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@@ -310,34 +309,35 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da h1:YKw1FZDyWyXXZcBxalRHz3CHieUKnKxanrbcO280Zwc=
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -347,31 +347,31 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -391,30 +391,30 @@ gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g=
modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU=
modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

17
pkg/storagetypes/batch.go Normal file
View File

@@ -0,0 +1,17 @@
package storagetypes
import "io"
// BatchItem describes one seekable file in a logical batch storage operation.
type BatchItem struct {
Reader io.ReadSeeker
StoragePath string
Size int64
// SourceGroupKey is empty for standalone source messages.
SourceGroupKey string
Caption string
// PreserveCaption distinguishes an intentionally empty source caption from
// the storage backend's default caption.
PreserveCaption bool
}

View File

@@ -0,0 +1,19 @@
package storagetypes
import "context"
type sourceCaptionContextKey struct{}
// WithSourceCaption records the source Telegram message caption for storage
// backends that can preserve it. Calling this function with an empty caption
// intentionally suppresses a backend-generated fallback caption.
func WithSourceCaption(ctx context.Context, caption string) context.Context {
return context.WithValue(ctx, sourceCaptionContextKey{}, caption)
}
// SourceCaptionFromContext returns the source caption and whether the caller
// explicitly supplied one.
func SourceCaptionFromContext(ctx context.Context) (string, bool) {
caption, ok := ctx.Value(sourceCaptionContextKey{}).(string)
return caption, ok
}

View File

@@ -0,0 +1,24 @@
package storagetypes
import (
"context"
"testing"
)
func TestSourceCaptionContext(t *testing.T) {
if _, ok := SourceCaptionFromContext(context.Background()); ok {
t.Fatal("caption unexpectedly present on empty context")
}
tests := []string{"original caption", ""}
for _, want := range tests {
ctx := WithSourceCaption(context.Background(), want)
got, ok := SourceCaptionFromContext(ctx)
if !ok {
t.Fatalf("caption %q was not marked as present", want)
}
if got != want {
t.Fatalf("caption = %q, want %q", got, want)
}
}
}

View File

@@ -1,6 +1,8 @@
package tcbdata
import (
"slices"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
"github.com/krau/SaveAny-Bot/pkg/parser"
"github.com/krau/SaveAny-Bot/pkg/telegraph"
@@ -31,12 +33,7 @@ func ConflictStrategyValues() []string {
}
func IsConflictStrategy(strategy string) bool {
for _, value := range ConflictStrategyValues() {
if strategy == value {
return true
}
}
return false
return slices.Contains(ConflictStrategyValues(), strategy)
}
// type TaskDataTGFiles struct {

View File

@@ -31,6 +31,38 @@ type StorageCannotStream interface {
CannotStream() string
}
// StorageBatchSaver can preserve relationships between files when saving a
// logical batch, such as a Telegram media album.
type StorageBatchSaver interface {
Storage
SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error
}
// StorageBatchProgressSaver reports confirmed upload progress for each item in
// a logical batch. The item index matches the items slice passed to
// SaveBatchWithProgress.
type StorageBatchProgressSaver interface {
StorageBatchSaver
SaveBatchWithProgress(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error
}
// StorageProgressSaver reports bytes after the backend has accepted them for
// upload. Backends with native progress support should implement this instead
// of relying on progress inferred from reads of the input stream.
type StorageProgressSaver interface {
Storage
SaveWithProgress(
ctx context.Context,
reader io.Reader,
storagePath string,
onProgress func(uploaded, total int64),
) error
}
// StorageListable 表示支持列举目录内容的存储
type StorageListable interface {
Storage
@@ -45,6 +77,9 @@ type StorageReadable interface {
var Storages = make(map[string]Storage)
var _ StorageProgressSaver = (*telegram.Telegram)(nil)
var _ StorageBatchProgressSaver = (*telegram.Telegram)(nil)
type StorageConstructor func() Storage
var storageConstructors = map[storenum.StorageType]StorageConstructor{

View File

@@ -0,0 +1,140 @@
package telegram
import (
"bytes"
"io"
"testing"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestPlanMediaGroups(t *testing.T) {
tests := []struct {
name string
items []batchMediaItem
wantSizes []int
}{
{
name: "same source album",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 1, true),
},
wantSizes: []int{2},
},
{
name: "different source albums",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("b", 1, true),
},
wantSizes: []int{1, 1},
},
{
name: "ungrouped messages",
items: []batchMediaItem{
albumItem("", 1, true),
albumItem("", 1, true),
},
wantSizes: []int{1, 1},
},
{
name: "different target chats",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 2, true),
},
wantSizes: []int{1, 1},
},
{
name: "ineligible media does not bridge albums",
items: []batchMediaItem{
albumItem("a", 1, true),
albumItem("a", 1, false),
albumItem("a", 1, true),
},
wantSizes: []int{1, 1, 1},
},
{
name: "maximum album size",
items: repeatedAlbumItems(11),
wantSizes: []int{10, 1},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
groups := planMediaGroups(tt.items)
if len(groups) != len(tt.wantSizes) {
t.Fatalf("got %d groups, want %d", len(groups), len(tt.wantSizes))
}
for i, want := range tt.wantSizes {
if got := len(groups[i]); got != want {
t.Errorf("group %d has %d items, want %d", i, got, want)
}
}
})
}
}
func TestMediaCaption(t *testing.T) {
empty := ""
original := "original caption"
tests := []struct {
name string
override *string
wantLen int
}{
{name: "filename fallback", wantLen: 1},
{name: "preserve empty source caption", override: &empty, wantLen: 0},
{name: "preserve source caption", override: &original, wantLen: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := len(mediaCaption("file.jpg", tt.override)); got != tt.wantLen {
t.Fatalf("got %d caption options, want %d", got, tt.wantLen)
}
})
}
}
func TestInspectBatchItemRewindsBeforeMimetypeDetection(t *testing.T) {
data := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
reader := bytes.NewReader(data)
if _, err := reader.Seek(4, io.SeekStart); err != nil {
t.Fatalf("failed to set initial reader offset: %v", err)
}
mediaItem, err := new(Telegram).inspectBatchItem(nil, storagetypes.BatchItem{
Reader: reader,
StoragePath: "photo.png",
Size: int64(len(data)),
})
if err != nil {
t.Fatalf("inspectBatchItem returned an error: %v", err)
}
if !mediaItem.albumEligible {
t.Fatal("albumEligible = false, want true for PNG input")
}
if offset, err := reader.Seek(0, io.SeekCurrent); err != nil {
t.Fatalf("failed to get final reader offset: %v", err)
} else if offset != 0 {
t.Fatalf("reader offset = %d, want 0", offset)
}
}
func albumItem(group string, chatID int64, eligible bool) batchMediaItem {
return batchMediaItem{
item: storagetypes.BatchItem{SourceGroupKey: group},
chatID: chatID,
albumEligible: eligible,
}
}
func repeatedAlbumItems(count int) []batchMediaItem {
items := make([]batchMediaItem, count)
for i := range items {
items[i] = albumItem("a", 1, true)
}
return items
}

View File

@@ -0,0 +1,60 @@
package telegram
import (
"context"
"sync"
"github.com/gotd/td/telegram/uploader"
)
var _ uploader.Progress = (*uploadProgress)(nil)
type uploadProgress struct {
mu sync.Mutex
onProgress func(uploaded, total int64)
total int64
uploaded int64
byID map[int64]int64
}
func newUploadProgress(total int64, onProgress func(uploaded, total int64)) *uploadProgress {
return &uploadProgress{
onProgress: onProgress,
total: total,
byID: make(map[int64]int64),
}
}
func (p *uploadProgress) Chunk(ctx context.Context, state uploader.ProgressState) error {
if err := ctx.Err(); err != nil {
return err
}
p.mu.Lock()
previous := p.byID[state.ID]
if state.Uploaded <= previous {
p.mu.Unlock()
return nil
}
p.byID[state.ID] = state.Uploaded
p.uploaded += state.Uploaded - previous
uploaded := p.uploaded
total := p.total
if total <= 0 {
total = state.Total
}
p.mu.Unlock()
if p.onProgress != nil {
p.onProgress(uploaded, total)
}
return nil
}
func (p *uploadProgress) reset(total int64) {
p.mu.Lock()
p.total = total
p.uploaded = 0
p.byID = make(map[int64]int64)
p.mu.Unlock()
}

View File

@@ -0,0 +1,80 @@
package telegram
import (
"context"
"testing"
"github.com/gotd/td/telegram/uploader"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestUploadProgressAggregatesUploaderParts(t *testing.T) {
type update struct {
uploaded int64
total int64
}
var updates []update
progress := newUploadProgress(100, func(uploaded, total int64) {
updates = append(updates, update{uploaded: uploaded, total: total})
})
states := []uploader.ProgressState{
{ID: 1, Uploaded: 20, Total: 60},
{ID: 1, Uploaded: 20, Total: 60},
{ID: 1, Uploaded: 60, Total: 60},
{ID: 2, Uploaded: 10, Total: 40},
{ID: 2, Uploaded: 40, Total: 40},
}
for _, state := range states {
if err := progress.Chunk(context.Background(), state); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
}
want := []update{{20, 100}, {60, 100}, {70, 100}, {100, 100}}
if len(updates) != len(want) {
t.Fatalf("got %d updates, want %d", len(updates), len(want))
}
for i := range want {
if updates[i] != want[i] {
t.Fatalf("update %d = %+v, want %+v", i, updates[i], want[i])
}
}
}
func TestUploadProgressResetForSplitFiles(t *testing.T) {
var uploaded, total int64
progress := newUploadProgress(100, func(current, size int64) {
uploaded, total = current, size
})
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 100, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
progress.reset(120)
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 2, Uploaded: 30, Total: 60}); err != nil {
t.Fatalf("Chunk() after reset failed: %v", err)
}
if uploaded != 30 || total != 120 {
t.Fatalf("progress after reset = %d/%d, want 30/120", uploaded, total)
}
}
func TestBatchItemUploadProgressPreservesItemIndex(t *testing.T) {
var gotIndex int
var gotUploaded, gotTotal int64
progress := batchItemUploadProgress(batchMediaItem{
index: 4,
item: storagetypes.BatchItem{Size: 100},
}, func(index int, uploaded, total int64) {
gotIndex = index
gotUploaded = uploaded
gotTotal = total
})
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 25, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
if gotIndex != 4 || gotUploaded != 25 || gotTotal != 100 {
t.Fatalf("batch progress = index %d, %d/%d; want index 4, 25/100", gotIndex, gotUploaded, gotTotal)
}
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/celestix/gotgproto/ext"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/retry"
"github.com/duke-git/lancet/v2/slice"
"github.com/duke-git/lancet/v2/validator"
"github.com/gabriel-vasile/mimetype"
@@ -26,14 +27,16 @@ import (
"github.com/krau/SaveAny-Bot/pkg/consts/tglimit"
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
"github.com/rs/xid"
"golang.org/x/time/rate"
)
const (
// https://core.telegram.org/api/config#upload-max-fileparts-default
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
PremiumMaxUploadFileSize = 8000 * 524288 // 8000 * 512 KB
)
type Telegram struct {
@@ -41,6 +44,20 @@ type Telegram struct {
limiter *rate.Limiter
}
type preparedMedia struct {
peer tg.InputPeerClass
uploader *uploader.Uploader
media message.MultiMediaOption
}
type batchMediaItem struct {
item storagetypes.BatchItem
index int
chatID int64
albumEligible bool
useSingleSave bool
}
func (t *Telegram) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
telegramConfig, ok := cfg.(*storconfig.TelegramStorageConfig)
if !ok {
@@ -71,37 +88,147 @@ func (t *Telegram) Exists(ctx context.Context, storagePath string) bool {
}
func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) error {
return t.save(ctx, r, storagePath, nil)
}
// SaveWithProgress saves a file while reporting Telegram-confirmed upload
// progress after each uploaded part.
func (t *Telegram) SaveWithProgress(
ctx context.Context,
r io.Reader,
storagePath string,
onProgress func(uploaded, total int64),
) error {
size := contentLength(ctx)
return t.save(ctx, r, storagePath, newUploadProgress(size, onProgress))
}
func (t *Telegram) save(ctx context.Context, r io.Reader, storagePath string, progress *uploadProgress) error {
storagePath = path.Clean(storagePath)
captionOverride := sourceCaptionOverride(ctx)
tctx := tgutil.ExtFromContext(ctx)
if tctx == nil {
return fmt.Errorf("failed to get telegram context")
}
size := func() int64 {
if length := ctx.Value(ctxkey.ContentLength); length != nil {
if l, ok := length.(int64); ok {
return l
}
}
return -1 // unknown size
}()
if t.config.SkipLarge && size > MaxUploadFileSize {
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", MaxUploadFileSize, size)
size := contentLength(ctx)
maxUploadSize := maxUploadFileSize(tctx)
if t.config.SkipLarge && size > maxUploadSize {
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", maxUploadSize, size)
return nil
}
rs, seekable := r.(io.ReadSeeker)
splitSize := t.config.SplitSizeMB * 1024 * 1024
if splitSize <= 0 {
splitSize = DefaultSplitSize
splitSize := t.splitSize(maxUploadSize)
if size > splitSize {
filename, chatID := t.target(tctx, storagePath)
if filename == "" {
if rs, ok := r.(io.ReadSeeker); ok {
mtype, err := mimetype.DetectReader(rs)
if err != nil {
return fmt.Errorf("failed to detect mimetype: %w", err)
}
filename = xid.New().String() + mtype.Extension()
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek reader: %w", err)
}
}
}
upler := t.newUploader(tctx, size, progress)
peer := tryGetInputPeer(tctx, chatID)
if peer == nil || peer.Zero() {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
if t.config.SplitLargeVideo {
rs, ok := r.(io.ReadSeeker)
if ok {
mtype, detectErr := mimetype.DetectReader(rs)
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek large file after mimetype detection: %w", seekErr)
}
if detectErr != nil {
log.FromContext(ctx).Warnf("Failed to detect large file type, falling back to ZIP split: %s", detectErr)
} else if strings.HasPrefix(mtype.String(), "video/") {
parts, cleanup, splitErr := createLosslessVideoParts(
ctx,
rs,
filename,
size,
splitSize,
)
if splitErr != nil {
if ctx.Err() != nil {
return ctx.Err()
}
log.FromContext(ctx).Warnf("Lossless video split failed, falling back to ZIP split: %s", splitErr)
} else {
defer cleanup()
log.FromContext(ctx).Infof("Uploading oversized video as %d lossless-playable parts", len(parts))
for _, part := range parts {
log.FromContext(ctx).Infof("Prepared lossless video part %s (%d bytes)", part.Name, part.Size)
}
return t.uploadLosslessVideoParts(ctx, tctx, storagePath, parts, captionOverride, progress)
}
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
return fmt.Errorf("failed to seek large video before ZIP fallback: %w", seekErr)
}
}
}
}
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize, progress)
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
prepared, err := t.prepareMedia(ctx, tctx, r, storagePath, size, nil, progress)
if err != nil {
return err
}
_, err = tctx.Sender.
WithUploader(prepared.uploader).
To(prepared.peer).
Media(ctx, prepared.media)
return err
}
func contentLength(ctx context.Context) int64 {
if length := ctx.Value(ctxkey.ContentLength); length != nil {
if size, ok := length.(int64); ok {
return size
}
}
return -1
}
func sourceCaptionOverride(ctx context.Context) *string {
caption, ok := storagetypes.SourceCaptionFromContext(ctx)
if !ok {
return nil
}
return &caption
}
func maxUploadFileSize(tctx *ext.Context) int64 {
if tctx != nil && tctx.Self != nil && !tctx.Self.GetBot() && tctx.Self.GetPremium() {
return PremiumMaxUploadFileSize
}
return MaxUploadFileSize
}
func (t *Telegram) splitSize(maxUploadSize int64) int64 {
splitSize := t.config.SplitSizeMB * 1024 * 1024
if splitSize <= 0 {
return maxUploadSize
}
return min(splitSize, maxUploadSize)
}
func (t *Telegram) target(tctx *ext.Context, storagePath string) (string, int64) {
// 去除前导斜杠并分隔路径, 当 len(parts):
// ==0, 存储到配置文件中的 chat_id, 随机文件名
// ==1, 视作只有文件名, 存储到配置文件中的 chat_id
// ==2, parts[0]: 视作要存储到的 chat_id, parts[1]: filename
// >=2, parts[0]: 视作要存储到的 chat_id, 最后一项为 filename
parts := slice.Compact(strings.Split(strings.TrimPrefix(storagePath, "/"), "/"))
filename := ""
chatID := t.config.ChatID
@@ -111,38 +238,66 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
if len(parts) >= 2 && validator.IsAlphaNumeric(parts[0]) {
cid, err := tgutil.ParseChatID(tctx, parts[0])
if err != nil {
// id不合法时使用配置文件中的 chat_id
log.FromContext(ctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
log.FromContext(tctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
cid = chatID
}
chatID = cid
}
return filename, chatID
}
func (t *Telegram) newUploader(tctx *ext.Context, size int64, progress *uploadProgress) *uploader.Uploader {
upler := uploader.NewUploader(tctx.Raw).
WithPartSize(tglimit.MaxUploadPartSize).
WithThreads(dlutil.BestThreads(size, config.C().Threads))
if progress != nil {
upler = upler.WithProgress(progress)
}
return upler
}
func mediaCaption(filename string, override *string) []message.StyledTextOption {
if override == nil {
return []message.StyledTextOption{styling.Plain(filename)}
}
if *override == "" {
return nil
}
return []message.StyledTextOption{styling.Plain(*override)}
}
func (t *Telegram) prepareMedia(
ctx context.Context,
tctx *ext.Context,
r io.Reader,
storagePath string,
size int64,
captionOverride *string,
progress *uploadProgress,
) (*preparedMedia, error) {
storagePath = path.Clean(storagePath)
filename, chatID := t.target(tctx, storagePath)
upler := t.newUploader(tctx, size, progress)
peer := tryGetInputPeer(tctx, chatID)
if peer == nil || peer.Zero() {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
return nil, fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
rs, seekable := r.(io.ReadSeeker)
var mtype *mimetype.MIME
if seekable {
var err error
mtype, err = mimetype.DetectReader(rs)
if err != nil {
return fmt.Errorf("failed to detect mimetype: %w", err)
return nil, fmt.Errorf("failed to detect mimetype: %w", err)
}
if filename == "" {
filename = xid.New().String() + mtype.Extension()
}
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek reader: %w", err)
return nil, fmt.Errorf("failed to seek reader: %w", err)
}
}
if size > splitSize {
// large file, use split uploader
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize)
}
var file tg.InputFileClass
var err error
@@ -152,21 +307,20 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
file, err = upler.Upload(ctx, uploader.NewUpload(filename, r, size))
}
if err != nil {
return fmt.Errorf("failed to upload file to telegram: %w", err)
return nil, fmt.Errorf("failed to upload file to telegram: %w", err)
}
caption := styling.Plain(filename)
caption := mediaCaption(filename, captionOverride)
forceFile := t.config.ForceFile
if mtype != nil && strings.HasPrefix(mtype.String(), "image/") && size >= tglimit.MaxPhotoSize {
forceFile = true
}
doc := message.UploadedDocument(file, caption).
doc := message.UploadedDocument(file, caption...).
Filename(filename).
ForceFile(forceFile)
if mtype != nil {
doc = doc.MIME(mtype.String())
}
var media message.MediaOption = doc
var media message.MultiMediaOption = doc
if mtype != nil && rs != nil {
switch mtypeStr := mtype.String(); {
case strings.HasPrefix(mtypeStr, "video/"):
@@ -205,19 +359,194 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
case strings.HasPrefix(mtypeStr, "audio/"):
media = doc.Audio().Title(filename)
case strings.HasPrefix(mtypeStr, "image/") && !strings.HasSuffix(mtypeStr, "webp"):
media = message.UploadedPhoto(file, caption)
media = message.UploadedPhoto(file, caption...)
}
}
sender := tctx.Sender
_, err = sender.WithUploader(upler).To(peer).Media(ctx, media)
return err
return &preparedMedia{
peer: peer,
uploader: upler,
media: media,
}, nil
}
// SaveBatch preserves each source photo/video group as a Telegram album.
func (t *Telegram) SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error {
return t.saveBatch(ctx, items, nil)
}
// SaveBatchWithProgress preserves source media groups while reporting native
// Telegram upload progress for each input item.
func (t *Telegram) SaveBatchWithProgress(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error {
return t.saveBatch(ctx, items, onProgress)
}
func (t *Telegram) saveBatch(
ctx context.Context,
items []storagetypes.BatchItem,
onProgress func(index int, uploaded, total int64),
) error {
tctx := tgutil.ExtFromContext(ctx)
if tctx == nil {
return fmt.Errorf("failed to get telegram context")
}
inspected := make([]batchMediaItem, 0, len(items))
for index, item := range items {
mediaItem, err := t.inspectBatchItem(tctx, item)
if err != nil {
return err
}
mediaItem.index = index
inspected = append(inspected, mediaItem)
}
for _, group := range planMediaGroups(inspected) {
if err := t.saveMediaGroup(ctx, tctx, group, onProgress); err != nil {
return err
}
}
return nil
}
func (t *Telegram) inspectBatchItem(tctx *ext.Context, item storagetypes.BatchItem) (batchMediaItem, error) {
_, chatID := t.target(tctx, path.Clean(item.StoragePath))
result := batchMediaItem{item: item, chatID: chatID}
maxUploadSize := maxUploadFileSize(tctx)
if (t.config.SkipLarge && item.Size > maxUploadSize) ||
item.Size > t.splitSize(maxUploadSize) {
result.useSingleSave = true
return result, nil
}
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return result, fmt.Errorf("failed to seek batch item before mimetype detection: %w", err)
}
mtype, err := mimetype.DetectReader(item.Reader)
if err != nil {
return result, fmt.Errorf("failed to detect batch item mimetype: %w", err)
}
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return result, fmt.Errorf("failed to seek batch item: %w", err)
}
mtypeStr := mtype.String()
forceFile := t.config.ForceFile || strings.HasPrefix(mtypeStr, "image/") && item.Size >= tglimit.MaxPhotoSize
result.albumEligible = !forceFile && (strings.HasPrefix(mtypeStr, "video/") ||
strings.HasPrefix(mtypeStr, "image/") && mtypeStr != "image/webp" && mtypeStr != "image/gif")
return result, nil
}
func planMediaGroups(items []batchMediaItem) [][]batchMediaItem {
groups := make([][]batchMediaItem, 0, len(items))
for i := 0; i < len(items); {
item := items[i]
if item.useSingleSave || !item.albumEligible || item.item.SourceGroupKey == "" {
groups = append(groups, items[i:i+1])
i++
continue
}
end := i + 1
for end < len(items) && end-i < 10 {
next := items[end]
if next.useSingleSave || !next.albumEligible || next.chatID != item.chatID || next.item.SourceGroupKey != item.item.SourceGroupKey {
break
}
end++
}
groups = append(groups, items[i:end])
i = end
}
return groups
}
func batchItemUploadProgress(
mediaItem batchMediaItem,
onProgress func(index int, uploaded, total int64),
) *uploadProgress {
if onProgress == nil {
return nil
}
return newUploadProgress(mediaItem.item.Size, func(uploaded, total int64) {
onProgress(mediaItem.index, uploaded, total)
})
}
func (t *Telegram) saveMediaGroup(
ctx context.Context,
tctx *ext.Context,
group []batchMediaItem,
onProgress func(index int, uploaded, total int64),
) error {
return retry.Retry(func() error {
if len(group) == 1 && group[0].useSingleSave {
mediaItem := group[0]
item := mediaItem.item
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek batch item: %w", err)
}
itemCtx := context.WithValue(ctx, ctxkey.ContentLength, item.Size)
if item.PreserveCaption {
itemCtx = storagetypes.WithSourceCaption(itemCtx, item.Caption)
}
if onProgress == nil {
return t.Save(itemCtx, item.Reader, item.StoragePath)
}
return t.SaveWithProgress(itemCtx, item.Reader, item.StoragePath, func(uploaded, total int64) {
onProgress(mediaItem.index, uploaded, total)
})
}
if err := t.limiter.Wait(ctx); err != nil {
return fmt.Errorf("rate limit failed: %w", err)
}
prepared := make([]preparedMedia, 0, len(group))
for _, mediaItem := range group {
item := mediaItem.item
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek batch item: %w", err)
}
var captionOverride *string
if item.PreserveCaption {
captionOverride = &item.Caption
}
progress := batchItemUploadProgress(mediaItem, onProgress)
media, err := t.prepareMedia(ctx, tctx, item.Reader, item.StoragePath, item.Size, captionOverride, progress)
if err != nil {
return err
}
prepared = append(prepared, *media)
}
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
if len(prepared) == 1 {
_, err := builder.Media(ctx, prepared[0].media)
return err
}
media := make([]message.MultiMediaOption, len(prepared))
for i := range prepared {
media[i] = prepared[i].media
}
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
return fmt.Errorf("failed to send media album: %w", err)
}
return nil
}, retry.Context(ctx), retry.RetryTimes(uint(config.C().Retry)))
}
func (t *Telegram) CannotStream() string {
return "Telegram storage must use a ReaderSeeker"
}
func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, upler *uploader.Uploader, peer tg.InputPeerClass, fileSize, splitSize int64) error {
func (t *Telegram) splitUpload(
ctx *ext.Context,
r io.Reader,
filename string,
upler *uploader.Uploader,
peer tg.InputPeerClass,
fileSize, splitSize int64,
progress *uploadProgress,
) error {
tempId := xid.New().String()
outputBase := filepath.Join(config.C().Temp.BasePath, tempId, strings.Split(filename, ".")[0])
defer func() {
@@ -234,6 +563,17 @@ func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, u
return fmt.Errorf("failed to glob split files: %w", err)
}
inputFiles := make([]tg.InputFileClass, 0, len(matched))
if progress != nil {
var uploadSize int64
for _, partPath := range matched {
partInfo, err := os.Stat(partPath)
if err != nil {
return fmt.Errorf("failed to stat split part %s: %w", partPath, err)
}
uploadSize += partInfo.Size()
}
progress.reset(uploadSize)
}
for _, partPath := range matched {
// 串行上传, 不然容易被tg风控
err = func() error {

View File

@@ -0,0 +1,119 @@
package telegram
import (
"bytes"
"testing"
"github.com/celestix/gotgproto/ext"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
)
func TestMaxUploadFileSize(t *testing.T) {
tests := []struct {
name string
ctx *ext.Context
want int64
}{
{name: "missing context", want: MaxUploadFileSize},
{name: "missing self", ctx: new(ext.Context), want: MaxUploadFileSize},
{
name: "bot",
ctx: uploadAccountContext(true, false),
want: MaxUploadFileSize,
},
{
name: "premium bot still uses bot limit",
ctx: uploadAccountContext(true, true),
want: MaxUploadFileSize,
},
{
name: "regular user",
ctx: uploadAccountContext(false, false),
want: MaxUploadFileSize,
},
{
name: "premium user",
ctx: uploadAccountContext(false, true),
want: PremiumMaxUploadFileSize,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := maxUploadFileSize(tt.ctx); got != tt.want {
t.Fatalf("maxUploadFileSize() = %d, want %d", got, tt.want)
}
})
}
}
func uploadAccountContext(bot, premium bool) *ext.Context {
self := new(tg.User)
self.SetBot(bot)
self.SetPremium(premium)
return &ext.Context{Self: self}
}
func TestSplitSizeUsesUploaderLimit(t *testing.T) {
tests := []struct {
name string
splitSizeMB int64
accountLimit int64
want int64
}{
{
name: "bot default",
accountLimit: MaxUploadFileSize,
want: MaxUploadFileSize,
},
{
name: "premium default",
accountLimit: PremiumMaxUploadFileSize,
want: PremiumMaxUploadFileSize,
},
{
name: "explicit lower limit",
splitSizeMB: 1500,
accountLimit: PremiumMaxUploadFileSize,
want: 1500 * 1024 * 1024,
},
{
name: "explicit limit is capped by account",
splitSizeMB: 5000,
accountLimit: PremiumMaxUploadFileSize,
want: PremiumMaxUploadFileSize,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
telegramStorage := Telegram{}
telegramStorage.config.SplitSizeMB = tt.splitSizeMB
if got := telegramStorage.splitSize(tt.accountLimit); got != tt.want {
t.Fatalf("splitSize() = %d, want %d", got, tt.want)
}
})
}
}
func TestBatchUploadLimitAppliesPerFile(t *testing.T) {
telegramStorage := Telegram{}
tctx := uploadAccountContext(true, false)
data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom")
itemSize := int64(MaxUploadFileSize/2 + 1)
for index := range 2 {
item, err := telegramStorage.inspectBatchItem(tctx, storagetypes.BatchItem{
Reader: bytes.NewReader(data),
StoragePath: "video.mp4",
Size: itemSize,
})
if err != nil {
t.Fatalf("inspectBatchItem() failed for item %d: %v", index, err)
}
if item.useSingleSave {
t.Fatalf("item %d was treated as oversized even though only the batch total exceeds the limit", index)
}
}
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"time"
"github.com/celestix/gotgproto/ext"
@@ -14,6 +15,34 @@ import (
"github.com/yapingcat/gomedia/go-mp4"
)
// sourceFile returns a filesystem path to the media for ffmpeg/ffprobe. Those
// tools need a SEEKABLE input: feeding them a pipe (pipe:0) fails for
// non-faststart MP4s whose moov atom is at the END of the file, because ffmpeg
// cannot seek backwards on a stream to read it. That silently broke thumbnail
// and metadata extraction for a large share of downloaded videos. If rs is
// already an *os.File we use it directly; otherwise we spool it to a temp file
// and return a cleanup func.
func sourceFile(rs io.ReadSeeker) (path string, cleanup func(), err error) {
noop := func() {}
if f, ok := rs.(*os.File); ok {
return f.Name(), noop, nil
}
if _, err = rs.Seek(0, io.SeekStart); err != nil {
return "", noop, err
}
tf, err := os.CreateTemp("", "saveany-media-*.tmp")
if err != nil {
return "", noop, err
}
if _, err = io.Copy(tf, rs); err != nil {
tf.Close()
os.Remove(tf.Name())
return "", noop, err
}
tf.Close()
return tf.Name(), func() { os.Remove(tf.Name()) }, nil
}
type VideoMetadata struct {
Duration int
Width int
@@ -52,16 +81,14 @@ func getMP4Meta(rs io.ReadSeeker) (metadata *VideoMetadata, err error) {
// getVideoMetadata uses ffprobe to get video metadata
func getVideoMetadata(rs io.ReadSeeker) (*VideoMetadata, error) {
pipeReader, pipeWriter := io.Pipe()
path, cleanup, err := sourceFile(rs)
if err != nil {
return nil, err
}
defer cleanup()
go func() {
defer pipeWriter.Close()
rs.Seek(0, io.SeekStart)
io.Copy(pipeWriter, rs)
}()
result, err := ffmpeg.ProbeReaderWithTimeout(
pipeReader,
result, err := ffmpeg.ProbeWithTimeout(
path,
time.Second*10,
ffmpeg.KwArgs{
"select_streams": "v:0",
@@ -114,25 +141,22 @@ func extractThumbFrame(rs io.ReadSeeker) ([]byte, error) {
}
func extractFrameAt(rs io.ReadSeeker, timestamp float64) ([]byte, error) {
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
rs.Seek(0, io.SeekStart)
io.Copy(pipeWriter, rs)
}()
path, cleanup, err := sourceFile(rs)
if err != nil {
return nil, err
}
defer cleanup()
var out bytes.Buffer
err := ffmpeg.
Input("pipe:0", ffmpeg.KwArgs{
err = ffmpeg.
Input(path, ffmpeg.KwArgs{
"ss": fmt.Sprintf("%.3f", timestamp),
}).
Output("pipe:1", ffmpeg.KwArgs{
"vframes": 1,
"f": "mjpeg",
}).
WithInput(pipeReader).
WithOutput(&out).
OverWriteOutput().
Run()

View File

@@ -0,0 +1,372 @@
package telegram
import (
"context"
"fmt"
"io"
"math"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/celestix/gotgproto/ext"
"github.com/charmbracelet/log"
"github.com/gotd/td/telegram/message"
"github.com/rs/xid"
"github.com/krau/SaveAny-Bot/config"
)
const (
videoPartTargetRatio = 0.95
videoSplitAttempts = 4
minSegmentDuration = 1.0
maxLosslessVideoParts = 10
)
type losslessVideoPart struct {
Path string
Name string
Size int64
}
type mediaToolRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
var runMediaTool mediaToolRunner = func(ctx context.Context, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
output, err := cmd.CombinedOutput()
if err != nil {
message := strings.TrimSpace(string(output))
if message == "" {
return nil, err
}
return nil, fmt.Errorf("%w: %s", err, message)
}
return output, nil
}
func createLosslessVideoParts(
ctx context.Context,
r io.ReadSeeker,
filename string,
fileSize, maxPartSize int64,
) ([]losslessVideoPart, func(), error) {
inputPath, sourceCleanup, err := sourceFile(r)
if err != nil {
return nil, func() {}, fmt.Errorf("failed to prepare seekable video source: %w", err)
}
tempBase := config.C().Temp.BasePath
if err := os.MkdirAll(tempBase, 0o755); err != nil {
sourceCleanup()
return nil, func() {}, fmt.Errorf("failed to create video split base directory: %w", err)
}
tempDir, err := os.MkdirTemp(tempBase, "telegram-video-split-*")
if err != nil {
sourceCleanup()
return nil, func() {}, fmt.Errorf("failed to create video split directory: %w", err)
}
cleanup := func() {
if err := os.RemoveAll(tempDir); err != nil {
log.FromContext(ctx).Warnf("Failed to clean lossless video parts: %s", err)
}
sourceCleanup()
}
parts, err := splitLosslessVideo(ctx, inputPath, tempDir, filename, fileSize, maxPartSize)
if err != nil {
cleanup()
return nil, func() {}, err
}
return parts, cleanup, nil
}
func splitLosslessVideo(
ctx context.Context,
inputPath, outputDir, filename string,
fileSize, maxPartSize int64,
) ([]losslessVideoPart, error) {
if fileSize <= maxPartSize {
return nil, fmt.Errorf("video size %d does not exceed part limit %d", fileSize, maxPartSize)
}
if maxPartSize <= 0 {
return nil, fmt.Errorf("invalid video part limit: %d", maxPartSize)
}
duration, err := probeMediaDuration(ctx, inputPath)
if err != nil {
return nil, fmt.Errorf("failed to probe source video duration: %w", err)
}
if duration < minSegmentDuration {
return nil, fmt.Errorf("invalid source video duration: %.3f", duration)
}
targetSize := int64(float64(maxPartSize) * videoPartTargetRatio)
segmentDuration := initialSegmentDuration(duration, fileSize, targetSize)
extension := videoPartExtension(filename)
outputPattern := filepath.Join(outputDir, "part-%03d"+extension)
var lastOversize int64
for range videoSplitAttempts {
if err := clearVideoParts(outputDir); err != nil {
return nil, err
}
if err := runFFmpegSegment(ctx, inputPath, outputPattern, extension, segmentDuration); err != nil {
return nil, fmt.Errorf("failed to losslessly split video: %w", err)
}
parts, largest, err := collectVideoParts(ctx, outputDir, filename)
if err != nil {
return nil, err
}
if len(parts) < 2 {
return nil, fmt.Errorf("video split produced %d part(s), expected at least 2", len(parts))
}
if len(parts) > maxLosslessVideoParts {
return nil, fmt.Errorf(
"video split produced %d parts, exceeding the single-album limit of %d",
len(parts),
maxLosslessVideoParts,
)
}
if largest <= maxPartSize {
return parts, nil
}
lastOversize = largest
segmentDuration *= float64(targetSize) / float64(largest)
if segmentDuration < minSegmentDuration {
break
}
}
return nil, fmt.Errorf(
"unable to keep lossless video parts below %d bytes; largest part was %d bytes",
maxPartSize,
lastOversize,
)
}
func initialSegmentDuration(duration float64, fileSize, targetSize int64) float64 {
partCount := math.Ceil(float64(fileSize) / float64(targetSize))
if partCount < 2 {
partCount = 2
}
segmentDuration := duration / partCount
if segmentDuration < minSegmentDuration {
return minSegmentDuration
}
return segmentDuration
}
func videoPartExtension(filename string) string {
extension := strings.ToLower(filepath.Ext(filepath.Base(filename)))
switch extension {
case ".avi", ".m4v", ".mkv", ".mov", ".mp4", ".ts", ".webm":
return extension
default:
return ".mp4"
}
}
func runFFmpegSegment(
ctx context.Context,
inputPath, outputPattern, extension string,
segmentDuration float64,
) error {
args := []string{
"-hide_banner",
"-loglevel", "error",
"-nostdin",
"-y",
"-i", inputPath,
"-map", "0",
"-map_metadata", "0",
"-c", "copy",
"-f", "segment",
"-segment_time", strconv.FormatFloat(segmentDuration, 'f', 3, 64),
"-segment_start_number", "1",
"-reset_timestamps", "1",
"-avoid_negative_ts", "make_zero",
}
if extension == ".mp4" || extension == ".m4v" || extension == ".mov" {
args = append(args, "-segment_format_options", "movflags=+faststart")
}
args = append(args, outputPattern)
_, err := runMediaTool(ctx, "ffmpeg", args...)
return err
}
func probeMediaDuration(ctx context.Context, filePath string) (float64, error) {
output, err := runMediaTool(
ctx,
"ffprobe",
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
filePath,
)
if err != nil {
return 0, err
}
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
if err != nil {
return 0, fmt.Errorf("invalid ffprobe duration %q: %w", strings.TrimSpace(string(output)), err)
}
return duration, nil
}
func clearVideoParts(outputDir string) error {
entries, err := os.ReadDir(outputDir)
if err != nil {
return fmt.Errorf("failed to read video split directory: %w", err)
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
if err := os.Remove(filepath.Join(outputDir, entry.Name())); err != nil {
return fmt.Errorf("failed to clear old video part %s: %w", entry.Name(), err)
}
}
return nil
}
func collectVideoParts(
ctx context.Context,
outputDir, filename string,
) ([]losslessVideoPart, int64, error) {
matches, err := filepath.Glob(filepath.Join(outputDir, "part-*"))
if err != nil {
return nil, 0, fmt.Errorf("failed to list video parts: %w", err)
}
sort.Strings(matches)
if len(matches) == 0 {
return nil, 0, fmt.Errorf("ffmpeg did not produce any video parts")
}
extension := videoPartExtension(filename)
base := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filepath.Base(filename)))
if base == "" || base == "." {
base = xid.New().String()
}
parts := make([]losslessVideoPart, 0, len(matches))
var largest int64
for index, match := range matches {
info, err := os.Stat(match)
if err != nil {
return nil, 0, fmt.Errorf("failed to stat video part %s: %w", match, err)
}
if info.Size() <= 0 {
return nil, 0, fmt.Errorf("video part %s is empty", match)
}
if _, err := probeMediaDuration(ctx, match); err != nil {
return nil, 0, fmt.Errorf("failed to validate video part %s: %w", match, err)
}
if info.Size() > largest {
largest = info.Size()
}
parts = append(parts, losslessVideoPart{
Path: match,
Name: fmt.Sprintf("%s.part%03d%s", base, index+1, extension),
Size: info.Size(),
})
}
return parts, largest, nil
}
func partStoragePath(storagePath, partName string) string {
directory := path.Dir(path.Clean(storagePath))
if directory == "." || directory == "/" {
return partName
}
return path.Join(directory, partName)
}
func (t *Telegram) uploadLosslessVideoParts(
ctx context.Context,
tctx *ext.Context,
storagePath string,
parts []losslessVideoPart,
sourceCaption *string,
progress *uploadProgress,
) error {
if len(parts) == 0 {
return fmt.Errorf("no lossless video parts to upload")
}
if len(parts) > maxLosslessVideoParts {
return fmt.Errorf(
"refusing to upload %d lossless video parts as multiple albums; maximum is %d",
len(parts),
maxLosslessVideoParts,
)
}
resetLosslessVideoUploadProgress(progress, parts)
prepared := make([]preparedMedia, 0, len(parts))
for index, part := range parts {
partFile, err := os.Open(part.Path)
if err != nil {
return fmt.Errorf("failed to open video part %s: %w", part.Name, err)
}
media, prepareErr := t.prepareMedia(
ctx,
tctx,
partFile,
partStoragePath(storagePath, part.Name),
part.Size,
videoPartCaption(sourceCaption, index),
progress,
)
closeErr := partFile.Close()
if prepareErr != nil {
return fmt.Errorf("failed to prepare video part %s: %w", part.Name, prepareErr)
}
if closeErr != nil {
return fmt.Errorf("failed to close video part %s: %w", part.Name, closeErr)
}
prepared = append(prepared, *media)
}
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
if len(prepared) == 1 {
if _, err := builder.Media(ctx, prepared[0].media); err != nil {
return fmt.Errorf("failed to send video part: %w", err)
}
return nil
}
media := make([]message.MultiMediaOption, len(prepared))
for index := range prepared {
media[index] = prepared[index].media
}
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
return fmt.Errorf("failed to send video parts as album: %w", err)
}
return nil
}
func resetLosslessVideoUploadProgress(progress *uploadProgress, parts []losslessVideoPart) {
if progress == nil {
return
}
var total int64
for _, part := range parts {
total += part.Size
}
progress.reset(total)
}
func videoPartCaption(sourceCaption *string, index int) *string {
if sourceCaption == nil {
return nil
}
if index == 0 {
return sourceCaption
}
empty := ""
return &empty
}

View File

@@ -0,0 +1,195 @@
package telegram
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/gotd/td/telegram/uploader"
)
func TestInitialSegmentDuration(t *testing.T) {
tests := []struct {
name string
duration float64
fileSize int64
targetSize int64
want float64
}{
{name: "two balanced parts", duration: 120, fileSize: 2200, targetSize: 1900, want: 60},
{name: "three balanced parts", duration: 120, fileSize: 3900, targetSize: 1900, want: 40},
{name: "minimum one second", duration: 1.5, fileSize: 3900, targetSize: 1900, want: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := initialSegmentDuration(tt.duration, tt.fileSize, tt.targetSize)
if got != tt.want {
t.Fatalf("initialSegmentDuration() = %v, want %v", got, tt.want)
}
})
}
}
func TestVideoPartExtension(t *testing.T) {
tests := map[string]string{
"clip.MOV": ".mov",
"clip.mkv": ".mkv",
"clip.exe": ".mp4",
"clip": ".mp4",
}
for input, want := range tests {
if got := videoPartExtension(input); got != want {
t.Fatalf("videoPartExtension(%q) = %q, want %q", input, got, want)
}
}
}
func TestPartStoragePath(t *testing.T) {
tests := map[string]string{
"clip.mov": "clip.part001.mov",
"123456/clip.mov": "123456/clip.part001.mov",
"/123456/clip.mov": "/123456/clip.part001.mov",
"folder/sub/clip.mov": "folder/sub/clip.part001.mov",
}
for input, want := range tests {
if got := partStoragePath(input, "clip.part001.mov"); got != want {
t.Fatalf("partStoragePath(%q) = %q, want %q", input, got, want)
}
}
}
func TestVideoPartCaption(t *testing.T) {
if got := videoPartCaption(nil, 0); got != nil {
t.Fatalf("caption without source = %q, want nil", *got)
}
source := "original caption"
first := videoPartCaption(&source, 0)
if first == nil || *first != source {
t.Fatalf("first part caption = %v, want %q", first, source)
}
second := videoPartCaption(&source, 1)
if second == nil || *second != "" {
t.Fatalf("second part caption = %v, want an explicit empty caption", second)
}
}
func TestResetLosslessVideoUploadProgressUsesCombinedPartSize(t *testing.T) {
var uploaded, total int64
progress := newUploadProgress(999, func(current, size int64) {
uploaded = current
total = size
})
resetLosslessVideoUploadProgress(progress, []losslessVideoPart{
{Size: 100},
{Size: 250},
})
if err := progress.Chunk(t.Context(), uploader.ProgressState{ID: 1, Uploaded: 50, Total: 100}); err != nil {
t.Fatalf("Chunk() failed: %v", err)
}
if uploaded != 50 || total != 350 {
t.Fatalf("lossless video progress = %d/%d, want 50/350", uploaded, total)
}
}
func TestSplitLosslessVideoRetriesOversizedPart(t *testing.T) {
tempDir := t.TempDir()
inputPath := filepath.Join(tempDir, "source.mov")
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
t.Fatal(err)
}
outputDir := filepath.Join(tempDir, "parts")
if err := os.Mkdir(outputDir, 0o755); err != nil {
t.Fatal(err)
}
originalRunner := runMediaTool
t.Cleanup(func() { runMediaTool = originalRunner })
ffmpegCalls := 0
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "ffprobe":
return []byte("120.0\n"), nil
case "ffmpeg":
ffmpegCalls++
pattern := args[len(args)-1]
sizes := []int{2001, 199}
if ffmpegCalls > 1 {
sizes = []int{1100, 1100}
}
for index, size := range sizes {
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
if err := os.WriteFile(partPath, make([]byte, size), 0o600); err != nil {
return nil, err
}
}
return nil, nil
default:
return nil, fmt.Errorf("unexpected media tool %q", name)
}
}
parts, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
if err != nil {
t.Fatalf("splitLosslessVideo() failed: %v", err)
}
if ffmpegCalls != 2 {
t.Fatalf("ffmpeg calls = %d, want 2", ffmpegCalls)
}
if len(parts) != 2 {
t.Fatalf("parts = %d, want 2", len(parts))
}
if parts[0].Name != "example.part001.mov" || parts[1].Name != "example.part002.mov" {
t.Fatalf("unexpected part names: %#v", parts)
}
for _, part := range parts {
if part.Size > 2000 {
t.Fatalf("part %s exceeds limit: %d", part.Name, part.Size)
}
}
}
func TestSplitLosslessVideoRejectsMultipleAlbums(t *testing.T) {
tempDir := t.TempDir()
inputPath := filepath.Join(tempDir, "source.mov")
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
t.Fatal(err)
}
outputDir := filepath.Join(tempDir, "parts")
if err := os.Mkdir(outputDir, 0o755); err != nil {
t.Fatal(err)
}
originalRunner := runMediaTool
t.Cleanup(func() { runMediaTool = originalRunner })
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
switch name {
case "ffprobe":
return []byte("120.0\n"), nil
case "ffmpeg":
pattern := args[len(args)-1]
for index := range maxLosslessVideoParts + 1 {
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
if err := os.WriteFile(partPath, make([]byte, 100), 0o600); err != nil {
return nil, err
}
}
return nil, nil
default:
return nil, fmt.Errorf("unexpected media tool %q", name)
}
}
_, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
if err == nil {
t.Fatal("splitLosslessVideo() succeeded with more than one album of parts")
}
if !strings.Contains(err.Error(), "single-album limit") {
t.Fatalf("splitLosslessVideo() error = %q, want single-album limit", err)
}
}

View File

@@ -119,21 +119,21 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
return nil
}
parts := strings.Split(dirPath, "/")
currentPath := ""
var currentPath strings.Builder
for i, part := range parts {
if i > 0 {
currentPath += "/"
currentPath.WriteString("/")
}
currentPath += part
currentPath.WriteString(part)
exists, err := c.Exists(ctx, currentPath)
exists, err := c.Exists(ctx, currentPath.String())
if err != nil {
return err
}
if exists {
continue
}
url := c.BaseURL + currentPath
url := c.BaseURL + currentPath.String()
resp, err := c.doRequest(ctx, WebdavMethodMkcol, url, nil)
if err != nil {
return err
@@ -141,7 +141,7 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("MKCOL %s: %s", currentPath, resp.Status)
return fmt.Errorf("MKCOL %s: %s", currentPath.String(), resp.Status)
}
}
return nil