mirror of
https://github.com/krau/SaveAny-Bot.git
synced 2026-08-14 00:44:01 +08:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a1bd2ed762 | ||
|
|
7e3e26fcf4 | ||
|
|
e49ebef977 | ||
|
|
da38f5bb88 | ||
|
|
0e6ed66ef5 | ||
|
|
e4144e73e6 | ||
|
|
1d794a7b9c | ||
|
|
52f880f0f2 | ||
|
|
fc11ca775f | ||
|
|
056e2fd546 | ||
|
|
c9bb6c9e3c |
@@ -7,6 +7,8 @@ ARG BuildTime="Unknown"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
@@ -31,5 +33,9 @@ FROM scratch
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/saveany-bot .
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
|
||||
ENV SSL_CERT_DIR=/etc/ssl/certs
|
||||
|
||||
ENTRYPOINT ["/app/saveany-bot"]
|
||||
|
||||
@@ -11,23 +11,23 @@ import (
|
||||
// guarded by mu. It implements taskevent.Sink so the task layer can update it
|
||||
// without knowing about the API.
|
||||
type TaskProgressInfo struct {
|
||||
mu sync.Mutex
|
||||
TaskID string
|
||||
Type string
|
||||
Status TaskStatus
|
||||
Title string
|
||||
TotalBytes int64
|
||||
DownloadedBytes int64
|
||||
TotalFiles int
|
||||
DownloadedFiles int
|
||||
Storage string
|
||||
Path string
|
||||
Error string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
StartedAt time.Time
|
||||
Webhook string
|
||||
webhookNotified bool
|
||||
mu sync.Mutex
|
||||
TaskID string
|
||||
Type string
|
||||
Status TaskStatus
|
||||
Title string
|
||||
TotalBytes int64
|
||||
DownloadedBytes int64
|
||||
TotalFiles int
|
||||
DownloadedFiles int
|
||||
Storage string
|
||||
Path string
|
||||
Error string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
StartedAt time.Time
|
||||
Webhook string
|
||||
webhookNotified bool
|
||||
}
|
||||
|
||||
// progressStore holds all API tasks. Entries are removed a fixed duration after
|
||||
@@ -208,9 +208,9 @@ func NewProgressTracker(taskID, taskType, storage, path, title, webhook string)
|
||||
return &ProgressTracker{}
|
||||
}
|
||||
|
||||
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
|
||||
func (p *ProgressTracker) OnStart(totalBytes int64, totalFiles int) {}
|
||||
func (p *ProgressTracker) OnProgress(downloadedBytes int64, downloadedFiles int) {}
|
||||
func (p *ProgressTracker) OnDone(err error) {}
|
||||
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
|
||||
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
|
||||
func (p *ProgressTracker) UpdateProgressFiles(files int) {}
|
||||
func (p *ProgressTracker) OnDone(err error) {}
|
||||
func (p *ProgressTracker) GetInfo() *TaskProgressInfo { return nil }
|
||||
func (p *ProgressTracker) UpdateProgressBytes(bytes int64) {}
|
||||
func (p *ProgressTracker) UpdateProgressFiles(files int) {}
|
||||
|
||||
@@ -60,6 +60,7 @@ func resolveChatID(_ context.Context, idOrUsername string) (int64, error) {
|
||||
}
|
||||
|
||||
// ParseMessageLink 解析 Telegram 消息链接
|
||||
// 支持的域名: t.me, telegram.me
|
||||
// 支持格式:
|
||||
// - https://t.me/username/123
|
||||
// - https://t.me/c/123456789/123
|
||||
@@ -268,5 +269,15 @@ func ExtractFilesFromLinks(ctx context.Context, links []string) ([]tfile.TGFileM
|
||||
|
||||
// isValidMessageLink 检查是否是有效的 Telegram 消息链接
|
||||
func isValidMessageLink(link string) bool {
|
||||
return strings.HasPrefix(link, "https://t.me/") || strings.HasPrefix(link, "http://t.me/")
|
||||
for _, prefix := range []string{
|
||||
"https://t.me/",
|
||||
"http://t.me/",
|
||||
"https://telegram.me/",
|
||||
"http://telegram.me/",
|
||||
} {
|
||||
if strings.HasPrefix(link, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
|
||||
"github.com/krau/SaveAny-Bot/database"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/database"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
"github.com/krau/SaveAny-Bot/storage"
|
||||
|
||||
@@ -3,8 +3,8 @@ package re
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
TgMessageLinkRegexString = `https?://t\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
|
||||
TgMessageLinkRegexString = `https?://(?:t|telegram)\.me/(?:c/\d+|[A-Za-z0-9_]+)/\d+(?:/\d+)?(?:\?[^\s#]*[A-Za-z0-9_])?\b`
|
||||
TgMessageLinkRegexp = regexp.MustCompile(TgMessageLinkRegexString)
|
||||
TelegraphUrlRegexString = `https://telegra.ph/.*`
|
||||
TelegraphUrlRegexString = `https://telegra\.ph/[^\s]+`
|
||||
TelegraphUrlRegexp = regexp.MustCompile(TelegraphUrlRegexString)
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@ package shortcut
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
@@ -179,21 +180,19 @@ type TelegraphResult struct {
|
||||
// return replied message, image urls, telegraph path(unescaped), error
|
||||
func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*types.Message, *TelegraphResult, error) {
|
||||
logger := log.FromContext(ctx)
|
||||
tphurl := re.TelegraphUrlRegexp.FindString(tgutil.ExtractMessageEntityUrlsText(update.EffectiveMessage.Message))
|
||||
tphurl := findTelegraphURL(update.EffectiveMessage.Message)
|
||||
if tphurl == "" {
|
||||
logger.Warnf("No telegraph url found but called handleTelegraph")
|
||||
return nil, nil, dispatcher.ContinueGroups
|
||||
}
|
||||
pagepath := strings.Split(tphurl, "/")[len(strings.Split(tphurl, "/"))-1]
|
||||
tphdir, err := url.PathUnescape(pagepath)
|
||||
pagepath, err := parseTelegraphPagePath(tphurl)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to unescape telegraph path: %s", err)
|
||||
logger.Errorf("Failed to parse telegraph path: %s", err)
|
||||
ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorParseTelegraphPathFailed, map[string]any{
|
||||
"Error": err.Error(),
|
||||
})), nil)
|
||||
return nil, nil, dispatcher.EndGroups
|
||||
}
|
||||
tphdir = strings.TrimSpace(tphdir)
|
||||
msg, err := ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoFetchingTelegraphPage, nil)), nil)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to reply to update: %s", err)
|
||||
@@ -244,7 +243,57 @@ func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*type
|
||||
}
|
||||
return msg, &TelegraphResult{
|
||||
Pics: imgs,
|
||||
TphDir: tphdir,
|
||||
TphDir: pagepath,
|
||||
Page: page,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func findTelegraphURL(msg *tg.Message) string {
|
||||
if msg == nil {
|
||||
return ""
|
||||
}
|
||||
var firstMatch string
|
||||
findValid := func(text string) string {
|
||||
for _, tphurl := range re.TelegraphUrlRegexp.FindAllString(text, -1) {
|
||||
if firstMatch == "" {
|
||||
firstMatch = tphurl
|
||||
}
|
||||
if _, err := parseTelegraphPagePath(tphurl); err == nil {
|
||||
return tphurl
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
for _, entityURL := range tgutil.ExtractMessageEntityUrls(msg) {
|
||||
if tphurl := findValid(entityURL); tphurl != "" {
|
||||
return tphurl
|
||||
}
|
||||
}
|
||||
if tphurl := findValid(msg.GetMessage()); tphurl != "" {
|
||||
return tphurl
|
||||
}
|
||||
return firstMatch
|
||||
}
|
||||
|
||||
func parseTelegraphPagePath(pageURL string) (string, error) {
|
||||
u, err := url.Parse(pageURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid telegraph URL: %w", err)
|
||||
}
|
||||
if u.Scheme != "https" || !strings.EqualFold(u.Hostname(), "telegra.ph") {
|
||||
return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host)
|
||||
}
|
||||
pagepath := strings.Trim(u.EscapedPath(), "/")
|
||||
if pagepath == "" || strings.Contains(pagepath, "/") {
|
||||
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
|
||||
}
|
||||
pagepath, err = url.PathUnescape(pagepath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to unescape telegraph path: %w", err)
|
||||
}
|
||||
pagepath = strings.TrimSpace(pagepath)
|
||||
if pagepath == "" || strings.Contains(pagepath, "/") {
|
||||
return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path)
|
||||
}
|
||||
return pagepath, nil
|
||||
}
|
||||
|
||||
163
client/bot/handlers/utils/shortcut/message_telegraph_test.go
Normal file
163
client/bot/handlers/utils/shortcut/message_telegraph_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package shortcut
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
func TestFindTelegraphURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
msg *tg.Message
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "single URL entity",
|
||||
msg: &tg.Message{
|
||||
Message: "https://telegra.ph/Example-01-02",
|
||||
Entities: []tg.MessageEntityClass{
|
||||
&tg.MessageEntityURL{Offset: 0, Length: 32},
|
||||
},
|
||||
},
|
||||
want: "https://telegra.ph/Example-01-02",
|
||||
},
|
||||
{
|
||||
name: "Telegraph URL before another URL",
|
||||
msg: &tg.Message{
|
||||
Message: "https://telegra.ph/Example-01-02 https://example.com/",
|
||||
Entities: []tg.MessageEntityClass{
|
||||
&tg.MessageEntityURL{Offset: 0, Length: 32},
|
||||
&tg.MessageEntityURL{Offset: 33, Length: 20},
|
||||
},
|
||||
},
|
||||
want: "https://telegra.ph/Example-01-02",
|
||||
},
|
||||
{
|
||||
name: "hidden Telegraph URL",
|
||||
msg: &tg.Message{
|
||||
Message: "article",
|
||||
Entities: []tg.MessageEntityClass{
|
||||
&tg.MessageEntityTextURL{
|
||||
Offset: 0,
|
||||
Length: 7,
|
||||
URL: "https://telegra.ph/Hidden-01-02",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: "https://telegra.ph/Hidden-01-02",
|
||||
},
|
||||
{
|
||||
name: "URL entity after non-BMP character",
|
||||
msg: &tg.Message{
|
||||
Message: "😀 https://telegra.ph/Emoji-01-02",
|
||||
Entities: []tg.MessageEntityClass{
|
||||
&tg.MessageEntityURL{Offset: 3, Length: 30},
|
||||
},
|
||||
},
|
||||
want: "https://telegra.ph/Emoji-01-02",
|
||||
},
|
||||
{
|
||||
name: "valid Telegraph URL after invalid candidate",
|
||||
msg: &tg.Message{
|
||||
Message: "https://telegra.ph/nested/Bad https://telegra.ph/Valid-01-02",
|
||||
Entities: []tg.MessageEntityClass{
|
||||
&tg.MessageEntityURL{Offset: 0, Length: 29},
|
||||
&tg.MessageEntityURL{Offset: 30, Length: 30},
|
||||
},
|
||||
},
|
||||
want: "https://telegra.ph/Valid-01-02",
|
||||
},
|
||||
{
|
||||
name: "plain message fallback",
|
||||
msg: &tg.Message{Message: "read https://telegra.ph/Plain-01-02 now"},
|
||||
want: "https://telegra.ph/Plain-01-02",
|
||||
},
|
||||
{
|
||||
name: "nil message",
|
||||
msg: nil,
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := findTelegraphURL(tt.msg); got != tt.want {
|
||||
t.Fatalf("findTelegraphURL() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTelegraphPagePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pageURL string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "plain path",
|
||||
pageURL: "https://telegra.ph/Example-01-02",
|
||||
want: "Example-01-02",
|
||||
},
|
||||
{
|
||||
name: "escaped path with query and fragment",
|
||||
pageURL: "https://telegra.ph/%E6%B5%8B%E8%AF%95-01-02?source=telegram#top",
|
||||
want: "测试-01-02",
|
||||
},
|
||||
{
|
||||
name: "trailing slash",
|
||||
pageURL: "https://telegra.ph/Example-01-02/",
|
||||
want: "Example-01-02",
|
||||
},
|
||||
{
|
||||
name: "root URL",
|
||||
pageURL: "https://telegra.ph/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong host",
|
||||
pageURL: "https://example.com/Example-01-02",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong scheme",
|
||||
pageURL: "http://telegra.ph/Example-01-02",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid percent escape",
|
||||
pageURL: "https://telegra.ph/Invalid-%zz",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "nested path",
|
||||
pageURL: "https://telegra.ph/nested/Example-01-02",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "encoded slash",
|
||||
pageURL: "https://telegra.ph/nested%2FExample-01-02",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseTelegraphPagePath(tt.pageURL)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("parseTelegraphPagePath(%q) returned no error", tt.pageURL)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseTelegraphPagePath(%q) failed: %v", tt.pageURL, err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("parseTelegraphPagePath(%q) = %q, want %q", tt.pageURL, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -150,28 +150,50 @@ const (
|
||||
BotMsgProgressAria2Downloading Key = "bot.msg.progress.aria2_downloading"
|
||||
BotMsgProgressAria2Start Key = "bot.msg.progress.aria2_start"
|
||||
BotMsgProgressAvgSpeedPrefix Key = "bot.msg.progress.avg_speed_prefix"
|
||||
BotMsgProgressBatchDonePrefix Key = "bot.msg.progress.batch_done_prefix"
|
||||
BotMsgProgressBatchProcessingPrefix Key = "bot.msg.progress.batch_processing_prefix"
|
||||
BotMsgProgressBatchStartPrefix Key = "bot.msg.progress.batch_start_prefix"
|
||||
BotMsgProgressBatchCanceled Key = "bot.msg.progress.batch_canceled"
|
||||
BotMsgProgressBatchDone Key = "bot.msg.progress.batch_done"
|
||||
BotMsgProgressBatchDoneWithSkipped Key = "bot.msg.progress.batch_done_with_skipped"
|
||||
BotMsgProgressBatchFailedGroup Key = "bot.msg.progress.batch_failed_group"
|
||||
BotMsgProgressBatchFailedItem Key = "bot.msg.progress.batch_failed_item"
|
||||
BotMsgProgressBatchFailedTask Key = "bot.msg.progress.batch_failed_task"
|
||||
BotMsgProgressBatchFailureStageBatchUpload Key = "bot.msg.progress.batch_failure_stage_batch_upload"
|
||||
BotMsgProgressBatchFailureStageCache Key = "bot.msg.progress.batch_failure_stage_cache"
|
||||
BotMsgProgressBatchFailureStageConfirm Key = "bot.msg.progress.batch_failure_stage_confirm"
|
||||
BotMsgProgressBatchFailureStageDownload Key = "bot.msg.progress.batch_failure_stage_download"
|
||||
BotMsgProgressBatchFailureStageInternal Key = "bot.msg.progress.batch_failure_stage_internal"
|
||||
BotMsgProgressBatchFailureStageUpload Key = "bot.msg.progress.batch_failure_stage_upload"
|
||||
BotMsgProgressBatchItemConfirming Key = "bot.msg.progress.batch_item_confirming"
|
||||
BotMsgProgressBatchItemDownloading Key = "bot.msg.progress.batch_item_downloading"
|
||||
BotMsgProgressBatchItemDownloadingUnknown Key = "bot.msg.progress.batch_item_downloading_unknown"
|
||||
BotMsgProgressBatchItemRetrying Key = "bot.msg.progress.batch_item_retrying"
|
||||
BotMsgProgressBatchItemTransferring Key = "bot.msg.progress.batch_item_transferring"
|
||||
BotMsgProgressBatchItemTransferringUnknown Key = "bot.msg.progress.batch_item_transferring_unknown"
|
||||
BotMsgProgressBatchItemUploading Key = "bot.msg.progress.batch_item_uploading"
|
||||
BotMsgProgressBatchStatusHeader Key = "bot.msg.progress.batch_status_header"
|
||||
BotMsgProgressBatchSummaryConfirming Key = "bot.msg.progress.batch_summary_confirming"
|
||||
BotMsgProgressBatchSummaryFailed Key = "bot.msg.progress.batch_summary_failed"
|
||||
BotMsgProgressBatchSummaryHiddenActive Key = "bot.msg.progress.batch_summary_hidden_active"
|
||||
BotMsgProgressBatchSummarySkipped Key = "bot.msg.progress.batch_summary_skipped"
|
||||
BotMsgProgressCurrentProgressPrefix Key = "bot.msg.progress.current_progress_prefix"
|
||||
BotMsgProgressCurrentSpeedPrefix Key = "bot.msg.progress.current_speed_prefix"
|
||||
BotMsgProgressDirectDonePrefix Key = "bot.msg.progress.direct_done_prefix"
|
||||
BotMsgProgressDirectStart Key = "bot.msg.progress.direct_start"
|
||||
BotMsgProgressDownloadDonePrefix Key = "bot.msg.progress.download_done_prefix"
|
||||
BotMsgProgressDownloadFailedPrefix Key = "bot.msg.progress.download_failed_prefix"
|
||||
BotMsgProgressDownloadedPrefix Key = "bot.msg.progress.downloaded_prefix"
|
||||
BotMsgProgressDownloadingPrefix Key = "bot.msg.progress.downloading_prefix"
|
||||
BotMsgProgressErrorPrefix Key = "bot.msg.progress.error_prefix"
|
||||
BotMsgProgressFileNamePrefix Key = "bot.msg.progress.file_name_prefix"
|
||||
BotMsgProgressFileProcessingPrefix Key = "bot.msg.progress.file_processing_prefix"
|
||||
BotMsgProgressFileSizePrefix Key = "bot.msg.progress.file_size_prefix"
|
||||
BotMsgProgressFileStartPrefix Key = "bot.msg.progress.file_start_prefix"
|
||||
BotMsgProgressParsedDonePrefix Key = "bot.msg.progress.parsed_done_prefix"
|
||||
BotMsgProgressParsedStartPrefix Key = "bot.msg.progress.parsed_start_prefix"
|
||||
BotMsgProgressProcessingListPrefix Key = "bot.msg.progress.processing_list_prefix"
|
||||
BotMsgProgressProcessingNone Key = "bot.msg.progress.processing_none"
|
||||
BotMsgProgressSavePathPrefix Key = "bot.msg.progress.save_path_prefix"
|
||||
BotMsgProgressTaskCanceled Key = "bot.msg.progress.task_canceled"
|
||||
BotMsgProgressSingleCanceled Key = "bot.msg.progress.single_canceled"
|
||||
BotMsgProgressSingleDone Key = "bot.msg.progress.single_done"
|
||||
BotMsgProgressSingleDownloading Key = "bot.msg.progress.single_downloading"
|
||||
BotMsgProgressSingleDownloadingUnknown Key = "bot.msg.progress.single_downloading_unknown"
|
||||
BotMsgProgressSingleFailed Key = "bot.msg.progress.single_failed"
|
||||
BotMsgProgressSingleStatusHeader Key = "bot.msg.progress.single_status_header"
|
||||
BotMsgProgressSingleUploading Key = "bot.msg.progress.single_uploading"
|
||||
BotMsgProgressSingleUploadRetrying Key = "bot.msg.progress.single_upload_retrying"
|
||||
BotMsgProgressTaskCanceledWithId Key = "bot.msg.progress.task_canceled_with_id"
|
||||
BotMsgProgressTaskFailedWithError Key = "bot.msg.progress.task_failed_with_error"
|
||||
BotMsgProgressTelegraphDonePrefix Key = "bot.msg.progress.telegraph_done_prefix"
|
||||
|
||||
@@ -351,32 +351,54 @@ bot:
|
||||
info_filename_prefix: "Filename: "
|
||||
info_prompt_select_storage: "\nPlease select storage"
|
||||
progress:
|
||||
batch_start_prefix: "Starting batch download task\nTotal size: "
|
||||
batch_processing_prefix: "Processing batch download task\nTotal size: "
|
||||
batch_status_header: "<b>📦 Processing</b>\n\nFiles: <code>{{.Total}}</code>\nStatus: ✅ <code>{{.Completed}}</code> | 📥 <code>{{.Downloaded}}</code> | ⏳ <code>{{.Waiting}}</code>\nTotal speed: ⬇️ <code>{{.DownloadSpeed}}</code> | ⬆️ <code>{{.UploadSpeed}}</code>"
|
||||
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
|
||||
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} Transferring</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown</blockquote>"
|
||||
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nRetry: <code>{{.Attempt}}/{{.Limit}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nReason: <code>{{.Reason}}</code></blockquote>"
|
||||
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} Waiting</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>Uploaded, awaiting remote confirmation</i></blockquote>"
|
||||
batch_summary_hidden_active: "🔄 <code>{{.Count}}</code> more files are active"
|
||||
batch_summary_confirming: "☁️ Uploaded, awaiting group send: <code>{{.Count}}</code>"
|
||||
batch_summary_failed: "❌ Failed: <code>{{.Count}}</code>"
|
||||
batch_summary_skipped: "⏭️ Skipped: <code>{{.Count}}</code>"
|
||||
batch_done: "<b>✅ Completed</b>\n\nFiles: <code>{{.Count}}</code>\nTotal size: <code>{{.Size}}</code>"
|
||||
batch_done_with_skipped: "<b>⚠️ Completed</b>\n\nSucceeded: <code>{{.Success}}</code>\nSkipped: <code>{{.Skipped}}</code>\nTotal size: <code>{{.Size}}</code>"
|
||||
batch_canceled: "<b>🚫 Task canceled</b>\n\nFiles: <code>{{.Total}}</code>\nCompleted: <code>{{.Completed}}</code>\nIncomplete: <code>{{.Incomplete}}</code>\nSkipped: <code>{{.Skipped}}</code>"
|
||||
batch_failed_item: "<b>❌ Processing failed</b>\n\nFailed file: <code>{{.Index}}. {{.Name}}</code>\nStage: <code>{{.Stage}}</code>\nProgress: <code>{{.Progress}}</code>\nSpeed before failure: <code>{{.Speed}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Failed: <code>{{.Failed}}</code>\n⏹️ Incomplete: <code>{{.Incomplete}}</code>"
|
||||
batch_failed_group: "<b>❌ Batch upload failed</b>\n\nAffected files: <code>{{.Affected}}</code>\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n❌ Batch failed: <code>{{.Failed}}</code>\n⏹️ Incomplete: <code>{{.Incomplete}}</code>"
|
||||
batch_failed_task: "<b>❌ Processing failed</b>\n\nReason: <code>{{.Reason}}</code>\n\n✅ Completed: <code>{{.Completed}}</code>\n⏹️ Incomplete: <code>{{.Incomplete}}</code>"
|
||||
batch_failure_stage_download: "download"
|
||||
batch_failure_stage_cache: "local cache"
|
||||
batch_failure_stage_upload: "upload"
|
||||
batch_failure_stage_confirm: "remote confirmation"
|
||||
batch_failure_stage_batch_upload: "batch upload"
|
||||
batch_failure_stage_internal: "internal task"
|
||||
single_status_header: "<b>📦 Processing</b>"
|
||||
single_downloading: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
|
||||
single_downloading_unknown: "<blockquote><b>⬇️ Downloading</b>\n<code>{{.Name}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / unknown\nSave to: <code>{{.Destination}}</code></blockquote>"
|
||||
single_uploading: "<blockquote><b>⬆️ Uploading</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
|
||||
single_upload_retrying: "<blockquote><b>🔁 Retrying upload</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\nAttempt: <code>{{.Attempt}}</code>\nSpeed: <code>{{.Speed}}</code>\nSize: <code>{{.Current}}</code> / <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code></blockquote>"
|
||||
single_done: "<b>✅ Completed</b>\n\nFilename: <code>{{.Name}}</code>\nTotal size: <code>{{.Size}}</code>\nSave to: <code>{{.Destination}}</code>"
|
||||
single_canceled: "<b>🚫 Task canceled</b>\n\nFilename: <code>{{.Name}}</code>"
|
||||
single_failed: "<b>❌ Processing failed</b>\n\nFilename: <code>{{.Name}}</code>\nReason: <code>{{.Reason}}</code>"
|
||||
downloading_prefix: "Downloading\nTotal size: "
|
||||
processing_list_prefix: "\nProcessing:\n"
|
||||
processing_none: " - None"
|
||||
avg_speed_prefix: "\nAverage speed: "
|
||||
current_progress_prefix: "\nCurrent progress: "
|
||||
task_canceled: "Task canceled"
|
||||
task_canceled_with_id: "Processing canceled: {{.TaskID}}"
|
||||
task_failed_with_error: "Processing failed: {{.Error}}"
|
||||
batch_done_prefix: "Completed\nFile count: "
|
||||
direct_done_prefix: "Completed, file count: "
|
||||
parsed_start_prefix: "Starting download from {{.Site}}\nTotal size: "
|
||||
parsed_done_prefix: "Completed, resource count: "
|
||||
telegraph_start_prefix: "Starting Telegraph download\nImage count: "
|
||||
telegraph_progress_prefix: "Downloading\nCurrent progress: "
|
||||
telegraph_done_prefix: "Completed\nImage count: "
|
||||
file_start_prefix: "Starting download\nFilename: "
|
||||
file_processing_prefix: "Processing download task\nFilename: "
|
||||
download_failed_prefix: "Download failed\nFilename: "
|
||||
download_done_prefix: "Download completed\nFilename: "
|
||||
file_size_prefix: "\nFile size: "
|
||||
save_path_prefix: "\nSave path: "
|
||||
total_size_prefix: "\nTotal size: "
|
||||
direct_start: "Starting download, total size: {{.SizeMB}} MB ({{.Count}} files)"
|
||||
file_name_prefix: "Filename: "
|
||||
error_prefix: "\nError: "
|
||||
aria2_start: "Waiting for Aria2 to complete download (GID: {{.GID}})..."
|
||||
aria2_downloading: "Aria2 downloading (GID: {{.GID}})\n"
|
||||
|
||||
@@ -352,32 +352,54 @@ bot:
|
||||
info_filename_prefix: "文件名: "
|
||||
info_prompt_select_storage: "\n请选择存储位置"
|
||||
progress:
|
||||
batch_start_prefix: "开始执行批量下载任务\n总大小: "
|
||||
batch_processing_prefix: "正在处理批量下载任务\n总大小: "
|
||||
batch_status_header: "<b>📦 正在处理</b>\n\n文件:<code>{{.Total}}</code>\n状态:✅ <code>{{.Completed}}</code> | 📥 <code>{{.Downloaded}}</code> | ⏳ <code>{{.Waiting}}</code>\n总速度:⬇️ <code>{{.DownloadSpeed}}</code> | ⬆️ <code>{{.UploadSpeed}}</code>"
|
||||
batch_item_downloading: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_downloading_unknown: "<blockquote><b>⬇️ {{.Index}}/{{.Total}} 下载中</b>\n<code>{{.Name}}</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / 未知</blockquote>"
|
||||
batch_item_transferring: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_transferring_unknown: "<blockquote><b>↕️ {{.Index}}/{{.Total}} 传输中</b>\n<code>{{.Name}}</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / 未知</blockquote>"
|
||||
batch_item_uploading: "<blockquote><b>⬆️ {{.Index}}/{{.Total}} 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code></blockquote>"
|
||||
batch_item_retrying: "<blockquote><b>🔁 {{.Index}}/{{.Total}} 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n重试次数:<code>{{.Attempt}}/{{.Limit}}</code>\n失败前速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code>\n原因:<code>{{.Reason}}</code></blockquote>"
|
||||
batch_item_confirming: "<blockquote><b>⏳ {{.Index}}/{{.Total}} 等待中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n<i>文件已上传,正在等待远端确认</i></blockquote>"
|
||||
batch_summary_hidden_active: "🔄 另有 <code>{{.Count}}</code> 个文件正在处理"
|
||||
batch_summary_confirming: "☁️ 已上传,等待整组发送:<code>{{.Count}}</code>"
|
||||
batch_summary_failed: "❌ 失败:<code>{{.Count}}</code>"
|
||||
batch_summary_skipped: "⏭️ 已跳过:<code>{{.Count}}</code>"
|
||||
batch_done: "<b>✅ 处理完成</b>\n\n文件数: <code>{{.Count}}</code>\n总大小: <code>{{.Size}}</code>"
|
||||
batch_done_with_skipped: "<b>⚠️ 处理完成</b>\n\n成功: <code>{{.Success}}</code>\n已跳过: <code>{{.Skipped}}</code>\n总大小: <code>{{.Size}}</code>"
|
||||
batch_canceled: "<b>🚫 任务已取消</b>\n\n文件数: <code>{{.Total}}</code>\n已完成: <code>{{.Completed}}</code>\n未完成: <code>{{.Incomplete}}</code>\n已跳过: <code>{{.Skipped}}</code>"
|
||||
batch_failed_item: "<b>❌ 处理失败</b>\n\n失败文件: <code>{{.Index}}. {{.Name}}</code>\n失败阶段: <code>{{.Stage}}</code>\n失败进度: <code>{{.Progress}}</code>\n失败前速度: <code>{{.Speed}}</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 失败: <code>{{.Failed}}</code>\n⏹️ 未完成: <code>{{.Incomplete}}</code>"
|
||||
batch_failed_group: "<b>❌ 批量上传失败</b>\n\n受影响文件: <code>{{.Affected}} 个</code>\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n❌ 批次失败: <code>{{.Failed}}</code>\n⏹️ 未完成: <code>{{.Incomplete}}</code>"
|
||||
batch_failed_task: "<b>❌ 处理失败</b>\n\n原因: <code>{{.Reason}}</code>\n\n✅ 已完成: <code>{{.Completed}}</code>\n⏹️ 未完成: <code>{{.Incomplete}}</code>"
|
||||
batch_failure_stage_download: "下载"
|
||||
batch_failure_stage_cache: "本地缓存"
|
||||
batch_failure_stage_upload: "上传"
|
||||
batch_failure_stage_confirm: "云端确认"
|
||||
batch_failure_stage_batch_upload: "批量上传"
|
||||
batch_failure_stage_internal: "任务内部"
|
||||
single_status_header: "<b>📦 正在处理</b>"
|
||||
single_downloading: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至:<code>{{.Destination}}</code></blockquote>"
|
||||
single_downloading_unknown: "<blockquote><b>⬇️ 下载中</b>\n<code>{{.Name}}</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / 未知\n保存至:<code>{{.Destination}}</code></blockquote>"
|
||||
single_uploading: "<blockquote><b>⬆️ 上传中</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至:<code>{{.Destination}}</code></blockquote>"
|
||||
single_upload_retrying: "<blockquote><b>🔁 上传重试</b>\n<code>{{.Name}}</code>\n{{.Bar}} <code>{{.Progress}}%</code>\n尝试次数:<code>{{.Attempt}}</code>\n速度:<code>{{.Speed}}</code>\n大小:<code>{{.Current}}</code> / <code>{{.Size}}</code>\n保存至:<code>{{.Destination}}</code></blockquote>"
|
||||
single_done: "<b>✅ 处理完成</b>\n\n文件名:<code>{{.Name}}</code>\n总大小:<code>{{.Size}}</code>\n保存至:<code>{{.Destination}}</code>"
|
||||
single_canceled: "<b>🚫 任务已取消</b>\n\n文件名:<code>{{.Name}}</code>"
|
||||
single_failed: "<b>❌ 处理失败</b>\n\n文件名:<code>{{.Name}}</code>\n原因:<code>{{.Reason}}</code>"
|
||||
downloading_prefix: "正在下载\n总大小: "
|
||||
processing_list_prefix: "\n正在处理:\n"
|
||||
processing_none: " - 无"
|
||||
avg_speed_prefix: "\n平均速度: "
|
||||
current_progress_prefix: "\n当前进度: "
|
||||
task_canceled: "任务已取消"
|
||||
task_canceled_with_id: "处理已取消: {{.TaskID}}"
|
||||
task_failed_with_error: "处理失败: {{.Error}}"
|
||||
batch_done_prefix: "处理完成\n文件数: "
|
||||
direct_done_prefix: "处理完成, 文件数量: "
|
||||
parsed_start_prefix: "开始下载 {{.Site}} 的资源\n总大小: "
|
||||
parsed_done_prefix: "处理完成, 资源数量: "
|
||||
telegraph_start_prefix: "开始下载Telegraph\n图片数量: "
|
||||
telegraph_progress_prefix: "正在下载\n当前进度: "
|
||||
telegraph_done_prefix: "处理完成\n图片数量: "
|
||||
file_start_prefix: "开始下载\n文件名: "
|
||||
file_processing_prefix: "正在处理下载任务\n文件名: "
|
||||
download_failed_prefix: "下载失败\n文件名: "
|
||||
download_done_prefix: "下载完成\n文件名: "
|
||||
file_size_prefix: "\n文件大小: "
|
||||
save_path_prefix: "\n保存路径: "
|
||||
total_size_prefix: "\n总大小: "
|
||||
direct_start: "开始下载, 总大小: {{.SizeMB}} MB ({{.Count}} 个文件)"
|
||||
file_name_prefix: "文件名: "
|
||||
error_prefix: "\n错误: "
|
||||
aria2_start: "等待 Aria2 下载完成 (GID: {{.GID}})..."
|
||||
aria2_downloading: "Aria2 正在下载 (GID: {{.GID}})\n"
|
||||
|
||||
42
common/utils/fsutil/file_test.go
Normal file
42
common/utils/fsutil/file_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package fsutil_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
)
|
||||
|
||||
func TestCloseAndRemove(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
preClose bool
|
||||
}{
|
||||
{name: "open file"},
|
||||
{name: "already closed file", preClose: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filePath := filepath.Join(t.TempDir(), "cache-file")
|
||||
file, err := fsutil.CreateFile(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateFile() failed: %v", err)
|
||||
}
|
||||
if tt.preClose {
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatalf("Close() failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := file.CloseAndRemove(); err != nil {
|
||||
t.Fatalf("CloseAndRemove() failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("cache file still exists after CloseAndRemove(): %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package fsutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -41,10 +42,11 @@ func (f *File) Remove() error {
|
||||
}
|
||||
|
||||
func (f *File) CloseAndRemove() error {
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
closeErr := f.Close()
|
||||
if errors.Is(closeErr, os.ErrClosed) {
|
||||
closeErr = nil
|
||||
}
|
||||
return f.Remove()
|
||||
return errors.Join(closeErr, f.Remove())
|
||||
}
|
||||
|
||||
func CreateFile(fp string) (*File, error) {
|
||||
|
||||
@@ -17,7 +17,11 @@ type ProgressReadSeeker struct {
|
||||
|
||||
// Seek implements io.ReadSeeker.
|
||||
func (pr *ProgressReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return pr.reader.Seek(offset, whence)
|
||||
position, err := pr.reader.Seek(offset, whence)
|
||||
if err == nil {
|
||||
pr.read.Store(position)
|
||||
}
|
||||
return position, err
|
||||
}
|
||||
|
||||
// NewProgressReader creates a new ProgressReader
|
||||
@@ -54,7 +58,7 @@ func (pr *ProgressReadSeeker) Progress() float64 {
|
||||
return float64(pr.read.Load()) / float64(pr.total.Load())
|
||||
}
|
||||
|
||||
// Read returns the number of bytes read so far
|
||||
// BytesRead returns the current tracked reader position.
|
||||
func (pr *ProgressReadSeeker) BytesRead() int64 {
|
||||
return pr.read.Load()
|
||||
}
|
||||
|
||||
50
common/utils/ioutil/progress_reader_test.go
Normal file
50
common/utils/ioutil/progress_reader_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package ioutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProgressReadSeekerTracksReads(t *testing.T) {
|
||||
var gotRead, gotTotal int64
|
||||
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, func(read, total int64) {
|
||||
gotRead = read
|
||||
gotTotal = total
|
||||
})
|
||||
|
||||
buffer := make([]byte, 4)
|
||||
if _, err := io.ReadFull(reader, buffer); err != nil {
|
||||
t.Fatalf("read failed: %v", err)
|
||||
}
|
||||
|
||||
if gotRead != 4 || gotTotal != 6 {
|
||||
t.Fatalf("progress = %d/%d, want 4/6", gotRead, gotTotal)
|
||||
}
|
||||
if reader.BytesRead() != 4 {
|
||||
t.Fatalf("BytesRead() = %d, want 4", reader.BytesRead())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressReadSeekerResetsPositionOnSeek(t *testing.T) {
|
||||
reader := NewProgressReader(bytes.NewReader([]byte("abcdef")), 6, nil)
|
||||
buffer := make([]byte, 4)
|
||||
if _, err := io.ReadFull(reader, buffer); err != nil {
|
||||
t.Fatalf("read failed: %v", err)
|
||||
}
|
||||
|
||||
position, err := reader.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
t.Fatalf("seek failed: %v", err)
|
||||
}
|
||||
if position != 0 || reader.BytesRead() != 0 {
|
||||
t.Fatalf("position after seek = %d (tracked %d), want 0", position, reader.BytesRead())
|
||||
}
|
||||
|
||||
if _, err := io.ReadFull(reader, buffer[:2]); err != nil {
|
||||
t.Fatalf("read after seek failed: %v", err)
|
||||
}
|
||||
if reader.BytesRead() != 2 {
|
||||
t.Fatalf("BytesRead() after seek and read = %d, want 2", reader.BytesRead())
|
||||
}
|
||||
}
|
||||
35
common/utils/tgutil/html.go
Normal file
35
common/utils/tgutil/html.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package tgutil
|
||||
|
||||
import (
|
||||
stdhtml "html"
|
||||
|
||||
"github.com/gotd/td/telegram/message/entity"
|
||||
messagehtml "github.com/gotd/td/telegram/message/html"
|
||||
"github.com/gotd/td/telegram/message/styling"
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
// EscapeHTMLTemplateData returns a copy of data with string values escaped for
|
||||
// interpolation into Telegram HTML templates.
|
||||
func EscapeHTMLTemplateData(data map[string]any) map[string]any {
|
||||
escaped := make(map[string]any, len(data))
|
||||
for key, value := range data {
|
||||
if text, ok := value.(string); ok {
|
||||
escaped[key] = stdhtml.EscapeString(text)
|
||||
continue
|
||||
}
|
||||
escaped[key] = value
|
||||
}
|
||||
return escaped
|
||||
}
|
||||
|
||||
// RenderHTML renders Telegram-compatible HTML into plain text and message
|
||||
// entities.
|
||||
func RenderHTML(markup string) (string, []tg.MessageEntityClass, error) {
|
||||
var builder entity.Builder
|
||||
if err := styling.Perform(&builder, messagehtml.String(nil, markup)); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
text, entities := builder.Complete()
|
||||
return text, entities, nil
|
||||
}
|
||||
54
common/utils/tgutil/html_test.go
Normal file
54
common/utils/tgutil/html_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package tgutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
func TestEscapeHTMLTemplateDataDoesNotMutateInput(t *testing.T) {
|
||||
input := map[string]any{
|
||||
"Text": `<b>A&B</b>`,
|
||||
"Count": 2,
|
||||
}
|
||||
escaped := EscapeHTMLTemplateData(input)
|
||||
|
||||
if got, want := escaped["Text"], "<b>A&B</b>"; got != want {
|
||||
t.Fatalf("escaped text = %q, want %q", got, want)
|
||||
}
|
||||
if got := input["Text"]; got != `<b>A&B</b>` {
|
||||
t.Fatalf("input was mutated: %q", got)
|
||||
}
|
||||
if got := escaped["Count"]; got != 2 {
|
||||
t.Fatalf("non-string value = %v, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderHTMLUsesTemplateStylesAndDecodesValues(t *testing.T) {
|
||||
data := EscapeHTMLTemplateData(map[string]any{"Name": `<b>A&B</b>.bin`})
|
||||
markup := `<blockquote><b>Uploading</b>
|
||||
<code>` + data["Name"].(string) + `</code></blockquote>`
|
||||
|
||||
text, entities, err := RenderHTML(markup)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderHTML() failed: %v", err)
|
||||
}
|
||||
if want := "Uploading\n<b>A&B</b>.bin"; text != want {
|
||||
t.Fatalf("rendered text = %q, want %q", text, want)
|
||||
}
|
||||
|
||||
var bold, code, blockquote int
|
||||
for _, messageEntity := range entities {
|
||||
switch messageEntity.(type) {
|
||||
case *tg.MessageEntityBold:
|
||||
bold++
|
||||
case *tg.MessageEntityCode:
|
||||
code++
|
||||
case *tg.MessageEntityBlockquote:
|
||||
blockquote++
|
||||
}
|
||||
}
|
||||
if bold != 1 || code != 1 || blockquote != 1 {
|
||||
t.Fatalf("entity counts = bold:%d code:%d blockquote:%d", bold, code, blockquote)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package tgutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
@@ -359,9 +360,16 @@ func GetGroupedMessages(ctx *ext.Context, chatID int64, msg *tg.Message) ([]*tg.
|
||||
groupedMessages = append(groupedMessages, m)
|
||||
}
|
||||
}
|
||||
sortMessagesByID(groupedMessages)
|
||||
return groupedMessages, nil
|
||||
}
|
||||
|
||||
func sortMessagesByID(messages []*tg.Message) {
|
||||
sort.Slice(messages, func(i, j int) bool {
|
||||
return messages[i].GetID() < messages[j].GetID()
|
||||
})
|
||||
}
|
||||
|
||||
func ExtractMessageEntityUrls(msg *tg.Message) []string {
|
||||
if len(msg.Entities) == 0 {
|
||||
return nil
|
||||
|
||||
18
common/utils/tgutil/message_test.go
Normal file
18
common/utils/tgutil/message_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package tgutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
func TestSortMessagesByID(t *testing.T) {
|
||||
messages := []*tg.Message{{ID: 9}, {ID: 3}, {ID: 7}}
|
||||
sortMessagesByID(messages)
|
||||
want := []int{3, 7, 9}
|
||||
for i := range messages {
|
||||
if messages[i].GetID() != want[i] {
|
||||
t.Fatalf("message %d has ID %d, want %d", i, messages[i].GetID(), want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,14 +8,16 @@ import (
|
||||
|
||||
type TelegramStorageConfig struct {
|
||||
BaseConfig
|
||||
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
|
||||
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
|
||||
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
|
||||
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
|
||||
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
|
||||
// split files larger than Telegram limit(2GB) into parts of specified size, in MB, leave 0 to set default(2000MB)
|
||||
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
|
||||
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
|
||||
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
|
||||
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
|
||||
SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB)
|
||||
SplitLargeVideo bool `toml:"split_large_video" mapstructure:"split_large_video" json:"split_large_video"`
|
||||
// split files larger than the uploader account limit into parts of specified size, in MB
|
||||
// leave 0 to use the account limit (2000MB for bots/regular users, 4000MB for Premium users)
|
||||
// only effective when SkipLarge is false
|
||||
// use zip when splitting
|
||||
// use zip when splitting non-video files or when lossless video splitting is disabled/unavailable
|
||||
SplitSizeMB int64 `toml:"split_size_mb" mapstructure:"split_size_mb" json:"split_size_mb"`
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
@@ -14,45 +15,269 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
"github.com/krau/SaveAny-Bot/pkg/taskevent"
|
||||
"github.com/krau/SaveAny-Bot/storage"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type executionGroup struct {
|
||||
elems []*TaskElement
|
||||
batchSaver storage.StorageBatchSaver
|
||||
}
|
||||
|
||||
func (g executionGroup) usesBatchSaver() bool {
|
||||
return g.batchSaver != nil
|
||||
}
|
||||
|
||||
func (t *Task) Execute(ctx context.Context) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("batch_file[%s]", t.ID))
|
||||
logger.Info("Starting batch file task")
|
||||
t.Progress.OnStart(ctx, t)
|
||||
workers := config.C().Workers
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(workers)
|
||||
for _, elem := range t.elems {
|
||||
eg.Go(func() error {
|
||||
t.processingMu.RLock()
|
||||
if t.processing[elem.ID] != nil {
|
||||
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
|
||||
groups := t.executionGroups()
|
||||
var err error
|
||||
for i := 0; i < len(groups); {
|
||||
if groups[i].usesBatchSaver() {
|
||||
err = t.processBatch(ctx, groups[i])
|
||||
i++
|
||||
} else {
|
||||
end := i + 1
|
||||
for end < len(groups) && !groups[end].usesBatchSaver() {
|
||||
end++
|
||||
}
|
||||
t.processingMu.RUnlock()
|
||||
t.processingMu.Lock()
|
||||
t.processing[elem.ID] = &elem
|
||||
t.processingMu.Unlock()
|
||||
defer func() {
|
||||
t.processingMu.Lock()
|
||||
delete(t.processing, elem.ID)
|
||||
t.processingMu.Unlock()
|
||||
}()
|
||||
return t.processElement(gctx, elem)
|
||||
})
|
||||
elems := make([]*TaskElement, 0, end-i)
|
||||
for _, group := range groups[i:end] {
|
||||
elems = append(elems, group.elems...)
|
||||
}
|
||||
err = t.processElements(ctx, elems)
|
||||
i = end
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
err := eg.Wait()
|
||||
if err != nil {
|
||||
logger.Errorf("Error during batch file processing: %v", err)
|
||||
} else {
|
||||
logger.Info("Batch file task completed successfully")
|
||||
}
|
||||
t.finishItems(err)
|
||||
t.Progress.OnDone(ctx, t, err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *Task) executionGroups() []executionGroup {
|
||||
groups := make([]executionGroup, 0, len(t.elems))
|
||||
for i := 0; i < len(t.elems); {
|
||||
elem := &t.elems[i]
|
||||
batchSaver, batchCapable := elem.Storage.(storage.StorageBatchSaver)
|
||||
if !batchCapable || elem.sourceGroupKey == "" {
|
||||
groups = append(groups, executionGroup{elems: []*TaskElement{elem}})
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
end := i + 1
|
||||
for end < len(t.elems) {
|
||||
next := &t.elems[end]
|
||||
if next.Storage != elem.Storage || next.sourceGroupKey != elem.sourceGroupKey {
|
||||
break
|
||||
}
|
||||
end++
|
||||
}
|
||||
elems := make([]*TaskElement, 0, end-i)
|
||||
for j := i; j < end; j++ {
|
||||
elems = append(elems, &t.elems[j])
|
||||
}
|
||||
groups = append(groups, executionGroup{elems: elems, batchSaver: batchSaver})
|
||||
i = end
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error {
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
for _, elem := range elems {
|
||||
eg.Go(func() error {
|
||||
if err := t.markProcessing(ctx, elem); err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.unmarkProcessing(elem.ID)
|
||||
return t.processElement(gctx, *elem)
|
||||
})
|
||||
}
|
||||
return eg.Wait()
|
||||
}
|
||||
|
||||
func (t *Task) processBatch(ctx context.Context, group executionGroup) error {
|
||||
defer func() {
|
||||
for _, elem := range group.elems {
|
||||
if err := os.Remove(elem.localPath); err != nil && !os.IsNotExist(err) {
|
||||
log.FromContext(ctx).Warnf("Failed to cleanup batch cache file %s: %v", elem.localPath, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
for _, elem := range group.elems {
|
||||
eg.Go(func() error {
|
||||
if err := t.markProcessing(ctx, elem); err != nil {
|
||||
return err
|
||||
}
|
||||
defer t.unmarkProcessing(elem.ID)
|
||||
return t.downloadElement(gctx, elem)
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
items := make([]storagetypes.BatchItem, 0, len(group.elems))
|
||||
openFiles := make([]*os.File, 0, len(group.elems))
|
||||
defer func() {
|
||||
for _, file := range openFiles {
|
||||
if err := file.Close(); err != nil {
|
||||
log.FromContext(ctx).Warnf("Failed to close batch cache file %s: %v", file.Name(), err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, elem := range group.elems {
|
||||
file, err := os.Open(elem.localPath)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageCache, err)
|
||||
return fmt.Errorf("failed to open cache file: %w", err)
|
||||
}
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
t.markItemFailed(elem.ID, FailureStageCache, err)
|
||||
return fmt.Errorf("failed to get cache file stat: %w", err)
|
||||
}
|
||||
openFiles = append(openFiles, file)
|
||||
items = append(items, storagetypes.BatchItem{
|
||||
Reader: file,
|
||||
StoragePath: elem.Path,
|
||||
Size: stat.Size(),
|
||||
SourceGroupKey: elem.sourceGroupKey,
|
||||
Caption: elem.sourceCaption,
|
||||
PreserveCaption: elem.preserveCaption,
|
||||
})
|
||||
}
|
||||
for index, item := range items {
|
||||
t.recordDownloadComplete(group.elems[index].ID, item.Size)
|
||||
}
|
||||
return t.saveBatchItems(ctx, group, items)
|
||||
}
|
||||
|
||||
func (t *Task) saveBatchItems(ctx context.Context, group executionGroup, items []storagetypes.BatchItem) error {
|
||||
t.startUpload(ctx)
|
||||
if progressSaver, ok := group.batchSaver.(storage.StorageBatchProgressSaver); ok {
|
||||
err := progressSaver.SaveBatchWithProgress(ctx, items, func(index int, uploaded, total int64) {
|
||||
if index < 0 || index >= len(group.elems) {
|
||||
return
|
||||
}
|
||||
t.uploadCallback(ctx, group.elems[index].ID)(uploaded, total)
|
||||
})
|
||||
if err != nil {
|
||||
for _, elem := range group.elems {
|
||||
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
|
||||
}
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to save batch: %w", err)
|
||||
}
|
||||
for index, elem := range group.elems {
|
||||
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
|
||||
t.markItemCompleted(elem.ID)
|
||||
}
|
||||
t.notifyStateChange(ctx)
|
||||
return nil
|
||||
}
|
||||
for i := range items {
|
||||
items[i].Reader = ioutil.NewProgressReader(
|
||||
items[i].Reader,
|
||||
items[i].Size,
|
||||
t.uploadCallback(ctx, group.elems[i].ID),
|
||||
)
|
||||
}
|
||||
if err := group.batchSaver.SaveBatch(ctx, items); err != nil {
|
||||
for _, elem := range group.elems {
|
||||
t.markItemFailed(elem.ID, FailureStageBatchUpload, err)
|
||||
}
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to save batch: %w", err)
|
||||
}
|
||||
for index, elem := range group.elems {
|
||||
t.uploadCallback(ctx, elem.ID)(items[index].Size, items[index].Size)
|
||||
t.markItemCompleted(elem.ID)
|
||||
}
|
||||
t.notifyStateChange(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) markProcessing(ctx context.Context, elem *TaskElement) error {
|
||||
t.processingMu.Lock()
|
||||
if t.processing[elem.ID] != nil {
|
||||
t.processingMu.Unlock()
|
||||
return fmt.Errorf("element with ID %s is already being processed", elem.ID)
|
||||
}
|
||||
t.processing[elem.ID] = elem
|
||||
t.processingMu.Unlock()
|
||||
t.markItemActive(elem.ID, elem.stream, time.Now())
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) unmarkProcessing(id string) {
|
||||
t.processingMu.Lock()
|
||||
delete(t.processing, id)
|
||||
t.processingMu.Unlock()
|
||||
}
|
||||
|
||||
func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
|
||||
logger.Info("Starting file download")
|
||||
localFile, err := fsutil.CreateFile(elem.localPath)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageCache, err)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to create local file: %w", err)
|
||||
}
|
||||
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
|
||||
t.recordItemDownload(elem.ID, int64(n), time.Now())
|
||||
downloaded := t.downloaded.Add(int64(n))
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
taskevent.Emit(ctx, taskevent.Event{
|
||||
TaskID: t.ID,
|
||||
Phase: taskevent.PhaseProgress,
|
||||
TotalBytes: t.totalSize,
|
||||
DownloadedBytes: downloaded,
|
||||
})
|
||||
})
|
||||
_, downloadErr := tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
|
||||
closeErr := localFile.Close()
|
||||
if downloadErr != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageDownload, downloadErr)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to download file: %w", downloadErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageCache, closeErr)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to close cache file: %w", closeErr)
|
||||
}
|
||||
logger.Info("File downloaded successfully")
|
||||
if path.Ext(elem.FileName()) == "" {
|
||||
if ext := fsutil.DetectFileExt(elem.localPath); ext != "" {
|
||||
elem.Path += ext
|
||||
}
|
||||
}
|
||||
t.markItemDownloaded(elem.ID)
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name()))
|
||||
if elem.stream {
|
||||
@@ -60,9 +285,15 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
defer pr.Close()
|
||||
errg, uploadCtx := errgroup.WithContext(ctx)
|
||||
errg.Go(func() error {
|
||||
return elem.Storage.Save(uploadCtx, pr, elem.Path)
|
||||
err := elem.Storage.Save(uploadCtx, pr, elem.Path)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageUpload, err)
|
||||
t.notifyStateChange(ctx)
|
||||
}
|
||||
return err
|
||||
})
|
||||
wr := ioutil.NewProgressWriter(pw, func(n int) {
|
||||
t.recordItemDownload(elem.ID, int64(n), time.Now())
|
||||
downloaded := t.downloaded.Add(int64(n))
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
taskevent.Emit(ctx, taskevent.Event{
|
||||
@@ -78,6 +309,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
_, err := tdler.NewDownloader(elem.File).Stream(uploadCtx, wr)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to download file: %v", err)
|
||||
t.markItemFailed(elem.ID, FailureStageDownload, err)
|
||||
t.notifyStateChange(ctx)
|
||||
pw.CloseWithError(err)
|
||||
}
|
||||
return err
|
||||
@@ -85,12 +318,17 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
if err := errg.Wait(); err != nil {
|
||||
return fmt.Errorf("failed to download file in stream mode: %w", err)
|
||||
}
|
||||
t.recordDownloadComplete(elem.ID, 0)
|
||||
t.markItemCompleted(elem.ID)
|
||||
t.notifyStateChange(ctx)
|
||||
logger.Info("File downloaded successfully in stream mode")
|
||||
return nil
|
||||
}
|
||||
logger.Info("Starting file download")
|
||||
localFile, err := fsutil.CreateFile(elem.localPath)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageCache, err)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to create local file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
@@ -99,6 +337,7 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
}
|
||||
}()
|
||||
wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) {
|
||||
t.recordItemDownload(elem.ID, int64(n), time.Now())
|
||||
downloaded := t.downloaded.Add(int64(n))
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
taskevent.Emit(ctx, taskevent.Event{
|
||||
@@ -110,6 +349,8 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
})
|
||||
_, err = tdler.NewDownloader(elem.File).Parallel(ctx, wrAt)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageDownload, err)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to download file: %w", err)
|
||||
}
|
||||
logger.Info("File downloaded successfully")
|
||||
@@ -122,21 +363,50 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error {
|
||||
var fileStat os.FileInfo
|
||||
fileStat, err = os.Stat(elem.localPath)
|
||||
if err != nil {
|
||||
t.markItemFailed(elem.ID, FailureStageCache, err)
|
||||
t.notifyStateChange(ctx)
|
||||
return fmt.Errorf("failed to get file stat: %w", err)
|
||||
}
|
||||
t.recordDownloadComplete(elem.ID, fileStat.Size())
|
||||
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
|
||||
t.startUpload(vctx)
|
||||
onProgress := t.uploadCallback(vctx, elem.ID)
|
||||
attempt := 0
|
||||
retryLimit := int(config.C().Retry)
|
||||
lastFailureStage := FailureStageUpload
|
||||
err = retry.Retry(func() error {
|
||||
attempt++
|
||||
var file *os.File
|
||||
file, err = os.Open(elem.localPath)
|
||||
if err != nil {
|
||||
lastFailureStage = FailureStageCache
|
||||
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
|
||||
t.notifyStateChange(vctx)
|
||||
return fmt.Errorf("failed to open cache file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err = elem.Storage.Save(vctx, file, elem.Path); err != nil {
|
||||
onProgress(0, fileStat.Size())
|
||||
if progressSaver, ok := elem.Storage.(storage.StorageProgressSaver); ok {
|
||||
err = progressSaver.SaveWithProgress(vctx, file, elem.Path, onProgress)
|
||||
} else {
|
||||
err = elem.Storage.Save(vctx, ioutil.NewProgressReader(file, fileStat.Size(), onProgress), elem.Path)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to save file: %s, retrying...", err)
|
||||
lastFailureStage = t.itemFailureStage(elem.ID)
|
||||
t.markItemRetry(elem.ID, lastFailureStage, attempt, retryLimit, err)
|
||||
t.notifyStateChange(vctx)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, retry.Context(vctx), retry.RetryTimes(uint(config.C().Retry)))
|
||||
if err == nil {
|
||||
onProgress(fileStat.Size(), fileStat.Size())
|
||||
t.markItemCompleted(elem.ID)
|
||||
t.notifyStateChange(vctx)
|
||||
} else {
|
||||
t.markItemFailed(elem.ID, lastFailureStage, err)
|
||||
t.notifyStateChange(vctx)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
57
core/tasks/batchtfile/execute_group_test.go
Normal file
57
core/tasks/batchtfile/execute_group_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package batchtfile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
tgstorage "github.com/krau/SaveAny-Bot/storage/telegram"
|
||||
)
|
||||
|
||||
func TestExecutionGroupsPreserveSourceAlbums(t *testing.T) {
|
||||
stor := new(tgstorage.Telegram)
|
||||
otherStor := new(tgstorage.Telegram)
|
||||
task := Task{elems: []TaskElement{
|
||||
{Storage: stor, sourceGroupKey: "album-1"},
|
||||
{Storage: stor, sourceGroupKey: "album-1"},
|
||||
{Storage: stor},
|
||||
{Storage: stor, sourceGroupKey: "album-2"},
|
||||
{Storage: stor, sourceGroupKey: "album-2"},
|
||||
{Storage: otherStor, sourceGroupKey: "album-2"},
|
||||
}}
|
||||
|
||||
groups := task.executionGroups()
|
||||
wantSizes := []int{2, 1, 2, 1}
|
||||
wantBatch := []bool{true, false, true, true}
|
||||
if len(groups) != len(wantSizes) {
|
||||
t.Fatalf("got %d groups, want %d", len(groups), len(wantSizes))
|
||||
}
|
||||
for i := range groups {
|
||||
if got := len(groups[i].elems); got != wantSizes[i] {
|
||||
t.Errorf("group %d has %d elements, want %d", i, got, wantSizes[i])
|
||||
}
|
||||
if got := groups[i].usesBatchSaver(); got != wantBatch[i] {
|
||||
t.Errorf("group %d batch=%v, want %v", i, got, wantBatch[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceMetadataPreservesAlbumIdentityAndCaption(t *testing.T) {
|
||||
msg := &tg.Message{
|
||||
PeerID: &tg.PeerChannel{ChannelID: 77},
|
||||
Message: "original caption",
|
||||
}
|
||||
msg.SetGroupedID(42)
|
||||
file := tfile.NewTGFile(nil, nil, 0, "photo.jpg", tfile.WithMessage(msg))
|
||||
|
||||
groupKey, caption, preserveCaption := sourceMetadata(file)
|
||||
if groupKey != "*tg.PeerChannel:77:42" {
|
||||
t.Fatalf("group key = %q, want %q", groupKey, "*tg.PeerChannel:77:42")
|
||||
}
|
||||
if caption != "original caption" {
|
||||
t.Fatalf("caption = %q, want original caption", caption)
|
||||
}
|
||||
if !preserveCaption {
|
||||
t.Fatal("preserveCaption = false, want true")
|
||||
}
|
||||
}
|
||||
360
core/tasks/batchtfile/item_progress.go
Normal file
360
core/tasks/batchtfile/item_progress.go
Normal file
@@ -0,0 +1,360 @@
|
||||
package batchtfile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
transferSpeedWindow = 5 * time.Second
|
||||
transferSamplePeriod = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// ItemPhase describes the current lifecycle stage of one batch item.
|
||||
type ItemPhase uint8
|
||||
|
||||
const (
|
||||
ItemPhaseWaiting ItemPhase = iota
|
||||
ItemPhaseDownloading
|
||||
ItemPhaseTransferring
|
||||
ItemPhaseDownloaded
|
||||
ItemPhaseUploading
|
||||
ItemPhaseRetrying
|
||||
ItemPhaseConfirming
|
||||
ItemPhaseCompleted
|
||||
ItemPhaseFailed
|
||||
ItemPhaseStopped
|
||||
)
|
||||
|
||||
// FailureStage identifies the operation that failed for one batch item.
|
||||
type FailureStage uint8
|
||||
|
||||
const (
|
||||
FailureStageNone FailureStage = iota
|
||||
FailureStageDownload
|
||||
FailureStageCache
|
||||
FailureStageUpload
|
||||
FailureStageConfirm
|
||||
FailureStageBatchUpload
|
||||
FailureStageInternal
|
||||
)
|
||||
|
||||
// TaskItemProgress is an immutable progress snapshot for one batch item.
|
||||
type TaskItemProgress struct {
|
||||
Index int
|
||||
ID string
|
||||
Name string
|
||||
Size int64
|
||||
Downloaded int64
|
||||
Uploaded int64
|
||||
DownloadSpeed float64
|
||||
UploadSpeed float64
|
||||
Phase ItemPhase
|
||||
FailureStage FailureStage
|
||||
RetryAttempt int
|
||||
RetryLimit int
|
||||
Error string
|
||||
}
|
||||
|
||||
type transferSample struct {
|
||||
at time.Time
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type transferMeter struct {
|
||||
samples []transferSample
|
||||
latest transferSample
|
||||
hasData bool
|
||||
}
|
||||
|
||||
func (m *transferMeter) record(now time.Time, transferred int64) {
|
||||
if m.hasData && transferred < m.latest.bytes {
|
||||
m.reset()
|
||||
}
|
||||
if m.hasData && now.Before(m.latest.at) {
|
||||
now = m.latest.at
|
||||
}
|
||||
sample := transferSample{at: now, bytes: transferred}
|
||||
m.latest = sample
|
||||
m.hasData = true
|
||||
if len(m.samples) == 0 {
|
||||
m.samples = append(m.samples, sample)
|
||||
return
|
||||
}
|
||||
if now.Sub(m.samples[len(m.samples)-1].at) >= transferSamplePeriod {
|
||||
m.samples = append(m.samples, sample)
|
||||
}
|
||||
cutoff := now.Add(-transferSpeedWindow)
|
||||
for len(m.samples) > 1 && m.samples[0].at.Before(cutoff) {
|
||||
m.samples = m.samples[1:]
|
||||
}
|
||||
}
|
||||
|
||||
func (m *transferMeter) speed() float64 {
|
||||
if !m.hasData || len(m.samples) == 0 {
|
||||
return 0
|
||||
}
|
||||
first := m.samples[0]
|
||||
last := m.latest
|
||||
elapsed := last.at.Sub(first.at).Seconds()
|
||||
if elapsed <= 0 || last.bytes <= first.bytes {
|
||||
return 0
|
||||
}
|
||||
return float64(last.bytes-first.bytes) / elapsed
|
||||
}
|
||||
|
||||
func (m *transferMeter) reset() {
|
||||
m.samples = m.samples[:0]
|
||||
m.latest = transferSample{}
|
||||
m.hasData = false
|
||||
}
|
||||
|
||||
type itemProgressState struct {
|
||||
index int
|
||||
id string
|
||||
name string
|
||||
expectedSize int64
|
||||
actualSize int64
|
||||
downloaded int64
|
||||
uploaded int64
|
||||
phase ItemPhase
|
||||
failureStage FailureStage
|
||||
retryAttempt int
|
||||
retryLimit int
|
||||
err string
|
||||
downloadMeter transferMeter
|
||||
uploadMeter transferMeter
|
||||
}
|
||||
|
||||
func newItemProgressStates(elems []TaskElement) ([]itemProgressState, map[string]int) {
|
||||
states := make([]itemProgressState, 0, len(elems))
|
||||
index := make(map[string]int, len(elems))
|
||||
for i, elem := range elems {
|
||||
name := ""
|
||||
size := int64(0)
|
||||
if elem.File != nil {
|
||||
name = elem.File.Name()
|
||||
size = elem.File.Size()
|
||||
}
|
||||
states = append(states, itemProgressState{
|
||||
index: i + 1,
|
||||
id: elem.ID,
|
||||
name: name,
|
||||
expectedSize: size,
|
||||
phase: ItemPhaseWaiting,
|
||||
})
|
||||
index[elem.ID] = i
|
||||
}
|
||||
return states, index
|
||||
}
|
||||
|
||||
func (t *Task) updateItem(id string, update func(*itemProgressState)) bool {
|
||||
t.itemMu.Lock()
|
||||
defer t.itemMu.Unlock()
|
||||
index, ok := t.itemIndex[id]
|
||||
if !ok || index < 0 || index >= len(t.itemStates) {
|
||||
return false
|
||||
}
|
||||
update(&t.itemStates[index])
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *Task) markItemActive(id string, stream bool, now time.Time) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
if stream {
|
||||
item.phase = ItemPhaseTransferring
|
||||
} else {
|
||||
item.phase = ItemPhaseDownloading
|
||||
}
|
||||
item.failureStage = FailureStageNone
|
||||
item.err = ""
|
||||
item.downloadMeter.record(now, item.downloaded)
|
||||
if stream {
|
||||
item.uploadMeter.record(now, item.uploaded)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) recordItemDownload(id string, n int64, now time.Time) {
|
||||
if n <= 0 {
|
||||
return
|
||||
}
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
item.downloaded += n
|
||||
item.downloadMeter.record(now, item.downloaded)
|
||||
if item.phase == ItemPhaseTransferring {
|
||||
item.uploaded += n
|
||||
item.uploadMeter.record(now, item.uploaded)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) markItemDownloaded(id string) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
item.phase = ItemPhaseDownloaded
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) recordItemDownloaded(id string, actualSize int64) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
if actualSize > 0 {
|
||||
item.actualSize = actualSize
|
||||
}
|
||||
if item.actualSize == 0 {
|
||||
item.actualSize = item.downloaded
|
||||
}
|
||||
if item.phase != ItemPhaseTransferring {
|
||||
item.phase = ItemPhaseDownloaded
|
||||
item.uploadMeter.reset()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) recordItemUpload(id string, uploaded, total int64, now time.Time) bool {
|
||||
becameConfirming := false
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
if uploaded < item.uploaded {
|
||||
if item.phase != ItemPhaseRetrying {
|
||||
return
|
||||
}
|
||||
item.uploadMeter.reset()
|
||||
}
|
||||
if total > 0 {
|
||||
item.actualSize = total
|
||||
}
|
||||
item.uploaded = uploaded
|
||||
item.uploadMeter.record(now, uploaded)
|
||||
if total > 0 && uploaded >= total {
|
||||
becameConfirming = item.phase != ItemPhaseConfirming
|
||||
item.phase = ItemPhaseConfirming
|
||||
return
|
||||
}
|
||||
item.phase = ItemPhaseUploading
|
||||
item.failureStage = FailureStageNone
|
||||
item.err = ""
|
||||
})
|
||||
return becameConfirming
|
||||
}
|
||||
|
||||
func (t *Task) markItemRetry(id string, stage FailureStage, attempt, limit int, err error) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
item.phase = ItemPhaseRetrying
|
||||
item.failureStage = stage
|
||||
item.retryAttempt = attempt
|
||||
item.retryLimit = limit
|
||||
item.err = compactError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) markItemFailed(id string, stage FailureStage, err error) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
if item.phase == ItemPhaseFailed || item.phase == ItemPhaseCompleted {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
item.phase = ItemPhaseStopped
|
||||
return
|
||||
}
|
||||
item.phase = ItemPhaseFailed
|
||||
item.failureStage = stage
|
||||
item.err = compactError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) markItemCompleted(id string) {
|
||||
t.updateItem(id, func(item *itemProgressState) {
|
||||
item.phase = ItemPhaseCompleted
|
||||
item.failureStage = FailureStageNone
|
||||
item.err = ""
|
||||
item.retryAttempt = 0
|
||||
item.retryLimit = 0
|
||||
if item.actualSize == 0 {
|
||||
item.actualSize = max(item.downloaded, item.uploaded)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) finishItems(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
t.itemMu.Lock()
|
||||
defer t.itemMu.Unlock()
|
||||
for i := range t.itemStates {
|
||||
item := &t.itemStates[i]
|
||||
if item.phase == ItemPhaseCompleted || item.phase == ItemPhaseFailed {
|
||||
continue
|
||||
}
|
||||
item.phase = ItemPhaseStopped
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Task) itemFailureStage(id string) FailureStage {
|
||||
t.itemMu.RLock()
|
||||
defer t.itemMu.RUnlock()
|
||||
index, ok := t.itemIndex[id]
|
||||
if !ok || index < 0 || index >= len(t.itemStates) {
|
||||
return FailureStageUpload
|
||||
}
|
||||
if t.itemStates[index].phase == ItemPhaseConfirming {
|
||||
return FailureStageConfirm
|
||||
}
|
||||
return FailureStageUpload
|
||||
}
|
||||
|
||||
func (t *Task) Items() []TaskItemProgress {
|
||||
t.itemMu.RLock()
|
||||
defer t.itemMu.RUnlock()
|
||||
items := make([]TaskItemProgress, 0, len(t.itemStates))
|
||||
for i := range t.itemStates {
|
||||
item := &t.itemStates[i]
|
||||
size := item.actualSize
|
||||
if size == 0 {
|
||||
size = item.expectedSize
|
||||
}
|
||||
items = append(items, TaskItemProgress{
|
||||
Index: item.index,
|
||||
ID: item.id,
|
||||
Name: item.name,
|
||||
Size: size,
|
||||
Downloaded: item.downloaded,
|
||||
Uploaded: item.uploaded,
|
||||
DownloadSpeed: item.downloadMeter.speed(),
|
||||
UploadSpeed: item.uploadMeter.speed(),
|
||||
Phase: item.phase,
|
||||
FailureStage: item.failureStage,
|
||||
RetryAttempt: item.retryAttempt,
|
||||
RetryLimit: item.retryLimit,
|
||||
Error: item.err,
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (t *Task) ActualTotalSize() int64 {
|
||||
items := t.Items()
|
||||
var total int64
|
||||
for _, item := range items {
|
||||
total += item.Size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
type stateProgressTracker interface {
|
||||
OnStateChange(ctx context.Context, info TaskInfo)
|
||||
}
|
||||
|
||||
func (t *Task) notifyStateChange(ctx context.Context) {
|
||||
if tracker, ok := t.Progress.(stateProgressTracker); ok {
|
||||
tracker.OnStateChange(ctx, t)
|
||||
}
|
||||
}
|
||||
|
||||
func compactError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(strings.Fields(err.Error()), " ")
|
||||
}
|
||||
@@ -4,20 +4,19 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"path"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/slice"
|
||||
"github.com/gotd/td/telegram/message/entity"
|
||||
"github.com/gotd/td/telegram/message/styling"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
)
|
||||
|
||||
type ProgressTracker interface {
|
||||
@@ -27,159 +26,471 @@ type ProgressTracker interface {
|
||||
}
|
||||
|
||||
type Progress struct {
|
||||
MessageID int
|
||||
ChatID int64
|
||||
start time.Time
|
||||
lastUpdatePercent atomic.Int32
|
||||
skippedFiles []string
|
||||
MessageID int
|
||||
ChatID int64
|
||||
updateMu sync.Mutex
|
||||
lastUpdateAt time.Time
|
||||
lastText string
|
||||
done bool
|
||||
skippedFiles []string
|
||||
}
|
||||
|
||||
type renderedBatchMessage struct {
|
||||
Text string
|
||||
Entities []tg.MessageEntityClass
|
||||
Err error
|
||||
}
|
||||
|
||||
const (
|
||||
progressRenderInterval = time.Second
|
||||
maxVisibleActiveItems = 5
|
||||
progressBarWidth = 10
|
||||
maxDisplayNameRunes = 36
|
||||
maxDisplayErrorRunes = 240
|
||||
)
|
||||
|
||||
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
|
||||
p.start = time.Now()
|
||||
p.lastUpdatePercent.Store(0)
|
||||
log.FromContext(ctx).Debugf("Batch task progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchStartPrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
|
||||
); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
return
|
||||
}
|
||||
p.render(ctx, info, true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) {
|
||||
if !shouldUpdateProgress(info.TotalSize(), info.Downloaded(), int(p.lastUpdatePercent.Load())) {
|
||||
p.render(ctx, info, false)
|
||||
}
|
||||
|
||||
func (p *Progress) OnStateChange(ctx context.Context, info TaskInfo) {
|
||||
p.render(ctx, info, true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, _ int64) {
|
||||
p.render(ctx, info, true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, _, _ int64) {
|
||||
p.render(ctx, info, false)
|
||||
}
|
||||
|
||||
func (p *Progress) render(ctx context.Context, info TaskInfo, priority bool) {
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
if p.done {
|
||||
return
|
||||
}
|
||||
percent := int((info.Downloaded() * 100) / info.TotalSize())
|
||||
if p.lastUpdatePercent.Load() == int32(percent) {
|
||||
now := time.Now()
|
||||
if !priority && !p.lastUpdateAt.IsZero() && now.Sub(p.lastUpdateAt) < progressRenderInterval {
|
||||
return
|
||||
}
|
||||
p.lastUpdatePercent.Store(int32(percent))
|
||||
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.TaskID(), info.Downloaded(), info.TotalSize())
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchProcessingPrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalSize())/(1024*1024), info.Count())),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressProcessingListPrefix, nil)),
|
||||
func() styling.StyledTextOption {
|
||||
var lines []string
|
||||
for _, elem := range info.Processing() {
|
||||
lines = append(lines, fmt.Sprintf(" - %s (%.2f MB)", elem.FileName(), float64(elem.FileSize())/(1024*1024)))
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
lines = append(lines, i18n.T(i18nk.BotMsgProgressProcessingNone, nil))
|
||||
}
|
||||
return styling.Plain(slice.Join(lines, "\n"))
|
||||
}(),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
|
||||
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(info.Downloaded(), p.start)/(1024*1024))),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
|
||||
styling.Bold(fmt.Sprintf("%.2f%%", float64(info.Downloaded())/float64(info.TotalSize())*100)),
|
||||
); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
message := buildBatchProgressMessage(info, p.skippedFiles, visibleActiveItems())
|
||||
if message.Err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
if message.Text == p.lastText {
|
||||
return
|
||||
}
|
||||
p.lastText = message.Text
|
||||
p.lastUpdateAt = now
|
||||
p.editMessage(ctx, info.TaskID(), message, true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
|
||||
if err != nil {
|
||||
log.FromContext(ctx).Errorf("Batch task %s failed: %s", info.TaskID(), err)
|
||||
} else {
|
||||
log.FromContext(ctx).Debugf("Batch task %s completed successfully", info.TaskID())
|
||||
}
|
||||
entityBuilder := entity.Builder{}
|
||||
var stylingErr error
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
|
||||
)
|
||||
} else {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskFailedWithError, map[string]any{
|
||||
"Error": "",
|
||||
})),
|
||||
styling.Code(err.Error()),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressBatchDonePrefix, nil)),
|
||||
styling.Code(strconv.Itoa(info.Count())),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressTotalSizePrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("%.2f MB", float64(info.TotalSize())/(1024*1024))),
|
||||
func() styling.StyledTextOption {
|
||||
if len(p.skippedFiles) == 0 {
|
||||
return styling.Plain("")
|
||||
}
|
||||
return styling.Plain("\n\n" + i18n.T(i18nk.BotMsgCommonInfoConflictFilesSkipped, map[string]any{
|
||||
"Skipped": strings.Join(p.skippedFiles, "\n"),
|
||||
}))
|
||||
}(),
|
||||
)
|
||||
}
|
||||
|
||||
if stylingErr != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
if p.done {
|
||||
return
|
||||
}
|
||||
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
p.done = true
|
||||
message := buildBatchDoneMessage(info, p.skippedFiles, err)
|
||||
if message.Err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to render final batch progress message: %v", message.Err)
|
||||
return
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
p.lastText = message.Text
|
||||
p.editMessage(ctx, info.TaskID(), message, false)
|
||||
}
|
||||
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedBatchMessage, cancellable bool) {
|
||||
if message.Err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to render batch progress message: %v", message.Err)
|
||||
return
|
||||
}
|
||||
req := buildBatchEditMessageRequest(p.MessageID, taskID, message, cancellable)
|
||||
if ext := tgutil.ExtFromContext(ctx); ext != nil {
|
||||
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to edit batch progress message: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildBatchEditMessageRequest(messageID int, taskID string, message renderedBatchMessage, cancellable bool) *tg.MessagesEditMessageRequest {
|
||||
req := &tg.MessagesEditMessageRequest{ID: messageID}
|
||||
req.SetMessage(message.Text)
|
||||
if len(message.Entities) > 0 {
|
||||
req.SetEntities(message.Entities)
|
||||
}
|
||||
if cancellable {
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
|
||||
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
|
||||
}}})
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func buildBatchProgressText(info TaskInfo, skipped []string, activeLimit int) string {
|
||||
return buildBatchProgressMessage(info, skipped, activeLimit).Text
|
||||
}
|
||||
|
||||
func buildBatchProgressMessage(info TaskInfo, skipped []string, activeLimit int) renderedBatchMessage {
|
||||
items := info.Items()
|
||||
completed, waiting, downloaded, failed := itemCounts(items)
|
||||
downloadSpeed, uploadSpeed := aggregateSpeeds(items)
|
||||
if activeLimit < 1 {
|
||||
activeLimit = 1
|
||||
}
|
||||
|
||||
total := len(items) + len(skipped)
|
||||
downloadSpeedText := formatSpeed(downloadSpeed)
|
||||
uploadSpeedText := formatSpeed(uploadSpeed)
|
||||
header := localizedProgressMarkup(i18nk.BotMsgProgressBatchStatusHeader, map[string]any{
|
||||
"Total": total,
|
||||
"Completed": completed,
|
||||
"Downloaded": downloaded,
|
||||
"Waiting": waiting,
|
||||
"DownloadSpeed": downloadSpeedText,
|
||||
"UploadSpeed": uploadSpeedText,
|
||||
})
|
||||
|
||||
var markup strings.Builder
|
||||
markup.WriteString(header)
|
||||
|
||||
visibleItems, hiddenTransfers, summarizedConfirming := visibleBatchItems(items, activeLimit)
|
||||
for _, item := range visibleItems {
|
||||
markup.WriteString("\n\n")
|
||||
markup.WriteString(formatActiveItemMarkup(item, len(items)))
|
||||
}
|
||||
|
||||
if hiddenTransfers > 0 {
|
||||
markup.WriteString("\n\n")
|
||||
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryHiddenActive, map[string]any{"Count": hiddenTransfers}))
|
||||
}
|
||||
if summarizedConfirming > 0 {
|
||||
markup.WriteString("\n\n")
|
||||
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryConfirming, map[string]any{"Count": summarizedConfirming}))
|
||||
}
|
||||
if failed > 0 {
|
||||
markup.WriteString("\n")
|
||||
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummaryFailed, map[string]any{"Count": failed}))
|
||||
}
|
||||
if len(skipped) > 0 {
|
||||
markup.WriteString("\n")
|
||||
markup.WriteString(localizedProgressMarkup(i18nk.BotMsgProgressBatchSummarySkipped, map[string]any{"Count": len(skipped)}))
|
||||
}
|
||||
return completeBatchMessage(markup.String())
|
||||
}
|
||||
|
||||
func buildBatchDoneMarkup(info TaskInfo, skipped []string, err error) string {
|
||||
items := info.Items()
|
||||
totalSize := info.ActualTotalSize()
|
||||
if totalSize == 0 {
|
||||
totalSize = info.TotalSize()
|
||||
}
|
||||
if err == nil {
|
||||
if len(skipped) > 0 {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDoneWithSkipped, map[string]any{
|
||||
"Success": len(items),
|
||||
"Skipped": len(skipped),
|
||||
"Size": dlutil.FormatSize(totalSize),
|
||||
})
|
||||
}
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchDone, map[string]any{
|
||||
"Count": len(items),
|
||||
"Size": dlutil.FormatSize(totalSize),
|
||||
})
|
||||
}
|
||||
completed, _, _, failed := itemCounts(items)
|
||||
incomplete := max(len(items)-completed-failed, 0)
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchCanceled, map[string]any{
|
||||
"Total": len(items) + len(skipped),
|
||||
"Completed": completed,
|
||||
"Incomplete": incomplete,
|
||||
"Skipped": len(skipped),
|
||||
})
|
||||
}
|
||||
|
||||
failedItems := make([]TaskItemProgress, 0, failed)
|
||||
for _, item := range items {
|
||||
if item.Phase == ItemPhaseFailed {
|
||||
failedItems = append(failedItems, item)
|
||||
}
|
||||
}
|
||||
if len(failedItems) > 1 && failedItems[0].FailureStage == FailureStageBatchUpload {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedGroup, map[string]any{
|
||||
"Affected": len(failedItems),
|
||||
"Reason": displayError(firstError(failedItems), err),
|
||||
"Completed": completed,
|
||||
"Failed": len(failedItems),
|
||||
"Incomplete": incomplete,
|
||||
})
|
||||
}
|
||||
if len(failedItems) == 0 {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedTask, map[string]any{
|
||||
"Reason": displayError("", err),
|
||||
"Completed": completed,
|
||||
"Incomplete": incomplete,
|
||||
})
|
||||
}
|
||||
item := failedItems[0]
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchFailedItem, map[string]any{
|
||||
"Index": item.Index,
|
||||
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
|
||||
"Stage": failureStageLabel(item.FailureStage),
|
||||
"Progress": failureProgress(item),
|
||||
"Speed": failureSpeed(item),
|
||||
"Reason": displayError(item.Error, err),
|
||||
"Completed": completed,
|
||||
"Failed": failed,
|
||||
"Incomplete": incomplete,
|
||||
})
|
||||
}
|
||||
|
||||
func buildBatchDoneMessage(info TaskInfo, skipped []string, err error) renderedBatchMessage {
|
||||
return completeBatchMessage(buildBatchDoneMarkup(info, skipped, err))
|
||||
}
|
||||
|
||||
func formatActiveItemMarkup(item TaskItemProgress, total int) string {
|
||||
data := map[string]any{
|
||||
"Index": item.Index,
|
||||
"Total": total,
|
||||
"Name": truncateFilename(item.Name, maxDisplayNameRunes),
|
||||
"Speed": formatSpeed(itemSpeed(item)),
|
||||
"Progress": itemPercent(item),
|
||||
"Bar": textProgressBar(itemPercent(item)),
|
||||
"Current": dlutil.FormatSize(itemBytes(item)),
|
||||
"Size": dlutil.FormatSize(item.Size),
|
||||
"Attempt": min(max(item.RetryAttempt, 1), max(item.RetryLimit, 1)),
|
||||
"Limit": max(item.RetryLimit, 1),
|
||||
"Reason": truncateRunes(item.Error, maxDisplayErrorRunes),
|
||||
}
|
||||
switch item.Phase {
|
||||
case ItemPhaseDownloading:
|
||||
if item.Size <= 0 {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloadingUnknown, data)
|
||||
}
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemDownloading, data)
|
||||
case ItemPhaseTransferring:
|
||||
if item.Size <= 0 {
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferringUnknown, data)
|
||||
}
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemTransferring, data)
|
||||
case ItemPhaseUploading:
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemUploading, data)
|
||||
case ItemPhaseRetrying:
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemRetrying, data)
|
||||
case ItemPhaseConfirming:
|
||||
return localizedProgressMarkup(i18nk.BotMsgProgressBatchItemConfirming, data)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
|
||||
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
|
||||
}
|
||||
|
||||
func visibleBatchItems(items []TaskItemProgress, limit int) (visible []TaskItemProgress, hiddenTransfers, summarizedConfirming int) {
|
||||
visible = make([]TaskItemProgress, 0, limit)
|
||||
transferCount := 0
|
||||
confirmingCount := 0
|
||||
for _, item := range items {
|
||||
switch {
|
||||
case isTransferPhase(item.Phase):
|
||||
transferCount++
|
||||
if len(visible) < limit {
|
||||
visible = append(visible, item)
|
||||
}
|
||||
case item.Phase == ItemPhaseConfirming:
|
||||
confirmingCount++
|
||||
}
|
||||
}
|
||||
hiddenTransfers = transferCount - len(visible)
|
||||
if confirmingCount == 1 && len(visible) < limit {
|
||||
for _, item := range items {
|
||||
if item.Phase == ItemPhaseConfirming {
|
||||
visible = append(visible, item)
|
||||
return visible, hiddenTransfers, 0
|
||||
}
|
||||
}
|
||||
}
|
||||
return visible, hiddenTransfers, confirmingCount
|
||||
}
|
||||
|
||||
func completeBatchMessage(markup string) renderedBatchMessage {
|
||||
text, entities, err := tgutil.RenderHTML(markup)
|
||||
return renderedBatchMessage{Text: text, Entities: entities, Err: err}
|
||||
}
|
||||
|
||||
func itemCounts(items []TaskItemProgress) (completed, waiting, downloaded, failed int) {
|
||||
for _, item := range items {
|
||||
switch item.Phase {
|
||||
case ItemPhaseCompleted:
|
||||
completed++
|
||||
case ItemPhaseWaiting:
|
||||
waiting++
|
||||
case ItemPhaseDownloaded:
|
||||
downloaded++
|
||||
case ItemPhaseFailed:
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func aggregateSpeeds(items []TaskItemProgress) (download, upload float64) {
|
||||
for _, item := range items {
|
||||
switch item.Phase {
|
||||
case ItemPhaseDownloading:
|
||||
download += item.DownloadSpeed
|
||||
case ItemPhaseTransferring:
|
||||
download += item.DownloadSpeed
|
||||
upload += item.UploadSpeed
|
||||
case ItemPhaseUploading:
|
||||
upload += item.UploadSpeed
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func isTransferPhase(phase ItemPhase) bool {
|
||||
switch phase {
|
||||
case ItemPhaseDownloading, ItemPhaseTransferring, ItemPhaseUploading, ItemPhaseRetrying:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func itemBytes(item TaskItemProgress) int64 {
|
||||
switch item.Phase {
|
||||
case ItemPhaseDownloading, ItemPhaseTransferring:
|
||||
return item.Downloaded
|
||||
default:
|
||||
return item.Uploaded
|
||||
}
|
||||
}
|
||||
|
||||
func itemSpeed(item TaskItemProgress) float64 {
|
||||
switch item.Phase {
|
||||
case ItemPhaseDownloading, ItemPhaseTransferring:
|
||||
return item.DownloadSpeed
|
||||
case ItemPhaseUploading:
|
||||
return item.UploadSpeed
|
||||
case ItemPhaseRetrying:
|
||||
return item.UploadSpeed
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func itemPercent(item TaskItemProgress) int {
|
||||
if item.Size <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int(min(itemBytes(item), item.Size) * 100 / item.Size)
|
||||
}
|
||||
|
||||
func textProgressBar(percent int) string {
|
||||
percent = min(max(percent, 0), 100)
|
||||
filled := percent * progressBarWidth / 100
|
||||
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", progressBarWidth-filled)
|
||||
}
|
||||
|
||||
func formatSpeed(speed float64) string {
|
||||
if speed <= 0 {
|
||||
return "0 B/s"
|
||||
}
|
||||
return dlutil.FormatSize(int64(speed)) + "/s"
|
||||
}
|
||||
|
||||
func truncateFilename(name string, limit int) string {
|
||||
if utf8.RuneCountInString(name) <= limit {
|
||||
return name
|
||||
}
|
||||
ext := path.Ext(name)
|
||||
if utf8.RuneCountInString(ext) >= limit-2 {
|
||||
return truncateRunes(name, limit-1) + "…"
|
||||
}
|
||||
base := strings.TrimSuffix(name, ext)
|
||||
baseLimit := limit - utf8.RuneCountInString(ext) - 1
|
||||
return truncateRunes(base, baseLimit) + "…" + ext
|
||||
}
|
||||
|
||||
func truncateRunes(value string, limit int) string {
|
||||
if limit <= 0 {
|
||||
return ""
|
||||
}
|
||||
runes := []rune(value)
|
||||
if len(runes) <= limit {
|
||||
return value
|
||||
}
|
||||
return string(runes[:limit])
|
||||
}
|
||||
|
||||
func displayError(itemError string, fallback error) string {
|
||||
if itemError == "" && fallback != nil {
|
||||
itemError = compactError(fallback)
|
||||
}
|
||||
return truncateRunes(itemError, maxDisplayErrorRunes)
|
||||
}
|
||||
|
||||
func firstError(items []TaskItemProgress) string {
|
||||
for _, item := range items {
|
||||
if item.Error != "" {
|
||||
return item.Error
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func failureStageLabel(stage FailureStage) string {
|
||||
switch stage {
|
||||
case FailureStageDownload:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageDownload, nil)
|
||||
case FailureStageCache:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageCache, nil)
|
||||
case FailureStageUpload:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageUpload, nil)
|
||||
case FailureStageConfirm:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageConfirm, nil)
|
||||
case FailureStageBatchUpload:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageBatchUpload, nil)
|
||||
default:
|
||||
return i18n.T(i18nk.BotMsgProgressBatchFailureStageInternal, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func failureProgress(item TaskItemProgress) string {
|
||||
if item.Size <= 0 {
|
||||
return dlutil.FormatSize(failureBytes(item))
|
||||
}
|
||||
return fmt.Sprintf("%d%%", min(failureBytes(item), item.Size)*100/item.Size)
|
||||
}
|
||||
|
||||
func failureSpeed(item TaskItemProgress) string {
|
||||
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
|
||||
return formatSpeed(item.DownloadSpeed)
|
||||
}
|
||||
return formatSpeed(item.UploadSpeed)
|
||||
}
|
||||
|
||||
func failureBytes(item TaskItemProgress) int64 {
|
||||
if item.FailureStage == FailureStageDownload || item.FailureStage == FailureStageCache {
|
||||
return item.Downloaded
|
||||
}
|
||||
return item.Uploaded
|
||||
}
|
||||
|
||||
func visibleActiveItems() int {
|
||||
return min(max(config.C().Workers, 1), maxVisibleActiveItems)
|
||||
}
|
||||
|
||||
func NewProgressTracker(messageID int, chatID int64) ProgressTracker {
|
||||
|
||||
311
core/tasks/batchtfile/progress_regression_test.go
Normal file
311
core/tasks/batchtfile/progress_regression_test.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package batchtfile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
"github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
)
|
||||
|
||||
type progressRegressionRecorder struct {
|
||||
mu sync.Mutex
|
||||
startTotal int64
|
||||
notifications []int64
|
||||
}
|
||||
|
||||
func (*progressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
|
||||
func (*progressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
|
||||
func (*progressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
|
||||
|
||||
func (r *progressRegressionRecorder) OnUploadStart(_ context.Context, _ TaskInfo, total int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.startTotal = total
|
||||
}
|
||||
|
||||
func (r *progressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.notifications = append(r.notifications, uploaded)
|
||||
}
|
||||
|
||||
type orderedProgressRegressionRecorder struct {
|
||||
firstEntered chan struct{}
|
||||
releaseFirst chan struct{}
|
||||
secondEntered chan struct{}
|
||||
mu sync.Mutex
|
||||
notifications []int64
|
||||
}
|
||||
|
||||
func (*orderedProgressRegressionRecorder) OnStart(context.Context, TaskInfo) {}
|
||||
func (*orderedProgressRegressionRecorder) OnProgress(context.Context, TaskInfo) {}
|
||||
func (*orderedProgressRegressionRecorder) OnDone(context.Context, TaskInfo, error) {}
|
||||
func (*orderedProgressRegressionRecorder) OnUploadStart(context.Context, TaskInfo, int64) {
|
||||
}
|
||||
|
||||
func (r *orderedProgressRegressionRecorder) OnUploadProgress(_ context.Context, _ TaskInfo, uploaded, _ int64) {
|
||||
if uploaded == 100 {
|
||||
close(r.firstEntered)
|
||||
<-r.releaseFirst
|
||||
}
|
||||
if uploaded == 200 {
|
||||
close(r.secondEntered)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.notifications = append(r.notifications, uploaded)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestBatchProgressShowsTransferSpeedAndSize(t *testing.T) {
|
||||
useProgressRegressionLocale(t)
|
||||
task := newProgressRegressionTask(nil,
|
||||
progressRegressionFile{"downloading", 1000},
|
||||
progressRegressionFile{"uploading", 1000},
|
||||
progressRegressionFile{"waiting", 1000},
|
||||
)
|
||||
started := time.Unix(100, 0)
|
||||
task.markItemActive("downloading", false, started)
|
||||
task.recordItemDownload("downloading", 500, started.Add(time.Second))
|
||||
task.recordItemDownloaded("uploading", 1000)
|
||||
task.recordItemUpload("uploading", 0, 1000, started.Add(time.Second))
|
||||
task.recordItemUpload("uploading", 250, 1000, started.Add(2*time.Second))
|
||||
|
||||
message := buildBatchProgressMessage(task, nil, 2)
|
||||
if message.Err != nil {
|
||||
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
|
||||
}
|
||||
assertProgressRegressionContains(t, message.Text,
|
||||
"状态:✅ 0 | 📥 0 | ⏳ 1",
|
||||
"⬇️ 1/3 下载中",
|
||||
"速度:500 B/s",
|
||||
"大小:500 B / 1000 B",
|
||||
"⬆️ 2/3 上传中",
|
||||
"速度:250 B/s",
|
||||
"大小:250 B / 1000 B",
|
||||
)
|
||||
bold, _, blockquote, _ := batchEntityCounts(message.Entities)
|
||||
if bold != 3 || blockquote != 2 {
|
||||
t.Fatalf("entity counts = bold:%d blockquote:%d, want bold:3 blockquote:2", bold, blockquote)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchProgressLimitsRowsWithoutHidingActiveUpload(t *testing.T) {
|
||||
useProgressRegressionLocale(t)
|
||||
task := newProgressRegressionTask(nil,
|
||||
progressRegressionFile{"confirm-01", 100},
|
||||
progressRegressionFile{"confirm-02", 100},
|
||||
progressRegressionFile{"uploading", 100},
|
||||
progressRegressionFile{"downloading", 100},
|
||||
)
|
||||
started := time.Unix(100, 0)
|
||||
task.recordItemUpload("confirm-01", 100, 100, started)
|
||||
task.recordItemUpload("confirm-02", 100, 100, started)
|
||||
task.recordItemUpload("uploading", 40, 100, started.Add(time.Second))
|
||||
task.markItemActive("downloading", false, started)
|
||||
|
||||
message := buildBatchProgressText(task, nil, 2)
|
||||
assertProgressRegressionContains(t, message,
|
||||
"uploading.bin",
|
||||
"downloading.bin",
|
||||
"☁️ 已上传,等待整组发送:2",
|
||||
)
|
||||
if strings.Contains(message, "confirm-01.bin") || strings.Contains(message, "confirm-02.bin") {
|
||||
t.Fatalf("confirmation rows displaced active transfers:\n%s", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
|
||||
useProgressRegressionLocale(t)
|
||||
fileID := `<b>A&B</b>`
|
||||
task := newProgressRegressionTask(nil, progressRegressionFile{fileID, 100})
|
||||
task.markItemRetry(fileID, FailureStageUpload, 1, 3, errors.New(`<i>remote & failed</i>`))
|
||||
|
||||
message := buildBatchProgressMessage(task, nil, 1)
|
||||
if message.Err != nil {
|
||||
t.Fatalf("buildBatchProgressMessage() failed: %v", message.Err)
|
||||
}
|
||||
assertProgressRegressionContains(t, message.Text,
|
||||
`<b>A&B</b>.bin`,
|
||||
`<i>remote & failed</i>`,
|
||||
)
|
||||
bold, _, blockquote, italic := batchEntityCounts(message.Entities)
|
||||
if bold != 2 || blockquote != 1 || italic != 0 {
|
||||
t.Fatalf("entity counts = bold:%d blockquote:%d italic:%d", bold, blockquote, italic)
|
||||
}
|
||||
|
||||
i18n.Init("en")
|
||||
english := buildBatchProgressMessage(task, nil, 1)
|
||||
if english.Err != nil {
|
||||
t.Fatalf("English batch template failed: %v", english.Err)
|
||||
}
|
||||
assertProgressRegressionContains(t, english.Text, "📦 Processing", "Retrying upload", `<b>A&B</b>.bin`)
|
||||
}
|
||||
|
||||
func TestDownloadProgressContinuesAfterUploadStarts(t *testing.T) {
|
||||
useProgressRegressionLocale(t)
|
||||
progress := new(Progress)
|
||||
task := newProgressRegressionTask(progress,
|
||||
progressRegressionFile{"uploading", 100},
|
||||
progressRegressionFile{"downloading", 100},
|
||||
)
|
||||
progress.OnStart(t.Context(), task)
|
||||
task.recordDownloadComplete("uploading", 100)
|
||||
task.uploadCallback(t.Context(), "uploading")(50, 100)
|
||||
|
||||
started := time.Unix(100, 0)
|
||||
task.markItemActive("downloading", false, started)
|
||||
task.recordItemDownload("downloading", 50, started.Add(time.Second))
|
||||
progress.updateMu.Lock()
|
||||
progress.lastUpdateAt = time.Now().Add(-progressRenderInterval)
|
||||
progress.updateMu.Unlock()
|
||||
progress.OnProgress(t.Context(), task)
|
||||
|
||||
progress.updateMu.Lock()
|
||||
text := progress.lastText
|
||||
progress.updateMu.Unlock()
|
||||
assertProgressRegressionContains(t, text,
|
||||
"uploading.bin",
|
||||
"🟩🟩🟩🟩🟩⬜️⬜️⬜️⬜️⬜️ 50%",
|
||||
"总速度:⬇️ 50 B/s | ⬆️ 0 B/s",
|
||||
"🔄 另有 1 个文件正在处理",
|
||||
)
|
||||
}
|
||||
|
||||
func TestBatchUploadIgnoresOutOfOrderBytesAndAllowsRetryReset(t *testing.T) {
|
||||
recorder := new(progressRegressionRecorder)
|
||||
task := newProgressRegressionTask(recorder, progressRegressionFile{"file", 100})
|
||||
task.recordDownloadComplete("file", 100)
|
||||
callback := task.uploadCallback(t.Context(), "file")
|
||||
callback(80, 100)
|
||||
callback(10, 100)
|
||||
|
||||
if got := task.Items()[0].Uploaded; got != 80 {
|
||||
t.Fatalf("out-of-order callback regressed item to %d, want 80", got)
|
||||
}
|
||||
task.markItemRetry("file", FailureStageUpload, 1, 3, context.DeadlineExceeded)
|
||||
callback(0, 100)
|
||||
callback(10, 100)
|
||||
if got := task.Items()[0].Uploaded; got != 10 {
|
||||
t.Fatalf("retry did not reset item progress: got %d, want 10", got)
|
||||
}
|
||||
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
for index := 1; index < len(recorder.notifications); index++ {
|
||||
if recorder.notifications[index] < recorder.notifications[index-1] {
|
||||
t.Fatalf("aggregate progress regressed: %v", recorder.notifications)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadProgressNotificationsRemainOrdered(t *testing.T) {
|
||||
recorder := &orderedProgressRegressionRecorder{
|
||||
firstEntered: make(chan struct{}),
|
||||
releaseFirst: make(chan struct{}),
|
||||
secondEntered: make(chan struct{}),
|
||||
}
|
||||
task := newProgressRegressionTask(recorder,
|
||||
progressRegressionFile{"first", 100},
|
||||
progressRegressionFile{"second", 100},
|
||||
)
|
||||
task.recordDownloadComplete("first", 100)
|
||||
task.recordDownloadComplete("second", 100)
|
||||
first := task.uploadCallback(t.Context(), "first")
|
||||
second := task.uploadCallback(t.Context(), "second")
|
||||
|
||||
var wait sync.WaitGroup
|
||||
wait.Go(func() {
|
||||
first(100, 100)
|
||||
})
|
||||
<-recorder.firstEntered
|
||||
wait.Go(func() {
|
||||
second(100, 100)
|
||||
})
|
||||
|
||||
overtook := false
|
||||
select {
|
||||
case <-recorder.secondEntered:
|
||||
overtook = true
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(recorder.releaseFirst)
|
||||
wait.Wait()
|
||||
if overtook {
|
||||
t.Fatal("later aggregate notification overtook the first callback")
|
||||
}
|
||||
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
if got := recorder.notifications; len(got) != 2 || got[0] != 100 || got[1] != 200 {
|
||||
t.Fatalf("upload notifications = %v, want [100 200]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchUploadUsesActualSizeWhenMetadataIsUnknown(t *testing.T) {
|
||||
recorder := new(progressRegressionRecorder)
|
||||
task := newProgressRegressionTask(recorder, progressRegressionFile{"photo", 0})
|
||||
task.recordDownloadComplete("photo", 25)
|
||||
task.uploadCallback(t.Context(), "photo")(25, 25)
|
||||
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
if recorder.startTotal != 25 {
|
||||
t.Fatalf("upload start total = %d, want actual size 25", recorder.startTotal)
|
||||
}
|
||||
if got := task.ActualTotalSize(); got != 25 {
|
||||
t.Fatalf("actual total size = %d, want 25", got)
|
||||
}
|
||||
}
|
||||
|
||||
type progressRegressionFile struct {
|
||||
id string
|
||||
size int64
|
||||
}
|
||||
|
||||
func newProgressRegressionTask(progress ProgressTracker, files ...progressRegressionFile) *Task {
|
||||
elems := make([]TaskElement, 0, len(files))
|
||||
for _, file := range files {
|
||||
elems = append(elems, TaskElement{
|
||||
ID: file.id,
|
||||
File: tfile.NewTGFile(nil, nil, file.size, file.id+".bin"),
|
||||
})
|
||||
}
|
||||
return NewBatchTGFileTask("progress-regression", context.Background(), elems, progress, true)
|
||||
}
|
||||
|
||||
func useProgressRegressionLocale(t *testing.T) {
|
||||
t.Helper()
|
||||
i18n.Init("zh-Hans")
|
||||
t.Cleanup(func() { i18n.Init("zh-Hans") })
|
||||
}
|
||||
|
||||
func assertProgressRegressionContains(t *testing.T, value string, wants ...string) {
|
||||
t.Helper()
|
||||
for _, want := range wants {
|
||||
if !strings.Contains(value, want) {
|
||||
t.Fatalf("text does not contain %q:\n%s", want, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func batchEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
|
||||
for _, messageEntity := range entities {
|
||||
switch messageEntity.(type) {
|
||||
case *tg.MessageEntityBold:
|
||||
bold++
|
||||
case *tg.MessageEntityCode:
|
||||
code++
|
||||
case *tg.MessageEntityBlockquote:
|
||||
blockquote++
|
||||
case *tg.MessageEntityItalic:
|
||||
italic++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/core"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
|
||||
@@ -18,25 +19,35 @@ import (
|
||||
var _ core.Executable = (*Task)(nil)
|
||||
|
||||
type TaskElement struct {
|
||||
ID string
|
||||
Storage storage.Storage
|
||||
Path string
|
||||
File tfile.TGFile
|
||||
localPath string
|
||||
stream bool
|
||||
ID string
|
||||
Storage storage.Storage
|
||||
Path string
|
||||
File tfile.TGFile
|
||||
localPath string
|
||||
stream bool
|
||||
sourceGroupKey string
|
||||
sourceCaption string
|
||||
preserveCaption bool
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
ID string
|
||||
ctx context.Context
|
||||
elems []TaskElement
|
||||
Progress ProgressTracker
|
||||
IgnoreErrors bool // if true, errors during processing will be ignored
|
||||
downloaded atomic.Int64
|
||||
totalSize int64
|
||||
processing map[string]TaskElementInfo
|
||||
processingMu sync.RWMutex
|
||||
failed map[string]error // [TODO] errors for each element
|
||||
ID string
|
||||
ctx context.Context
|
||||
elems []TaskElement
|
||||
Progress ProgressTracker
|
||||
IgnoreErrors bool // if true, errors during processing will be ignored
|
||||
downloaded atomic.Int64
|
||||
totalSize int64
|
||||
uploadTotalSize atomic.Int64
|
||||
processing map[string]TaskElementInfo
|
||||
processingMu sync.RWMutex
|
||||
itemStates []itemProgressState
|
||||
itemIndex map[string]int
|
||||
itemMu sync.RWMutex
|
||||
uploadOnce sync.Once
|
||||
uploadMu sync.Mutex
|
||||
uploaded map[string]int64
|
||||
failed map[string]error // [TODO] errors for each element
|
||||
}
|
||||
|
||||
// Title implements core.Exectable.
|
||||
@@ -54,6 +65,7 @@ func NewTaskElement(
|
||||
file tfile.TGFile,
|
||||
) (*TaskElement, error) {
|
||||
id := xid.New().String()
|
||||
groupKey, caption, preserveCaption := sourceMetadata(file)
|
||||
_, ok := stor.(storage.StorageCannotStream)
|
||||
if !config.C().Stream || ok {
|
||||
cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name())))
|
||||
@@ -61,22 +73,42 @@ func NewTaskElement(
|
||||
return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
|
||||
}
|
||||
return &TaskElement{
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
localPath: cachePath,
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
localPath: cachePath,
|
||||
sourceGroupKey: groupKey,
|
||||
sourceCaption: caption,
|
||||
preserveCaption: preserveCaption,
|
||||
}, nil
|
||||
}
|
||||
return &TaskElement{
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
stream: true,
|
||||
ID: id,
|
||||
Storage: stor,
|
||||
Path: path,
|
||||
File: file,
|
||||
stream: true,
|
||||
sourceGroupKey: groupKey,
|
||||
sourceCaption: caption,
|
||||
preserveCaption: preserveCaption,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sourceMetadata(file tfile.TGFile) (groupKey, caption string, preserveCaption bool) {
|
||||
messageFile, ok := file.(tfile.TGFileMessage)
|
||||
if !ok || messageFile.Message() == nil {
|
||||
return "", "", false
|
||||
}
|
||||
msg := messageFile.Message()
|
||||
groupID, grouped := msg.GetGroupedID()
|
||||
if !grouped || groupID == 0 {
|
||||
return "", "", false
|
||||
}
|
||||
chatID := tgutil.ChatIdFromPeer(msg.GetPeerID())
|
||||
return fmt.Sprintf("%T:%d:%d", msg.GetPeerID(), chatID, groupID), msg.GetMessage(), true
|
||||
}
|
||||
|
||||
func NewBatchTGFileTask(
|
||||
id string,
|
||||
ctx context.Context,
|
||||
@@ -84,6 +116,7 @@ func NewBatchTGFileTask(
|
||||
progress ProgressTracker,
|
||||
ignoreErrors bool,
|
||||
) *Task {
|
||||
itemStates, itemIndex := newItemProgressStates(files)
|
||||
task := &Task{
|
||||
ID: id,
|
||||
ctx: ctx,
|
||||
@@ -98,6 +131,9 @@ func NewBatchTGFileTask(
|
||||
return total
|
||||
}(),
|
||||
processing: make(map[string]TaskElementInfo),
|
||||
itemStates: itemStates,
|
||||
itemIndex: itemIndex,
|
||||
uploaded: make(map[string]int64),
|
||||
IgnoreErrors: ignoreErrors,
|
||||
processingMu: sync.RWMutex{},
|
||||
failed: make(map[string]error),
|
||||
|
||||
@@ -27,8 +27,10 @@ type TaskInfo interface {
|
||||
TaskID() string
|
||||
TotalSize() int64
|
||||
Downloaded() int64
|
||||
ActualTotalSize() int64
|
||||
Count() int
|
||||
Processing() []TaskElementInfo
|
||||
Items() []TaskItemProgress
|
||||
}
|
||||
|
||||
func (t *Task) TaskID() string {
|
||||
|
||||
70
core/tasks/batchtfile/upload_progress.go
Normal file
70
core/tasks/batchtfile/upload_progress.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package batchtfile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// UploadProgressTracker optionally extends a batch progress tracker with a
|
||||
// distinct aggregate upload phase.
|
||||
type UploadProgressTracker interface {
|
||||
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
|
||||
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
|
||||
}
|
||||
|
||||
func (t *Task) startUpload(ctx context.Context) {
|
||||
tracker, ok := t.Progress.(UploadProgressTracker)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
t.uploadMu.Lock()
|
||||
defer t.uploadMu.Unlock()
|
||||
t.uploadOnce.Do(func() {
|
||||
tracker.OnUploadStart(ctx, t, t.uploadTotalSize.Load())
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Task) uploadCallback(ctx context.Context, id string) func(uploaded, total int64) {
|
||||
return func(uploaded, total int64) {
|
||||
tracker, ok := t.Progress.(UploadProgressTracker)
|
||||
if !ok || uploaded < 0 {
|
||||
return
|
||||
}
|
||||
t.startUpload(ctx)
|
||||
if total > 0 && uploaded > total {
|
||||
uploaded = total
|
||||
}
|
||||
|
||||
t.uploadMu.Lock()
|
||||
defer t.uploadMu.Unlock()
|
||||
becameConfirming := t.recordItemUpload(id, uploaded, total, time.Now())
|
||||
if t.uploaded == nil {
|
||||
t.uploaded = make(map[string]int64)
|
||||
}
|
||||
previous, tracked := t.uploaded[id]
|
||||
if !tracked || uploaded > previous {
|
||||
t.uploaded[id] = uploaded
|
||||
}
|
||||
var aggregate int64
|
||||
for _, current := range t.uploaded {
|
||||
aggregate += current
|
||||
}
|
||||
uploadTotal := t.uploadTotalSize.Load()
|
||||
if aggregate > uploadTotal {
|
||||
aggregate = uploadTotal
|
||||
}
|
||||
tracker.OnUploadProgress(ctx, t, aggregate, uploadTotal)
|
||||
if becameConfirming {
|
||||
t.notifyStateChange(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Task) recordDownloadComplete(id string, uploadSize int64) {
|
||||
t.uploadMu.Lock()
|
||||
defer t.uploadMu.Unlock()
|
||||
if uploadSize > 0 {
|
||||
t.uploadTotalSize.Add(uploadSize)
|
||||
}
|
||||
t.recordItemDownloaded(id, uploadSize)
|
||||
}
|
||||
42
core/tasks/tfile/caption_test.go
Normal file
42
core/tasks/tfile/caption_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package tfile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
)
|
||||
|
||||
func TestSourceCaption(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
file tfilepkg.TGFile
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{
|
||||
name: "original caption",
|
||||
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{Message: "original caption"})),
|
||||
want: "original caption",
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "empty caption suppresses storage fallback",
|
||||
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{})),
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "file without source message",
|
||||
file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := sourceCaption(tt.file)
|
||||
if ok != tt.ok || got != tt.want {
|
||||
t.Fatalf("sourceCaption() = (%q, %v), want (%q, %v)", got, ok, tt.want, tt.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package tfile
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
@@ -10,8 +11,12 @@ import (
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/krau/SaveAny-Bot/common/tdler"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile"
|
||||
"github.com/krau/SaveAny-Bot/storage"
|
||||
)
|
||||
|
||||
func (t *Task) Execute(ctx context.Context) error {
|
||||
@@ -57,13 +62,34 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
return fmt.Errorf("failed to get file stat: %w", err)
|
||||
}
|
||||
vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size())
|
||||
if caption, ok := sourceCaption(t.File); ok {
|
||||
vctx = storagetypes.WithSourceCaption(vctx, caption)
|
||||
}
|
||||
err = retry.Retry(func() error {
|
||||
file, err := os.Open(t.localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open cache file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
|
||||
uploadProgress, tracksUpload := t.Progress.(UploadProgressTracker)
|
||||
if !tracksUpload {
|
||||
if err = t.Storage.Save(vctx, file, t.Path); err != nil {
|
||||
return fmt.Errorf("failed to save file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
uploadProgress.OnUploadStart(vctx, t, fileStat.Size())
|
||||
onProgress := func(uploaded, total int64) {
|
||||
uploadProgress.OnUploadProgress(vctx, t, uploaded, total)
|
||||
}
|
||||
if progressSaver, ok := t.Storage.(storage.StorageProgressSaver); ok {
|
||||
err = progressSaver.SaveWithProgress(vctx, file, t.Path, onProgress)
|
||||
} else {
|
||||
var reader io.Reader = ioutil.NewProgressReader(file, fileStat.Size(), onProgress)
|
||||
err = t.Storage.Save(vctx, reader, t.Path)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save file: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -73,3 +99,11 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sourceCaption(file tfilepkg.TGFile) (string, bool) {
|
||||
messageFile, ok := file.(tfilepkg.TGFileMessage)
|
||||
if !ok || messageFile.Message() == nil {
|
||||
return "", false
|
||||
}
|
||||
return messageFile.Message().GetMessage(), true
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/gotd/td/telegram/message/entity"
|
||||
"github.com/gotd/td/telegram/message/styling"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
|
||||
@@ -23,153 +23,319 @@ type ProgressTracker interface {
|
||||
OnDone(ctx context.Context, info TaskInfo, err error)
|
||||
}
|
||||
|
||||
// UploadProgressTracker optionally extends a task progress tracker with a
|
||||
// distinct upload phase. Keeping it separate preserves compatibility with
|
||||
// custom download-only trackers.
|
||||
type UploadProgressTracker interface {
|
||||
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
|
||||
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
|
||||
}
|
||||
|
||||
type Progress struct {
|
||||
MessageID int
|
||||
ChatID int64
|
||||
start time.Time
|
||||
lastUpdatePercent atomic.Int32
|
||||
lastUpdateAt atomic.Int64
|
||||
updateMu sync.Mutex
|
||||
uploadAttempt int
|
||||
uploadedBytes int64
|
||||
actualSize int64
|
||||
hasActualSize bool
|
||||
}
|
||||
|
||||
const (
|
||||
uploadProgressMinInterval = time.Second
|
||||
uploadProgressMaxInterval = 3 * time.Second
|
||||
singleProgressBarWidth = 10
|
||||
maxSingleErrorRunes = 240
|
||||
)
|
||||
|
||||
type singleProgressPhase int
|
||||
|
||||
const (
|
||||
singlePhaseDownloading singleProgressPhase = iota
|
||||
singlePhaseUploading
|
||||
singlePhaseRetrying
|
||||
)
|
||||
|
||||
type renderedSingleMessage struct {
|
||||
Text string
|
||||
Entities []tg.MessageEntityClass
|
||||
Err error
|
||||
}
|
||||
|
||||
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
p.start = time.Now()
|
||||
p.lastUpdatePercent.Store(0)
|
||||
p.lastUpdateAt.Store(0)
|
||||
p.uploadAttempt = 0
|
||||
p.uploadedBytes = 0
|
||||
p.actualSize = 0
|
||||
p.hasActualSize = false
|
||||
log.FromContext(ctx).Debugf("Progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressFileStartPrefix, nil)),
|
||||
styling.Code(info.FileName()),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("%.2f MB", float64(info.FileSize())/(1024*1024))),
|
||||
); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
return
|
||||
}
|
||||
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, singlePhaseDownloading, 0, info.FileSize(), 0, 0), true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo, downloaded, total int64) {
|
||||
if !shouldUpdateProgress(total, downloaded, int(p.lastUpdatePercent.Load())) {
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
now := time.Now()
|
||||
elapsed := uploadProgressMaxInterval
|
||||
if lastUpdateAt := p.lastUpdateAt.Load(); lastUpdateAt > 0 {
|
||||
elapsed = now.Sub(time.Unix(0, lastUpdateAt))
|
||||
}
|
||||
if !shouldUpdateSingleDownloadProgress(total, downloaded, int(p.lastUpdatePercent.Load()), elapsed) {
|
||||
return
|
||||
}
|
||||
percent := int32((downloaded * 100) / total)
|
||||
if p.lastUpdatePercent.Load() == percent {
|
||||
return
|
||||
if total > 0 {
|
||||
percent := int32((downloaded * 100) / total)
|
||||
if p.lastUpdatePercent.Load() == percent {
|
||||
return
|
||||
}
|
||||
p.lastUpdatePercent.Store(percent)
|
||||
}
|
||||
p.lastUpdatePercent.Store(percent)
|
||||
p.lastUpdateAt.Store(now.UnixNano())
|
||||
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.FileName(), downloaded, total)
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressFileProcessingPrefix, nil)),
|
||||
styling.Code(info.FileName()),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressFileSizePrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("%.2f MB", float64(total)/(1024*1024))),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressAvgSpeedPrefix, nil)),
|
||||
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(downloaded, p.start)/(1024*1024))),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressCurrentProgressPrefix, nil)),
|
||||
styling.Bold(fmt.Sprintf("%.2f%%", float64(downloaded)/float64(total)*100)),
|
||||
); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
|
||||
info,
|
||||
singlePhaseDownloading,
|
||||
downloaded,
|
||||
total,
|
||||
dlutil.GetSpeed(downloaded, p.start),
|
||||
0,
|
||||
), true)
|
||||
}
|
||||
|
||||
func shouldUpdateSingleDownloadProgress(total, downloaded int64, lastPercent int, elapsed time.Duration) bool {
|
||||
if total > 0 {
|
||||
return shouldUpdateProgress(total, downloaded, lastPercent)
|
||||
}
|
||||
return downloaded > 0 && elapsed >= uploadProgressMaxInterval
|
||||
}
|
||||
|
||||
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, total int64) {
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
p.start = time.Now()
|
||||
p.lastUpdatePercent.Store(0)
|
||||
p.lastUpdateAt.Store(p.start.UnixNano())
|
||||
p.uploadAttempt++
|
||||
p.uploadedBytes = 0
|
||||
p.actualSize = max(total, 0)
|
||||
p.hasActualSize = true
|
||||
log.FromContext(ctx).Debugf("Upload progress tracking started: %s", info.FileName())
|
||||
phase := singleUploadPhase(p.uploadAttempt)
|
||||
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, phase, 0, total, 0, p.uploadAttempt), true)
|
||||
}
|
||||
|
||||
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64) {
|
||||
if total <= 0 || uploaded <= 0 {
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
if uploaded > total {
|
||||
uploaded = total
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
if uploaded < p.uploadedBytes {
|
||||
return
|
||||
}
|
||||
p.uploadedBytes = uploaded
|
||||
|
||||
now := time.Now()
|
||||
lastUpdateAt := time.Unix(0, p.lastUpdateAt.Load())
|
||||
lastPercent := int(p.lastUpdatePercent.Load())
|
||||
if !shouldUpdateUploadProgress(total, uploaded, lastPercent, now.Sub(lastUpdateAt)) {
|
||||
return
|
||||
}
|
||||
|
||||
percent := int32((uploaded * 100) / total)
|
||||
p.lastUpdatePercent.Store(percent)
|
||||
p.lastUpdateAt.Store(now.UnixNano())
|
||||
log.FromContext(ctx).Debugf("Upload progress update: %s, %d/%d", info.FileName(), uploaded, total)
|
||||
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
|
||||
info,
|
||||
singleUploadPhase(p.uploadAttempt),
|
||||
uploaded,
|
||||
total,
|
||||
dlutil.GetSpeed(uploaded, p.start),
|
||||
p.uploadAttempt,
|
||||
), true)
|
||||
}
|
||||
|
||||
func shouldUpdateUploadProgress(total, uploaded int64, lastPercent int, elapsed time.Duration) bool {
|
||||
if total <= 0 || uploaded <= 0 {
|
||||
return false
|
||||
}
|
||||
if uploaded >= total {
|
||||
return lastPercent < 100 && elapsed >= uploadProgressMinInterval
|
||||
}
|
||||
percent := int((uploaded * 100) / total)
|
||||
if percent < lastPercent {
|
||||
return false
|
||||
}
|
||||
if elapsed < uploadProgressMinInterval {
|
||||
return false
|
||||
}
|
||||
if percent == lastPercent {
|
||||
return elapsed >= uploadProgressMaxInterval
|
||||
}
|
||||
return shouldUpdateProgress(total, uploaded, lastPercent) || elapsed >= uploadProgressMaxInterval
|
||||
}
|
||||
|
||||
func singleUploadPhase(attempt int) singleProgressPhase {
|
||||
if attempt > 1 {
|
||||
return singlePhaseRetrying
|
||||
}
|
||||
return singlePhaseUploading
|
||||
}
|
||||
|
||||
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
|
||||
p.updateMu.Lock()
|
||||
defer p.updateMu.Unlock()
|
||||
if err != nil {
|
||||
log.FromContext(ctx).Errorf("Progress error for file [%s]: %v", info.FileName(), err)
|
||||
} else {
|
||||
log.FromContext(ctx).Debugf("Progress done for file [%s]", info.FileName())
|
||||
}
|
||||
|
||||
entityBuilder := entity.Builder{}
|
||||
var stylingErr error
|
||||
p.editMessage(ctx, info.TaskID(), buildSingleDoneMessage(info, p.doneSize(info), err), false)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressTaskCanceled, nil)),
|
||||
styling.Plain("\n"),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressFileNamePrefix, nil)),
|
||||
styling.Code(info.FileName()),
|
||||
)
|
||||
} else {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadFailedPrefix, nil)),
|
||||
styling.Code(info.FileName()),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressErrorPrefix, nil)),
|
||||
styling.Bold(err.Error()),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
stylingErr = styling.Perform(&entityBuilder,
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressDownloadDonePrefix, nil)),
|
||||
styling.Code(info.FileName()),
|
||||
styling.Plain(i18n.T(i18nk.BotMsgProgressSavePathPrefix, nil)),
|
||||
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
|
||||
)
|
||||
func (p *Progress) doneSize(info TaskInfo) int64 {
|
||||
if p.hasActualSize {
|
||||
return p.actualSize
|
||||
}
|
||||
return max(info.FileSize(), 0)
|
||||
}
|
||||
|
||||
if stylingErr != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", stylingErr)
|
||||
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedSingleMessage, cancellable bool) {
|
||||
if message.Err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to render file progress message: %v", message.Err)
|
||||
return
|
||||
}
|
||||
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.MessageID,
|
||||
req := buildSingleEditMessageRequest(p.MessageID, taskID, message, cancellable)
|
||||
if ext := tgutil.ExtFromContext(ctx); ext != nil {
|
||||
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to edit file progress message: %v", err)
|
||||
}
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
}
|
||||
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.ChatID, req)
|
||||
func buildSingleEditMessageRequest(messageID int, taskID string, message renderedSingleMessage, cancellable bool) *tg.MessagesEditMessageRequest {
|
||||
req := &tg.MessagesEditMessageRequest{ID: messageID}
|
||||
req.SetMessage(message.Text)
|
||||
if len(message.Entities) > 0 {
|
||||
req.SetEntities(message.Entities)
|
||||
}
|
||||
if cancellable {
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
|
||||
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
|
||||
}}})
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func buildSingleProgressMessage(
|
||||
info TaskInfo,
|
||||
phase singleProgressPhase,
|
||||
current int64,
|
||||
total int64,
|
||||
speed float64,
|
||||
attempt int,
|
||||
) renderedSingleMessage {
|
||||
if current < 0 {
|
||||
current = 0
|
||||
}
|
||||
if total > 0 && current > total {
|
||||
current = total
|
||||
}
|
||||
percent := singleProgressPercent(current, total)
|
||||
destination := fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())
|
||||
data := map[string]any{
|
||||
"Name": info.FileName(),
|
||||
"Bar": singleProgressBar(percent),
|
||||
"Progress": percent,
|
||||
"Speed": singleProgressSpeed(speed),
|
||||
"Current": dlutil.FormatSize(current),
|
||||
"Size": dlutil.FormatSize(total),
|
||||
"Destination": destination,
|
||||
"Attempt": max(attempt, 1),
|
||||
}
|
||||
|
||||
var key i18nk.Key
|
||||
switch phase {
|
||||
case singlePhaseUploading:
|
||||
key = i18nk.BotMsgProgressSingleUploading
|
||||
case singlePhaseRetrying:
|
||||
key = i18nk.BotMsgProgressSingleUploadRetrying
|
||||
default:
|
||||
key = i18nk.BotMsgProgressSingleDownloading
|
||||
if total <= 0 {
|
||||
key = i18nk.BotMsgProgressSingleDownloadingUnknown
|
||||
}
|
||||
}
|
||||
markup := i18n.T(i18nk.BotMsgProgressSingleStatusHeader, nil) + "\n\n" + localizedProgressMarkup(key, data)
|
||||
return completeSingleMessage(markup)
|
||||
}
|
||||
|
||||
func buildSingleDoneMessage(info TaskInfo, size int64, err error) renderedSingleMessage {
|
||||
data := map[string]any{
|
||||
"Name": info.FileName(),
|
||||
"Size": dlutil.FormatSize(max(size, 0)),
|
||||
"Destination": fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath()),
|
||||
}
|
||||
var key i18nk.Key
|
||||
switch {
|
||||
case err == nil:
|
||||
key = i18nk.BotMsgProgressSingleDone
|
||||
case errors.Is(err, context.Canceled):
|
||||
key = i18nk.BotMsgProgressSingleCanceled
|
||||
default:
|
||||
data["Reason"] = truncateSingleError(err.Error())
|
||||
key = i18nk.BotMsgProgressSingleFailed
|
||||
}
|
||||
return completeSingleMessage(localizedProgressMarkup(key, data))
|
||||
}
|
||||
|
||||
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
|
||||
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
|
||||
}
|
||||
|
||||
func completeSingleMessage(markup string) renderedSingleMessage {
|
||||
text, entities, err := tgutil.RenderHTML(markup)
|
||||
return renderedSingleMessage{Text: text, Entities: entities, Err: err}
|
||||
}
|
||||
|
||||
func singleProgressPercent(current, total int64) int {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int(min(max(current, 0), total) * 100 / total)
|
||||
}
|
||||
|
||||
func singleProgressBar(percent int) string {
|
||||
percent = min(max(percent, 0), 100)
|
||||
filled := percent * singleProgressBarWidth / 100
|
||||
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", singleProgressBarWidth-filled)
|
||||
}
|
||||
|
||||
func singleProgressSpeed(speed float64) string {
|
||||
if speed <= 0 {
|
||||
return "0 B/s"
|
||||
}
|
||||
return dlutil.FormatSize(int64(speed)) + "/s"
|
||||
}
|
||||
|
||||
func truncateSingleError(value string) string {
|
||||
runes := []rune(value)
|
||||
if len(runes) <= maxSingleErrorRunes {
|
||||
return value
|
||||
}
|
||||
return string(runes[:maxSingleErrorRunes])
|
||||
}
|
||||
|
||||
type ProgressOption func(*Progress)
|
||||
|
||||
178
core/tasks/tfile/progress_test.go
Normal file
178
core/tasks/tfile/progress_test.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package tfile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/i18n"
|
||||
)
|
||||
|
||||
type progressTestTaskInfo struct{}
|
||||
|
||||
func (progressTestTaskInfo) TaskID() string { return "task" }
|
||||
func (progressTestTaskInfo) FileName() string { return "file.bin" }
|
||||
func (progressTestTaskInfo) FileSize() int64 { return 100 << 20 }
|
||||
func (progressTestTaskInfo) StoragePath() string { return "file.bin" }
|
||||
func (progressTestTaskInfo) StorageName() string { return "test" }
|
||||
|
||||
func TestShouldUpdateUploadProgress(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
total int64
|
||||
uploaded int64
|
||||
lastPercent int
|
||||
elapsed time.Duration
|
||||
want bool
|
||||
}{
|
||||
{name: "invalid total", total: 0, uploaded: 1, want: false},
|
||||
{name: "no uploaded bytes", total: 100, uploaded: 0, want: false},
|
||||
{name: "percentage threshold", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval, want: true},
|
||||
{name: "percentage threshold rate limited", total: 100 << 20, uploaded: 10 << 20, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
|
||||
{name: "maximum time threshold", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval, want: true},
|
||||
{name: "below thresholds", total: 100 << 20, uploaded: 1 << 20, elapsed: uploadProgressMaxInterval - time.Millisecond, want: false},
|
||||
{name: "completion", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval, want: true},
|
||||
{name: "completion rate limited", total: 100, uploaded: 100, lastPercent: 99, elapsed: uploadProgressMinInterval - time.Millisecond, want: false},
|
||||
{name: "completion already reported", total: 100, uploaded: 100, lastPercent: 100, elapsed: uploadProgressMinInterval, want: false},
|
||||
{name: "out of order callback", total: 100, uploaded: 40, lastPercent: 60, elapsed: uploadProgressMaxInterval, want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := shouldUpdateUploadProgress(tt.total, tt.uploaded, tt.lastPercent, tt.elapsed)
|
||||
if got != tt.want {
|
||||
t.Fatalf("shouldUpdateUploadProgress() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadProgressConcurrentCallbacks(t *testing.T) {
|
||||
progress := new(Progress)
|
||||
ctx := context.Background()
|
||||
info := progressTestTaskInfo{}
|
||||
const total = int64(100 << 20)
|
||||
|
||||
progress.OnUploadStart(ctx, info, total)
|
||||
progress.lastUpdateAt.Store(time.Now().Add(-uploadProgressMaxInterval).UnixNano())
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for uploaded := int64(1 << 20); uploaded <= total; uploaded += 1 << 20 {
|
||||
uploaded := uploaded
|
||||
wg.Go(func() {
|
||||
progress.OnUploadProgress(ctx, info, uploaded, total)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
percent := progress.lastUpdatePercent.Load()
|
||||
if percent <= 0 || percent > 100 {
|
||||
t.Fatalf("last upload percentage = %d, want a value in (0, 100]", percent)
|
||||
}
|
||||
if progress.uploadedBytes != total {
|
||||
t.Fatalf("maximum uploaded bytes = %d, want %d", progress.uploadedBytes, total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleUploadRetryKeepsRichProgressLayout(t *testing.T) {
|
||||
i18n.Init("zh-Hans")
|
||||
t.Cleanup(func() { i18n.Init("zh-Hans") })
|
||||
|
||||
message := buildSingleProgressMessage(
|
||||
progressTestTaskInfo{},
|
||||
singleUploadPhase(2),
|
||||
25<<20,
|
||||
100<<20,
|
||||
5<<20,
|
||||
2,
|
||||
)
|
||||
if message.Err != nil {
|
||||
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"🔁 上传重试",
|
||||
"🟩🟩⬜️⬜️⬜️⬜️⬜️⬜️⬜️⬜️ 25%",
|
||||
"尝试次数:2",
|
||||
"速度:5.00 MB/s",
|
||||
"大小:25.00 MB / 100.00 MB",
|
||||
} {
|
||||
if !strings.Contains(message.Text, want) {
|
||||
t.Fatalf("retry progress does not contain %q:\n%s", want, message.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleProgressTemplateOwnsStylesAndEscapesValues(t *testing.T) {
|
||||
i18n.Init("en")
|
||||
t.Cleanup(func() { i18n.Init("zh-Hans") })
|
||||
info := htmlProgressTestTaskInfo{}
|
||||
|
||||
message := buildSingleProgressMessage(info, singlePhaseDownloading, 50, 100, 25, 0)
|
||||
if message.Err != nil {
|
||||
t.Fatalf("buildSingleProgressMessage() failed: %v", message.Err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`<b>A&B</b>.bin`,
|
||||
`[store<&>]:dir/<i>x</i>&`,
|
||||
"Speed: 25 B/s",
|
||||
} {
|
||||
if !strings.Contains(message.Text, want) {
|
||||
t.Fatalf("progress text does not contain %q:\n%s", want, message.Text)
|
||||
}
|
||||
}
|
||||
bold, code, blockquote, italic := singleEntityCounts(message.Entities)
|
||||
if bold != 2 || code != 6 || blockquote != 1 || italic != 0 {
|
||||
t.Fatalf("progress entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
|
||||
}
|
||||
|
||||
failure := buildSingleDoneMessage(info, 100, errors.New(`<i>remote & failed</i>`))
|
||||
if failure.Err != nil {
|
||||
t.Fatalf("buildSingleDoneMessage() failed: %v", failure.Err)
|
||||
}
|
||||
if !strings.Contains(failure.Text, `<i>remote & failed</i>`) {
|
||||
t.Fatalf("failure reason was not preserved literally:\n%s", failure.Text)
|
||||
}
|
||||
bold, code, blockquote, italic = singleEntityCounts(failure.Entities)
|
||||
if bold != 1 || code != 2 || blockquote != 0 || italic != 0 {
|
||||
t.Fatalf("failure entity counts = bold:%d code:%d blockquote:%d italic:%d", bold, code, blockquote, italic)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleDoneSizeUsesActualUploadSize(t *testing.T) {
|
||||
progress := new(Progress)
|
||||
info := progressTestTaskInfo{}
|
||||
progress.OnStart(context.Background(), info)
|
||||
progress.OnUploadStart(context.Background(), info, 2048)
|
||||
|
||||
if got := progress.doneSize(info); got != 2048 {
|
||||
t.Fatalf("done size = %d, want actual upload size 2048", got)
|
||||
}
|
||||
}
|
||||
|
||||
type htmlProgressTestTaskInfo struct{}
|
||||
|
||||
func (htmlProgressTestTaskInfo) TaskID() string { return "html-task" }
|
||||
func (htmlProgressTestTaskInfo) FileName() string { return `<b>A&B</b>.bin` }
|
||||
func (htmlProgressTestTaskInfo) FileSize() int64 { return 100 }
|
||||
func (htmlProgressTestTaskInfo) StoragePath() string { return `dir/<i>x</i>&` }
|
||||
func (htmlProgressTestTaskInfo) StorageName() string { return `store<&>` }
|
||||
|
||||
func singleEntityCounts(entities []tg.MessageEntityClass) (bold, code, blockquote, italic int) {
|
||||
for _, messageEntity := range entities {
|
||||
switch messageEntity.(type) {
|
||||
case *tg.MessageEntityBold:
|
||||
bold++
|
||||
case *tg.MessageEntityCode:
|
||||
code++
|
||||
case *tg.MessageEntityBlockquote:
|
||||
blockquote++
|
||||
case *tg.MessageEntityItalic:
|
||||
italic++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -113,6 +113,51 @@ secret = "your-rpc-secret"
|
||||
remove_after_transfer = true
|
||||
```
|
||||
|
||||
### yt-dlp Configuration
|
||||
|
||||
Configures the behavior of the `/ytdlp` command and the `ytdlp` HTTP-API task type when no custom flags are passed.
|
||||
|
||||
- `max_height`: Default maximum video resolution by height in pixels (e.g. `1080`, `720`). `0` means no limit (best available). Ignored when `format` is set.
|
||||
- `format`: A raw yt-dlp format selector (`-f`). When set, it takes precedence over `max_height` and gives you full control, e.g. `bv*[height<=720]+ba/b`.
|
||||
- `recode`: The target video container yt-dlp recodes into after download (e.g. `mp4`). Leave empty to disable recoding.
|
||||
|
||||
{{< hint info >}}
|
||||
These defaults only apply when using the `/ytdlp` command without passing any custom flags. Passing custom flags on the command (or `flags` in the API) overrides them.
|
||||
{{< /hint >}}
|
||||
|
||||
```toml
|
||||
[ytdlp]
|
||||
max_height = 1080
|
||||
format = "" # e.g. "bv*[height<=720]+ba/b"
|
||||
recode = "mp4" # empty disables recoding
|
||||
```
|
||||
|
||||
### HTTP API Configuration
|
||||
|
||||
When enabled, SaveAny-Bot exposes an HTTP API for creating/querying/canceling tasks programmatically. See [HTTP API](../../usage/api) for the full endpoint reference.
|
||||
|
||||
- `enable`: Whether to enable the HTTP API server, default is `false`.
|
||||
- `host`: Bind address, default `0.0.0.0`.
|
||||
- `port`: Listen port, default `8080`.
|
||||
- `token`: Authentication token. **Strongly recommended** — if empty, the API is exposed without any authentication.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
enable = false
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
token = "your-token"
|
||||
```
|
||||
|
||||
### Log Configuration
|
||||
|
||||
- `level`: Log level. One of `debug`, `info`, `warn`, `error`, `fatal`. Default is `info`.
|
||||
|
||||
```toml
|
||||
[log]
|
||||
level = "info"
|
||||
```
|
||||
|
||||
### Storage Endpoints List
|
||||
|
||||
The storage endpoints list is used to define the storage locations supported by the Bot. Each storage endpoint needs to specify a name, type, and related configuration, using the double bracket syntax `[[storages]]`.
|
||||
|
||||
@@ -79,7 +79,8 @@ Stream mode is not supported.
|
||||
chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat
|
||||
force_file = false # Force sending as file, default is false
|
||||
skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded.
|
||||
spilt_size_mb = 2000 # Split size in MB, default is 2000 MB (2 GB). Files larger than this will be split into multiple parts (zip format). Ignored when skip_large is true.
|
||||
split_large_video = false # Losslessly split oversized videos into one album of up to 10 independently playable parts. Falls back to ZIP parts on failure or when more than 10 parts are required.
|
||||
split_size_mb = 0 # Split size in MB. 0 uses the uploader account limit: 2000 MB for bots/regular users and 4000 MB for Premium users. Oversized non-video files use ZIP parts. Ignored when skip_large is true.
|
||||
```
|
||||
|
||||
## Rclone
|
||||
@@ -136,4 +137,4 @@ remote = "myremote"
|
||||
base_path = "/backup"
|
||||
config_path = "/path/to/rclone.conf"
|
||||
flags = ["--progress"]
|
||||
```
|
||||
```
|
||||
|
||||
90
docs/content/en/usage/cli.md
Normal file
90
docs/content/en/usage/cli.md
Normal file
@@ -0,0 +1,90 @@
|
||||
---
|
||||
title: "CLI Subcommands"
|
||||
weight: 21
|
||||
---
|
||||
|
||||
# CLI Subcommands
|
||||
|
||||
Besides running the Telegram bot with `./saveany-bot` (no subcommand), the binary exposes two helper subcommands for moving local files into a storage backend: `upload` (one-shot) and `watch` (continuous).
|
||||
|
||||
These subcommands load the same `config.toml` as the bot, initialize the database and caches, then perform their task. They do **not** start the Telegram bot itself, although storages of type `telegram` will spin up the bot client just for the upload.
|
||||
|
||||
## `upload` — Upload a Single File
|
||||
|
||||
```
|
||||
saveany-bot upload -f <file> -s <storage> [-d <dir>] [--no-progress]
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Required | Description |
|
||||
|---|---|---|
|
||||
| `-f, --file` | Yes | Path to the local file to upload |
|
||||
| `-s, --storage` | Yes | Target storage name (must exist in `config.toml`) |
|
||||
| `-d, --dir` | No | Destination directory within the storage. Defaults to the storage's `base_path` |
|
||||
| `--no-progress` | No | Disable the terminal progress bar |
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
# Upload a file to the default dir of storage "MyAlist"
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist
|
||||
|
||||
# Upload into a specific subdirectory
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
|
||||
|
||||
# Upload via Telegram storage without a progress bar
|
||||
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
|
||||
```
|
||||
|
||||
## `watch` — Watch a Directory and Auto-Upload
|
||||
|
||||
The `watch` subcommand continuously monitors a local directory and uploads created or modified files to a storage backend, preserving the relative directory structure from the watch root.
|
||||
|
||||
```
|
||||
saveany-bot watch -p <path> -s <storage> [-d <dir>] [options]
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Default | Description |
|
||||
|---|---|---|
|
||||
| `-p, --path` | *(required)* | Local directory to watch |
|
||||
| `-s, --storage` | *(required)* | Target storage name |
|
||||
| `-d, --dir` | storage's `base_path` | Destination directory within the storage |
|
||||
| `-r, --recursive` | `false` | Watch subdirectories recursively |
|
||||
| `--overwrite` | `false` | Overwrite existing files on the storage instead of skipping them |
|
||||
| `--initial-scan` | `false` | Upload files already present in the directory on startup |
|
||||
| `--debounce` | `2s` | How long to wait after the last write before uploading a file |
|
||||
| `--upload-workers` | `config.workers` | Number of concurrent uploads |
|
||||
| `--retry-delay` | `3s` | Delay between upload retries |
|
||||
|
||||
{{< hint info >}}
|
||||
Write-completion detection: the watcher debounces per file and only uploads once the file size stays unchanged across the debounce window, so partial/write-in-progress files are not uploaded.
|
||||
<br />
|
||||
If a file changes while being uploaded, it is re-uploaded once after the current upload finishes (instead of being queued multiple times).
|
||||
{{< /hint >}}
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
# Watch ./inbox and upload new files to "MyAlist" recursively
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -r
|
||||
|
||||
# Watch with a custom destination dir and overwrite
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
|
||||
|
||||
# On startup, also upload everything already in ./inbox
|
||||
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
|
||||
```
|
||||
|
||||
### Behavior notes
|
||||
|
||||
- Relative directory structure is preserved under the destination directory. A file written to `./inbox/sub/file.txt` with `--path ./inbox` is uploaded to `<dest_dir>/sub/file.txt`.
|
||||
- `watch` runs until interrupted (e.g. `Ctrl-C` / `SIGINT`); in-flight uploads are drained before exit.
|
||||
- Retries follow the global `retry` value from `config.toml`, with `--retry-delay` between attempts.
|
||||
- Telegram-type storages will start the bot client automatically to perform uploads.
|
||||
|
||||
{{< hint warning >}}
|
||||
`watch` is unrelated to the in-bot `/watch` command (which watches Telegram chats). This subcommand watches a **local filesystem directory** and uploads to a storage backend, independent of Telegram.
|
||||
{{< /hint >}}
|
||||
75
docs/content/en/usage/config.md
Normal file
75
docs/content/en/usage/config.md
Normal file
@@ -0,0 +1,75 @@
|
||||
---
|
||||
title: "File Naming & Conflict Strategies"
|
||||
weight: 11
|
||||
---
|
||||
|
||||
# File Naming & Conflict Strategies
|
||||
|
||||
SaveAny-Bot lets you customize how saved files are named and how collisions with existing files are resolved, directly in Telegram via the `/config` and `/fnametmpl` commands.
|
||||
|
||||
## `/config` — User Configuration
|
||||
|
||||
The `/config` command opens an inline menu where you can change two per-user settings:
|
||||
|
||||
- **Filename strategy** — how the saved file is named
|
||||
- **Duplicate file strategy** — what happens when a file with the same name already exists in the target storage
|
||||
|
||||
Settings are stored per user and apply to all of that user's subsequent save/transfer tasks.
|
||||
|
||||
### Filename strategy
|
||||
|
||||
| Option | Behavior |
|
||||
|---|---|
|
||||
| `Default` | Use the original media filename, or a generated name when no original filename is available |
|
||||
| `Gen From Msg First` | Generate the filename from the message content (e.g. caption, text) and prefer that over the original filename |
|
||||
| `Template` | Render the filename from a custom template you define with `/fnametmpl` |
|
||||
|
||||
### Duplicate file strategy
|
||||
|
||||
| Option | Behavior |
|
||||
|---|---|
|
||||
| `Always rename` (default) | Keep the existing file and save the new one with an alternate name |
|
||||
| `Ask every time` | Prompt you with inline buttons each time a collision occurs |
|
||||
| `Always overwrite` | Replace the existing file with the new one |
|
||||
| `Always skip` | Do nothing for conflicting files |
|
||||
|
||||
{{< hint info >}}
|
||||
The conflict strategy only kicks in for storage backends that can detect the existence of a file. Backends that do not support existence checks will fall back to overwriting.
|
||||
{{< /hint >}}
|
||||
|
||||
## `/fnametmpl` — Custom Filename Template
|
||||
|
||||
When the filename strategy is set to `Template`, SaveAny-Bot renders each saved file's name using the template configured via `/fnametmpl`.
|
||||
|
||||
```
|
||||
/fnametmpl [template]
|
||||
```
|
||||
|
||||
- Running `/fnametmpl` without arguments shows your current template and the help text.
|
||||
- Running it with a template string sets that template as your filename template.
|
||||
|
||||
The template uses Go [`text/template`](https://pkg.go.dev/text/template) syntax. The available variables are:
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `{{.msgid}}` | Telegram message ID |
|
||||
| `{{.msgtags}}` | Hashtags found in the message, joined with `_` |
|
||||
| `{{.msggen}}` | Filename generated from the message |
|
||||
| `{{.msgdate}}` | Message date, formatted `YYYY-MM-DD_HH-MM-SS` |
|
||||
| `{{.msgraw}}` | Raw, unprocessed message text |
|
||||
| `{{.origname}}` | The media's original filename (if any) |
|
||||
| `{{.chatid}}` | Chat ID of the message |
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
# Fixed prefix + message id + date
|
||||
/fnametmpl Image_{{.msgid}}_{{.msgdate}}.jpg
|
||||
|
||||
# Use original name if available, otherwise a generated name
|
||||
/fnametmpl {{.origname}}
|
||||
```
|
||||
|
||||
{{< hint warning >}}
|
||||
The template only takes effect when the filename strategy is set to `Template`. If template parsing fails, SaveAny-Bot falls back to the default filename naming logic.
|
||||
{{< /hint >}}
|
||||
@@ -27,6 +27,45 @@ Pay attention to spaces; the bot can only parse correctly formatted syntax. Belo
|
||||
|
||||
In addition, if `CHOSEN` is used as the storage name in the rule, it means files will be stored under the path of the storage you selected by clicking the inline button.
|
||||
|
||||
You can also toggle whether rules are applied with `/rule switch`. When rule mode is off, all files go to the default storage.
|
||||
|
||||
## Preset Rules
|
||||
|
||||
Manually writing regex rules for common file types is tedious, so the bot ships a built-in set of preset categories (video, image, audio, document, archive) that you can import in one command:
|
||||
|
||||
```
|
||||
/rule preset <storage> [base_path]
|
||||
```
|
||||
|
||||
Parameters:
|
||||
|
||||
- `storage`: Target storage name (must exist and be accessible to you)
|
||||
- `base_path`: Optional. Each preset category's subdirectory is created under this path. If omitted, the default category directory names are used directly.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
# Import preset rules into "MyAlist" with the default directory layout
|
||||
/rule preset MyAlist
|
||||
|
||||
# Import preset rules with a custom base path "downloads/sorted"
|
||||
/rule preset MyAlist downloads/sorted
|
||||
```
|
||||
|
||||
This will create `FILENAME-REGEX` rules for each category, routing matched files to the corresponding subdirectory under `base_path`:
|
||||
|
||||
| Category | Matched extensions | Default directory |
|
||||
|---|---|---|
|
||||
| video | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
|
||||
| image | jpg, jpeg, png, gif, webp, bmp | `图片` |
|
||||
| audio | mp3, flac, wav, aac, m4a, ogg | `音频` |
|
||||
| document | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
|
||||
| archive | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
|
||||
|
||||
{{< hint info >}}
|
||||
Preset rules are regular `FILENAME-REGEX` rules once imported. You can view, edit, or delete them individually with `/rule` and `/rule del <id>` like any other rule.
|
||||
{{< /hint >}}
|
||||
|
||||
Rule types:
|
||||
|
||||
## FILENAME-REGEX
|
||||
|
||||
@@ -30,6 +30,7 @@ base_path = "./downloads"
|
||||
|
||||
### 全局配置
|
||||
|
||||
- `lang`: Bot 使用的语言, 默认为 `zh-CN` (简体中文), 设为 `en` 则使用英语.
|
||||
- `stream`: 是否启用 Stream 模式, 默认为 `false`. 启用后 Bot 将直接将文件流式传输到存储端(若存储端支持), 不需要下载到本地
|
||||
{{< hint warning >}}
|
||||
Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一些弊端:
|
||||
@@ -47,6 +48,7 @@ Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一
|
||||
- `proxy`: 全局代理配置, 配置后程序内一切网络连接将会尝试使用该代理, 可选.
|
||||
|
||||
```toml
|
||||
lang = "zh-CN"
|
||||
stream = false
|
||||
workers = 3
|
||||
threads = 4
|
||||
@@ -111,6 +113,51 @@ secret = "your-rpc-secret"
|
||||
remove_after_transfer = true
|
||||
```
|
||||
|
||||
### yt-dlp 配置
|
||||
|
||||
用于配置 `/ytdlp` 命令以及 HTTP API 中 `ytdlp` 任务类型在未传自定义参数时的默认行为.
|
||||
|
||||
- `max_height`: 默认下载的最高视频清晰度 (按高度限制), 如 `1080`, `720`, `480`; `0` 表示不限制 (下载最佳画质). 当设置了 `format` 时此项被忽略.
|
||||
- `format`: 直接指定 yt-dlp format 选择表达式, 设置后优先级高于 `max_height`, 例如 `bv*[height<=720]+ba/b`.
|
||||
- `recode`: 下载后转封装的视频容器格式 (如 `mp4`), 留空则不转封装.
|
||||
|
||||
{{< hint info >}}
|
||||
这些默认值仅在使用 `/ytdlp` 命令且未传任何自定义参数时生效. 在命令上传递自定义参数 (或在 API 中传 `flags`) 会覆盖这些默认值.
|
||||
{{< /hint >}}
|
||||
|
||||
```toml
|
||||
[ytdlp]
|
||||
max_height = 1080
|
||||
format = "" # 例如 "bv*[height<=720]+ba/b"
|
||||
recode = "mp4" # 留空则不转封装
|
||||
```
|
||||
|
||||
### HTTP API 配置
|
||||
|
||||
启用后, SaveAny-Bot 会暴露一套 HTTP API, 用于以编程方式创建/查询/取消任务. 完整的接口说明见 [HTTP API](../../usage/api).
|
||||
|
||||
- `enable`: 是否启用 HTTP API 服务, 默认为 `false`.
|
||||
- `host`: 监听地址, 默认 `0.0.0.0`.
|
||||
- `port`: 监听端口, 默认 `8080`.
|
||||
- `token`: 鉴权 Token, **强烈建议设置** — 若为空, API 将在无任何鉴权的情况下暴露.
|
||||
|
||||
```toml
|
||||
[api]
|
||||
enable = false
|
||||
host = "0.0.0.0"
|
||||
port = 8080
|
||||
token = "your-token"
|
||||
```
|
||||
|
||||
### 日志配置
|
||||
|
||||
- `level`: 日志级别, 可选 `debug`, `info`, `warn`, `error`, `fatal`. 默认为 `info`.
|
||||
|
||||
```toml
|
||||
[log]
|
||||
level = "info"
|
||||
```
|
||||
|
||||
### 存储端列表
|
||||
|
||||
存储端列表用于定义 Bot 支持的存储位置, 每个存储端需要指定名称、类型和相关配置, 使用双中括号语法 `[[storages]]` 定义.
|
||||
|
||||
@@ -82,10 +82,12 @@ chat_id = "123456789"
|
||||
force_file = false
|
||||
# 是否跳过大文件, 默认为 false. 如果启用, 超过 Telegram 限制的文件将不会上传.
|
||||
skip_large = false
|
||||
# 分卷大小, 单位 MB, 默认为 2000 MB (2 GB).
|
||||
# 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式)
|
||||
# 超限视频是否使用 FFmpeg 无损分割成一个媒体组(最多 10 个)内可独立播放的小视频;失败或需要超过 10 段时回退到 ZIP 分卷.
|
||||
split_large_video = false
|
||||
# 分卷大小, 单位 MB. 设为 0 时使用实际上传账号的限制:Bot/普通用户为 2000 MB,Premium 用户为 4000 MB.
|
||||
# 超过该大小的文件将被分割成多个部分上传;非视频使用 ZIP 格式.
|
||||
# 当 skip_large 启用时, 该选项无效.
|
||||
spilt_size_mb = 2000
|
||||
split_size_mb = 0
|
||||
```
|
||||
|
||||
## Rclone
|
||||
@@ -142,4 +144,4 @@ remote = "myremote"
|
||||
base_path = "/backup"
|
||||
config_path = "/path/to/rclone.conf"
|
||||
flags = ["--progress"]
|
||||
```
|
||||
```
|
||||
|
||||
90
docs/content/zh/usage/cli.md
Normal file
90
docs/content/zh/usage/cli.md
Normal file
@@ -0,0 +1,90 @@
|
||||
---
|
||||
title: "命令行子命令"
|
||||
weight: 21
|
||||
---
|
||||
|
||||
# 命令行子命令
|
||||
|
||||
除了直接运行 `./saveany-bot` (不带子命令) 启动 Telegram Bot 外, 这个二进制文件还提供两个把本地文件上传到存储后端的辅助子命令: `upload` (一次性) 和 `watch` (持续监听).
|
||||
|
||||
这些子命令会读取与 Bot 相同的 `config.toml`, 初始化数据库和缓存, 然后执行任务. 它们**不会**启动 Telegram Bot 本身, 但 `telegram` 类型的存储会在需要上传时临时启动 Bot 客户端来执行上传.
|
||||
|
||||
## `upload` — 上传单个文件
|
||||
|
||||
```
|
||||
saveany-bot upload -f <文件> -s <存储名> [-d <目录>] [--no-progress]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
| 参数 | 必填 | 说明 |
|
||||
|---|---|---|
|
||||
| `-f, --file` | 是 | 待上传的本地文件路径 |
|
||||
| `-s, --storage` | 是 | 目标存储名 (必须存在于 `config.toml`) |
|
||||
| `-d, --dir` | 否 | 存储中的目标目录, 默认使用存储的 `base_path` |
|
||||
| `--no-progress` | 否 | 关闭终端进度条 |
|
||||
|
||||
示例:
|
||||
|
||||
```bash
|
||||
# 上传文件到 "MyAlist" 的默认目录
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist
|
||||
|
||||
# 上传到指定子目录
|
||||
./saveany-bot upload -f ./movie.mp4 -s MyAlist -d movies/2026
|
||||
|
||||
# 通过 Telegram 存储上传并关闭进度条
|
||||
./saveany-bot upload -f ./photo.jpg -s MyChannel --no-progress
|
||||
```
|
||||
|
||||
## `watch` — 监听目录并自动上传
|
||||
|
||||
`watch` 子命令持续监听一个本地目录, 将新建或修改的文件上传到存储后端, 并保留相对监听根目录的子目录结构.
|
||||
|
||||
```
|
||||
saveany-bot watch -p <路径> -s <存储名> [-d <目录>] [选项]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `-p, --path` | *(必填)* | 要监听的本地目录 |
|
||||
| `-s, --storage` | *(必填)* | 目标存储名 |
|
||||
| `-d, --dir` | 存储的 `base_path` | 存储中的目标目录 |
|
||||
| `-r, --recursive` | `false` | 是否递归监听子目录 |
|
||||
| `--overwrite` | `false` | 覆盖存储上已有的文件, 而非跳过 |
|
||||
| `--initial-scan` | `false` | 启动时将目录中已存在的文件也上传 |
|
||||
| `--debounce` | `2s` | 文件最后一次写入后, 等待多久再上传 |
|
||||
| `--upload-workers` | `config.workers` | 并发上传数 |
|
||||
| `--retry-delay` | `3s` | 上传重试之间的延迟 |
|
||||
|
||||
{{< hint info >}}
|
||||
写入完成检测: 监听器会按文件做防抖处理, 仅当文件大小在一个 debounce 窗口内保持不变时才上传, 因此不会上传未写完的半成品文件.
|
||||
<br />
|
||||
若某文件在上传过程中又被修改, 它会在当前上传完成后再上传一次, 而不是被重复排队.
|
||||
{{< /hint >}}
|
||||
|
||||
示例:
|
||||
|
||||
```bash
|
||||
# 递归监听 ./inbox 并且把新文件上传到 "MyAlist"
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -r
|
||||
|
||||
# 自定义目标目录并覆盖已有文件
|
||||
./saveany-bot watch -p ./inbox -s MyAlist -d backup --overwrite
|
||||
|
||||
# 启动时把 ./inbox 中已有的内容也一并上传
|
||||
./saveany-bot watch -p ./inbox -s MyAlist --initial-scan
|
||||
```
|
||||
|
||||
### 行为说明
|
||||
|
||||
- 相对子目录结构会被保留: 以 `--path ./inbox` 为例, 写入 `./inbox/sub/file.txt` 的文件会被上传到 `<目标目录>/sub/file.txt`.
|
||||
- `watch` 会一直运行直到被中断 (如 `Ctrl-C` / `SIGINT`), 退出前会等待所有进行中的上传完成.
|
||||
- 重试次数遵循 `config.toml` 中的全局 `retry` 值, 各次重试之间间隔 `--retry-delay`.
|
||||
- `telegram` 类型的存储会自动启动 Bot 客户端来执行上传.
|
||||
|
||||
{{< hint warning >}}
|
||||
`watch` 子命令与 Bot 内的 `/watch` 命令 (监听 Telegram 聊天) 无关. 本子命令监听的是**本地文件系统目录**, 不依赖 Telegram.
|
||||
{{< /hint >}}
|
||||
75
docs/content/zh/usage/config.md
Normal file
75
docs/content/zh/usage/config.md
Normal file
@@ -0,0 +1,75 @@
|
||||
---
|
||||
title: "文件命名与重名策略"
|
||||
weight: 11
|
||||
---
|
||||
|
||||
# 文件命名与重名策略
|
||||
|
||||
SaveAny-Bot 支持在 Telegram 中通过 `/config` 和 `/fnametmpl` 命令自定义保存文件的命名方式, 以及处理与已存在文件重名时的冲突策略.
|
||||
|
||||
## `/config` — 用户配置
|
||||
|
||||
`/config` 命令会弹出一个内联菜单, 你可以在其中修改以下两项用户级设置:
|
||||
|
||||
- **文件名策略** — 保存文件的命名方式
|
||||
- **重名文件保存策略** — 目标存储中已存在同名文件时的处理方式
|
||||
|
||||
设置按用户分别保存, 对该用户后续所有的保存/转存任务生效.
|
||||
|
||||
### 文件名策略
|
||||
|
||||
| 选项 | 行为 |
|
||||
|---|---|
|
||||
| `默认` | 使用媒体原始文件名, 没有原始文件名时使用生成名 |
|
||||
| `优先从消息生成` | 优先根据消息内容 (如 caption、文本) 生成文件名, 而非原始文件名 |
|
||||
| `自定义模板` | 使用 `/fnametmpl` 设置的自定义模板渲染文件名 |
|
||||
|
||||
### 重名文件保存策略
|
||||
|
||||
| 选项 | 行为 |
|
||||
|---|---|
|
||||
| `始终重命名` (默认) | 保留已有文件, 将新文件以另一个名字保存 |
|
||||
| `每次询问` | 每次遇到重名时通过内联按钮提示你选择 |
|
||||
| `始终覆盖` | 用新文件替换已有文件 |
|
||||
| `始终跳过` | 对重名文件不做处理 |
|
||||
|
||||
{{< hint info >}}
|
||||
重名策略仅在能够检测文件是否已存在的存储后端生效. 不支持检测文件是否存在的存储后端会退化为覆盖行为.
|
||||
{{< /hint >}}
|
||||
|
||||
## `/fnametmpl` — 自定义文件名模板
|
||||
|
||||
当文件名策略设置为 `自定义模板` 时, SaveAny-Bot 会用 `/fnametmpl` 配置的模板来渲染所保存文件的文件名.
|
||||
|
||||
```
|
||||
/fnametmpl [模板]
|
||||
```
|
||||
|
||||
- 不带参数运行 `/fnametmpl` 会显示当前模板以及帮助说明
|
||||
- 带模板字符串运行则会把它设为你的文件名模板
|
||||
|
||||
模板使用 Go [`text/template`](https://pkg.go.dev/text/template) 语法. 可用变量如下:
|
||||
|
||||
| 变量 | 说明 |
|
||||
|---|---|
|
||||
| `{{.msgid}}` | Telegram 消息 ID |
|
||||
| `{{.msgtags}}` | 消息中的标签, 以 `_` 连接输出 |
|
||||
| `{{.msggen}}` | 根据消息生成的文件名 |
|
||||
| `{{.msgdate}}` | 消息日期, 格式 `YYYY-MM-DD_HH-MM-SS` |
|
||||
| `{{.msgraw}}` | 消息的原始文本内容 (不做处理) |
|
||||
| `{{.origname}}` | 媒体的原始文件名 (如有) |
|
||||
| `{{.chatid}}` | 消息所在聊天的 ID |
|
||||
|
||||
示例:
|
||||
|
||||
```
|
||||
# 固定前缀 + 消息 ID + 日期
|
||||
/fnametmpl 图片_{{.msgid}}_{{.msgdate}}.jpg
|
||||
|
||||
# 优先使用原始文件名, 没有则用生成名
|
||||
/fnametmpl {{.origname}}
|
||||
```
|
||||
|
||||
{{< hint warning >}}
|
||||
模板仅在文件名策略设置为 `自定义模板` 时生效. 如果模板解析失败, SaveAny-Bot 会回退到默认的文件名生成逻辑.
|
||||
{{< /hint >}}
|
||||
@@ -27,6 +27,45 @@ weight: 3
|
||||
|
||||
此外, 规则中的存储名若使用 "CHOSEN" , 则表示存储到点击按钮选择的存储端的路径下
|
||||
|
||||
你也可以使用 `/rule switch` 来开关规则模式. 关闭规则模式时, 所有文件都将保存到默认存储.
|
||||
|
||||
## 预设规则
|
||||
|
||||
为常见文件类型手动编写正则规则比较繁琐, 因此 Bot 内置了一组预设分类 (视频、图片、音频、文档、压缩包), 可以通过一条命令批量导入:
|
||||
|
||||
```
|
||||
/rule preset <存储名> [基础路径]
|
||||
```
|
||||
|
||||
参数:
|
||||
|
||||
- `存储名`: 目标存储名 (必须存在且你有权访问)
|
||||
- `基础路径`: 可选. 各预设分类的子目录会创建在此路径下; 若不填则直接使用默认分类目录名
|
||||
|
||||
示例:
|
||||
|
||||
```
|
||||
# 导入预设规则到 "MyAlist", 使用默认目录布局
|
||||
/rule preset MyAlist
|
||||
|
||||
# 在自定义基础路径 "downloads/sorted" 下导入预设规则
|
||||
/rule preset MyAlist downloads/sorted
|
||||
```
|
||||
|
||||
此命令会为每个分类创建 `FILENAME-REGEX` 规则, 将匹配的文件路由到 `基础路径` 下对应的子目录:
|
||||
|
||||
| 分类 | 匹配的扩展名 | 默认目录 |
|
||||
|---|---|---|
|
||||
| 视频 | mp4, mkv, ts, avi, flv, mov, webm, wmv, rmvb, m2ts | `视频` |
|
||||
| 图片 | jpg, jpeg, png, gif, webp, bmp | `图片` |
|
||||
| 音频 | mp3, flac, wav, aac, m4a, ogg | `音频` |
|
||||
| 文档 | pdf, doc, docx, xls, xlsx, ppt, pptx, txt, md, csv, epub, mobi, azw3, chm | `文档` |
|
||||
| 压缩包 | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` |
|
||||
|
||||
{{< hint info >}}
|
||||
导入后的预设规则就是普通的 `FILENAME-REGEX` 规则. 你可以像其他规则一样通过 `/rule` 查看或用 `/rule del <id>` 单独删除/编辑它们.
|
||||
{{< /hint >}}
|
||||
|
||||
规则类型:
|
||||
|
||||
## FILENAME-REGEX
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
{"Target":"book.min.a22f4c7d8c2bdc5e3d6e34ba11cb59ab50ea5772594e71305bfd5a595dc78b7e.css","MediaType":"text/css","Data":{"Integrity":"sha256-oi9MfYwr3F49bjS6EctZq1DqV3JZTnEwW/1aWV3Hi34="}}
|
||||
{"Target":"book.min.a643d39733d3a0ac48d6369128a52703207c5e11a74c3a70cfebfe0c15838ab8.css","MediaType":"text/css","Data":{"Integrity":"sha256-pkPTlzPToKxI1jaRKKUnAyB8XhGnTDpwz+v+DBWDirg="}}
|
||||
80
go.mod
80
go.mod
@@ -11,22 +11,22 @@ require (
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/charmbracelet/log v1.0.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/gabriel-vasile/mimetype v1.4.13
|
||||
github.com/gabriel-vasile/mimetype v1.4.15
|
||||
github.com/goccy/go-yaml v1.19.2
|
||||
github.com/gotd/contrib v0.21.1
|
||||
github.com/gotd/td v0.143.0
|
||||
github.com/gotd/td v0.149.0
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3
|
||||
github.com/krau/ffmpeg-go v0.6.0
|
||||
github.com/lrstanley/go-ytdlp v1.3.5
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/playwright-community/playwright-go v0.5700.1
|
||||
github.com/minio/minio-go/v7 v7.2.1
|
||||
github.com/playwright-community/playwright-go v0.6000.0
|
||||
github.com/rs/xid v1.6.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/unvgo/ghselfupdate v1.0.1
|
||||
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/term v0.44.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/time v0.15.0
|
||||
)
|
||||
|
||||
@@ -43,16 +43,16 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.4 // indirect
|
||||
github.com/cloudflare/circl v1.6.5 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
github.com/deckarep/golang-set/v2 v2.9.0 // indirect
|
||||
github.com/dlclark/regexp2 v1.12.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.6.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/glebarez/go-sqlite v1.22.0 // indirect
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/glebarez/go-sqlite v1.23.0 // indirect
|
||||
github.com/go-faster/errors v0.8.0 // indirect
|
||||
github.com/go-faster/jx v1.2.0 // indirect
|
||||
github.com/go-faster/xor v1.0.0 // indirect
|
||||
github.com/go-faster/yaml v0.4.6 // indirect
|
||||
@@ -63,29 +63,29 @@ require (
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/google/go-github/v30 v30.1.0 // indirect
|
||||
github.com/google/go-querystring v1.2.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 // indirect
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gotd/ige v0.2.2 // indirect
|
||||
github.com/gotd/ige v0.3.0 // indirect
|
||||
github.com/gotd/neo v0.1.5 // indirect
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.27 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 // indirect
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncruces/julianday v1.0.0 // indirect
|
||||
github.com/ogen-go/ogen v1.22.0 // indirect
|
||||
github.com/ogen-go/ogen v1.24.0 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
@@ -94,52 +94,52 @@ require (
|
||||
github.com/segmentio/asm v1.2.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/yuin/goldmark v1.8.2 // indirect
|
||||
github.com/ulikunitz/xz v0.5.16 // indirect
|
||||
github.com/yuin/goldmark v1.8.5 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/zap v1.28.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/tools v0.46.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/mod v0.39.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/libc v1.75.3 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.53.0 // indirect
|
||||
modernc.org/memory v1.12.0 // indirect
|
||||
modernc.org/sqlite v1.56.0 // indirect
|
||||
rsc.io/qr v0.2.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0
|
||||
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.2
|
||||
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6
|
||||
github.com/duke-git/lancet/v2 v2.3.9
|
||||
github.com/fsnotify/fsnotify v1.10.1
|
||||
github.com/glebarez/sqlite v1.11.0
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/ncruces/go-sqlite3 v0.35.1 // indirect
|
||||
github.com/ncruces/go-sqlite3 v0.35.3 // indirect
|
||||
github.com/ncruces/go-sqlite3/gormlite v0.34.0
|
||||
github.com/nicksnyder/go-i18n/v2 v2.6.1
|
||||
github.com/pelletier/go-toml/v2 v2.4.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/xo/terminfo v1.0.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.38.0
|
||||
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da // indirect
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0
|
||||
gorm.io/gorm v1.31.2
|
||||
)
|
||||
|
||||
180
go.sum
180
go.sum
@@ -66,8 +66,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
|
||||
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U=
|
||||
github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY=
|
||||
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
|
||||
github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c=
|
||||
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
|
||||
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
@@ -76,16 +76,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4M5yOCKI=
|
||||
github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.2 h1:x0cvjmUKxt764Yxdk2nr94we1AvPPAMh1rh5TQ+Jo80=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.2/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
|
||||
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0=
|
||||
github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59 h1:DjKLmvKK9u15djHZ88N8M0DhgnHVgJJ8bnEe0h7Lga8=
|
||||
github.com/dop251/goja v0.0.0-20260618133527-c9b2ea77db59/go.mod h1:Sc+QOu1WruvaaeT/cxFez/pXHpI9ZDjg/E8QNfSVveI=
|
||||
github.com/dlclark/regexp2/v2 v2.6.0 h1:KugbSrpXcRpziHIcqqFEXwKwi09LbPkTzvLRItf2mo8=
|
||||
github.com/dlclark/regexp2/v2 v2.6.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6 h1:Oh2rRG1un7tLlC3/NJDzKppZ4CeZGkVFJCUOTRwLpfw=
|
||||
github.com/dop251/goja v0.0.0-20260806115107-493f22071ef6/go.mod h1:LiIEzozrcvNXorsG/3+ypGqdTUAqZryhzSsqi0oU/Qg=
|
||||
github.com/duke-git/lancet/v2 v2.3.9 h1:ZxUvfoEY7YbsGIeoXRxHWIkRCAt6VN7UBKWgCCqBB3U=
|
||||
github.com/duke-git/lancet/v2 v2.3.9/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
@@ -98,19 +98,18 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
|
||||
github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
|
||||
github.com/glebarez/go-sqlite v1.23.0 h1:FyhIq4jqmgphQAUlY79zPldYGwISEZikaDfhiGWkkaI=
|
||||
github.com/glebarez/go-sqlite v1.23.0/go.mod h1:IIYrOH3L0rHY3jb4IXOHoWdklNajSGUN2eJcvK8WrnI=
|
||||
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
|
||||
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
|
||||
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
|
||||
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/go-faster/errors v0.8.0 h1:9T9eJrM+72dFk7n4DfhuaDDe6cyuFCSW2oNUkN77Yqc=
|
||||
github.com/go-faster/errors v0.8.0/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/go-faster/jx v1.2.0 h1:T2YHJPrFaYu21fJtUxC9GzmluKu8rVIFDwwGBKTDseI=
|
||||
github.com/go-faster/jx v1.2.0/go.mod h1:UWLOVDmMG597a5tBFPLIWJdUxz5/2emOpfsj9Neg0PE=
|
||||
github.com/go-faster/xor v0.3.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
|
||||
github.com/go-faster/xor v1.0.0 h1:2o8vTOgErSGHP3/7XwA5ib1FTtUsNtwCoLLBjl31X38=
|
||||
github.com/go-faster/xor v1.0.0/go.mod h1:x5CaDY9UKErKzqfRfFZdfu+OSTfoZny3w5Ak7UxcipQ=
|
||||
github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
|
||||
@@ -119,8 +118,8 @@ github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQ
|
||||
github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
|
||||
github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE=
|
||||
github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
@@ -141,18 +140,18 @@ github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQF
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
|
||||
github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
|
||||
github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE=
|
||||
github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gotd/contrib v0.21.1 h1:NSF+0YEnosQ34QEo2o4s6MA5YFDAor1LVvLhN1L3H1M=
|
||||
github.com/gotd/contrib v0.21.1/go.mod h1:trVJBP9Q/TJbjmJbVnLc0cnX/8T4N0RpQBULVa3BNnE=
|
||||
github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk=
|
||||
github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0=
|
||||
github.com/gotd/ige v0.3.0 h1:4f6LEHWsVDLBG0bT9wWG2/9TZb5aWm265G8ZlTXmRRU=
|
||||
github.com/gotd/ige v0.3.0/go.mod h1:FE9bTaQtvfArizAcZuI4sS6gXaEUBmixdUufVHoCKac=
|
||||
github.com/gotd/neo v0.1.5 h1:oj0iQfMbGClP8xI59x7fE/uHoTJD7NZH9oV1WNuPukQ=
|
||||
github.com/gotd/neo v0.1.5/go.mod h1:9A2a4bn9zL6FADufBdt7tZt+WMhvZoc5gWXihOPoiBQ=
|
||||
github.com/gotd/td v0.143.0 h1:p0U/Nn92zXmAsahDn5CIVzay2kQ36lBBENT/FlWR2nQ=
|
||||
github.com/gotd/td v0.143.0/go.mod h1:8GA5ecTI5iswLwBAlqf0u6/+j+BqSWUARSrX2Xk1usQ=
|
||||
github.com/gotd/td v0.149.0 h1:vXzNO99FFzWKKyI0vizvtgkpI7AqmbcXHLWjxkD+69o=
|
||||
github.com/gotd/td v0.149.0/go.mod h1:+s0fRWlKL+RqoKJAMnCRoj0sWtxzRlWm886elNZEpJQ=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
@@ -165,11 +164,11 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3 h1:2713fQZ560HxoNVgfJH41GKzjMjIG+DW4hH6nYXfXW8=
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250916175020-ebf3e50324d3/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -180,24 +179,24 @@ github.com/krau/ffmpeg-go v0.6.0 h1:F4HWvOrKXQsfLsFTOnUfP0HY6WISJqOrsAFGSIzkKto=
|
||||
github.com/krau/ffmpeg-go v0.6.0/go.mod h1:sa7/bWHB6fO9j4lhmxnWQ1U07o+dE1leFjhctotxU7A=
|
||||
github.com/lrstanley/go-ytdlp v1.3.5 h1:eT+29mK3Lp+XPMQOH25+jVerrrjifYW1o3IkTYJ9SMs=
|
||||
github.com/lrstanley/go-ytdlp v1.3.5/go.mod h1:VgjnTrvkTf+23JuySjyPq1iQ8ijSovBtTPpXH5XrLtI=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss=
|
||||
github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
|
||||
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
|
||||
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs=
|
||||
github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw=
|
||||
github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
@@ -206,10 +205,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/ncruces/go-sqlite3 v0.35.1 h1:h/LaVyQwIvBBT0+2JmVe2tbYyWjUQ093/pYhpBqdxJo=
|
||||
github.com/ncruces/go-sqlite3 v0.35.1/go.mod h1:fXOSIkWwN5NXgbJk+7Zls8QIW4xOflmgh11OFvcY+J0=
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302 h1:Cew7/eNAMd1zhpXYBjofBua/63pFvbvB2h4PM/p6gKU=
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.1.35302/go.mod h1:xe0CfafDUxfh+fSVKjHHMiAxoG9KALt5nFtbGNb/jRs=
|
||||
github.com/ncruces/go-sqlite3 v0.35.3 h1:Ei07Zv1qfV/vyXzelhFsyS5Oh9TArBZHsmFk14Xv3GY=
|
||||
github.com/ncruces/go-sqlite3 v0.35.3/go.mod h1:i1rhym/NIiB5xeEfzbN+e24Y+i7NGUpf7C2xZ3Dpwks=
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304 h1:5NoQAewtgKNK3G4bjNPxVoGXu6F6NzLXWCTdD5FFAEY=
|
||||
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35304/go.mod h1:o8gr9w/50fXA5TDskg6bNUjvqmFfw4KaXth4q+yDSjg=
|
||||
github.com/ncruces/go-sqlite3/gormlite v0.34.0 h1:QLlOy/i7OabsFUQ+d5KyXmq2hw9sMh/CRW435+eQMRY=
|
||||
github.com/ncruces/go-sqlite3/gormlite v0.34.0/go.mod h1:CMv+6YhqLmPBXYACiQtrWA0q/JLIMTKB4E65SUfLgF0=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
@@ -218,24 +217,24 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
|
||||
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
|
||||
github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
|
||||
github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
|
||||
github.com/ogen-go/ogen v1.22.0 h1:7wU+jcIKg/JBAhM95909ULLdAkGr43KQOuvNpJ7Mxb4=
|
||||
github.com/ogen-go/ogen v1.22.0/go.mod h1:7BOh9a51QiPCC92RMrj1LlkLjejhBAyPhR+oMc6lR9g=
|
||||
github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
|
||||
github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/ogen-go/ogen v1.24.0 h1:NehBG/8s0JeM6jYHPJeFJntBG3cE/xQgZd1q8BchPNM=
|
||||
github.com/ogen-go/ogen v1.24.0/go.mod h1:hcg4aTzLcu/MS1SGGahUriPBEOZB3c82RGIC6dYcdFU=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/playwright-community/playwright-go v0.5700.1 h1:PNFb1byWqrTT720rEO0JL88C6Ju0EmUnR5deFLvtP/U=
|
||||
github.com/playwright-community/playwright-go v0.5700.1/go.mod h1:MlSn1dZrx8rszbCxY6x3qK89ZesJUYVx21B2JnkoNF0=
|
||||
github.com/playwright-community/playwright-go v0.6000.0 h1:R7sENcRI6n0Zd5ZoW8EKdVF1ZVJgvTubfJeqKHNGvsw=
|
||||
github.com/playwright-community/playwright-go v0.6000.0/go.mod h1:z/YpFVdU4LAi+0f9VPOCkGvmdH6dCrtza9nxnXFXgiE=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
||||
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
@@ -273,17 +272,17 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/unvgo/ghselfupdate v1.0.1 h1:4clbOkfPbfEmRnnYxVXDSBs0JG12DO+0FfqplJckreU=
|
||||
github.com/unvgo/ghselfupdate v1.0.1/go.mod h1:3snWV5vEHGXQqqhY7FwKjPOtH6e7cFdHYN7UMAihhxs=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY=
|
||||
github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c h1:xA2TJS9Hu/ivzaZIrDcwvpJ3Fnpsk5fDOJ4iSnL6J0w=
|
||||
github.com/yapingcat/gomedia v0.0.0-20240906162731-17feea57090c/go.mod h1:WSZ59bidJOO40JSJmLqlkBJrjZCtjbKKkygEMfzY/kc=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA=
|
||||
github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
@@ -294,12 +293,12 @@ go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8=
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
@@ -310,34 +309,35 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
|
||||
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da h1:YKw1FZDyWyXXZcBxalRHz3CHieUKnKxanrbcO280Zwc=
|
||||
golang.org/x/exp v0.0.0-20260810151157-a8b543ca52da/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
|
||||
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -347,31 +347,31 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
|
||||
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
|
||||
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -391,30 +391,30 @@ gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
|
||||
gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo=
|
||||
gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
|
||||
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
|
||||
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
|
||||
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
|
||||
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
|
||||
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
|
||||
modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
|
||||
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
|
||||
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
|
||||
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
|
||||
modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g=
|
||||
modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU=
|
||||
modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
|
||||
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
|
||||
modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
|
||||
modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
|
||||
17
pkg/storagetypes/batch.go
Normal file
17
pkg/storagetypes/batch.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package storagetypes
|
||||
|
||||
import "io"
|
||||
|
||||
// BatchItem describes one seekable file in a logical batch storage operation.
|
||||
type BatchItem struct {
|
||||
Reader io.ReadSeeker
|
||||
StoragePath string
|
||||
Size int64
|
||||
|
||||
// SourceGroupKey is empty for standalone source messages.
|
||||
SourceGroupKey string
|
||||
Caption string
|
||||
// PreserveCaption distinguishes an intentionally empty source caption from
|
||||
// the storage backend's default caption.
|
||||
PreserveCaption bool
|
||||
}
|
||||
19
pkg/storagetypes/context.go
Normal file
19
pkg/storagetypes/context.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package storagetypes
|
||||
|
||||
import "context"
|
||||
|
||||
type sourceCaptionContextKey struct{}
|
||||
|
||||
// WithSourceCaption records the source Telegram message caption for storage
|
||||
// backends that can preserve it. Calling this function with an empty caption
|
||||
// intentionally suppresses a backend-generated fallback caption.
|
||||
func WithSourceCaption(ctx context.Context, caption string) context.Context {
|
||||
return context.WithValue(ctx, sourceCaptionContextKey{}, caption)
|
||||
}
|
||||
|
||||
// SourceCaptionFromContext returns the source caption and whether the caller
|
||||
// explicitly supplied one.
|
||||
func SourceCaptionFromContext(ctx context.Context) (string, bool) {
|
||||
caption, ok := ctx.Value(sourceCaptionContextKey{}).(string)
|
||||
return caption, ok
|
||||
}
|
||||
24
pkg/storagetypes/context_test.go
Normal file
24
pkg/storagetypes/context_test.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package storagetypes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSourceCaptionContext(t *testing.T) {
|
||||
if _, ok := SourceCaptionFromContext(context.Background()); ok {
|
||||
t.Fatal("caption unexpectedly present on empty context")
|
||||
}
|
||||
|
||||
tests := []string{"original caption", ""}
|
||||
for _, want := range tests {
|
||||
ctx := WithSourceCaption(context.Background(), want)
|
||||
got, ok := SourceCaptionFromContext(ctx)
|
||||
if !ok {
|
||||
t.Fatalf("caption %q was not marked as present", want)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("caption = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package tcbdata
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
|
||||
"github.com/krau/SaveAny-Bot/pkg/parser"
|
||||
"github.com/krau/SaveAny-Bot/pkg/telegraph"
|
||||
@@ -31,12 +33,7 @@ func ConflictStrategyValues() []string {
|
||||
}
|
||||
|
||||
func IsConflictStrategy(strategy string) bool {
|
||||
for _, value := range ConflictStrategyValues() {
|
||||
if strategy == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.Contains(ConflictStrategyValues(), strategy)
|
||||
}
|
||||
|
||||
// type TaskDataTGFiles struct {
|
||||
|
||||
@@ -31,6 +31,38 @@ type StorageCannotStream interface {
|
||||
CannotStream() string
|
||||
}
|
||||
|
||||
// StorageBatchSaver can preserve relationships between files when saving a
|
||||
// logical batch, such as a Telegram media album.
|
||||
type StorageBatchSaver interface {
|
||||
Storage
|
||||
SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error
|
||||
}
|
||||
|
||||
// StorageBatchProgressSaver reports confirmed upload progress for each item in
|
||||
// a logical batch. The item index matches the items slice passed to
|
||||
// SaveBatchWithProgress.
|
||||
type StorageBatchProgressSaver interface {
|
||||
StorageBatchSaver
|
||||
SaveBatchWithProgress(
|
||||
ctx context.Context,
|
||||
items []storagetypes.BatchItem,
|
||||
onProgress func(index int, uploaded, total int64),
|
||||
) error
|
||||
}
|
||||
|
||||
// StorageProgressSaver reports bytes after the backend has accepted them for
|
||||
// upload. Backends with native progress support should implement this instead
|
||||
// of relying on progress inferred from reads of the input stream.
|
||||
type StorageProgressSaver interface {
|
||||
Storage
|
||||
SaveWithProgress(
|
||||
ctx context.Context,
|
||||
reader io.Reader,
|
||||
storagePath string,
|
||||
onProgress func(uploaded, total int64),
|
||||
) error
|
||||
}
|
||||
|
||||
// StorageListable 表示支持列举目录内容的存储
|
||||
type StorageListable interface {
|
||||
Storage
|
||||
@@ -45,6 +77,9 @@ type StorageReadable interface {
|
||||
|
||||
var Storages = make(map[string]Storage)
|
||||
|
||||
var _ StorageProgressSaver = (*telegram.Telegram)(nil)
|
||||
var _ StorageBatchProgressSaver = (*telegram.Telegram)(nil)
|
||||
|
||||
type StorageConstructor func() Storage
|
||||
|
||||
var storageConstructors = map[storenum.StorageType]StorageConstructor{
|
||||
|
||||
140
storage/telegram/media_group_test.go
Normal file
140
storage/telegram/media_group_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
)
|
||||
|
||||
func TestPlanMediaGroups(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
items []batchMediaItem
|
||||
wantSizes []int
|
||||
}{
|
||||
{
|
||||
name: "same source album",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 1, true),
|
||||
},
|
||||
wantSizes: []int{2},
|
||||
},
|
||||
{
|
||||
name: "different source albums",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("b", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "ungrouped messages",
|
||||
items: []batchMediaItem{
|
||||
albumItem("", 1, true),
|
||||
albumItem("", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "different target chats",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 2, true),
|
||||
},
|
||||
wantSizes: []int{1, 1},
|
||||
},
|
||||
{
|
||||
name: "ineligible media does not bridge albums",
|
||||
items: []batchMediaItem{
|
||||
albumItem("a", 1, true),
|
||||
albumItem("a", 1, false),
|
||||
albumItem("a", 1, true),
|
||||
},
|
||||
wantSizes: []int{1, 1, 1},
|
||||
},
|
||||
{
|
||||
name: "maximum album size",
|
||||
items: repeatedAlbumItems(11),
|
||||
wantSizes: []int{10, 1},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
groups := planMediaGroups(tt.items)
|
||||
if len(groups) != len(tt.wantSizes) {
|
||||
t.Fatalf("got %d groups, want %d", len(groups), len(tt.wantSizes))
|
||||
}
|
||||
for i, want := range tt.wantSizes {
|
||||
if got := len(groups[i]); got != want {
|
||||
t.Errorf("group %d has %d items, want %d", i, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaCaption(t *testing.T) {
|
||||
empty := ""
|
||||
original := "original caption"
|
||||
tests := []struct {
|
||||
name string
|
||||
override *string
|
||||
wantLen int
|
||||
}{
|
||||
{name: "filename fallback", wantLen: 1},
|
||||
{name: "preserve empty source caption", override: &empty, wantLen: 0},
|
||||
{name: "preserve source caption", override: &original, wantLen: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := len(mediaCaption("file.jpg", tt.override)); got != tt.wantLen {
|
||||
t.Fatalf("got %d caption options, want %d", got, tt.wantLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectBatchItemRewindsBeforeMimetypeDetection(t *testing.T) {
|
||||
data := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
|
||||
reader := bytes.NewReader(data)
|
||||
if _, err := reader.Seek(4, io.SeekStart); err != nil {
|
||||
t.Fatalf("failed to set initial reader offset: %v", err)
|
||||
}
|
||||
|
||||
mediaItem, err := new(Telegram).inspectBatchItem(nil, storagetypes.BatchItem{
|
||||
Reader: reader,
|
||||
StoragePath: "photo.png",
|
||||
Size: int64(len(data)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspectBatchItem returned an error: %v", err)
|
||||
}
|
||||
if !mediaItem.albumEligible {
|
||||
t.Fatal("albumEligible = false, want true for PNG input")
|
||||
}
|
||||
if offset, err := reader.Seek(0, io.SeekCurrent); err != nil {
|
||||
t.Fatalf("failed to get final reader offset: %v", err)
|
||||
} else if offset != 0 {
|
||||
t.Fatalf("reader offset = %d, want 0", offset)
|
||||
}
|
||||
}
|
||||
|
||||
func albumItem(group string, chatID int64, eligible bool) batchMediaItem {
|
||||
return batchMediaItem{
|
||||
item: storagetypes.BatchItem{SourceGroupKey: group},
|
||||
chatID: chatID,
|
||||
albumEligible: eligible,
|
||||
}
|
||||
}
|
||||
|
||||
func repeatedAlbumItems(count int) []batchMediaItem {
|
||||
items := make([]batchMediaItem, count)
|
||||
for i := range items {
|
||||
items[i] = albumItem("a", 1, true)
|
||||
}
|
||||
return items
|
||||
}
|
||||
60
storage/telegram/progress.go
Normal file
60
storage/telegram/progress.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/gotd/td/telegram/uploader"
|
||||
)
|
||||
|
||||
var _ uploader.Progress = (*uploadProgress)(nil)
|
||||
|
||||
type uploadProgress struct {
|
||||
mu sync.Mutex
|
||||
onProgress func(uploaded, total int64)
|
||||
total int64
|
||||
uploaded int64
|
||||
byID map[int64]int64
|
||||
}
|
||||
|
||||
func newUploadProgress(total int64, onProgress func(uploaded, total int64)) *uploadProgress {
|
||||
return &uploadProgress{
|
||||
onProgress: onProgress,
|
||||
total: total,
|
||||
byID: make(map[int64]int64),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *uploadProgress) Chunk(ctx context.Context, state uploader.ProgressState) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
previous := p.byID[state.ID]
|
||||
if state.Uploaded <= previous {
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
p.byID[state.ID] = state.Uploaded
|
||||
p.uploaded += state.Uploaded - previous
|
||||
uploaded := p.uploaded
|
||||
total := p.total
|
||||
if total <= 0 {
|
||||
total = state.Total
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.onProgress != nil {
|
||||
p.onProgress(uploaded, total)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *uploadProgress) reset(total int64) {
|
||||
p.mu.Lock()
|
||||
p.total = total
|
||||
p.uploaded = 0
|
||||
p.byID = make(map[int64]int64)
|
||||
p.mu.Unlock()
|
||||
}
|
||||
80
storage/telegram/progress_test.go
Normal file
80
storage/telegram/progress_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/telegram/uploader"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
)
|
||||
|
||||
func TestUploadProgressAggregatesUploaderParts(t *testing.T) {
|
||||
type update struct {
|
||||
uploaded int64
|
||||
total int64
|
||||
}
|
||||
var updates []update
|
||||
progress := newUploadProgress(100, func(uploaded, total int64) {
|
||||
updates = append(updates, update{uploaded: uploaded, total: total})
|
||||
})
|
||||
|
||||
states := []uploader.ProgressState{
|
||||
{ID: 1, Uploaded: 20, Total: 60},
|
||||
{ID: 1, Uploaded: 20, Total: 60},
|
||||
{ID: 1, Uploaded: 60, Total: 60},
|
||||
{ID: 2, Uploaded: 10, Total: 40},
|
||||
{ID: 2, Uploaded: 40, Total: 40},
|
||||
}
|
||||
for _, state := range states {
|
||||
if err := progress.Chunk(context.Background(), state); err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
want := []update{{20, 100}, {60, 100}, {70, 100}, {100, 100}}
|
||||
if len(updates) != len(want) {
|
||||
t.Fatalf("got %d updates, want %d", len(updates), len(want))
|
||||
}
|
||||
for i := range want {
|
||||
if updates[i] != want[i] {
|
||||
t.Fatalf("update %d = %+v, want %+v", i, updates[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadProgressResetForSplitFiles(t *testing.T) {
|
||||
var uploaded, total int64
|
||||
progress := newUploadProgress(100, func(current, size int64) {
|
||||
uploaded, total = current, size
|
||||
})
|
||||
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 100, Total: 100}); err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
|
||||
progress.reset(120)
|
||||
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 2, Uploaded: 30, Total: 60}); err != nil {
|
||||
t.Fatalf("Chunk() after reset failed: %v", err)
|
||||
}
|
||||
if uploaded != 30 || total != 120 {
|
||||
t.Fatalf("progress after reset = %d/%d, want 30/120", uploaded, total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchItemUploadProgressPreservesItemIndex(t *testing.T) {
|
||||
var gotIndex int
|
||||
var gotUploaded, gotTotal int64
|
||||
progress := batchItemUploadProgress(batchMediaItem{
|
||||
index: 4,
|
||||
item: storagetypes.BatchItem{Size: 100},
|
||||
}, func(index int, uploaded, total int64) {
|
||||
gotIndex = index
|
||||
gotUploaded = uploaded
|
||||
gotTotal = total
|
||||
})
|
||||
if err := progress.Chunk(context.Background(), uploader.ProgressState{ID: 1, Uploaded: 25, Total: 100}); err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
if gotIndex != 4 || gotUploaded != 25 || gotTotal != 100 {
|
||||
t.Fatalf("batch progress = index %d, %d/%d; want index 4, 25/100", gotIndex, gotUploaded, gotTotal)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/duke-git/lancet/v2/slice"
|
||||
"github.com/duke-git/lancet/v2/validator"
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
@@ -26,14 +27,16 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/pkg/consts/tglimit"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
"github.com/rs/xid"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
// https://core.telegram.org/api/config#upload-max-fileparts-default
|
||||
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
|
||||
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
|
||||
DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB
|
||||
MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB
|
||||
PremiumMaxUploadFileSize = 8000 * 524288 // 8000 * 512 KB
|
||||
)
|
||||
|
||||
type Telegram struct {
|
||||
@@ -41,6 +44,20 @@ type Telegram struct {
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
type preparedMedia struct {
|
||||
peer tg.InputPeerClass
|
||||
uploader *uploader.Uploader
|
||||
media message.MultiMediaOption
|
||||
}
|
||||
|
||||
type batchMediaItem struct {
|
||||
item storagetypes.BatchItem
|
||||
index int
|
||||
chatID int64
|
||||
albumEligible bool
|
||||
useSingleSave bool
|
||||
}
|
||||
|
||||
func (t *Telegram) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
|
||||
telegramConfig, ok := cfg.(*storconfig.TelegramStorageConfig)
|
||||
if !ok {
|
||||
@@ -71,37 +88,147 @@ func (t *Telegram) Exists(ctx context.Context, storagePath string) bool {
|
||||
}
|
||||
|
||||
func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) error {
|
||||
return t.save(ctx, r, storagePath, nil)
|
||||
}
|
||||
|
||||
// SaveWithProgress saves a file while reporting Telegram-confirmed upload
|
||||
// progress after each uploaded part.
|
||||
func (t *Telegram) SaveWithProgress(
|
||||
ctx context.Context,
|
||||
r io.Reader,
|
||||
storagePath string,
|
||||
onProgress func(uploaded, total int64),
|
||||
) error {
|
||||
size := contentLength(ctx)
|
||||
return t.save(ctx, r, storagePath, newUploadProgress(size, onProgress))
|
||||
}
|
||||
|
||||
func (t *Telegram) save(ctx context.Context, r io.Reader, storagePath string, progress *uploadProgress) error {
|
||||
storagePath = path.Clean(storagePath)
|
||||
captionOverride := sourceCaptionOverride(ctx)
|
||||
tctx := tgutil.ExtFromContext(ctx)
|
||||
if tctx == nil {
|
||||
return fmt.Errorf("failed to get telegram context")
|
||||
}
|
||||
size := func() int64 {
|
||||
if length := ctx.Value(ctxkey.ContentLength); length != nil {
|
||||
if l, ok := length.(int64); ok {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return -1 // unknown size
|
||||
}()
|
||||
if t.config.SkipLarge && size > MaxUploadFileSize {
|
||||
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", MaxUploadFileSize, size)
|
||||
size := contentLength(ctx)
|
||||
maxUploadSize := maxUploadFileSize(tctx)
|
||||
if t.config.SkipLarge && size > maxUploadSize {
|
||||
log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", maxUploadSize, size)
|
||||
return nil
|
||||
}
|
||||
rs, seekable := r.(io.ReadSeeker)
|
||||
splitSize := t.config.SplitSizeMB * 1024 * 1024
|
||||
if splitSize <= 0 {
|
||||
splitSize = DefaultSplitSize
|
||||
splitSize := t.splitSize(maxUploadSize)
|
||||
if size > splitSize {
|
||||
filename, chatID := t.target(tctx, storagePath)
|
||||
if filename == "" {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
mtype, err := mimetype.DetectReader(rs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
}
|
||||
filename = xid.New().String() + mtype.Extension()
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek reader: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
upler := t.newUploader(tctx, size, progress)
|
||||
peer := tryGetInputPeer(tctx, chatID)
|
||||
if peer == nil || peer.Zero() {
|
||||
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
}
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
if t.config.SplitLargeVideo {
|
||||
rs, ok := r.(io.ReadSeeker)
|
||||
if ok {
|
||||
mtype, detectErr := mimetype.DetectReader(rs)
|
||||
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return fmt.Errorf("failed to seek large file after mimetype detection: %w", seekErr)
|
||||
}
|
||||
if detectErr != nil {
|
||||
log.FromContext(ctx).Warnf("Failed to detect large file type, falling back to ZIP split: %s", detectErr)
|
||||
} else if strings.HasPrefix(mtype.String(), "video/") {
|
||||
parts, cleanup, splitErr := createLosslessVideoParts(
|
||||
ctx,
|
||||
rs,
|
||||
filename,
|
||||
size,
|
||||
splitSize,
|
||||
)
|
||||
if splitErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
log.FromContext(ctx).Warnf("Lossless video split failed, falling back to ZIP split: %s", splitErr)
|
||||
} else {
|
||||
defer cleanup()
|
||||
log.FromContext(ctx).Infof("Uploading oversized video as %d lossless-playable parts", len(parts))
|
||||
for _, part := range parts {
|
||||
log.FromContext(ctx).Infof("Prepared lossless video part %s (%d bytes)", part.Name, part.Size)
|
||||
}
|
||||
return t.uploadLosslessVideoParts(ctx, tctx, storagePath, parts, captionOverride, progress)
|
||||
}
|
||||
if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return fmt.Errorf("failed to seek large video before ZIP fallback: %w", seekErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize, progress)
|
||||
}
|
||||
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
prepared, err := t.prepareMedia(ctx, tctx, r, storagePath, size, nil, progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tctx.Sender.
|
||||
WithUploader(prepared.uploader).
|
||||
To(prepared.peer).
|
||||
Media(ctx, prepared.media)
|
||||
return err
|
||||
}
|
||||
|
||||
func contentLength(ctx context.Context) int64 {
|
||||
if length := ctx.Value(ctxkey.ContentLength); length != nil {
|
||||
if size, ok := length.(int64); ok {
|
||||
return size
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func sourceCaptionOverride(ctx context.Context) *string {
|
||||
caption, ok := storagetypes.SourceCaptionFromContext(ctx)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &caption
|
||||
}
|
||||
|
||||
func maxUploadFileSize(tctx *ext.Context) int64 {
|
||||
if tctx != nil && tctx.Self != nil && !tctx.Self.GetBot() && tctx.Self.GetPremium() {
|
||||
return PremiumMaxUploadFileSize
|
||||
}
|
||||
return MaxUploadFileSize
|
||||
}
|
||||
|
||||
func (t *Telegram) splitSize(maxUploadSize int64) int64 {
|
||||
splitSize := t.config.SplitSizeMB * 1024 * 1024
|
||||
if splitSize <= 0 {
|
||||
return maxUploadSize
|
||||
}
|
||||
return min(splitSize, maxUploadSize)
|
||||
}
|
||||
|
||||
func (t *Telegram) target(tctx *ext.Context, storagePath string) (string, int64) {
|
||||
// 去除前导斜杠并分隔路径, 当 len(parts):
|
||||
// ==0, 存储到配置文件中的 chat_id, 随机文件名
|
||||
// ==1, 视作只有文件名, 存储到配置文件中的 chat_id
|
||||
// ==2, parts[0]: 视作要存储到的 chat_id, parts[1]: filename
|
||||
// >=2, parts[0]: 视作要存储到的 chat_id, 最后一项为 filename
|
||||
parts := slice.Compact(strings.Split(strings.TrimPrefix(storagePath, "/"), "/"))
|
||||
filename := ""
|
||||
chatID := t.config.ChatID
|
||||
@@ -111,38 +238,66 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
if len(parts) >= 2 && validator.IsAlphaNumeric(parts[0]) {
|
||||
cid, err := tgutil.ParseChatID(tctx, parts[0])
|
||||
if err != nil {
|
||||
// id不合法时使用配置文件中的 chat_id
|
||||
log.FromContext(ctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
|
||||
log.FromContext(tctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
|
||||
cid = chatID
|
||||
}
|
||||
chatID = cid
|
||||
}
|
||||
return filename, chatID
|
||||
}
|
||||
|
||||
func (t *Telegram) newUploader(tctx *ext.Context, size int64, progress *uploadProgress) *uploader.Uploader {
|
||||
upler := uploader.NewUploader(tctx.Raw).
|
||||
WithPartSize(tglimit.MaxUploadPartSize).
|
||||
WithThreads(dlutil.BestThreads(size, config.C().Threads))
|
||||
if progress != nil {
|
||||
upler = upler.WithProgress(progress)
|
||||
}
|
||||
return upler
|
||||
}
|
||||
|
||||
func mediaCaption(filename string, override *string) []message.StyledTextOption {
|
||||
if override == nil {
|
||||
return []message.StyledTextOption{styling.Plain(filename)}
|
||||
}
|
||||
if *override == "" {
|
||||
return nil
|
||||
}
|
||||
return []message.StyledTextOption{styling.Plain(*override)}
|
||||
}
|
||||
|
||||
func (t *Telegram) prepareMedia(
|
||||
ctx context.Context,
|
||||
tctx *ext.Context,
|
||||
r io.Reader,
|
||||
storagePath string,
|
||||
size int64,
|
||||
captionOverride *string,
|
||||
progress *uploadProgress,
|
||||
) (*preparedMedia, error) {
|
||||
storagePath = path.Clean(storagePath)
|
||||
filename, chatID := t.target(tctx, storagePath)
|
||||
upler := t.newUploader(tctx, size, progress)
|
||||
peer := tryGetInputPeer(tctx, chatID)
|
||||
if peer == nil || peer.Zero() {
|
||||
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
return nil, fmt.Errorf("failed to get input peer for chat ID %d", chatID)
|
||||
}
|
||||
|
||||
rs, seekable := r.(io.ReadSeeker)
|
||||
var mtype *mimetype.MIME
|
||||
if seekable {
|
||||
var err error
|
||||
mtype, err = mimetype.DetectReader(rs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
return nil, fmt.Errorf("failed to detect mimetype: %w", err)
|
||||
}
|
||||
if filename == "" {
|
||||
filename = xid.New().String() + mtype.Extension()
|
||||
}
|
||||
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek reader: %w", err)
|
||||
return nil, fmt.Errorf("failed to seek reader: %w", err)
|
||||
}
|
||||
}
|
||||
if size > splitSize {
|
||||
// large file, use split uploader
|
||||
return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize)
|
||||
}
|
||||
|
||||
var file tg.InputFileClass
|
||||
var err error
|
||||
@@ -152,21 +307,20 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
file, err = upler.Upload(ctx, uploader.NewUpload(filename, r, size))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file to telegram: %w", err)
|
||||
return nil, fmt.Errorf("failed to upload file to telegram: %w", err)
|
||||
}
|
||||
caption := styling.Plain(filename)
|
||||
caption := mediaCaption(filename, captionOverride)
|
||||
forceFile := t.config.ForceFile
|
||||
|
||||
if mtype != nil && strings.HasPrefix(mtype.String(), "image/") && size >= tglimit.MaxPhotoSize {
|
||||
forceFile = true
|
||||
}
|
||||
doc := message.UploadedDocument(file, caption).
|
||||
doc := message.UploadedDocument(file, caption...).
|
||||
Filename(filename).
|
||||
ForceFile(forceFile)
|
||||
if mtype != nil {
|
||||
doc = doc.MIME(mtype.String())
|
||||
}
|
||||
var media message.MediaOption = doc
|
||||
var media message.MultiMediaOption = doc
|
||||
if mtype != nil && rs != nil {
|
||||
switch mtypeStr := mtype.String(); {
|
||||
case strings.HasPrefix(mtypeStr, "video/"):
|
||||
@@ -205,19 +359,194 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
|
||||
case strings.HasPrefix(mtypeStr, "audio/"):
|
||||
media = doc.Audio().Title(filename)
|
||||
case strings.HasPrefix(mtypeStr, "image/") && !strings.HasSuffix(mtypeStr, "webp"):
|
||||
media = message.UploadedPhoto(file, caption)
|
||||
media = message.UploadedPhoto(file, caption...)
|
||||
}
|
||||
}
|
||||
sender := tctx.Sender
|
||||
_, err = sender.WithUploader(upler).To(peer).Media(ctx, media)
|
||||
return err
|
||||
return &preparedMedia{
|
||||
peer: peer,
|
||||
uploader: upler,
|
||||
media: media,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SaveBatch preserves each source photo/video group as a Telegram album.
|
||||
func (t *Telegram) SaveBatch(ctx context.Context, items []storagetypes.BatchItem) error {
|
||||
return t.saveBatch(ctx, items, nil)
|
||||
}
|
||||
|
||||
// SaveBatchWithProgress preserves source media groups while reporting native
|
||||
// Telegram upload progress for each input item.
|
||||
func (t *Telegram) SaveBatchWithProgress(
|
||||
ctx context.Context,
|
||||
items []storagetypes.BatchItem,
|
||||
onProgress func(index int, uploaded, total int64),
|
||||
) error {
|
||||
return t.saveBatch(ctx, items, onProgress)
|
||||
}
|
||||
|
||||
func (t *Telegram) saveBatch(
|
||||
ctx context.Context,
|
||||
items []storagetypes.BatchItem,
|
||||
onProgress func(index int, uploaded, total int64),
|
||||
) error {
|
||||
tctx := tgutil.ExtFromContext(ctx)
|
||||
if tctx == nil {
|
||||
return fmt.Errorf("failed to get telegram context")
|
||||
}
|
||||
|
||||
inspected := make([]batchMediaItem, 0, len(items))
|
||||
for index, item := range items {
|
||||
mediaItem, err := t.inspectBatchItem(tctx, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mediaItem.index = index
|
||||
inspected = append(inspected, mediaItem)
|
||||
}
|
||||
for _, group := range planMediaGroups(inspected) {
|
||||
if err := t.saveMediaGroup(ctx, tctx, group, onProgress); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Telegram) inspectBatchItem(tctx *ext.Context, item storagetypes.BatchItem) (batchMediaItem, error) {
|
||||
_, chatID := t.target(tctx, path.Clean(item.StoragePath))
|
||||
result := batchMediaItem{item: item, chatID: chatID}
|
||||
maxUploadSize := maxUploadFileSize(tctx)
|
||||
if (t.config.SkipLarge && item.Size > maxUploadSize) ||
|
||||
item.Size > t.splitSize(maxUploadSize) {
|
||||
result.useSingleSave = true
|
||||
return result, nil
|
||||
}
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("failed to seek batch item before mimetype detection: %w", err)
|
||||
}
|
||||
mtype, err := mimetype.DetectReader(item.Reader)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to detect batch item mimetype: %w", err)
|
||||
}
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return result, fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
mtypeStr := mtype.String()
|
||||
forceFile := t.config.ForceFile || strings.HasPrefix(mtypeStr, "image/") && item.Size >= tglimit.MaxPhotoSize
|
||||
result.albumEligible = !forceFile && (strings.HasPrefix(mtypeStr, "video/") ||
|
||||
strings.HasPrefix(mtypeStr, "image/") && mtypeStr != "image/webp" && mtypeStr != "image/gif")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func planMediaGroups(items []batchMediaItem) [][]batchMediaItem {
|
||||
groups := make([][]batchMediaItem, 0, len(items))
|
||||
for i := 0; i < len(items); {
|
||||
item := items[i]
|
||||
if item.useSingleSave || !item.albumEligible || item.item.SourceGroupKey == "" {
|
||||
groups = append(groups, items[i:i+1])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
end := i + 1
|
||||
for end < len(items) && end-i < 10 {
|
||||
next := items[end]
|
||||
if next.useSingleSave || !next.albumEligible || next.chatID != item.chatID || next.item.SourceGroupKey != item.item.SourceGroupKey {
|
||||
break
|
||||
}
|
||||
end++
|
||||
}
|
||||
groups = append(groups, items[i:end])
|
||||
i = end
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func batchItemUploadProgress(
|
||||
mediaItem batchMediaItem,
|
||||
onProgress func(index int, uploaded, total int64),
|
||||
) *uploadProgress {
|
||||
if onProgress == nil {
|
||||
return nil
|
||||
}
|
||||
return newUploadProgress(mediaItem.item.Size, func(uploaded, total int64) {
|
||||
onProgress(mediaItem.index, uploaded, total)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Telegram) saveMediaGroup(
|
||||
ctx context.Context,
|
||||
tctx *ext.Context,
|
||||
group []batchMediaItem,
|
||||
onProgress func(index int, uploaded, total int64),
|
||||
) error {
|
||||
return retry.Retry(func() error {
|
||||
if len(group) == 1 && group[0].useSingleSave {
|
||||
mediaItem := group[0]
|
||||
item := mediaItem.item
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
itemCtx := context.WithValue(ctx, ctxkey.ContentLength, item.Size)
|
||||
if item.PreserveCaption {
|
||||
itemCtx = storagetypes.WithSourceCaption(itemCtx, item.Caption)
|
||||
}
|
||||
if onProgress == nil {
|
||||
return t.Save(itemCtx, item.Reader, item.StoragePath)
|
||||
}
|
||||
return t.SaveWithProgress(itemCtx, item.Reader, item.StoragePath, func(uploaded, total int64) {
|
||||
onProgress(mediaItem.index, uploaded, total)
|
||||
})
|
||||
}
|
||||
if err := t.limiter.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("rate limit failed: %w", err)
|
||||
}
|
||||
|
||||
prepared := make([]preparedMedia, 0, len(group))
|
||||
for _, mediaItem := range group {
|
||||
item := mediaItem.item
|
||||
if _, err := item.Reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("failed to seek batch item: %w", err)
|
||||
}
|
||||
var captionOverride *string
|
||||
if item.PreserveCaption {
|
||||
captionOverride = &item.Caption
|
||||
}
|
||||
progress := batchItemUploadProgress(mediaItem, onProgress)
|
||||
media, err := t.prepareMedia(ctx, tctx, item.Reader, item.StoragePath, item.Size, captionOverride, progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prepared = append(prepared, *media)
|
||||
}
|
||||
|
||||
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
|
||||
if len(prepared) == 1 {
|
||||
_, err := builder.Media(ctx, prepared[0].media)
|
||||
return err
|
||||
}
|
||||
media := make([]message.MultiMediaOption, len(prepared))
|
||||
for i := range prepared {
|
||||
media[i] = prepared[i].media
|
||||
}
|
||||
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
|
||||
return fmt.Errorf("failed to send media album: %w", err)
|
||||
}
|
||||
return nil
|
||||
}, retry.Context(ctx), retry.RetryTimes(uint(config.C().Retry)))
|
||||
}
|
||||
|
||||
func (t *Telegram) CannotStream() string {
|
||||
return "Telegram storage must use a ReaderSeeker"
|
||||
}
|
||||
|
||||
func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, upler *uploader.Uploader, peer tg.InputPeerClass, fileSize, splitSize int64) error {
|
||||
func (t *Telegram) splitUpload(
|
||||
ctx *ext.Context,
|
||||
r io.Reader,
|
||||
filename string,
|
||||
upler *uploader.Uploader,
|
||||
peer tg.InputPeerClass,
|
||||
fileSize, splitSize int64,
|
||||
progress *uploadProgress,
|
||||
) error {
|
||||
tempId := xid.New().String()
|
||||
outputBase := filepath.Join(config.C().Temp.BasePath, tempId, strings.Split(filename, ".")[0])
|
||||
defer func() {
|
||||
@@ -234,6 +563,17 @@ func (t *Telegram) splitUpload(ctx *ext.Context, r io.Reader, filename string, u
|
||||
return fmt.Errorf("failed to glob split files: %w", err)
|
||||
}
|
||||
inputFiles := make([]tg.InputFileClass, 0, len(matched))
|
||||
if progress != nil {
|
||||
var uploadSize int64
|
||||
for _, partPath := range matched {
|
||||
partInfo, err := os.Stat(partPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stat split part %s: %w", partPath, err)
|
||||
}
|
||||
uploadSize += partInfo.Size()
|
||||
}
|
||||
progress.reset(uploadSize)
|
||||
}
|
||||
for _, partPath := range matched {
|
||||
// 串行上传, 不然容易被tg风控
|
||||
err = func() error {
|
||||
|
||||
119
storage/telegram/upload_limit_test.go
Normal file
119
storage/telegram/upload_limit_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/pkg/storagetypes"
|
||||
)
|
||||
|
||||
func TestMaxUploadFileSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx *ext.Context
|
||||
want int64
|
||||
}{
|
||||
{name: "missing context", want: MaxUploadFileSize},
|
||||
{name: "missing self", ctx: new(ext.Context), want: MaxUploadFileSize},
|
||||
{
|
||||
name: "bot",
|
||||
ctx: uploadAccountContext(true, false),
|
||||
want: MaxUploadFileSize,
|
||||
},
|
||||
{
|
||||
name: "premium bot still uses bot limit",
|
||||
ctx: uploadAccountContext(true, true),
|
||||
want: MaxUploadFileSize,
|
||||
},
|
||||
{
|
||||
name: "regular user",
|
||||
ctx: uploadAccountContext(false, false),
|
||||
want: MaxUploadFileSize,
|
||||
},
|
||||
{
|
||||
name: "premium user",
|
||||
ctx: uploadAccountContext(false, true),
|
||||
want: PremiumMaxUploadFileSize,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := maxUploadFileSize(tt.ctx); got != tt.want {
|
||||
t.Fatalf("maxUploadFileSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func uploadAccountContext(bot, premium bool) *ext.Context {
|
||||
self := new(tg.User)
|
||||
self.SetBot(bot)
|
||||
self.SetPremium(premium)
|
||||
return &ext.Context{Self: self}
|
||||
}
|
||||
|
||||
func TestSplitSizeUsesUploaderLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
splitSizeMB int64
|
||||
accountLimit int64
|
||||
want int64
|
||||
}{
|
||||
{
|
||||
name: "bot default",
|
||||
accountLimit: MaxUploadFileSize,
|
||||
want: MaxUploadFileSize,
|
||||
},
|
||||
{
|
||||
name: "premium default",
|
||||
accountLimit: PremiumMaxUploadFileSize,
|
||||
want: PremiumMaxUploadFileSize,
|
||||
},
|
||||
{
|
||||
name: "explicit lower limit",
|
||||
splitSizeMB: 1500,
|
||||
accountLimit: PremiumMaxUploadFileSize,
|
||||
want: 1500 * 1024 * 1024,
|
||||
},
|
||||
{
|
||||
name: "explicit limit is capped by account",
|
||||
splitSizeMB: 5000,
|
||||
accountLimit: PremiumMaxUploadFileSize,
|
||||
want: PremiumMaxUploadFileSize,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
telegramStorage := Telegram{}
|
||||
telegramStorage.config.SplitSizeMB = tt.splitSizeMB
|
||||
if got := telegramStorage.splitSize(tt.accountLimit); got != tt.want {
|
||||
t.Fatalf("splitSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchUploadLimitAppliesPerFile(t *testing.T) {
|
||||
telegramStorage := Telegram{}
|
||||
tctx := uploadAccountContext(true, false)
|
||||
data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom")
|
||||
itemSize := int64(MaxUploadFileSize/2 + 1)
|
||||
|
||||
for index := range 2 {
|
||||
item, err := telegramStorage.inspectBatchItem(tctx, storagetypes.BatchItem{
|
||||
Reader: bytes.NewReader(data),
|
||||
StoragePath: "video.mp4",
|
||||
Size: itemSize,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("inspectBatchItem() failed for item %d: %v", index, err)
|
||||
}
|
||||
if item.useSingleSave {
|
||||
t.Fatalf("item %d was treated as oversized even though only the batch total exceeds the limit", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
@@ -14,6 +15,34 @@ import (
|
||||
"github.com/yapingcat/gomedia/go-mp4"
|
||||
)
|
||||
|
||||
// sourceFile returns a filesystem path to the media for ffmpeg/ffprobe. Those
|
||||
// tools need a SEEKABLE input: feeding them a pipe (pipe:0) fails for
|
||||
// non-faststart MP4s whose moov atom is at the END of the file, because ffmpeg
|
||||
// cannot seek backwards on a stream to read it. That silently broke thumbnail
|
||||
// and metadata extraction for a large share of downloaded videos. If rs is
|
||||
// already an *os.File we use it directly; otherwise we spool it to a temp file
|
||||
// and return a cleanup func.
|
||||
func sourceFile(rs io.ReadSeeker) (path string, cleanup func(), err error) {
|
||||
noop := func() {}
|
||||
if f, ok := rs.(*os.File); ok {
|
||||
return f.Name(), noop, nil
|
||||
}
|
||||
if _, err = rs.Seek(0, io.SeekStart); err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
tf, err := os.CreateTemp("", "saveany-media-*.tmp")
|
||||
if err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
if _, err = io.Copy(tf, rs); err != nil {
|
||||
tf.Close()
|
||||
os.Remove(tf.Name())
|
||||
return "", noop, err
|
||||
}
|
||||
tf.Close()
|
||||
return tf.Name(), func() { os.Remove(tf.Name()) }, nil
|
||||
}
|
||||
|
||||
type VideoMetadata struct {
|
||||
Duration int
|
||||
Width int
|
||||
@@ -52,16 +81,14 @@ func getMP4Meta(rs io.ReadSeeker) (metadata *VideoMetadata, err error) {
|
||||
|
||||
// getVideoMetadata uses ffprobe to get video metadata
|
||||
func getVideoMetadata(rs io.ReadSeeker) (*VideoMetadata, error) {
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
path, cleanup, err := sourceFile(rs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
rs.Seek(0, io.SeekStart)
|
||||
io.Copy(pipeWriter, rs)
|
||||
}()
|
||||
|
||||
result, err := ffmpeg.ProbeReaderWithTimeout(
|
||||
pipeReader,
|
||||
result, err := ffmpeg.ProbeWithTimeout(
|
||||
path,
|
||||
time.Second*10,
|
||||
ffmpeg.KwArgs{
|
||||
"select_streams": "v:0",
|
||||
@@ -114,25 +141,22 @@ func extractThumbFrame(rs io.ReadSeeker) ([]byte, error) {
|
||||
}
|
||||
|
||||
func extractFrameAt(rs io.ReadSeeker, timestamp float64) ([]byte, error) {
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
rs.Seek(0, io.SeekStart)
|
||||
io.Copy(pipeWriter, rs)
|
||||
}()
|
||||
path, cleanup, err := sourceFile(rs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
err := ffmpeg.
|
||||
Input("pipe:0", ffmpeg.KwArgs{
|
||||
err = ffmpeg.
|
||||
Input(path, ffmpeg.KwArgs{
|
||||
"ss": fmt.Sprintf("%.3f", timestamp),
|
||||
}).
|
||||
Output("pipe:1", ffmpeg.KwArgs{
|
||||
"vframes": 1,
|
||||
"f": "mjpeg",
|
||||
}).
|
||||
WithInput(pipeReader).
|
||||
WithOutput(&out).
|
||||
OverWriteOutput().
|
||||
Run()
|
||||
|
||||
372
storage/telegram/video_split.go
Normal file
372
storage/telegram/video_split.go
Normal file
@@ -0,0 +1,372 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/celestix/gotgproto/ext"
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/gotd/td/telegram/message"
|
||||
"github.com/rs/xid"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
)
|
||||
|
||||
const (
|
||||
videoPartTargetRatio = 0.95
|
||||
videoSplitAttempts = 4
|
||||
minSegmentDuration = 1.0
|
||||
maxLosslessVideoParts = 10
|
||||
)
|
||||
|
||||
type losslessVideoPart struct {
|
||||
Path string
|
||||
Name string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type mediaToolRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
|
||||
var runMediaTool mediaToolRunner = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
message := strings.TrimSpace(string(output))
|
||||
if message == "" {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", err, message)
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func createLosslessVideoParts(
|
||||
ctx context.Context,
|
||||
r io.ReadSeeker,
|
||||
filename string,
|
||||
fileSize, maxPartSize int64,
|
||||
) ([]losslessVideoPart, func(), error) {
|
||||
inputPath, sourceCleanup, err := sourceFile(r)
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("failed to prepare seekable video source: %w", err)
|
||||
}
|
||||
|
||||
tempBase := config.C().Temp.BasePath
|
||||
if err := os.MkdirAll(tempBase, 0o755); err != nil {
|
||||
sourceCleanup()
|
||||
return nil, func() {}, fmt.Errorf("failed to create video split base directory: %w", err)
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(tempBase, "telegram-video-split-*")
|
||||
if err != nil {
|
||||
sourceCleanup()
|
||||
return nil, func() {}, fmt.Errorf("failed to create video split directory: %w", err)
|
||||
}
|
||||
cleanup := func() {
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
log.FromContext(ctx).Warnf("Failed to clean lossless video parts: %s", err)
|
||||
}
|
||||
sourceCleanup()
|
||||
}
|
||||
|
||||
parts, err := splitLosslessVideo(ctx, inputPath, tempDir, filename, fileSize, maxPartSize)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return parts, cleanup, nil
|
||||
}
|
||||
|
||||
func splitLosslessVideo(
|
||||
ctx context.Context,
|
||||
inputPath, outputDir, filename string,
|
||||
fileSize, maxPartSize int64,
|
||||
) ([]losslessVideoPart, error) {
|
||||
if fileSize <= maxPartSize {
|
||||
return nil, fmt.Errorf("video size %d does not exceed part limit %d", fileSize, maxPartSize)
|
||||
}
|
||||
if maxPartSize <= 0 {
|
||||
return nil, fmt.Errorf("invalid video part limit: %d", maxPartSize)
|
||||
}
|
||||
|
||||
duration, err := probeMediaDuration(ctx, inputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to probe source video duration: %w", err)
|
||||
}
|
||||
if duration < minSegmentDuration {
|
||||
return nil, fmt.Errorf("invalid source video duration: %.3f", duration)
|
||||
}
|
||||
|
||||
targetSize := int64(float64(maxPartSize) * videoPartTargetRatio)
|
||||
segmentDuration := initialSegmentDuration(duration, fileSize, targetSize)
|
||||
extension := videoPartExtension(filename)
|
||||
outputPattern := filepath.Join(outputDir, "part-%03d"+extension)
|
||||
|
||||
var lastOversize int64
|
||||
for range videoSplitAttempts {
|
||||
if err := clearVideoParts(outputDir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := runFFmpegSegment(ctx, inputPath, outputPattern, extension, segmentDuration); err != nil {
|
||||
return nil, fmt.Errorf("failed to losslessly split video: %w", err)
|
||||
}
|
||||
|
||||
parts, largest, err := collectVideoParts(ctx, outputDir, filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(parts) < 2 {
|
||||
return nil, fmt.Errorf("video split produced %d part(s), expected at least 2", len(parts))
|
||||
}
|
||||
if len(parts) > maxLosslessVideoParts {
|
||||
return nil, fmt.Errorf(
|
||||
"video split produced %d parts, exceeding the single-album limit of %d",
|
||||
len(parts),
|
||||
maxLosslessVideoParts,
|
||||
)
|
||||
}
|
||||
if largest <= maxPartSize {
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
lastOversize = largest
|
||||
segmentDuration *= float64(targetSize) / float64(largest)
|
||||
if segmentDuration < minSegmentDuration {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"unable to keep lossless video parts below %d bytes; largest part was %d bytes",
|
||||
maxPartSize,
|
||||
lastOversize,
|
||||
)
|
||||
}
|
||||
|
||||
func initialSegmentDuration(duration float64, fileSize, targetSize int64) float64 {
|
||||
partCount := math.Ceil(float64(fileSize) / float64(targetSize))
|
||||
if partCount < 2 {
|
||||
partCount = 2
|
||||
}
|
||||
segmentDuration := duration / partCount
|
||||
if segmentDuration < minSegmentDuration {
|
||||
return minSegmentDuration
|
||||
}
|
||||
return segmentDuration
|
||||
}
|
||||
|
||||
func videoPartExtension(filename string) string {
|
||||
extension := strings.ToLower(filepath.Ext(filepath.Base(filename)))
|
||||
switch extension {
|
||||
case ".avi", ".m4v", ".mkv", ".mov", ".mp4", ".ts", ".webm":
|
||||
return extension
|
||||
default:
|
||||
return ".mp4"
|
||||
}
|
||||
}
|
||||
|
||||
func runFFmpegSegment(
|
||||
ctx context.Context,
|
||||
inputPath, outputPattern, extension string,
|
||||
segmentDuration float64,
|
||||
) error {
|
||||
args := []string{
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-nostdin",
|
||||
"-y",
|
||||
"-i", inputPath,
|
||||
"-map", "0",
|
||||
"-map_metadata", "0",
|
||||
"-c", "copy",
|
||||
"-f", "segment",
|
||||
"-segment_time", strconv.FormatFloat(segmentDuration, 'f', 3, 64),
|
||||
"-segment_start_number", "1",
|
||||
"-reset_timestamps", "1",
|
||||
"-avoid_negative_ts", "make_zero",
|
||||
}
|
||||
if extension == ".mp4" || extension == ".m4v" || extension == ".mov" {
|
||||
args = append(args, "-segment_format_options", "movflags=+faststart")
|
||||
}
|
||||
args = append(args, outputPattern)
|
||||
_, err := runMediaTool(ctx, "ffmpeg", args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func probeMediaDuration(ctx context.Context, filePath string) (float64, error) {
|
||||
output, err := runMediaTool(
|
||||
ctx,
|
||||
"ffprobe",
|
||||
"-v", "error",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
filePath,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid ffprobe duration %q: %w", strings.TrimSpace(string(output)), err)
|
||||
}
|
||||
return duration, nil
|
||||
}
|
||||
|
||||
func clearVideoParts(outputDir string) error {
|
||||
entries, err := os.ReadDir(outputDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read video split directory: %w", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(filepath.Join(outputDir, entry.Name())); err != nil {
|
||||
return fmt.Errorf("failed to clear old video part %s: %w", entry.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectVideoParts(
|
||||
ctx context.Context,
|
||||
outputDir, filename string,
|
||||
) ([]losslessVideoPart, int64, error) {
|
||||
matches, err := filepath.Glob(filepath.Join(outputDir, "part-*"))
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to list video parts: %w", err)
|
||||
}
|
||||
sort.Strings(matches)
|
||||
if len(matches) == 0 {
|
||||
return nil, 0, fmt.Errorf("ffmpeg did not produce any video parts")
|
||||
}
|
||||
|
||||
extension := videoPartExtension(filename)
|
||||
base := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filepath.Base(filename)))
|
||||
if base == "" || base == "." {
|
||||
base = xid.New().String()
|
||||
}
|
||||
parts := make([]losslessVideoPart, 0, len(matches))
|
||||
var largest int64
|
||||
for index, match := range matches {
|
||||
info, err := os.Stat(match)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to stat video part %s: %w", match, err)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return nil, 0, fmt.Errorf("video part %s is empty", match)
|
||||
}
|
||||
if _, err := probeMediaDuration(ctx, match); err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to validate video part %s: %w", match, err)
|
||||
}
|
||||
if info.Size() > largest {
|
||||
largest = info.Size()
|
||||
}
|
||||
parts = append(parts, losslessVideoPart{
|
||||
Path: match,
|
||||
Name: fmt.Sprintf("%s.part%03d%s", base, index+1, extension),
|
||||
Size: info.Size(),
|
||||
})
|
||||
}
|
||||
return parts, largest, nil
|
||||
}
|
||||
|
||||
func partStoragePath(storagePath, partName string) string {
|
||||
directory := path.Dir(path.Clean(storagePath))
|
||||
if directory == "." || directory == "/" {
|
||||
return partName
|
||||
}
|
||||
return path.Join(directory, partName)
|
||||
}
|
||||
|
||||
func (t *Telegram) uploadLosslessVideoParts(
|
||||
ctx context.Context,
|
||||
tctx *ext.Context,
|
||||
storagePath string,
|
||||
parts []losslessVideoPart,
|
||||
sourceCaption *string,
|
||||
progress *uploadProgress,
|
||||
) error {
|
||||
if len(parts) == 0 {
|
||||
return fmt.Errorf("no lossless video parts to upload")
|
||||
}
|
||||
if len(parts) > maxLosslessVideoParts {
|
||||
return fmt.Errorf(
|
||||
"refusing to upload %d lossless video parts as multiple albums; maximum is %d",
|
||||
len(parts),
|
||||
maxLosslessVideoParts,
|
||||
)
|
||||
}
|
||||
resetLosslessVideoUploadProgress(progress, parts)
|
||||
|
||||
prepared := make([]preparedMedia, 0, len(parts))
|
||||
for index, part := range parts {
|
||||
partFile, err := os.Open(part.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open video part %s: %w", part.Name, err)
|
||||
}
|
||||
media, prepareErr := t.prepareMedia(
|
||||
ctx,
|
||||
tctx,
|
||||
partFile,
|
||||
partStoragePath(storagePath, part.Name),
|
||||
part.Size,
|
||||
videoPartCaption(sourceCaption, index),
|
||||
progress,
|
||||
)
|
||||
closeErr := partFile.Close()
|
||||
if prepareErr != nil {
|
||||
return fmt.Errorf("failed to prepare video part %s: %w", part.Name, prepareErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("failed to close video part %s: %w", part.Name, closeErr)
|
||||
}
|
||||
prepared = append(prepared, *media)
|
||||
}
|
||||
|
||||
builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer)
|
||||
if len(prepared) == 1 {
|
||||
if _, err := builder.Media(ctx, prepared[0].media); err != nil {
|
||||
return fmt.Errorf("failed to send video part: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
media := make([]message.MultiMediaOption, len(prepared))
|
||||
for index := range prepared {
|
||||
media[index] = prepared[index].media
|
||||
}
|
||||
if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil {
|
||||
return fmt.Errorf("failed to send video parts as album: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resetLosslessVideoUploadProgress(progress *uploadProgress, parts []losslessVideoPart) {
|
||||
if progress == nil {
|
||||
return
|
||||
}
|
||||
var total int64
|
||||
for _, part := range parts {
|
||||
total += part.Size
|
||||
}
|
||||
progress.reset(total)
|
||||
}
|
||||
|
||||
func videoPartCaption(sourceCaption *string, index int) *string {
|
||||
if sourceCaption == nil {
|
||||
return nil
|
||||
}
|
||||
if index == 0 {
|
||||
return sourceCaption
|
||||
}
|
||||
empty := ""
|
||||
return &empty
|
||||
}
|
||||
195
storage/telegram/video_split_test.go
Normal file
195
storage/telegram/video_split_test.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package telegram
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gotd/td/telegram/uploader"
|
||||
)
|
||||
|
||||
func TestInitialSegmentDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
duration float64
|
||||
fileSize int64
|
||||
targetSize int64
|
||||
want float64
|
||||
}{
|
||||
{name: "two balanced parts", duration: 120, fileSize: 2200, targetSize: 1900, want: 60},
|
||||
{name: "three balanced parts", duration: 120, fileSize: 3900, targetSize: 1900, want: 40},
|
||||
{name: "minimum one second", duration: 1.5, fileSize: 3900, targetSize: 1900, want: 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := initialSegmentDuration(tt.duration, tt.fileSize, tt.targetSize)
|
||||
if got != tt.want {
|
||||
t.Fatalf("initialSegmentDuration() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoPartExtension(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"clip.MOV": ".mov",
|
||||
"clip.mkv": ".mkv",
|
||||
"clip.exe": ".mp4",
|
||||
"clip": ".mp4",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := videoPartExtension(input); got != want {
|
||||
t.Fatalf("videoPartExtension(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartStoragePath(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"clip.mov": "clip.part001.mov",
|
||||
"123456/clip.mov": "123456/clip.part001.mov",
|
||||
"/123456/clip.mov": "/123456/clip.part001.mov",
|
||||
"folder/sub/clip.mov": "folder/sub/clip.part001.mov",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := partStoragePath(input, "clip.part001.mov"); got != want {
|
||||
t.Fatalf("partStoragePath(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoPartCaption(t *testing.T) {
|
||||
if got := videoPartCaption(nil, 0); got != nil {
|
||||
t.Fatalf("caption without source = %q, want nil", *got)
|
||||
}
|
||||
|
||||
source := "original caption"
|
||||
first := videoPartCaption(&source, 0)
|
||||
if first == nil || *first != source {
|
||||
t.Fatalf("first part caption = %v, want %q", first, source)
|
||||
}
|
||||
second := videoPartCaption(&source, 1)
|
||||
if second == nil || *second != "" {
|
||||
t.Fatalf("second part caption = %v, want an explicit empty caption", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetLosslessVideoUploadProgressUsesCombinedPartSize(t *testing.T) {
|
||||
var uploaded, total int64
|
||||
progress := newUploadProgress(999, func(current, size int64) {
|
||||
uploaded = current
|
||||
total = size
|
||||
})
|
||||
resetLosslessVideoUploadProgress(progress, []losslessVideoPart{
|
||||
{Size: 100},
|
||||
{Size: 250},
|
||||
})
|
||||
|
||||
if err := progress.Chunk(t.Context(), uploader.ProgressState{ID: 1, Uploaded: 50, Total: 100}); err != nil {
|
||||
t.Fatalf("Chunk() failed: %v", err)
|
||||
}
|
||||
if uploaded != 50 || total != 350 {
|
||||
t.Fatalf("lossless video progress = %d/%d, want 50/350", uploaded, total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitLosslessVideoRetriesOversizedPart(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
inputPath := filepath.Join(tempDir, "source.mov")
|
||||
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputDir := filepath.Join(tempDir, "parts")
|
||||
if err := os.Mkdir(outputDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
originalRunner := runMediaTool
|
||||
t.Cleanup(func() { runMediaTool = originalRunner })
|
||||
ffmpegCalls := 0
|
||||
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
switch name {
|
||||
case "ffprobe":
|
||||
return []byte("120.0\n"), nil
|
||||
case "ffmpeg":
|
||||
ffmpegCalls++
|
||||
pattern := args[len(args)-1]
|
||||
sizes := []int{2001, 199}
|
||||
if ffmpegCalls > 1 {
|
||||
sizes = []int{1100, 1100}
|
||||
}
|
||||
for index, size := range sizes {
|
||||
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
|
||||
if err := os.WriteFile(partPath, make([]byte, size), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected media tool %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
parts, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
|
||||
if err != nil {
|
||||
t.Fatalf("splitLosslessVideo() failed: %v", err)
|
||||
}
|
||||
if ffmpegCalls != 2 {
|
||||
t.Fatalf("ffmpeg calls = %d, want 2", ffmpegCalls)
|
||||
}
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("parts = %d, want 2", len(parts))
|
||||
}
|
||||
if parts[0].Name != "example.part001.mov" || parts[1].Name != "example.part002.mov" {
|
||||
t.Fatalf("unexpected part names: %#v", parts)
|
||||
}
|
||||
for _, part := range parts {
|
||||
if part.Size > 2000 {
|
||||
t.Fatalf("part %s exceeds limit: %d", part.Name, part.Size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitLosslessVideoRejectsMultipleAlbums(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
inputPath := filepath.Join(tempDir, "source.mov")
|
||||
if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputDir := filepath.Join(tempDir, "parts")
|
||||
if err := os.Mkdir(outputDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
originalRunner := runMediaTool
|
||||
t.Cleanup(func() { runMediaTool = originalRunner })
|
||||
runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
switch name {
|
||||
case "ffprobe":
|
||||
return []byte("120.0\n"), nil
|
||||
case "ffmpeg":
|
||||
pattern := args[len(args)-1]
|
||||
for index := range maxLosslessVideoParts + 1 {
|
||||
partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1)
|
||||
if err := os.WriteFile(partPath, make([]byte, 100), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected media tool %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000)
|
||||
if err == nil {
|
||||
t.Fatal("splitLosslessVideo() succeeded with more than one album of parts")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "single-album limit") {
|
||||
t.Fatalf("splitLosslessVideo() error = %q, want single-album limit", err)
|
||||
}
|
||||
}
|
||||
@@ -119,21 +119,21 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(dirPath, "/")
|
||||
currentPath := ""
|
||||
var currentPath strings.Builder
|
||||
for i, part := range parts {
|
||||
if i > 0 {
|
||||
currentPath += "/"
|
||||
currentPath.WriteString("/")
|
||||
}
|
||||
currentPath += part
|
||||
currentPath.WriteString(part)
|
||||
|
||||
exists, err := c.Exists(ctx, currentPath)
|
||||
exists, err := c.Exists(ctx, currentPath.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
url := c.BaseURL + currentPath
|
||||
url := c.BaseURL + currentPath.String()
|
||||
resp, err := c.doRequest(ctx, WebdavMethodMkcol, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -141,7 +141,7 @@ func (c *Client) MkDir(ctx context.Context, dirPath string) error {
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("MKCOL %s: %s", currentPath, resp.Status)
|
||||
return fmt.Errorf("MKCOL %s: %s", currentPath.String(), resp.Status)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user