mirror of
https://github.com/krau/SaveAny-Bot.git
synced 2026-08-31 05:06:42 +08:00
* fix: prevent queue deadlock after cancelling tasks Get no longer recurses while holding the mutex. Cancelled queued tasks leave the map, making their IDs reusable. Closed empty queues return ErrQueueClosed. * fix: init task queue lazily and close on shutdown AddTask is safe before Run by initializing the queue once. Close unblocks workers waiting in Get. * fix: make resource fingerprints deterministic Sort map keys before hashing so Resource.ID is stable. * fix: harden per-item processing tracking in tasks Use the resource fingerprint as the dedup key on insert and delete. Check and set processing entries atomically instead of TOCTOU. Count only successful downloads. * fix: honor IgnoreErrors in batch tasks Element failures no longer cancel sibling elements or stop later groups. * fix: report streamed upload bytes in batch tasks Stream downloads report their byte count as the upload total. * fix: validate i18n key parity across locales geni18n fails when a language file misses any key. Align the syncpeers completion key between en and zh-Hans. Translate three untranslated parse keys in zh-Hans. * refactor: share progress throttling helpers Move size-tiered and count-based throttling into progressutil. Localize hardcoded Chinese progress strings. Drop five duplicated implementations and dead local copies. * refactor: share unique filename logic Storage backends use fsutil.UniquePath instead of local loops. * fix: sanitize local storage paths Reject absolute paths and dot-dot escapes in Save. Check close errors and wrap creation failures. * fix: preserve webdav error causes Wrap mkdir and write failures with %w and drop dead error values. * fix: kill rclone subprocess on reader close Prevent cat processes from hanging after the pipe is closed. * fix: fail alist init instead of exiting Replace log.Fatalf with wrapped errors so a bad alist cannot kill the bot. Cancel token refresh with the init context and re-login on 401. * fix: enforce telegram album limits and track saved paths Use tglimit.MaxAlbumItems for album batching and video splitting. Exists now reports previously saved paths instead of always false. * fix: guard storage registry maps Protect Storages and UserStorages with mutexes and expose read accessors. * fix: harden JS parser plugin runtime Validate semver instead of panicking and require canHandle. Recover plugin worker panics and time out CanHandle calls. Return a copy from the registry and sanitize install filenames. * fix: guard kemono parser against nil fields Skip sparse preview and attachment entries instead of panicking. * refactor: remove commented-out dead code Drop unused kemono legacy types and commented response structs. * test: cover invalid plugin version rejection * fix: show all queued tasks in /task list Render up to ten tasks and append the truncation note once. * fix: guard callback data parsing Reject malformed callback payloads before indexing split parts. * fix: isolate media groups per user Key pending groups by chat, user and group id. * fix: require permission for callback handlers * fix: initialize userbot context once Replace the racy lazy init with sync.OnceValue. * fix: avoid leaking raw errors in /dir reply * fix: notify users on invalid update version * fix: fail fast when API listen fails Bind synchronously and surface errors instead of logging them. * fix: add timeouts and backoff to webhook delivery * refactor: remove dead code from api and bot Drop the empty ProgressTracker shim, unused token context key and a redundant SetBotCommands call. * fix: load remote config without local lookup Skip the local file search after reading a config URL and add a timeout. * refactor: drop unused hook config * docs: document parser plugin config * ci: fix BuildTime formatting and align checkout Actions format does not format dates; pass the raw timestamp. * chore: ignore cache directory * fix: make cache init idempotent * fix: upload only downloaded batch elements Failed elements keep partial cache files and never reach the backend. Successfully downloaded siblings still upload when one element fails. * fix: record telegram saved paths only after upload Skip-large returns a sentinel so skipped files are not marked as saved. * fix: deduplicate alist token refreshes Guard token access with a mutex and merge concurrent logins. Reuse a recent refresh to avoid login storms. * test: cover concurrent alist 401 retry Ten parallel uploads share a single re-login under -race. * fix: count parsed resources in progress text * fix: localize storage lookup errors in /dir Use the shared i18n key and escape the dynamic error. * fix: deduplicate concurrent storage initialization singleflight merges first-time inits so side effects are not duplicated. * fix: guard nil progress trackers in api-created tasks Batch, telegraph and transfer tasks run without a Telegram tracker when created through the API; their callbacks must not panic. * fix: keep album order when filtering failed batch items Download results are stored by original index so surviving elements keep their source order. * test: cover nil-tracker task execution * fix: report partial failure in batch done message IgnoreErrors runs with failed elements show success and failed counts instead of claiming every file completed. * fix: skip login refresh for token-only alist storage 401 responses surface the auth error instead of sending a credential-less login; the refresh window never exceeds TokenExp. * test: cover alist refresh semantics Concurrent refresh uses username/password; token-only storage never attempts a login on 401. * fix: call tracker from notifyProgress instead of recursing The helper called itself, overflowing the stack on any batch task with a progress tracker. * fix: propagate cancellation past IgnoreErrors Cancelled tasks must not be reported as successful: only ordinary element failures are ignored. * test: cover notifyProgress tracker call * test: cover cancellation with IgnoreErrors * fix: refresh alist token after startup 401s The init login no longer satisfies the dedup window, so an early 401 triggers a real refresh; later 401s reuse it. Clarify that streaming uploads cannot replay their body. * test: exercise the alist 401 refresh path Concurrent uploads now reject the init token, share one refresh and retry with the new token; token-only stays inert. * style: trim verbose comments Drop process-style explanations; keep one-line behavior notes. * style: gofmt test files * fix: drop unbounded saved-path cache in telegram storage Telegram cannot reliably query remote file existence, so the cache answered a question it could not answer and grew without bound. Exists returns false again, as before. * style: format codes
358 lines
10 KiB
Go
358 lines
10 KiB
Go
package tfile
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/charmbracelet/log"
|
|
"github.com/gotd/td/tg"
|
|
"github.com/krau/SaveAny-Bot/common/i18n"
|
|
"github.com/krau/SaveAny-Bot/common/i18n/i18nk"
|
|
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
|
|
"github.com/krau/SaveAny-Bot/common/utils/progressutil"
|
|
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
|
)
|
|
|
|
type ProgressTracker interface {
|
|
OnStart(ctx context.Context, info TaskInfo)
|
|
OnProgress(ctx context.Context, info TaskInfo, downloaded, total int64)
|
|
OnDone(ctx context.Context, info TaskInfo, err error)
|
|
}
|
|
|
|
// UploadProgressTracker optionally extends a task progress tracker with a
|
|
// distinct upload phase. Keeping it separate preserves compatibility with
|
|
// custom download-only trackers.
|
|
type UploadProgressTracker interface {
|
|
OnUploadStart(ctx context.Context, info TaskInfo, total int64)
|
|
OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64)
|
|
}
|
|
|
|
type Progress struct {
|
|
MessageID int
|
|
ChatID int64
|
|
start time.Time
|
|
lastUpdatePercent atomic.Int32
|
|
lastUpdateAt atomic.Int64
|
|
updateMu sync.Mutex
|
|
uploadAttempt int
|
|
uploadedBytes int64
|
|
actualSize int64
|
|
hasActualSize bool
|
|
}
|
|
|
|
const (
|
|
uploadProgressMinInterval = time.Second
|
|
uploadProgressMaxInterval = 3 * time.Second
|
|
singleProgressBarWidth = 10
|
|
maxSingleErrorRunes = 240
|
|
)
|
|
|
|
type singleProgressPhase int
|
|
|
|
const (
|
|
singlePhaseDownloading singleProgressPhase = iota
|
|
singlePhaseUploading
|
|
singlePhaseRetrying
|
|
)
|
|
|
|
type renderedSingleMessage struct {
|
|
Text string
|
|
Entities []tg.MessageEntityClass
|
|
Err error
|
|
}
|
|
|
|
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
|
|
p.updateMu.Lock()
|
|
defer p.updateMu.Unlock()
|
|
p.start = time.Now()
|
|
p.lastUpdatePercent.Store(0)
|
|
p.lastUpdateAt.Store(0)
|
|
p.uploadAttempt = 0
|
|
p.uploadedBytes = 0
|
|
p.actualSize = 0
|
|
p.hasActualSize = false
|
|
log.FromContext(ctx).Debugf("Progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
|
|
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, singlePhaseDownloading, 0, info.FileSize(), 0, 0), true)
|
|
}
|
|
|
|
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo, downloaded, total int64) {
|
|
p.updateMu.Lock()
|
|
defer p.updateMu.Unlock()
|
|
now := time.Now()
|
|
elapsed := uploadProgressMaxInterval
|
|
if lastUpdateAt := p.lastUpdateAt.Load(); lastUpdateAt > 0 {
|
|
elapsed = now.Sub(time.Unix(0, lastUpdateAt))
|
|
}
|
|
if !shouldUpdateSingleDownloadProgress(total, downloaded, int(p.lastUpdatePercent.Load()), elapsed) {
|
|
return
|
|
}
|
|
if total > 0 {
|
|
percent := int32((downloaded * 100) / total)
|
|
if p.lastUpdatePercent.Load() == percent {
|
|
return
|
|
}
|
|
p.lastUpdatePercent.Store(percent)
|
|
}
|
|
p.lastUpdateAt.Store(now.UnixNano())
|
|
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.FileName(), downloaded, total)
|
|
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
|
|
info,
|
|
singlePhaseDownloading,
|
|
downloaded,
|
|
total,
|
|
dlutil.GetSpeed(downloaded, p.start),
|
|
0,
|
|
), true)
|
|
}
|
|
|
|
func shouldUpdateSingleDownloadProgress(total, downloaded int64, lastPercent int, elapsed time.Duration) bool {
|
|
if total > 0 {
|
|
return progressutil.ShouldUpdate(total, downloaded, lastPercent)
|
|
}
|
|
return downloaded > 0 && elapsed >= uploadProgressMaxInterval
|
|
}
|
|
|
|
func (p *Progress) OnUploadStart(ctx context.Context, info TaskInfo, total int64) {
|
|
p.updateMu.Lock()
|
|
defer p.updateMu.Unlock()
|
|
p.start = time.Now()
|
|
p.lastUpdatePercent.Store(0)
|
|
p.lastUpdateAt.Store(p.start.UnixNano())
|
|
p.uploadAttempt++
|
|
p.uploadedBytes = 0
|
|
p.actualSize = max(total, 0)
|
|
p.hasActualSize = true
|
|
log.FromContext(ctx).Debugf("Upload progress tracking started: %s", info.FileName())
|
|
phase := singleUploadPhase(p.uploadAttempt)
|
|
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(info, phase, 0, total, 0, p.uploadAttempt), true)
|
|
}
|
|
|
|
func (p *Progress) OnUploadProgress(ctx context.Context, info TaskInfo, uploaded, total int64) {
|
|
if total <= 0 || uploaded <= 0 {
|
|
return
|
|
}
|
|
p.updateMu.Lock()
|
|
defer p.updateMu.Unlock()
|
|
if uploaded > total {
|
|
uploaded = total
|
|
}
|
|
if uploaded < p.uploadedBytes {
|
|
return
|
|
}
|
|
p.uploadedBytes = uploaded
|
|
|
|
now := time.Now()
|
|
lastUpdateAt := time.Unix(0, p.lastUpdateAt.Load())
|
|
lastPercent := int(p.lastUpdatePercent.Load())
|
|
if !shouldUpdateUploadProgress(total, uploaded, lastPercent, now.Sub(lastUpdateAt)) {
|
|
return
|
|
}
|
|
|
|
percent := int32((uploaded * 100) / total)
|
|
p.lastUpdatePercent.Store(percent)
|
|
p.lastUpdateAt.Store(now.UnixNano())
|
|
log.FromContext(ctx).Debugf("Upload progress update: %s, %d/%d", info.FileName(), uploaded, total)
|
|
p.editMessage(ctx, info.TaskID(), buildSingleProgressMessage(
|
|
info,
|
|
singleUploadPhase(p.uploadAttempt),
|
|
uploaded,
|
|
total,
|
|
dlutil.GetSpeed(uploaded, p.start),
|
|
p.uploadAttempt,
|
|
), true)
|
|
}
|
|
|
|
func shouldUpdateUploadProgress(total, uploaded int64, lastPercent int, elapsed time.Duration) bool {
|
|
if total <= 0 || uploaded <= 0 {
|
|
return false
|
|
}
|
|
if uploaded >= total {
|
|
return lastPercent < 100 && elapsed >= uploadProgressMinInterval
|
|
}
|
|
percent := int((uploaded * 100) / total)
|
|
if percent < lastPercent {
|
|
return false
|
|
}
|
|
if elapsed < uploadProgressMinInterval {
|
|
return false
|
|
}
|
|
if percent == lastPercent {
|
|
return elapsed >= uploadProgressMaxInterval
|
|
}
|
|
return progressutil.ShouldUpdate(total, uploaded, lastPercent) || elapsed >= uploadProgressMaxInterval
|
|
}
|
|
|
|
func singleUploadPhase(attempt int) singleProgressPhase {
|
|
if attempt > 1 {
|
|
return singlePhaseRetrying
|
|
}
|
|
return singlePhaseUploading
|
|
}
|
|
|
|
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
|
|
p.updateMu.Lock()
|
|
defer p.updateMu.Unlock()
|
|
if err != nil {
|
|
log.FromContext(ctx).Errorf("Progress error for file [%s]: %v", info.FileName(), err)
|
|
} else {
|
|
log.FromContext(ctx).Debugf("Progress done for file [%s]", info.FileName())
|
|
}
|
|
|
|
p.editMessage(ctx, info.TaskID(), buildSingleDoneMessage(info, p.doneSize(info), err), false)
|
|
}
|
|
|
|
func (p *Progress) doneSize(info TaskInfo) int64 {
|
|
if p.hasActualSize {
|
|
return p.actualSize
|
|
}
|
|
return max(info.FileSize(), 0)
|
|
}
|
|
|
|
func (p *Progress) editMessage(ctx context.Context, taskID string, message renderedSingleMessage, cancellable bool) {
|
|
if message.Err != nil {
|
|
log.FromContext(ctx).Errorf("Failed to render file progress message: %v", message.Err)
|
|
return
|
|
}
|
|
req := buildSingleEditMessageRequest(p.MessageID, taskID, message, cancellable)
|
|
if ext := tgutil.ExtFromContext(ctx); ext != nil {
|
|
if _, err := ext.EditMessage(p.ChatID, req); err != nil {
|
|
log.FromContext(ctx).Errorf("Failed to edit file progress message: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func buildSingleEditMessageRequest(messageID int, taskID string, message renderedSingleMessage, cancellable bool) *tg.MessagesEditMessageRequest {
|
|
req := &tg.MessagesEditMessageRequest{ID: messageID}
|
|
req.SetMessage(message.Text)
|
|
if len(message.Entities) > 0 {
|
|
req.SetEntities(message.Entities)
|
|
}
|
|
if cancellable {
|
|
req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{
|
|
Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)},
|
|
}}})
|
|
}
|
|
return req
|
|
}
|
|
|
|
func buildSingleProgressMessage(
|
|
info TaskInfo,
|
|
phase singleProgressPhase,
|
|
current int64,
|
|
total int64,
|
|
speed float64,
|
|
attempt int,
|
|
) renderedSingleMessage {
|
|
if current < 0 {
|
|
current = 0
|
|
}
|
|
if total > 0 && current > total {
|
|
current = total
|
|
}
|
|
percent := singleProgressPercent(current, total)
|
|
destination := fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())
|
|
data := map[string]any{
|
|
"Name": info.FileName(),
|
|
"Bar": singleProgressBar(percent),
|
|
"Progress": percent,
|
|
"Speed": singleProgressSpeed(speed),
|
|
"Current": dlutil.FormatSize(current),
|
|
"Size": dlutil.FormatSize(total),
|
|
"Destination": destination,
|
|
"Attempt": max(attempt, 1),
|
|
}
|
|
|
|
var key i18nk.Key
|
|
switch phase {
|
|
case singlePhaseUploading:
|
|
key = i18nk.BotMsgProgressSingleUploading
|
|
case singlePhaseRetrying:
|
|
key = i18nk.BotMsgProgressSingleUploadRetrying
|
|
default:
|
|
key = i18nk.BotMsgProgressSingleDownloading
|
|
if total <= 0 {
|
|
key = i18nk.BotMsgProgressSingleDownloadingUnknown
|
|
}
|
|
}
|
|
markup := i18n.T(i18nk.BotMsgProgressSingleStatusHeader, nil) + "\n\n" + localizedProgressMarkup(key, data)
|
|
return completeSingleMessage(markup)
|
|
}
|
|
|
|
func buildSingleDoneMessage(info TaskInfo, size int64, err error) renderedSingleMessage {
|
|
data := map[string]any{
|
|
"Name": info.FileName(),
|
|
"Size": dlutil.FormatSize(max(size, 0)),
|
|
"Destination": fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath()),
|
|
}
|
|
var key i18nk.Key
|
|
switch {
|
|
case err == nil:
|
|
key = i18nk.BotMsgProgressSingleDone
|
|
case errors.Is(err, context.Canceled):
|
|
key = i18nk.BotMsgProgressSingleCanceled
|
|
default:
|
|
data["Reason"] = truncateSingleError(err.Error())
|
|
key = i18nk.BotMsgProgressSingleFailed
|
|
}
|
|
return completeSingleMessage(localizedProgressMarkup(key, data))
|
|
}
|
|
|
|
func localizedProgressMarkup(key i18nk.Key, data map[string]any) string {
|
|
return i18n.T(key, tgutil.EscapeHTMLTemplateData(data))
|
|
}
|
|
|
|
func completeSingleMessage(markup string) renderedSingleMessage {
|
|
text, entities, err := tgutil.RenderHTML(markup)
|
|
return renderedSingleMessage{Text: text, Entities: entities, Err: err}
|
|
}
|
|
|
|
func singleProgressPercent(current, total int64) int {
|
|
if total <= 0 {
|
|
return 0
|
|
}
|
|
return int(min(max(current, 0), total) * 100 / total)
|
|
}
|
|
|
|
func singleProgressBar(percent int) string {
|
|
percent = min(max(percent, 0), 100)
|
|
filled := percent * singleProgressBarWidth / 100
|
|
return strings.Repeat("🟩", filled) + strings.Repeat("⬜️", singleProgressBarWidth-filled)
|
|
}
|
|
|
|
func singleProgressSpeed(speed float64) string {
|
|
if speed <= 0 {
|
|
return "0 B/s"
|
|
}
|
|
return dlutil.FormatSize(int64(speed)) + "/s"
|
|
}
|
|
|
|
func truncateSingleError(value string) string {
|
|
runes := []rune(value)
|
|
if len(runes) <= maxSingleErrorRunes {
|
|
return value
|
|
}
|
|
return string(runes[:maxSingleErrorRunes])
|
|
}
|
|
|
|
type ProgressOption func(*Progress)
|
|
|
|
func NewProgressTrack(
|
|
messageID int,
|
|
chatID int64,
|
|
opts ...ProgressOption,
|
|
) ProgressTracker {
|
|
p := &Progress{
|
|
MessageID: messageID,
|
|
ChatID: chatID,
|
|
}
|
|
for _, opt := range opts {
|
|
opt(p)
|
|
}
|
|
return p
|
|
}
|