mirror of
https://github.com/krau/SaveAny-Bot.git
synced 2026-07-07 07:01:21 +08:00
feat: add direct links download functionality
- Implemented a new task type for handling direct links downloads. - Added command handler for downloading multiple links via /dl command. - Introduced progress tracking for direct link downloads. - Enhanced filename parsing to support various encoding scenarios. - Updated enums to include direct links as a task type. - Refactored existing task structures to accommodate new functionality. - Improved error handling and logging throughout the download process.
This commit is contained in:
@@ -24,7 +24,7 @@ func (t *Task) Execute(ctx context.Context) error {
|
||||
workers := config.C().Workers
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(workers)
|
||||
for _, elem := range t.Elems {
|
||||
for _, elem := range t.elems {
|
||||
eg.Go(func() error {
|
||||
t.processingMu.RLock()
|
||||
if t.processing[elem.ID] != nil {
|
||||
|
||||
@@ -25,8 +25,8 @@ type TaskElement struct {
|
||||
|
||||
type Task struct {
|
||||
ID string
|
||||
Ctx context.Context
|
||||
Elems []TaskElement
|
||||
ctx context.Context
|
||||
elems []TaskElement
|
||||
Progress ProgressTracker
|
||||
IgnoreErrors bool // if true, errors during processing will be ignored
|
||||
downloaded atomic.Int64
|
||||
@@ -78,8 +78,8 @@ func NewBatchTGFileTask(
|
||||
) *Task {
|
||||
task := &Task{
|
||||
ID: id,
|
||||
Ctx: ctx,
|
||||
Elems: files,
|
||||
ctx: ctx,
|
||||
elems: files,
|
||||
Progress: progress,
|
||||
downloaded: atomic.Int64{},
|
||||
totalSize: func() int64 {
|
||||
|
||||
@@ -44,11 +44,11 @@ func (t *Task) Downloaded() int64 {
|
||||
}
|
||||
|
||||
func (t *Task) Count() int {
|
||||
return len(t.Elems)
|
||||
return len(t.elems)
|
||||
}
|
||||
|
||||
func (t *Task) Processing() []TaskElementInfo {
|
||||
processing := make([]TaskElementInfo, 0, len(t.Elems))
|
||||
processing := make([]TaskElementInfo, 0, len(t.elems))
|
||||
for _, elem := range t.processing {
|
||||
processing = append(processing, elem)
|
||||
}
|
||||
|
||||
167
core/tasks/directlinks/execute.go
Normal file
167
core/tasks/directlinks/execute.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package directlinks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/retry"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/ioutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func (t *Task) Execute(ctx context.Context) error {
|
||||
logger := log.FromContext(ctx)
|
||||
logger.Infof("Starting directlinks task %s", t.ID)
|
||||
if t.Progress != nil {
|
||||
t.Progress.OnStart(ctx, t)
|
||||
}
|
||||
// head all links to get file info
|
||||
eg, gctx := errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
fetchedTotalBytes := atomic.Int64{}
|
||||
for _, file := range t.files {
|
||||
eg.Go(func() error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, file.URL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create HEAD request for %s: %w", file.URL, err)
|
||||
}
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to HEAD %s: %w", file.URL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("HEAD %s returned status %d", file.URL, resp.StatusCode)
|
||||
}
|
||||
fetchedTotalBytes.Add(resp.ContentLength)
|
||||
file.Size = resp.ContentLength
|
||||
if name := resp.Header.Get("Content-Disposition"); name != "" {
|
||||
// Set file name
|
||||
filename := parseFilename(name)
|
||||
file.Name = filename
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
err := eg.Wait()
|
||||
if err != nil {
|
||||
logger.Errorf("Error during HEAD requests: %v", err)
|
||||
if t.Progress != nil {
|
||||
t.Progress.OnDone(ctx, t, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
t.totalBytes = fetchedTotalBytes.Load()
|
||||
// start downloading
|
||||
eg, gctx = errgroup.WithContext(ctx)
|
||||
eg.SetLimit(config.C().Workers)
|
||||
for _, file := range t.files {
|
||||
eg.Go(func() error {
|
||||
t.processingMu.RLock()
|
||||
if _, ok := t.processing[file.URL]; ok {
|
||||
return fmt.Errorf("file %s is already being processed", file.URL)
|
||||
}
|
||||
t.processingMu.RUnlock()
|
||||
t.processingMu.Lock()
|
||||
t.processing[file.URL] = file
|
||||
t.processingMu.Unlock()
|
||||
defer func() {
|
||||
t.processingMu.Lock()
|
||||
delete(t.processing, file.URL)
|
||||
t.processingMu.Unlock()
|
||||
}()
|
||||
err := t.processLink(gctx, file)
|
||||
t.downloaded.Add(1)
|
||||
if errors.Is(err, context.Canceled) {
|
||||
logger.Debug("Link processing canceled")
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("Error processing link %s: %v", file.URL, err)
|
||||
return fmt.Errorf("failed to process link %s: %w", file.URL, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
err = eg.Wait()
|
||||
if err != nil {
|
||||
logger.Errorf("Error during directlinks task execution: %v", err)
|
||||
} else {
|
||||
logger.Infof("Directlinks task %s completed successfully", t.ID)
|
||||
}
|
||||
if t.Progress != nil {
|
||||
t.Progress.OnDone(ctx, t, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *Task) processLink(ctx context.Context, file *File) error {
|
||||
logger := log.FromContext(ctx)
|
||||
err := retry.Retry(func() error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, file.URL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create GET request for %s: %w", file.URL, err)
|
||||
}
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to GET %s: %w", file.URL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("GET %s returned status %d", file.URL, resp.StatusCode)
|
||||
}
|
||||
ctx = context.WithValue(ctx, ctxkey.ContentLength, file.Size)
|
||||
if t.stream {
|
||||
return t.Storage.Save(ctx, resp.Body, filepath.Join(t.StorPath, file.Name))
|
||||
}
|
||||
cacheFile, err := fsutil.CreateFile(filepath.Join(config.C().Temp.BasePath,
|
||||
fmt.Sprintf("direct_%s_%s", t.ID, file.Name)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := cacheFile.CloseAndRemove(); err != nil {
|
||||
logger.Errorf("Failed to close and remove cache file: %v", err)
|
||||
}
|
||||
}()
|
||||
wr := ioutil.NewProgressWriter(cacheFile, func(n int) {
|
||||
t.downloadedBytes.Add(int64(n))
|
||||
if t.Progress != nil {
|
||||
t.Progress.OnProgress(ctx, t)
|
||||
}
|
||||
})
|
||||
|
||||
copyResultCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := io.Copy(wr, resp.Body)
|
||||
copyResultCh <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-copyResultCh:
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file %s to cache file: %w", file.URL, err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
_, err = cacheFile.Seek(0, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to seek cache file for resource %s: %w", file.URL, err)
|
||||
}
|
||||
return t.Storage.Save(ctx, cacheFile, filepath.Join(t.StorPath, file.Name))
|
||||
}, retry.RetryTimes(uint(config.C().Retry)), retry.Context(ctx))
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return err
|
||||
}
|
||||
196
core/tasks/directlinks/progress.go
Normal file
196
core/tasks/directlinks/progress.go
Normal file
@@ -0,0 +1,196 @@
|
||||
package directlinks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/duke-git/lancet/v2/slice"
|
||||
"github.com/gotd/td/telegram/message/entity"
|
||||
"github.com/gotd/td/telegram/message/styling"
|
||||
"github.com/gotd/td/tg"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/dlutil"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
)
|
||||
|
||||
type TaskInfo interface {
|
||||
TotalBytes() int64
|
||||
TotalFiles() int
|
||||
TaskID() string
|
||||
StorageName() string
|
||||
StoragePath() string
|
||||
DownloadedBytes() int64
|
||||
Processing() []FileInfo
|
||||
}
|
||||
|
||||
type FileInfo interface {
|
||||
FileName() string
|
||||
FileSize() int64
|
||||
}
|
||||
|
||||
type ProgressTracker interface {
|
||||
OnStart(ctx context.Context, info TaskInfo)
|
||||
OnProgress(ctx context.Context, info TaskInfo)
|
||||
OnDone(ctx context.Context, info TaskInfo, err error)
|
||||
}
|
||||
|
||||
type Progress struct {
|
||||
msgID int
|
||||
chatID int64
|
||||
start time.Time
|
||||
lastUpdatePercent atomic.Int32
|
||||
}
|
||||
|
||||
// OnDone implements ProgressTracker.
|
||||
func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
|
||||
logger := log.FromContext(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
logger.Infof("Parsed task %s was canceled", info.TaskID())
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.chatID, &tg.MessagesEditMessageRequest{
|
||||
ID: p.msgID,
|
||||
Message: fmt.Sprintf("处理已取消: %s", info.TaskID()),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
logger.Errorf("Parsed task %s failed: %s", info.TaskID(), err)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.chatID, &tg.MessagesEditMessageRequest{
|
||||
ID: p.msgID,
|
||||
Message: fmt.Sprintf("处理失败: %s", err.Error()),
|
||||
})
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
logger.Infof("Parsed task %s completed successfully", info.TaskID())
|
||||
|
||||
entityBuilder := entity.Builder{}
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain("处理完成, 文件数量: "),
|
||||
styling.Code(fmt.Sprintf("%d", info.TotalFiles())),
|
||||
styling.Plain("\n保存路径: "),
|
||||
styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
|
||||
); err != nil {
|
||||
logger.Errorf("Failed to build entities: %s", err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.msgID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.chatID, req)
|
||||
}
|
||||
}
|
||||
|
||||
// OnProgress implements ProgressTracker.
|
||||
func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) {
|
||||
if !shouldUpdateProgress(info.TotalBytes(), info.DownloadedBytes(), int(p.lastUpdatePercent.Load())) {
|
||||
return
|
||||
}
|
||||
percent := int((info.DownloadedBytes() * 100) / info.TotalBytes())
|
||||
if p.lastUpdatePercent.Load() == int32(percent) {
|
||||
return
|
||||
}
|
||||
p.lastUpdatePercent.Store(int32(percent))
|
||||
log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.TaskID(), info.DownloadedBytes(), info.TotalBytes())
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain("正在下载\n总大小: "),
|
||||
styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalBytes())/(1024*1024), info.TotalFiles())),
|
||||
styling.Plain("\n正在处理:\n"),
|
||||
func() styling.StyledTextOption {
|
||||
var lines []string
|
||||
for _, elem := range info.Processing() {
|
||||
lines = append(lines, fmt.Sprintf(" - %s (%.2f MB)", elem.FileName(), float64(elem.FileSize())/(1024*1024)))
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
lines = append(lines, " - 无")
|
||||
}
|
||||
return styling.Plain(slice.Join(lines, "\n"))
|
||||
}(),
|
||||
styling.Plain("\n平均速度: "),
|
||||
styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(info.DownloadedBytes(), p.start)/(1024*1024))),
|
||||
styling.Plain("\n当前进度: "),
|
||||
styling.Bold(fmt.Sprintf("%.2f%%", float64(info.DownloadedBytes())/float64(info.TotalBytes())*100)),
|
||||
); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.msgID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext != nil {
|
||||
ext.EditMessage(p.chatID, req)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// OnStart implements ProgressTracker.
|
||||
func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
|
||||
logger := log.FromContext(ctx)
|
||||
p.start = time.Now()
|
||||
p.lastUpdatePercent.Store(0)
|
||||
logger.Infof("Direct links task started: message_id=%d, chat_id=%d", p.msgID, p.chatID)
|
||||
ext := tgutil.ExtFromContext(ctx)
|
||||
if ext == nil {
|
||||
return
|
||||
}
|
||||
entityBuilder := entity.Builder{}
|
||||
var entities []tg.MessageEntityClass
|
||||
if err := styling.Perform(&entityBuilder,
|
||||
styling.Plain(fmt.Sprintf("开始下载, 总大小: %.2f MB (%d 个文件)", float64(info.TotalBytes())/(1024*1024), info.TotalFiles()))); err != nil {
|
||||
log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
|
||||
return
|
||||
}
|
||||
text, entities := entityBuilder.Complete()
|
||||
req := &tg.MessagesEditMessageRequest{
|
||||
ID: p.msgID,
|
||||
}
|
||||
req.SetMessage(text)
|
||||
req.SetEntities(entities)
|
||||
req.SetReplyMarkup(&tg.ReplyInlineMarkup{
|
||||
Rows: []tg.KeyboardButtonRow{
|
||||
{
|
||||
Buttons: []tg.KeyboardButtonClass{
|
||||
tgutil.BuildCancelButton(info.TaskID()),
|
||||
},
|
||||
},
|
||||
}},
|
||||
)
|
||||
ext.EditMessage(p.chatID, req)
|
||||
}
|
||||
|
||||
var _ ProgressTracker = (*Progress)(nil)
|
||||
|
||||
func NewProgress(msgID int, userID int64) ProgressTracker {
|
||||
return &Progress{
|
||||
msgID: msgID,
|
||||
chatID: userID,
|
||||
}
|
||||
}
|
||||
121
core/tasks/directlinks/task.go
Normal file
121
core/tasks/directlinks/task.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package directlinks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
|
||||
"github.com/krau/SaveAny-Bot/storage"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Name string
|
||||
URL string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func (f *File) FileName() string {
|
||||
return f.Name
|
||||
}
|
||||
|
||||
func (f *File) FileSize() int64 {
|
||||
return f.Size
|
||||
}
|
||||
|
||||
type Task struct {
|
||||
ID string
|
||||
ctx context.Context
|
||||
files []*File
|
||||
Storage storage.Storage
|
||||
StorPath string
|
||||
Progress ProgressTracker
|
||||
|
||||
client *http.Client // [TODO] parallel download
|
||||
stream bool
|
||||
totalBytes int64 // total bytes to download
|
||||
downloadedBytes atomic.Int64 // downloaded bytes
|
||||
totalFiles int64 // total files to download
|
||||
downloaded atomic.Int64 // downloaded files count
|
||||
processing map[string]*File // {"url": File}
|
||||
processingMu sync.RWMutex
|
||||
failed map[string]error // [TODO] errors for each file
|
||||
}
|
||||
|
||||
// DownloadedBytes implements TaskInfo.
|
||||
func (t *Task) DownloadedBytes() int64 {
|
||||
return t.downloadedBytes.Load()
|
||||
}
|
||||
|
||||
// Processing implements TaskInfo.
|
||||
func (t *Task) Processing() []FileInfo {
|
||||
t.processingMu.RLock()
|
||||
defer t.processingMu.RUnlock()
|
||||
infos := make([]FileInfo, 0, len(t.processing))
|
||||
for _, f := range t.processing {
|
||||
infos = append(infos, f)
|
||||
}
|
||||
return infos
|
||||
}
|
||||
|
||||
// StorageName implements TaskInfo.
|
||||
func (t *Task) StorageName() string {
|
||||
return t.Storage.Name()
|
||||
}
|
||||
|
||||
// StoragePath implements TaskInfo.
|
||||
func (t *Task) StoragePath() string {
|
||||
return t.StorPath
|
||||
}
|
||||
|
||||
// TotalBytes implements TaskInfo.
|
||||
func (t *Task) TotalBytes() int64 {
|
||||
return t.totalBytes
|
||||
}
|
||||
|
||||
// TotalFiles implements TaskInfo.
|
||||
func (t *Task) TotalFiles() int {
|
||||
return int(t.totalFiles)
|
||||
}
|
||||
|
||||
func (t *Task) Type() tasktype.TaskType {
|
||||
return tasktype.TaskTypeDirectlinks
|
||||
}
|
||||
|
||||
func (t *Task) TaskID() string {
|
||||
return t.ID
|
||||
}
|
||||
|
||||
func NewTask(
|
||||
id string,
|
||||
ctx context.Context,
|
||||
links []string,
|
||||
stor storage.Storage,
|
||||
storPath string,
|
||||
progressTracker ProgressTracker,
|
||||
) *Task {
|
||||
_, ok := stor.(storage.StorageCannotStream)
|
||||
stream := config.C().Stream && !ok
|
||||
files := make([]*File, 0, len(links))
|
||||
for _, link := range links {
|
||||
files = append(files, &File{
|
||||
URL: link,
|
||||
})
|
||||
}
|
||||
return &Task{
|
||||
ID: id,
|
||||
ctx: ctx,
|
||||
files: files,
|
||||
Storage: stor,
|
||||
StorPath: storPath,
|
||||
Progress: progressTracker,
|
||||
stream: stream,
|
||||
client: http.DefaultClient,
|
||||
processing: make(map[string]*File),
|
||||
processingMu: sync.RWMutex{},
|
||||
failed: make(map[string]error),
|
||||
totalFiles: int64(len(files)),
|
||||
}
|
||||
}
|
||||
205
core/tasks/directlinks/util.go
Normal file
205
core/tasks/directlinks/util.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package directlinks
|
||||
|
||||
import (
|
||||
"mime"
|
||||
"net/url"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// parseFilename extracts filename from Content-Disposition header
|
||||
// It handles multiple encoding scenarios:
|
||||
// 1. RFC 5987/RFC 2231 format: filename*=UTF-8”%E6%B5%8B%E8%AF%95.zip (preferred, checked first)
|
||||
// 2. MIME encoded-word: filename="=?UTF-8?B?5rWL6K+VLnppcA==?="
|
||||
// 3. URL-encoded: filename="%E6%B5%8B%E8%AF%95.zip"
|
||||
// 4. Plain ASCII filename
|
||||
//
|
||||
// The key fix is checking filename*= first before mime.ParseMediaType, because
|
||||
// some servers send Content-Disposition headers with invalid characters that cause
|
||||
// mime.ParseMediaType to fail, but the filename*= parameter is still valid.
|
||||
func parseFilename(contentDisposition string) string {
|
||||
// First, try to find filename*= (RFC 5987 format, most reliable for non-ASCII)
|
||||
if filename := parseFilenameExtended(contentDisposition); filename != "" {
|
||||
return filename
|
||||
}
|
||||
|
||||
// Try standard MIME parsing for regular filename= parameter
|
||||
_, params, err := mime.ParseMediaType(contentDisposition)
|
||||
if err == nil {
|
||||
if filename := params["filename"]; filename != "" {
|
||||
return decodeFilenameParam(filename)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: manual parsing if mime.ParseMediaType fails
|
||||
return parseFilenameFallback(contentDisposition)
|
||||
}
|
||||
|
||||
// parseFilenameExtended parses RFC 5987/RFC 2231 extended parameter format
|
||||
// Format: filename*=charset'language'value (e.g., UTF-8”%E6%B5%8B%E8%AF%95.zip)
|
||||
func parseFilenameExtended(cd string) string {
|
||||
// Look for filename*= (case-insensitive)
|
||||
lower := strings.ToLower(cd)
|
||||
idx := strings.Index(lower, "filename*=")
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Extract the value after filename*=
|
||||
value := cd[idx+len("filename*="):]
|
||||
|
||||
// Find the end of the value (next ; or end of string)
|
||||
if endIdx := strings.Index(value, ";"); endIdx != -1 {
|
||||
value = value[:endIdx]
|
||||
}
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
// Parse charset'language'encoded-value format
|
||||
// Common format: UTF-8''%E6%B5%8B%E8%AF%95.zip
|
||||
parts := strings.SplitN(value, "''", 2)
|
||||
if len(parts) == 2 {
|
||||
// parts[0] is charset (e.g., "UTF-8")
|
||||
// parts[1] is percent-encoded value
|
||||
decoded, err := url.QueryUnescape(parts[1])
|
||||
if err == nil {
|
||||
return decoded
|
||||
}
|
||||
}
|
||||
|
||||
// Try with single quote delimiter as well (some servers use this)
|
||||
parts = strings.SplitN(value, "'", 3)
|
||||
if len(parts) >= 3 {
|
||||
decoded, err := url.QueryUnescape(parts[2])
|
||||
if err == nil {
|
||||
return decoded
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// TryUrlQueryUnescape tries to unescape a URL-encoded string.
|
||||
//
|
||||
// If unescaping fails, it returns the original string.
|
||||
func tryUrlQueryUnescape(s string) string {
|
||||
if decoded, err := url.QueryUnescape(s); err == nil {
|
||||
return decoded
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// decodeFilenameParam decodes a filename parameter value
|
||||
// Handles MIME encoded-word, URL encoding, and GBK encoding fallback
|
||||
func decodeFilenameParam(filename string) string {
|
||||
// Check if the filename is MIME encoded-word (e.g., =?UTF-8?B?...?=)
|
||||
if strings.HasPrefix(filename, "=?") {
|
||||
decoder := new(mime.WordDecoder)
|
||||
// Some servers use "UTF8" instead of "UTF-8", create a normalized copy
|
||||
normalizedFilename := strings.Replace(filename, "UTF8", "UTF-8", 1)
|
||||
if decoded, err := decoder.Decode(normalizedFilename); err == nil {
|
||||
return decoded
|
||||
}
|
||||
}
|
||||
|
||||
// Try URL decoding
|
||||
decoded := tryUrlQueryUnescape(filename)
|
||||
|
||||
// Check if the result is valid UTF-8. If not, try GBK decoding.
|
||||
// This handles the case where Chinese Windows servers send GBK-encoded filenames
|
||||
// which appear as garbled characters (e.g., "下载地址.zip" -> "<22><><EFBFBD>ص<EFBFBD>ַ.zip")
|
||||
if !utf8.ValidString(decoded) {
|
||||
if gbkDecoded := tryDecodeGBK(decoded); gbkDecoded != "" {
|
||||
return gbkDecoded
|
||||
}
|
||||
}
|
||||
|
||||
return decoded
|
||||
}
|
||||
|
||||
// gbkDecoder is a reusable GBK decoder for better performance
|
||||
var gbkDecoder = simplifiedchinese.GBK.NewDecoder()
|
||||
|
||||
// tryDecodeGBK attempts to decode a string as GBK/GB2312/GB18030 encoding
|
||||
// Returns empty string if decoding fails or result is not valid UTF-8
|
||||
func tryDecodeGBK(s string) string {
|
||||
// GBK uses 1-2 bytes per character. Single-byte chars are 0x00-0x7F (ASCII compatible).
|
||||
// Double-byte chars have first byte 0x81-0xFE and second byte 0x40-0xFE.
|
||||
// Skip if string is empty or all ASCII (valid UTF-8)
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Create a fresh decoder since the transform state may be corrupted
|
||||
decoder := gbkDecoder
|
||||
decoded, err := decoder.Bytes([]byte(s))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
result := string(decoded)
|
||||
if utf8.ValidString(result) {
|
||||
return result
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseFilenameFallback manually parses filename= when mime.ParseMediaType fails
|
||||
func parseFilenameFallback(cd string) string {
|
||||
// Look for filename= (case-insensitive)
|
||||
lower := strings.ToLower(cd)
|
||||
idx := strings.Index(lower, "filename=")
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Skip "filename=" prefix
|
||||
value := cd[idx+len("filename="):]
|
||||
|
||||
// Find the end of the value
|
||||
if endIdx := strings.Index(value, ";"); endIdx != -1 {
|
||||
value = value[:endIdx]
|
||||
}
|
||||
value = strings.TrimSpace(value)
|
||||
|
||||
// Remove quotes if present
|
||||
if len(value) >= 2 {
|
||||
if (value[0] == '"' && value[len(value)-1] == '"') ||
|
||||
(value[0] == '\'' && value[len(value)-1] == '\'') {
|
||||
value = value[1 : len(value)-1]
|
||||
}
|
||||
}
|
||||
|
||||
return decodeFilenameParam(value)
|
||||
}
|
||||
|
||||
var progressUpdatesLevels = []struct {
|
||||
size int64 // 文件大小阈值
|
||||
stepPercent int // 每多少 % 更新一次
|
||||
}{
|
||||
{10 << 20, 100},
|
||||
{50 << 20, 50},
|
||||
{200 << 20, 20},
|
||||
{500 << 20, 10},
|
||||
}
|
||||
|
||||
func shouldUpdateProgress(total, downloaded int64, lastUpdatePercent int) bool {
|
||||
if total <= 0 || downloaded <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
percent := int((downloaded * 100) / total)
|
||||
if percent <= lastUpdatePercent {
|
||||
return false
|
||||
}
|
||||
|
||||
step := progressUpdatesLevels[len(progressUpdatesLevels)-1].stepPercent
|
||||
for _, lvl := range progressUpdatesLevels {
|
||||
if total < lvl.size {
|
||||
step = lvl.stepPercent
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return percent >= lastUpdatePercent+step
|
||||
}
|
||||
@@ -19,9 +19,9 @@ type Task struct {
|
||||
Stor storage.Storage
|
||||
StorPath string
|
||||
item *parser.Item
|
||||
httpClient *http.Client
|
||||
progress ProgressTracker
|
||||
stream bool
|
||||
httpClient *http.Client // [TODO] btorrent support?
|
||||
progress ProgressTracker
|
||||
stream bool
|
||||
|
||||
totalResources int64
|
||||
downloaded atomic.Int64 // downloaded resources count
|
||||
|
||||
Reference in New Issue
Block a user