feat: parse url with js plugins support (#96)

* feat: WIP. add parser functionality and text message handling * fix: use json to marshal js result * feat: add metadata handling and version validation for jsParser * refactor: rename parser package to parsers and restructure parser handling * refactor: core code struct and impl parse task handle * feat: impl parsed download * fix: seek cache file when processing tph picture * feat: implement parsed task handling and progress tracking * feat: enhance task processing with concurrency control and progress tracking * feat: add resource ID generation and improve resource processing handling * feat: improve message formatting in parsed text and progress completion * feat: add example js plugin * feat: implement Twitter parser * fix: twitter parse video json decode error * feat: impl stream mode for parse task
2026-06-27 18:21:35 +08:00 · 2025-08-21 23:48:17 +08:00
parent 79386bdd7d
commit 302db2fe75
47 changed files with 1348 additions and 47 deletions
--- a/core/tasks/batchtfile/task.go
+++ b/core/tasks/batchtfile/task.go
@@ -0,0 +1,98 @@
+package batchtfile
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+
+	"github.com/krau/SaveAny-Bot/config"
+	"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
+	"github.com/krau/SaveAny-Bot/pkg/tfile"
+	"github.com/krau/SaveAny-Bot/storage"
+	"github.com/rs/xid"
+)
+
+type TaskElement struct {
+	ID        string
+	Storage   storage.Storage
+	Path      string
+	File      tfile.TGFile
+	localPath string
+	stream    bool
+}
+
+type Task struct {
+	ID           string
+	Ctx          context.Context
+	Elems        []TaskElement
+	Progress     ProgressTracker
+	IgnoreErrors bool // if true, errors during processing will be ignored
+	downloaded   atomic.Int64
+	totalSize    int64
+	processing   map[string]TaskElementInfo
+	processingMu sync.RWMutex
+	failed       map[string]error // [TODO] errors for each element
+}
+
+func (t *Task) Type() tasktype.TaskType {
+	return tasktype.TaskTypeTgfiles
+}
+
+func NewTaskElement(
+	stor storage.Storage,
+	path string,
+	file tfile.TGFile,
+) (*TaskElement, error) {
+	id := xid.New().String()
+	_, ok := stor.(storage.StorageCannotStream)
+	if !config.Cfg.Stream || ok {
+		cachePath, err := filepath.Abs(filepath.Join(config.Cfg.Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name())))
+		if err != nil {
+			return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
+		}
+		return &TaskElement{
+			ID:        id,
+			Storage:   stor,
+			Path:      path,
+			File:      file,
+			localPath: cachePath,
+		}, nil
+	}
+	return &TaskElement{
+		ID:      id,
+		Storage: stor,
+		Path:    path,
+		File:    file,
+		stream:  true,
+	}, nil
+}
+
+func NewBatchTGFileTask(
+	id string,
+	ctx context.Context,
+	files []TaskElement,
+	progress ProgressTracker,
+	ignoreErrors bool,
+) *Task {
+	task := &Task{
+		ID:         id,
+		Ctx:        ctx,
+		Elems:      files,
+		Progress:   progress,
+		downloaded: atomic.Int64{},
+		totalSize: func() int64 {
+			var total int64
+			for _, elem := range files {
+				total += elem.File.Size()
+			}
+			return total
+		}(),
+		processing:   make(map[string]TaskElementInfo),
+		IgnoreErrors: ignoreErrors,
+		processingMu: sync.RWMutex{},
+		failed:       make(map[string]error),
+	}
+	return task
+}