feat: parse url with js plugins support (#96)

* feat: WIP. add parser functionality and text message handling

* fix: use json to marshal js result

* feat: add metadata handling and version validation for jsParser

* refactor: rename parser package to parsers and restructure parser handling

* refactor: core code struct and impl parse task handle

* feat: impl parsed download

* fix: seek cache file when processing tph picture

* feat: implement parsed task handling and progress tracking

* feat: enhance task processing with concurrency control and progress tracking

* feat: add resource ID generation and improve resource processing handling

* feat: improve message formatting in parsed text and progress completion

* feat: add example js plugin

* feat: implement Twitter parser

* fix: twitter parse video json decode error

* feat: impl stream mode for parse task

core/tasks/batchtfile/execute.go

@@ -1,4 +1,4 @@
-package batchtftask
+package batchtfile
 
 import (
 	"context"
@@ -25,14 +25,19 @@ func (t *Task) Execute(ctx context.Context) error {
 	eg, gctx := errgroup.WithContext(ctx)
 	eg.SetLimit(workers)
 	for _, elem := range t.Elems {
-		elem := elem
 		eg.Go(func() error {
+			t.processingMu.RLock()
 			if t.processing[elem.ID] != nil {
 				return fmt.Errorf("element with ID %s is already being processed", elem.ID)
 			}
+			t.processingMu.RUnlock()
+			t.processingMu.Lock()
 			t.processing[elem.ID] = &elem
+			t.processingMu.Unlock()
 			defer func() {
+				t.processingMu.Lock()
 				delete(t.processing, elem.ID)
+				t.processingMu.Unlock()
 			}()
 			return t.processElement(gctx, elem)
 		})

core/tasks/batchtfile/progress.go

@@ -1,4 +1,4 @@
-package batchtftask
+package batchtfile
 
 import (
 	"context"

core/tasks/batchtfile/task.go

@@ -1,9 +1,10 @@
-package batchtftask
+package batchtfile
 
 import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 
 	"github.com/krau/SaveAny-Bot/config"
@@ -31,7 +32,8 @@ type Task struct {
 	downloaded   atomic.Int64
 	totalSize    int64
 	processing   map[string]TaskElementInfo
-	failed       map[string]error // errors for each element
+	processingMu sync.RWMutex
+	failed       map[string]error // [TODO] errors for each element
 }
 
 func (t *Task) Type() tasktype.TaskType {
@@ -89,6 +91,7 @@ func NewBatchTGFileTask(
 		}(),
 		processing:   make(map[string]TaskElementInfo),
 		IgnoreErrors: ignoreErrors,
+		processingMu: sync.RWMutex{},
 		failed:       make(map[string]error),
 	}
 	return task

core/tasks/batchtfile/taskinfo.go

@@ -1,4 +1,4 @@
-package batchtftask
+package batchtfile
 
 type TaskElementInfo interface {
 	FileName() string

core/tasks/batchtfile/utils.go

@@ -1,4 +1,4 @@
-package batchtftask
+package batchtfile
 
 var progressUpdatesLevels = []struct {
 	size        int64 // 文件大小阈值

core/tasks/parsed/execute.go

@@ -0,0 +1,139 @@
+package parsed
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"path/filepath"
+
+	"github.com/charmbracelet/log"
+	"github.com/duke-git/lancet/v2/retry"
+	"github.com/krau/SaveAny-Bot/common/utils/fsutil"
+	"github.com/krau/SaveAny-Bot/common/utils/ioutil"
+	"github.com/krau/SaveAny-Bot/config"
+	"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
+	"github.com/krau/SaveAny-Bot/pkg/parser"
+	"golang.org/x/sync/errgroup"
+)
+
+func (t *Task) Execute(ctx context.Context) error {
+	logger := log.FromContext(ctx)
+	logger.Infof("Starting Parsed item task %s", t.item.Title)
+	if t.progress != nil {
+		t.progress.OnStart(ctx, t)
+	}
+	eg, gctx := errgroup.WithContext(ctx)
+	eg.SetLimit(config.Cfg.Workers)
+	for _, resource := range t.item.Resources {
+		eg.Go(func() error {
+			t.processingMu.RLock()
+			if t.processing[resource.ID()] != nil {
+				return fmt.Errorf("resource %s is already being processed", resource.ID())
+			}
+			t.processingMu.RUnlock()
+			t.processingMu.Lock()
+			t.processing[resource.ID()] = &resource
+			t.processingMu.Unlock()
+			defer func() {
+				t.processingMu.Lock()
+				delete(t.processing, resource.URL)
+				t.processingMu.Unlock()
+			}()
+			err := t.processResource(gctx, resource)
+			t.downloaded.Add(1)
+			if errors.Is(err, context.Canceled) {
+				logger.Debug("Resource processing canceled")
+				return err
+			}
+			if err != nil {
+				logger.Errorf("Error processing resource %s: %v", resource.URL, err)
+				return fmt.Errorf("failed to process resource %s: %w", resource.URL, err)
+			}
+			return nil
+		})
+	}
+	err := eg.Wait()
+	if err != nil {
+		logger.Errorf("Error during Parsed item task execution: %v", err)
+	} else {
+		logger.Infof("Parsed item task %s completed successfully", t.item.Title)
+	}
+	if t.progress != nil {
+		t.progress.OnDone(ctx, t, err)
+	}
+	return err
+}
+
+func (t *Task) processResource(ctx context.Context, resource parser.Resource) error {
+	logger := log.FromContext(ctx)
+	err := retry.Retry(func() error {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, resource.URL, nil)
+		if err != nil {
+			return err
+		}
+		if resource.Headers != nil {
+			for k, v := range resource.Headers {
+				req.Header.Set(k, v)
+			}
+		}
+		resp, err := t.httpClient.Do(req)
+		if err != nil {
+			return fmt.Errorf("failed to download resource %s: %w", resource.URL, err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("failed to download resource %s: %s", resource.URL, resp.Status)
+		}
+		ctx = context.WithValue(ctx, ctxkey.ContentLength, func() int64 {
+			if resource.Size > 0 {
+				return resource.Size
+			}
+			return resp.ContentLength
+		}())
+		if t.stream {
+			return t.Stor.Save(ctx, resp.Body, path.Join(t.StorPath, resource.Filename))
+		}
+		cacheFile, err := fsutil.CreateFile(filepath.Join(config.Cfg.Temp.BasePath,
+			fmt.Sprintf("resource_%s_%s", t.ID, resource.Filename)))
+		if err != nil {
+			return fmt.Errorf("failed to create cache file for resource %s: %w", resource.URL, err)
+		}
+		defer func() {
+			if err := cacheFile.CloseAndRemove(); err != nil {
+				logger.Errorf("Failed to close and remove cache file: %v", err)
+			}
+		}()
+		wr := ioutil.NewProgressWriter(cacheFile, func(n int) {
+			t.downloadedBytes.Add(int64(n))
+			if t.progress != nil {
+				t.progress.OnProgress(ctx, t)
+			}
+		})
+
+		copyResultCh := make(chan error, 1)
+		go func() {
+			_, err := io.Copy(wr, resp.Body)
+			copyResultCh <- err
+		}()
+		select {
+		case err := <-copyResultCh:
+			if err != nil {
+				return fmt.Errorf("failed to copy resource %s to cache file: %w", resource.URL, err)
+			}
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+		_, err = cacheFile.Seek(0, 0)
+		if err != nil {
+			return fmt.Errorf("failed to seek cache file for resource %s: %w", resource.URL, err)
+		}
+		return t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, resource.Filename))
+	}, retry.Context(ctx), retry.RetryTimes(uint(config.Cfg.Retry)))
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	return err
+}

core/tasks/parsed/progress.go

@@ -0,0 +1,209 @@
+package parsed
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/charmbracelet/log"
+	"github.com/duke-git/lancet/v2/slice"
+	"github.com/gotd/td/telegram/message/entity"
+	"github.com/gotd/td/telegram/message/styling"
+	"github.com/gotd/td/tg"
+	"github.com/krau/SaveAny-Bot/common/utils/dlutil"
+	"github.com/krau/SaveAny-Bot/common/utils/tgutil"
+)
+
+var progressUpdatesLevels = []struct {
+	size        int64 // 文件大小阈值
+	stepPercent int   // 每多少 % 更新一次
+}{
+	{10 << 20, 100},
+	{50 << 20, 50},
+	{200 << 20, 20},
+	{500 << 20, 10},
+}
+
+func shouldUpdateProgress(total, downloaded int64, lastUpdatePercent int) bool {
+	if total <= 0 || downloaded <= 0 {
+		return false
+	}
+
+	percent := int((downloaded * 100) / total)
+	if percent <= lastUpdatePercent {
+		return false
+	}
+
+	step := progressUpdatesLevels[len(progressUpdatesLevels)-1].stepPercent
+	for _, lvl := range progressUpdatesLevels {
+		if total < lvl.size {
+			step = lvl.stepPercent
+			break
+		}
+	}
+
+	return percent >= lastUpdatePercent+step
+}
+
+type ProgressTracker interface {
+	OnStart(ctx context.Context, info TaskInfo)
+	OnProgress(ctx context.Context, info TaskInfo)
+	OnDone(ctx context.Context, info TaskInfo, err error)
+}
+
+type Progress struct {
+	MessageID         int
+	ChatID            int64
+	start             time.Time
+	lastUpdatePercent atomic.Int32
+}
+
+func (p *Progress) OnStart(ctx context.Context, info TaskInfo) {
+	logger := log.FromContext(ctx)
+	p.start = time.Now()
+	p.lastUpdatePercent.Store(0)
+	logger.Debugf("Parsed task progress tracking started for message %d in chat %d", p.MessageID, p.ChatID)
+	entityBuilder := entity.Builder{}
+	var entities []tg.MessageEntityClass
+	if err := styling.Perform(&entityBuilder,
+		styling.Plain(fmt.Sprintf("开始下载 %s 的资源\n总大小: ", info.Site())),
+		styling.Code(fmt.Sprintf("%.2f MB (%d个资源)", float64(info.TotalBytes())/(1024*1024), info.TotalResources())),
+	); err != nil {
+		log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
+		return
+	}
+	text, entities := entityBuilder.Complete()
+	req := &tg.MessagesEditMessageRequest{
+		ID: p.MessageID,
+	}
+	req.SetMessage(text)
+	req.SetEntities(entities)
+	req.SetReplyMarkup(&tg.ReplyInlineMarkup{
+		Rows: []tg.KeyboardButtonRow{
+			{
+				Buttons: []tg.KeyboardButtonClass{
+					tgutil.BuildCancelButton(info.TaskID()),
+				},
+			},
+		}},
+	)
+	ext := tgutil.ExtFromContext(ctx)
+	if ext != nil {
+		ext.EditMessage(p.ChatID, req)
+		return
+	}
+}
+
+func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) {
+	if !shouldUpdateProgress(info.TotalBytes(), info.DownloadedBytes(), int(p.lastUpdatePercent.Load())) {
+		return
+	}
+	percent := int((info.DownloadedBytes() * 100) / info.TotalBytes())
+	if p.lastUpdatePercent.Load() == int32(percent) {
+		return
+	}
+	p.lastUpdatePercent.Store(int32(percent))
+	log.FromContext(ctx).Debugf("Progress update: %s, %d/%d", info.TaskID(), info.DownloadedBytes(), info.TotalBytes())
+	entityBuilder := entity.Builder{}
+	var entities []tg.MessageEntityClass
+	if err := styling.Perform(&entityBuilder,
+		styling.Plain("正在下载\n总大小: "),
+		styling.Code(fmt.Sprintf("%.2f MB (%d个文件)", float64(info.TotalBytes())/(1024*1024), info.TotalResources())),
+		styling.Plain("\n正在处理:\n"),
+		func() styling.StyledTextOption {
+			var lines []string
+			for _, elem := range info.Processing() {
+				lines = append(lines, fmt.Sprintf("  - %s (%.2f MB)", elem.FileName(), float64(elem.FileSize())/(1024*1024)))
+			}
+			if len(lines) == 0 {
+				lines = append(lines, "  - 无")
+			}
+			return styling.Plain(slice.Join(lines, "\n"))
+		}(),
+		styling.Plain("\n平均速度: "),
+		styling.Bold(fmt.Sprintf("%.2f MB/s", dlutil.GetSpeed(info.DownloadedBytes(), p.start)/(1024*1024))),
+		styling.Plain("\n当前进度: "),
+		styling.Bold(fmt.Sprintf("%.2f%%", float64(info.DownloadedBytes())/float64(info.TotalBytes())*100)),
+	); err != nil {
+		log.FromContext(ctx).Errorf("Failed to build entities: %s", err)
+		return
+	}
+	text, entities := entityBuilder.Complete()
+	req := &tg.MessagesEditMessageRequest{
+		ID: p.MessageID,
+	}
+	req.SetMessage(text)
+	req.SetEntities(entities)
+	req.SetReplyMarkup(&tg.ReplyInlineMarkup{
+		Rows: []tg.KeyboardButtonRow{
+			{
+				Buttons: []tg.KeyboardButtonClass{
+					tgutil.BuildCancelButton(info.TaskID()),
+				},
+			},
+		}},
+	)
+	ext := tgutil.ExtFromContext(ctx)
+	if ext != nil {
+		ext.EditMessage(p.ChatID, req)
+		return
+	}
+}
+
+func (p *Progress) OnDone(ctx context.Context, info TaskInfo, err error) {
+	logger := log.FromContext(ctx)
+	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			logger.Infof("Parsed task %s was canceled", info.TaskID())
+			ext := tgutil.ExtFromContext(ctx)
+			if ext != nil {
+				ext.EditMessage(p.ChatID, &tg.MessagesEditMessageRequest{
+					ID:      p.MessageID,
+					Message: fmt.Sprintf("处理已取消: %s", info.TaskID()),
+				})
+			}
+		} else {
+			logger.Errorf("Parsed task %s failed: %s", info.TaskID(), err)
+			ext := tgutil.ExtFromContext(ctx)
+			if ext != nil {
+				ext.EditMessage(p.ChatID, &tg.MessagesEditMessageRequest{
+					ID:      p.MessageID,
+					Message: fmt.Sprintf("处理失败: %s", err.Error()),
+				})
+			}
+		}
+		return
+	}
+	logger.Infof("Parsed task %s completed successfully", info.TaskID())
+
+	entityBuilder := entity.Builder{}
+	if err := styling.Perform(&entityBuilder,
+		styling.Plain("处理完成, 资源数量: "),
+		styling.Code(fmt.Sprintf("%d", info.TotalResources())),
+		styling.Plain("\n保存路径: "),
+		styling.Code(fmt.Sprintf("[%s]:%s", info.StorageName(), info.StoragePath())),
+	); err != nil {
+		logger.Errorf("Failed to build entities: %s", err)
+		return
+	}
+	text, entities := entityBuilder.Complete()
+	req := &tg.MessagesEditMessageRequest{
+		ID: p.MessageID,
+	}
+	req.SetMessage(text)
+	req.SetEntities(entities)
+
+	ext := tgutil.ExtFromContext(ctx)
+	if ext != nil {
+		ext.EditMessage(p.ChatID, req)
+	}
+}
+
+func NewProgress(messageID int, chatID int64) *Progress {
+	return &Progress{
+		MessageID: messageID,
+		ChatID:    chatID,
+	}
+}

core/tasks/parsed/task.go

@@ -0,0 +1,84 @@
+package parsed
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"sync/atomic"
+
+	"github.com/krau/SaveAny-Bot/config"
+	"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
+	"github.com/krau/SaveAny-Bot/pkg/parser"
+	"github.com/krau/SaveAny-Bot/storage"
+)
+
+type Task struct {
+	ID         string
+	Ctx        context.Context
+	Stor       storage.Storage
+	StorPath   string
+	item       *parser.Item
+	httpClient *http.Client
+	progress   ProgressTracker
+	stream     bool
+
+	totalResources  int64
+	downloaded      atomic.Int64 // downloaded resources count
+	totalBytes      int64        // total bytes to download
+	downloadedBytes atomic.Int64 // downloaded bytes count
+	processing      map[string]ResourceInfo
+	processingMu    sync.RWMutex
+	failed          map[string]error // [TODO] errors for each resource
+}
+
+func (t *Task) Type() tasktype.TaskType {
+	return tasktype.TaskTypeParseditem
+}
+
+func (t *Task) TaskID() string {
+	return t.ID
+}
+
+func NewTask(
+	id string,
+	ctx context.Context,
+	stor storage.Storage,
+	storPath string,
+	item *parser.Item,
+	progressTracker ProgressTracker,
+) *Task {
+	client := &http.Client{
+		Transport: &http.Transport{
+			// [TODO] configure it via config
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
+	_, ok := stor.(storage.StorageCannotStream)
+	stream := config.Cfg.Stream && !ok
+	return &Task{
+		ID:             id,
+		Ctx:            ctx,
+		Stor:           stor,
+		StorPath:       storPath,
+		item:           item,
+		totalResources: int64(len(item.Resources)),
+		downloaded:     atomic.Int64{},
+		totalBytes: func() int64 {
+			var total int64
+			for _, res := range item.Resources {
+				if res.Size < 0 {
+					continue // skip resources with unknown size
+				}
+				total += res.Size
+			}
+			return total
+		}(),
+		stream:          stream,
+		downloadedBytes: atomic.Int64{},
+		httpClient:      client,
+		progress:        progressTracker,
+		processing:      make(map[string]ResourceInfo),
+		processingMu:    sync.RWMutex{},
+		failed:          make(map[string]error),
+	}
+}

core/tasks/parsed/taskinfo.go

@@ -0,0 +1,51 @@
+package parsed
+
+type TaskInfo interface {
+	TaskID() string
+	Site() string
+	TotalResources() int64
+	Downloaded() int64
+	TotalBytes() int64
+	DownloadedBytes() int64
+	Processing() map[string]ResourceInfo
+	StorageName() string
+	StoragePath() string
+}
+
+func (t *Task) StoragePath() string {
+	return t.StorPath
+}
+func (t *Task) TotalResources() int64 {
+	return t.totalResources
+}
+
+func (t *Task) Downloaded() int64 {
+	return t.downloaded.Load()
+}
+
+func (t *Task) StorageName() string {
+	return t.Stor.Name()
+}
+
+func (t *Task) Site() string {
+	return t.item.Site
+}
+
+func (t *Task) TotalBytes() int64 {
+	return t.totalBytes
+}
+
+func (t *Task) DownloadedBytes() int64 {
+	return t.downloadedBytes.Load()
+}
+
+func (t *Task) Processing() map[string]ResourceInfo {
+	t.processingMu.RLock()
+	defer t.processingMu.RUnlock()
+	return t.processing
+}
+
+type ResourceInfo interface {
+	FileName() string
+	FileSize() int64
+}

core/tasks/telegraph/execute.go

@@ -1,4 +1,4 @@
-package tphtask
+package telegraph
 
 import (
 	"context"
@@ -22,8 +22,6 @@ func (t *Task) Execute(ctx context.Context) error {
 	eg, gctx := errgroup.WithContext(ctx)
 	eg.SetLimit(config.Cfg.Workers)
 	for i, pic := range t.Pics {
-		pic := pic
-		i := i
 		eg.Go(func() error {
 			err := t.processPic(gctx, pic, i)
 			if err != nil {
@@ -79,6 +77,11 @@ func (t *Task) processPic(ctx context.Context, picUrl string, index int) error {
 				lastErr = fmt.Errorf("failed to copy picture %s to cache file: %w", filename, lastErr)
 				return lastErr
 			}
+			_, err = cacheFile.Seek(0, 0)
+			if err != nil {
+				lastErr = fmt.Errorf("failed to seek cache file for picture %s: %w", filename, err)
+				return lastErr
+			}
 			lastErr = t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, filename))
 		} else {
 			lastErr = t.Stor.Save(ctx, body, path.Join(t.StorPath, filename))

core/tasks/telegraph/progress.go

@@ -1,4 +1,4 @@
-package tphtask
+package telegraph
 
 import (
 	"context"

core/tasks/telegraph/task.go

@@ -1,4 +1,4 @@
-package tphtask
+package telegraph
 
 import (
 	"context"
@@ -39,7 +39,7 @@ func NewTask(
 	progress ProgressTracker,
 ) *Task {
 	_, cannotStream := stor.(storage.StorageCannotStream)
-	tphtask := &Task{
+	telegraph := &Task{
 		ID:           id,
 		Ctx:          ctx,
 		PhPath:       phPath,
@@ -52,5 +52,5 @@ func NewTask(
 		totalpics:    len(pics),
 		downloaded:   atomic.Int64{},
 	}
-	return tphtask
+	return telegraph
 }

core/tasks/telegraph/taskinfo.go

@@ -1,4 +1,4 @@
-package tphtask
+package telegraph
 
 type TaskInfo interface {
 	TaskID() string

core/tasks/telegraph/utils.go

@@ -1,4 +1,4 @@
-package tphtask
+package telegraph
 
 func shouldUpdateProgress(downloaded int64, total int64) bool {
 	if total <= 0 || downloaded <= 0 {

core/tasks/tfile/execute.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 import (
 	"context"

core/tasks/tfile/progress.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 import (
 	"context"

core/tasks/tfile/stream.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 import (
 	"context"

core/tasks/tfile/taskinfo.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 type TaskInfo interface {
 	TaskID() string

core/tasks/tfile/tftask.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 import (
 	"context"
@@ -40,7 +40,7 @@ func NewTGFileTask(
 		if err != nil {
 			return nil, fmt.Errorf("failed to get absolute path for cache: %w", err)
 		}
-		tftask := &Task{
+		tfile := &Task{
 			ID:        id,
 			Ctx:       ctx,
 			File:      file,
@@ -49,7 +49,7 @@ func NewTGFileTask(
 			Progress:  progress,
 			localPath: cachePath,
 		}
-		return tftask, nil
+		return tfile, nil
 	}
 	tfileTask := &Task{
 		ID:       id,

core/tasks/tfile/util.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 var progressUpdatesLevels = []struct {
 	size        int64 // 文件大小阈值

core/tasks/tfile/writer.go

@@ -1,4 +1,4 @@
-package tftask
+package tfile
 
 import (
 	"context"