feat: parse url with js plugins support (#96)

* feat: WIP. add parser functionality and text message handling

* fix: use json to marshal js result

* feat: add metadata handling and version validation for jsParser

* refactor: rename parser package to parsers and restructure parser handling

* refactor: core code struct and impl parse task handle

* feat: impl parsed download

* fix: seek cache file when processing tph picture

* feat: implement parsed task handling and progress tracking

* feat: enhance task processing with concurrency control and progress tracking

* feat: add resource ID generation and improve resource processing handling

* feat: improve message formatting in parsed text and progress completion

* feat: add example js plugin

* feat: implement Twitter parser

* fix: twitter parse video json decode error

* feat: impl stream mode for parse task

client/bot/handlers/utils/shortcut/message.go

@@ -32,8 +32,7 @@ func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message *
 	media := message.Media
 	supported := mediautil.IsSupported(media)
 	if !supported {
-		ctx.Reply(update, ext.ReplyTextString("不支持的消息类型"), nil)
-		return nil, nil, dispatcher.EndGroups
+		return nil, nil, dispatcher.ContinueGroups
 	}
 
 	replied, err = ctx.Reply(update, ext.ReplyTextString("正在获取文件信息..."), nil)

client/bot/handlers/utils/shortcut/parsed.go

@@ -0,0 +1,35 @@
+package shortcut
+
+import (
+	"github.com/celestix/gotgproto/dispatcher"
+	"github.com/celestix/gotgproto/ext"
+	"github.com/charmbracelet/log"
+	"github.com/gotd/td/tg"
+	"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem"
+	"github.com/krau/SaveAny-Bot/common/utils/tgutil"
+	"github.com/krau/SaveAny-Bot/core"
+	"github.com/krau/SaveAny-Bot/core/tasks/parsed"
+	"github.com/krau/SaveAny-Bot/pkg/parser"
+	"github.com/krau/SaveAny-Bot/storage"
+	"github.com/rs/xid"
+)
+
+func CreateAndAddParsedTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, item *parser.Item, msgID int, userID int64) error {
+	injectCtx := tgutil.ExtWithContext(ctx.Context, ctx)
+	task := parsed.NewTask(xid.New().String(), injectCtx, stor, stor.JoinStoragePath(dirPath), item, parsed.NewProgress(msgID, userID))
+	if err := core.AddTask(injectCtx, task); err != nil {
+		log.FromContext(ctx).Errorf("Failed to add task: %s", err)
+		ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
+			ID:      msgID,
+			Message: "任务添加失败: " + err.Error(),
+		})
+		return dispatcher.EndGroups
+	}
+	text, entities := msgelem.BuildTaskAddedEntities(ctx, item.Title, core.GetLength(ctx))
+	ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
+		ID:       msgID,
+		Message:  text,
+		Entities: entities,
+	})
+	return dispatcher.EndGroups
+}

client/bot/handlers/utils/shortcut/tftask.go

@@ -13,15 +13,15 @@ import (
 	"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/ruleutil"
 	"github.com/krau/SaveAny-Bot/common/utils/tgutil"
 	"github.com/krau/SaveAny-Bot/core"
-	"github.com/krau/SaveAny-Bot/core/batchtftask"
-	"github.com/krau/SaveAny-Bot/core/tftask"
+	"github.com/krau/SaveAny-Bot/core/tasks/batchtfile"
+	tftask "github.com/krau/SaveAny-Bot/core/tasks/tfile"
 	"github.com/krau/SaveAny-Bot/database"
 	"github.com/krau/SaveAny-Bot/pkg/tfile"
 	"github.com/krau/SaveAny-Bot/storage"
 	"github.com/rs/xid"
 )
 
-// 创建一个 tftask.TGFileTask 并添加到任务队列中, 以编辑消息的方式反馈结果
+// 创建一个 tfile.TGFileTask 并添加到任务队列中, 以编辑消息的方式反馈结果
 func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage.Storage, dirPath string, file tfile.TGFileMessage, trackMsgID int) error {
 	logger := log.FromContext(ctx)
 	user, err := database.GetUserByChatID(ctx, userID)
@@ -82,7 +82,7 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage
 	return dispatcher.EndGroups
 }
 
-// 创建一个 batchtftask.BatchTGFileTask 并添加到任务队列中, 以编辑消息的方式反馈结果
+// 创建一个 batchtfile.BatchTGFileTask 并添加到任务队列中, 以编辑消息的方式反馈结果
 func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage.Storage, dirPath string, files []tfile.TGFileMessage, trackMsgID int) error {
 	logger := log.FromContext(ctx)
 	user, err := database.GetUserByChatID(ctx, userID)
@@ -110,7 +110,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
 		return storname, dirP
 	}
 
-	elems := make([]batchtftask.TaskElement, 0, len(files))
+	elems := make([]batchtfile.TaskElement, 0, len(files))
 	type albumFile struct {
 		file    tfile.TGFileMessage
 		storage storage.Storage
@@ -132,7 +132,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
 		}
 		if !dirPath.NeedNewForAlbum() {
 			storPath := fileStor.JoinStoragePath(path.Join(dirPath.String(), file.Name()))
-			elem, err := batchtftask.NewTaskElement(fileStor, storPath, file)
+			elem, err := batchtfile.NewTaskElement(fileStor, storPath, file)
 			if err != nil {
 				logger.Errorf("Failed to create task element: %s", err)
 				ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
@@ -167,7 +167,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
 		albumStor := afiles[0].storage
 		for _, af := range afiles {
 			afstorPath := af.storage.JoinStoragePath(path.Join(dirPath, albumDir, af.file.Name()))
-			elem, err := batchtftask.NewTaskElement(albumStor, afstorPath, af.file)
+			elem, err := batchtfile.NewTaskElement(albumStor, afstorPath, af.file)
 			if err != nil {
 				logger.Errorf("Failed to create task element for album file: %s", err)
 				ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
@@ -182,7 +182,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
 
 	injectCtx := tgutil.ExtWithContext(ctx.Context, ctx)
 	taskid := xid.New().String()
-	task := batchtftask.NewBatchTGFileTask(taskid, injectCtx, elems, batchtftask.NewProgressTracker(trackMsgID, userID), true)
+	task := batchtfile.NewBatchTGFileTask(taskid, injectCtx, elems, batchtfile.NewProgressTracker(trackMsgID, userID), true)
 	if err := core.AddTask(injectCtx, task); err != nil {
 		logger.Errorf("Failed to add batch task: %s", err)
 		ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{

client/bot/handlers/utils/shortcut/tphtask.go

@@ -9,19 +9,21 @@ import (
 	"github.com/krau/SaveAny-Bot/common/utils/tgutil"
 	"github.com/krau/SaveAny-Bot/common/utils/tphutil"
 	"github.com/krau/SaveAny-Bot/core"
-	"github.com/krau/SaveAny-Bot/core/tphtask"
+	tphtask "github.com/krau/SaveAny-Bot/core/tasks/telegraph"
 	"github.com/krau/SaveAny-Bot/pkg/telegraph"
 	"github.com/krau/SaveAny-Bot/storage"
 	"github.com/rs/xid"
 )
 
-func CreateAndAddTphTaskWithEdit(ctx *ext.Context,
+func CreateAndAddtelegraphWithEdit(
+	ctx *ext.Context,
 	userID int64,
 	tphpage *telegraph.Page,
 	dirPath string, // unescaped ph path for file storage
 	pics []string,
 	stor storage.Storage,
 	trackMsgID int) error {
+		
 	injectCtx := tgutil.ExtWithContext(ctx.Context, ctx)
 	task := tphtask.NewTask(xid.New().String(),
 		injectCtx,