Compare commits

...

20 Commits

Author SHA1 Message Date
krau
b7d3ec6230 docs: update upgrade command 2025-08-24 14:26:47 +08:00
krau
f812990e1c feat: update help command to include usage and feedback group links 2025-08-24 14:18:43 +08:00
krau
492900bbef feat: add update command and callback for version checking and upgrading 2025-08-24 14:16:26 +08:00
krau
764be2a083 fix: improve error handling in config initialization 2025-08-24 14:16:23 +08:00
krau
46c21b77e9 feat: enhance Save method to validate chat ID and adjust ForceFile logic for image uploads 2025-08-24 12:02:27 +08:00
krau
8b389a58d5 fix: improve chat ID parsing in Save method and fallback to configured chat_id 2025-08-24 11:46:12 +08:00
krau
25ad9befa0 feat: add ForceFile option to TelegramStorageConfig and update Save method 2025-08-24 11:38:44 +08:00
krau
e824b210d1 feat: improve telegram storage chat ID parsing logic 2025-08-24 11:29:19 +08:00
krau
ae0aa7db3f fix: skip overwrite dirpath and storage when rule miss match 2025-08-24 11:28:37 +08:00
krau
226c15ef08 feat: add NormalizePathname function and update task handling for parsed items 2025-08-24 10:28:50 +08:00
krau
9b3f955e48 feat: use default HTTP client from netutil for task creation 2025-08-24 09:37:52 +08:00
krau
4997ec408f docs: update parsers 2025-08-23 20:42:51 +08:00
krau
0756cc9eb1 fix: improve parser configuration handling and default values 2025-08-23 20:40:13 +08:00
krau
37c32a23d4 feat: add Kemono parser with download info extraction and API handling 2025-08-23 20:18:02 +08:00
krau
3aa1e2eaed feat: enhance text message handling and parser configuration 2025-08-23 20:17:56 +08:00
krau
b87dd68880 feat: proxy client for parser 2025-08-23 20:17:24 +08:00
krau
68e5a51300 feat: file name staregy 2025-08-23 17:16:51 +08:00
krau
7300e54c40 refactor: rule package 2025-08-23 16:14:12 +08:00
krau
94f796d0e8 refactor: move version to config package 2025-08-23 16:10:02 +08:00
krau
c023fd869d feat: refactor jsParser to use ParserMethod constants and remove redundant locking in ParseWithContext 2025-08-23 16:04:32 +08:00
55 changed files with 1004 additions and 182 deletions

View File

@@ -63,9 +63,9 @@ jobs:
README.md
ldflags: >-
-s -w
-X "github.com/krau/SaveAny-Bot/pkg/consts.Version=${{ env.VERSION }}"
-X "github.com/krau/SaveAny-Bot/pkg/consts.BuildTime=${{ format(github.event.repository.updated_at, 'yyyy-MM-dd HH:mm:ss') }}"
-X "github.com/krau/SaveAny-Bot/pkg/consts.GitCommit=${{ github.sha }}"
-X "github.com/krau/SaveAny-Bot/config.Version=${{ env.VERSION }}"
-X "github.com/krau/SaveAny-Bot/config.BuildTime=${{ format(github.event.repository.updated_at, 'yyyy-MM-dd HH:mm:ss') }}"
-X "github.com/krau/SaveAny-Bot/config.GitCommit=${{ github.sha }}"
binary_name: saveany-bot
env:
VERSION: ${{ env.VERSION }}

View File

@@ -17,9 +17,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
go build -trimpath \
-ldflags=" \
-s -w \
-X 'github.com/krau/SaveAny-Bot/common.Version=${VERSION}' \
-X 'github.com/krau/SaveAny-Bot/common.GitCommit=${GitCommit}' \
-X 'github.com/krau/SaveAny-Bot/common.BuildTime=${BuildTime}' \
-X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \
-X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \
-X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \
" \
-o saveany-bot .

View File

@@ -19,12 +19,13 @@ import (
"golang.org/x/net/proxy"
)
func Init(ctx context.Context) {
func Init(ctx context.Context) (<-chan struct{}) {
log.FromContext(ctx).Info("初始化 Bot...")
resultChan := make(chan struct {
client *gotgproto.Client
err error
})
shouldRestart := make(chan struct{})
go func() {
var resolver dcs.Resolver
if config.C().Telegram.Proxy.Enable && config.C().Telegram.Proxy.URL != "" {
@@ -55,7 +56,11 @@ func Init(ctx context.Context) {
MaxRetries: config.C().Telegram.RpcRetry,
AutoFetchReply: true,
ErrorHandler: func(ctx *ext.Context, u *ext.Update, s string) error {
log.FromContext(ctx).Errorf("Unhandled error: %s", s)
if s == "SAVEANTBOT-RESTART" {
shouldRestart <- struct{}{}
return dispatcher.EndGroups
}
log.FromContext(ctx).Errorf("unhandled error: %s", s)
return dispatcher.EndGroups
},
},
@@ -103,4 +108,5 @@ func Init(ctx context.Context) {
handlers.Register(result.client.Dispatcher)
log.FromContext(ctx).Info("Bot 初始化完成")
}
return shouldRestart
}

View File

@@ -3,6 +3,7 @@ package handlers
import (
"errors"
"fmt"
"path"
"strings"
"github.com/celestix/gotgproto/dispatcher"
@@ -11,6 +12,7 @@ import (
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
@@ -74,6 +76,9 @@ func handleAddCallback(ctx *ext.Context, update *ext.Update) error {
case tasktype.TaskTypeTphpics:
return shortcut.CreateAndAddtelegraphWithEdit(ctx, userID, data.TphPageNode, data.TphDirPath, data.TphPics, selectedStorage, msgID)
case tasktype.TaskTypeParseditem:
if len(data.ParsedItem.Resources) > 1 {
dirPath = path.Join(dirPath, fsutil.NormalizePathname(data.ParsedItem.Title))
}
shortcut.CreateAndAddParsedTaskWithEdit(ctx, selectedStorage, dirPath, data.ParsedItem, msgID, userID)
default:
log.FromContext(ctx).Errorf("Unsupported task type: %s", data.TaskType)

View File

@@ -0,0 +1,103 @@
package handlers
import (
"fmt"
"strings"
"github.com/celestix/gotgproto/dispatcher"
"github.com/celestix/gotgproto/ext"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/enums/fnamest"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
)
func handleConfigCmd(ctx *ext.Context, update *ext.Update) error {
ctx.Reply(update, ext.ReplyTextString("请选择要配置的选项"), &ext.ReplyOpts{
Markup: &tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
&tg.KeyboardButtonCallback{
Text: "文件名策略",
Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeConfig, "fnamest"),
},
},
},
},
},
})
return dispatcher.EndGroups
}
func handleConfigCallback(ctx *ext.Context, update *ext.Update) error {
args := strings.Fields(string(update.CallbackQuery.Data))
invaildDataAnswer := func() error {
ctx.AnswerCallback(&tg.MessagesSetBotCallbackAnswerRequest{
QueryID: update.CallbackQuery.GetQueryID(),
Alert: true,
Message: "无效的回调数据",
CacheTime: 5,
})
return dispatcher.EndGroups
}
if len(args) < 2 {
return invaildDataAnswer()
}
switch args[1] {
case "fnamest":
return handleConfigFnameSTCallback(ctx, update)
default:
return invaildDataAnswer()
}
}
func handleConfigFnameSTCallback(ctx *ext.Context, update *ext.Update) error {
userID := update.CallbackQuery.GetUserID()
user, err := database.GetUserByChatID(ctx, userID)
if err != nil {
return err
}
args := strings.Fields(string(update.CallbackQuery.Data))
if len(args) == 3 {
selected := args[2]
st, err := fnamest.ParseFnameST(selected)
if err != nil {
return err
}
user.FilenameStrategy = st.String()
if err := database.UpdateUser(ctx, user); err != nil {
return err
}
ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
ID: update.CallbackQuery.GetMsgID(),
Message: fmt.Sprintf("已将文件名策略设置为: %s", fnamest.FnameSTDisplay[st]),
})
return dispatcher.EndGroups
}
opts := fnamest.FnameSTValues()
buttons := make([]tg.KeyboardButtonClass, 0, len(opts))
for _, opt := range opts {
buttons = append(buttons, &tg.KeyboardButtonCallback{
Text: fnamest.FnameSTDisplay[opt],
Data: fmt.Appendf(nil, "%s %s %s", tcbdata.TypeConfig, "fnamest", opt),
})
}
markup := &tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{
{Buttons: buttons},
}}
currentStStr := user.FilenameStrategy
if currentStStr == "" {
currentStStr = fnamest.Default.String()
}
currentSt, err := fnamest.ParseFnameST(currentStStr)
if err != nil {
currentSt = fnamest.Default
}
ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
ID: update.CallbackQuery.GetMsgID(),
Message: fmt.Sprintf("请选择文件名策略, 当前策略: %s", fnamest.FnameSTDisplay[currentSt]),
ReplyMarkup: markup,
})
return dispatcher.EndGroups
}

View File

@@ -5,7 +5,7 @@ import (
"github.com/celestix/gotgproto/dispatcher"
"github.com/celestix/gotgproto/ext"
"github.com/krau/SaveAny-Bot/pkg/consts"
"github.com/krau/SaveAny-Bot/config"
)
func handleHelpCmd(ctx *ext.Context, update *ext.Update) error {
@@ -21,13 +21,15 @@ Save Any Bot - 转存你的 Telegram 文件
/save [自定义文件名] - 保存文件
/dir - 管理存储目录
/rule - 管理规则
/update - 检查更新并升级
使用帮助: https://sabot.unv.app/usage/
使用帮助: https://sabot.unv.app/usage
反馈群组: https://t.me/ProjectSaveAny
`
shortHash := consts.GitCommit
shortHash := config.GitCommit
if len(shortHash) > 7 {
shortHash = shortHash[:7]
}
ctx.Reply(update, ext.ReplyTextString(fmt.Sprintf(helpText, consts.Version, shortHash)), nil)
ctx.Reply(update, ext.ReplyTextString(fmt.Sprintf(helpText, config.Version, shortHash)), nil)
return dispatcher.EndGroups
}

View File

@@ -13,6 +13,8 @@ import (
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/enums/fnamest"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
"github.com/krau/SaveAny-Bot/pkg/tfile"
"github.com/krau/SaveAny-Bot/storage"
@@ -26,12 +28,22 @@ func handleMediaMessage(ctx *ext.Context, update *ext.Update) error {
return handleGroupMediaMessage(ctx, update, message, groupID)
}
logger.Debugf("Got media: %s", message.Media.TypeName())
msg, file, err := shortcut.GetFileFromMessageWithReply(ctx, update, message)
userId := update.GetUserChat().GetID()
userDB, err := database.GetUserByChatID(ctx, userId)
if err != nil {
return err
}
userId := update.GetUserChat().GetID()
tfOpts := make([]tfile.TGFileOption, 0)
switch userDB.FilenameStrategy {
case fnamest.Message.String():
tfOpts = append(tfOpts, tfile.WithName(tgutil.GenFileNameFromMessage(*message)))
default:
}
msg, file, err := shortcut.GetFileFromMessageWithReply(ctx, update, message, tfOpts...)
if err != nil {
return err
}
stors := storage.GetUserStorages(ctx, userId)
req, err := msgelem.BuildAddOneSelectStorageMessage(ctx, stors, file, msg.ID)
if err != nil {
@@ -58,7 +70,17 @@ func handleSilentSaveMedia(ctx *ext.Context, update *ext.Update) error {
}
logger.Debugf("Got media: %s", message.Media.TypeName())
userID := update.GetUserChat().GetID()
msg, file, err := shortcut.GetFileFromMessageWithReply(ctx, update, message)
userDB, err := database.GetUserByChatID(ctx, userID)
if err != nil {
return err
}
tfOpts := make([]tfile.TGFileOption, 0)
switch userDB.FilenameStrategy {
case fnamest.Message.String():
tfOpts = append(tfOpts, tfile.WithName(tgutil.GenFileNameFromMessage(*message)))
default:
}
msg, file, err := shortcut.GetFileFromMessageWithReply(ctx, update, message, tfOpts...)
if err != nil {
return err
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
"github.com/krau/SaveAny-Bot/parsers"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
"github.com/krau/SaveAny-Bot/pkg/tcbdata"
@@ -20,7 +21,16 @@ import (
func handleTextMessage(ctx *ext.Context, u *ext.Update) error {
logger := log.FromContext(ctx)
text := u.EffectiveMessage.Text
item, err := parsers.ParseWithContext(ctx, text)
ok, pser := parsers.CanHandle(text)
if !ok {
return dispatcher.EndGroups
}
msg, err := ctx.Reply(u, ext.ReplyTextString("正在解析..."), nil)
if err != nil {
return err
}
item, err := pser.Parse(ctx, text)
if errors.Is(err, parsers.ErrNoParserFound) {
return dispatcher.EndGroups
}
@@ -29,7 +39,7 @@ func handleTextMessage(ctx *ext.Context, u *ext.Update) error {
ctx.Reply(u, ext.ReplyTextString("Failed to parse text: "+err.Error()), nil)
return dispatcher.EndGroups
}
logger.Debug("Parsed item from text message", "text", text, "item", item)
logger.Debug("Parsed item from text message", "title", item.Title, "url", item.URL)
userID := u.GetUserChat().GetID()
markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, userID), tcbdata.Add{
TaskType: tasktype.TaskTypeParseditem,
@@ -46,14 +56,11 @@ func handleTextMessage(ctx *ext.Context, u *ext.Update) error {
ctx.Reply(u, ext.ReplyTextString("Failed to build parsed text entity: "+err.Error()), nil)
return dispatcher.EndGroups
}
ctx.SendMessage(userID, &tg.MessagesSendMessageRequest{
ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{
Message: text,
ReplyMarkup: markup,
Entities: entities,
ReplyTo: &tg.InputReplyToMessage{
ReplyToMsgID: u.EffectiveMessage.ID,
ReplyToPeerID: u.GetUserChat().AsInputPeer(),
},
ID: msg.ID,
})
return dispatcher.EndGroups
@@ -80,7 +87,7 @@ func handleSilentSaveText(ctx *ext.Context, u *ext.Update) error {
ctx.Reply(u, ext.ReplyTextString("Failed to parse text: "+err.Error()), nil)
return dispatcher.EndGroups
}
logger.Debug("Parsed item from text message", "text", text, "item", item)
logger.Debug("Parsed item from text message", "title", item.Title, "url", item.URL)
userID := u.GetUserChat().GetID()
text, entities, err := msgelem.BuildParsedTextEntity(*item)
if err != nil {
@@ -100,5 +107,9 @@ func handleSilentSaveText(ctx *ext.Context, u *ext.Update) error {
logger.Errorf("Failed to send message: %s", err)
return dispatcher.EndGroups
}
return shortcut.CreateAndAddParsedTaskWithEdit(ctx, stor, "", item, msg.ID, userID)
dirPath := ""
if len(item.Resources) > 1 {
dirPath = fsutil.NormalizePathname(item.Title)
}
return shortcut.CreateAndAddParsedTaskWithEdit(ctx, stor, dirPath, item, msg.ID, userID)
}

View File

@@ -40,9 +40,13 @@ func Register(disp dispatcher.Dispatcher) {
disp.AddHandler(handlers.NewCommand("watch", handleWatchCmd))
disp.AddHandler(handlers.NewCommand("unwatch", handleUnwatchCmd))
disp.AddHandler(handlers.NewCommand("save", handleSilentMode(handleSaveCmd, handleSilentSaveReplied)))
disp.AddHandler(handlers.NewCommand("config", handleConfigCmd))
disp.AddHandler(handlers.NewCommand("update", handleUpdateCmd))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix("update"), handleUpdateCallback))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeAdd), handleAddCallback))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeSetDefault), handleSetDefaultCallback))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix("cancel"), handleCancelCallback))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeCancel), handleCancelCallback))
disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeConfig), handleConfigCallback))
linkRegexFilter, err := filters.Message.Regex(re.TgMessageLinkRegexString)
if err != nil {
panic("failed to create regex filter: " + err.Error())
@@ -110,7 +114,10 @@ func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent) {
}
var dirPath string
if user.ApplyRule && user.Rules != nil {
matchedStorageName, matchedDirPath := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
matched, matchedStorageName, matchedDirPath := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
if !matched {
goto startCreateTask
}
dirPath = matchedDirPath.String()
if matchedStorageName.IsUsable() {
stor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, matchedStorageName.String())
@@ -120,6 +127,7 @@ func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent) {
}
}
}
startCreateTask:
storagePath := stor.JoinStoragePath(path.Join(dirPath, file.Name()))
injectCtx := tgutil.ExtWithContext(ctx.Context, ctx)
taskid := xid.New().String()

View File

@@ -11,7 +11,7 @@ import (
"github.com/duke-git/lancet/v2/slice"
"github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/enums/rule"
"github.com/krau/SaveAny-Bot/pkg/rule"
)
func handleRuleCmd(ctx *ext.Context, update *ext.Update) error {

View File

@@ -0,0 +1,105 @@
package handlers
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/blang/semver"
"github.com/celestix/gotgproto/dispatcher"
"github.com/celestix/gotgproto/ext"
"github.com/gotd/td/telegram/message/html"
"github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/config"
"github.com/rhysd/go-github-selfupdate/selfupdate"
)
func handleUpdateCmd(ctx *ext.Context, u *ext.Update) error {
currentV, err := semver.Parse(config.Version)
if err != nil {
ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("You are in dev or the version var failed to inject: %v", err)), nil)
return dispatcher.EndGroups
}
latest, ok, err := selfupdate.DetectLatest(config.GitRepo)
if err != nil {
ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("检测最新版本失败: %v", err)), nil)
return dispatcher.EndGroups
}
if !ok {
ctx.Reply(u, ext.ReplyTextString("没有找到版本信息"), nil)
return dispatcher.EndGroups
}
if latest.Version.Equals(currentV) {
return dispatcher.EndGroups
}
if latest.Version.LT(currentV) || latest.Version.Equals(currentV) {
ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("当前已经是最新版本: %s", config.Version)), nil)
return dispatcher.EndGroups
}
ctx.Sender.To(u.GetUserChat().AsInputPeer()).StyledText(ctx, html.String(nil, func() string {
md := latest.ReleaseNotes
md = regexp.MustCompile(`(?m)^###\s+&nbsp;&nbsp;&nbsp;(.+)$`).ReplaceAllString(md, "<b>$1</b>")
md = regexp.MustCompile(`(?m)^#####\s+&nbsp;&nbsp;&nbsp;&nbsp;(.+)$`).ReplaceAllString(md, "<i>$1</i>")
md = regexp.MustCompile(`(?m)^- `).ReplaceAllString(md, "• ")
md = regexp.MustCompile(`\[\((\w{6,})\)\]\((https?://[^\s)]+)\)`).ReplaceAllString(md, `(<a href="$2">$1</a>)`)
md = regexp.MustCompile(`\[(.+?)\]\((https?://[^\s)]+)\)`).ReplaceAllString(md, `<a href="$2">$1</a>`)
md = strings.ReplaceAll(md, "&nbsp;", " ")
return `<blockquote expandable>` + md + `</blockquote>`
}()))
text := fmt.Sprintf(`发现新版本: %s
当前版本: %s
文件大小: %.2f MB
下载链接: %s
发布时间: %s
升级将重启 Bot , 是否升级?`, latest.Version, config.Version,
float64(latest.AssetByteSize)/(1024*1024), latest.AssetURL,
latest.PublishedAt.Format("2006-01-02 15:04:05"),
)
ctx.Reply(u, ext.ReplyTextString(text), &ext.ReplyOpts{
Markup: &tg.ReplyInlineMarkup{
Rows: []tg.KeyboardButtonRow{
{
Buttons: []tg.KeyboardButtonClass{
&tg.KeyboardButtonCallback{
Text: "升级",
Data: []byte("update"),
},
},
},
},
},
})
return dispatcher.EndGroups
}
func handleUpdateCallback(ctx *ext.Context, u *ext.Update) error {
currentV, err := semver.Parse(config.Version)
if err != nil {
return err
}
ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{
ID: u.CallbackQuery.GetMsgID(),
Message: fmt.Sprintf("正在升级中, 当前版本: %s", config.Version),
})
latest, err := selfupdate.UpdateSelf(currentV, config.GitRepo)
if err != nil {
ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{
ID: u.CallbackQuery.GetMsgID(),
Message: fmt.Sprintf("升级失败: %v", err),
})
return dispatcher.EndGroups
}
ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{
ID: u.CallbackQuery.GetMsgID(),
Message: fmt.Sprintf("已升级至版本 %s\n若 Bot 未自动重启请手动启动", latest.Version),
})
return errors.New("SAVEANTBOT-RESTART")
}

View File

@@ -7,8 +7,6 @@ import (
"github.com/charmbracelet/log"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/consts"
ruleenum "github.com/krau/SaveAny-Bot/pkg/enums/rule"
"github.com/krau/SaveAny-Bot/pkg/rule"
"github.com/krau/SaveAny-Bot/pkg/tfile"
)
@@ -37,7 +35,7 @@ func (m matchedStorName) String() string {
// can we use this storage name directly?
func (m matchedStorName) IsUsable() bool {
return m != "" && m != consts.RuleStorNameChosen
return m != "" && m != rule.RuleStorNameChosen
}
type MatchedDirPath string
@@ -47,17 +45,17 @@ func (m MatchedDirPath) String() string {
}
func (m MatchedDirPath) NeedNewForAlbum() bool {
return m != "" && m == consts.RuleDirPathNewForAlbum
return m != "" && m == rule.RuleDirPathNewForAlbum
}
func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (matchedStorageName matchedStorName, dirPath MatchedDirPath) {
func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (matched bool, matchedStorageName matchedStorName, dirPath MatchedDirPath) {
if inputs == nil || len(rules) == 0 {
return "", ""
return false, "", ""
}
logger := log.FromContext(ctx)
for _, ur := range rules {
switch ur.Type {
case ruleenum.FileNameRegex.String():
case rule.FileNameRegex.String():
ru, err := rule.NewRuleFileNameRegex(ur.StorageName, ur.DirPath, ur.Data)
if err != nil {
logger.Errorf("Failed to create rule: %s", err)
@@ -72,7 +70,7 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m
dirPath = MatchedDirPath(ru.StoragePath())
matchedStorageName = matchedStorName(ru.StorageName())
}
case ruleenum.MessageRegex.String():
case rule.MessageRegex.String():
ru, err := rule.NewRuleMessageRegex(ur.StorageName, ur.DirPath, ur.Data)
if err != nil {
logger.Errorf("Failed to create rule: %s", err)
@@ -87,7 +85,7 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m
dirPath = MatchedDirPath(ru.StoragePath())
matchedStorageName = matchedStorName(ru.StorageName())
}
case ruleenum.IsAlbum.String():
case rule.IsAlbum.String():
matchAlbum, err := convertor.ToBool(ur.Data)
if err != nil {
matchAlbum = false
@@ -108,5 +106,8 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m
}
}
}
return
if matchedStorageName != "" || dirPath != "" {
return true, matchedStorageName, dirPath
}
return false, "", ""
}

View File

@@ -20,12 +20,14 @@ import (
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
"github.com/krau/SaveAny-Bot/common/utils/tphutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/database"
"github.com/krau/SaveAny-Bot/pkg/enums/fnamest"
"github.com/krau/SaveAny-Bot/pkg/telegraph"
"github.com/krau/SaveAny-Bot/pkg/tfile"
)
// 获取消息中的文件并回复等待消息, 返回等待消息, 获取到的文件
func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message *tg.Message, tfileopts ...tfile.TGFileOptions) (replied *types.Message,
func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message *tg.Message, tfileopts ...tfile.TGFileOption) (replied *types.Message,
file tfile.TGFileMessage, err error,
) {
logger := log.FromContext(ctx)
@@ -40,7 +42,7 @@ func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message *
logger.Errorf("Failed to reply: %s", err)
return nil, nil, dispatcher.EndGroups
}
options := []tfile.TGFileOptions{
options := []tfile.TGFileOption{
tfile.WithMessage(message),
}
if len(tfileopts) > 0 {
@@ -81,7 +83,12 @@ func GetFilesFromUpdateLinkMessageWithReplyEdit(ctx *ext.Context, update *ext.Up
logger.Errorf("failed to edit message: %s", err)
}
}
user, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID())
if err != nil {
logger.Errorf("failed to get user from db: %s", err)
editReplied("获取用户信息失败: "+err.Error(), nil)
return nil, nil, nil, dispatcher.EndGroups
}
files = make([]tfile.TGFileMessage, 0, len(msgLinks))
addFile := func(client downloader.Client, msg *tg.Message) {
if msg == nil || msg.Media == nil {
@@ -93,7 +100,14 @@ func GetFilesFromUpdateLinkMessageWithReplyEdit(ctx *ext.Context, update *ext.Up
logger.Debugf("message %d has no media", msg.GetID())
return
}
file, err := tfile.FromMediaMessage(media, client, msg, tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*msg)))
var opt tfile.TGFileOption
switch user.FilenameStrategy {
case fnamest.Message.String():
opt = tfile.WithName(tgutil.GenFileNameFromMessage(*msg))
default:
opt = tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*msg))
}
file, err := tfile.FromMediaMessage(media, client, msg, opt)
if err != nil {
logger.Errorf("failed to create file from media: %s", err)
return

View File

@@ -34,8 +34,13 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage
return dispatcher.EndGroups
}
if user.ApplyRule && user.Rules != nil {
matchedStorageName, matchedDirPath := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
dirPath = matchedDirPath.String()
matched, matchedStorageName, matchedDirPath := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
if !matched {
goto startCreateTask
}
if matchedDirPath != "" {
dirPath = matchedDirPath.String()
}
if matchedStorageName.IsUsable() {
stor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, matchedStorageName.String())
if err != nil {
@@ -48,7 +53,7 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage
}
}
}
startCreateTask:
storagePath := stor.JoinStoragePath(path.Join(dirPath, file.Name()))
injectCtx := tgutil.ExtWithContext(ctx.Context, ctx)
taskid := xid.New().String()
@@ -101,8 +106,10 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st
if !useRule {
return stor.Name(), ruleutil.MatchedDirPath(dirPath)
}
storName, dirP := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
matched, storName, dirP := ruleutil.ApplyRule(ctx, user.Rules, ruleutil.NewInput(file))
if !matched {
return stor.Name(), ruleutil.MatchedDirPath(dirPath)
}
storname := storName.String()
if !storName.IsUsable() {
storname = stor.Name()

View File

@@ -14,7 +14,7 @@ import (
func handleWatchCmd(ctx *ext.Context, update *ext.Update) error {
logger := log.FromContext(ctx)
args := strings.Split(string(update.EffectiveMessage.Text), " ")
args := strings.Split(update.EffectiveMessage.Text, " ")
if len(args) < 2 {
ctx.Reply(update, ext.ReplyTextString(msgelem.WatchHelpText), nil)
return dispatcher.EndGroups
@@ -82,7 +82,7 @@ func handleWatchCmd(ctx *ext.Context, update *ext.Update) error {
func handleUnwatchCmd(ctx *ext.Context, update *ext.Update) error {
logger := log.FromContext(ctx)
args := strings.Split(string(update.EffectiveMessage.Text), " ")
args := strings.Split(update.EffectiveMessage.Text, " ")
if len(args) < 2 {
ctx.Reply(update, ext.ReplyTextString("请提供要取消监听的聊天ID或用户名"), nil)
return dispatcher.EndGroups

View File

@@ -25,7 +25,7 @@ import (
)
func Run(cmd *cobra.Command, _ []string) {
ctx := cmd.Context()
ctx, cancel := context.WithCancel(cmd.Context())
logger := log.NewWithOptions(os.Stdout, log.Options{
Level: log.DebugLevel,
ReportTimestamp: true,
@@ -34,7 +34,15 @@ func Run(cmd *cobra.Command, _ []string) {
})
ctx = log.WithContext(ctx, logger)
initAll(ctx)
exitChan, err := initAll(ctx)
if err != nil {
logger.Fatal("Failed to initialize", "error", err)
}
go func() {
<-exitChan
cancel()
}()
core.Run(ctx)
<-ctx.Done()
@@ -43,10 +51,10 @@ func Run(cmd *cobra.Command, _ []string) {
cleanCache()
}
func initAll(ctx context.Context) {
func initAll(ctx context.Context) (<-chan struct{}, error) {
if err := config.Init(ctx); err != nil {
fmt.Println("Failed to load config:", err)
os.Exit(1)
return nil, err
}
cache.Init()
logger := log.FromContext(ctx)
@@ -69,7 +77,7 @@ func initAll(ctx context.Context) {
logger.Fatalf("User client login failed: %s", err)
}
}
bot.Init(ctx)
return bot.Init(ctx), nil
}
func cleanCache() {

View File

@@ -4,7 +4,7 @@ import (
"fmt"
"runtime"
"github.com/krau/SaveAny-Bot/pkg/consts"
"github.com/krau/SaveAny-Bot/config"
"github.com/rhysd/go-github-selfupdate/selfupdate"
"github.com/blang/semver"
@@ -16,7 +16,7 @@ var VersionCmd = &cobra.Command{
Aliases: []string{"v"},
Short: "Print the version number of saveany-bot",
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("saveany-bot version: %s %s/%s\nBuildTime: %s, Commit: %s\n", consts.Version, runtime.GOOS, runtime.GOARCH, consts.BuildTime, consts.GitCommit)
fmt.Printf("saveany-bot version: %s %s/%s\nBuildTime: %s, Commit: %s\n", config.Version, runtime.GOOS, runtime.GOARCH, config.BuildTime, config.GitCommit)
},
}
@@ -25,14 +25,14 @@ var upgradeCmd = &cobra.Command{
Aliases: []string{"up"},
Short: "Upgrade saveany-bot to the latest version",
Run: func(cmd *cobra.Command, args []string) {
v := semver.MustParse(consts.Version)
latest, err := selfupdate.UpdateSelf(v, "krau/SaveAny-Bot")
v := semver.MustParse(config.Version)
latest, err := selfupdate.UpdateSelf(v, config.GitRepo)
if err != nil {
fmt.Println("Binary update failed:", err)
return
}
if latest.Version.Equals(v) {
fmt.Println("Current binary is the latest version", consts.Version)
fmt.Println("Current binary is the latest version", config.Version)
} else {
fmt.Println("Successfully updated to version", latest.Version)
fmt.Println("Release note:\n", latest.ReleaseNotes)

View File

@@ -3,6 +3,8 @@ package fsutil
import (
"os"
"path/filepath"
"strings"
"unicode"
"github.com/gabriel-vasile/mimetype"
)
@@ -55,3 +57,21 @@ func CreateFile(fp string) (*File, error) {
}
return &File{File: file}, nil
}
func NormalizePathname(s string) string {
specials := `\/:*?"<>|` + "\n\r\t"
var builder strings.Builder
for _, ch := range s {
if strings.ContainsRune(specials, ch) || unicode.IsControl(ch) {
builder.WriteRune('_')
} else {
builder.WriteRune(ch)
}
}
result := strings.TrimRightFunc(builder.String(), func(r rune) bool {
return r == '.' || r == '_' || unicode.IsSpace(r)
})
return result
}

View File

@@ -0,0 +1,46 @@
package fsutil_test
import (
"testing"
"github.com/krau/SaveAny-Bot/common/utils/fsutil"
)
func TestNormalizePathname(t *testing.T) {
tests := []struct {
input string
expected string
}{
{
input: "hello/world?.txt ",
expected: "hello_world_.txt",
},
{
input: "bad|name:\nfile\r.",
expected: "bad_name__file",
},
{
input: "normal.txt",
expected: "normal.txt",
},
{
input: "test.... ",
expected: "test",
},
{
input: "abc<>def",
expected: "abc__def",
},
{
input: "with\tcontrol",
expected: "with_control",
},
}
for _, tc := range tests {
got := fsutil.NormalizePathname(tc.input)
if got != tc.expected {
t.Errorf("NormalizePathname(%q) = %q; want %q", tc.input, got, tc.expected)
}
}
}

View File

@@ -6,7 +6,10 @@ import (
"net"
"net/http"
"net/url"
"sync"
"github.com/charmbracelet/log"
"github.com/krau/SaveAny-Bot/config"
"golang.org/x/net/proxy"
)
@@ -20,7 +23,11 @@ func NewProxyDialer(proxyUrl string) (proxy.Dialer, error) {
func NewProxyHTTPClient(proxyUrl string) (*http.Client, error) {
if proxyUrl == "" {
return http.DefaultClient, nil
return &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
},
}, nil
}
u, err := url.Parse(proxyUrl)
@@ -52,3 +59,21 @@ func NewProxyHTTPClient(proxyUrl string) (*http.Client, error) {
return nil, fmt.Errorf("unsupported proxy scheme: %s", u.Scheme)
}
}
var (
defaultProxyHttpClient *http.Client
onceLoadDefaultProxyHttpClient sync.Once
)
func DefaultParserHTTPClient() *http.Client {
onceLoadDefaultProxyHttpClient.Do(func() {
client, err := NewProxyHTTPClient(config.C().Parser.Proxy)
if err != nil {
log.Warn("Failed to create default proxy HTTP client, using http.DefaultClient", "error", err)
defaultProxyHttpClient = http.DefaultClient
} else {
defaultProxyHttpClient = client
}
})
return defaultProxyHttpClient
}

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"unicode"
"github.com/celestix/gotgproto/ext"
"github.com/duke-git/lancet/v2/maputil"
@@ -61,16 +62,12 @@ func GenFileNameFromMessage(message tg.Message) string {
return fmt.Sprintf("%s_%s", tagStr, strconv.Itoa(message.GetID()))
}
text = lcstrutil.Substring(strings.Map(func(r rune) rune {
if r < 0x20 || r == 0x7F {
return '_'
}
switch r {
// invalid characters
case '/', '\\',
':', '*', '?', '"', '<', '>', '|':
return '_'
// empty
case ' ', '\t', '\r', '\n':
}
if unicode.IsControl(r) || unicode.IsSpace(r) {
return '_'
}
if validator.IsPrintable(string(r)) {

View File

@@ -86,7 +86,7 @@ func ParseMessageLink(ctx *ext.Context, link string) (int64, int, error) {
return chatID, msgID, nil
case 3:
// https://t.me/c/123456789/123
// https://t.me/acherkrau/123/456 , 456: message thread ID
// https://t.me/acherkrau/123/456 , 123: topic id
chatPart, msgPart := paths[1], paths[2]
if paths[0] != "c" {
chatPart = paths[0]

View File

@@ -1,10 +1,10 @@
package config
type parserConfig struct {
PluginEnable bool `toml:"plugin_enable" mapstructure:"plugin_enable" json:"plugin_enable"`
PluginDirs []string `toml:"plugin_dirs" mapstructure:"plugin_dirs" json:"plugin_dirs"`
ParserCfgs map[string]map[string]any `mapstructure:",remain"`
PluginEnable bool `toml:"plugin_enable" mapstructure:"plugin_enable" json:"plugin_enable"`
PluginDirs []string `toml:"plugin_dirs" mapstructure:"plugin_dirs" json:"plugin_dirs"`
Proxy string `toml:"proxy" mapstructure:"proxy" json:"proxy"`
ParserCfgs map[string]map[string]any `mapstructure:",remain"`
}
func (c Config) GetParserConfigByName(name string) map[string]any {

View File

@@ -9,6 +9,7 @@ import (
type TelegramStorageConfig struct {
BaseConfig
ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"`
ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"`
RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"`
RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"`
}

13
config/version.go Normal file
View File

@@ -0,0 +1,13 @@
package config
// inject version by '-X' flag
// go build -ldflags "-X github.com/krau/SaveAny-Bot/config.Version=${{ env.VERSION }}"
var (
Version string = "dev"
BuildTime string = "unknown"
GitCommit string = "unknown"
)
const (
GitRepo = "krau/SaveAny-Bot"
)

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"os"
"strings"
"github.com/duke-git/lancet/v2/slice"
@@ -96,12 +95,12 @@ func Init(ctx context.Context) error {
if err := viper.ReadInConfig(); err != nil {
fmt.Println("Error reading config file, ", err)
os.Exit(1)
return err
}
if err := viper.Unmarshal(cfg); err != nil {
fmt.Println("Error unmarshalling config file, ", err)
os.Exit(1)
return err
}
storagesConfig, err := storage.LoadStorageConfigs(viper.GetViper())

View File

@@ -6,6 +6,7 @@ import (
"sync"
"sync/atomic"
"github.com/krau/SaveAny-Bot/common/utils/netutil"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/pkg/enums/tasktype"
"github.com/krau/SaveAny-Bot/pkg/parser"
@@ -47,12 +48,7 @@ func NewTask(
item *parser.Item,
progressTracker ProgressTracker,
) *Task {
client := &http.Client{
Transport: &http.Transport{
// [TODO] configure it via config
Proxy: http.ProxyFromEnvironment,
},
}
client := netutil.DefaultParserHTTPClient()
_, ok := stor.(storage.StorageCannotStream)
stream := config.C().Stream && !ok
return &Task{

View File

@@ -6,13 +6,14 @@ import (
type User struct {
gorm.Model
ChatID int64 `gorm:"uniqueIndex;not null"`
Silent bool
DefaultStorage string
Dirs []Dir
ApplyRule bool
Rules []Rule
WatchChats []WatchChat
ChatID int64 `gorm:"uniqueIndex;not null"`
Silent bool
DefaultStorage string
Dirs []Dir
ApplyRule bool
Rules []Rule
WatchChats []WatchChat
FilenameStrategy string
}
type WatchChat struct {

View File

@@ -29,6 +29,6 @@ weight: 20
1.`parsers` 目录下新建一个包, 编写解析器实现
2.`parsers/parser.go``init` 中注册解析器
如果使用 JavaScript 编写, 请参考 `plugins/example_parser.js` 的实现, 并在该文件夹下新建一个 js 文件, 实现你的解析逻辑.
如果使用 JavaScript 编写, 请参考 `plugins/example_parser_basic.js` 的实现, 并在该文件夹下新建一个 js 文件, 实现你的解析逻辑.
需要注意, `plugins` 目录下解析器默认不会被编译到二进制文件中, 用户需要手动下载它们并放到本地指定目录下以启用它们.

View File

@@ -61,7 +61,9 @@ Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一
{{< hint warning >}}
启用 userbot 集成后, bot 可以下载私密频道和群组的文件, 但具有无法避免的账号被封禁的风险.
<br />
开启 userbot 集成后第一次启动 bot 时需要通过终端交互输入手机号, 2FA 和验证码, 如果你使用 docker 部署, 请进入容器内执行相关操作.
开启 userbot 集成后第一次启动 bot 时需要通过终端交互输入手机号, 2FA 和验证码.
<br />
如果你使用 docker 部署, 请进入容器内执行相关操作.
{{< /hint >}}
```toml

View File

@@ -4,7 +4,7 @@ title: "安装与更新"
# 安装与更新
## 从预编译文件部署
## 从预编译文件部署(推荐)
在 [Release](https://github.com/krau/SaveAny-Bot/releases) 页面下载对应平台的二进制文件.
@@ -131,13 +131,13 @@ docker run -d --name saveany-bot \
## 更新
使用 `upgrade``up` 升级到最新版
向 Bot 发送 `/update` 指令检查更新并升级, 或者使用 CLI 命令更新:
```bash
./saveany-bot upgrade
./saveany-bot up
```
如果是 Docker 部署, 使用以下命令更新:
如果是 Docker 部署, 还可以使用以下命令更新:
```bash
docker pull ghcr.io/krau/saveany-bot:latest

View File

@@ -120,4 +120,5 @@ IS-ALBUM true MyWebdav NEW-FOR-ALBUM
只需向 Bot 发送符合解析器要求的链接即可使用, 当前内置的解析器:
- Twitter
- Twitter
- Kemono

View File

@@ -15,18 +15,6 @@ import (
"github.com/krau/SaveAny-Bot/pkg/parser"
)
var (
LatestParserVersion = semver.MustParse("1.0.0")
MinimumParserVersion = semver.MustParse("1.0.0")
)
type PluginMeta struct {
Name string `json:"name"`
Version string `json:"version"` // [TODO] 分版本解析, 但是我们现在只有 v1 所以先不写
Description string `json:"description"`
Author string `json:"author"`
}
type jsParser struct {
meta PluginMeta
vm *goja.Runtime
@@ -34,7 +22,7 @@ type jsParser struct {
}
type jsParserReq struct {
method string
method ParserMethod
url string
respCh chan jsParserResp
}
@@ -47,14 +35,14 @@ type jsParserResp struct {
func (p *jsParser) CanHandle(url string) bool {
respCh := make(chan jsParserResp, 1)
p.reqCh <- jsParserReq{method: "canHandle", url: url, respCh: respCh}
p.reqCh <- jsParserReq{method: ParserMethodCanHandle, url: url, respCh: respCh}
resp := <-respCh
return resp.ok && resp.err == nil
}
func (p *jsParser) Parse(ctx context.Context, url string) (*parser.Item, error) {
respCh := make(chan jsParserResp, 1)
p.reqCh <- jsParserReq{method: "parse", url: url, respCh: respCh}
p.reqCh <- jsParserReq{method: ParserMethodParse, url: url, respCh: respCh}
select {
case resp := <-respCh:
return resp.item, resp.err
@@ -73,7 +61,7 @@ func newJSParser(vm *goja.Runtime, canHandleFunc, parseFunc goja.Value, metadata
go func() {
for req := range p.reqCh {
switch req.method {
case "canHandle":
case ParserMethodCanHandle:
fn, _ := goja.AssertFunction(canHandleFunc)
res, err := fn(goja.Undefined(), p.vm.ToValue(req.url))
if err != nil {
@@ -81,7 +69,7 @@ func newJSParser(vm *goja.Runtime, canHandleFunc, parseFunc goja.Value, metadata
continue
}
req.respCh <- jsParserResp{ok: res.ToBoolean()}
case "parse":
case ParserMethodParse:
fn, _ := goja.AssertFunction(parseFunc)
result, err := fn(goja.Undefined(), p.vm.ToValue(req.url))
if err != nil {

View File

@@ -0,0 +1,36 @@
package kemono
import (
"net/url"
"strings"
)
type DownloadInfo struct {
ServiceName string
UserID string
PostID string
}
func extractDownloadInfoFromURL(u string) *DownloadInfo {
if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
u = "https://" + u
}
url, err := url.Parse(u)
if err != nil {
return nil
}
parts := strings.Split(strings.Trim(url.Path, "/"), "/")
if len(parts) == 3 {
return &DownloadInfo{
ServiceName: parts[0],
UserID: parts[2],
}
} else if len(parts) == 5 && parts[3] == "post" {
return &DownloadInfo{
ServiceName: parts[0],
UserID: parts[2],
PostID: parts[4],
}
}
return nil
}

173
parsers/kemono/kemono.go Normal file
View File

@@ -0,0 +1,173 @@
package kemono
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"github.com/duke-git/lancet/v2/strutil"
"github.com/krau/SaveAny-Bot/common/utils/netutil"
"github.com/krau/SaveAny-Bot/pkg/parser"
)
type KemonoParser struct{}
var (
kemonoDomains = []string{
"kemono.su",
"kemono.cr",
}
ErrFailedToExtractInfo = errors.New("failed to extract download info from URL")
)
const (
kemonoApiBase = "https://kemono.cr/api/v1"
)
func (k *KemonoParser) CanHandle(text string) bool {
text = strings.TrimPrefix(text, "https://")
text = strings.TrimPrefix(text, "http://")
var matchesDomain bool
for _, domain := range kemonoDomains {
if strings.Contains(text, domain) {
matchesDomain = true
break
}
}
if !matchesDomain {
return false
}
var path string
for _, domain := range kemonoDomains {
if idx := strings.Index(text, domain); idx != -1 {
remaining := text[idx+len(domain):]
if len(remaining) > 0 && remaining[0] == '/' {
path = remaining[1:]
}
break
}
}
if path == "" {
return false
}
parts := strings.Split(path, "/")
// servicename/user/id (user profile page)
// servicename/user/id/post/id (post page)
return len(parts) == 3 || (len(parts) == 5 && parts[3] == "post")
}
func (k *KemonoParser) Parse(ctx context.Context, u string) (*parser.Item, error) {
info := extractDownloadInfoFromURL(u)
if info == nil {
return nil, ErrFailedToExtractInfo
}
if info.PostID != "" {
return k.parseOne(ctx, info)
}
return k.parseUserPage(ctx, info)
}
func (k *KemonoParser) parseOne(ctx context.Context, info *DownloadInfo) (*parser.Item, error) {
client := netutil.DefaultParserHTTPClient()
endpoint := fmt.Sprintf("%s/%s/user/%s/post/%s", kemonoApiBase, info.ServiceName, info.UserID, info.PostID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request to Kemono API: %w", err)
}
req.Header.Set("Accept", "text/css")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch Kemono API: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to fetch Kemono API, status code: %d", resp.StatusCode)
}
var postInfo PostInfo
if err := json.NewDecoder(resp.Body).Decode(&postInfo); err != nil {
return nil, fmt.Errorf("failed to decode Kemono API response: %w", err)
}
item := &parser.Item{
Site: "kemono",
Title: postInfo.Post.Title,
URL: fmt.Sprintf("https://kemono.cr/%s/user/%s/post/%s", info.ServiceName, info.UserID, info.PostID),
Author: postInfo.Post.User, // [TODO] request user profile
Description: postInfo.Post.Content,
Tags: func() []string {
if postInfo.Post.Tags != nil {
return *postInfo.Post.Tags
}
return nil
}(),
}
resources := make([]parser.Resource, 0)
for _, attachment := range postInfo.Attachments {
if attachment.Server == nil || attachment.Path == nil || attachment.Name == nil {
continue
}
var size int64
fileUrl := fmt.Sprintf("%s/data%s", *attachment.Server, *attachment.Path)
headReq, err := http.NewRequestWithContext(ctx, http.MethodHead, fileUrl, nil)
if err == nil {
resp, err := client.Do(headReq)
if err == nil {
size = resp.ContentLength
resp.Body.Close()
}
}
resources = append(resources, parser.Resource{
URL: fmt.Sprintf("%s/data%s", *attachment.Server, *attachment.Path),
Filename: *attachment.Name,
Size: size,
})
}
picCdnMap := make(map[string]string)
for _, preview := range postInfo.Previews {
if preview.Type == nil || *preview.Type != "thumbnail" {
continue
}
picCdnMap[*preview.Path] = *preview.Server
}
for _, attachment := range postInfo.Post.Attachments {
if !isImageExt(*attachment.Path) {
continue
}
picUrl, err := url.JoinPath(picCdnMap[*attachment.Path], "data", *attachment.Path)
if err != nil {
continue
}
var size int64
headReq, err := http.NewRequestWithContext(ctx, http.MethodHead, picUrl, nil)
if err == nil {
resp, err := client.Do(headReq)
if err == nil {
size = resp.ContentLength
resp.Body.Close()
}
}
resources = append(resources, parser.Resource{
URL: picUrl,
Filename: *attachment.Name,
Size: size,
})
}
item.Resources = resources
return item, nil
}
func (k *KemonoParser) parseUserPage(_ context.Context, _ *DownloadInfo) (*parser.Item, error) {
return nil, errors.New("kemono user page not implemented")
}
func isImageExt(attachmentPath string) bool {
return strutil.HasSuffixAny(path.Ext(strings.Split(attachmentPath, "?")[0]), []string{".jpg", ".jpeg", ".png", ".webp"})
}

View File

@@ -0,0 +1,62 @@
// https://github.com/kemono-rs/kemono
package kemono
type PostInfo struct {
Post Post `json:"post"`
Attachments []AttachmentLike `json:"attachments"`
Previews []AttachmentLike `json:"previews"`
}
type AttachmentLike struct {
Type *string `json:"type,omitempty"`
Server *string `json:"server,omitempty"`
Name *string `json:"name,omitempty"`
Path *string `json:"path,omitempty"`
}
type Post struct {
ID string `json:"id"`
User string `json:"user"`
Service string `json:"service"`
Title string `json:"title"`
Content string `json:"content"`
Embed Embed `json:"embed"`
SharedFile bool `json:"shared_file"`
Added *string `json:"added,omitempty"`
Published string `json:"published"`
Edited *string `json:"edited,omitempty"`
File File `json:"file"`
Attachments []AttachmentLike `json:"attachments"`
Poll *Poll `json:"poll,omitempty"`
Captions *string `json:"captions,omitempty"`
Tags *[]string `json:"tags,omitempty"`
Next *string `json:"next,omitempty"`
Prev *string `json:"prev,omitempty"`
}
type File struct {
Name *string `json:"name,omitempty"`
Path *string `json:"path,omitempty"`
}
type Embed struct {
URL *string `json:"url,omitempty"`
Subject *string `json:"subject,omitempty"`
Description *string `json:"description,omitempty"`
}
type Poll struct {
Title string `json:"title"`
Choices []Choice `json:"choices"`
ClosesAt *string `json:"closes_at,omitempty"`
CreatedAt string `json:"created_at"`
Description *string `json:"description,omitempty"`
AllowsMultiple bool `json:"allows_multiple"`
TotalVotes int64 `json:"total_votes"`
}
type Choice struct {
Text string `json:"text"`
Votes int64 `json:"votes"`
}

View File

@@ -0,0 +1,16 @@
package kemono
type PostLegacy struct {
Props Props `json:"props"`
Results []Result `json:"results"`
}
type Props struct {
Count uint `json:"count"`
Limit uint `json:"limit"`
}
type Result struct {
ID string `json:"id"`
Title string `json:"title"`
}

View File

@@ -0,0 +1,8 @@
package kemono
type UserProfile struct {
ID string `json:"id"`
Name string `json:"name"`
Service string `json:"service"`
PublicID *string `json:"public_id,omitempty"`
}

View File

@@ -6,14 +6,28 @@ import (
"sync"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/parsers/kemono"
"github.com/krau/SaveAny-Bot/parsers/twitter"
"github.com/krau/SaveAny-Bot/pkg/parser"
)
var (
parsers []parser.Parser
parsersMu sync.Mutex
doConfig sync.Once
parsers []parser.Parser
parsersMu sync.Mutex
doConfig sync.Once
configParsers = func() {
if len(parsers) == 0 {
return
}
for _, pser := range parsers {
if configurable, ok := pser.(parser.ConfigurableParser); ok {
cfg := config.C().GetParserConfigByName(configurable.Name())
if err := configurable.Configure(cfg); err != nil {
fmt.Printf("Error configuring parser %s: %v\n", configurable.Name(), err)
}
}
}
}
)
func AddParser(p ...parser.Parser) {
@@ -23,7 +37,7 @@ func AddParser(p ...parser.Parser) {
}
func init() {
AddParser(new(twitter.TwitterParser))
AddParser(new(twitter.TwitterParser), new(kemono.KemonoParser))
}
var (
@@ -31,23 +45,7 @@ var (
)
func ParseWithContext(ctx context.Context, url string) (*parser.Item, error) {
doConfig.Do(func() {
parsersMu.Lock()
defer parsersMu.Unlock()
if len(parsers) == 0 {
return
}
for _, pser := range parsers {
if configurable, ok := pser.(parser.ConfigurableParser); ok {
cfg := config.C().GetParserConfigByName(configurable.Name())
if cfg != nil {
if err := configurable.Configure(cfg); err != nil {
fmt.Printf("Error configuring parser %s: %v\n", configurable.Name(), err)
}
}
}
}
})
doConfig.Do(configParsers)
ch := make(chan *parser.Item, 1)
errCh := make(chan error, 1)
@@ -76,3 +74,13 @@ func ParseWithContext(ctx context.Context, url string) (*parser.Item, error) {
return nil, ctx.Err()
}
}
func CanHandle(url string) (bool, parser.Parser) {
doConfig.Do(configParsers)
for _, pser := range parsers {
if pser.CanHandle(url) {
return true, pser
}
}
return false, nil
}

23
parsers/plugin.go Normal file
View File

@@ -0,0 +1,23 @@
package parsers
import "github.com/blang/semver"
var (
LatestParserVersion = semver.MustParse("1.0.0")
MinimumParserVersion = semver.MustParse("1.0.0")
)
type PluginMeta struct {
Name string `json:"name"`
Version string `json:"version"` // [TODO] 分版本解析, 但是我们现在只有 v1 所以先不写
Description string `json:"description"`
Author string `json:"author"`
}
type ParserMethod uint
const (
_ ParserMethod = iota
ParserMethodCanHandle
ParserMethodParse
)

View File

@@ -68,7 +68,7 @@ func (p *TwitterParser) Parse(ctx context.Context, u string) (*parser.Item, erro
resources := make([]parser.Resource, 0, len(fxResp.Tweet.Media.All))
for _, media := range fxResp.Tweet.Media.All {
var size int64
resp, err := p.client.Get(media.URL)
resp, err := p.client.Head(media.URL)
if err == nil {
size = resp.ContentLength
resp.Body.Close()
@@ -101,6 +101,11 @@ func (p *TwitterParser) Name() string {
}
func (p *TwitterParser) Configure(config map[string]any) error {
if config == nil {
p.apiDomain = fxTwitterApi
p.client = *netutil.DefaultParserHTTPClient()
return nil
}
if domain, ok := config["api_domain"].(string); ok && domain != "" {
p.apiDomain = domain
} else {

View File

@@ -7,4 +7,5 @@ import (
const (
MaxPartSize = 1024 * 1024
MaxUploadPartSize = uploader.MaximumPartSize
MaxPhotoSize = 10 * 1024 * 1024
)

View File

@@ -1,9 +0,0 @@
package consts
// inject version by '-X' flag
// go build -ldflags "-X github.com/krau/SaveAny-Bot/pkg/consts.Version=${{ env.VERSION }}"
var (
Version string = "dev"
BuildTime string = "unknown"
GitCommit string = "unknown"
)

View File

@@ -0,0 +1,14 @@
package fnamest
//go:generate go-enum --values --names --noprefix --flag --nocase
// FnameST
/* ENUM(
default, message
) */
type FnameST string
var FnameSTDisplay = map[FnameST]string{
Default: "默认",
Message: "优先从消息生成",
}

View File

@@ -0,0 +1,87 @@
// Code generated by go-enum DO NOT EDIT.
// Version: 0.6.1
// Revision: a6f63bddde05aca4221df9c8e9e6d7d9674b1cb4
// Build Date: 2025-03-18T23:42:14Z
// Built By: goreleaser
package fnamest
import (
"fmt"
"strings"
)
const (
// Default is a FnameST of type default.
Default FnameST = "default"
// Message is a FnameST of type message.
Message FnameST = "message"
)
var ErrInvalidFnameST = fmt.Errorf("not a valid FnameST, try [%s]", strings.Join(_FnameSTNames, ", "))
var _FnameSTNames = []string{
string(Default),
string(Message),
}
// FnameSTNames returns a list of possible string values of FnameST.
func FnameSTNames() []string {
tmp := make([]string, len(_FnameSTNames))
copy(tmp, _FnameSTNames)
return tmp
}
// FnameSTValues returns a list of the values for FnameST
func FnameSTValues() []FnameST {
return []FnameST{
Default,
Message,
}
}
// String implements the Stringer interface.
func (x FnameST) String() string {
return string(x)
}
// IsValid provides a quick way to determine if the typed value is
// part of the allowed enumerated values
func (x FnameST) IsValid() bool {
_, err := ParseFnameST(string(x))
return err == nil
}
var _FnameSTValue = map[string]FnameST{
"default": Default,
"message": Message,
}
// ParseFnameST attempts to convert a string to a FnameST.
func ParseFnameST(name string) (FnameST, error) {
if x, ok := _FnameSTValue[name]; ok {
return x, nil
}
// Case insensitive parse, do a separate lookup to prevent unnecessary cost of lowercasing a string if we don't need to.
if x, ok := _FnameSTValue[strings.ToLower(name)]; ok {
return x, nil
}
return FnameST(""), fmt.Errorf("%s is %w", name, ErrInvalidFnameST)
}
// Set implements the Golang flag.Value interface func.
func (x *FnameST) Set(val string) error {
v, err := ParseFnameST(val)
*x = v
return err
}
// Get implements the Golang flag.Getter interface func.
func (x *FnameST) Get() interface{} {
return *x
}
// Type implements the github.com/spf13/pFlag Value interface.
func (x *FnameST) Type() string {
return "FnameST"
}

View File

@@ -1,4 +1,4 @@
package consts
package rule
const (
RuleStorNameChosen = "CHOSEN"

View File

@@ -3,7 +3,6 @@ package rule
import (
"regexp"
ruleenum "github.com/krau/SaveAny-Bot/pkg/enums/rule"
"github.com/krau/SaveAny-Bot/pkg/tfile"
)
@@ -14,8 +13,8 @@ type RuleFileNameRegex struct {
var _ RuleClass[tfile.TGFile] = (*RuleFileNameRegex)(nil)
func (r RuleFileNameRegex) Type() ruleenum.RuleType {
return ruleenum.FileNameRegex
func (r RuleFileNameRegex) Type() RuleType {
return FileNameRegex
}
func (r RuleFileNameRegex) Match(input tfile.TGFile) (bool, error) {

View File

@@ -1,9 +1,5 @@
package rule
import (
ruleenum "github.com/krau/SaveAny-Bot/pkg/enums/rule"
)
var _ RuleClass[bool] = (*RuleMediaType)(nil)
type RuleMediaType struct {
@@ -11,8 +7,8 @@ type RuleMediaType struct {
matchAlbum bool
}
func (r RuleMediaType) Type() ruleenum.RuleType {
return ruleenum.IsAlbum
func (r RuleMediaType) Type() RuleType {
return IsAlbum
}
func (r RuleMediaType) Match(input bool) (bool, error) {

View File

@@ -2,8 +2,6 @@ package rule
import (
"regexp"
ruleenum "github.com/krau/SaveAny-Bot/pkg/enums/rule"
)
var _ RuleClass[string] = (*RuleMessageRegex)(nil)
@@ -13,8 +11,8 @@ type RuleMessageRegex struct {
regex *regexp.Regexp
}
func (r RuleMessageRegex) Type() ruleenum.RuleType {
return ruleenum.MessageRegex
func (r RuleMessageRegex) Type() RuleType {
return MessageRegex
}
func (r RuleMessageRegex) Match(input string) (bool, error) {

View File

@@ -1,11 +1,7 @@
package rule
import (
ruleenum "github.com/krau/SaveAny-Bot/pkg/enums/rule"
)
type RuleClass[InputType any] interface {
Type() ruleenum.RuleType
Type() RuleType
Match(input InputType) (bool, error)
StorageName() string
StoragePath() string

View File

@@ -10,6 +10,8 @@ import (
const (
TypeAdd = "add"
TypeSetDefault = "setdefault"
TypeConfig = "config"
TypeCancel = "cancel"
)
// type TaskDataTGFiles struct {

View File

@@ -2,20 +2,21 @@ package tfile
import "github.com/gotd/td/tg"
type TGFileOptions func(*tgFile)
type TGFileOption func(*tgFile)
func WithMessage(msg *tg.Message) TGFileOptions {
func WithMessage(msg *tg.Message) TGFileOption {
return func(f *tgFile) {
f.message = msg
}
}
func WithName(name string) TGFileOptions {
func WithName(name string) TGFileOption {
return func(f *tgFile) {
f.name = name
}
}
func WithNameIfEmpty(name string) TGFileOptions {
func WithNameIfEmpty(name string) TGFileOption {
return func(f *tgFile) {
if f.name == "" {
f.name = name
@@ -23,13 +24,13 @@ func WithNameIfEmpty(name string) TGFileOptions {
}
}
func WithSize(size int64) TGFileOptions {
func WithSize(size int64) TGFileOption {
return func(f *tgFile) {
f.size = size
}
}
func WithSizeIfZero(size int64) TGFileOptions {
func WithSizeIfZero(size int64) TGFileOption {
return func(f *tgFile) {
if f.size == 0 {
f.size = size

View File

@@ -54,7 +54,7 @@ func NewTGFile(
dler downloader.Client,
size int64,
name string,
opts ...TGFileOptions,
opts ...TGFileOption,
) TGFile {
f := &tgFile{
location: location,
@@ -68,7 +68,7 @@ func NewTGFile(
return f
}
func FromMedia(media tg.MessageMediaClass, client downloader.Client, opts ...TGFileOptions) (TGFile, error) {
func FromMedia(media tg.MessageMediaClass, client downloader.Client, opts ...TGFileOption) (TGFile, error) {
switch m := media.(type) {
case *tg.MessageMediaDocument:
document, ok := m.Document.AsNotEmpty()
@@ -125,7 +125,7 @@ func FromMedia(media tg.MessageMediaClass, client downloader.Client, opts ...TGF
return nil, fmt.Errorf("unsupported media type: %T", media)
}
func FromMediaMessage(media tg.MessageMediaClass, client downloader.Client, msg *tg.Message, opts ...TGFileOptions) (TGFileMessage, error) {
func FromMediaMessage(media tg.MessageMediaClass, client downloader.Client, msg *tg.Message, opts ...TGFileOption) (TGFileMessage, error) {
file, err := FromMedia(media, client, opts...)
if err != nil {
return nil, err

View File

@@ -5,12 +5,14 @@ import (
"fmt"
"io"
"path"
"strconv"
"strings"
"time"
"github.com/duke-git/lancet/v2/convertor"
"github.com/charmbracelet/log"
"github.com/duke-git/lancet/v2/slice"
"github.com/duke-git/lancet/v2/validator"
"github.com/gabriel-vasile/mimetype"
"github.com/gotd/td/constant"
"github.com/gotd/td/telegram/message"
"github.com/gotd/td/telegram/message/styling"
"github.com/gotd/td/telegram/uploader"
@@ -75,26 +77,46 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
if tctx == nil {
return fmt.Errorf("failed to get telegram context")
}
// 去除前导斜杠并分隔路径, 当 len(parts):
// ==0, 存储到配置文件中的 chat_id, 随机文件名
// ==1, 视作只有文件名, 存储到配置文件中的 chat_id
// ==2, parts[0]: 视作要存储到的 chat_id, parts[1]: filename
parts := slice.Compact(strings.Split(strings.TrimPrefix(storagePath, "/"), "/"))
filename := ""
chatID := t.config.ChatID
if after, ok0 := strings.CutPrefix(convertor.ToString(chatID), "-100"); ok0 {
cid, err := strconv.ParseInt(after, 10, 64)
if len(parts) >= 1 {
filename = parts[len(parts)-1]
}
if len(parts) >= 2 && validator.IsAlphaNumeric(parts[0]) {
cid, err := tgutil.ParseChatID(tctx, parts[0])
if err != nil {
return fmt.Errorf("failed to parse chat ID: %w", err)
// id不合法时使用配置文件中的 chat_id
log.FromContext(ctx).Warnf("Failed to parse chat ID from path, using configured chat_id: %s", err)
cid = chatID
} else {
if cid > constant.MaxTDLibChannelID || cid > constant.MaxTDLibChatID || cid > constant.MaxTDLibUserID {
cid = chatID
}
}
chatID = cid
}
peer := tctx.PeerStorage.GetInputPeerById(chatID)
if peer == nil {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
mtype, err := mimetype.DetectReader(rs)
if err != nil {
return fmt.Errorf("failed to detect mimetype: %w", err)
}
filename := path.Base(storagePath)
if filename == "" {
filename = xid.New().String() + mtype.Extension()
}
if chatID < 0 {
chatID = chatID - constant.ZeroTDLibChannelID
}
peer := tctx.PeerStorage.GetInputPeerById(chatID)
if peer == nil {
return fmt.Errorf("failed to get input peer for chat ID %d", chatID)
}
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek reader: %w", err)
}
@@ -120,9 +142,13 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
return fmt.Errorf("failed to upload file to telegram: %w", err)
}
caption := styling.Plain(filename)
forceFile := t.config.ForceFile
if strings.HasPrefix(mtype.String(), "image/") && size >= tglimit.MaxPhotoSize {
forceFile = true
}
docb := message.UploadedDocument(file, caption).
Filename(filename).
ForceFile(false).
ForceFile(forceFile).
MIME(mtype.String())
var media message.MediaOption = docb
@@ -135,7 +161,6 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er
case strings.HasPrefix(mtypeStr, "image/") && !strings.HasSuffix(mtypeStr, "webp"):
media = message.UploadedPhoto(file, caption)
}
sender := tctx.Sender
_, err = sender.WithUploader(upler).To(peer).Media(ctx, media)
return err