Compare commits

..

7 Commits

15 changed files with 393 additions and 161 deletions

View File

@@ -2,7 +2,7 @@
# <img src="docs/logo.jpg" width="45" align="center"> Save Any Bot
**简体中文** | [English](README_EN.md)
**简体中文** | [English](README_EN.md)
把 Telegram 的文件保存到各类存储端.
@@ -50,7 +50,7 @@ WantedBy=multi-user.target
systemctl enable --now saveany-bot
```
#### 为OpenWrt及衍生系统添加开机自启动服务
#### 为 OpenWrt 及衍生系统添加开机自启动服务
创建文件 ` /etc/init.d/saveanybot` ,参考[saveanybot](./docs/saveanybot)自行修改.
@@ -60,7 +60,7 @@ systemctl enable --now saveany-bot
`chmod +x /etc/rc.d/S99saveanybot`
#### 为OpenWrt及衍生系统添加快捷指令
#### 为 OpenWrt 及衍生系统添加快捷指令
创建文件` /usr/bin/sabot` ,参考[sabot](./docs/sabot)自行配置修改,注意此处文件编码仅支持 ANSI 936 .
@@ -68,7 +68,6 @@ systemctl enable --now saveany-bot
之后,终端输入`sabot start|stop|restart|status|enable|disable`即可.
### 使用 Docker 部署
#### Docker Compose
@@ -111,6 +110,14 @@ docker restart saveany-bot
---
## 赞助
本项目受到 [YxVM](https://yxvm.com/) 与 [NodeSupport](https://github.com/NodeSeekDev/NodeSupport) 的支持.
如果这个项目对你有帮助, 你可以考虑通过以下方式赞助我:
- [爱发电](https://afdian.com/a/acherkrau)
## Thanks
- [gotd](https://github.com/gotd/td)

View File

@@ -92,6 +92,14 @@ Send (forward) files to the Bot and follow the prompts.
---
## Sponsors
This project is supported by [YxVM](https://yxvm.com/) and [NodeSupport](https://github.com/NodeSeekDev/NodeSupport).
You can consider sponsoring me if this project helps you:
- [Afdian](https://afdian.com/a/acherkrau)
## Thanks
- [gotd](https://github.com/gotd/td)

View File

@@ -27,10 +27,10 @@ func newProxyDialer(proxyUrl string) (proxy.Dialer, error) {
}
func Init() {
InitTelegraphClient()
common.Log.Info("初始化 Telegram 客户端...")
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.Cfg.Telegram.Timeout)*time.Second)
defer cancel()
go InitTelegraphClient()
resultChan := make(chan struct {
client *gotgproto.Client
err error

View File

@@ -13,6 +13,9 @@ token = ""
# app_id = 123456
# app_hash = "0123456789abcdef0123456789abcdef"
# 初始化超时时间, 单位: 秒
timeout = 60
[telegram.proxy]
# 启用代理连接 telegram, 只支持 socks5
enable = false
@@ -30,12 +33,6 @@ enable = true
# 文件保存根路径
base_path = "./downloads"
[[storages]]
name = "本机2"
type = "local"
enable = true
base_path = "./downloads/2"
[[storages]]
name = "MyAlist"
type = "alist"
@@ -49,7 +46,6 @@ token_exp = 86400 # 86400--1天 604800--7天 1296000--15天 2592000--30
# 请自行在 alist 侧配置合理的 token 过期时间
# token = ""
[[storages]]
name = "MyWebdav"
type = "webdav"

View File

@@ -44,6 +44,7 @@ type telegramConfig struct {
Token string `toml:"token" mapstructure:"token"`
AppID int `toml:"app_id" mapstructure:"app_id" json:"app_id"`
AppHash string `toml:"app_hash" mapstructure:"app_hash" json:"app_hash"`
Timeout int `toml:"timeout" mapstructure:"timeout" json:"timeout"`
Proxy proxyConfig `toml:"proxy" mapstructure:"proxy"`
// Deprecated
@@ -82,6 +83,7 @@ func Init() error {
viper.SetDefault("telegram.app_id", 1025907)
viper.SetDefault("telegram.app_hash", "452b0359b988148995f22ff0f4229750")
viper.SetDefault("telegram.timeout", 60)
viper.SetDefault("temp.base_path", "cache/")
viper.SetDefault("temp.cache_ttl", 3600)

View File

@@ -59,41 +59,50 @@ func processPendingTask(task *types.Task) error {
downloadBuilder := Downloader.Download(bot.Client.API(), task.File.Location).WithThreads(getTaskThreads(task.File.FileSize))
notsupportStreamStorage, notsupportStream := taskStorage.(storage.StorageNotSupportStream)
cancelMarkUp := getCancelTaskMarkup(task)
if config.Cfg.Stream {
if !notsupportStream {
text, entities := buildProgressMessageEntity(task, 0, task.StartTime, 0)
ctx.EditMessage(task.ReplyChatID, &tg.MessagesEditMessageRequest{
Message: text,
Entities: entities,
ID: task.ReplyMessageID,
ReplyMarkup: cancelMarkUp,
})
text, entities := buildProgressMessageEntity(task, 0, task.StartTime, 0)
ctx.EditMessage(task.ReplyChatID, &tg.MessagesEditMessageRequest{
Message: text,
Entities: entities,
ID: task.ReplyMessageID,
ReplyMarkup: getCancelTaskMarkup(task),
})
pr, pw := io.Pipe()
defer pr.Close()
pr, pw := io.Pipe()
defer pr.Close()
task.StartTime = time.Now()
progressCallback := buildProgressCallback(ctx, task, getProgressUpdateCount(task.File.FileSize))
task.StartTime = time.Now()
progressCallback := buildProgressCallback(ctx, task, getProgressUpdateCount(task.File.FileSize))
progressStream := NewProgressStream(pw, task.File.FileSize, progressCallback)
progressStream := NewProgressStream(pw, task.File.FileSize, progressCallback)
eg, uploadCtx := errgroup.WithContext(cancelCtx)
eg, uploadCtx := errgroup.WithContext(cancelCtx)
eg.Go(func() error {
return taskStorage.Save(uploadCtx, pr, task.StoragePath)
})
eg.Go(func() error {
_, err := downloadBuilder.Stream(uploadCtx, progressStream)
if closeErr := pw.CloseWithError(err); closeErr != nil {
common.Log.Errorf("Failed to close pipe writer: %v", closeErr)
eg.Go(func() error {
return taskStorage.Save(uploadCtx, pr, task.StoragePath)
})
eg.Go(func() error {
_, err := downloadBuilder.Stream(uploadCtx, progressStream)
if closeErr := pw.CloseWithError(err); closeErr != nil {
common.Log.Errorf("Failed to close pipe writer: %v", closeErr)
}
return err
})
if err := eg.Wait(); err != nil {
return err
}
return err
})
if err := eg.Wait(); err != nil {
return err
}
return nil
return nil
}
common.Log.Warnf("存储 %s 不支持流式传输: %s", task.StorageName, notsupportStreamStorage.NotSupportStream())
ctx.EditMessage(task.ReplyChatID, &tg.MessagesEditMessageRequest{
Message: fmt.Sprintf("存储 %s 不支持流式传输: %s\n正在使用普通下载...", task.StorageName, notsupportStreamStorage.NotSupportStream()),
ID: task.ReplyMessageID,
ReplyMarkup: cancelMarkUp,
})
}
cacheDestPath := filepath.Join(config.Cfg.Temp.BasePath, task.FileName())
@@ -110,7 +119,7 @@ func processPendingTask(task *types.Task) error {
Message: text,
Entities: entities,
ID: task.ReplyMessageID,
ReplyMarkup: getCancelTaskMarkup(task),
ReplyMarkup: cancelMarkUp,
})
progressCallback := buildProgressCallback(ctx, task, getProgressUpdateCount(task.File.FileSize))
@@ -191,7 +200,7 @@ func processTelegraph(extCtx *ext.Context, cancelCtx context.Context, task *type
if len(node.Children) != 0 {
for _, child := range node.Children {
imgs = append(imgs, GetImages(child)...)
imgs = append(imgs, getNodeImages(child)...)
}
}
@@ -265,27 +274,3 @@ func processTelegraph(extCtx *ext.Context, cancelCtx context.Context, task *type
return cancelCtx.Err()
}
}
func GetImages(node telegraph.Node) []string {
var srcs []string
var nodeElement telegraph.NodeElement
data, err := json.Marshal(node)
if err != nil {
return srcs
}
err = json.Unmarshal(data, &nodeElement)
if err != nil {
return srcs
}
if nodeElement.Tag == "img" {
if src, exists := nodeElement.Attrs["src"]; exists {
srcs = append(srcs, src)
}
}
for _, child := range nodeElement.Children {
srcs = append(srcs, GetImages(child)...)
}
return srcs
}

View File

@@ -72,7 +72,7 @@ func TestGetImgSrcs(t *testing.T) {
"https://example.com/image4.png",
}
got := GetImages(complexStructure)
got := getNodeImages(complexStructure)
if !reflect.DeepEqual(expected, got) {
t.Errorf("expected %vgot %v", expected, got)

View File

@@ -3,6 +3,7 @@ package core
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
@@ -10,6 +11,7 @@ import (
"time"
"github.com/celestix/gotgproto/ext"
"github.com/celestix/telegraph-go/v2"
"github.com/gabriel-vasile/mimetype"
"github.com/gotd/td/telegram/message/entity"
"github.com/gotd/td/telegram/message/styling"
@@ -22,22 +24,33 @@ import (
)
func saveFileWithRetry(ctx context.Context, storagePath string, taskStorage storage.Storage, cacheFilePath string) error {
file, err := os.Open(cacheFilePath)
if err != nil {
return fmt.Errorf("failed to open cache file: %w", err)
}
defer file.Close()
fileStat, err := file.Stat()
if err != nil {
return fmt.Errorf("failed to get file stat: %w", err)
}
vctx := context.WithValue(ctx, types.ContextKeyContentLength, fileStat.Size())
for i := 0; i <= config.Cfg.Retry; i++ {
if err := ctx.Err(); err != nil {
if err := vctx.Err(); err != nil {
return fmt.Errorf("context canceled while saving file: %w", err)
}
file, err := os.Open(cacheFilePath)
if err != nil {
return fmt.Errorf("failed to open cache file: %w", err)
}
if err := taskStorage.Save(ctx, file, storagePath); err != nil {
defer file.Close()
if err := taskStorage.Save(vctx, file, storagePath); err != nil {
if i == config.Cfg.Retry {
return fmt.Errorf("failed to save file: %w", err)
}
common.Log.Errorf("Failed to save file: %s, retrying...", err)
select {
case <-ctx.Done():
return fmt.Errorf("context canceled during retry delay: %w", ctx.Err())
case <-vctx.Done():
return fmt.Errorf("context canceled during retry delay: %w", vctx.Err())
case <-time.After(time.Duration(i*500) * time.Millisecond):
}
continue
@@ -256,3 +269,27 @@ func NewProgressStream(writer io.Writer, size int64, callback func(bytesRead, co
interval: interval,
}
}
func getNodeImages(node telegraph.Node) []string {
var srcs []string
var nodeElement telegraph.NodeElement
data, err := json.Marshal(node)
if err != nil {
return srcs
}
err = json.Unmarshal(data, &nodeElement)
if err != nil {
return srcs
}
if nodeElement.Tag == "img" {
if src, exists := nodeElement.Attrs["src"]; exists {
srcs = append(srcs, src)
}
}
for _, child := range nodeElement.Children {
srcs = append(srcs, getNodeImages(child)...)
}
return srcs
}

View File

@@ -6,10 +6,7 @@ Bot 接受两种消息: 文件和链接.
支持以下链接:
1. 公开频道 (具有用户名) 的消息链接, 例如: `https://t.me/acherkrau/1097`.
**即使频道禁止了转发和保存, Bot 依然可以下载其文件.**
1. 公开频道 (具有用户名) 的消息链接, 例如: `https://t.me/acherkrau/1097`. **即使频道禁止了转发和保存, Bot 依然可以下载其文件.**
2. Telegra.ph 的文章链接, Bot 将下载其中的所有图片
## 静默模式 (silent)
@@ -35,3 +32,7 @@ Bot 接受两种消息: 文件和链接.
- 无法使用多线程从 telegram 下载文件, 速度较慢.
- 网络不稳定时, 任务失败率高.
- 无法在中间层对文件进行处理, 例如自动文件类型识别.
**不支持** Stream 模式的存储端:
- alist

View File

@@ -106,6 +106,12 @@ func (a *Alist) Save(ctx context.Context, reader io.Reader, storagePath string)
req.Header.Set("Authorization", a.token)
req.Header.Set("File-Path", url.PathEscape(storagePath))
req.Header.Set("Content-Type", "application/octet-stream")
if length := ctx.Value(types.ContextKeyContentLength); length != nil {
length, ok := length.(int64)
if ok {
req.ContentLength = length
}
}
resp, err := a.client.Do(req)
if err != nil {
@@ -134,6 +140,10 @@ func (a *Alist) Save(ctx context.Context, reader io.Reader, storagePath string)
return nil
}
func (a *Alist) NotSupportStream() string {
return "Alist does not support chunked transfer encoding"
}
func (a *Alist) JoinStoragePath(task types.Task) string {
return path.Join(a.config.BasePath, task.StoragePath)
}

View File

@@ -23,6 +23,11 @@ type Storage interface {
Save(ctx context.Context, reader io.Reader, storagePath string) error
}
type StorageNotSupportStream interface {
Storage
NotSupportStream() string
}
var Storages = make(map[string]Storage)
var UserStorages = make(map[int64][]Storage)

View File

@@ -0,0 +1,130 @@
package webdav
import (
"context"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
"testing"
"golang.org/x/net/webdav"
)
func setupWebDAVServer(t *testing.T) (*httptest.Server, string) {
t.Helper()
tempDir, err := os.MkdirTemp("", "webdav_test")
if err != nil {
t.Fatalf("mk temp dir failed: %v", err)
}
handler := &webdav.Handler{
Prefix: "/",
FileSystem: webdav.Dir(tempDir),
LockSystem: webdav.NewMemLS(),
}
server := httptest.NewServer(handler)
return server, tempDir
}
func TestMkDirAndExists(t *testing.T) {
server, tempDir := setupWebDAVServer(t)
defer os.RemoveAll(tempDir)
defer server.Close()
client := NewClient(server.URL, "", "", nil)
ctx := context.Background()
testpaths := []string{"testdir", "testdir/subdir", "testdir/子目录", "/testdir/测试路径/测试路径2"}
for _, p := range testpaths {
exists, err := client.Exists(ctx, p)
if err != nil {
t.Fatalf("Call Exists Err: %v", err)
}
if exists {
t.Fatalf("Dir should not exist")
}
if err := client.MkDir(ctx, p); err != nil {
t.Fatalf("Call MkDir Err: %v", err)
}
exists, err = client.Exists(ctx, p)
if err != nil {
t.Fatalf("Call Exists Err: %v", err)
}
if !exists {
t.Fatalf("Dir should exist")
}
}
}
func TestWriteFile(t *testing.T) {
server, tempDir := setupWebDAVServer(t)
defer os.RemoveAll(tempDir)
defer server.Close()
client := NewClient(server.URL, "", "", nil)
ctx := context.Background()
testCases := []struct {
remotePath string
content string
}{
{
remotePath: "hello.txt",
content: "Hello webdav",
},
{
remotePath: "nested/dir/test.txt",
content: "Nested file",
},
{
remotePath: "empty.txt",
content: "",
},
{
remotePath: "unicode.txt",
content: "测试",
},
}
for _, tc := range testCases {
t.Run(tc.remotePath, func(t *testing.T) {
dir := path.Dir(tc.remotePath)
if dir != "." {
if err := client.MkDir(ctx, dir); err != nil {
t.Fatalf("创建目录 %s 失败: %v", dir, err)
}
}
if err := client.WriteFile(ctx, tc.remotePath, strings.NewReader(tc.content)); err != nil {
t.Fatalf("写入文件 %s 失败: %v", tc.remotePath, err)
}
localPath := filepath.Join(tempDir, tc.remotePath)
data, err := os.ReadFile(localPath)
if err != nil {
t.Fatalf("读取文件 %s 失败: %v", localPath, err)
}
if string(data) != tc.content {
t.Fatalf("文件内容不匹配: got %s, want %s", string(data), tc.content)
}
appended := tc.content + " Overwritten."
if err := client.WriteFile(ctx, tc.remotePath, strings.NewReader(appended)); err != nil {
t.Fatalf("覆盖写入文件 %s 失败: %v", tc.remotePath, err)
}
data, err = os.ReadFile(localPath)
if err != nil {
t.Fatalf("读取覆盖后的文件 %s 失败: %v", localPath, err)
}
if string(data) != appended {
t.Fatalf("文件覆盖后的内容不匹配: got %s, want %s", string(data), appended)
}
})
}
}

View File

@@ -6,6 +6,8 @@ import (
"io"
"net/http"
"strings"
"github.com/krau/SaveAny-Bot/types"
)
type Client struct {
@@ -38,21 +40,63 @@ func (c *Client) doRequest(ctx context.Context, method, url string, body io.Read
if c.Username != "" && c.Password != "" {
req.SetBasicAuth(c.Username, c.Password)
}
if length := ctx.Value(types.ContextKeyContentLength); length != nil {
if l, ok := length.(int64); ok {
req.ContentLength = l
}
}
return c.httpClient.Do(req)
}
func (c *Client) MkDir(ctx context.Context, dirPath string) error {
url := c.BaseURL + dirPath
resp, err := c.doRequest(ctx, "MKCOL", url, nil)
func (c *Client) Exists(ctx context.Context, remotePath string) (bool, error) {
url := c.BaseURL + remotePath
resp, err := c.doRequest(ctx, "PROPFIND", url, nil)
if err != nil {
return err
return false, err
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
return false, fmt.Errorf("PROPFIND: %s", resp.Status)
}
func (c *Client) MkDir(ctx context.Context, dirPath string) error {
dirPath = strings.Trim(dirPath, "/")
if dirPath == "" {
return nil
}
return fmt.Errorf("MKCOL: %s", resp.Status)
parts := strings.Split(dirPath, "/")
currentPath := ""
for i, part := range parts {
if i > 0 {
currentPath += "/"
}
currentPath += part
exists, err := c.Exists(ctx, currentPath)
if err != nil {
return err
}
if exists {
continue
}
url := c.BaseURL + currentPath
resp, err := c.doRequest(ctx, "MKCOL", url, nil)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("MKCOL %s: %s", currentPath, resp.Status)
}
}
return nil
}
func (c *Client) WriteFile(ctx context.Context, remotePath string, content io.Reader) error {

82
types/task.go Normal file
View File

@@ -0,0 +1,82 @@
package types
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"net/url"
"strings"
"time"
"github.com/gotd/td/tg"
)
type Task struct {
Ctx context.Context
Cancel context.CancelFunc
Error error
Status TaskStatus
StorageName string
StoragePath string
StartTime time.Time
File *File
FileMessageID int
FileChatID int64
IsTelegraph bool
TelegraphURL string
// to track the reply message
ReplyMessageID int
ReplyChatID int64
UserID int64
}
func (t Task) Key() string {
if t.IsTelegraph {
return hashStr(t.TelegraphURL)
}
return fmt.Sprintf("%d:%d", t.FileChatID, t.FileMessageID)
}
func (t Task) String() string {
if t.IsTelegraph {
return fmt.Sprintf("[telegraph]:%s", t.TelegraphURL)
}
return fmt.Sprintf("[%d:%d]:%s", t.FileChatID, t.FileMessageID, t.File.FileName)
}
func (t Task) FileName() string {
if t.IsTelegraph {
tgphPath := strings.Split(t.TelegraphURL, "/")[len(strings.Split(t.TelegraphURL, "/"))-1]
tgphPathUnescaped, err := url.PathUnescape(tgphPath)
if err != nil {
return tgphPath
}
return tgphPathUnescaped
}
return t.File.FileName
}
type File struct {
Location tg.InputFileLocationClass
FileSize int64
FileName string
}
func (f File) Hash() string {
locationBytes := []byte(f.Location.String())
fileSizeBytes := []byte(fmt.Sprintf("%d", f.FileSize))
fileNameBytes := []byte(f.FileName)
structBytes := append(locationBytes, fileSizeBytes...)
structBytes = append(structBytes, fileNameBytes...)
hash := md5.New()
hash.Write(structBytes)
hashBytes := hash.Sum(nil)
return hex.EncodeToString(hashBytes)
}

View File

@@ -1,20 +1,8 @@
package types
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"net/url"
"strings"
"time"
"github.com/gotd/td/tg"
)
type TaskStatus string
var (
const (
Pending TaskStatus = "pending"
Succeeded TaskStatus = "succeeded"
Failed TaskStatus = "failed"
@@ -23,7 +11,7 @@ var (
type StorageType string
var (
const (
StorageTypeLocal StorageType = "local"
StorageTypeWebdav StorageType = "webdav"
StorageTypeAlist StorageType = "alist"
@@ -38,71 +26,8 @@ var StorageTypeDisplay = map[StorageType]string{
StorageTypeMinio: "Minio",
}
type Task struct {
Ctx context.Context
Cancel context.CancelFunc
Error error
Status TaskStatus
StorageName string
StoragePath string
StartTime time.Time
type ContextKey string
File *File
FileMessageID int
FileChatID int64
IsTelegraph bool
TelegraphURL string
// to track the reply message
ReplyMessageID int
ReplyChatID int64
UserID int64
}
func (t Task) Key() string {
if t.IsTelegraph {
return hashStr(t.TelegraphURL)
}
return fmt.Sprintf("%d:%d", t.FileChatID, t.FileMessageID)
}
func (t Task) String() string {
if t.IsTelegraph {
return fmt.Sprintf("[telegraph]:%s", t.TelegraphURL)
}
return fmt.Sprintf("[%d:%d]:%s", t.FileChatID, t.FileMessageID, t.File.FileName)
}
func (t Task) FileName() string {
if t.IsTelegraph {
tgphPath := strings.Split(t.TelegraphURL, "/")[len(strings.Split(t.TelegraphURL, "/"))-1]
tgphPathUnescaped, err := url.PathUnescape(tgphPath)
if err != nil {
return tgphPath
}
return tgphPathUnescaped
}
return t.File.FileName
}
type File struct {
Location tg.InputFileLocationClass
FileSize int64
FileName string
}
func (f File) Hash() string {
locationBytes := []byte(f.Location.String())
fileSizeBytes := []byte(fmt.Sprintf("%d", f.FileSize))
fileNameBytes := []byte(f.FileName)
structBytes := append(locationBytes, fileSizeBytes...)
structBytes = append(structBytes, fileNameBytes...)
hash := md5.New()
hash.Write(structBytes)
hashBytes := hash.Sum(nil)
return hex.EncodeToString(hashBytes)
}
const (
ContextKeyContentLength ContextKey = "content-length"
)