Compare commits

...

9 Commits

13 changed files with 254 additions and 82 deletions

60
.github/workflows/build-docker.yml vendored Normal file
View File

@@ -0,0 +1,60 @@
name: Build and Publish Docker Image
on:
push:
tags:
- "v*"
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build-and-push:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata for Docker
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
type=raw,value=latest
type=ref,event=branch
type=ref,event=tag
labels: |
org.opencontainers.image.title=${{ env.IMAGE_NAME }}
org.opencontainers.image.source=https://github.com/krau/SaveAny-Bot
org.opencontainers.image.url=https://github.com/krau/SaveAny-Bot
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

View File

@@ -23,7 +23,7 @@ jobs:
- name: Setup node - name: Setup node
uses: actions/setup-node@v4 uses: actions/setup-node@v4
with: with:
node-version: 20 node-version: 22
- run: npx changelogithub - run: npx changelogithub
env: env:
@@ -49,7 +49,7 @@ jobs:
- name: Release Go Binary - name: Release Go Binary
uses: wangyoucao577/go-release-action@v1 uses: wangyoucao577/go-release-action@v1
with: with:
pre_command: export pre_command: export CGO_ENABLED=0
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }} goarch: ${{ matrix.goarch }}
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}

18
Dockerfile Normal file
View File

@@ -0,0 +1,18 @@
FROM golang:alpine AS builder
WORKDIR /app
COPY go.* ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o saveany-bot .
FROM alpine:latest
WORKDIR /app
COPY --from=builder /app/saveany-bot .
CMD ["./saveany-bot"]

View File

@@ -29,6 +29,14 @@ chmod +x saveany-bot
./saveany-bot ./saveany-bot
``` ```
## 更新
使用 `upgrade``up` 升级到最新版
```bash
./saveany-bot upgrade
```
## 使用 ## 使用
向 Bot 发送(转发)文件, 按照提示操作. 向 Bot 发送(转发)文件, 按照提示操作.

View File

@@ -107,6 +107,7 @@ func setDefaultStorage(ctx *ext.Context, update *ext.Update) error {
text = append(text, styling.Plain("\n")) text = append(text, styling.Plain("\n"))
text = append(text, styling.Code(name)) text = append(text, styling.Code(name))
} }
text = append(text, styling.Plain("\n示例: /storage local"))
ctx.Reply(update, ext.ReplyTextStyledTextArray(text), nil) ctx.Reply(update, ext.ReplyTextStyledTextArray(text), nil)
return dispatcher.EndGroups return dispatcher.EndGroups
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"crypto/md5" "crypto/md5"
"fmt" "fmt"
"time"
"github.com/celestix/gotgproto" "github.com/celestix/gotgproto"
"github.com/celestix/gotgproto/dispatcher" "github.com/celestix/gotgproto/dispatcher"
@@ -21,10 +22,8 @@ func supportedMediaFilter(m *tg.Message) (bool, error) {
switch m.Media.(type) { switch m.Media.(type) {
case *tg.MessageMediaDocument: case *tg.MessageMediaDocument:
return true, nil return true, nil
case *tg.MessageMediaWebPage: case *tg.MessageMediaPhoto:
return false, dispatcher.EndGroups return true, nil
case tg.MessageMediaClass:
return false, dispatcher.EndGroups
default: default:
return false, nil return false, nil
} }
@@ -80,7 +79,7 @@ func FileFromMedia(media tg.MessageMediaClass) (*types.File, error) {
case *tg.MessageMediaDocument: case *tg.MessageMediaDocument:
document, ok := media.Document.AsNotEmpty() document, ok := media.Document.AsNotEmpty()
if !ok { if !ok {
return nil, fmt.Errorf("unexpected type %T", media) return nil, fmt.Errorf("document is empty")
} }
var fileName string var fileName string
for _, attribute := range document.Attributes { for _, attribute := range document.Attributes {
@@ -97,9 +96,32 @@ func FileFromMedia(media tg.MessageMediaClass) (*types.File, error) {
Location: document.AsInputDocumentFileLocation(), Location: document.AsInputDocumentFileLocation(),
FileSize: document.Size, FileSize: document.Size,
FileName: fileName, FileName: fileName,
MimeType: document.MimeType,
ID: document.ID,
}, nil }, nil
case *tg.MessageMediaPhoto:
photo, ok := media.Photo.AsNotEmpty()
if !ok {
return nil, fmt.Errorf("photo is empty")
}
sizes := photo.Sizes
if len(sizes) == 0 {
return nil, fmt.Errorf("photo sizes is empty")
}
photoSize := sizes[len(sizes)-1]
size, ok := photoSize.AsNotEmpty()
if !ok {
return nil, fmt.Errorf("photo size is empty")
}
location := new(tg.InputPhotoFileLocation)
location.ID = photo.GetID()
location.AccessHash = photo.GetAccessHash()
location.FileReference = photo.GetFileReference()
location.ThumbSize = size.GetType()
return &types.File{
Location: location,
FileSize: 0,
FileName: fmt.Sprintf("photo_%s_%d.jpg", time.Now().Format("2006-01-02_15-04-05"), photo.GetID()),
}, nil
} }
return nil, fmt.Errorf("unexpected type %T", media) return nil, fmt.Errorf("unexpected type %T", media)
} }

View File

@@ -20,6 +20,7 @@ var Cache *CommonCache
func initCache() { func initCache() {
gob.Register(types.File{}) gob.Register(types.File{})
gob.Register(tg.InputDocumentFileLocation{}) gob.Register(tg.InputDocumentFileLocation{})
gob.Register(tg.InputPhotoFileLocation{})
Cache = &CommonCache{cache: freecache.NewCache(10 * 1024 * 1024)} Cache = &CommonCache{cache: freecache.NewCache(10 * 1024 * 1024)}
} }

View File

@@ -9,6 +9,15 @@ import (
"github.com/krau/SaveAny-Bot/logger" "github.com/krau/SaveAny-Bot/logger"
) )
// 创建文件, 自动创建目录
func MkFile(path string, data []byte) error {
err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
if err != nil {
return err
}
return os.WriteFile(path, data, os.ModePerm)
}
// 删除文件, 并清理空目录. 如果文件不存在则返回 nil // 删除文件, 并清理空目录. 如果文件不存在则返回 nil
func PurgeFile(path string) error { func PurgeFile(path string) error {
if err := os.Remove(path); err != nil { if err := os.Remove(path); err != nil {

View File

@@ -3,13 +3,14 @@ package config
import ( import (
"fmt" "fmt"
"os" "os"
"strings"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
type Config struct { type Config struct {
Workers int `toml:"workers" mapstructure:"workers"` Workers int `toml:"workers" mapstructure:"workers"`
Retry int `toml:"retry" mapstructure:"retry"` // Retry times for failed tasks Retry int `toml:"retry" mapstructure:"retry"`
Temp tempConfig `toml:"temp" mapstructure:"temp"` Temp tempConfig `toml:"temp" mapstructure:"temp"`
Log logConfig `toml:"log" mapstructure:"log"` Log logConfig `toml:"log" mapstructure:"log"`
@@ -79,7 +80,12 @@ var Cfg *Config
func Init() { func Init() {
viper.SetConfigName("config") viper.SetConfigName("config")
viper.AddConfigPath(".") viper.AddConfigPath(".")
viper.AddConfigPath("/etc/saveany/")
viper.SetConfigType("toml") viper.SetConfigType("toml")
viper.SetEnvPrefix("SAVEANY")
viper.AutomaticEnv()
replacer := strings.NewReplacer(".", "_")
viper.SetEnvKeyReplacer(replacer)
viper.SetDefault("workers", 3) viper.SetDefault("workers", 3)
viper.SetDefault("retry", 3) viper.SetDefault("retry", 3)

View File

@@ -7,71 +7,86 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"time"
"github.com/celestix/gotgproto/ext" "github.com/celestix/gotgproto/ext"
"github.com/gotd/td/tg" "github.com/gotd/td/tg"
"github.com/krau/SaveAny-Bot/bot" "github.com/krau/SaveAny-Bot/bot"
"github.com/krau/SaveAny-Bot/common"
"github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/logger" "github.com/krau/SaveAny-Bot/logger"
"github.com/krau/SaveAny-Bot/queue" "github.com/krau/SaveAny-Bot/queue"
"github.com/krau/SaveAny-Bot/storage"
"github.com/krau/SaveAny-Bot/types" "github.com/krau/SaveAny-Bot/types"
) )
func processPendingTask(task *types.Task) error { func processPendingTask(task *types.Task) error {
logger.L.Debugf("Start processing task: %s", task.String()) logger.L.Debugf("Start processing task: %s", task.String())
os.MkdirAll(config.Cfg.Temp.BasePath, os.ModePerm) os.MkdirAll(config.Cfg.Temp.BasePath, os.ModePerm)
task.Ctx.(*ext.Context).EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
ctx := task.Ctx.(*ext.Context)
ctx.EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
Message: "正在下载: " + task.String(), Message: "正在下载: " + task.String(),
ID: task.ReplyMessageID, ID: task.ReplyMessageID,
}) })
barTotalCount := 5 destPath := filepath.Join(config.Cfg.Temp.BasePath, task.File.FileName)
if task.File.FileSize > 1024*1024*200 { if task.StoragePath == "" {
barTotalCount = 10 task.StoragePath = task.File.FileName
} else if task.File.FileSize > 1024*1024*500 {
barTotalCount = 20
} else if task.File.FileSize > 1024*1024*1000 {
barTotalCount = 50
} }
readCloser, err := NewTelegramReader(task.Ctx, bot.Client, task.File.Location, 0, task.File.FileSize-1, task.File.FileSize, func(bytesRead, contentLength int64) { // process photo
if task.File.FileSize == 0 {
res, err := bot.Client.API().UploadGetFile(task.Ctx, &tg.UploadGetFileRequest{
Location: task.File.Location,
Offset: 0,
Limit: 1024 * 1024,
})
if err != nil {
return fmt.Errorf("Failed to get file: %w", err)
}
result, ok := res.(*tg.UploadFile)
if !ok {
return fmt.Errorf("unexpected type %T", res)
}
if err := os.WriteFile(destPath, result.Bytes, os.ModePerm); err != nil {
return fmt.Errorf("Failed to write file: %w", err)
}
defer cleanCacheFile(destPath)
logger.L.Infof("Downloaded file: %s", destPath)
return saveFileWithRetry(task, destPath)
}
barTotalCount := calculateBarTotalCount(task.File.FileSize)
progressCallback := func(bytesRead, contentLength int64) {
progress := float64(bytesRead) / float64(contentLength) * 100 progress := float64(bytesRead) / float64(contentLength) * 100
logger.L.Tracef("Downloading %s: %.2f%%", task.String(), progress) logger.L.Tracef("Downloading %s: %.2f%%", task.String(), progress)
if task.File.FileSize < 1024*1024*50 { if task.File.FileSize < 1024*1024*50 || int(progress)%(100/barTotalCount) != 0 {
return return
} }
text := fmt.Sprintf("正在下载: %s\n[%s] %.2f%%",
barSize := 100 / barTotalCount task.String(),
if int(progress)%barSize != 0 { getProgressBar(progress, barTotalCount),
return progress,
} )
ctx.EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
text := fmt.Sprintf("正在下载: %s\n[%s] %.2f%%", task.String(), func() string {
bar := ""
for i := 0; i < barTotalCount; i++ {
if int(progress)/barSize > i {
bar += "█"
} else {
bar += "░"
}
}
return bar
}(), progress)
task.Ctx.(*ext.Context).EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
Message: text, Message: text,
ID: task.ReplyMessageID, ID: task.ReplyMessageID,
}) })
}, task.File.FileSize/100) }
readCloser, err := NewTelegramReader(task.Ctx, bot.Client, &task.File.Location,
0, task.File.FileSize-1, task.File.FileSize,
progressCallback, task.File.FileSize/100)
if err != nil { if err != nil {
return fmt.Errorf("Failed to create reader: %w", err) return fmt.Errorf("Failed to create reader: %w", err)
} }
defer readCloser.Close() defer readCloser.Close()
dest, err := os.Create(filepath.Join(config.Cfg.Temp.BasePath, task.File.FileName)) dest, err := os.Create(destPath)
if err != nil { if err != nil {
return fmt.Errorf("Failed to create file: %w", err) return fmt.Errorf("Failed to create file: %w", err)
} }
@@ -80,44 +95,16 @@ func processPendingTask(task *types.Task) error {
if _, err := io.CopyN(dest, readCloser, task.File.FileSize); err != nil { if _, err := io.CopyN(dest, readCloser, task.File.FileSize); err != nil {
return fmt.Errorf("Failed to download file: %w", err) return fmt.Errorf("Failed to download file: %w", err)
} }
destName := dest.Name()
defer func() { defer cleanCacheFile(destPath)
if config.Cfg.Temp.CacheTTL > 0 {
common.RmFileAfter(destName, time.Duration(config.Cfg.Temp.CacheTTL)*time.Second)
} else {
if err := os.Remove(destName); err != nil {
logger.L.Errorf("Failed to purge file: %s", err)
}
}
}()
if task.StoragePath == "" { logger.L.Infof("Downloaded file: %s", destPath)
task.StoragePath = task.File.FileName ctx.EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
}
logger.L.Infof("Downloaded file: %s", dest.Name())
task.Ctx.(*ext.Context).EditMessage(task.ChatID, &tg.MessagesEditMessageRequest{
Message: fmt.Sprintf("下载完成: %s\n正在转存文件...", task.FileName()), Message: fmt.Sprintf("下载完成: %s\n正在转存文件...", task.FileName()),
ID: task.ReplyMessageID, ID: task.ReplyMessageID,
}) })
if config.Cfg.Retry <= 0 {
if err := storage.Save(task.Storage, task.Ctx, dest.Name(), task.StoragePath); err != nil { return saveFileWithRetry(task, destPath)
return fmt.Errorf("Failed to save file: %w", err)
}
} else {
for i := 0; i < config.Cfg.Retry; i++ {
if err := storage.Save(task.Storage, task.Ctx, dest.Name(), task.StoragePath); err != nil {
logger.L.Errorf("Failed to save file: %s, retrying...", err)
if i == config.Cfg.Retry-1 {
return fmt.Errorf("Failed to save file: %w", err)
}
} else {
break
}
}
}
return nil
} }
func worker(queue *queue.TaskQueue, semaphore chan struct{}) { func worker(queue *queue.TaskQueue, semaphore chan struct{}) {

View File

@@ -13,7 +13,7 @@ import (
type telegramReader struct { type telegramReader struct {
client *gotgproto.Client client *gotgproto.Client
location *tg.InputDocumentFileLocation location *tg.InputFileLocationClass
bytesread int64 bytesread int64
chunkSize int64 chunkSize int64
i int64 i int64
@@ -67,7 +67,7 @@ func (r *telegramReader) Read(p []byte) (n int, err error) {
func NewTelegramReader( func NewTelegramReader(
ctx context.Context, ctx context.Context,
client *gotgproto.Client, client *gotgproto.Client,
location *tg.InputDocumentFileLocation, location *tg.InputFileLocationClass,
start int64, start int64,
end int64, end int64,
contentLength int64, contentLength int64,
@@ -97,7 +97,7 @@ func (r *telegramReader) chunk(offset int64, limit int64) ([]byte, error) {
req := &tg.UploadGetFileRequest{ req := &tg.UploadGetFileRequest{
Offset: offset, Offset: offset,
Limit: int(limit), Limit: int(limit),
Location: r.location, Location: *r.location,
} }
res, err := r.client.API().UploadGetFile(r.ctx, req) res, err := r.client.API().UploadGetFile(r.ctx, req)
if err != nil { if err != nil {

62
core/utils.go Normal file
View File

@@ -0,0 +1,62 @@
package core
import (
"fmt"
"os"
"time"
"github.com/krau/SaveAny-Bot/common"
"github.com/krau/SaveAny-Bot/config"
"github.com/krau/SaveAny-Bot/logger"
"github.com/krau/SaveAny-Bot/storage"
"github.com/krau/SaveAny-Bot/types"
)
func saveFileWithRetry(task *types.Task, destPath string) error {
for i := 0; i <= config.Cfg.Retry; i++ {
if err := storage.Save(task.Storage, task.Ctx, destPath, task.StoragePath); err != nil {
if i == config.Cfg.Retry {
return fmt.Errorf("Failed to save file: %w", err)
}
logger.L.Errorf("Failed to save file: %s, retrying...", err)
continue
}
return nil
}
return nil
}
func getProgressBar(progress float64, totalCount int) string {
bar := ""
barSize := 100 / totalCount
for i := 0; i < totalCount; i++ {
if int(progress)/barSize > i {
bar += "█"
} else {
bar += "░"
}
}
return bar
}
func cleanCacheFile(destPath string) {
if config.Cfg.Temp.CacheTTL > 0 {
common.RmFileAfter(destPath, time.Duration(config.Cfg.Temp.CacheTTL)*time.Second)
} else {
if err := os.Remove(destPath); err != nil {
logger.L.Errorf("Failed to purge file: %s", err)
}
}
}
func calculateBarTotalCount(fileSize int64) int {
barTotalCount := 5
if fileSize > 1024*1024*1000 {
barTotalCount = 50
} else if fileSize > 1024*1024*500 {
barTotalCount = 20
} else if fileSize > 1024*1024*200 {
barTotalCount = 10
}
return barTotalCount
}

View File

@@ -49,9 +49,7 @@ func (t Task) FileName() string {
} }
type File struct { type File struct {
Location *tg.InputDocumentFileLocation Location tg.InputFileLocationClass
FileSize int64 FileSize int64
FileName string FileName string
MimeType string
ID int64
} }