mirror of
https://github.com/krau/SaveAny-Bot.git
synced 2026-08-29 04:06:42 +08:00
* fix: prevent queue deadlock after cancelling tasks Get no longer recurses while holding the mutex. Cancelled queued tasks leave the map, making their IDs reusable. Closed empty queues return ErrQueueClosed. * fix: init task queue lazily and close on shutdown AddTask is safe before Run by initializing the queue once. Close unblocks workers waiting in Get. * fix: make resource fingerprints deterministic Sort map keys before hashing so Resource.ID is stable. * fix: harden per-item processing tracking in tasks Use the resource fingerprint as the dedup key on insert and delete. Check and set processing entries atomically instead of TOCTOU. Count only successful downloads. * fix: honor IgnoreErrors in batch tasks Element failures no longer cancel sibling elements or stop later groups. * fix: report streamed upload bytes in batch tasks Stream downloads report their byte count as the upload total. * fix: validate i18n key parity across locales geni18n fails when a language file misses any key. Align the syncpeers completion key between en and zh-Hans. Translate three untranslated parse keys in zh-Hans. * refactor: share progress throttling helpers Move size-tiered and count-based throttling into progressutil. Localize hardcoded Chinese progress strings. Drop five duplicated implementations and dead local copies. * refactor: share unique filename logic Storage backends use fsutil.UniquePath instead of local loops. * fix: sanitize local storage paths Reject absolute paths and dot-dot escapes in Save. Check close errors and wrap creation failures. * fix: preserve webdav error causes Wrap mkdir and write failures with %w and drop dead error values. * fix: kill rclone subprocess on reader close Prevent cat processes from hanging after the pipe is closed. * fix: fail alist init instead of exiting Replace log.Fatalf with wrapped errors so a bad alist cannot kill the bot. Cancel token refresh with the init context and re-login on 401. * fix: enforce telegram album limits and track saved paths Use tglimit.MaxAlbumItems for album batching and video splitting. Exists now reports previously saved paths instead of always false. * fix: guard storage registry maps Protect Storages and UserStorages with mutexes and expose read accessors. * fix: harden JS parser plugin runtime Validate semver instead of panicking and require canHandle. Recover plugin worker panics and time out CanHandle calls. Return a copy from the registry and sanitize install filenames. * fix: guard kemono parser against nil fields Skip sparse preview and attachment entries instead of panicking. * refactor: remove commented-out dead code Drop unused kemono legacy types and commented response structs. * test: cover invalid plugin version rejection * fix: show all queued tasks in /task list Render up to ten tasks and append the truncation note once. * fix: guard callback data parsing Reject malformed callback payloads before indexing split parts. * fix: isolate media groups per user Key pending groups by chat, user and group id. * fix: require permission for callback handlers * fix: initialize userbot context once Replace the racy lazy init with sync.OnceValue. * fix: avoid leaking raw errors in /dir reply * fix: notify users on invalid update version * fix: fail fast when API listen fails Bind synchronously and surface errors instead of logging them. * fix: add timeouts and backoff to webhook delivery * refactor: remove dead code from api and bot Drop the empty ProgressTracker shim, unused token context key and a redundant SetBotCommands call. * fix: load remote config without local lookup Skip the local file search after reading a config URL and add a timeout. * refactor: drop unused hook config * docs: document parser plugin config * ci: fix BuildTime formatting and align checkout Actions format does not format dates; pass the raw timestamp. * chore: ignore cache directory * fix: make cache init idempotent * fix: upload only downloaded batch elements Failed elements keep partial cache files and never reach the backend. Successfully downloaded siblings still upload when one element fails. * fix: record telegram saved paths only after upload Skip-large returns a sentinel so skipped files are not marked as saved. * fix: deduplicate alist token refreshes Guard token access with a mutex and merge concurrent logins. Reuse a recent refresh to avoid login storms. * test: cover concurrent alist 401 retry Ten parallel uploads share a single re-login under -race. * fix: count parsed resources in progress text * fix: localize storage lookup errors in /dir Use the shared i18n key and escape the dynamic error. * fix: deduplicate concurrent storage initialization singleflight merges first-time inits so side effects are not duplicated. * fix: guard nil progress trackers in api-created tasks Batch, telegraph and transfer tasks run without a Telegram tracker when created through the API; their callbacks must not panic. * fix: keep album order when filtering failed batch items Download results are stored by original index so surviving elements keep their source order. * test: cover nil-tracker task execution * fix: report partial failure in batch done message IgnoreErrors runs with failed elements show success and failed counts instead of claiming every file completed. * fix: skip login refresh for token-only alist storage 401 responses surface the auth error instead of sending a credential-less login; the refresh window never exceeds TokenExp. * test: cover alist refresh semantics Concurrent refresh uses username/password; token-only storage never attempts a login on 401. * fix: call tracker from notifyProgress instead of recursing The helper called itself, overflowing the stack on any batch task with a progress tracker. * fix: propagate cancellation past IgnoreErrors Cancelled tasks must not be reported as successful: only ordinary element failures are ignored. * test: cover notifyProgress tracker call * test: cover cancellation with IgnoreErrors * fix: refresh alist token after startup 401s The init login no longer satisfies the dedup window, so an early 401 triggers a real refresh; later 401s reuse it. Clarify that streaming uploads cannot replay their body. * test: exercise the alist 401 refresh path Concurrent uploads now reject the init token, share one refresh and retry with the new token; token-only stays inert. * style: trim verbose comments Drop process-style explanations; keep one-line behavior notes. * style: gofmt test files * fix: drop unbounded saved-path cache in telegram storage Telegram cannot reliably query remote file existence, so the cache answered a question it could not answer and grew without bound. Exists returns false again, as before. * style: format codes
242 lines
6.8 KiB
Go
242 lines
6.8 KiB
Go
package config
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/charmbracelet/log"
|
|
"github.com/duke-git/lancet/v2/slice"
|
|
"github.com/krau/SaveAny-Bot/config/storage"
|
|
"github.com/spf13/viper"
|
|
"golang.org/x/net/proxy"
|
|
)
|
|
|
|
type Config struct {
|
|
Lang string `toml:"lang" mapstructure:"lang" json:"lang"`
|
|
Workers int `toml:"workers" mapstructure:"workers"`
|
|
Retry int `toml:"retry" mapstructure:"retry"`
|
|
NoCleanCache bool `toml:"no_clean_cache" mapstructure:"no_clean_cache" json:"no_clean_cache"`
|
|
Threads int `toml:"threads" mapstructure:"threads" json:"threads"`
|
|
Stream bool `toml:"stream" mapstructure:"stream" json:"stream"`
|
|
Proxy string `toml:"proxy" mapstructure:"proxy" json:"proxy"`
|
|
Log logConfig `toml:"log" mapstructure:"log" json:"log"`
|
|
Aria2 aria2Config `toml:"aria2" mapstructure:"aria2" json:"aria2"`
|
|
API apiConfig `toml:"api" mapstructure:"api" json:"api"`
|
|
|
|
Cache cacheConfig `toml:"cache" mapstructure:"cache" json:"cache"`
|
|
Users []userConfig `toml:"users" mapstructure:"users" json:"users"`
|
|
Temp tempConfig `toml:"temp" mapstructure:"temp"`
|
|
DB dbConfig `toml:"db" mapstructure:"db"`
|
|
Telegram telegramConfig `toml:"telegram" mapstructure:"telegram"`
|
|
Storages []storage.StorageConfig `toml:"-" mapstructure:"-" json:"storages"`
|
|
Parser parserConfig `toml:"parser" mapstructure:"parser" json:"parser"`
|
|
Hook hookConfig `toml:"hook" mapstructure:"hook" json:"hook"`
|
|
Ytdlp YtdlpConfig `toml:"ytdlp" mapstructure:"ytdlp" json:"ytdlp"`
|
|
}
|
|
|
|
type aria2Config struct {
|
|
Enable bool `toml:"enable" mapstructure:"enable" json:"enable"`
|
|
Url string `toml:"url" mapstructure:"url" json:"url"`
|
|
Secret string `toml:"secret" mapstructure:"secret" json:"secret"`
|
|
KeepFile bool `toml:"keep_file" mapstructure:"keep_file" json:"keep_file"`
|
|
}
|
|
|
|
type apiConfig struct {
|
|
Enable bool `toml:"enable" mapstructure:"enable" json:"enable"`
|
|
Host string `toml:"host" mapstructure:"host" json:"host"`
|
|
Port int `toml:"port" mapstructure:"port" json:"port"`
|
|
Token string `toml:"token" mapstructure:"token" json:"token"`
|
|
}
|
|
|
|
var cfg = &Config{}
|
|
|
|
func C() Config {
|
|
return *cfg
|
|
}
|
|
|
|
func (c Config) GetStorageByName(name string) storage.StorageConfig {
|
|
for _, storage := range c.Storages {
|
|
if storage.GetName() == name {
|
|
return storage
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Init(ctx context.Context, configFile ...string) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
// Reset side tables for re-init.
|
|
storages = nil
|
|
userIDs = nil
|
|
userStorages = make(map[int64][]string)
|
|
|
|
viper.SetConfigType("toml")
|
|
viper.SetEnvPrefix("SAVEANY")
|
|
viper.AutomaticEnv()
|
|
replacer := strings.NewReplacer(".", "_")
|
|
viper.SetEnvKeyReplacer(replacer)
|
|
|
|
// 如果指定了配置文件路径,则使用指定的配置文件
|
|
// 配置文件支持传入一个 http(s) URL 地址
|
|
loadedFromURL := false
|
|
if len(configFile) > 0 && configFile[0] != "" {
|
|
cfg := configFile[0]
|
|
if strings.HasPrefix(cfg, "http://") || strings.HasPrefix(cfg, "https://") {
|
|
// 使用远程配置文件
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
resp, err := client.Get(cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch remote config file: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("failed to fetch remote config file: status code %d", resp.StatusCode)
|
|
}
|
|
if err := viper.ReadConfig(resp.Body); err != nil {
|
|
return fmt.Errorf("failed to read remote config file: %w", err)
|
|
}
|
|
loadedFromURL = true
|
|
} else {
|
|
viper.SetConfigFile(cfg)
|
|
}
|
|
} else {
|
|
viper.SetConfigName("config")
|
|
viper.AddConfigPath(".")
|
|
viper.AddConfigPath("/etc/saveany/")
|
|
}
|
|
|
|
defaultConfigs := map[string]any{
|
|
// 基础配置
|
|
"lang": "zh-Hans",
|
|
"workers": 3,
|
|
"retry": 3,
|
|
"threads": 4,
|
|
"log.level": "debug",
|
|
|
|
// 缓存配置
|
|
"cache.ttl": 86400,
|
|
"cache.num_counters": 1e5,
|
|
"cache.max_cost": 1e6,
|
|
|
|
// Telegram
|
|
"telegram.app_id": 1025907,
|
|
"telegram.app_hash": "452b0359b988148995f22ff0f4229750",
|
|
"telegram.rpc_retry": 5,
|
|
"telegram.userbot.enable": false,
|
|
"telegram.userbot.session": "data/usersession.db",
|
|
|
|
// 临时目录
|
|
"temp.base_path": "cache/",
|
|
|
|
// 数据库
|
|
"db.path": "data/saveany.db",
|
|
"db.session": "data/session.db",
|
|
|
|
// API
|
|
"api.enable": false,
|
|
"api.host": "0.0.0.0",
|
|
"api.port": 8080,
|
|
"api.token": "",
|
|
|
|
// yt-dlp
|
|
"ytdlp.recode": "mp4",
|
|
}
|
|
|
|
for key, value := range defaultConfigs {
|
|
viper.SetDefault(key, value)
|
|
}
|
|
|
|
if !loadedFromURL {
|
|
if err := viper.ReadInConfig(); err != nil {
|
|
logger.Errorf("Error reading config file: %v", err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := viper.Unmarshal(cfg); err != nil {
|
|
logger.Errorf("Error unmarshalling config file: %v", err)
|
|
return err
|
|
}
|
|
|
|
storagesConfig, err := storage.LoadStorageConfigs(viper.GetViper())
|
|
if err != nil {
|
|
return fmt.Errorf("error loading storage configs: %w", err)
|
|
}
|
|
cfg.Storages = storagesConfig
|
|
|
|
storageNames := make(map[string]struct{})
|
|
for _, storage := range cfg.Storages {
|
|
if _, ok := storageNames[storage.GetName()]; ok {
|
|
return fmt.Errorf("duplicate storage name: %s", storage.GetName())
|
|
}
|
|
storageNames[storage.GetName()] = struct{}{}
|
|
}
|
|
|
|
if cfg.Workers < 1 {
|
|
cfg.Workers = 1
|
|
}
|
|
if cfg.Threads < 1 {
|
|
cfg.Threads = 1
|
|
}
|
|
if cfg.Retry < 1 {
|
|
cfg.Retry = 1
|
|
}
|
|
|
|
for _, storage := range cfg.Storages {
|
|
storages = append(storages, storage.GetName())
|
|
}
|
|
for _, user := range cfg.Users {
|
|
userIDs = append(userIDs, user.ID)
|
|
if user.Blacklist {
|
|
userStorages[user.ID] = slice.Compact(slice.Difference(storages, user.Storages))
|
|
} else {
|
|
userStorages[user.ID] = user.Storages
|
|
}
|
|
}
|
|
if cfg.Proxy != "" {
|
|
http.DefaultTransport, err = newProxyTransport(cfg.Proxy)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create proxy transport: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newProxyTransport(proxyStr string) (*http.Transport, error) {
|
|
proxyURL, err := url.Parse(proxyStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
transport := &http.Transport{
|
|
ForceAttemptHTTP2: true,
|
|
MaxIdleConns: 100,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
}
|
|
switch proxyURL.Scheme {
|
|
case "http", "https":
|
|
transport.Proxy = http.ProxyURL(proxyURL)
|
|
|
|
case "socks5", "socks5h":
|
|
dialer, err := proxy.FromURL(proxyURL, proxy.Direct)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
return dialer.(proxy.ContextDialer).DialContext(ctx, network, addr)
|
|
}
|
|
|
|
default:
|
|
return nil, fmt.Errorf("unsupported proxy type: %s", proxyURL.Scheme)
|
|
}
|
|
|
|
return transport, nil
|
|
}
|