diff --git a/config/storage/telegram.go b/config/storage/telegram.go index e7e7861..c897f80 100644 --- a/config/storage/telegram.go +++ b/config/storage/telegram.go @@ -8,14 +8,16 @@ import ( type TelegramStorageConfig struct { BaseConfig - ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"` - ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"` - RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"` - RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"` - SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB) - // split files larger than Telegram limit(2GB) into parts of specified size, in MB, leave 0 to set default(2000MB) + ChatID int64 `toml:"chat_id" mapstructure:"chat_id" json:"chat_id"` + ForceFile bool `toml:"force_file" mapstructure:"force_file" json:"force_file"` + RateLimit int `toml:"rate_limit" mapstructure:"rate_limit" json:"rate_limit"` + RateBurst int `toml:"rate_burst" mapstructure:"rate_burst" json:"rate_burst"` + SkipLarge bool `toml:"skip_large" mapstructure:"skip_large" json:"skip_large"` // skip files larger than Telegram limit(2GB) + SplitLargeVideo bool `toml:"split_large_video" mapstructure:"split_large_video" json:"split_large_video"` + // split files larger than the uploader account limit into parts of specified size, in MB + // leave 0 to use the account limit (2000MB for bots/regular users, 4000MB for Premium users) // only effective when SkipLarge is false - // use zip when splitting + // use zip when splitting non-video files or when lossless video splitting is disabled/unavailable SplitSizeMB int64 `toml:"split_size_mb" mapstructure:"split_size_mb" json:"split_size_mb"` } diff --git a/core/tasks/tfile/caption_test.go b/core/tasks/tfile/caption_test.go new file mode 100644 index 0000000..6326f1d --- /dev/null +++ b/core/tasks/tfile/caption_test.go @@ -0,0 +1,42 @@ +package tfile + +import ( + "testing" + + "github.com/gotd/td/tg" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" +) + +func TestSourceCaption(t *testing.T) { + tests := []struct { + name string + file tfilepkg.TGFile + want string + ok bool + }{ + { + name: "original caption", + file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{Message: "original caption"})), + want: "original caption", + ok: true, + }, + { + name: "empty caption suppresses storage fallback", + file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov", tfilepkg.WithMessage(&tg.Message{})), + ok: true, + }, + { + name: "file without source message", + file: tfilepkg.NewTGFile(nil, nil, 0, "video.mov"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := sourceCaption(tt.file) + if ok != tt.ok || got != tt.want { + t.Fatalf("sourceCaption() = (%q, %v), want (%q, %v)", got, ok, tt.want, tt.ok) + } + }) + } +} diff --git a/core/tasks/tfile/execute.go b/core/tasks/tfile/execute.go index 4efbfc0..d1c6098 100644 --- a/core/tasks/tfile/execute.go +++ b/core/tasks/tfile/execute.go @@ -12,6 +12,8 @@ import ( "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" + "github.com/krau/SaveAny-Bot/pkg/storagetypes" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" ) func (t *Task) Execute(ctx context.Context) error { @@ -57,6 +59,9 @@ func (t *Task) Execute(ctx context.Context) error { return fmt.Errorf("failed to get file stat: %w", err) } vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size()) + if caption, ok := sourceCaption(t.File); ok { + vctx = storagetypes.WithSourceCaption(vctx, caption) + } err = retry.Retry(func() error { file, err := os.Open(t.localPath) if err != nil { @@ -73,3 +78,11 @@ func (t *Task) Execute(ctx context.Context) error { } return nil } + +func sourceCaption(file tfilepkg.TGFile) (string, bool) { + messageFile, ok := file.(tfilepkg.TGFileMessage) + if !ok || messageFile.Message() == nil { + return "", false + } + return messageFile.Message().GetMessage(), true +} diff --git a/docs/content/en/deployment/configuration/storages.md b/docs/content/en/deployment/configuration/storages.md index 73daed6..44d7780 100644 --- a/docs/content/en/deployment/configuration/storages.md +++ b/docs/content/en/deployment/configuration/storages.md @@ -79,7 +79,8 @@ Stream mode is not supported. chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat force_file = false # Force sending as file, default is false skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded. -spilt_size_mb = 2000 # Split size in MB, default is 2000 MB (2 GB). Files larger than this will be split into multiple parts (zip format). Ignored when skip_large is true. +split_large_video = false # Losslessly split oversized videos into one album of up to 10 independently playable parts. Falls back to ZIP parts on failure or when more than 10 parts are required. +split_size_mb = 0 # Split size in MB. 0 uses the uploader account limit: 2000 MB for bots/regular users and 4000 MB for Premium users. Oversized non-video files use ZIP parts. Ignored when skip_large is true. ``` ## Rclone @@ -136,4 +137,4 @@ remote = "myremote" base_path = "/backup" config_path = "/path/to/rclone.conf" flags = ["--progress"] -``` \ No newline at end of file +``` diff --git a/docs/content/zh/deployment/configuration/storages.md b/docs/content/zh/deployment/configuration/storages.md index 37a8564..4dce946 100644 --- a/docs/content/zh/deployment/configuration/storages.md +++ b/docs/content/zh/deployment/configuration/storages.md @@ -82,10 +82,12 @@ chat_id = "123456789" force_file = false # 是否跳过大文件, 默认为 false. 如果启用, 超过 Telegram 限制的文件将不会上传. skip_large = false -# 分卷大小, 单位 MB, 默认为 2000 MB (2 GB). -# 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式) +# 超限视频是否使用 FFmpeg 无损分割成一个媒体组（最多 10 个）内可独立播放的小视频；失败或需要超过 10 段时回退到 ZIP 分卷. +split_large_video = false +# 分卷大小, 单位 MB. 设为 0 时使用实际上传账号的限制：Bot/普通用户为 2000 MB，Premium 用户为 4000 MB. +# 超过该大小的文件将被分割成多个部分上传；非视频使用 ZIP 格式. # 当 skip_large 启用时, 该选项无效. -spilt_size_mb = 2000 +split_size_mb = 0 ``` ## Rclone @@ -142,4 +144,4 @@ remote = "myremote" base_path = "/backup" config_path = "/path/to/rclone.conf" flags = ["--progress"] -``` \ No newline at end of file +``` diff --git a/pkg/storagetypes/context.go b/pkg/storagetypes/context.go new file mode 100644 index 0000000..f90d3bd --- /dev/null +++ b/pkg/storagetypes/context.go @@ -0,0 +1,19 @@ +package storagetypes + +import "context" + +type sourceCaptionContextKey struct{} + +// WithSourceCaption records the source Telegram message caption for storage +// backends that can preserve it. Calling this function with an empty caption +// intentionally suppresses a backend-generated fallback caption. +func WithSourceCaption(ctx context.Context, caption string) context.Context { + return context.WithValue(ctx, sourceCaptionContextKey{}, caption) +} + +// SourceCaptionFromContext returns the source caption and whether the caller +// explicitly supplied one. +func SourceCaptionFromContext(ctx context.Context) (string, bool) { + caption, ok := ctx.Value(sourceCaptionContextKey{}).(string) + return caption, ok +} diff --git a/pkg/storagetypes/context_test.go b/pkg/storagetypes/context_test.go new file mode 100644 index 0000000..0e069eb --- /dev/null +++ b/pkg/storagetypes/context_test.go @@ -0,0 +1,24 @@ +package storagetypes + +import ( + "context" + "testing" +) + +func TestSourceCaptionContext(t *testing.T) { + if _, ok := SourceCaptionFromContext(context.Background()); ok { + t.Fatal("caption unexpectedly present on empty context") + } + + tests := []string{"original caption", ""} + for _, want := range tests { + ctx := WithSourceCaption(context.Background(), want) + got, ok := SourceCaptionFromContext(ctx) + if !ok { + t.Fatalf("caption %q was not marked as present", want) + } + if got != want { + t.Fatalf("caption = %q, want %q", got, want) + } + } +} diff --git a/storage/telegram/telegram.go b/storage/telegram/telegram.go index 6938e87..b86cad7 100644 --- a/storage/telegram/telegram.go +++ b/storage/telegram/telegram.go @@ -34,8 +34,9 @@ import ( const ( // https://core.telegram.org/api/config#upload-max-fileparts-default - DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB - MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB + DefaultSplitSize = 4000 * 524288 // 4000 * 512 KB + MaxUploadFileSize = 4000 * 524288 // 4000 * 512 KB + PremiumMaxUploadFileSize = 8000 * 524288 // 8000 * 512 KB ) type Telegram struct { @@ -86,17 +87,21 @@ func (t *Telegram) Exists(ctx context.Context, storagePath string) bool { } func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) error { + storagePath = path.Clean(storagePath) + captionOverride := sourceCaptionOverride(ctx) tctx := tgutil.ExtFromContext(ctx) if tctx == nil { return fmt.Errorf("failed to get telegram context") } size := contentLength(ctx) - if t.config.SkipLarge && size > MaxUploadFileSize { - log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", MaxUploadFileSize, size) + maxUploadSize := maxUploadFileSize(tctx) + if t.config.SkipLarge && size > maxUploadSize { + log.FromContext(ctx).Warnf("Skipping file larger than Telegram limit (%d bytes): %d bytes", maxUploadSize, size) return nil } - if size > t.splitSize() { - filename, chatID := t.target(tctx, path.Clean(storagePath)) + splitSize := t.splitSize(maxUploadSize) + if size > splitSize { + filename, chatID := t.target(tctx, storagePath) if filename == "" { if rs, ok := r.(io.ReadSeeker); ok { mtype, err := mimetype.DetectReader(rs) @@ -117,7 +122,43 @@ func (t *Telegram) Save(ctx context.Context, r io.Reader, storagePath string) er if err := t.limiter.Wait(ctx); err != nil { return fmt.Errorf("rate limit failed: %w", err) } - return t.splitUpload(tctx, r, filename, upler, peer, size, t.splitSize()) + if t.config.SplitLargeVideo { + rs, ok := r.(io.ReadSeeker) + if ok { + mtype, detectErr := mimetype.DetectReader(rs) + if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil { + return fmt.Errorf("failed to seek large file after mimetype detection: %w", seekErr) + } + if detectErr != nil { + log.FromContext(ctx).Warnf("Failed to detect large file type, falling back to ZIP split: %s", detectErr) + } else if strings.HasPrefix(mtype.String(), "video/") { + parts, cleanup, splitErr := createLosslessVideoParts( + ctx, + rs, + filename, + size, + splitSize, + ) + if splitErr != nil { + if ctx.Err() != nil { + return ctx.Err() + } + log.FromContext(ctx).Warnf("Lossless video split failed, falling back to ZIP split: %s", splitErr) + } else { + defer cleanup() + log.FromContext(ctx).Infof("Uploading oversized video as %d lossless-playable parts", len(parts)) + for _, part := range parts { + log.FromContext(ctx).Infof("Prepared lossless video part %s (%d bytes)", part.Name, part.Size) + } + return t.uploadLosslessVideoParts(ctx, tctx, storagePath, parts, captionOverride) + } + if _, seekErr := rs.Seek(0, io.SeekStart); seekErr != nil { + return fmt.Errorf("failed to seek large video before ZIP fallback: %w", seekErr) + } + } + } + } + return t.splitUpload(tctx, r, filename, upler, peer, size, splitSize) } if err := t.limiter.Wait(ctx); err != nil { @@ -143,12 +184,27 @@ func contentLength(ctx context.Context) int64 { return -1 } -func (t *Telegram) splitSize() int64 { +func sourceCaptionOverride(ctx context.Context) *string { + caption, ok := storagetypes.SourceCaptionFromContext(ctx) + if !ok { + return nil + } + return &caption +} + +func maxUploadFileSize(tctx *ext.Context) int64 { + if tctx != nil && tctx.Self != nil && !tctx.Self.GetBot() && tctx.Self.GetPremium() { + return PremiumMaxUploadFileSize + } + return MaxUploadFileSize +} + +func (t *Telegram) splitSize(maxUploadSize int64) int64 { splitSize := t.config.SplitSizeMB * 1024 * 1024 if splitSize <= 0 { - return DefaultSplitSize + return maxUploadSize } - return splitSize + return min(splitSize, maxUploadSize) } func (t *Telegram) target(tctx *ext.Context, storagePath string) (string, int64) { @@ -310,7 +366,9 @@ func (t *Telegram) SaveBatch(ctx context.Context, items []storagetypes.BatchItem func (t *Telegram) inspectBatchItem(tctx *ext.Context, item storagetypes.BatchItem) (batchMediaItem, error) { _, chatID := t.target(tctx, path.Clean(item.StoragePath)) result := batchMediaItem{item: item, chatID: chatID} - if (t.config.SkipLarge && item.Size > MaxUploadFileSize) || item.Size > t.splitSize() { + maxUploadSize := maxUploadFileSize(tctx) + if (t.config.SkipLarge && item.Size > maxUploadSize) || + item.Size > t.splitSize(maxUploadSize) { result.useSingleSave = true return result, nil } @@ -362,6 +420,9 @@ func (t *Telegram) saveMediaGroup(ctx context.Context, tctx *ext.Context, group return fmt.Errorf("failed to seek batch item: %w", err) } itemCtx := context.WithValue(ctx, ctxkey.ContentLength, item.Size) + if item.PreserveCaption { + itemCtx = storagetypes.WithSourceCaption(itemCtx, item.Caption) + } return t.Save(itemCtx, item.Reader, item.StoragePath) } if err := t.limiter.Wait(ctx); err != nil { diff --git a/storage/telegram/upload_limit_test.go b/storage/telegram/upload_limit_test.go new file mode 100644 index 0000000..9038fc4 --- /dev/null +++ b/storage/telegram/upload_limit_test.go @@ -0,0 +1,119 @@ +package telegram + +import ( + "bytes" + "testing" + + "github.com/celestix/gotgproto/ext" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/pkg/storagetypes" +) + +func TestMaxUploadFileSize(t *testing.T) { + tests := []struct { + name string + ctx *ext.Context + want int64 + }{ + {name: "missing context", want: MaxUploadFileSize}, + {name: "missing self", ctx: new(ext.Context), want: MaxUploadFileSize}, + { + name: "bot", + ctx: uploadAccountContext(true, false), + want: MaxUploadFileSize, + }, + { + name: "premium bot still uses bot limit", + ctx: uploadAccountContext(true, true), + want: MaxUploadFileSize, + }, + { + name: "regular user", + ctx: uploadAccountContext(false, false), + want: MaxUploadFileSize, + }, + { + name: "premium user", + ctx: uploadAccountContext(false, true), + want: PremiumMaxUploadFileSize, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maxUploadFileSize(tt.ctx); got != tt.want { + t.Fatalf("maxUploadFileSize() = %d, want %d", got, tt.want) + } + }) + } +} + +func uploadAccountContext(bot, premium bool) *ext.Context { + self := new(tg.User) + self.SetBot(bot) + self.SetPremium(premium) + return &ext.Context{Self: self} +} + +func TestSplitSizeUsesUploaderLimit(t *testing.T) { + tests := []struct { + name string + splitSizeMB int64 + accountLimit int64 + want int64 + }{ + { + name: "bot default", + accountLimit: MaxUploadFileSize, + want: MaxUploadFileSize, + }, + { + name: "premium default", + accountLimit: PremiumMaxUploadFileSize, + want: PremiumMaxUploadFileSize, + }, + { + name: "explicit lower limit", + splitSizeMB: 1500, + accountLimit: PremiumMaxUploadFileSize, + want: 1500 * 1024 * 1024, + }, + { + name: "explicit limit is capped by account", + splitSizeMB: 5000, + accountLimit: PremiumMaxUploadFileSize, + want: PremiumMaxUploadFileSize, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + telegramStorage := Telegram{} + telegramStorage.config.SplitSizeMB = tt.splitSizeMB + if got := telegramStorage.splitSize(tt.accountLimit); got != tt.want { + t.Fatalf("splitSize() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestBatchUploadLimitAppliesPerFile(t *testing.T) { + telegramStorage := Telegram{} + tctx := uploadAccountContext(true, false) + data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") + itemSize := int64(MaxUploadFileSize/2 + 1) + + for index := 0; index < 2; index++ { + item, err := telegramStorage.inspectBatchItem(tctx, storagetypes.BatchItem{ + Reader: bytes.NewReader(data), + StoragePath: "video.mp4", + Size: itemSize, + }) + if err != nil { + t.Fatalf("inspectBatchItem() failed for item %d: %v", index, err) + } + if item.useSingleSave { + t.Fatalf("item %d was treated as oversized even though only the batch total exceeds the limit", index) + } + } +} diff --git a/storage/telegram/video_split.go b/storage/telegram/video_split.go new file mode 100644 index 0000000..fb1d1a5 --- /dev/null +++ b/storage/telegram/video_split.go @@ -0,0 +1,358 @@ +package telegram + +import ( + "context" + "fmt" + "io" + "math" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/celestix/gotgproto/ext" + "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/message" + "github.com/rs/xid" + + "github.com/krau/SaveAny-Bot/config" +) + +const ( + videoPartTargetRatio = 0.95 + videoSplitAttempts = 4 + minSegmentDuration = 1.0 + maxLosslessVideoParts = 10 +) + +type losslessVideoPart struct { + Path string + Name string + Size int64 +} + +type mediaToolRunner func(ctx context.Context, name string, args ...string) ([]byte, error) + +var runMediaTool mediaToolRunner = func(ctx context.Context, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + output, err := cmd.CombinedOutput() + if err != nil { + message := strings.TrimSpace(string(output)) + if message == "" { + return nil, err + } + return nil, fmt.Errorf("%w: %s", err, message) + } + return output, nil +} + +func createLosslessVideoParts( + ctx context.Context, + r io.ReadSeeker, + filename string, + fileSize, maxPartSize int64, +) ([]losslessVideoPart, func(), error) { + inputPath, sourceCleanup, err := sourceFile(r) + if err != nil { + return nil, func() {}, fmt.Errorf("failed to prepare seekable video source: %w", err) + } + + tempBase := config.C().Temp.BasePath + if err := os.MkdirAll(tempBase, 0o755); err != nil { + sourceCleanup() + return nil, func() {}, fmt.Errorf("failed to create video split base directory: %w", err) + } + tempDir, err := os.MkdirTemp(tempBase, "telegram-video-split-*") + if err != nil { + sourceCleanup() + return nil, func() {}, fmt.Errorf("failed to create video split directory: %w", err) + } + cleanup := func() { + if err := os.RemoveAll(tempDir); err != nil { + log.FromContext(ctx).Warnf("Failed to clean lossless video parts: %s", err) + } + sourceCleanup() + } + + parts, err := splitLosslessVideo(ctx, inputPath, tempDir, filename, fileSize, maxPartSize) + if err != nil { + cleanup() + return nil, func() {}, err + } + return parts, cleanup, nil +} + +func splitLosslessVideo( + ctx context.Context, + inputPath, outputDir, filename string, + fileSize, maxPartSize int64, +) ([]losslessVideoPart, error) { + if fileSize <= maxPartSize { + return nil, fmt.Errorf("video size %d does not exceed part limit %d", fileSize, maxPartSize) + } + if maxPartSize <= 0 { + return nil, fmt.Errorf("invalid video part limit: %d", maxPartSize) + } + + duration, err := probeMediaDuration(ctx, inputPath) + if err != nil { + return nil, fmt.Errorf("failed to probe source video duration: %w", err) + } + if duration < minSegmentDuration { + return nil, fmt.Errorf("invalid source video duration: %.3f", duration) + } + + targetSize := int64(float64(maxPartSize) * videoPartTargetRatio) + segmentDuration := initialSegmentDuration(duration, fileSize, targetSize) + extension := videoPartExtension(filename) + outputPattern := filepath.Join(outputDir, "part-%03d"+extension) + + var lastOversize int64 + for attempt := 0; attempt < videoSplitAttempts; attempt++ { + if err := clearVideoParts(outputDir); err != nil { + return nil, err + } + if err := runFFmpegSegment(ctx, inputPath, outputPattern, extension, segmentDuration); err != nil { + return nil, fmt.Errorf("failed to losslessly split video: %w", err) + } + + parts, largest, err := collectVideoParts(ctx, outputDir, filename) + if err != nil { + return nil, err + } + if len(parts) < 2 { + return nil, fmt.Errorf("video split produced %d part(s), expected at least 2", len(parts)) + } + if len(parts) > maxLosslessVideoParts { + return nil, fmt.Errorf( + "video split produced %d parts, exceeding the single-album limit of %d", + len(parts), + maxLosslessVideoParts, + ) + } + if largest <= maxPartSize { + return parts, nil + } + + lastOversize = largest + segmentDuration *= float64(targetSize) / float64(largest) + if segmentDuration < minSegmentDuration { + break + } + } + + return nil, fmt.Errorf( + "unable to keep lossless video parts below %d bytes; largest part was %d bytes", + maxPartSize, + lastOversize, + ) +} + +func initialSegmentDuration(duration float64, fileSize, targetSize int64) float64 { + partCount := math.Ceil(float64(fileSize) / float64(targetSize)) + if partCount < 2 { + partCount = 2 + } + segmentDuration := duration / partCount + if segmentDuration < minSegmentDuration { + return minSegmentDuration + } + return segmentDuration +} + +func videoPartExtension(filename string) string { + extension := strings.ToLower(filepath.Ext(filepath.Base(filename))) + switch extension { + case ".avi", ".m4v", ".mkv", ".mov", ".mp4", ".ts", ".webm": + return extension + default: + return ".mp4" + } +} + +func runFFmpegSegment( + ctx context.Context, + inputPath, outputPattern, extension string, + segmentDuration float64, +) error { + args := []string{ + "-hide_banner", + "-loglevel", "error", + "-nostdin", + "-y", + "-i", inputPath, + "-map", "0", + "-map_metadata", "0", + "-c", "copy", + "-f", "segment", + "-segment_time", strconv.FormatFloat(segmentDuration, 'f', 3, 64), + "-segment_start_number", "1", + "-reset_timestamps", "1", + "-avoid_negative_ts", "make_zero", + } + if extension == ".mp4" || extension == ".m4v" || extension == ".mov" { + args = append(args, "-segment_format_options", "movflags=+faststart") + } + args = append(args, outputPattern) + _, err := runMediaTool(ctx, "ffmpeg", args...) + return err +} + +func probeMediaDuration(ctx context.Context, filePath string) (float64, error) { + output, err := runMediaTool( + ctx, + "ffprobe", + "-v", "error", + "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", + filePath, + ) + if err != nil { + return 0, err + } + duration, err := strconv.ParseFloat(strings.TrimSpace(string(output)), 64) + if err != nil { + return 0, fmt.Errorf("invalid ffprobe duration %q: %w", strings.TrimSpace(string(output)), err) + } + return duration, nil +} + +func clearVideoParts(outputDir string) error { + entries, err := os.ReadDir(outputDir) + if err != nil { + return fmt.Errorf("failed to read video split directory: %w", err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if err := os.Remove(filepath.Join(outputDir, entry.Name())); err != nil { + return fmt.Errorf("failed to clear old video part %s: %w", entry.Name(), err) + } + } + return nil +} + +func collectVideoParts( + ctx context.Context, + outputDir, filename string, +) ([]losslessVideoPart, int64, error) { + matches, err := filepath.Glob(filepath.Join(outputDir, "part-*")) + if err != nil { + return nil, 0, fmt.Errorf("failed to list video parts: %w", err) + } + sort.Strings(matches) + if len(matches) == 0 { + return nil, 0, fmt.Errorf("ffmpeg did not produce any video parts") + } + + extension := videoPartExtension(filename) + base := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filepath.Base(filename))) + if base == "" || base == "." { + base = xid.New().String() + } + parts := make([]losslessVideoPart, 0, len(matches)) + var largest int64 + for index, match := range matches { + info, err := os.Stat(match) + if err != nil { + return nil, 0, fmt.Errorf("failed to stat video part %s: %w", match, err) + } + if info.Size() <= 0 { + return nil, 0, fmt.Errorf("video part %s is empty", match) + } + if _, err := probeMediaDuration(ctx, match); err != nil { + return nil, 0, fmt.Errorf("failed to validate video part %s: %w", match, err) + } + if info.Size() > largest { + largest = info.Size() + } + parts = append(parts, losslessVideoPart{ + Path: match, + Name: fmt.Sprintf("%s.part%03d%s", base, index+1, extension), + Size: info.Size(), + }) + } + return parts, largest, nil +} + +func partStoragePath(storagePath, partName string) string { + directory := path.Dir(path.Clean(storagePath)) + if directory == "." || directory == "/" { + return partName + } + return path.Join(directory, partName) +} + +func (t *Telegram) uploadLosslessVideoParts( + ctx context.Context, + tctx *ext.Context, + storagePath string, + parts []losslessVideoPart, + sourceCaption *string, +) error { + if len(parts) == 0 { + return fmt.Errorf("no lossless video parts to upload") + } + if len(parts) > maxLosslessVideoParts { + return fmt.Errorf( + "refusing to upload %d lossless video parts as multiple albums; maximum is %d", + len(parts), + maxLosslessVideoParts, + ) + } + + prepared := make([]preparedMedia, 0, len(parts)) + for index, part := range parts { + partFile, err := os.Open(part.Path) + if err != nil { + return fmt.Errorf("failed to open video part %s: %w", part.Name, err) + } + media, prepareErr := t.prepareMedia( + ctx, + tctx, + partFile, + partStoragePath(storagePath, part.Name), + part.Size, + videoPartCaption(sourceCaption, index), + ) + closeErr := partFile.Close() + if prepareErr != nil { + return fmt.Errorf("failed to prepare video part %s: %w", part.Name, prepareErr) + } + if closeErr != nil { + return fmt.Errorf("failed to close video part %s: %w", part.Name, closeErr) + } + prepared = append(prepared, *media) + } + + builder := tctx.Sender.WithUploader(prepared[0].uploader).To(prepared[0].peer) + if len(prepared) == 1 { + if _, err := builder.Media(ctx, prepared[0].media); err != nil { + return fmt.Errorf("failed to send video part: %w", err) + } + return nil + } + media := make([]message.MultiMediaOption, len(prepared)) + for index := range prepared { + media[index] = prepared[index].media + } + if _, err := builder.Album(ctx, media[0], media[1:]...); err != nil { + return fmt.Errorf("failed to send video parts as album: %w", err) + } + return nil +} + +func videoPartCaption(sourceCaption *string, index int) *string { + if sourceCaption == nil { + return nil + } + if index == 0 { + return sourceCaption + } + empty := "" + return &empty +} diff --git a/storage/telegram/video_split_test.go b/storage/telegram/video_split_test.go new file mode 100644 index 0000000..617b912 --- /dev/null +++ b/storage/telegram/video_split_test.go @@ -0,0 +1,174 @@ +package telegram + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestInitialSegmentDuration(t *testing.T) { + tests := []struct { + name string + duration float64 + fileSize int64 + targetSize int64 + want float64 + }{ + {name: "two balanced parts", duration: 120, fileSize: 2200, targetSize: 1900, want: 60}, + {name: "three balanced parts", duration: 120, fileSize: 3900, targetSize: 1900, want: 40}, + {name: "minimum one second", duration: 1.5, fileSize: 3900, targetSize: 1900, want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := initialSegmentDuration(tt.duration, tt.fileSize, tt.targetSize) + if got != tt.want { + t.Fatalf("initialSegmentDuration() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestVideoPartExtension(t *testing.T) { + tests := map[string]string{ + "clip.MOV": ".mov", + "clip.mkv": ".mkv", + "clip.exe": ".mp4", + "clip": ".mp4", + } + for input, want := range tests { + if got := videoPartExtension(input); got != want { + t.Fatalf("videoPartExtension(%q) = %q, want %q", input, got, want) + } + } +} + +func TestPartStoragePath(t *testing.T) { + tests := map[string]string{ + "clip.mov": "clip.part001.mov", + "123456/clip.mov": "123456/clip.part001.mov", + "/123456/clip.mov": "/123456/clip.part001.mov", + "folder/sub/clip.mov": "folder/sub/clip.part001.mov", + } + for input, want := range tests { + if got := partStoragePath(input, "clip.part001.mov"); got != want { + t.Fatalf("partStoragePath(%q) = %q, want %q", input, got, want) + } + } +} + +func TestVideoPartCaption(t *testing.T) { + if got := videoPartCaption(nil, 0); got != nil { + t.Fatalf("caption without source = %q, want nil", *got) + } + + source := "original caption" + first := videoPartCaption(&source, 0) + if first == nil || *first != source { + t.Fatalf("first part caption = %v, want %q", first, source) + } + second := videoPartCaption(&source, 1) + if second == nil || *second != "" { + t.Fatalf("second part caption = %v, want an explicit empty caption", second) + } +} + +func TestSplitLosslessVideoRetriesOversizedPart(t *testing.T) { + tempDir := t.TempDir() + inputPath := filepath.Join(tempDir, "source.mov") + if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil { + t.Fatal(err) + } + outputDir := filepath.Join(tempDir, "parts") + if err := os.Mkdir(outputDir, 0o755); err != nil { + t.Fatal(err) + } + + originalRunner := runMediaTool + t.Cleanup(func() { runMediaTool = originalRunner }) + ffmpegCalls := 0 + runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) { + switch name { + case "ffprobe": + return []byte("120.0\n"), nil + case "ffmpeg": + ffmpegCalls++ + pattern := args[len(args)-1] + sizes := []int{2001, 199} + if ffmpegCalls > 1 { + sizes = []int{1100, 1100} + } + for index, size := range sizes { + partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1) + if err := os.WriteFile(partPath, make([]byte, size), 0o600); err != nil { + return nil, err + } + } + return nil, nil + default: + return nil, fmt.Errorf("unexpected media tool %q", name) + } + } + + parts, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000) + if err != nil { + t.Fatalf("splitLosslessVideo() failed: %v", err) + } + if ffmpegCalls != 2 { + t.Fatalf("ffmpeg calls = %d, want 2", ffmpegCalls) + } + if len(parts) != 2 { + t.Fatalf("parts = %d, want 2", len(parts)) + } + if parts[0].Name != "example.part001.mov" || parts[1].Name != "example.part002.mov" { + t.Fatalf("unexpected part names: %#v", parts) + } + for _, part := range parts { + if part.Size > 2000 { + t.Fatalf("part %s exceeds limit: %d", part.Name, part.Size) + } + } +} + +func TestSplitLosslessVideoRejectsMultipleAlbums(t *testing.T) { + tempDir := t.TempDir() + inputPath := filepath.Join(tempDir, "source.mov") + if err := os.WriteFile(inputPath, []byte("source"), 0o600); err != nil { + t.Fatal(err) + } + outputDir := filepath.Join(tempDir, "parts") + if err := os.Mkdir(outputDir, 0o755); err != nil { + t.Fatal(err) + } + + originalRunner := runMediaTool + t.Cleanup(func() { runMediaTool = originalRunner }) + runMediaTool = func(ctx context.Context, name string, args ...string) ([]byte, error) { + switch name { + case "ffprobe": + return []byte("120.0\n"), nil + case "ffmpeg": + pattern := args[len(args)-1] + for index := 0; index < maxLosslessVideoParts+1; index++ { + partPath := strings.Replace(pattern, "%03d", fmt.Sprintf("%03d", index+1), 1) + if err := os.WriteFile(partPath, make([]byte, 100), 0o600); err != nil { + return nil, err + } + } + return nil, nil + default: + return nil, fmt.Errorf("unexpected media tool %q", name) + } + } + + _, err := splitLosslessVideo(t.Context(), inputPath, outputDir, "example.mov", 2200, 2000) + if err == nil { + t.Fatal("splitLosslessVideo() succeeded with more than one album of parts") + } + if !strings.Contains(err.Error(), "single-album limit") { + t.Fatalf("splitLosslessVideo() error = %q, want single-album limit", err) + } +}