feat: add directory watching and auto-upload functionality (#219)

cmd/watch/uploader.go

@@ -0,0 +1,227 @@
+package watch
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/charmbracelet/log"
+	"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
+	"github.com/krau/SaveAny-Bot/storage"
+)
+
+type uploadJob struct {
+	// localPath is the absolute path of the local file.
+	localPath string
+	// relPath is relative to the watch root, used to preserve directory structure on storage.
+	relPath string
+}
+
+// Uploader uploads local files to the target storage via a worker pool.
+// If a file changes while being uploaded, it is re-uploaded once after the
+// current upload finishes, instead of being queued multiple times.
+type Uploader struct {
+	stor       storage.Storage
+	destDir    string
+	overwrite  bool
+	retry      int
+	retryDelay time.Duration
+	logger     *log.Logger
+
+	jobs chan uploadJob
+	wg   sync.WaitGroup
+
+	mu sync.Mutex
+	// inflight maps in-progress (or queued) file paths. A true value means the
+	// file changed during upload and must be re-queued once done.
+	inflight map[string]bool
+}
+
+type UploaderOptions struct {
+	Storage    storage.Storage
+	DestDir    string
+	Overwrite  bool
+	Workers    int
+	Retry      int
+	RetryDelay time.Duration
+	QueueSize  int
+}
+
+// NewUploader creates and starts an Uploader. The caller must call Close when done.
+func NewUploader(ctx context.Context, opts UploaderOptions) *Uploader {
+	if opts.Workers < 1 {
+		opts.Workers = 1
+	}
+	if opts.Retry < 1 {
+		opts.Retry = 1
+	}
+	if opts.RetryDelay <= 0 {
+		opts.RetryDelay = 3 * time.Second
+	}
+	if opts.QueueSize < opts.Workers {
+		opts.QueueSize = opts.Workers * 64
+	}
+
+	u := &Uploader{
+		stor:       opts.Storage,
+		destDir:    opts.DestDir,
+		overwrite:  opts.Overwrite,
+		retry:      opts.Retry,
+		retryDelay: opts.RetryDelay,
+		logger:     log.FromContext(ctx).WithPrefix("uploader"),
+		jobs:       make(chan uploadJob, opts.QueueSize),
+		inflight:   make(map[string]bool),
+	}
+
+	for i := 0; i < opts.Workers; i++ {
+		u.wg.Add(1)
+		go u.worker(ctx)
+	}
+
+	return u
+}
+
+// Submit enqueues an upload job. If the same file is already in flight, it is
+// marked for re-upload instead of being queued again. Returns false if ctx is
+// cancelled before the job can be enqueued.
+func (u *Uploader) Submit(ctx context.Context, job uploadJob) bool {
+	u.mu.Lock()
+	if _, ok := u.inflight[job.localPath]; ok {
+		u.inflight[job.localPath] = true
+		u.mu.Unlock()
+		u.logger.Debugf("file %s already in flight, marked for re-upload", job.localPath)
+		return true
+	}
+	u.inflight[job.localPath] = false
+	u.mu.Unlock()
+
+	select {
+	case u.jobs <- job:
+		return true
+	case <-ctx.Done():
+		u.mu.Lock()
+		delete(u.inflight, job.localPath)
+		u.mu.Unlock()
+		return false
+	}
+}
+
+func (u *Uploader) worker(ctx context.Context) {
+	defer u.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case job, ok := <-u.jobs:
+			if !ok {
+				return
+			}
+			u.process(ctx, job)
+		}
+	}
+}
+
+func (u *Uploader) process(ctx context.Context, job uploadJob) {
+	if err := u.uploadWithRetry(ctx, job); err != nil {
+		if ctx.Err() != nil {
+			u.clearInflight(job.localPath)
+			return
+		}
+		u.logger.Errorf("failed to upload %s after %d attempt(s): %v", job.localPath, u.retry, err)
+	}
+
+	// Re-queue if the file changed again while it was being uploaded.
+	u.mu.Lock()
+	needReupload := u.inflight[job.localPath]
+	if needReupload {
+		u.inflight[job.localPath] = false
+	} else {
+		delete(u.inflight, job.localPath)
+	}
+	u.mu.Unlock()
+
+	if needReupload {
+		select {
+		case u.jobs <- job:
+			u.logger.Debugf("re-queued %s due to changes during upload", job.localPath)
+		case <-ctx.Done():
+			u.clearInflight(job.localPath)
+		}
+	}
+}
+
+func (u *Uploader) clearInflight(localPath string) {
+	u.mu.Lock()
+	delete(u.inflight, localPath)
+	u.mu.Unlock()
+}
+
+func (u *Uploader) uploadWithRetry(ctx context.Context, job uploadJob) error {
+	var lastErr error
+	for attempt := 1; attempt <= u.retry; attempt++ {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		err := u.upload(ctx, job)
+		if err == nil {
+			return nil
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		lastErr = err
+		u.logger.Warnf("upload %s failed (attempt %d/%d): %v", job.localPath, attempt, u.retry, err)
+		if attempt < u.retry {
+			select {
+			case <-time.After(u.retryDelay):
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+	}
+	return lastErr
+}
+
+func (u *Uploader) upload(ctx context.Context, job uploadJob) error {
+	file, err := os.Open(filepath.Clean(job.localPath))
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat file: %w", err)
+	}
+	if info.IsDir() {
+		return fmt.Errorf("path is a directory, not a file")
+	}
+
+	// Keep the relative directory structure on the storage side.
+	storagePath := path.Join(u.destDir, filepath.ToSlash(job.relPath))
+
+	uploadCtx := context.WithValue(ctx, ctxkey.ContentLength, info.Size())
+	if u.overwrite {
+		uploadCtx = storage.WithOverwrite(uploadCtx)
+	} else if u.stor.Exists(uploadCtx, storagePath) {
+		u.logger.Infof("skip existing file: %s", storagePath)
+		return nil
+	}
+
+	u.logger.Infof("uploading %s -> %s (%d bytes)", job.localPath, storagePath, info.Size())
+	if err := u.stor.Save(uploadCtx, file, storagePath); err != nil {
+		return fmt.Errorf("failed to save to storage: %w", err)
+	}
+	u.logger.Infof("uploaded %s", storagePath)
+	return nil
+}
+
+// Close stops accepting jobs and waits for in-flight uploads to finish.
+func (u *Uploader) Close() {
+	close(u.jobs)
+	u.wg.Wait()
+}