feat: support import files from storages to telegram (#183)

* feat: add import command and batch import functionality - Implemented the `/import` command to allow users to import files from storage to Telegram. - Added support for listing files in storage and filtering based on regex patterns. - Created a batch import task to handle multiple file uploads concurrently. - Introduced progress tracking for batch imports, providing real-time updates to users. - Enhanced storage interfaces to support file listing and reading capabilities. - Updated localization files for the new import command and its usage instructions. - Added utility functions for file size formatting and speed calculation. - Refactored Telegram storage handling to support reading from non-seekable streams. * feat: add i18n for import command * feat: implement ListFiles and OpenFile methods for WebDAV and Alist storage * Update common/i18n/locale/zh-Hans.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update core/tasks/batchimport/progress.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update core/tasks/batchimport/execute.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update storage/alist/alist.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update common/i18n/locale/en.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update pkg/storagetypes/fileinfo.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update common/i18n/locale/en.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update core/tasks/batchimport/execute.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update storage/webdav/webdav.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update storage/telegram/telegram.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update core/tasks/batchimport/execute.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update storage/webdav/webdav.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: missing progress stats i18n * refactor: use strutil to parse args * chore: update generated code files for consistency --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-05-11 22:19:40 +08:00 · 2026-01-19 17:39:47 +08:00
parent 3e20dc2c5f
commit f92c43b9c8
22 changed files with 1396 additions and 72 deletions
--- a/storage/webdav/client.go
+++ b/storage/webdav/client.go
@@ -2,6 +2,7 @@ package webdav

 import (
 	"context"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
@@ -25,8 +26,40 @@ const (
 	WebdavMethodMkcol    WebdavMethod = "MKCOL"
 	WebdavMethodPropfind WebdavMethod = "PROPFIND"
 	WebdavMethodPut      WebdavMethod = "PUT"
+	WebdavMethodGet      WebdavMethod = "GET"
 )

+// WebDAV XML structures for PROPFIND response
+type Multistatus struct {
+	XMLName   xml.Name   `xml:"multistatus"`
+	Responses []Response `xml:"response"`
+}
+
+type Response struct {
+	Href     string   `xml:"href"`
+	Propstat Propstat `xml:"propstat"`
+}
+
+type Propstat struct {
+	Prop Prop   `xml:"prop"`
+	Status string `xml:"status"`
+}
+
+type Prop struct {
+	ResourceType  ResourceType `xml:"resourcetype"`
+	GetContentLength int64     `xml:"getcontentlength"`
+	GetLastModified  string    `xml:"getlastmodified"`
+	DisplayName      string    `xml:"displayname"`
+}
+
+type ResourceType struct {
+	Collection *struct{} `xml:"collection"`
+}
+
+func (rt ResourceType) IsCollection() bool {
+	return rt.Collection != nil
+}
+
 func NewClient(baseURL, username, password string, httpClient *http.Client) *Client {
 	if !strings.HasSuffix(baseURL, "/") {
 		baseURL += "/"
@@ -131,5 +164,79 @@ func (c *Client) WriteFile(ctx context.Context, remotePath string, content io.Re
 		return nil
 	}
 	return fmt.Errorf("PUT: %s", resp.Status)
-
+}
+
+// ListDir lists files and directories in the given path
+func (c *Client) ListDir(ctx context.Context, dirPath string) ([]Response, error) {
+	dirPath = strings.Trim(dirPath, "/")
+	u, err := url.Parse(c.BaseURL)
+	if err != nil {
+		return nil, err
+	}
+	u.Path = path.Join(u.Path, dirPath)
+	if !strings.HasSuffix(u.Path, "/") {
+		u.Path += "/"
+	}
+
+	resp, err := c.doRequest(ctx, WebdavMethodPropfind, u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusMultiStatus {
+		return nil, fmt.Errorf("PROPFIND: %s", resp.Status)
+	}
+
+	var multistatus Multistatus
+	if err := xml.NewDecoder(resp.Body).Decode(&multistatus); err != nil {
+		return nil, fmt.Errorf("failed to decode PROPFIND response: %w", err)
+	}
+
+	// Filter out the directory itself from results
+	var results []Response
+	basePath := u.Path
+	for _, r := range multistatus.Responses {
+		decodedHref, err := url.PathUnescape(r.Href)
+		if err != nil {
+			decodedHref = r.Href
+		}
+		// Skip the directory itself
+		if strings.TrimSuffix(decodedHref, "/") == strings.TrimSuffix(basePath, "/") {
+			continue
+		}
+		results = append(results, r)
+	}
+
+	return results, nil
+}
+
+// ReadFile downloads a file and returns a ReadCloser
+func (c *Client) ReadFile(ctx context.Context, filePath string) (io.ReadCloser, int64, error) {
+	filePath = strings.Trim(filePath, "/")
+	u, err := url.Parse(c.BaseURL)
+	if err != nil {
+		return nil, 0, err
+	}
+	u.Path = path.Join(u.Path, filePath)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	if c.Username != "" && c.Password != "" {
+		req.SetBasicAuth(c.Username, c.Password)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, 0, fmt.Errorf("GET: %s", resp.Status)
+	}
+
+	return resp.Body, resp.ContentLength, nil
 }
--- a/storage/webdav/webdav.go
+++ b/storage/webdav/webdav.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/charmbracelet/log"
 	config "github.com/krau/SaveAny-Bot/config/storage"
 	storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
+	"github.com/krau/SaveAny-Bot/pkg/storagetypes"
 	"github.com/rs/xid"
 )

@@ -84,3 +86,77 @@ func (w *Webdav) Exists(ctx context.Context, storagePath string) bool {
 	}
 	return exists
 }
+
+// ListFiles implements storage.StorageListable
+func (w *Webdav) ListFiles(ctx context.Context, dirPath string) ([]storagetypes.FileInfo, error) {
+	w.logger.Infof("Listing files in %s", dirPath)
+	
+	// Join with base path
+	fullPath := path.Join(w.config.BasePath, dirPath)
+	
+	responses, err := w.client.ListDir(ctx, fullPath)
+	if err != nil {
+		w.logger.Errorf("Failed to list directory %s: %v", fullPath, err)
+		return nil, fmt.Errorf("failed to list directory: %w", err)
+	}
+
+	files := make([]storagetypes.FileInfo, 0, len(responses))
+	for _, resp := range responses {
+		// Parse the href to get the file name
+		decodedHref, err := url.PathUnescape(resp.Href)
+		if err != nil {
+			w.logger.Warnf("Failed to unescape href %q: %v; using original value", resp.Href, err)
+			decodedHref = resp.Href
+		}
+		
+		// Extract filename from href
+		name := path.Base(strings.TrimSuffix(decodedHref, "/"))
+		if name == "" || name == "." {
+			continue
+		}
+
+		// Parse modification time
+		var modTime time.Time
+		if resp.Propstat.Prop.GetLastModified != "" {
+			// Try RFC1123 format (standard for WebDAV)
+			parsedTime, err := time.Parse(time.RFC1123, resp.Propstat.Prop.GetLastModified)
+			if err != nil {
+				w.logger.Warnf("Failed to parse last modified time %q for %s: %v", resp.Propstat.Prop.GetLastModified, decodedHref, err)
+			} else {
+				modTime = parsedTime
+			}
+		}
+
+		isDir := resp.Propstat.Prop.ResourceType.IsCollection()
+		
+		fileInfo := storagetypes.FileInfo{
+			Name:    name,
+			Path:    strings.TrimPrefix(decodedHref, w.config.BasePath),
+			Size:    resp.Propstat.Prop.GetContentLength,
+			IsDir:   isDir,
+			ModTime: modTime,
+		}
+		
+		files = append(files, fileInfo)
+	}
+
+	w.logger.Debugf("Found %d files/directories in %s", len(files), dirPath)
+	return files, nil
+}
+
+// OpenFile implements storage.StorageReadable
+func (w *Webdav) OpenFile(ctx context.Context, filePath string) (io.ReadCloser, int64, error) {
+	w.logger.Infof("Opening file %s", filePath)
+	
+	// Join with base path
+	fullPath := path.Join(w.config.BasePath, filePath)
+	
+	reader, size, err := w.client.ReadFile(ctx, fullPath)
+	if err != nil {
+		w.logger.Errorf("Failed to open file %s: %v", fullPath, err)
+		return nil, 0, fmt.Errorf("failed to open file: %w", err)
+	}
+
+	w.logger.Debugf("Opened file %s (size: %d bytes)", filePath, size)
+	return reader, size, nil
+}