import json import subprocess import threading import time from collections import OrderedDict from pathlib import Path from typing import Optional, List, Union from app.schemas.file import StorageUsage as _SchemaStorageUsage from app.schemas.workflow import FileItem as _SchemaFileItem from app.runtime.settings import RuntimeSettingsCompat settings = RuntimeSettingsCompat() from app.runtime.log import logger from app.modules.filemanager.storages import StorageBase, transfer_process from app.schemas.exception import StorageQueryError from app.schemas.types import StorageSchema from app.foundation import temporal as time_tools from app.adapters.system.host import SystemUtils _MAX_FOLDER_LOCKS = 4096 _folder_locks: OrderedDict[str, threading.Lock] = OrderedDict() _folder_locks_guard = threading.Lock() def _evict_unused_folder_locks_locked() -> None: """ 在持有全局锁表互斥锁时淘汰旧路径锁,避免大量不同目录导致锁表无限增长。 """ while len(_folder_locks) >= _MAX_FOLDER_LOCKS: for key, lock in list(_folder_locks.items()): if not lock.locked(): _folder_locks.pop(key, None) break else: break class Rclone(StorageBase): """ rclone相关操作 """ # 存储类型 schema = StorageSchema.Rclone # 支持的整理方式 transtype = { "move": "移动", "copy": "复制" } snapshot_check_folder_modtime = settings.RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME def init_storage(self): """ 初始化 """ pass def set_config(self, conf: dict): """ 设置配置 """ super().set_config(conf) filepath = conf.get("filepath") if not filepath: logger.warn("【rclone】保存配置失败:未设置配置文件路径") logger.info(f"【rclone】配置写入文件:{filepath}") path = Path(filepath) if not path.parent.exists(): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(conf.get('content'), encoding='utf-8') @staticmethod def __get_hidden_shell(): if SystemUtils.is_windows(): st = subprocess.STARTUPINFO() st.dwFlags = subprocess.STARTF_USESHOWWINDOW st.wShowWindow = subprocess.SW_HIDE return st else: return None @staticmethod def __parse_rclone_progress(line: str) -> Optional[float]: """ 解析rclone进度输出 """ if not line: return None line = line.strip() # 检查是否包含百分比 if '%' not in line: return None try: # 尝试多种进度输出格式 if 'ETA' in line: # 格式: "Transferred: 1.234M / 5.678M, 22%, 1.234MB/s, ETA 2m3s" percent_str = line.split('%')[0].split()[-1] return float(percent_str) elif 'Transferred:' in line and '100%' in line: # 传输完成 return 100.0 else: # 其他包含百分比的格式 parts = line.split() for part in parts: if '%' in part: percent_str = part.replace('%', '') return float(percent_str) except (ValueError, IndexError): pass return None def __get_rcloneitem(self, item: dict, parent: Optional[str] = "/") -> _SchemaFileItem: """ 获取rclone文件项 """ if not item: return _SchemaFileItem() if item.get("IsDir"): return _SchemaFileItem( storage=self.schema.value, type="dir", path=f"{parent}{item.get('Name')}" + "/", name=item.get("Name"), basename=item.get("Name"), modify_time=time_tools.parse_timestamp(item.get("ModTime")) ) else: return _SchemaFileItem( storage=self.schema.value, type="file", path=f"{parent}{item.get('Name')}", name=item.get("Name"), basename=Path(item.get("Name")).stem, extension=Path(item.get("Name")).suffix[1:], size=item.get("Size"), modify_time=time_tools.parse_timestamp(item.get("ModTime")) ) @staticmethod def __normalize_remote_path(path: Union[Path, str]) -> str: """ 规范化远端路径,统一目录锁键值。 """ path_str = Path(str(path or "/")).as_posix() if not path_str.startswith("/"): path_str = f"/{path_str}" if path_str != "/": path_str = path_str.rstrip("/") return path_str or "/" @staticmethod def __get_path_lock(path: Union[Path, str]) -> threading.Lock: """ 获取指定远端路径的模块级锁。 """ normalized = Rclone.__normalize_remote_path(path) with _folder_locks_guard: lock = _folder_locks.get(normalized) if lock: _folder_locks.move_to_end(normalized) return lock _evict_unused_folder_locks_locked() lock = threading.Lock() _folder_locks[normalized] = lock return lock def __wait_for_item( self, path: Path, retries: int = 3, delay: float = 0.2 ) -> Optional[_SchemaFileItem]: """ 等待目录或文件在远端可见,兼容云盘最终一致性延迟。 """ for attempt in range(retries): item = self.get_item(path) if item: return item if attempt < retries - 1: time.sleep(delay) return None def check(self) -> bool: """ 检查存储是否可用 """ try: retcode = subprocess.run( ['rclone', 'lsf', 'MP:'], startupinfo=self.__get_hidden_shell() ).returncode if retcode == 0: return True except Exception as err: logger.error(f"【rclone】存储检查失败:{err}") return False def list(self, fileitem: _SchemaFileItem) -> List[_SchemaFileItem]: """ 浏览文件 """ if fileitem.type == "file": return [fileitem] try: ret = subprocess.run( [ 'rclone', 'lsjson', f'MP:{fileitem.path}' ], capture_output=True, startupinfo=self.__get_hidden_shell() ) if ret.returncode == 0: items = json.loads(ret.stdout) return [self.__get_rcloneitem(item, parent=fileitem.path) for item in items] except Exception as err: logger.error(f"【rclone】浏览文件失败:{err}") return [] def create_folder(self, fileitem: _SchemaFileItem, name: str) -> Optional[_SchemaFileItem]: """ 创建目录 :param fileitem: 父目录 :param name: 目录名 """ path = Path(self.__normalize_remote_path(Path(fileitem.path) / name)) try: retcode = subprocess.run( [ 'rclone', 'mkdir', f'MP:{path}' ], startupinfo=self.__get_hidden_shell() ).returncode if retcode == 0: folder = self.__wait_for_item(path) if folder: return folder logger.warn(f"【rclone】目录 {path} 创建成功后暂未可见") return None folder = self.__wait_for_item(path, retries=2) if folder: logger.info(f"【rclone】目录 {path} 已存在,忽略重复创建") return folder except Exception as err: logger.error(f"【rclone】创建目录失败:{err}") folder = self.__wait_for_item(path, retries=2) if folder: logger.info(f"【rclone】目录 {path} 已存在,忽略创建异常") return folder return None def get_folder(self, path: Path) -> Optional[_SchemaFileItem]: """ 根据文件路程获取目录,不存在则创建 """ normalized = Path(self.__normalize_remote_path(path)) # 是否已存在 folder = self.get_item(normalized) if folder: return folder # 逐级查找和创建目录 fileitem = _SchemaFileItem(storage=self.schema.value, type="dir", path="/") for part in normalized.parts[1:]: current_path = Path(self.__normalize_remote_path(Path(fileitem.path) / part)) with self.__get_path_lock(current_path): dir_file = self.get_item(current_path) if not dir_file: dir_file = self.create_folder(fileitem, part) if not dir_file: logger.warn(f"【rclone】创建目录 {current_path} 失败!") return None fileitem = dir_file return fileitem def get_item(self, path: Path) -> Optional[_SchemaFileItem]: """ 获取文件或目录,不存在返回None """ try: ret = subprocess.run( [ 'rclone', 'lsjson', f'MP:{path.parent}' ], capture_output=True, startupinfo=self.__get_hidden_shell() ) if ret.returncode == 0: items = json.loads(ret.stdout) for item in items: if item.get("Name") == path.name: return self.__get_rcloneitem(item, parent=str(path.parent) + "/") return None except Exception as err: logger.debug(f"【rclone】获取文件项失败:{err}") return None def get_item_strict(self, path: Path) -> Optional[_SchemaFileItem]: """ 获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。 rclone 用退出码 3/4 表示目录/文件不存在,其余非零退出无法区分 「不存在」与「查询失败」,必须保守失败以免覆盖保护被绕过。 """ try: ret = subprocess.run( [ 'rclone', 'lsjson', f'MP:{path.parent}' ], capture_output=True, startupinfo=self.__get_hidden_shell() ) except Exception as err: raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {err}") from err if ret.returncode in (3, 4): # 目录或文件不存在,是确定结果 return None if ret.returncode != 0: errmsg = (ret.stderr or b"").decode(errors="ignore").strip() raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {errmsg}") try: items = json.loads(ret.stdout) except Exception as err: raise StorageQueryError(f"【rclone】解析查询结果失败: {path} - {err}") from err for item in items: if item.get("Name") == path.name: return self.__get_rcloneitem(item, parent=str(path.parent) + "/") return None def delete(self, fileitem: _SchemaFileItem) -> bool: """ 删除文件 """ try: retcode = subprocess.run( [ 'rclone', 'deletefile', f'MP:{fileitem.path}' ], startupinfo=self.__get_hidden_shell() ).returncode if retcode == 0: return True except Exception as err: logger.error(f"【rclone】删除文件失败:{err}") return False def rename(self, fileitem: _SchemaFileItem, name: str) -> bool: """ 重命名文件 """ try: retcode = subprocess.run( [ 'rclone', 'moveto', f'MP:{fileitem.path}', f'MP:{Path(fileitem.path).parent / name}' ], startupinfo=self.__get_hidden_shell() ).returncode if retcode == 0: return True except Exception as err: logger.error(f"【rclone】重命名文件失败:{err}") return False def download(self, fileitem: _SchemaFileItem, path: Path = None) -> Optional[Path]: """ 带实时进度显示的下载 """ local_path = self._build_download_path(fileitem, path or settings.TEMP_PATH) if not local_path: return None # 初始化进度条 logger.info(f"【rclone】开始下载: {fileitem.name} -> {local_path}") progress_callback = transfer_process(Path(fileitem.path).as_posix()) try: # 使用rclone的进度显示功能 process = subprocess.Popen( [ 'rclone', 'copyto', '--progress', # 启用进度显示 '--stats', '1s', # 每秒更新一次统计信息 f'MP:{fileitem.path}', f'{local_path}' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=self.__get_hidden_shell(), universal_newlines=True, bufsize=1 ) # 监控进度输出 last_progress = 0 for line in process.stdout: if line: # 解析rclone的进度输出 progress = self.__parse_rclone_progress(line) if progress is not None and progress > last_progress: progress_callback(progress) last_progress = progress if progress >= 100: break # 等待进程完成 retcode = process.wait() if retcode == 0: logger.info(f"【rclone】下载完成: {fileitem.name}") return local_path else: logger.error(f"【rclone】下载失败: {fileitem.name}") return None except Exception as err: logger.error(f"【rclone】下载失败: {fileitem.name} - {err}") # 删除可能部分下载的文件 if local_path.exists(): local_path.unlink() return None def upload(self, fileitem: _SchemaFileItem, path: Path, new_name: Optional[str] = None) -> Optional[_SchemaFileItem]: """ 带实时进度显示的上传 :param fileitem: 上传目录项 :param path: 本地文件路径 :param new_name: 上传后文件名 """ target_name = new_name or path.name new_path = Path(fileitem.path) / target_name # 初始化进度条 logger.info(f"【rclone】开始上传: {path} -> {new_path}") progress_callback = transfer_process(path.as_posix()) try: # 使用rclone的进度显示功能 process = subprocess.Popen( [ 'rclone', 'copyto', '--progress', # 启用进度显示 '--stats', '1s', # 每秒更新一次统计信息 path.as_posix(), f'MP:{new_path}' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=self.__get_hidden_shell(), universal_newlines=True, bufsize=1 ) # 监控进度输出 last_progress = 0 for line in process.stdout: if line: # 解析rclone的进度输出 progress = self.__parse_rclone_progress(line) if progress is not None and progress > last_progress: progress_callback(progress) last_progress = progress if progress >= 100: break # 等待进程完成 retcode = process.wait() if retcode == 0: logger.info(f"【rclone】上传完成: {target_name}") return self.get_item(new_path) else: logger.error(f"【rclone】上传失败: {target_name}") return None except Exception as err: logger.error(f"【rclone】上传失败: {target_name} - {err}") return None def detail(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]: """ 获取文件详情 """ try: ret = subprocess.run( [ 'rclone', 'lsjson', f'MP:{fileitem.path}' ], capture_output=True, startupinfo=self.__get_hidden_shell() ) if ret.returncode == 0: items = json.loads(ret.stdout) return self.__get_rcloneitem(items[0]) except Exception as err: logger.error(f"【rclone】获取文件详情失败:{err}") return None def move(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool: """ 移动文件 :param fileitem: 文件项 :param path: 目标目录 :param new_name: 新文件名 """ target_path = path / new_name # 初始化进度条 logger.info(f"【rclone】开始移动: {fileitem.path} -> {target_path}") progress_callback = transfer_process(Path(fileitem.path).as_posix()) try: # 使用rclone的进度显示功能 process = subprocess.Popen( [ 'rclone', 'moveto', '--progress', # 启用进度显示 '--stats', '1s', # 每秒更新一次统计信息 f'MP:{fileitem.path}', f'MP:{target_path}' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=self.__get_hidden_shell(), universal_newlines=True, bufsize=1 ) # 监控进度输出 last_progress = 0 for line in process.stdout: if line: # 解析rclone的进度输出 progress = self.__parse_rclone_progress(line) if progress is not None and progress > last_progress: progress_callback(progress) last_progress = progress if progress >= 100: break # 等待进程完成 retcode = process.wait() if retcode == 0: logger.info(f"【rclone】移动完成: {fileitem.name}") return True else: logger.error(f"【rclone】移动失败: {fileitem.name}") return False except Exception as err: logger.error(f"【rclone】移动失败: {fileitem.name} - {err}") return False def copy(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool: """ 复制文件 :param fileitem: 文件项 :param path: 目标目录 :param new_name: 新文件名 """ target_path = path / new_name # 初始化进度条 logger.info(f"【rclone】开始复制: {fileitem.path} -> {target_path}") progress_callback = transfer_process(Path(fileitem.path).as_posix()) try: # 使用rclone的进度显示功能 process = subprocess.Popen( [ 'rclone', 'copyto', '--progress', # 启用进度显示 '--stats', '1s', # 每秒更新一次统计信息 f'MP:{fileitem.path}', f'MP:{target_path}' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=self.__get_hidden_shell(), universal_newlines=True, bufsize=1 ) # 监控进度输出 last_progress = 0 for line in process.stdout: if line: # 解析rclone的进度输出 progress = self.__parse_rclone_progress(line) if progress is not None and progress > last_progress: progress_callback(progress) last_progress = progress if progress >= 100: break # 等待进程完成 retcode = process.wait() if retcode == 0: logger.info(f"【rclone】复制完成: {fileitem.name}") return True else: logger.error(f"【rclone】复制失败: {fileitem.name}") return False except Exception as err: logger.error(f"【rclone】复制失败: {fileitem.name} - {err}") return False def link(self, fileitem: _SchemaFileItem, target_file: Path) -> bool: pass def softlink(self, fileitem: _SchemaFileItem, target_file: Path) -> bool: pass def usage(self) -> Optional[_SchemaStorageUsage]: """ 存储使用情况 """ conf = self.get_config() if not conf: return None file_path = conf.config.get("filepath") if not file_path or not Path(file_path).exists(): return None # 读取rclone文件,检查是否有[MP]节点配置 with open(file_path, "r", encoding="utf-8", errors="replace") as f: lines = f.readlines() if not lines: return None if not any("[MP]" in line.strip() for line in lines): return None try: ret = subprocess.run( [ 'rclone', 'about', 'MP:/', '--json' ], capture_output=True, startupinfo=self.__get_hidden_shell() ) if ret.returncode == 0: items = json.loads(ret.stdout) return _SchemaStorageUsage( total=items.get("total"), available=items.get("free") ) except Exception as err: logger.error(f"【rclone】获取存储使用情况失败:{err}") return None