mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-30 20:54:32 +08:00
660 lines
23 KiB
Python
660 lines
23 KiB
Python
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
from typing import Optional, List, Union
|
|
|
|
from app.schemas.file import StorageUsage as _SchemaStorageUsage
|
|
from app.schemas.workflow import FileItem as _SchemaFileItem
|
|
from app.runtime.settings import RuntimeSettingsCompat
|
|
|
|
settings = RuntimeSettingsCompat()
|
|
from app.runtime.log import logger
|
|
from app.modules.filemanager.storages import StorageBase, transfer_process
|
|
from app.schemas.exception import StorageQueryError
|
|
from app.schemas.types import StorageSchema
|
|
from app.foundation import temporal as time_tools
|
|
from app.adapters.system.host import SystemUtils
|
|
|
|
_MAX_FOLDER_LOCKS = 4096
|
|
_folder_locks: OrderedDict[str, threading.Lock] = OrderedDict()
|
|
_folder_locks_guard = threading.Lock()
|
|
|
|
|
|
def _evict_unused_folder_locks_locked() -> None:
|
|
"""
|
|
在持有全局锁表互斥锁时淘汰旧路径锁，避免大量不同目录导致锁表无限增长。
|
|
"""
|
|
while len(_folder_locks) >= _MAX_FOLDER_LOCKS:
|
|
for key, lock in list(_folder_locks.items()):
|
|
if not lock.locked():
|
|
_folder_locks.pop(key, None)
|
|
break
|
|
else:
|
|
break
|
|
|
|
|
|
class Rclone(StorageBase):
|
|
"""
|
|
rclone相关操作
|
|
"""
|
|
|
|
# 存储类型
|
|
schema = StorageSchema.Rclone
|
|
|
|
# 支持的整理方式
|
|
transtype = {
|
|
"move": "移动",
|
|
"copy": "复制"
|
|
}
|
|
|
|
snapshot_check_folder_modtime = settings.RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME
|
|
|
|
def init_storage(self):
|
|
"""
|
|
初始化
|
|
"""
|
|
pass
|
|
|
|
def set_config(self, conf: dict):
|
|
"""
|
|
设置配置
|
|
"""
|
|
super().set_config(conf)
|
|
filepath = conf.get("filepath")
|
|
if not filepath:
|
|
logger.warn("【rclone】保存配置失败：未设置配置文件路径")
|
|
logger.info(f"【rclone】配置写入文件：{filepath}")
|
|
path = Path(filepath)
|
|
if not path.parent.exists():
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(conf.get('content'), encoding='utf-8')
|
|
|
|
@staticmethod
|
|
def __get_hidden_shell():
|
|
if SystemUtils.is_windows():
|
|
st = subprocess.STARTUPINFO()
|
|
st.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
|
st.wShowWindow = subprocess.SW_HIDE
|
|
return st
|
|
else:
|
|
return None
|
|
|
|
@staticmethod
|
|
def __parse_rclone_progress(line: str) -> Optional[float]:
|
|
"""
|
|
解析rclone进度输出
|
|
"""
|
|
if not line:
|
|
return None
|
|
|
|
line = line.strip()
|
|
|
|
# 检查是否包含百分比
|
|
if '%' not in line:
|
|
return None
|
|
|
|
try:
|
|
# 尝试多种进度输出格式
|
|
if 'ETA' in line:
|
|
# 格式: "Transferred: 1.234M / 5.678M, 22%, 1.234MB/s, ETA 2m3s"
|
|
percent_str = line.split('%')[0].split()[-1]
|
|
return float(percent_str)
|
|
elif 'Transferred:' in line and '100%' in line:
|
|
# 传输完成
|
|
return 100.0
|
|
else:
|
|
# 其他包含百分比的格式
|
|
parts = line.split()
|
|
for part in parts:
|
|
if '%' in part:
|
|
percent_str = part.replace('%', '')
|
|
return float(percent_str)
|
|
except (ValueError, IndexError):
|
|
pass
|
|
|
|
return None
|
|
|
|
def __get_rcloneitem(self, item: dict, parent: Optional[str] = "/") -> _SchemaFileItem:
|
|
"""
|
|
获取rclone文件项
|
|
"""
|
|
if not item:
|
|
return _SchemaFileItem()
|
|
if item.get("IsDir"):
|
|
return _SchemaFileItem(
|
|
storage=self.schema.value,
|
|
type="dir",
|
|
path=f"{parent}{item.get('Name')}" + "/",
|
|
name=item.get("Name"),
|
|
basename=item.get("Name"),
|
|
modify_time=time_tools.parse_timestamp(item.get("ModTime"))
|
|
)
|
|
else:
|
|
return _SchemaFileItem(
|
|
storage=self.schema.value,
|
|
type="file",
|
|
path=f"{parent}{item.get('Name')}",
|
|
name=item.get("Name"),
|
|
basename=Path(item.get("Name")).stem,
|
|
extension=Path(item.get("Name")).suffix[1:],
|
|
size=item.get("Size"),
|
|
modify_time=time_tools.parse_timestamp(item.get("ModTime"))
|
|
)
|
|
|
|
@staticmethod
|
|
def __normalize_remote_path(path: Union[Path, str]) -> str:
|
|
"""
|
|
规范化远端路径，统一目录锁键值。
|
|
"""
|
|
path_str = Path(str(path or "/")).as_posix()
|
|
if not path_str.startswith("/"):
|
|
path_str = f"/{path_str}"
|
|
if path_str != "/":
|
|
path_str = path_str.rstrip("/")
|
|
return path_str or "/"
|
|
|
|
@staticmethod
|
|
def __get_path_lock(path: Union[Path, str]) -> threading.Lock:
|
|
"""
|
|
获取指定远端路径的模块级锁。
|
|
"""
|
|
normalized = Rclone.__normalize_remote_path(path)
|
|
with _folder_locks_guard:
|
|
lock = _folder_locks.get(normalized)
|
|
if lock:
|
|
_folder_locks.move_to_end(normalized)
|
|
return lock
|
|
_evict_unused_folder_locks_locked()
|
|
lock = threading.Lock()
|
|
_folder_locks[normalized] = lock
|
|
return lock
|
|
|
|
def __wait_for_item(
|
|
self, path: Path, retries: int = 3, delay: float = 0.2
|
|
) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
等待目录或文件在远端可见，兼容云盘最终一致性延迟。
|
|
"""
|
|
for attempt in range(retries):
|
|
item = self.get_item(path)
|
|
if item:
|
|
return item
|
|
if attempt < retries - 1:
|
|
time.sleep(delay)
|
|
return None
|
|
|
|
def check(self) -> bool:
|
|
"""
|
|
检查存储是否可用
|
|
"""
|
|
try:
|
|
retcode = subprocess.run(
|
|
['rclone', 'lsf', 'MP:'],
|
|
startupinfo=self.__get_hidden_shell()
|
|
).returncode
|
|
if retcode == 0:
|
|
return True
|
|
except Exception as err:
|
|
logger.error(f"【rclone】存储检查失败：{err}")
|
|
return False
|
|
|
|
def list(self, fileitem: _SchemaFileItem) -> List[_SchemaFileItem]:
|
|
"""
|
|
浏览文件
|
|
"""
|
|
if fileitem.type == "file":
|
|
return [fileitem]
|
|
try:
|
|
ret = subprocess.run(
|
|
[
|
|
'rclone', 'lsjson',
|
|
f'MP:{fileitem.path}'
|
|
],
|
|
capture_output=True,
|
|
startupinfo=self.__get_hidden_shell()
|
|
)
|
|
if ret.returncode == 0:
|
|
items = json.loads(ret.stdout)
|
|
return [self.__get_rcloneitem(item, parent=fileitem.path) for item in items]
|
|
except Exception as err:
|
|
logger.error(f"【rclone】浏览文件失败：{err}")
|
|
return []
|
|
|
|
def create_folder(self, fileitem: _SchemaFileItem, name: str) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
创建目录
|
|
:param fileitem: 父目录
|
|
:param name: 目录名
|
|
"""
|
|
path = Path(self.__normalize_remote_path(Path(fileitem.path) / name))
|
|
try:
|
|
retcode = subprocess.run(
|
|
[
|
|
'rclone', 'mkdir',
|
|
f'MP:{path}'
|
|
],
|
|
startupinfo=self.__get_hidden_shell()
|
|
).returncode
|
|
if retcode == 0:
|
|
folder = self.__wait_for_item(path)
|
|
if folder:
|
|
return folder
|
|
logger.warn(f"【rclone】目录 {path} 创建成功后暂未可见")
|
|
return None
|
|
folder = self.__wait_for_item(path, retries=2)
|
|
if folder:
|
|
logger.info(f"【rclone】目录 {path} 已存在，忽略重复创建")
|
|
return folder
|
|
except Exception as err:
|
|
logger.error(f"【rclone】创建目录失败：{err}")
|
|
folder = self.__wait_for_item(path, retries=2)
|
|
if folder:
|
|
logger.info(f"【rclone】目录 {path} 已存在，忽略创建异常")
|
|
return folder
|
|
return None
|
|
|
|
def get_folder(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
根据文件路程获取目录，不存在则创建
|
|
"""
|
|
normalized = Path(self.__normalize_remote_path(path))
|
|
|
|
# 是否已存在
|
|
folder = self.get_item(normalized)
|
|
if folder:
|
|
return folder
|
|
# 逐级查找和创建目录
|
|
fileitem = _SchemaFileItem(storage=self.schema.value, type="dir", path="/")
|
|
for part in normalized.parts[1:]:
|
|
current_path = Path(self.__normalize_remote_path(Path(fileitem.path) / part))
|
|
with self.__get_path_lock(current_path):
|
|
dir_file = self.get_item(current_path)
|
|
if not dir_file:
|
|
dir_file = self.create_folder(fileitem, part)
|
|
if not dir_file:
|
|
logger.warn(f"【rclone】创建目录 {current_path} 失败！")
|
|
return None
|
|
fileitem = dir_file
|
|
return fileitem
|
|
|
|
def get_item(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件或目录，不存在返回None
|
|
"""
|
|
try:
|
|
ret = subprocess.run(
|
|
[
|
|
'rclone', 'lsjson',
|
|
f'MP:{path.parent}'
|
|
],
|
|
capture_output=True,
|
|
startupinfo=self.__get_hidden_shell()
|
|
)
|
|
if ret.returncode == 0:
|
|
items = json.loads(ret.stdout)
|
|
for item in items:
|
|
if item.get("Name") == path.name:
|
|
return self.__get_rcloneitem(item, parent=str(path.parent) + "/")
|
|
return None
|
|
except Exception as err:
|
|
logger.debug(f"【rclone】获取文件项失败：{err}")
|
|
return None
|
|
|
|
def get_item_strict(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件或目录，确认不存在返回None；无法确认状态时抛出 StorageQueryError。
|
|
rclone 用退出码 3/4 表示目录/文件不存在，其余非零退出无法区分
|
|
「不存在」与「查询失败」，必须保守失败以免覆盖保护被绕过。
|
|
"""
|
|
try:
|
|
ret = subprocess.run(
|
|
[
|
|
'rclone', 'lsjson',
|
|
f'MP:{path.parent}'
|
|
],
|
|
capture_output=True,
|
|
startupinfo=self.__get_hidden_shell()
|
|
)
|
|
except Exception as err:
|
|
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {err}") from err
|
|
if ret.returncode in (3, 4):
|
|
# 目录或文件不存在，是确定结果
|
|
return None
|
|
if ret.returncode != 0:
|
|
errmsg = (ret.stderr or b"").decode(errors="ignore").strip()
|
|
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {errmsg}")
|
|
try:
|
|
items = json.loads(ret.stdout)
|
|
except Exception as err:
|
|
raise StorageQueryError(f"【rclone】解析查询结果失败: {path} - {err}") from err
|
|
for item in items:
|
|
if item.get("Name") == path.name:
|
|
return self.__get_rcloneitem(item, parent=str(path.parent) + "/")
|
|
return None
|
|
|
|
def delete(self, fileitem: _SchemaFileItem) -> bool:
|
|
"""
|
|
删除文件
|
|
"""
|
|
try:
|
|
retcode = subprocess.run(
|
|
[
|
|
'rclone', 'deletefile',
|
|
f'MP:{fileitem.path}'
|
|
],
|
|
startupinfo=self.__get_hidden_shell()
|
|
).returncode
|
|
if retcode == 0:
|
|
return True
|
|
except Exception as err:
|
|
logger.error(f"【rclone】删除文件失败：{err}")
|
|
return False
|
|
|
|
def rename(self, fileitem: _SchemaFileItem, name: str) -> bool:
|
|
"""
|
|
重命名文件
|
|
"""
|
|
try:
|
|
retcode = subprocess.run(
|
|
[
|
|
'rclone', 'moveto',
|
|
f'MP:{fileitem.path}',
|
|
f'MP:{Path(fileitem.path).parent / name}'
|
|
],
|
|
startupinfo=self.__get_hidden_shell()
|
|
).returncode
|
|
if retcode == 0:
|
|
return True
|
|
except Exception as err:
|
|
logger.error(f"【rclone】重命名文件失败：{err}")
|
|
return False
|
|
|
|
def download(self, fileitem: _SchemaFileItem, path: Path = None) -> Optional[Path]:
|
|
"""
|
|
带实时进度显示的下载
|
|
"""
|
|
local_path = self._build_download_path(fileitem, path or settings.TEMP_PATH)
|
|
if not local_path:
|
|
return None
|
|
|
|
# 初始化进度条
|
|
logger.info(f"【rclone】开始下载: {fileitem.name} -> {local_path}")
|
|
progress_callback = transfer_process(Path(fileitem.path).as_posix())
|
|
|
|
try:
|
|
# 使用rclone的进度显示功能
|
|
process = subprocess.Popen(
|
|
[
|
|
'rclone', 'copyto',
|
|
'--progress', # 启用进度显示
|
|
'--stats', '1s', # 每秒更新一次统计信息
|
|
f'MP:{fileitem.path}',
|
|
f'{local_path}'
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
startupinfo=self.__get_hidden_shell(),
|
|
universal_newlines=True,
|
|
bufsize=1
|
|
)
|
|
|
|
# 监控进度输出
|
|
last_progress = 0
|
|
for line in process.stdout:
|
|
if line:
|
|
# 解析rclone的进度输出
|
|
progress = self.__parse_rclone_progress(line)
|
|
if progress is not None and progress > last_progress:
|
|
progress_callback(progress)
|
|
last_progress = progress
|
|
if progress >= 100:
|
|
break
|
|
|
|
# 等待进程完成
|
|
retcode = process.wait()
|
|
if retcode == 0:
|
|
logger.info(f"【rclone】下载完成: {fileitem.name}")
|
|
return local_path
|
|
else:
|
|
logger.error(f"【rclone】下载失败: {fileitem.name}")
|
|
return None
|
|
|
|
except Exception as err:
|
|
logger.error(f"【rclone】下载失败: {fileitem.name} - {err}")
|
|
# 删除可能部分下载的文件
|
|
if local_path.exists():
|
|
local_path.unlink()
|
|
return None
|
|
|
|
def upload(self, fileitem: _SchemaFileItem, path: Path,
|
|
new_name: Optional[str] = None) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
带实时进度显示的上传
|
|
:param fileitem: 上传目录项
|
|
:param path: 本地文件路径
|
|
:param new_name: 上传后文件名
|
|
"""
|
|
target_name = new_name or path.name
|
|
new_path = Path(fileitem.path) / target_name
|
|
|
|
# 初始化进度条
|
|
logger.info(f"【rclone】开始上传: {path} -> {new_path}")
|
|
progress_callback = transfer_process(path.as_posix())
|
|
|
|
try:
|
|
# 使用rclone的进度显示功能
|
|
process = subprocess.Popen(
|
|
[
|
|
'rclone', 'copyto',
|
|
'--progress', # 启用进度显示
|
|
'--stats', '1s', # 每秒更新一次统计信息
|
|
path.as_posix(),
|
|
f'MP:{new_path}'
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
startupinfo=self.__get_hidden_shell(),
|
|
universal_newlines=True,
|
|
bufsize=1
|
|
)
|
|
|
|
# 监控进度输出
|
|
last_progress = 0
|
|
for line in process.stdout:
|
|
if line:
|
|
# 解析rclone的进度输出
|
|
progress = self.__parse_rclone_progress(line)
|
|
if progress is not None and progress > last_progress:
|
|
progress_callback(progress)
|
|
last_progress = progress
|
|
if progress >= 100:
|
|
break
|
|
|
|
# 等待进程完成
|
|
retcode = process.wait()
|
|
if retcode == 0:
|
|
logger.info(f"【rclone】上传完成: {target_name}")
|
|
return self.get_item(new_path)
|
|
else:
|
|
logger.error(f"【rclone】上传失败: {target_name}")
|
|
return None
|
|
|
|
except Exception as err:
|
|
logger.error(f"【rclone】上传失败: {target_name} - {err}")
|
|
return None
|
|
|
|
def detail(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件详情
|
|
"""
|
|
try:
|
|
ret = subprocess.run(
|
|
[
|
|
'rclone', 'lsjson',
|
|
f'MP:{fileitem.path}'
|
|
],
|
|
capture_output=True,
|
|
startupinfo=self.__get_hidden_shell()
|
|
)
|
|
if ret.returncode == 0:
|
|
items = json.loads(ret.stdout)
|
|
return self.__get_rcloneitem(items[0])
|
|
except Exception as err:
|
|
logger.error(f"【rclone】获取文件详情失败：{err}")
|
|
return None
|
|
|
|
def move(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool:
|
|
"""
|
|
移动文件
|
|
:param fileitem: 文件项
|
|
:param path: 目标目录
|
|
:param new_name: 新文件名
|
|
"""
|
|
target_path = path / new_name
|
|
|
|
# 初始化进度条
|
|
logger.info(f"【rclone】开始移动: {fileitem.path} -> {target_path}")
|
|
progress_callback = transfer_process(Path(fileitem.path).as_posix())
|
|
|
|
try:
|
|
# 使用rclone的进度显示功能
|
|
process = subprocess.Popen(
|
|
[
|
|
'rclone', 'moveto',
|
|
'--progress', # 启用进度显示
|
|
'--stats', '1s', # 每秒更新一次统计信息
|
|
f'MP:{fileitem.path}',
|
|
f'MP:{target_path}'
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
startupinfo=self.__get_hidden_shell(),
|
|
universal_newlines=True,
|
|
bufsize=1
|
|
)
|
|
|
|
# 监控进度输出
|
|
last_progress = 0
|
|
for line in process.stdout:
|
|
if line:
|
|
# 解析rclone的进度输出
|
|
progress = self.__parse_rclone_progress(line)
|
|
if progress is not None and progress > last_progress:
|
|
progress_callback(progress)
|
|
last_progress = progress
|
|
if progress >= 100:
|
|
break
|
|
|
|
# 等待进程完成
|
|
retcode = process.wait()
|
|
if retcode == 0:
|
|
logger.info(f"【rclone】移动完成: {fileitem.name}")
|
|
return True
|
|
else:
|
|
logger.error(f"【rclone】移动失败: {fileitem.name}")
|
|
return False
|
|
|
|
except Exception as err:
|
|
logger.error(f"【rclone】移动失败: {fileitem.name} - {err}")
|
|
return False
|
|
|
|
def copy(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool:
|
|
"""
|
|
复制文件
|
|
:param fileitem: 文件项
|
|
:param path: 目标目录
|
|
:param new_name: 新文件名
|
|
"""
|
|
target_path = path / new_name
|
|
|
|
# 初始化进度条
|
|
logger.info(f"【rclone】开始复制: {fileitem.path} -> {target_path}")
|
|
progress_callback = transfer_process(Path(fileitem.path).as_posix())
|
|
|
|
try:
|
|
# 使用rclone的进度显示功能
|
|
process = subprocess.Popen(
|
|
[
|
|
'rclone', 'copyto',
|
|
'--progress', # 启用进度显示
|
|
'--stats', '1s', # 每秒更新一次统计信息
|
|
f'MP:{fileitem.path}',
|
|
f'MP:{target_path}'
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
startupinfo=self.__get_hidden_shell(),
|
|
universal_newlines=True,
|
|
bufsize=1
|
|
)
|
|
|
|
# 监控进度输出
|
|
last_progress = 0
|
|
for line in process.stdout:
|
|
if line:
|
|
# 解析rclone的进度输出
|
|
progress = self.__parse_rclone_progress(line)
|
|
if progress is not None and progress > last_progress:
|
|
progress_callback(progress)
|
|
last_progress = progress
|
|
if progress >= 100:
|
|
break
|
|
|
|
# 等待进程完成
|
|
retcode = process.wait()
|
|
if retcode == 0:
|
|
logger.info(f"【rclone】复制完成: {fileitem.name}")
|
|
return True
|
|
else:
|
|
logger.error(f"【rclone】复制失败: {fileitem.name}")
|
|
return False
|
|
|
|
except Exception as err:
|
|
logger.error(f"【rclone】复制失败: {fileitem.name} - {err}")
|
|
return False
|
|
|
|
def link(self, fileitem: _SchemaFileItem, target_file: Path) -> bool:
|
|
pass
|
|
|
|
def softlink(self, fileitem: _SchemaFileItem, target_file: Path) -> bool:
|
|
pass
|
|
|
|
def usage(self) -> Optional[_SchemaStorageUsage]:
|
|
"""
|
|
存储使用情况
|
|
"""
|
|
conf = self.get_config()
|
|
if not conf:
|
|
return None
|
|
file_path = conf.config.get("filepath")
|
|
if not file_path or not Path(file_path).exists():
|
|
return None
|
|
# 读取rclone文件，检查是否有[MP]节点配置
|
|
with open(file_path, "r", encoding="utf-8", errors="replace") as f:
|
|
lines = f.readlines()
|
|
if not lines:
|
|
return None
|
|
if not any("[MP]" in line.strip() for line in lines):
|
|
return None
|
|
try:
|
|
ret = subprocess.run(
|
|
[
|
|
'rclone', 'about',
|
|
'MP:/', '--json'
|
|
],
|
|
capture_output=True,
|
|
startupinfo=self.__get_hidden_shell()
|
|
)
|
|
if ret.returncode == 0:
|
|
items = json.loads(ret.stdout)
|
|
return _SchemaStorageUsage(
|
|
total=items.get("total"),
|
|
available=items.get("free")
|
|
)
|
|
except Exception as err:
|
|
logger.error(f"【rclone】获取存储使用情况失败：{err}")
|
|
return None
|