mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-30 20:54:32 +08:00
361 lines
11 KiB
Python
361 lines
11 KiB
Python
from abc import ABCMeta, abstractmethod
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Optional, List, Dict, Tuple, Callable, Union
|
|
|
|
from tqdm import tqdm
|
|
|
|
from app.schemas.file import StorageUsage as _SchemaStorageUsage
|
|
from app.schemas.system import StorageConf as _SchemaStorageConf
|
|
from app.schemas.workflow import FileItem as _SchemaFileItem
|
|
from app.runtime.progress import ProgressHelper
|
|
from app.application.storage import StorageHelper
|
|
from app.runtime.log import logger
|
|
from app.schemas.exception import StorageQueryError
|
|
from app.foundation.crypto import HashUtils
|
|
|
|
|
|
def transfer_process(path: str) -> Callable[[int | float], None]:
|
|
"""
|
|
传输进度回调
|
|
"""
|
|
pbar = tqdm(total=100, desc="进度", unit="%")
|
|
progress = ProgressHelper(HashUtils.md5(path))
|
|
progress.start()
|
|
|
|
def update_progress(percent: Union[int, float]) -> None:
|
|
"""
|
|
更新进度百分比
|
|
"""
|
|
percent_value = round(percent, 2) if isinstance(percent, float) else percent
|
|
pbar.n = percent_value
|
|
# 更新进度
|
|
pbar.refresh()
|
|
progress.update(value=percent_value, text=f"{path} 进度：{percent_value}%")
|
|
# 完成时结束
|
|
if percent_value >= 100:
|
|
progress.end()
|
|
pbar.close()
|
|
|
|
return update_progress
|
|
|
|
|
|
class StorageBase(metaclass=ABCMeta):
|
|
"""
|
|
存储基类
|
|
"""
|
|
schema = None
|
|
transtype = {}
|
|
snapshot_check_folder_modtime = True
|
|
|
|
def __init__(self):
|
|
self.storagehelper = StorageHelper()
|
|
|
|
@abstractmethod
|
|
def init_storage(self):
|
|
"""
|
|
初始化
|
|
"""
|
|
pass
|
|
|
|
def generate_qrcode(self, *args, **kwargs) -> Optional[Tuple[dict, str]]:
|
|
"""生成存储登录二维码"""
|
|
pass
|
|
|
|
def generate_auth_url(self, *args, **kwargs) -> Optional[Tuple[dict, str]]:
|
|
"""
|
|
生成 OAuth2 授权 URL
|
|
"""
|
|
return {}, "此存储不支持 OAuth2 授权"
|
|
|
|
def check_login(self, *args, **kwargs) -> Optional[Dict[str, str]]:
|
|
"""检查存储登录状态"""
|
|
pass
|
|
|
|
def get_config(self) -> Optional[_SchemaStorageConf]:
|
|
"""
|
|
获取配置
|
|
"""
|
|
return self.storagehelper.get_storage(self.schema.value)
|
|
|
|
def get_conf(self) -> dict:
|
|
"""
|
|
获取配置
|
|
"""
|
|
conf = self.get_config()
|
|
return conf.config if conf else {}
|
|
|
|
def set_config(self, conf: dict):
|
|
"""
|
|
设置配置
|
|
"""
|
|
self.storagehelper.set_storage(self.schema.value, conf)
|
|
self.init_storage()
|
|
|
|
def support_transtype(self) -> dict:
|
|
"""
|
|
支持的整理方式
|
|
"""
|
|
return self.transtype
|
|
|
|
def is_support_transtype(self, transtype: str) -> bool:
|
|
"""
|
|
是否支持整理方式
|
|
"""
|
|
return transtype in self.transtype
|
|
|
|
def reset_config(self):
|
|
"""
|
|
重置置配置
|
|
"""
|
|
self.storagehelper.reset_storage(self.schema.value)
|
|
self.init_storage()
|
|
|
|
@staticmethod
|
|
def _safe_download_name(name: Optional[str]) -> Optional[str]:
|
|
"""
|
|
提取可安全落盘的文件名。
|
|
"""
|
|
if not name:
|
|
return None
|
|
|
|
safe_name = PurePosixPath(str(name).replace("\\", "/")).name
|
|
if safe_name in ("", ".", ".."):
|
|
return None
|
|
return safe_name
|
|
|
|
def _build_download_path(
|
|
self, fileitem: _SchemaFileItem, path: Path
|
|
) -> Optional[Path]:
|
|
"""
|
|
构造本地下载路径，避免远端文件名携带目录片段时越过目标目录。
|
|
"""
|
|
safe_name = self._safe_download_name(fileitem.name)
|
|
if not safe_name:
|
|
logger.error(f"【存储】下载文件名无效：{fileitem.name}")
|
|
return None
|
|
|
|
local_path = path / safe_name
|
|
try:
|
|
local_path.resolve().relative_to(path.resolve())
|
|
except ValueError:
|
|
logger.error(f"【存储】下载路径越界：{fileitem.name} -> {local_path}")
|
|
return None
|
|
return local_path
|
|
|
|
@abstractmethod
|
|
def check(self) -> bool:
|
|
"""
|
|
检查存储是否可用
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def list(self, fileitem: _SchemaFileItem) -> List[_SchemaFileItem]:
|
|
"""
|
|
浏览文件
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_folder(self, fileitem: _SchemaFileItem, name: str) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
创建目录
|
|
:param fileitem: 父目录
|
|
:param name: 目录名
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_folder(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取目录，如目录不存在则创建
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_item(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件或目录，不存在返回None
|
|
"""
|
|
pass
|
|
|
|
def get_item_strict(self, path: Path) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件或目录，确认不存在返回None；无法确认状态时抛出 StorageQueryError。
|
|
|
|
默认保守失败：未覆写的存储无法区分「不存在」与「查询失败」，沿用
|
|
get_item() 会让 overwrite_mode=size 的覆盖保护在查询失败时被绕过，
|
|
把「无法确认」当成「目标不存在」而放行覆盖。具体存储必须先实现
|
|
「确认不存在」的判定，再覆写本方法。
|
|
"""
|
|
raise StorageQueryError(f"存储 {self.schema} 未实现严格查询，无法确认目标状态: {path}")
|
|
|
|
def get_parent(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取父目录
|
|
"""
|
|
return self.get_item(Path(fileitem.path).parent)
|
|
|
|
@abstractmethod
|
|
def delete(self, fileitem: _SchemaFileItem) -> bool:
|
|
"""
|
|
删除文件
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def rename(self, fileitem: _SchemaFileItem, name: str) -> bool:
|
|
"""
|
|
重命名文件
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def download(self, fileitem: _SchemaFileItem, path: Path = None) -> Path:
|
|
"""
|
|
下载文件，保存到本地，返回本地临时文件地址
|
|
:param fileitem: 文件项
|
|
:param path: 文件保存路径
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def upload(self, fileitem: _SchemaFileItem, path: Path,
|
|
new_name: Optional[str] = None) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
上传文件
|
|
:param fileitem: 上传目录项
|
|
:param path: 本地文件路径
|
|
:param new_name: 上传后文件名
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def detail(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]:
|
|
"""
|
|
获取文件详情
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def copy(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool:
|
|
"""
|
|
复制文件
|
|
:param fileitem: 文件项
|
|
:param path: 目标目录
|
|
:param new_name: 新文件名
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def move(self, fileitem: _SchemaFileItem, path: Path, new_name: str) -> bool:
|
|
"""
|
|
移动文件
|
|
:param fileitem: 文件项
|
|
:param path: 目标目录
|
|
:param new_name: 新文件名
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def link(self, fileitem: _SchemaFileItem, target_file: Path) -> bool:
|
|
"""
|
|
硬链接文件
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def softlink(self, fileitem: _SchemaFileItem, target_file: Path) -> bool:
|
|
"""
|
|
软链接文件
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def usage(self) -> Optional[_SchemaStorageUsage]:
|
|
"""
|
|
存储使用情况
|
|
"""
|
|
pass
|
|
|
|
def snapshot(self, path: Path, last_snapshot_time: float = None, max_depth: int = 5,
|
|
previous_snapshot: Optional[Dict[str, Dict]] = None) -> Dict[str, Dict]:
|
|
"""
|
|
快照文件系统，输出所有层级文件信息（不含目录）
|
|
:param path: 路径
|
|
:param last_snapshot_time: 上次快照时间，用于增量快照
|
|
:param max_depth: 最大递归深度，避免过深遍历
|
|
:param previous_snapshot: 上次完整快照，用于保留未变化目录并清理已删除文件
|
|
"""
|
|
root_path = PurePosixPath(path.as_posix())
|
|
files_info = {
|
|
file_path: file_info
|
|
for file_path, file_info in (previous_snapshot or {}).items()
|
|
if PurePosixPath(file_path).is_relative_to(root_path)
|
|
}
|
|
|
|
def __remove_deleted_children(_fileitm: _SchemaFileItem,
|
|
sub_files: List[_SchemaFileItem]) -> None:
|
|
"""
|
|
清理已确认遍历目录中不再存在的直接子项。
|
|
未变化的子目录仍保留旧基线，避免增量遍历将其误删。
|
|
"""
|
|
directory_path = PurePosixPath(_fileitm.path)
|
|
child_paths = {PurePosixPath(sub_file.path) for sub_file in sub_files}
|
|
for old_file_path in list(files_info):
|
|
try:
|
|
relative_path = PurePosixPath(old_file_path).relative_to(directory_path)
|
|
except ValueError:
|
|
continue
|
|
if not relative_path.parts:
|
|
continue
|
|
direct_child_path = directory_path / relative_path.parts[0]
|
|
if direct_child_path not in child_paths:
|
|
files_info.pop(old_file_path, None)
|
|
|
|
def __snapshot_file(_fileitm: _SchemaFileItem, current_depth: int = 0):
|
|
"""
|
|
递归获取文件信息
|
|
"""
|
|
try:
|
|
if _fileitm.type == "dir":
|
|
# 检查递归深度限制
|
|
if current_depth >= max_depth:
|
|
return
|
|
|
|
# 根目录每轮至少列举一次，用于清理已移走的直接子项；子目录仍按修改时间增量遍历
|
|
if (current_depth > 0 and
|
|
self.snapshot_check_folder_modtime and
|
|
last_snapshot_time and
|
|
_fileitm.modify_time and
|
|
_fileitm.modify_time <= last_snapshot_time):
|
|
return
|
|
|
|
# 只有目录列表成功返回后才清理旧基线，查询异常时继续保留待下轮重试
|
|
sub_files = self.list(_fileitm)
|
|
if sub_files is None:
|
|
return
|
|
sub_files = list(sub_files)
|
|
__remove_deleted_children(_fileitm, sub_files)
|
|
for sub_file in sub_files:
|
|
__snapshot_file(sub_file, current_depth + 1)
|
|
else:
|
|
# 记录文件的完整信息用于比对（始终包含所有文件，由 compare_snapshots 负责检测变化）
|
|
files_info[_fileitm.path] = {
|
|
'size': _fileitm.size or 0,
|
|
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
|
'fileid': getattr(_fileitm, 'fileid', None),
|
|
'type': _fileitm.type
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Snapshot error for {_fileitm.path}: {e}")
|
|
|
|
fileitem = self.get_item(path)
|
|
if not fileitem:
|
|
return {}
|
|
|
|
__snapshot_file(fileitem)
|
|
|
|
return files_info
|