mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-21 16:23:34 +08:00
fix(monitor,transfer): 修复 FUSE 挂载无响应导致的监控冻死、整理链锁死与漏件 (#6276)
* wip(v3): 移植监控与整理韧性修复到 v3 基线 包含:监控看门狗隔离/挂载探测、整理队列持久化、文件系统子进程代理、 写入原子化。迁移重挂到 v3 链 8a4c7e1d2f90 -> 7f5c1d2e3a4b -> e3d9f4b7c806。 tmdb 相关测试尚未通过,待定位。 * fix(v3): 修正移植引入的 16 项测试失败 - poller.py:合并时我方保留的行仍用旧变量名 merged_snapshot,而 v3 已统一 改名为 current_snapshot,导致 NameError 被外层 except 吞掉、快照从未保存 - smb.py:采纳 f-string 拆分写法,恢复 Python 3.11 可解析 - dispatcher 测试:历史查重由 _should_skip_by_history 统一承担,mock 点随之调整 - tmdb 缓存测试:补充 v3 新增的 media_source/media_id 字段 - tmdb 重试测试:为 fake 补充 match_multi/async_match_multi 尚余 3 项与 v3 识别流程的连接失败处理有关,待单独判断。 * fix(v3): 测试适配 v3 的 media_source/media_id 重构 v3 将媒体标识从 tmdbid 统一重构为 media_source + media_id,recognize_media 的 tmdbid 参数已被 **kwargs 静默吞掉——传了也不生效,流程会误降级到名称搜索。 tmdb 重试用例改用新参数后恢复正确路径。 同时修正 fake 的 match_multi 语义:真实实现(tmdbapi.match_multi)吞掉所有 异常并返回 None,连接失败与「未找到」在该路径上本就不可区分,fake 需保持一致。 至此移植引入的 19 项失败全部清零。 --------- Co-authored-by: Aqr-K <Aqr-K@users.noreply.github.com>
This commit is contained in:
411
app/modules/filemanager/fsproxy.py
Normal file
411
app/modules/filemanager/fsproxy.py
Normal file
@@ -0,0 +1,411 @@
|
||||
"""
|
||||
本地文件系统操作代理。
|
||||
|
||||
FUSE/网络挂载有两种故障形态:crash 型(调用抛错,可捕获、可重试)和 block 型
|
||||
(调用既不返回错误也不返回结果,永久悬挂)。**Python 无法中断一个已经发出的
|
||||
系统调用,也无法强杀线程**,所以 block 型故障下阻塞的线程永远无法回收——这正是
|
||||
整理消费线程停摆、监控自愈路径自冻的根因。
|
||||
|
||||
本模块把这些调用放进一个常驻子进程执行。子进程可以被 SIGKILL,因此超时后能真正
|
||||
回收;对调用方而言,超时表现为一个普通的 OSError 子类(FileSystemTimeout)。
|
||||
换句话说:**把不可处理的 block 型故障,转换成系统各层已经能正确处理的 crash 型
|
||||
故障**——退避重启、登记待重试这些既有机制立刻就能接管。
|
||||
|
||||
第一版只放行安全的操作:
|
||||
- 只读(stat/exists/listdir)——强杀不产生任何副作用
|
||||
- 同存储 rename——内核保证原子性,强杀后要么完全成功要么完全没发生
|
||||
跨存储的复制+删除不在此列,它需要单独的可恢复语义(临时名 + 完成后 rename)。
|
||||
"""
|
||||
import errno as errno_module
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
# worker 脚本路径。用绝对路径直接执行,而不是 -m 或 import:
|
||||
# 直接执行文件不会触发 app/__init__.py 的导入链,代理启动才是毫秒级的
|
||||
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
|
||||
# 单次快操作(stat/listdir/rename/unlink 等)的默认超时秒数
|
||||
DEFAULT_TIMEOUT = 30
|
||||
# 长耗时操作(复制)两次进度上报之间的最长间隔秒数。
|
||||
# worker 每秒上报一次心跳,因此这个阈值判定的是「传输完全没有推进」,
|
||||
# 而不是「传输很慢」——大文件复制几小时也不会误杀
|
||||
DEFAULT_STALL_TIMEOUT = 120
|
||||
# 强杀代理后等待它消失的宽限秒数,不能无限等待
|
||||
_KILL_GRACE = 5
|
||||
|
||||
|
||||
class FileSystemTimeout(OSError):
|
||||
"""
|
||||
文件系统操作在代理中超时未返回,判定挂载无响应。
|
||||
|
||||
继承 OSError 是刻意的:整理链、监控 watcher 等各层对 OSError 已有完整的
|
||||
退避重试与登记逻辑,block 型故障经此转换后可以直接复用它们。
|
||||
"""
|
||||
|
||||
|
||||
class FileSystemProxy:
|
||||
"""
|
||||
常驻子进程文件系统代理。
|
||||
|
||||
请求-响应严格串行(一个代理同时只处理一个请求),由锁保证。超时即强杀代理,
|
||||
下一次请求自动重启一个新的——启动成本是毫秒级,因为 worker 只依赖标准库。
|
||||
"""
|
||||
|
||||
def __init__(self, timeout: Optional[float] = None,
|
||||
stall_timeout: Optional[float] = None):
|
||||
"""
|
||||
:param timeout: 单次快操作的超时秒数,None 表示实时跟随系统设置
|
||||
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数,
|
||||
None 表示实时跟随系统设置
|
||||
"""
|
||||
self._timeout_override = timeout
|
||||
self._stall_timeout_override = stall_timeout
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._selector: Optional[selectors.BaseSelector] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 对外操作
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def stat(self, path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
读取路径属性。
|
||||
:param path: 目标路径
|
||||
:return: {"size", "mtime", "is_dir", "is_file"}
|
||||
"""
|
||||
return self._call("stat", path=str(path))
|
||||
|
||||
def exists(self, path: Path) -> bool:
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
只有 FileNotFoundError 才算「不存在」;其余 OSError(含超时)原样抛出,
|
||||
避免像 Path.exists() 那样把挂载抖动误判成文件消失。
|
||||
:param path: 目标路径
|
||||
:return: 是否存在
|
||||
"""
|
||||
try:
|
||||
self._call("exists", path=str(path))
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def listdir(self, path: Path) -> List[str]:
|
||||
"""
|
||||
列出目录条目名。
|
||||
:param path: 目标目录
|
||||
:return: 条目名列表
|
||||
"""
|
||||
return self._call("listdir", path=str(path))
|
||||
|
||||
def rename(self, src: Path, dst: Path) -> bool:
|
||||
"""
|
||||
同一存储内重命名/移动。跨存储会抛 OSError(EXDEV),由调用方走原有路径。
|
||||
:param src: 源路径
|
||||
:param dst: 目标路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rename", src=str(src), dst=str(dst))
|
||||
|
||||
def copy(self, src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]] = None,
|
||||
cancel_cb: Optional[Callable[[], bool]] = None,
|
||||
chunk_size: Optional[int] = None) -> Any:
|
||||
"""
|
||||
复制文件内容并保留时间戳,按「进度无推进」判定挂死。
|
||||
|
||||
复制大文件可能持续几小时,固定超时无法区分「正常但慢」和「已经挂死」。
|
||||
worker 每秒上报一次进度作为心跳,这里判定的是**两次上报之间的间隔**:
|
||||
超过 stall 阈值收不到任何一行,才认定挂载无响应并强杀 worker。
|
||||
|
||||
取消检查放在父进程:worker 里读不到 global_vars 的传输取消标记,而父进程
|
||||
每收到一次进度就能检查一次,要取消直接杀掉 worker 即可,比在子进程里
|
||||
轮询标记更干净。
|
||||
:param src: 源文件
|
||||
:param dst: 目标文件(调用方应传临时名,完成后自行原子替换)
|
||||
:param progress_cb: 进度回调,入参为百分比
|
||||
:param cancel_cb: 取消检查回调,返回 True 表示应中止
|
||||
:param chunk_size: 分块大小
|
||||
:return: 成功时为 {"copied", "total"},被取消或通信失败时为 False
|
||||
"""
|
||||
payload = {"src": str(src), "dst": str(dst)}
|
||||
if chunk_size:
|
||||
payload["chunk_size"] = chunk_size
|
||||
if not self._enabled():
|
||||
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request_stream(payload, progress_cb, cancel_cb)
|
||||
except FileSystemTimeout:
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
|
||||
self._shutdown()
|
||||
return False
|
||||
|
||||
def _request_stream(self, payload: Dict[str, Any],
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]]) -> Any:
|
||||
"""
|
||||
发起一次流式请求,逐行消费进度直到终态。
|
||||
:param payload: 请求参数
|
||||
:param progress_cb: 进度回调
|
||||
:param cancel_cb: 取消检查回调
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": "copy", **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
while True:
|
||||
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
|
||||
progress = response.get("progress")
|
||||
if progress is not None:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
logger.info(f"复制已取消: {payload.get('src')}")
|
||||
# 取消就地生效:杀掉 worker 立刻中断传输,不必等它读完整个文件
|
||||
self._shutdown()
|
||||
return False
|
||||
if progress_cb is not None:
|
||||
total = progress.get("total") or 0
|
||||
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
|
||||
continue
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
@staticmethod
|
||||
def _direct_copy(src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]],
|
||||
chunk_size: Optional[int]) -> bool:
|
||||
"""
|
||||
不经代理直接复制,供代理关闭时使用。
|
||||
"""
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
return False
|
||||
buf = fsrc.read(chunk_size or 1024 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(copied / total * 100)
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return True
|
||||
|
||||
def unlink(self, path: Path) -> bool:
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作,强杀后没有中间状态。
|
||||
:param path: 目标文件
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("unlink", path=str(path))
|
||||
|
||||
def rmtree(self, path: Path) -> bool:
|
||||
"""
|
||||
递归删除目录,容忍部分失败(可重复执行直到成功)。
|
||||
:param path: 目标目录
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rmtree", path=str(path))
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
关闭代理进程。
|
||||
"""
|
||||
with self._lock:
|
||||
self._shutdown()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内部实现
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _timeout(self) -> float:
|
||||
"""
|
||||
单次快操作的超时秒数。
|
||||
|
||||
实时读取而不是构造时固定:这三项都暴露在前端设置里,用户改完保存后
|
||||
必须立刻生效,否则会出现「改了没反应」的困惑。
|
||||
"""
|
||||
if self._timeout_override is not None:
|
||||
return self._timeout_override
|
||||
return float(getattr(settings, "FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
|
||||
|
||||
@property
|
||||
def _stall_timeout(self) -> float:
|
||||
"""
|
||||
长耗时操作两次进度上报之间的最长间隔秒数,同样实时跟随系统设置。
|
||||
"""
|
||||
if self._stall_timeout_override is not None:
|
||||
return self._stall_timeout_override
|
||||
return float(getattr(settings, "FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT))
|
||||
|
||||
@staticmethod
|
||||
def _enabled() -> bool:
|
||||
"""
|
||||
代理是否启用。关闭时退回直接调用,行为与引入代理之前完全一致。
|
||||
"""
|
||||
return bool(getattr(settings, "FS_PROXY_ENABLED", True))
|
||||
|
||||
@staticmethod
|
||||
def _direct(op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
不经代理直接执行操作,供代理关闭时使用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if op == "stat":
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
if op == "exists":
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
if op == "listdir":
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
if op == "rename":
|
||||
os.rename(payload["src"], payload["dst"])
|
||||
return True
|
||||
if op == "unlink":
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
if op == "rmtree":
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
raise ValueError(f"unknown op: {op}")
|
||||
|
||||
def _call(self, op: str, **payload) -> Any:
|
||||
"""
|
||||
执行一次代理调用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if not self._enabled():
|
||||
return self._direct(op, payload)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request(op, payload)
|
||||
except FileSystemTimeout:
|
||||
# 超时说明挂载正在挂死,重试只会再冻一次,直接上报给调用方
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
# 代理进程意外退出或响应损坏,重启后重试一次
|
||||
logger.debug(f"文件系统代理通信异常,重启后重试: {op} - {err}")
|
||||
self._shutdown()
|
||||
return self._request(op, payload)
|
||||
|
||||
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
发送请求并等待响应。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": op, **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
response = json.loads(self._read_line().decode("utf-8"))
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类,
|
||||
# 调用方沿用原有的异常分支即可,无需感知代理的存在
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
def _read_line(self, timeout: Optional[float] = None) -> bytes:
|
||||
"""
|
||||
读取一行响应,超时即强杀代理。
|
||||
:param timeout: 本次读取的超时秒数,默认用单次操作超时
|
||||
:return: 响应行
|
||||
"""
|
||||
timeout = self._timeout if timeout is None else timeout
|
||||
if not self._selector.select(timeout=timeout):
|
||||
logger.error(f"文件系统操作 {timeout} 秒无响应,判定挂载挂死,正在回收代理进程")
|
||||
self._shutdown()
|
||||
raise FileSystemTimeout(
|
||||
errno_module.ETIMEDOUT,
|
||||
f"文件系统操作超过 {timeout} 秒无响应,挂载可能已无响应"
|
||||
)
|
||||
line = self._process.stdout.readline()
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _ensure_worker(self):
|
||||
"""
|
||||
确保代理进程可用,不可用时重新启动。
|
||||
"""
|
||||
if self._process is not None and self._process.poll() is None:
|
||||
return
|
||||
self._shutdown()
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, str(_WORKER_PATH)],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
bufsize=0,
|
||||
)
|
||||
self._selector = selectors.DefaultSelector()
|
||||
self._selector.register(self._process.stdout, selectors.EVENT_READ)
|
||||
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
|
||||
|
||||
def _shutdown(self):
|
||||
"""
|
||||
回收代理进程。冻在挂载上的进程用 SIGKILL,且不无限等待它消失
|
||||
——否则「可放弃的代理」又变回一次不可放弃的阻塞。
|
||||
"""
|
||||
if self._selector is not None:
|
||||
try:
|
||||
self._selector.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._selector = None
|
||||
process, self._process = self._process, None
|
||||
if process is None:
|
||||
return
|
||||
for stream in (process.stdin, process.stdout):
|
||||
try:
|
||||
if stream:
|
||||
stream.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=_KILL_GRACE)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warn(f"文件系统代理进程未能及时退出,交由系统回收: pid={process.pid}")
|
||||
except Exception as err: # noqa: BLE001
|
||||
logger.debug(f"回收文件系统代理进程失败: {err}")
|
||||
|
||||
|
||||
# 全局单例:local 存储本身是单例,代理也只需要一个
|
||||
# 不传超时参数:让它实时跟随系统设置,前端改完保存即刻生效
|
||||
fsproxy = FileSystemProxy()
|
||||
180
app/modules/filemanager/fsworker.py
Normal file
180
app/modules/filemanager/fsworker.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
文件系统操作代理 worker。
|
||||
|
||||
**本文件不能被 import,只能作为独立脚本执行**(fsproxy 用
|
||||
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动)。直接执行文件
|
||||
路径不会触发 `app/__init__.py` 的导入链,因此这个进程只依赖标准库、启动是
|
||||
毫秒级的;一旦走 import 就会把整个应用的依赖拉进来,代理被强杀后的重启成本
|
||||
会高到无法接受。
|
||||
|
||||
存在的理由:FUSE/网络挂载进入 block 型故障时,`stat`/`listdir`/`rename` 这类
|
||||
系统调用既不返回错误也不返回结果,而 Python 没有中断线程的手段——阻塞其上的
|
||||
线程永远无法回收。放进独立进程后,父进程可以在超时后 SIGKILL 掉它,把
|
||||
「不可处理的 block」转换成「可处理的 crash」。
|
||||
|
||||
协议:stdin/stdout 逐行 JSON。
|
||||
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
|
||||
成功 {"ok": true, "result": {...}}
|
||||
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _stat(payload, _emit):
|
||||
"""
|
||||
读取路径的基本属性。
|
||||
"""
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
|
||||
|
||||
def _exists(payload, _emit):
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
用 os.stat 而不是 os.path.exists:后者会把任意 OSError 都归为「不存在」,
|
||||
挂载抖动会被误判成文件消失。这里让异常原样抛出,由父进程按 errno 区分。
|
||||
"""
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _listdir(payload, _emit):
|
||||
"""
|
||||
列出目录下的条目名。
|
||||
"""
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
|
||||
|
||||
def _copy(payload, emit):
|
||||
"""
|
||||
分块复制文件内容并周期上报进度。
|
||||
|
||||
进度上报同时充当心跳:复制大文件可能持续几小时,父进程无法用固定超时判断
|
||||
挂死,只能看「两次上报之间隔了多久」。因此这里按固定时间间隔上报,即使
|
||||
某一秒没读到数据也照常发——一旦挂载卡住,read/write 不返回,上报自然断流,
|
||||
父进程据此判定并强杀本进程。
|
||||
|
||||
只复制内容和时间戳,不复制权限:目标目录的默认权限与继承 ACL 是媒体库的
|
||||
访问策略,用源文件权限覆盖会清除已继承的 ACL。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
chunk_size = payload.get("chunk_size") or 1024 * 1024
|
||||
interval = payload.get("progress_interval") or 1.0
|
||||
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
# 先发一次 0%:既让心跳立刻开始,也保证父进程在传输开始前就有一次检查
|
||||
# 取消的机会——否则小文件会在首次定时上报之前就复制完,取消形同虚设
|
||||
emit({"ok": True, "progress": {"copied": 0, "total": total}})
|
||||
last_emit = time.monotonic()
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
buf = fsrc.read(chunk_size)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= interval:
|
||||
last_emit = now
|
||||
emit({"ok": True, "progress": {"copied": copied, "total": total}})
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return {"copied": copied, "total": total}
|
||||
|
||||
|
||||
def _rename(payload, _emit):
|
||||
"""
|
||||
同一存储内重命名/移动。
|
||||
|
||||
这是第一版唯一放行的写操作:同文件系统内的 rename 由内核保证原子性,
|
||||
进程被强杀后要么完全成功要么完全没发生,不存在需要清理的中间状态。
|
||||
跨存储的复制+删除不走这里,它需要单独的可恢复语义。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
|
||||
raise OSError(18, "Cross-device rename is not handled by the proxy")
|
||||
os.rename(src, dst)
|
||||
return True
|
||||
|
||||
|
||||
def _unlink(payload, _emit):
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作,强杀后要么删掉了要么没删,没有中间状态。
|
||||
"""
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _rmtree(payload, _emit):
|
||||
"""
|
||||
递归删除目录。
|
||||
|
||||
这一项不是原子的,强杀可能只删掉一部分。放行的理由是:删除被中断的后果
|
||||
(残留若干文件)远轻于写入被中断(留下叫最终文件名的半成品),而且调用方
|
||||
本来就以 ignore_errors 容忍部分失败、可以重复执行直到成功。
|
||||
"""
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
|
||||
|
||||
_HANDLERS = {
|
||||
"stat": _stat,
|
||||
"exists": _exists,
|
||||
"listdir": _listdir,
|
||||
"copy": _copy,
|
||||
"rename": _rename,
|
||||
"unlink": _unlink,
|
||||
"rmtree": _rmtree,
|
||||
"ping": lambda _payload, _emit: True,
|
||||
}
|
||||
|
||||
|
||||
def _write(message):
|
||||
"""
|
||||
输出一行响应。
|
||||
"""
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
请求循环:每读一行处理一个请求,直到 stdin 关闭。
|
||||
|
||||
一个请求可能对应多行响应:长耗时操作先流式发若干 progress 行,最后发一行
|
||||
终态(result 或 error)。父进程据此区分「还在推进」和「已经挂死」。
|
||||
"""
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
handler = _HANDLERS.get(payload.get("op"))
|
||||
if handler is None:
|
||||
response = {"ok": False, "errno": 0,
|
||||
"error": f"unknown op: {payload.get('op')}"}
|
||||
else:
|
||||
response = {"ok": True, "result": handler(payload, _write)}
|
||||
except OSError as err:
|
||||
response = {"ok": False, "errno": err.errno or 0,
|
||||
"error": err.strerror or str(err)}
|
||||
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
|
||||
response = {"ok": False, "errno": 0, "error": str(err)}
|
||||
_write(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -8,6 +8,7 @@ from app import schemas
|
||||
from app.helper.progress import ProgressHelper
|
||||
from app.helper.storage import StorageHelper
|
||||
from app.log import logger
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.utils.crypto import HashUtils
|
||||
|
||||
|
||||
@@ -179,9 +180,13 @@ class StorageBase(metaclass=ABCMeta):
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
默认实现不区分「不存在」与「查询失败」,由具体存储按需覆写。
|
||||
|
||||
默认保守失败:未覆写的存储无法区分「不存在」与「查询失败」,沿用
|
||||
get_item() 会让 overwrite_mode=size 的覆盖保护在查询失败时被绕过,
|
||||
把「无法确认」当成「目标不存在」而放行覆盖。具体存储必须先实现
|
||||
「确认不存在」的判定,再覆写本方法。
|
||||
"""
|
||||
return self.get_item(path)
|
||||
raise StorageQueryError(f"存储 {self.schema} 未实现严格查询,无法确认目标状态: {path}")
|
||||
|
||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
@@ -337,6 +342,7 @@ class StorageBase(metaclass=ABCMeta):
|
||||
files_info[_fileitm.path] = {
|
||||
'size': _fileitm.size or 0,
|
||||
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
||||
'fileid': getattr(_fileitm, 'fileid', None),
|
||||
'type': _fileitm.type
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from app.core.cache import cached
|
||||
from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import OperationInterrupted
|
||||
from app.schemas.exception import OperationInterrupted, StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import WeakSingleton
|
||||
@@ -471,18 +471,58 @@ class Alist(StorageBase, metaclass=WeakSingleton):
|
||||
)
|
||||
return None
|
||||
|
||||
return self.__build_fileitem(path, result["data"])
|
||||
|
||||
def __build_fileitem(self, path: Path, data: dict) -> schemas.FileItem:
|
||||
"""
|
||||
根据接口返回数据构建文件项。
|
||||
:param path: 文件路径
|
||||
:param data: 接口返回的 data 字段
|
||||
:return: 文件项
|
||||
"""
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
type="dir" if result["data"]["is_dir"] else "file",
|
||||
path=path.as_posix() + ("/" if result["data"]["is_dir"] else ""),
|
||||
name=result["data"]["name"],
|
||||
basename=Path(result["data"]["name"]).stem,
|
||||
extension=Path(result["data"]["name"]).suffix[1:],
|
||||
size=result["data"]["size"],
|
||||
modify_time=self.__parse_timestamp(result["data"]["modified"]),
|
||||
thumbnail=result["data"]["thumb"],
|
||||
type="dir" if data["is_dir"] else "file",
|
||||
path=path.as_posix() + ("/" if data["is_dir"] else ""),
|
||||
name=data["name"],
|
||||
basename=Path(data["name"]).stem,
|
||||
extension=Path(data["name"]).suffix[1:],
|
||||
size=data["size"],
|
||||
modify_time=self.__parse_timestamp(data["modified"]),
|
||||
thumbnail=data["thumb"],
|
||||
)
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
只有接口明确回报「对象不存在」才是确定结果,连接失败、HTTP 异常与其他
|
||||
业务错误都无法确认目标状态,必须保守失败以免覆盖保护被绕过。
|
||||
"""
|
||||
resp = RequestUtils(headers=self.__get_header_with_token()).post_res(
|
||||
self.__get_api_url("/api/fs/get"),
|
||||
json={
|
||||
"path": path.as_posix(),
|
||||
"password": "",
|
||||
"page": 1,
|
||||
"per_page": 0,
|
||||
"refresh": False,
|
||||
},
|
||||
)
|
||||
if resp is None:
|
||||
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,无法连接服务")
|
||||
if resp.status_code != 200:
|
||||
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,状态码:{resp.status_code}")
|
||||
try:
|
||||
result = resp.json()
|
||||
except Exception as err:
|
||||
raise StorageQueryError(f"【OpenList】解析查询结果失败: {path} - {err}") from err
|
||||
if result.get("code") != 200:
|
||||
message = str(result.get("message") or "")
|
||||
if "not found" in message.lower() or "not exist" in message.lower():
|
||||
return None
|
||||
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败:{message}")
|
||||
return self.__build_fileitem(path, result["data"])
|
||||
|
||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取父目录
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
@@ -7,6 +8,7 @@ from app import schemas
|
||||
from app.core.config import global_vars, settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.fsproxy import fsproxy
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
@@ -47,6 +49,10 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
获取文件项
|
||||
"""
|
||||
# 走代理读取:挂载挂死时这一步会在超时后抛 OSError,而不是永久悬挂线程。
|
||||
# 顺带只 stat 一次——原先 size 与 modify_time 各 stat 一次,在网络挂载上
|
||||
# 等于把这个热点路径的开销翻倍
|
||||
info = fsproxy.stat(path)
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
type="file",
|
||||
@@ -54,8 +60,8 @@ class LocalStorage(StorageBase):
|
||||
name=path.name,
|
||||
basename=path.stem,
|
||||
extension=path.suffix[1:],
|
||||
size=path.stat().st_size,
|
||||
modify_time=path.stat().st_mtime,
|
||||
size=info["size"],
|
||||
modify_time=info["mtime"],
|
||||
)
|
||||
|
||||
def __get_diritem(self, path: Path) -> schemas.FileItem:
|
||||
@@ -68,7 +74,7 @@ class LocalStorage(StorageBase):
|
||||
path=path.as_posix() + "/",
|
||||
name=path.name,
|
||||
basename=path.stem,
|
||||
modify_time=path.stat().st_mtime,
|
||||
modify_time=fsproxy.stat(path)["mtime"],
|
||||
)
|
||||
|
||||
def list(self, fileitem: schemas.FileItem) -> List[schemas.FileItem]:
|
||||
@@ -100,12 +106,14 @@ class LocalStorage(StorageBase):
|
||||
|
||||
# 遍历目录
|
||||
path_obj = Path(path)
|
||||
if not path_obj.exists():
|
||||
try:
|
||||
info = fsproxy.stat(path_obj)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
logger.warn(f"【本地】目录不存在:{path}")
|
||||
return []
|
||||
|
||||
# 如果是文件
|
||||
if path_obj.is_file():
|
||||
if info["is_file"]:
|
||||
ret_items.append(self.__get_fileitem(path_obj))
|
||||
return ret_items
|
||||
|
||||
@@ -143,9 +151,11 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
获取文件或目录,不存在返回None
|
||||
"""
|
||||
if not path.exists():
|
||||
try:
|
||||
info = fsproxy.stat(path)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
if path.is_file():
|
||||
if info["is_file"]:
|
||||
return self.__get_fileitem(path)
|
||||
return self.__get_diritem(path)
|
||||
|
||||
@@ -154,9 +164,11 @@ class LocalStorage(StorageBase):
|
||||
获取文件或目录,无法确认状态时抛出 StorageQueryError。
|
||||
Path.exists() 会把部分 errno(如 EBADF/ELOOP)归入「不存在」,
|
||||
网络/FUSE 挂载抖动时会误判,这里用 stat 显式区分。
|
||||
挂载完全无响应时代理会超时并抛 FileSystemTimeout(OSError 子类),
|
||||
同样落入下面的分支,转化成调用方能处理的查询失败。
|
||||
"""
|
||||
try:
|
||||
path.stat()
|
||||
fsproxy.stat(path)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as e:
|
||||
@@ -182,13 +194,18 @@ class LocalStorage(StorageBase):
|
||||
if not fileitem.path:
|
||||
return False
|
||||
path_obj = Path(fileitem.path)
|
||||
if not path_obj.exists():
|
||||
return True
|
||||
try:
|
||||
if path_obj.is_file():
|
||||
path_obj.unlink()
|
||||
info = fsproxy.stat(path_obj)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error(f"【本地】读取待删除文件状态失败:{e}")
|
||||
return False
|
||||
try:
|
||||
if info["is_file"]:
|
||||
fsproxy.unlink(path_obj)
|
||||
else:
|
||||
shutil.rmtree(path_obj, ignore_errors=True)
|
||||
fsproxy.rmtree(path_obj)
|
||||
except Exception as e:
|
||||
logger.error(f"【本地】删除文件失败:{e}")
|
||||
return False
|
||||
@@ -199,10 +216,10 @@ class LocalStorage(StorageBase):
|
||||
重命名文件
|
||||
"""
|
||||
path_obj = Path(fileitem.path)
|
||||
if not path_obj.exists():
|
||||
return False
|
||||
try:
|
||||
path_obj.rename(path_obj.parent / name)
|
||||
fsproxy.rename(path_obj, path_obj.parent / name)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"【本地】重命名文件失败:{e}")
|
||||
return False
|
||||
@@ -214,6 +231,93 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
return Path(fileitem.path)
|
||||
|
||||
# 写入中的临时文件后缀。点开头(隐藏)+ 专用后缀双重保证:即使进程被
|
||||
# SIGKILL、临时文件残留,媒体库也不会把半成品当成媒体收录
|
||||
PARTIAL_SUFFIX = ".mp-partial"
|
||||
# 临时文件被认定为中断残留的时长(秒)。正常失败路径会自行清理,只有被
|
||||
# 强杀才会残留;阈值取得宽松,避免误删仍在写入的大文件
|
||||
PARTIAL_STALE_SECONDS = 24 * 3600
|
||||
|
||||
@classmethod
|
||||
def _partial_path(cls, dest: Path) -> Path:
|
||||
"""
|
||||
生成写入中的临时文件路径。
|
||||
|
||||
必须与目标同目录:os.replace 只有在同一文件系统内才是原子的,放到
|
||||
/tmp 之类的地方会退化成一次完整拷贝,原子性荡然无存。带 PID 是为了
|
||||
避免多进程同时写同一目标时互相踩踏。
|
||||
:param dest: 目标文件路径
|
||||
:return: 临时文件路径
|
||||
"""
|
||||
return dest.parent / f".{dest.name}.{os.getpid()}{cls.PARTIAL_SUFFIX}"
|
||||
|
||||
@classmethod
|
||||
def _cleanup_stale_partials(cls, directory: Path):
|
||||
"""
|
||||
清理目录下中断残留的临时文件。
|
||||
|
||||
只做局部清理而不是全库扫描:在网络挂载上遍历整个媒体库代价不可接受,
|
||||
而残留只可能出现在曾经写入过的目录里,因此每次写入时顺带清理即可。
|
||||
本方法是尽力而为的旁路操作,任何失败都不影响主流程。
|
||||
:param directory: 目标目录
|
||||
"""
|
||||
try:
|
||||
threshold = time.time() - cls.PARTIAL_STALE_SECONDS
|
||||
for item in directory.glob(f"*{cls.PARTIAL_SUFFIX}"):
|
||||
try:
|
||||
if item.stat().st_mtime < threshold:
|
||||
item.unlink()
|
||||
logger.info(f"【本地】已清理中断残留的临时文件:{item}")
|
||||
except OSError:
|
||||
continue
|
||||
except Exception as err:
|
||||
logger.debug(f"【本地】清理临时文件失败:{directory} - {err}")
|
||||
|
||||
def _write_atomically(self, src: Path, dest: Path) -> bool:
|
||||
"""
|
||||
以「写临时名 → os.replace」的方式把源文件内容落到目标。
|
||||
|
||||
直接写目标路径的话,进程被杀(OOM、重启、宿主断电、SIGKILL)会在媒体库
|
||||
里留下一个**叫最终文件名的半截文件**:媒体库会把它扫进去,后续的
|
||||
「目标已存在」判断也会把它当成完成品。os.replace 在同目录内由内核保证
|
||||
原子性,因此目标要么完整存在,要么根本不存在。
|
||||
:param src: 源文件路径
|
||||
:param dest: 目标文件路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
self._cleanup_stale_partials(dest.parent)
|
||||
partial = self._partial_path(dest)
|
||||
# 进度只在需要展示时才回调 UI,但代理内部始终按固定间隔上报——那是判定
|
||||
# 「传输是否还在推进」的心跳,不能因为不展示进度就关掉
|
||||
progress_callback = (
|
||||
transfer_process(src.as_posix())
|
||||
if self.__should_show_progress(src, dest) else None
|
||||
)
|
||||
try:
|
||||
copied = fsproxy.copy(
|
||||
src, partial,
|
||||
progress_cb=progress_callback,
|
||||
cancel_cb=lambda: global_vars.is_transfer_stopped(src.as_posix()),
|
||||
chunk_size=self.chunk_size,
|
||||
)
|
||||
if not copied:
|
||||
logger.info(f"【本地】{src} 复制未完成")
|
||||
return False
|
||||
os.replace(partial, dest)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】复制文件失败:{err}")
|
||||
return False
|
||||
finally:
|
||||
if progress_callback:
|
||||
progress_callback(100)
|
||||
# 失败路径留下的临时文件就地清掉;成功时 replace 已经把它移走
|
||||
try:
|
||||
if partial.exists():
|
||||
partial.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _copy_with_target_permissions(src: Path, dest: Path) -> Path:
|
||||
"""
|
||||
@@ -276,12 +380,13 @@ class LocalStorage(StorageBase):
|
||||
try:
|
||||
dir_path = Path(fileitem.path)
|
||||
target_path = dir_path / (new_name or path.name)
|
||||
if self._copy_with_progress(path, target_path):
|
||||
# 先原子地把内容落到目标,确认完整之后才删源
|
||||
if self._write_atomically(path, target_path):
|
||||
# 上传删除源文件
|
||||
path.unlink()
|
||||
return self.get_item(target_path)
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】移动文件失败:{err}")
|
||||
logger.error(f"【本地】上传文件失败:{err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
@@ -304,18 +409,7 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
复制文件(带进度)
|
||||
"""
|
||||
try:
|
||||
src = Path(fileitem.path)
|
||||
dest = path / new_name
|
||||
if self.__should_show_progress(src, dest):
|
||||
if self._copy_with_progress(src, dest):
|
||||
return True
|
||||
else:
|
||||
self._copy_with_target_permissions(src, dest)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】复制文件失败:{err}")
|
||||
return False
|
||||
return self._write_atomically(Path(fileitem.path), path / new_name)
|
||||
|
||||
def move(
|
||||
self,
|
||||
@@ -326,23 +420,28 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
移动文件(带进度)
|
||||
"""
|
||||
src = Path(fileitem.path)
|
||||
dest = path / new_name
|
||||
if src == dest:
|
||||
# 目标和源文件相同,直接返回成功,不做任何操作
|
||||
return True
|
||||
try:
|
||||
src = Path(fileitem.path)
|
||||
dest = path / new_name
|
||||
if src == dest:
|
||||
# 目标和源文件相同,直接返回成功,不做任何操作
|
||||
return True
|
||||
if self.__should_show_progress(src, dest):
|
||||
if self._copy_with_progress(src, dest):
|
||||
# 复制成功删除源文件
|
||||
src.unlink()
|
||||
return True
|
||||
else:
|
||||
shutil.move(src, dest, copy_function=self._copy_with_target_permissions)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】移动文件失败:{err}")
|
||||
return False
|
||||
# 同一文件系统内 rename 是原子操作:中断后要么完全成功、要么完全
|
||||
# 没发生,既不需要临时文件也不会留下半成品。直接尝试而不预先比较
|
||||
# st_dev,省掉挂载上的两次 stat——跨设备会以 EXDEV 失败并落到下面
|
||||
os.replace(src, dest)
|
||||
return True
|
||||
except OSError as err:
|
||||
logger.debug(f"【本地】直接移动未成功,降级为复制:{src} -> {dest} - {err}")
|
||||
# 跨文件系统:先原子地把内容落到目标,确认完整之后才删源。
|
||||
# 顺序不能反——先删源再失败就是永久丢件
|
||||
if not self._write_atomically(src, dest):
|
||||
return False
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError as err:
|
||||
logger.warn(f"【本地】移动已完成但删除源文件失败:{src} - {err}")
|
||||
return True
|
||||
|
||||
def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool:
|
||||
"""
|
||||
|
||||
@@ -10,6 +10,7 @@ from app import schemas
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
@@ -299,6 +300,38 @@ class Rclone(StorageBase):
|
||||
logger.debug(f"【rclone】获取文件项失败:{err}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
rclone 用退出码 3/4 表示目录/文件不存在,其余非零退出无法区分
|
||||
「不存在」与「查询失败」,必须保守失败以免覆盖保护被绕过。
|
||||
"""
|
||||
try:
|
||||
ret = subprocess.run(
|
||||
[
|
||||
'rclone', 'lsjson',
|
||||
f'MP:{path.parent}'
|
||||
],
|
||||
capture_output=True,
|
||||
startupinfo=self.__get_hidden_shell()
|
||||
)
|
||||
except Exception as err:
|
||||
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {err}") from err
|
||||
if ret.returncode in (3, 4):
|
||||
# 目录或文件不存在,是确定结果
|
||||
return None
|
||||
if ret.returncode != 0:
|
||||
errmsg = (ret.stderr or b"").decode(errors="ignore").strip()
|
||||
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {errmsg}")
|
||||
try:
|
||||
items = json.loads(ret.stdout)
|
||||
except Exception as err:
|
||||
raise StorageQueryError(f"【rclone】解析查询结果失败: {path} - {err}") from err
|
||||
for item in items:
|
||||
if item.get("Name") == path.name:
|
||||
return self.__get_rcloneitem(item, parent=str(path.parent) + "/")
|
||||
return None
|
||||
|
||||
def delete(self, fileitem: schemas.FileItem) -> bool:
|
||||
"""
|
||||
删除文件
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import errno
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -16,6 +17,7 @@ from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.modules.filemanager.storages import transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
@@ -163,7 +165,8 @@ class SMB(StorageBase, metaclass=WeakSingleton):
|
||||
|
||||
# 构建完整的SMB路径
|
||||
if path_str:
|
||||
return f"{self._server_path}\\{path_str.replace('/', '\\')}"
|
||||
normalized_path = path_str.replace("/", "\\")
|
||||
return f"{self._server_path}\\{normalized_path}"
|
||||
else:
|
||||
return self._server_path
|
||||
|
||||
@@ -379,6 +382,39 @@ class SMB(StorageBase, metaclass=WeakSingleton):
|
||||
logger.debug(f"【SMB】获取文件项失败: {e}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
只有 ENOENT/ENOTDIR 才是「确认不存在」,连接中断、认证失败等都无法确认
|
||||
目标状态,必须保守失败以免覆盖保护被绕过。
|
||||
"""
|
||||
try:
|
||||
self._check_connection()
|
||||
|
||||
# 处理根目录
|
||||
if str(path) == "/":
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
type="dir",
|
||||
path="/",
|
||||
name="",
|
||||
basename="",
|
||||
modify_time=int(time.time()),
|
||||
)
|
||||
|
||||
smb_path = self._normalize_path(str(path).rstrip("/"))
|
||||
try:
|
||||
stat_result = smbclient.stat(smb_path)
|
||||
except OSError as err:
|
||||
if err.errno in (errno.ENOENT, errno.ENOTDIR):
|
||||
return None
|
||||
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {err}") from err
|
||||
return self._create_fileitem(stat_result, smb_path, Path(path).name)
|
||||
except StorageQueryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {e}") from e
|
||||
|
||||
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件详情
|
||||
|
||||
@@ -545,6 +545,7 @@ class TransHandler:
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
overwrite_skipped=True,
|
||||
)
|
||||
return result
|
||||
elif overwrite_mode == "always":
|
||||
@@ -571,6 +572,7 @@ class TransHandler:
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
overwrite_skipped=True,
|
||||
)
|
||||
return result
|
||||
else:
|
||||
@@ -614,6 +616,7 @@ class TransHandler:
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
overwrite_skipped=True,
|
||||
)
|
||||
return result
|
||||
elif overwrite_mode == "latest":
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.modules.themoviedb.category import CategoryHelper
|
||||
from app.modules.themoviedb.scraper import TmdbScraper
|
||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||
from app.modules.themoviedb.tmdbapi import TmdbApi
|
||||
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbConnectionError
|
||||
from app.schemas.category import CategoryConfig
|
||||
from app.schemas.types import (
|
||||
MediaImageType,
|
||||
@@ -193,16 +194,42 @@ class TheMovieDbModule(_ModuleBase):
|
||||
media.season = meta.begin_season
|
||||
return medias
|
||||
|
||||
def _safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
|
||||
"""
|
||||
查询指定类型的媒体详情,将"确认TMDB连接失败"与"确认查无此项"区分开。
|
||||
|
||||
:param mtype: 媒体类型:电影或电视剧
|
||||
:param tmdbid: TMDB的ID
|
||||
:return: (媒体信息或None, 本次查询是否因TMDB连接失败而没有得到确定结果)
|
||||
"""
|
||||
try:
|
||||
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
|
||||
except TMDbConnectionError:
|
||||
return None, True
|
||||
|
||||
async def _async_safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
|
||||
"""
|
||||
查询指定类型的媒体详情,将"确认TMDB连接失败"与"确认查无此项"区分开(异步版本)
|
||||
"""
|
||||
try:
|
||||
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
|
||||
except TMDbConnectionError:
|
||||
return None, True
|
||||
|
||||
def _get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
||||
meta: Optional[MetaBase]) -> Optional[dict]:
|
||||
"""
|
||||
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧
|
||||
|
||||
:raises TMDbConnectionError: 电影、电视剧两路查询都没有得到确定结果,且至少一路
|
||||
是因TMDB连接失败导致的,此时不能断言"条目不存在",交由上层报网络故障
|
||||
"""
|
||||
if mtype:
|
||||
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid)
|
||||
# 类型未知,分别查询电影和电视剧
|
||||
info_tv = self.tmdb.get_info(mtype=MediaType.TV, tmdbid=tmdbid)
|
||||
info_movie = self.tmdb.get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid)
|
||||
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
|
||||
# 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
|
||||
# 避免一路瞬时抖动掩盖另一路已经得到的确定结果
|
||||
info_tv, tv_conn_error = self._safe_get_info_by_type(MediaType.TV, tmdbid)
|
||||
info_movie, movie_conn_error = self._safe_get_info_by_type(MediaType.MOVIE, tmdbid)
|
||||
if info_tv and info_movie:
|
||||
# 同时存在,尝试通过元数据消歧
|
||||
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
||||
@@ -210,18 +237,26 @@ class TheMovieDbModule(_ModuleBase):
|
||||
return result
|
||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||
return None
|
||||
return info_tv or info_movie or None
|
||||
if info_tv or info_movie:
|
||||
return info_tv or info_movie
|
||||
if tv_conn_error or movie_conn_error:
|
||||
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
|
||||
return None
|
||||
|
||||
async def _async_get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
||||
meta: Optional[MetaBase]) -> Optional[dict]:
|
||||
"""
|
||||
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧(异步版本)
|
||||
|
||||
:raises TMDbConnectionError: 电影、电视剧两路查询都没有得到确定结果,且至少一路
|
||||
是因TMDB连接失败导致的,此时不能断言"条目不存在",交由上层报网络故障
|
||||
"""
|
||||
if mtype:
|
||||
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid)
|
||||
# 类型未知,分别查询电影和电视剧
|
||||
info_tv = await self.tmdb.async_get_info(mtype=MediaType.TV, tmdbid=tmdbid)
|
||||
info_movie = await self.tmdb.async_get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid)
|
||||
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
|
||||
# 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
|
||||
# 避免一路瞬时抖动掩盖另一路已经得到的确定结果
|
||||
info_tv, tv_conn_error = await self._async_safe_get_info_by_type(MediaType.TV, tmdbid)
|
||||
info_movie, movie_conn_error = await self._async_safe_get_info_by_type(MediaType.MOVIE, tmdbid)
|
||||
if info_tv and info_movie:
|
||||
# 同时存在,尝试通过元数据消歧
|
||||
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
||||
@@ -229,7 +264,11 @@ class TheMovieDbModule(_ModuleBase):
|
||||
return result
|
||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||
return None
|
||||
return info_tv or info_movie or None
|
||||
if info_tv or info_movie:
|
||||
return info_tv or info_movie
|
||||
if tv_conn_error or movie_conn_error:
|
||||
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _disambiguate_by_meta(info_tv: dict, info_movie: dict,
|
||||
@@ -525,10 +564,15 @@ class TheMovieDbModule(_ModuleBase):
|
||||
# 识别匹配
|
||||
if not cache_info or not cache:
|
||||
info = None
|
||||
connection_error = False
|
||||
# 缓存没有或者强制不使用缓存
|
||||
if tmdbid:
|
||||
# 直接查询详情,支持同ID电影/电视剧消歧
|
||||
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||
try:
|
||||
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(f"tmdb_id:{tmdbid} {err}")
|
||||
connection_error = True
|
||||
if not info and meta and not tmdbid:
|
||||
# 准备搜索名称
|
||||
names = self._prepare_search_names(meta)
|
||||
@@ -542,7 +586,11 @@ class TheMovieDbModule(_ModuleBase):
|
||||
info = self.tmdb.get_info(mtype=info.get("media_type"),
|
||||
tmdbid=info.get("id"))
|
||||
elif not info:
|
||||
if tmdbid:
|
||||
if connection_error:
|
||||
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
|
||||
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
|
||||
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
|
||||
elif tmdbid:
|
||||
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
||||
else:
|
||||
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
||||
@@ -624,10 +672,15 @@ class TheMovieDbModule(_ModuleBase):
|
||||
# 识别匹配
|
||||
if not cache_info or not cache:
|
||||
info = None
|
||||
connection_error = False
|
||||
# 缓存没有或者强制不使用缓存
|
||||
if tmdbid:
|
||||
# 直接查询详情,支持同ID电影/电视剧消歧
|
||||
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||
try:
|
||||
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(f"tmdb_id:{tmdbid} {err}")
|
||||
connection_error = True
|
||||
if not info and meta and not tmdbid:
|
||||
# 准备搜索名称
|
||||
names = self._prepare_search_names(meta)
|
||||
@@ -641,7 +694,11 @@ class TheMovieDbModule(_ModuleBase):
|
||||
info = await self.tmdb.async_get_info(mtype=info.get("media_type"),
|
||||
tmdbid=info.get("id"))
|
||||
elif not info:
|
||||
if tmdbid:
|
||||
if connection_error:
|
||||
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
|
||||
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
|
||||
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
|
||||
elif tmdbid:
|
||||
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
||||
else:
|
||||
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
||||
|
||||
@@ -3,6 +3,7 @@ import traceback
|
||||
from math import ceil
|
||||
from threading import RLock
|
||||
from time import time
|
||||
from typing import Any
|
||||
|
||||
from app.core.cache import FileCache, TTLCache
|
||||
from app.core.config import settings
|
||||
@@ -144,6 +145,30 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
media_id = meta.media_id if meta.media_source == MediaSource.TMDB else None
|
||||
return f"[{meta.type.value if meta.type else '未知'}][{settings.TMDB_LOCALE}]{media_id or meta.name}-{meta.year}-{meta.begin_season}"
|
||||
|
||||
@staticmethod
|
||||
def __is_type_conflicted(meta: MetaBase, media_type: Any, tmdb_id: Any) -> bool:
|
||||
"""
|
||||
判断媒体类型是否与元数据声明的类型冲突。
|
||||
|
||||
只有「元数据判定为电视剧、结果却是电影」才算冲突。反向不算:名称识别在
|
||||
电影分支查不到时会回退到电视剧查询,识别缓存正是用来记住这个纠正结果,
|
||||
一律要求 key 与 value 类型一致会让这类条目每次都被丢弃、反复回源。而电视
|
||||
剧分支恒定写入电视剧类型,`[电视剧]` 键下出现电影只可能来自 tmdbid 消歧
|
||||
或共享识别回填的脏写,会让整季剧集被当成电影反复整理失败。
|
||||
:param meta: 元数据
|
||||
:param media_type: 待校验的媒体类型
|
||||
:param tmdb_id: 对应的 TMDB ID,为空表示负缓存,不带类型信息
|
||||
:return: 是否冲突
|
||||
"""
|
||||
if meta.type != MediaType.TV or not tmdb_id:
|
||||
return False
|
||||
if not isinstance(media_type, MediaType):
|
||||
try:
|
||||
media_type = MediaType(media_type)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return media_type == MediaType.MOVIE
|
||||
|
||||
def get(self, meta: MetaBase):
|
||||
"""
|
||||
根据KEY值获取缓存值
|
||||
@@ -154,7 +179,17 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
cache_data = self._cache.get(key)
|
||||
if not cache_data and self._expires_at.pop(key, None) is not None:
|
||||
self._dirty = True
|
||||
return cache_data or {}
|
||||
if not cache_data or not isinstance(cache_data, dict):
|
||||
return {}
|
||||
if self.__is_type_conflicted(meta, cache_data.get("type"), cache_data.get("id")):
|
||||
# 脏条目不丢弃就会被无限期沿用,正确的识别逻辑永远没有执行机会
|
||||
logger.warn(f"识别缓存类型与元数据冲突,已丢弃并重新识别:{key} -> "
|
||||
f"{cache_data.get('title')}({cache_data.get('type')})")
|
||||
self._cache.delete(key)
|
||||
self._expires_at.pop(key, None)
|
||||
self._dirty = True
|
||||
return {}
|
||||
return cache_data
|
||||
|
||||
def delete(self, key: str) -> dict:
|
||||
"""
|
||||
@@ -193,6 +228,11 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
"""
|
||||
key = self.__get_key(meta)
|
||||
if info:
|
||||
if self.__is_type_conflicted(meta, info.get("media_type"), info.get("id")):
|
||||
# 拒绝写入而不是改写键:识别结果照常返回,只是不把矛盾条目留给下一次
|
||||
logger.warn(f"识别结果类型与元数据冲突,不写入识别缓存:{key} -> "
|
||||
f"{info.get('title')}({info.get('media_type')})")
|
||||
return
|
||||
# 缓存标题
|
||||
cache_title = info.get("title") \
|
||||
if info.get("media_type") == MediaType.MOVIE else info.get("name")
|
||||
|
||||
@@ -8,7 +8,7 @@ from app.schemas.types import MediaType
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.zhconv import convert as zhconv_convert
|
||||
from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
|
||||
from .tmdbv3api.exceptions import TMDbException
|
||||
from .tmdbv3api.exceptions import TMDbException, TMDbConnectionError
|
||||
|
||||
|
||||
class TmdbApi:
|
||||
@@ -584,11 +584,14 @@ class TmdbApi:
|
||||
|
||||
def get_info(self,
|
||||
mtype: MediaType,
|
||||
tmdbid: int) -> dict:
|
||||
tmdbid: int,
|
||||
raise_on_connection_error: bool = False) -> dict:
|
||||
"""
|
||||
给定TMDB号,查询一条媒体信息
|
||||
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
||||
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
||||
:param raise_on_connection_error: 为True时,遇到TMDB连接失败(区别于404等业务错误)
|
||||
将抛出TMDbConnectionError而不是吞掉返回None;默认False,与既有调用方行为完全一致
|
||||
"""
|
||||
|
||||
def __get_genre_ids(genres: list) -> list:
|
||||
@@ -604,16 +607,16 @@ class TmdbApi:
|
||||
|
||||
# 查询TMDB详情
|
||||
if mtype == MediaType.MOVIE:
|
||||
tmdb_info = self.__get_movie_detail(tmdbid)
|
||||
tmdb_info = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info:
|
||||
tmdb_info['media_type'] = MediaType.MOVIE
|
||||
elif mtype == MediaType.TV:
|
||||
tmdb_info = self.__get_tv_detail(tmdbid)
|
||||
tmdb_info = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info:
|
||||
tmdb_info['media_type'] = MediaType.TV
|
||||
else:
|
||||
tmdb_info_tv = self.__get_tv_detail(tmdbid)
|
||||
tmdb_info_movie = self.__get_movie_detail(tmdbid)
|
||||
tmdb_info_tv = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
tmdb_info_movie = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info_tv and tmdb_info_movie:
|
||||
tmdb_info = None
|
||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||
@@ -797,10 +800,13 @@ class TmdbApi:
|
||||
"alternative_titles,"
|
||||
"translations,"
|
||||
"release_dates,"
|
||||
"external_ids") -> Optional[dict]:
|
||||
"external_ids",
|
||||
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||
"""
|
||||
获取电影的详情
|
||||
:param tmdbid: TMDB ID
|
||||
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||
:return: TMDB信息
|
||||
"""
|
||||
"""
|
||||
@@ -899,6 +905,11 @@ class TmdbApi:
|
||||
if tmdbinfo:
|
||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
||||
return tmdbinfo or {}
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(str(err))
|
||||
if raise_on_connection_error:
|
||||
raise
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
return None
|
||||
@@ -911,10 +922,13 @@ class TmdbApi:
|
||||
"translations,"
|
||||
"content_ratings,"
|
||||
"external_ids,"
|
||||
"episode_groups") -> Optional[dict]:
|
||||
"episode_groups",
|
||||
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||
"""
|
||||
获取电视剧的详情
|
||||
:param tmdbid: TMDB ID
|
||||
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||
:return: TMDB信息
|
||||
"""
|
||||
"""
|
||||
@@ -1084,6 +1098,11 @@ class TmdbApi:
|
||||
if tmdbinfo:
|
||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
||||
return tmdbinfo or {}
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(str(err))
|
||||
if raise_on_connection_error:
|
||||
raise
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
return None
|
||||
@@ -1666,10 +1685,13 @@ class TmdbApi:
|
||||
"alternative_titles,"
|
||||
"translations,"
|
||||
"release_dates,"
|
||||
"external_ids") -> Optional[dict]:
|
||||
"external_ids",
|
||||
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||
"""
|
||||
获取电影的详情(异步版本)
|
||||
:param tmdbid: TMDB ID
|
||||
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||
:return: TMDB信息
|
||||
"""
|
||||
if not self.movie:
|
||||
@@ -1680,6 +1702,11 @@ class TmdbApi:
|
||||
if tmdbinfo:
|
||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
||||
return tmdbinfo or {}
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(str(err))
|
||||
if raise_on_connection_error:
|
||||
raise
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
return None
|
||||
@@ -1692,10 +1719,13 @@ class TmdbApi:
|
||||
"translations,"
|
||||
"content_ratings,"
|
||||
"external_ids,"
|
||||
"episode_groups") -> Optional[dict]:
|
||||
"episode_groups",
|
||||
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||
"""
|
||||
获取电视剧的详情(异步版本)
|
||||
:param tmdbid: TMDB ID
|
||||
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||
:return: TMDB信息
|
||||
"""
|
||||
if not self.tv:
|
||||
@@ -1706,6 +1736,11 @@ class TmdbApi:
|
||||
if tmdbinfo:
|
||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
||||
return tmdbinfo or {}
|
||||
except TMDbConnectionError as err:
|
||||
logger.error(str(err))
|
||||
if raise_on_connection_error:
|
||||
raise
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
return None
|
||||
@@ -1922,11 +1957,14 @@ class TmdbApi:
|
||||
|
||||
async def async_get_info(self,
|
||||
mtype: MediaType,
|
||||
tmdbid: int) -> dict:
|
||||
tmdbid: int,
|
||||
raise_on_connection_error: bool = False) -> dict:
|
||||
"""
|
||||
给定TMDB号,查询一条媒体信息(异步版本)
|
||||
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
||||
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
||||
:param raise_on_connection_error: 为True时,遇到TMDB连接失败(区别于404等业务错误)
|
||||
将抛出TMDbConnectionError而不是吞掉返回None;默认False,与既有调用方行为完全一致
|
||||
"""
|
||||
|
||||
def __get_genre_ids(genres: list) -> list:
|
||||
@@ -1942,16 +1980,16 @@ class TmdbApi:
|
||||
|
||||
# 查询TMDB详情
|
||||
if mtype == MediaType.MOVIE:
|
||||
tmdb_info = await self.__async_get_movie_detail(tmdbid)
|
||||
tmdb_info = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info:
|
||||
tmdb_info['media_type'] = MediaType.MOVIE
|
||||
elif mtype == MediaType.TV:
|
||||
tmdb_info = await self.__async_get_tv_detail(tmdbid)
|
||||
tmdb_info = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info:
|
||||
tmdb_info['media_type'] = MediaType.TV
|
||||
else:
|
||||
tmdb_info_tv = await self.__async_get_tv_detail(tmdbid)
|
||||
tmdb_info_movie = await self.__async_get_movie_detail(tmdbid)
|
||||
tmdb_info_tv = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
tmdb_info_movie = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||
if tmdb_info_tv and tmdb_info_movie:
|
||||
tmdb_info = None
|
||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||
|
||||
@@ -1,2 +1,16 @@
|
||||
class TMDbException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TMDbConnectionError(TMDbException):
|
||||
"""
|
||||
TMDB连接失败异常。
|
||||
|
||||
仅在确认为传输层/响应格式问题(如底层HTTP请求失败、响应无法解析为JSON)时抛出,
|
||||
与TMDB业务层明确返回的错误(如404条目不存在、参数错误等,仍抛出普通TMDbException)
|
||||
区分开,便于上层区分"网络故障,请重试"与"条目确实不存在"两类完全不同的处理与文案。
|
||||
|
||||
继承自TMDbException,因此现有 `except TMDbException` 代码路径无需修改即可
|
||||
继续捕获本异常,保持向后兼容。
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -12,10 +12,30 @@ import requests.exceptions
|
||||
from app.core.cache import cached, fresh, async_fresh
|
||||
from app.core.config import settings
|
||||
from app.utils.http import RequestUtils, AsyncRequestUtils
|
||||
from .exceptions import TMDbException
|
||||
from .exceptions import TMDbException, TMDbConnectionError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 单次重试前的退避等待时间(秒)。NAS+FUSE网盘等环境下TMDB连接的失败大多是数秒内
|
||||
# 可自愈的瞬时抖动,零间隔重试(或异步完全不重试)基本无法穿越这类抖动窗口;
|
||||
# 识别链路是同步阻塞调用,1-3秒的等待可接受,超出3秒则会让识别耗时明显变长,
|
||||
# 故取区间内的经验值。
|
||||
RETRY_BACKOFF_SECONDS = 2
|
||||
|
||||
|
||||
def _is_business_failure_snapshot(snapshot) -> bool:
|
||||
"""
|
||||
判断响应快照是否为TMDB业务失败(success=false,如404/限流/服务端错误的合法JSON)。
|
||||
|
||||
这类响应若入缓存会把瞬时失败固化整个TTL周期(如12小时),期间同key请求
|
||||
直接命中失败快照;跳过缓存让下次请求重新确认。真正的负缓存(条目确认
|
||||
不存在)由上层 TmdbCache 负责,request 层不做失败记忆。
|
||||
"""
|
||||
if not isinstance(snapshot, dict):
|
||||
return False
|
||||
json_data = snapshot.get("json")
|
||||
return isinstance(json_data, dict) and json_data.get("success") is False
|
||||
|
||||
|
||||
class TMDb(object):
|
||||
_RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__"
|
||||
@@ -136,15 +156,21 @@ class TMDb(object):
|
||||
def wait_on_rate_limit(self, wait_on_rate_limit):
|
||||
self._wait_on_rate_limit = bool(wait_on_rate_limit)
|
||||
|
||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True)
|
||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||
skip_if=_is_business_failure_snapshot)
|
||||
def request(self, method, url, data, json, **kwargs):
|
||||
req = self._request_once(method, url, data, json)
|
||||
if req is None and method == "GET" and self._owns_session:
|
||||
logger.debug("TMDB同步请求失败,重建会话后重试一次")
|
||||
logger.debug(f"TMDB同步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重建会话重试一次")
|
||||
# 同步阻塞识别链线程;1-3秒的退避等待可接受,能显著提升对瞬时抖动的容错,
|
||||
# 详见模块级常量 RETRY_BACKOFF_SECONDS 的说明。
|
||||
time.sleep(RETRY_BACKOFF_SECONDS)
|
||||
self._reset_owned_session()
|
||||
req = self._request_once(method, url, data, json)
|
||||
if req is None:
|
||||
raise TMDbException("无法连接TheMovieDb,请检查网络连接!")
|
||||
# 抛出更具体的连接异常子类,供上层(如TMDB详情查询)区分"网络故障"
|
||||
# 与"TMDB业务层明确返回的错误"(如404条目不存在),两者不能混为一谈。
|
||||
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||
return self._snapshot_response(req)
|
||||
|
||||
def _request_once(self, method, url, data, json):
|
||||
@@ -155,16 +181,28 @@ class TMDb(object):
|
||||
return self._req.get_res(url, params=data, json=json)
|
||||
return self._req.post_res(url, data=data, json=json)
|
||||
|
||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True)
|
||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||
skip_if=_is_business_failure_snapshot)
|
||||
async def async_request(self, method, url, data, json, **kwargs):
|
||||
if method == "GET":
|
||||
req = await self._async_req.get_res(url, params=data, json=json)
|
||||
else:
|
||||
req = await self._async_req.post_res(url, data=data, json=json)
|
||||
req = await self._async_request_once(method, url, data, json)
|
||||
if req is None:
|
||||
raise TMDbException("无法连接TheMovieDb,请检查网络连接!")
|
||||
logger.debug(f"TMDB异步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重试一次")
|
||||
# 异步会话(AsyncRequestUtils)不像同步会话那样支持按需重建,
|
||||
# 这里退化为原会话上的纯重试,同样以退避等待应对瞬时抖动。
|
||||
await asyncio.sleep(RETRY_BACKOFF_SECONDS)
|
||||
req = await self._async_request_once(method, url, data, json)
|
||||
if req is None:
|
||||
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||
return self._snapshot_response(req)
|
||||
|
||||
async def _async_request_once(self, method, url, data, json):
|
||||
"""
|
||||
执行一次TMDB异步请求,调用方负责决定是否重试。
|
||||
"""
|
||||
if method == "GET":
|
||||
return await self._async_req.get_res(url, params=data, json=json)
|
||||
return await self._async_req.post_res(url, data=data, json=json)
|
||||
|
||||
@classmethod
|
||||
def _snapshot_response(cls, response):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user