mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
refactor(architecture): 修复模块依赖违规并强化架构守护
- 字幕编排上移 DownloadChain.download_site_subtitles，SubtitleModule 仅保留站点链接解析 - TransferChain.recommend_name 上移 TV episodes_info 获取，filemanager 模块不再导入 TmdbChain - endpoint 穿透修复：WXBizMsgCrypt3 迁至 adapters/external/wechat_crypt.py； music/tmdb 缓存管理、listenbrainz 常量、TMDbException、WechatClawBot 辅助统一经 chain 包装 - RuleParser 与 builtin_rules 合并为 application/filter_rules.py； fsproxy/fsworker 迁至 adapters/system/ - chain/__init__.py 删除 qbittorrentapi/transmission_rpc 导入，消除后端协议类型泄漏 - 架构守护测试新增三项检查：模块间隔离、入口层穿透、下载器 SDK 泄漏 - 文档同步：05-architecture.md 记录 DB/Oper 聚合例外与迁移文件位置，AGENTS.md 更新所有权表
This commit is contained in:
jxxghp
@@ -1,7 +1,6 @@
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Tuple, Union, Dict, Callable
|
||||
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.runtime.config import settings
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
@@ -136,32 +135,18 @@ class FileManagerModule(_ModuleBase):
|
||||
return storage_oper.support_transtype()
|
||||
|
||||
@staticmethod
|
||||
def recommend_name(meta: MetaBase, mediainfo: MediaInfo) -> Optional[str]:
|
||||
def recommend_name(meta: MetaBase, mediainfo: MediaInfo,
|
||||
episodes_info: Optional[List[TmdbEpisode]] = None) -> Optional[str]:
|
||||
"""
|
||||
获取重命名后的名称
|
||||
:param meta: 元数据
|
||||
:param mediainfo: 媒体信息
|
||||
:param episodes_info: 集信息，由调用方链层预先获取
|
||||
:return: 重命名后的名称（含目录）
|
||||
"""
|
||||
handler = TransHandler()
|
||||
# 重命名格式
|
||||
rename_format = settings.RENAME_FORMAT(mediainfo.type)
|
||||
# 获取集信息
|
||||
episodes_info: Optional[List[TmdbEpisode]] = None
|
||||
if mediainfo.type == MediaType.TV:
|
||||
# 判断注意season为0的情况
|
||||
season_num = mediainfo.season
|
||||
if season_num is None and meta.season_seq:
|
||||
if meta.season_seq.isdigit():
|
||||
season_num = int(meta.season_seq)
|
||||
# 默认值1
|
||||
if season_num is None:
|
||||
season_num = 1
|
||||
episodes_info = TmdbChain().tmdb_episodes(
|
||||
tmdbid=mediainfo.tmdb_id,
|
||||
season=season_num,
|
||||
episode_group=mediainfo.episode_group,
|
||||
)
|
||||
# 获取重命名后的名称
|
||||
path = handler.get_rename_path(
|
||||
template_string=rename_format,
|
||||
|
||||
@@ -1,428 +0,0 @@
|
||||
"""
|
||||
本地文件系统操作代理。
|
||||
|
||||
FUSE/网络挂载有两种故障形态：crash 型（调用抛错，可捕获、可重试）和 block 型
|
||||
（调用既不返回错误也不返回结果，永久悬挂）。**Python 无法中断一个已经发出的
|
||||
系统调用，也无法强杀线程**，所以 block 型故障下阻塞的线程永远无法回收——这正是
|
||||
整理消费线程停摆、监控自愈路径自冻的根因。
|
||||
|
||||
本模块把这些调用放进一个常驻子进程执行。子进程可以被 SIGKILL，因此超时后能真正
|
||||
回收；对调用方而言，超时表现为一个普通的 OSError 子类（FileSystemTimeout）。
|
||||
换句话说：**把不可处理的 block 型故障，转换成系统各层已经能正确处理的 crash 型
|
||||
故障**——退避重启、登记待重试这些既有机制立刻就能接管。
|
||||
|
||||
第一版只放行安全的操作：
|
||||
- 只读（stat/exists/listdir）——强杀不产生任何副作用
|
||||
- 同存储 rename——内核保证原子性，强杀后要么完全成功要么完全没发生
|
||||
跨存储的复制+删除不在此列，它需要单独的可恢复语义（临时名 + 完成后 rename）。
|
||||
"""
|
||||
import errno as errno_module
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.log import logger
|
||||
|
||||
# worker 脚本路径。用绝对路径直接执行，而不是 -m 或 import：
|
||||
# 直接执行文件不会触发 app/__init__.py 的导入链，代理启动才是毫秒级的
|
||||
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
|
||||
# 单次快操作（stat/listdir/rename/unlink 等）的默认超时秒数
|
||||
DEFAULT_TIMEOUT = 30
|
||||
# 长耗时操作（复制）两次进度上报之间的最长间隔秒数。
|
||||
# worker 每秒上报一次心跳，因此这个阈值判定的是「传输完全没有推进」，
|
||||
# 而不是「传输很慢」——大文件复制几小时也不会误杀
|
||||
DEFAULT_STALL_TIMEOUT = 120
|
||||
# 强杀代理后等待它消失的宽限秒数，不能无限等待
|
||||
_KILL_GRACE = 5
|
||||
|
||||
|
||||
class FileSystemTimeout(OSError):
|
||||
"""
|
||||
文件系统操作在代理中超时未返回，判定挂载无响应。
|
||||
|
||||
继承 OSError 是刻意的：整理链、监控 watcher 等各层对 OSError 已有完整的
|
||||
退避重试与登记逻辑，block 型故障经此转换后可以直接复用它们。
|
||||
"""
|
||||
|
||||
|
||||
class FileSystemProxy:
|
||||
"""
|
||||
常驻子进程文件系统代理。
|
||||
|
||||
请求-响应严格串行（一个代理同时只处理一个请求），由锁保证。超时即强杀代理，
|
||||
下一次请求自动重启一个新的——启动成本是毫秒级，因为 worker 只依赖标准库。
|
||||
"""
|
||||
|
||||
def __init__(self, timeout: Optional[float] = None,
|
||||
stall_timeout: Optional[float] = None):
|
||||
"""
|
||||
:param timeout: 单次快操作的超时秒数，None 表示实时跟随系统设置
|
||||
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数，
|
||||
None 表示实时跟随系统设置
|
||||
"""
|
||||
self._timeout_override = timeout
|
||||
self._stall_timeout_override = stall_timeout
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._selector: Optional[selectors.BaseSelector] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 对外操作
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def stat(self, path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
读取路径属性。
|
||||
:param path: 目标路径
|
||||
:return: {"size", "mtime", "is_dir", "is_file"}
|
||||
"""
|
||||
return self._call("stat", path=str(path))
|
||||
|
||||
def exists(self, path: Path) -> bool:
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
只有 FileNotFoundError 才算「不存在」；其余 OSError（含超时）原样抛出，
|
||||
避免像 Path.exists() 那样把挂载抖动误判成文件消失。
|
||||
:param path: 目标路径
|
||||
:return: 是否存在
|
||||
"""
|
||||
try:
|
||||
self._call("exists", path=str(path))
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def listdir(self, path: Path) -> List[str]:
|
||||
"""
|
||||
列出目录条目名。
|
||||
:param path: 目标目录
|
||||
:return: 条目名列表
|
||||
"""
|
||||
return self._call("listdir", path=str(path))
|
||||
|
||||
def count_entries(self, path: Path, max_check: int = 10000) -> Dict[str, int]:
|
||||
"""
|
||||
统计目录规模。整棵树的遍历在子进程内一次完成，超时可整体放弃。
|
||||
:param path: 目标目录
|
||||
:param max_check: 文件数上限，超过即提前结束
|
||||
:return: {"file_count", "dir_count"}
|
||||
"""
|
||||
return self._call("count_entries", path=str(path), max_check=max_check)
|
||||
|
||||
def rename(self, src: Path, dst: Path) -> bool:
|
||||
"""
|
||||
同一存储内重命名/移动。跨存储会抛 OSError(EXDEV)，由调用方走原有路径。
|
||||
:param src: 源路径
|
||||
:param dst: 目标路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rename", src=str(src), dst=str(dst))
|
||||
|
||||
def copy(self, src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]] = None,
|
||||
cancel_cb: Optional[Callable[[], bool]] = None,
|
||||
chunk_size: Optional[int] = None) -> Any:
|
||||
"""
|
||||
复制文件内容并保留时间戳，按「进度无推进」判定挂死。
|
||||
|
||||
复制大文件可能持续几小时，固定超时无法区分「正常但慢」和「已经挂死」。
|
||||
worker 每秒上报一次进度作为心跳，这里判定的是**两次上报之间的间隔**：
|
||||
超过 stall 阈值收不到任何一行，才认定挂载无响应并强杀 worker。
|
||||
|
||||
取消检查放在父进程：worker 里读不到 global_vars 的传输取消标记，而父进程
|
||||
每收到一次进度就能检查一次，要取消直接杀掉 worker 即可，比在子进程里
|
||||
轮询标记更干净。
|
||||
:param src: 源文件
|
||||
:param dst: 目标文件（调用方应传临时名，完成后自行原子替换）
|
||||
:param progress_cb: 进度回调，入参为百分比
|
||||
:param cancel_cb: 取消检查回调，返回 True 表示应中止
|
||||
:param chunk_size: 分块大小
|
||||
:return: 成功时为 {"copied", "total"}，被取消或通信失败时为 False
|
||||
"""
|
||||
payload = {"src": str(src), "dst": str(dst)}
|
||||
if chunk_size:
|
||||
payload["chunk_size"] = chunk_size
|
||||
if not self._enabled():
|
||||
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request_stream(payload, progress_cb, cancel_cb)
|
||||
except FileSystemTimeout:
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
|
||||
self._shutdown()
|
||||
return False
|
||||
|
||||
def _request_stream(self, payload: Dict[str, Any],
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]]) -> Any:
|
||||
"""
|
||||
发起一次流式请求，逐行消费进度直到终态。
|
||||
:param payload: 请求参数
|
||||
:param progress_cb: 进度回调
|
||||
:param cancel_cb: 取消检查回调
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": "copy", **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
while True:
|
||||
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
|
||||
progress = response.get("progress")
|
||||
if progress is not None:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
logger.info(f"复制已取消: {payload.get('src')}")
|
||||
# 取消就地生效：杀掉 worker 立刻中断传输，不必等它读完整个文件
|
||||
self._shutdown()
|
||||
return False
|
||||
if progress_cb is not None:
|
||||
total = progress.get("total") or 0
|
||||
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
|
||||
continue
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
@staticmethod
|
||||
def _direct_copy(src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]],
|
||||
chunk_size: Optional[int]) -> bool:
|
||||
"""
|
||||
不经代理直接复制，供代理关闭时使用。
|
||||
"""
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
return False
|
||||
buf = fsrc.read(chunk_size or 1024 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(copied / total * 100)
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return True
|
||||
|
||||
def unlink(self, path: Path) -> bool:
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作，强杀后没有中间状态。
|
||||
:param path: 目标文件
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("unlink", path=str(path))
|
||||
|
||||
def rmtree(self, path: Path) -> bool:
|
||||
"""
|
||||
递归删除目录，容忍部分失败（可重复执行直到成功）。
|
||||
:param path: 目标目录
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rmtree", path=str(path))
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
关闭代理进程。
|
||||
"""
|
||||
with self._lock:
|
||||
self._shutdown()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内部实现
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _timeout(self) -> float:
|
||||
"""
|
||||
单次快操作的超时秒数。
|
||||
|
||||
实时读取而不是构造时固定：这三项都暴露在前端设置里，用户改完保存后
|
||||
必须立刻生效，否则会出现「改了没反应」的困惑。
|
||||
"""
|
||||
if self._timeout_override is not None:
|
||||
return self._timeout_override
|
||||
return float(getattr(settings, "FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
|
||||
|
||||
@property
|
||||
def _stall_timeout(self) -> float:
|
||||
"""
|
||||
长耗时操作两次进度上报之间的最长间隔秒数，同样实时跟随系统设置。
|
||||
"""
|
||||
if self._stall_timeout_override is not None:
|
||||
return self._stall_timeout_override
|
||||
return float(getattr(settings, "FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT))
|
||||
|
||||
@staticmethod
|
||||
def _enabled() -> bool:
|
||||
"""
|
||||
代理是否启用。关闭时退回直接调用，行为与引入代理之前完全一致。
|
||||
"""
|
||||
return bool(getattr(settings, "FS_PROXY_ENABLED", True))
|
||||
|
||||
@staticmethod
|
||||
def _direct(op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
不经代理直接执行操作，供代理关闭时使用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if op == "stat":
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
if op == "exists":
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
if op == "listdir":
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
if op == "count_entries":
|
||||
file_count = dir_count = 0
|
||||
for _, dirs, files in os.walk(payload["path"]):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > (payload.get("max_check") or 10000):
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
if op == "rename":
|
||||
os.rename(payload["src"], payload["dst"])
|
||||
return True
|
||||
if op == "unlink":
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
if op == "rmtree":
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
raise ValueError(f"unknown op: {op}")
|
||||
|
||||
def _call(self, op: str, **payload) -> Any:
|
||||
"""
|
||||
执行一次代理调用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if not self._enabled():
|
||||
return self._direct(op, payload)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request(op, payload)
|
||||
except FileSystemTimeout:
|
||||
# 超时说明挂载正在挂死，重试只会再冻一次，直接上报给调用方
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
# 代理进程意外退出或响应损坏，重启后重试一次
|
||||
logger.debug(f"文件系统代理通信异常，重启后重试: {op} - {err}")
|
||||
self._shutdown()
|
||||
return self._request(op, payload)
|
||||
|
||||
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
发送请求并等待响应。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": op, **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
response = json.loads(self._read_line().decode("utf-8"))
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类，
|
||||
# 调用方沿用原有的异常分支即可，无需感知代理的存在
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
def _read_line(self, timeout: Optional[float] = None) -> bytes:
|
||||
"""
|
||||
读取一行响应，超时即强杀代理。
|
||||
:param timeout: 本次读取的超时秒数，默认用单次操作超时
|
||||
:return: 响应行
|
||||
"""
|
||||
timeout = self._timeout if timeout is None else timeout
|
||||
if not self._selector.select(timeout=timeout):
|
||||
logger.error(f"文件系统操作 {timeout} 秒无响应，判定挂载挂死，正在回收代理进程")
|
||||
self._shutdown()
|
||||
raise FileSystemTimeout(
|
||||
errno_module.ETIMEDOUT,
|
||||
f"文件系统操作超过 {timeout} 秒无响应，挂载可能已无响应"
|
||||
)
|
||||
line = self._process.stdout.readline()
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _ensure_worker(self):
|
||||
"""
|
||||
确保代理进程可用，不可用时重新启动。
|
||||
"""
|
||||
if self._process is not None and self._process.poll() is None:
|
||||
return
|
||||
self._shutdown()
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, str(_WORKER_PATH)],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
bufsize=0,
|
||||
)
|
||||
self._selector = selectors.DefaultSelector()
|
||||
self._selector.register(self._process.stdout, selectors.EVENT_READ)
|
||||
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
|
||||
|
||||
def _shutdown(self):
|
||||
"""
|
||||
回收代理进程。冻在挂载上的进程用 SIGKILL，且不无限等待它消失
|
||||
——否则「可放弃的代理」又变回一次不可放弃的阻塞。
|
||||
"""
|
||||
if self._selector is not None:
|
||||
try:
|
||||
self._selector.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._selector = None
|
||||
process, self._process = self._process, None
|
||||
if process is None:
|
||||
return
|
||||
for stream in (process.stdin, process.stdout):
|
||||
try:
|
||||
if stream:
|
||||
stream.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=_KILL_GRACE)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warn(f"文件系统代理进程未能及时退出，交由系统回收: pid={process.pid}")
|
||||
except Exception as err: # noqa: BLE001
|
||||
logger.debug(f"回收文件系统代理进程失败: {err}")
|
||||
|
||||
|
||||
# 全局单例：local 存储本身是单例，代理也只需要一个
|
||||
# 不传超时参数：让它实时跟随系统设置，前端改完保存即刻生效
|
||||
fsproxy = FileSystemProxy()
|
||||
@@ -1,201 +0,0 @@
|
||||
"""
|
||||
文件系统操作代理 worker。
|
||||
|
||||
**本文件不能被 import，只能作为独立脚本执行**（fsproxy 用
|
||||
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动）。直接执行文件
|
||||
路径不会触发 `app/__init__.py` 的导入链，因此这个进程只依赖标准库、启动是
|
||||
毫秒级的；一旦走 import 就会把整个应用的依赖拉进来，代理被强杀后的重启成本
|
||||
会高到无法接受。
|
||||
|
||||
存在的理由：FUSE/网络挂载进入 block 型故障时，`stat`/`listdir`/`rename` 这类
|
||||
系统调用既不返回错误也不返回结果，而 Python 没有中断线程的手段——阻塞其上的
|
||||
线程永远无法回收。放进独立进程后，父进程可以在超时后 SIGKILL 掉它，把
|
||||
「不可处理的 block」转换成「可处理的 crash」。
|
||||
|
||||
协议：stdin/stdout 逐行 JSON。
|
||||
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
|
||||
成功 {"ok": true, "result": {...}}
|
||||
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _stat(payload, _emit):
|
||||
"""
|
||||
读取路径的基本属性。
|
||||
"""
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
|
||||
|
||||
def _exists(payload, _emit):
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
用 os.stat 而不是 os.path.exists：后者会把任意 OSError 都归为「不存在」，
|
||||
挂载抖动会被误判成文件消失。这里让异常原样抛出，由父进程按 errno 区分。
|
||||
"""
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _listdir(payload, _emit):
|
||||
"""
|
||||
列出目录下的条目名。
|
||||
"""
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
|
||||
|
||||
def _copy(payload, emit):
|
||||
"""
|
||||
分块复制文件内容并周期上报进度。
|
||||
|
||||
进度上报同时充当心跳：复制大文件可能持续几小时，父进程无法用固定超时判断
|
||||
挂死，只能看「两次上报之间隔了多久」。因此这里按固定时间间隔上报，即使
|
||||
某一秒没读到数据也照常发——一旦挂载卡住，read/write 不返回，上报自然断流，
|
||||
父进程据此判定并强杀本进程。
|
||||
|
||||
只复制内容和时间戳，不复制权限：目标目录的默认权限与继承 ACL 是媒体库的
|
||||
访问策略，用源文件权限覆盖会清除已继承的 ACL。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
chunk_size = payload.get("chunk_size") or 1024 * 1024
|
||||
interval = payload.get("progress_interval") or 1.0
|
||||
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
# 先发一次 0%：既让心跳立刻开始，也保证父进程在传输开始前就有一次检查
|
||||
# 取消的机会——否则小文件会在首次定时上报之前就复制完，取消形同虚设
|
||||
emit({"ok": True, "progress": {"copied": 0, "total": total}})
|
||||
last_emit = time.monotonic()
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
buf = fsrc.read(chunk_size)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= interval:
|
||||
last_emit = now
|
||||
emit({"ok": True, "progress": {"copied": copied, "total": total}})
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return {"copied": copied, "total": total}
|
||||
|
||||
|
||||
def _count_entries(payload, _emit):
|
||||
"""
|
||||
统计目录下的文件与子目录数量，超过上限即提前结束。
|
||||
|
||||
放在子进程里做而不是逐层 listdir 走 IPC：递归遍历一棵大目录树会产生成千
|
||||
上万次往返，代价不可接受；一次调用在子进程内跑完 os.walk，父进程只需对
|
||||
这一次调用设超时即可整体放弃。
|
||||
"""
|
||||
directory = payload["path"]
|
||||
max_check = payload.get("max_check") or 10000
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
for _, dirs, files in os.walk(directory):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > max_check:
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
|
||||
|
||||
def _rename(payload, _emit):
|
||||
"""
|
||||
同一存储内重命名/移动。
|
||||
|
||||
这是第一版唯一放行的写操作：同文件系统内的 rename 由内核保证原子性，
|
||||
进程被强杀后要么完全成功要么完全没发生，不存在需要清理的中间状态。
|
||||
跨存储的复制+删除不走这里，它需要单独的可恢复语义。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
|
||||
raise OSError(18, "Cross-device rename is not handled by the proxy")
|
||||
os.rename(src, dst)
|
||||
return True
|
||||
|
||||
|
||||
def _unlink(payload, _emit):
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作，强杀后要么删掉了要么没删，没有中间状态。
|
||||
"""
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _rmtree(payload, _emit):
|
||||
"""
|
||||
递归删除目录。
|
||||
|
||||
这一项不是原子的，强杀可能只删掉一部分。放行的理由是：删除被中断的后果
|
||||
（残留若干文件）远轻于写入被中断（留下叫最终文件名的半成品），而且调用方
|
||||
本来就以 ignore_errors 容忍部分失败、可以重复执行直到成功。
|
||||
"""
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
|
||||
|
||||
_HANDLERS = {
|
||||
"stat": _stat,
|
||||
"exists": _exists,
|
||||
"listdir": _listdir,
|
||||
"copy": _copy,
|
||||
"count_entries": _count_entries,
|
||||
"rename": _rename,
|
||||
"unlink": _unlink,
|
||||
"rmtree": _rmtree,
|
||||
"ping": lambda _payload, _emit: True,
|
||||
}
|
||||
|
||||
|
||||
def _write(message):
|
||||
"""
|
||||
输出一行响应。
|
||||
"""
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
请求循环：每读一行处理一个请求，直到 stdin 关闭。
|
||||
|
||||
一个请求可能对应多行响应：长耗时操作先流式发若干 progress 行，最后发一行
|
||||
终态（result 或 error）。父进程据此区分「还在推进」和「已经挂死」。
|
||||
"""
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
handler = _HANDLERS.get(payload.get("op"))
|
||||
if handler is None:
|
||||
response = {"ok": False, "errno": 0,
|
||||
"error": f"unknown op: {payload.get('op')}"}
|
||||
else:
|
||||
response = {"ok": True, "result": handler(payload, _write)}
|
||||
except OSError as err:
|
||||
response = {"ok": False, "errno": err.errno or 0,
|
||||
"error": err.strerror or str(err)}
|
||||
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
|
||||
response = {"ok": False, "errno": 0, "error": str(err)}
|
||||
_write(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -8,7 +8,7 @@ from app import schemas
|
||||
from app.runtime.config import global_vars, settings
|
||||
from app.application.directory import DirectoryHelper
|
||||
from app.runtime.log import logger
|
||||
from app.modules.filemanager.fsproxy import fsproxy
|
||||
from app.adapters.system.fsproxy import fsproxy
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import threading
|
||||
|
||||
from pyparsing import Forward, Literal, Word, alphas, infix_notation, opAssoc, alphanums, Combine, nums, ParseResults
|
||||
|
||||
from app.adapters.system import rust as rust_accel
|
||||
|
||||
|
||||
class RuleParser:
|
||||
|
||||
_lock = threading.Lock()
|
||||
_thread_local = threading.local()
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
定义语法规则
|
||||
"""
|
||||
with self._lock:
|
||||
if not hasattr(self._thread_local, 'initialized'):
|
||||
# 表达式
|
||||
expr: Forward = Forward()
|
||||
# 原子
|
||||
atom: Combine = Combine(Word(alphas, alphanums) | (Word(nums) + Word(alphas, alphanums)))
|
||||
# 逻辑非操作符
|
||||
operator_not: Literal = Literal('!').set_parse_action(lambda t: 'not')
|
||||
# 逻辑或操作符
|
||||
operator_or: Literal = Literal('|').set_parse_action(lambda t: 'or')
|
||||
# 逻辑与操作符
|
||||
operator_and: Literal = Literal('&').set_parse_action(lambda t: 'and')
|
||||
# 定义表达式的语法规则
|
||||
expr <<= (operator_not + expr) | atom | ('(' + expr + ')')
|
||||
|
||||
# 运算符优先级
|
||||
self.expr = infix_notation(expr,
|
||||
[(operator_not, 1, opAssoc.RIGHT),
|
||||
(operator_and, 2, opAssoc.LEFT),
|
||||
(operator_or, 2, opAssoc.LEFT)])
|
||||
|
||||
self._thread_local.expr = self.expr
|
||||
self._thread_local.initialized = True
|
||||
else:
|
||||
self.expr = self._thread_local.expr
|
||||
|
||||
def parse(self, expression: str) -> ParseResults:
|
||||
"""
|
||||
解析给定的表达式。
|
||||
|
||||
参数:
|
||||
expression -- 要解析的表达式
|
||||
|
||||
返回:
|
||||
解析结果
|
||||
"""
|
||||
rust_result = rust_accel.parse_filter_rule(expression)
|
||||
if rust_result is not None:
|
||||
return _RustParseResults(rust_result)
|
||||
return self.expr.parse_string(expression)
|
||||
|
||||
|
||||
class _RustParseResults(list):
|
||||
"""
|
||||
包装 Rust 解析结果，提供本模块调用方使用的 as_list/asList 接口。
|
||||
"""
|
||||
|
||||
def as_list(self) -> list:
|
||||
"""
|
||||
返回兼容 pyparsing.ParseResults.as_list 的列表结构。
|
||||
"""
|
||||
return list(self)
|
||||
|
||||
def asList(self) -> list: # noqa: N802
|
||||
"""
|
||||
返回兼容 pyparsing.ParseResults.asList 的列表结构。
|
||||
"""
|
||||
return self.as_list()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 测试代码
|
||||
expression_str = """
|
||||
SPECSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & 60FPS & !DOLBY & !SDR & !3D > CNSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & 60FPS & !DOLBY & !SDR & !3D > SPECSUB & 4K & !BLU & !REMUX & !WEBDL & 60FPS & !DOLBY & !SDR & !3D > CNSUB & 4K & !BLU & !REMUX & !WEBDL & 60FPS & !DOLBY & !SDR & !3D > SPECSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > CNSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > CNSUB & CNVOI & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > SPECSUB & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > CNSUB & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > CNSUB & 4K & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > SPECSUB & CNVOI & 4K & WEBDL & 60FPS & !DOLBY & !SDR & !3D > CNSUB & CNVOI & 4K & WEBDL & 60FPS & !DOLBY & !SDR & !3D > SPECSUB & 4K & WEBDL & 60FPS & !DOLBY & !SDR & !3D > CNSUB & 4K & WEBDL & 60FPS & !DOLBY & !SDR & !3D > SPECSUB & CNVOI & 4K & WEBDL & !DOLBY & HDR & !3D > CNSUB & CNVOI & 4K & WEBDL & !DOLBY & HDR & !3D > SPECSUB & CNVOI & 4K & WEBDL & !DOLBY & !3D > CNSUB & CNVOI & 4K & WEBDL & !DOLBY & !3D > SPECSUB & 4K & WEBDL & !DOLBY & HDR & !3D > CNSUB & 4K & WEBDL & !DOLBY & HDR & !3D > SPECSUB & 4K & WEBDL & !DOLBY & !3D > CNSUB & 4K & WEBDL & !DOLBY & !3D > SPECSUB & CNVOI & 4K & !BLU & !WEBDL & !DOLBY & HDR & !3D > CNSUB & CNVOI & 4K & !BLU & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & CNVOI & 4K & !BLU & !WEBDL & !DOLBY & !3D > CNSUB & CNVOI & 4K & !BLU & !WEBDL & !DOLBY & !3D > SPECSUB & 4K & !BLU & !WEBDL & !DOLBY & HDR & !3D > CNSUB & 4K & !BLU & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & 4K & !BLU & !WEBDL & !DOLBY & !SDR & !3D > CNSUB & 4K & !BLU & !WEBDL & !DOLBY & !SDR & !3D > 4K & !BLU & !REMUX & !DOLBY & HDR & !3D > 4K & !BLURAY & !REMUX & !DOLBY & !3D > SPECSUB & 1080P & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > CNSUB & 1080P & !BLU & !REMUX & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & 1080P & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > CNSUB & 1080P & !BLU & !REMUX & !WEBDL & !DOLBY & !3D > SPECSUB & 1080P & !BLU & !WEBDL & !DOLBY & HDR & !3D > CNSUB & 1080P & !BLU & !WEBDL & !DOLBY & HDR & !3D > SPECSUB & 1080P & !BLU & !WEBDL & !DOLBY & !3D > CNSUB & 1080P & !BLU & !WEBDL & !DOLBY & !3D > SPECSUB & 1080P & WEBDL & !DOLBY & HDR & !3D > CNSUB & 1080P & WEBDL & !DOLBY & HDR & !3D > SPECSUB & 1080P & WEBDL & !DOLBY & !3D > CNSUB & 1080P & WEBDL & !DOLBY & !3D > 1080P & !BLU & !REMUX & !DOLBY & HDR & !3D > 1080P & !BLU & !REMUX & !DOLBY & !3D
|
||||
"""
|
||||
for exp in expression_str.split('>'):
|
||||
parsed_expr = RuleParser().parse(exp.strip())
|
||||
print(parsed_expr.asList())
|
||||
@@ -8,8 +8,8 @@ from app.domain.metainfo import MetaInfo, clear_rust_parse_options_cache, _rust_
|
||||
from app.application.filter import RuleHelper
|
||||
from app.runtime.log import logger
|
||||
from app.modules import _ModuleBase
|
||||
from app.modules.filter.RuleParser import RuleParser
|
||||
from app.modules.filter.builtin_rules import BUILTIN_RULE_SET
|
||||
from app.application.filter_rules import RuleParser
|
||||
from app.application.filter_rules import BUILTIN_RULE_SET
|
||||
from app.schemas.types import ModuleType, OtherModulesType, SystemConfigKey
|
||||
from app.adapters.system import rust as rust_accel
|
||||
from app.foundation import size as size_tools
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
"""过滤器内置规则定义。"""
|
||||
|
||||
from typing import Dict
|
||||
|
||||
# 内置规则只在这里维护一份，便于过滤模块和 Agent 工具共享同一套事实来源。
|
||||
BUILTIN_RULE_SET: Dict[str, dict] = {
|
||||
# 蓝光原盘
|
||||
"BLU": {
|
||||
"include": [
|
||||
r"(?i)(\bBlu-?Ray\b.*\b(?:VC-?1|AVC|MPEG-?2)\b|\b(?:UHD|4K|2160p)\b(?:.*Blu-?Ray)?.*\b(?:HEVC|H\.?265)\b|\bBlu-?Ray\b.*\b(?:UHD|4K|2160p)\b.*\b(?:HEVC|H\.?265)\b|\b(?:COMPLETE|FULL)\b.*\b(?:(?:UHD|4K|2160p)\b.*)?Blu-?Ray\b|\b(BD25|BD50|BD66|BD100|BDMV|MiniBD)\b)"
|
||||
],
|
||||
"exclude": [
|
||||
r"(?i)(\b[XH]\.?264\b|\b[XH]\.?265\b|\bWEB-?DL\b|\bWEB-?RIP\b|\bHDTV(?:RIP)?\b|\bREMUX\b|\bBDRip\b|\bBRRip\b|\bHDRip\b|\bENCODE\b|\b(?<!WEB-|HDTV)RIP\b)"
|
||||
],
|
||||
},
|
||||
# 4K
|
||||
"4K": {
|
||||
"include": [r"4k|2160p|x2160"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 1080P
|
||||
"1080P": {
|
||||
"include": [r"1080[pi]|x1080"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 720P
|
||||
"720P": {
|
||||
"include": [r"720[pi]|x720"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 中字
|
||||
"CNSUB": {
|
||||
"include": [
|
||||
r"[中国國繁简](/|\s|\\|\|)?[繁简英粤]|[英简繁](/|\s|\\|\|)?[中繁简]"
|
||||
r"|繁體|简体|[中国國][字配]|国语|國語|中文|中字|简日|繁日|简繁|繁体"
|
||||
r"|([\s,.-\[])(chs|cht)(|[\s,.-\]])"
|
||||
r"|(?<![a-z0-9])(?<!\d\s)(gb|big5)(?![a-z0-9])"
|
||||
],
|
||||
"exclude": [],
|
||||
"tmdb": {
|
||||
"original_language": "zh,cn",
|
||||
},
|
||||
},
|
||||
# 官种
|
||||
"GZ": {
|
||||
"include": [r"官方", r"官种", r"官组"],
|
||||
"match": ["labels"],
|
||||
},
|
||||
# 特效字幕
|
||||
"SPECSUB": {
|
||||
"include": [r"特效"],
|
||||
"exclude": [],
|
||||
},
|
||||
# BluRay
|
||||
"BLURAY": {
|
||||
"include": [r"Blu-?Ray"],
|
||||
"exclude": [],
|
||||
},
|
||||
# UHD
|
||||
"UHD": {
|
||||
"include": [r"UHD|UltraHD"],
|
||||
"exclude": [],
|
||||
},
|
||||
# H265
|
||||
"H265": {
|
||||
"include": [r"[Hx].?265|HEVC"],
|
||||
"exclude": [],
|
||||
},
|
||||
# H264
|
||||
"H264": {
|
||||
"include": [r"[Hx].?264|AVC"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 杜比视界
|
||||
"DOLBY": {
|
||||
"include": [r"Dolby[\s.]+Vision|DOVI|[\s.]+DV[\s.]+|杜比视界"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 杜比全景声
|
||||
"ATMOS": {
|
||||
"include": [r"Dolby[\s.+]+Atmos|Atmos|杜比全景[声聲]"],
|
||||
"exclude": [],
|
||||
},
|
||||
# HDR
|
||||
"HDR": {
|
||||
"include": [r"[\s.]+HDR[\s.]+|HDR10|HDR10\+|HDRVivid"],
|
||||
"exclude": [],
|
||||
},
|
||||
# SDR
|
||||
"SDR": {
|
||||
"include": [r"[\s.]+SDR[\s.]+"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 重编码
|
||||
"REMUX": {
|
||||
"include": [r"REMUX"],
|
||||
"exclude": [],
|
||||
},
|
||||
# WEB-DL
|
||||
"WEBDL": {
|
||||
"include": [r"WEB-?DL|WEB-?RIP"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 免费
|
||||
"FREE": {
|
||||
"downloadvolumefactor": 0,
|
||||
},
|
||||
# 国语配音
|
||||
"CNVOI": {
|
||||
"include": [r"[国國][语語]配音|[国國]配|[国國][语語]"],
|
||||
"exclude": [],
|
||||
"tmdb": {
|
||||
"original_language": "zh",
|
||||
},
|
||||
},
|
||||
# 粤语配音
|
||||
"HKVOI": {
|
||||
"include": [r"粤语配音|粤语"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 60FPS
|
||||
"60FPS": {
|
||||
"include": [r"60fps|60帧"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 3D
|
||||
"3D": {
|
||||
"include": [r"3D"],
|
||||
"exclude": [],
|
||||
},
|
||||
# Hi-Res 无损音频
|
||||
"HIRES": {
|
||||
"include": [r"(?i)\b(?:Hi[ ._-]?Res(?:olution)?|DSD(?:64|128|256|512)?)\b|高解析|(?:24|32)\s*(?:-?bit|位)"],
|
||||
"exclude": [],
|
||||
},
|
||||
# 无损音频
|
||||
"LOSSLESS": {
|
||||
"include": [r"(?i)\b(?:Lossless|FLAC|ALAC|APE|WAV|WAVE|AIFF?|PCM|DSD|DSF|DFF)\b|无损"],
|
||||
"exclude": [],
|
||||
},
|
||||
"FLAC": {"include": [r"(?i)(?<![A-Z0-9])FLAC(?![A-Z0-9])"], "exclude": []},
|
||||
"ALAC": {"include": [r"(?i)(?<![A-Z0-9])ALAC(?![A-Z0-9])"], "exclude": []},
|
||||
"APE": {"include": [r"(?i)(?<![A-Z0-9])APE(?![A-Z0-9])"], "exclude": []},
|
||||
"WAV": {"include": [r"(?i)(?<![A-Z0-9])WAV(?:E)?(?![A-Z0-9])"], "exclude": []},
|
||||
"DSD": {"include": [r"(?i)(?<![A-Z0-9])(?:DSD(?:64|128|256|512)?|DSF|DFF)(?![A-Z0-9])"], "exclude": []},
|
||||
"MP3": {"include": [r"(?i)(?<![A-Z0-9])MP3(?![A-Z0-9])"], "exclude": []},
|
||||
"AAC": {"include": [r"(?i)(?<![A-Z0-9])(?:AAC|M4A)(?![A-Z0-9])"], "exclude": []},
|
||||
"OPUS": {"include": [r"(?i)(?<![A-Z0-9])OPUS(?![A-Z0-9])"], "exclude": []},
|
||||
"BITRATE320": {"include": [r"(?i)(?<!\d)320\s*k(?:bps?|b(?:it)?/?s?)?(?![a-z])"], "exclude": []},
|
||||
"BITRATE256": {"include": [r"(?i)(?<!\d)256\s*k(?:bps?|b(?:it)?/?s?)?(?![a-z])"], "exclude": []},
|
||||
"BITRATE192": {"include": [r"(?i)(?<!\d)192\s*k(?:bps?|b(?:it)?/?s?)?(?![a-z])"], "exclude": []},
|
||||
}
|
||||
@@ -1,26 +1,15 @@
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from app.chain.storage import StorageChain
|
||||
from app.runtime.config import settings
|
||||
from app.domain.context import Context
|
||||
from app.db.oper.site import SiteOper
|
||||
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
||||
from app.application.torrent import TorrentHelper
|
||||
from app.runtime.log import logger
|
||||
from app.modules import _ModuleBase
|
||||
from app.modules.indexer.spider.mtorrent import MTorrentSpider
|
||||
from app.schemas import TorrentInfo
|
||||
from app.schemas.file import FileURI
|
||||
from app.schemas.types import ModuleType, OtherModulesType
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.adapters.system.host import SystemUtils
|
||||
|
||||
|
||||
class SubtitleModule(_ModuleBase):
|
||||
@@ -28,11 +17,6 @@ class SubtitleModule(_ModuleBase):
|
||||
字幕下载模块
|
||||
"""
|
||||
|
||||
_SUBTITLE_ARCHIVE_FORMATS = {
|
||||
".zip": "zip",
|
||||
".rar": "rar",
|
||||
}
|
||||
|
||||
# 站点详情页字幕下载元素识别XPATH
|
||||
_SITE_SUBTITLE_XPATH = [
|
||||
'//td[@class="rowhead"][text()="字幕"]/following-sibling::td//a[not(@class)]',
|
||||
@@ -140,20 +124,15 @@ class SubtitleModule(_ModuleBase):
|
||||
break
|
||||
return sublink_list
|
||||
|
||||
def _get_subtitle_links(self, torrent: TorrentInfo):
|
||||
def site_subtitle_links(self, context: Context) -> Optional[List[str]]:
|
||||
"""
|
||||
获取字幕链接
|
||||
解析普通站点详情页获取字幕下载链接
|
||||
:param context: 上下文，包括识别信息、媒体信息、种子信息
|
||||
:return: 字幕下载链接列表，无法访问页面时返回None
|
||||
"""
|
||||
# API请求方式的站点需要特殊处理
|
||||
if torrent.site is not None:
|
||||
site = SiteOper().get(torrent.site)
|
||||
if indexer := SitesHelper().get_indexer(site.domain):
|
||||
if indexer.get("parser") == "mTorrent":
|
||||
return MTorrentSpider(indexer).get_subtitle_links(
|
||||
torrent.page_url
|
||||
)
|
||||
# TODO 其它采用API访问的站点
|
||||
# 普通站点通过解析网站代码的方式获取
|
||||
torrent = context.torrent_info
|
||||
if not torrent.page_url:
|
||||
return None
|
||||
request = RequestUtils(
|
||||
cookies=torrent.site_cookie,
|
||||
ua=torrent.site_ua,
|
||||
@@ -175,125 +154,3 @@ class SubtitleModule(_ModuleBase):
|
||||
else:
|
||||
logger.warn(f"无法打开链接：{torrent.page_url}")
|
||||
return None
|
||||
|
||||
def download_added(self, context: Context, download_dir: Path, torrent_content: Union[str, bytes] = None):
|
||||
"""
|
||||
添加下载任务成功后，从站点下载字幕，保存到下载目录
|
||||
:param context: 上下文，包括识别信息、媒体信息、种子信息
|
||||
:param download_dir: 下载目录
|
||||
:param torrent_content: 种子内容，如果是种子文件，则为文件内容，否则为种子字符串
|
||||
:return: None，该方法可被多个模块同时处理
|
||||
"""
|
||||
if not settings.DOWNLOAD_SUBTITLE:
|
||||
return
|
||||
|
||||
# 没有种子文件不处理
|
||||
if not torrent_content:
|
||||
return
|
||||
|
||||
# 没有详情页不处理
|
||||
torrent = context.torrent_info
|
||||
if not torrent.page_url:
|
||||
return
|
||||
# 字幕下载目录
|
||||
logger.info("开始从站点下载字幕：%s" % torrent.page_url)
|
||||
# 获取种子信息
|
||||
folder_name, _ = TorrentHelper().get_fileinfo_from_torrent_content(torrent_content)
|
||||
# 文件保存目录，如果是单文件种子，则folder_name是空，此时文件保存目录就是下载目录
|
||||
storageChain = StorageChain()
|
||||
# 等待目录存在
|
||||
working_dir_item = None
|
||||
# split download_dir into storage and path
|
||||
fileURI = FileURI.from_uri(download_dir.as_posix())
|
||||
storage = fileURI.storage
|
||||
download_dir = Path(fileURI.path)
|
||||
for _ in range(30):
|
||||
found = storageChain.get_file_item(storage, download_dir / folder_name)
|
||||
if found:
|
||||
working_dir_item = found
|
||||
break
|
||||
time.sleep(1)
|
||||
# 目录仍然不存在，且有文件夹名，则创建目录
|
||||
if not working_dir_item and folder_name:
|
||||
parent_dir_item = storageChain.get_folder(storage, download_dir)
|
||||
if parent_dir_item:
|
||||
working_dir_item = storageChain.create_folder(
|
||||
parent_dir_item,
|
||||
folder_name
|
||||
)
|
||||
else:
|
||||
logger.error(f"下载根目录不存在，无法创建字幕文件夹：{download_dir}")
|
||||
return
|
||||
if not working_dir_item:
|
||||
logger.error(f"下载目录不存在，无法保存字幕：{download_dir / folder_name}")
|
||||
return
|
||||
# 读取网站代码
|
||||
sublink_list = self._get_subtitle_links(torrent)
|
||||
if not sublink_list:
|
||||
logger.warn(f"{torrent.page_url} 页面未找到字幕下载链接")
|
||||
return
|
||||
# 下载所有字幕文件
|
||||
request = RequestUtils(
|
||||
cookies=torrent.site_cookie,
|
||||
ua=torrent.site_ua,
|
||||
proxies=settings.PROXY if torrent.site_proxy else None,
|
||||
)
|
||||
settings.TEMP_PATH.mkdir(parents=True, exist_ok=True)
|
||||
for sublink in sublink_list:
|
||||
logger.info(f"找到字幕下载链接：{sublink}，开始下载...")
|
||||
# 下载
|
||||
ret = request.get_res(sublink)
|
||||
if ret and ret.status_code == 200:
|
||||
file_name = TorrentHelper.get_url_filename(ret, sublink)
|
||||
if not file_name:
|
||||
logger.warn(f"链接不是字幕文件：{sublink}")
|
||||
continue
|
||||
archive_format = self._SUBTITLE_ARCHIVE_FORMATS.get(Path(file_name).suffix.lower())
|
||||
if archive_format:
|
||||
archive_file = settings.TEMP_PATH / file_name
|
||||
# 保存
|
||||
archive_file.write_bytes(ret.content)
|
||||
# 解压路径
|
||||
archive_path = archive_file.with_name(archive_file.stem)
|
||||
try:
|
||||
# 解压文件
|
||||
SystemUtils.unpack_archive(
|
||||
archive_file,
|
||||
archive_path,
|
||||
archive_format=archive_format,
|
||||
)
|
||||
# 遍历转移文件
|
||||
for sub_file in SystemUtils.list_files(archive_path, settings.RMT_SUBEXT):
|
||||
target_sub_file = Path(working_dir_item.path) / Path(sub_file.name)
|
||||
if storageChain.get_file_item(storage, target_sub_file):
|
||||
logger.info(f"字幕文件已存在：{target_sub_file}")
|
||||
continue
|
||||
logger.info(f"转移字幕 {sub_file} 到 {target_sub_file} ...")
|
||||
storageChain.upload_file(working_dir_item, sub_file)
|
||||
except Exception as err:
|
||||
logger.error(f"字幕压缩包解压失败：{archive_file} - {str(err)}")
|
||||
# 删除临时文件
|
||||
try:
|
||||
if archive_path.exists():
|
||||
shutil.rmtree(archive_path)
|
||||
if archive_file.exists():
|
||||
archive_file.unlink()
|
||||
except Exception as err:
|
||||
logger.error(f"删除临时文件失败：{str(err)}")
|
||||
else:
|
||||
if Path(file_name).suffix.lower() not in settings.RMT_SUBEXT:
|
||||
logger.warn(f"链接不是支持的字幕文件：{sublink} - {file_name}")
|
||||
continue
|
||||
sub_file = settings.TEMP_PATH / file_name
|
||||
# 保存
|
||||
sub_file.write_bytes(ret.content)
|
||||
target_sub_file = Path(working_dir_item.path) / Path(sub_file.name)
|
||||
if storageChain.get_file_item(storage, target_sub_file):
|
||||
logger.info(f"字幕文件已存在：{target_sub_file}")
|
||||
continue
|
||||
logger.info(f"转移字幕 {sub_file} 到 {target_sub_file} ...")
|
||||
storageChain.upload_file(working_dir_item, sub_file)
|
||||
else:
|
||||
logger.error(f"下载字幕文件失败：{sublink}")
|
||||
continue
|
||||
logger.info(f"{torrent.page_url} 页面字幕下载完成")
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- encoding:utf-8 -*-
|
||||
|
||||
""" 对企业微信发送给企业后台的消息加解密示例代码.
|
||||
@copyright: Copyright (c) 1998-2014 Tencent Inc.
|
||||
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
# ------------------------------------------------------------------------
|
||||
import logging
|
||||
import random
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import xml.etree.cElementTree as ET
|
||||
|
||||
from Crypto.Cipher import AES
|
||||
|
||||
# Description:定义错误码含义
|
||||
#########################################################################
|
||||
WXBizMsgCrypt_OK = 0
|
||||
WXBizMsgCrypt_ValidateSignature_Error = -40001
|
||||
WXBizMsgCrypt_ParseXml_Error = -40002
|
||||
WXBizMsgCrypt_ComputeSignature_Error = -40003
|
||||
WXBizMsgCrypt_IllegalAesKey = -40004
|
||||
WXBizMsgCrypt_ValidateCorpid_Error = -40005
|
||||
WXBizMsgCrypt_EncryptAES_Error = -40006
|
||||
WXBizMsgCrypt_DecryptAES_Error = -40007
|
||||
WXBizMsgCrypt_IllegalBuffer = -40008
|
||||
WXBizMsgCrypt_EncodeBase64_Error = -40009
|
||||
WXBizMsgCrypt_DecodeBase64_Error = -40010
|
||||
WXBizMsgCrypt_GenReturnXml_Error = -40011
|
||||
|
||||
"""
|
||||
关于Crypto.Cipher模块，ImportError: No module named 'Crypto'解决方案
|
||||
请到官方网站 https://www.dlitz.net/software/pycrypto/ 下载pycrypto。
|
||||
下载后，按照README中的“Installation”小节的提示进行pycrypto安装。
|
||||
"""
|
||||
|
||||
|
||||
class FormatException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def throw_exception(message, exception_class=FormatException):
|
||||
"""my define raise exception function"""
|
||||
raise exception_class(message)
|
||||
|
||||
|
||||
class SHA1:
|
||||
"""计算企业微信的消息签名接口"""
|
||||
|
||||
@staticmethod
|
||||
def getSHA1(token, timestamp, nonce, encrypt):
|
||||
"""用SHA1算法生成安全签名
|
||||
@param token: 票据
|
||||
@param timestamp: 时间戳
|
||||
@param encrypt: 密文
|
||||
@param nonce: 随机字符串
|
||||
@return: 安全签名
|
||||
"""
|
||||
try:
|
||||
sortlist = [token, timestamp, nonce, encrypt]
|
||||
sortlist.sort()
|
||||
sha = hashlib.sha1()
|
||||
sha.update("".join(sortlist).encode())
|
||||
return WXBizMsgCrypt_OK, sha.hexdigest()
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return WXBizMsgCrypt_ComputeSignature_Error, None
|
||||
|
||||
|
||||
class XMLParse:
|
||||
"""提供提取消息格式中的密文及生成回复消息格式的接口"""
|
||||
|
||||
# xml消息模板
|
||||
AES_TEXT_RESPONSE_TEMPLATE = """<xml>
|
||||
<Encrypt><![CDATA[%(msg_encrypt)s]]></Encrypt>
|
||||
<MsgSignature><![CDATA[%(msg_signaturet)s]]></MsgSignature>
|
||||
<TimeStamp>%(timestamp)s</TimeStamp>
|
||||
<Nonce><![CDATA[%(nonce)s]]></Nonce>
|
||||
</xml>"""
|
||||
|
||||
@staticmethod
|
||||
def extract(xmltext):
|
||||
"""提取出xml数据包中的加密消息
|
||||
@param xmltext: 待提取的xml字符串
|
||||
@return: 提取出的加密消息字符串
|
||||
"""
|
||||
try:
|
||||
xml_tree = ET.fromstring(xmltext)
|
||||
encrypt = xml_tree.find("Encrypt")
|
||||
return WXBizMsgCrypt_OK, encrypt.text
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return WXBizMsgCrypt_ParseXml_Error, None
|
||||
|
||||
def generate(self, encrypt, signature, timestamp, nonce):
|
||||
"""生成xml消息
|
||||
@param encrypt: 加密后的消息密文
|
||||
@param signature: 安全签名
|
||||
@param timestamp: 时间戳
|
||||
@param nonce: 随机字符串
|
||||
@return: 生成的xml字符串
|
||||
"""
|
||||
resp_dict = {
|
||||
'msg_encrypt': encrypt,
|
||||
'msg_signaturet': signature,
|
||||
'timestamp': timestamp,
|
||||
'nonce': nonce,
|
||||
}
|
||||
resp_xml = self.AES_TEXT_RESPONSE_TEMPLATE % resp_dict
|
||||
return resp_xml
|
||||
|
||||
|
||||
class PKCS7Encoder:
|
||||
"""提供基于PKCS7算法的加解密接口"""
|
||||
|
||||
block_size = 32
|
||||
|
||||
def encode(self, text):
|
||||
""" 对需要加密的明文进行填充补位
|
||||
@param text: 需要进行填充补位操作的明文
|
||||
@return: 补齐明文字符串
|
||||
"""
|
||||
text_length = len(text)
|
||||
# 计算需要填充的位数
|
||||
amount_to_pad = self.block_size - (text_length % self.block_size)
|
||||
if amount_to_pad == 0:
|
||||
amount_to_pad = self.block_size
|
||||
# 获得补位所用的字符
|
||||
pad = chr(amount_to_pad)
|
||||
return text + (pad * amount_to_pad).encode()
|
||||
|
||||
@staticmethod
|
||||
def decode(decrypted):
|
||||
"""删除解密后明文的补位字符
|
||||
@param decrypted: 解密后的明文
|
||||
@return: 删除补位字符后的明文
|
||||
"""
|
||||
pad = ord(decrypted[-1])
|
||||
if pad < 1 or pad > 32:
|
||||
pad = 0
|
||||
return decrypted[:-pad]
|
||||
|
||||
|
||||
class Prpcrypt(object):
|
||||
"""提供接收和推送给企业微信消息的加解密接口"""
|
||||
|
||||
def __init__(self, key):
|
||||
|
||||
# self.key = base64.b64decode(key+"=")
|
||||
self.key = key
|
||||
# 设置加解密模式为AES的CBC模式
|
||||
self.mode = AES.MODE_CBC
|
||||
|
||||
def encrypt(self, text, receiveid):
|
||||
"""对明文进行加密
|
||||
@param text: 需要加密的明文
|
||||
@param receiveid: receiveid
|
||||
@return: 加密得到的字符串
|
||||
"""
|
||||
# 16位随机字符串添加到明文开头
|
||||
text = text.encode()
|
||||
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode()
|
||||
|
||||
# 使用自定义的填充方式对明文进行补位填充
|
||||
pkcs7 = PKCS7Encoder()
|
||||
text = pkcs7.encode(text)
|
||||
# 加密
|
||||
cryptor = AES.new(self.key, self.mode, self.key[:16])
|
||||
try:
|
||||
ciphertext = cryptor.encrypt(text)
|
||||
# 使用BASE64对加密后的字符串进行编码
|
||||
return WXBizMsgCrypt_OK, base64.b64encode(ciphertext)
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return WXBizMsgCrypt_EncryptAES_Error, None
|
||||
|
||||
def decrypt(self, text, receiveid):
|
||||
"""对解密后的明文进行补位删除
|
||||
@param text: 密文
|
||||
@param receiveid: receiveid
|
||||
@return: 删除填充补位后的明文
|
||||
"""
|
||||
try:
|
||||
cryptor = AES.new(self.key, self.mode, self.key[:16])
|
||||
# 使用BASE64对密文进行解码，然后AES-CBC解密
|
||||
plain_text = cryptor.decrypt(base64.b64decode(text))
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return WXBizMsgCrypt_DecryptAES_Error, None
|
||||
try:
|
||||
pad = plain_text[-1]
|
||||
# 去掉补位字符串
|
||||
# pkcs7 = PKCS7Encoder()
|
||||
# plain_text = pkcs7.encode(plain_text)
|
||||
# 去除16位随机字符串
|
||||
content = plain_text[16:-pad]
|
||||
xml_len = socket.ntohl(struct.unpack("I", content[: 4])[0])
|
||||
xml_content = content[4: xml_len + 4]
|
||||
from_receiveid = content[xml_len + 4:]
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return WXBizMsgCrypt_IllegalBuffer, None
|
||||
|
||||
if from_receiveid.decode('utf8') != receiveid:
|
||||
return WXBizMsgCrypt_ValidateCorpid_Error, None
|
||||
return 0, xml_content
|
||||
|
||||
@staticmethod
|
||||
def get_random_str():
|
||||
""" 随机生成16位字符串
|
||||
@return: 16位字符串
|
||||
"""
|
||||
return str(random.randint(1000000000000000, 9999999999999999)).encode()
|
||||
|
||||
|
||||
class WXBizMsgCrypt(object):
|
||||
# 构造函数
|
||||
def __init__(self, sToken, sEncodingAESKey, sReceiveId):
|
||||
try:
|
||||
self.key = base64.b64decode(sEncodingAESKey + "=")
|
||||
assert len(self.key) == 32
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
throw_exception("[error]: EncodingAESKey unvalid !", FormatException)
|
||||
# return WXBizMsgCrypt_IllegalAesKey,None
|
||||
self.m_sToken = sToken
|
||||
self.m_sReceiveId = sReceiveId
|
||||
|
||||
# 验证URL
|
||||
# @param sMsgSignature: 签名串，对应URL参数的msg_signature
|
||||
# @param sTimeStamp: 时间戳，对应URL参数的timestamp
|
||||
# @param sNonce: 随机串，对应URL参数的nonce
|
||||
# @param sEchoStr: 随机串，对应URL参数的echostr
|
||||
# @param sReplyEchoStr: 解密之后的echostr，当return返回0时有效
|
||||
# @return：成功0，失败返回对应的错误码
|
||||
|
||||
def VerifyURL(self, sMsgSignature, sTimeStamp, sNonce, sEchoStr):
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, sTimeStamp, sNonce, sEchoStr)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if not signature == sMsgSignature:
|
||||
return WXBizMsgCrypt_ValidateSignature_Error, None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, sReplyEchoStr = pc.decrypt(sEchoStr, self.m_sReceiveId)
|
||||
return ret, sReplyEchoStr
|
||||
|
||||
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
|
||||
# 将企业回复用户的消息加密打包
|
||||
# @param sReplyMsg: 企业号待回复用户的消息，xml格式的字符串
|
||||
# @param sTimeStamp: 时间戳，可以自己生成，也可以用URL参数的timestamp,如为None则自动用当前时间
|
||||
# @param sNonce: 随机串，可以自己生成，也可以用URL参数的nonce
|
||||
# sEncryptMsg: 加密后的可以直接回复用户的密文，包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串,
|
||||
# return：成功0，sEncryptMsg,失败返回对应的错误码None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
|
||||
encrypt = encrypt.decode('utf8')
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if timestamp is None:
|
||||
timestamp = str(int(time.time()))
|
||||
# 生成安全签名
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, timestamp, sNonce, encrypt)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
xmlParse = XMLParse()
|
||||
return ret, xmlParse.generate(encrypt, signature, timestamp, sNonce)
|
||||
|
||||
def DecryptMsg(self, sPostData, sMsgSignature, sTimeStamp, sNonce):
|
||||
# 检验消息的真实性，并且获取解密后的明文
|
||||
# @param sMsgSignature: 签名串，对应URL参数的msg_signature
|
||||
# @param sTimeStamp: 时间戳，对应URL参数的timestamp
|
||||
# @param sNonce: 随机串，对应URL参数的nonce
|
||||
# @param sPostData: 密文，对应POST请求的数据
|
||||
# xml_content: 解密后的原文，当return返回0时有效
|
||||
# @return: 成功0，失败返回对应的错误码
|
||||
# 验证安全签名
|
||||
xmlParse = XMLParse()
|
||||
ret, encrypt = xmlParse.extract(sPostData)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, sTimeStamp, sNonce, encrypt)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if not signature == sMsgSignature:
|
||||
return WXBizMsgCrypt_ValidateSignature_Error, None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, xml_content = pc.decrypt(encrypt, self.m_sReceiveId)
|
||||
return ret, xml_content
|
||||
@@ -14,7 +14,7 @@ from app.application.messaging.agent import (
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.modules import _ModuleBase, _MessageBase
|
||||
from app.modules.wechat.WXBizMsgCrypt3 import WXBizMsgCrypt
|
||||
from app.adapters.external.wechat_crypt import WXBizMsgCrypt
|
||||
from app.modules.wechat.wechat import WeChat
|
||||
from app.modules.wechat.wechatbot import WeChatBot
|
||||
from app.schemas import MessageChannel, CommingMessage, Notification, CommandRegisterEventData
|
||||
|
||||
Reference in New Issue
Block a user