mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-06-04 15:09:43 +08:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3653164924 | ||
|
|
ca0127cc87 | ||
|
|
092666f9d2 | ||
|
|
7b97e2039f | ||
|
|
e168e31a8f | ||
|
|
3ee601574c | ||
|
|
0ee9fec1d2 | ||
|
|
9069dccb2a |
32
.pylintrc
32
.pylintrc
@@ -5,38 +5,30 @@ init-hook='import sys; sys.path.append(".")'
|
||||
# 忽略的文件和目录
|
||||
ignore=.git,__pycache__,.venv,build,dist,tests,docs
|
||||
|
||||
# 通过 `pylint app/` 检查主程序时不扫描内置插件目录,
|
||||
# 插件依赖和动态模型较多,容易产生与主程序无关的误报。
|
||||
ignore-paths=^app/plugins(/|$)
|
||||
|
||||
# 并行作业数量
|
||||
jobs=0
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
# 只关注错误级别的问题,禁用警告、约定和重构建议
|
||||
# E = Error (错误) - 会导致构建失败
|
||||
# W = Warning (警告) - 仅显示,不会失败
|
||||
# R = Refactor (重构建议) - 仅显示,不会失败
|
||||
# C = Convention (约定) - 仅显示,不会失败
|
||||
# I = Information (信息) - 仅显示,不会失败
|
||||
|
||||
# 禁用大部分警告、约定和重构建议,只保留错误和重要警告
|
||||
# 只启用确定性较强的严重问题检查,避免 SQLAlchemy、FastAPI 依赖注入、
|
||||
# 第三方 SDK 等动态对象被 Pylint 推断成误报。
|
||||
disable=all
|
||||
enable=E,
|
||||
syntax-error,
|
||||
enable=syntax-error,
|
||||
undefined-variable,
|
||||
used-before-assignment,
|
||||
possibly-used-before-assignment,
|
||||
unreachable,
|
||||
return-outside-function,
|
||||
yield-outside-function,
|
||||
continue-in-finally,
|
||||
nonlocal-without-binding,
|
||||
undefined-loop-variable,
|
||||
redefined-builtin,
|
||||
not-callable,
|
||||
assignment-from-no-return,
|
||||
no-value-for-parameter,
|
||||
too-many-function-args,
|
||||
unexpected-keyword-arg,
|
||||
redundant-keyword-arg,
|
||||
import-error,
|
||||
relative-beyond-top-level
|
||||
relative-beyond-top-level,
|
||||
no-name-in-module
|
||||
|
||||
[REPORTS]
|
||||
# 设置报告格式
|
||||
@@ -80,4 +72,6 @@ ignore-imports=yes
|
||||
|
||||
[TYPECHECK]
|
||||
# 生成缺失成员提示的类列表
|
||||
generated-members=requests.packages.urllib3
|
||||
generated-members=requests.packages.urllib3
|
||||
# app.helper.sites 会主动隐藏模块属性枚举,避免误报 no-name-in-module
|
||||
ignored-modules=app.helper.sites
|
||||
|
||||
@@ -157,7 +157,7 @@ def _parse_skill_metadata( # noqa: C901
|
||||
MAX_SKILL_COMPATIBILITY_LENGTH,
|
||||
skill_path,
|
||||
)
|
||||
compatibility_str = compatibility_str[:MAX_SKILL_COMPATIBILITY_LENGTH]
|
||||
compatibility_str = str(compatibility_str)[:MAX_SKILL_COMPATIBILITY_LENGTH]
|
||||
|
||||
# 版本号,默认为 0(表示未设置版本)
|
||||
raw_version = frontmatter_data.get("version")
|
||||
|
||||
@@ -236,7 +236,8 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
Returns:
|
||||
str: 友好的提示消息,如果返回 None 或空字符串则使用 explanation
|
||||
"""
|
||||
return None
|
||||
explanation = kwargs.get("explanation")
|
||||
return str(explanation) if explanation else None
|
||||
|
||||
@abstractmethod
|
||||
async def run(self, **kwargs) -> str:
|
||||
|
||||
@@ -26,9 +26,11 @@ class UserChoiceOptionInput(BaseModel):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_option(self):
|
||||
if not self.label.strip():
|
||||
label = str(self.label)
|
||||
value = str(self.value)
|
||||
if not label.strip():
|
||||
raise ValueError("label 不能为空")
|
||||
if not self.value.strip():
|
||||
if not value.strip():
|
||||
raise ValueError("value 不能为空")
|
||||
return self
|
||||
|
||||
@@ -55,7 +57,8 @@ class AskUserChoiceInput(BaseModel):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_payload(self):
|
||||
if not self.message.strip():
|
||||
message = str(self.message)
|
||||
if not message.strip():
|
||||
raise ValueError("message 不能为空")
|
||||
if not self.options:
|
||||
raise ValueError("options 至少需要提供一个")
|
||||
|
||||
@@ -198,68 +198,62 @@ class BrowseWebpageTool(MoviePilotTool):
|
||||
cookies: Optional[str],
|
||||
user_agent: Optional[str],
|
||||
) -> str:
|
||||
"""在同步上下文中执行 Playwright 浏览器操作"""
|
||||
from playwright.sync_api import sync_playwright
|
||||
"""在同步上下文中执行 CloakBrowser 浏览器操作"""
|
||||
from cloakbrowser import launch_context
|
||||
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = None
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
# 启动浏览器
|
||||
browser_type = settings.PLAYWRIGHT_BROWSER_TYPE or "chromium"
|
||||
browser = playwright[browser_type].launch(headless=True)
|
||||
|
||||
# 创建上下文
|
||||
context_kwargs = {}
|
||||
if user_agent:
|
||||
context_kwargs["user_agent"] = user_agent
|
||||
# 设置视口大小
|
||||
context_kwargs["viewport"] = {
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
context_kwargs = {
|
||||
"viewport": {
|
||||
"width": SCREENSHOT_MAX_WIDTH,
|
||||
"height": SCREENSHOT_MAX_HEIGHT,
|
||||
}
|
||||
}
|
||||
if user_agent:
|
||||
context_kwargs["user_agent"] = user_agent
|
||||
|
||||
context = browser.new_context(**context_kwargs)
|
||||
page = context.new_page()
|
||||
page.set_default_timeout(timeout * 1000)
|
||||
context = launch_context(
|
||||
headless=True,
|
||||
humanize=settings.CLOAKBROWSER_HUMANIZE,
|
||||
human_preset=settings.CLOAKBROWSER_HUMAN_PRESET,
|
||||
**context_kwargs,
|
||||
)
|
||||
page = context.new_page()
|
||||
page.set_default_timeout(timeout * 1000)
|
||||
|
||||
# 设置 cookies
|
||||
if cookies:
|
||||
page.set_extra_http_headers({"cookie": cookies})
|
||||
# 设置 cookies
|
||||
if cookies:
|
||||
page.set_extra_http_headers({"cookie": cookies})
|
||||
|
||||
# 对于非 goto 操作,如果提供了 url 先导航
|
||||
if url and browser_action != BrowserAction.GOTO:
|
||||
page.goto(
|
||||
url, wait_until="domcontentloaded", timeout=timeout * 1000
|
||||
)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
# 对于非 goto 操作,如果提供了 url 先导航
|
||||
if url and browser_action != BrowserAction.GOTO:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
|
||||
# 执行具体操作
|
||||
result = self._do_action(
|
||||
page,
|
||||
browser_action,
|
||||
url,
|
||||
selector,
|
||||
value,
|
||||
script,
|
||||
content_type,
|
||||
timeout,
|
||||
)
|
||||
return result
|
||||
# 执行具体操作
|
||||
result = self._do_action(
|
||||
page,
|
||||
browser_action,
|
||||
url,
|
||||
selector,
|
||||
value,
|
||||
script,
|
||||
content_type,
|
||||
timeout,
|
||||
)
|
||||
return result
|
||||
|
||||
finally:
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
if browser:
|
||||
browser.close()
|
||||
finally:
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Playwright 执行失败: {e}", exc_info=True)
|
||||
return f"Playwright 执行失败: {str(e)}"
|
||||
logger.error(f"CloakBrowser 执行失败: {e}", exc_info=True)
|
||||
return f"CloakBrowser 执行失败: {str(e)}"
|
||||
|
||||
def _do_action(
|
||||
self,
|
||||
|
||||
@@ -6,7 +6,7 @@ import signal
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Optional, TextIO, Type
|
||||
from typing import Any, Optional, TextIO, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -188,7 +188,7 @@ class ExecuteCommandTool(MoviePilotTool):
|
||||
output.append(stream_name, chunk.decode("utf-8", errors="replace"))
|
||||
|
||||
@staticmethod
|
||||
def _terminate_process(process: asyncio.subprocess.Process, sig: int):
|
||||
def _terminate_process(process: Any, sig: int):
|
||||
"""向进程组发送终止信号;不支持进程组的平台回退为单进程终止。"""
|
||||
try:
|
||||
if os.name == "posix":
|
||||
@@ -203,7 +203,7 @@ class ExecuteCommandTool(MoviePilotTool):
|
||||
@classmethod
|
||||
async def _cleanup_process(
|
||||
cls,
|
||||
process: asyncio.subprocess.Process,
|
||||
process: Any,
|
||||
wait_task: asyncio.Task,
|
||||
) -> None:
|
||||
"""先温和终止,失败后强杀,避免超时 shell 遗留子进程。"""
|
||||
|
||||
@@ -159,7 +159,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
处理插件模块执行错误
|
||||
"""
|
||||
if kwargs.get("raise_exception"):
|
||||
raise
|
||||
raise err
|
||||
logger.error(
|
||||
f"运行插件 {plugin_id} 模块 {method} 出错:{str(err)}\n{traceback.format_exc()}"
|
||||
)
|
||||
@@ -185,7 +185,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
处理系统模块执行错误
|
||||
"""
|
||||
if kwargs.get("raise_exception"):
|
||||
raise
|
||||
raise err
|
||||
logger.error(
|
||||
f"运行模块 {module_id}.{method} 出错:{str(err)}\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.core.metainfo import MetaInfo
|
||||
from app.db.downloadhistory_oper import DownloadHistoryOper
|
||||
from app.db.mediaserver_oper import MediaServerOper
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.thread import ThreadHelper
|
||||
from app.helper.torrent import TorrentHelper
|
||||
from app.log import logger
|
||||
from app.schemas import ExistMediaInfo, FileURI, NotExistMediaInfo, DownloadingTorrent, Notification, ResourceSelectionEventData, \
|
||||
@@ -32,6 +33,31 @@ class DownloadChain(ChainBase):
|
||||
下载处理链
|
||||
"""
|
||||
|
||||
def _submit_download_added_task(
|
||||
self,
|
||||
context: Context,
|
||||
download_dir: Path,
|
||||
torrent_content: Union[str, bytes],
|
||||
) -> None:
|
||||
"""
|
||||
后台执行下载成功后的附加处理,避免站点字幕下载阻塞添加下载响应。
|
||||
"""
|
||||
|
||||
def _run_download_added() -> None:
|
||||
try:
|
||||
self.download_added(
|
||||
context=context,
|
||||
download_dir=download_dir,
|
||||
torrent_content=torrent_content,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"执行下载成功后处理失败:{str(err)}")
|
||||
|
||||
try:
|
||||
ThreadHelper().submit(_run_download_added)
|
||||
except Exception as err:
|
||||
logger.error(f"提交下载成功后处理后台任务失败:{str(err)}")
|
||||
|
||||
def download_torrent(self, torrent: TorrentInfo,
|
||||
channel: MessageChannel = None,
|
||||
source: Optional[str] = None,
|
||||
@@ -371,7 +397,11 @@ class DownloadChain(ChainBase):
|
||||
username=username,
|
||||
)
|
||||
# 下载成功后处理
|
||||
self.download_added(context=context, download_dir=download_dir, torrent_content=torrent_content)
|
||||
self._submit_download_added_task(
|
||||
context=context,
|
||||
download_dir=download_dir,
|
||||
torrent_content=torrent_content,
|
||||
)
|
||||
# 广播事件
|
||||
self.eventmanager.send_event(EventType.DownloadAdded, {
|
||||
"hash": _hash,
|
||||
|
||||
@@ -788,7 +788,7 @@ class SkillsChain(ChainBase):
|
||||
if skill.source_type == "registry":
|
||||
text_lines.append("社区源,安装前请自行甄别安全性")
|
||||
|
||||
if any(skill.source_type == "registry" for skill in page_items):
|
||||
if any(skill.source_type == "registry" for skill in items):
|
||||
text_lines.extend(
|
||||
[
|
||||
"",
|
||||
|
||||
@@ -2245,6 +2245,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
)
|
||||
if not built_meta:
|
||||
return None
|
||||
if not meta:
|
||||
# _build_path_meta 已经应用过手动季集/自定义格式覆盖;
|
||||
# 这里避免再次偏移集数,导致手动整理的集数偏移翻倍。
|
||||
return built_meta
|
||||
return _apply_meta_overrides(built_meta, source_path)
|
||||
|
||||
def _build_path_meta(
|
||||
|
||||
@@ -998,7 +998,8 @@ def config_list(show_secrets: bool) -> None:
|
||||
@click.argument("key")
|
||||
def config_get(key: str) -> None:
|
||||
"""读取单个配置项"""
|
||||
if key not in Settings.model_fields and not hasattr(settings, key):
|
||||
setting_fields = Settings.model_fields.keys()
|
||||
if key not in setting_fields and not hasattr(settings, key):
|
||||
raise click.ClickException(f"配置项不存在:{key}")
|
||||
click.echo(_format_value(getattr(settings, key)))
|
||||
|
||||
|
||||
@@ -331,8 +331,12 @@ class ConfigModel(BaseModel):
|
||||
NO_CACHE_SITE_KEY: str = "m-team"
|
||||
# OCR服务器地址,用于识别站点验证码
|
||||
OCR_HOST: str = "https://movie-pilot.org"
|
||||
# 仿真类型:playwright 或 flaresolverr
|
||||
BROWSER_EMULATION: str = "playwright"
|
||||
# 仿真类型:cloakbrowser 或 flaresolverr,其他值按 cloakbrowser 处理
|
||||
BROWSER_EMULATION: str = "cloakbrowser"
|
||||
# CloakBrowser 是否启用拟人化输入
|
||||
CLOAKBROWSER_HUMANIZE: bool = True
|
||||
# CloakBrowser 拟人化输入预设:default 或 careful
|
||||
CLOAKBROWSER_HUMAN_PRESET: str = "default"
|
||||
# FlareSolverr 服务地址,例如 http://127.0.0.1:8191
|
||||
FLARESOLVERR_URL: Optional[str] = None
|
||||
|
||||
@@ -526,7 +530,7 @@ class ConfigModel(BaseModel):
|
||||
# ==================== Docker配置 ====================
|
||||
# Docker Client API地址
|
||||
DOCKER_CLIENT_API: Optional[str] = "tcp://127.0.0.1:38379"
|
||||
# Playwright浏览器类型,chromium/firefox
|
||||
# Playwright浏览器类型,供智能体浏览器工具和插件直接使用 Playwright 时读取
|
||||
PLAYWRIGHT_BROWSER_TYPE: str = "chromium"
|
||||
|
||||
# ==================== AI智能体配置 ====================
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from datetime import datetime
|
||||
from builtins import list as builtin_list
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, JSON, String, Index, and_, or_, select
|
||||
@@ -34,9 +35,9 @@ class Workflow(Base):
|
||||
# 已执行次数
|
||||
run_count = Column(Integer, default=0)
|
||||
# 任务列表
|
||||
actions = Column(JSON, default=list)
|
||||
actions = Column(JSON, default=builtin_list)
|
||||
# 任务流
|
||||
flows = Column(JSON, default=list)
|
||||
flows = Column(JSON, default=builtin_list)
|
||||
# 执行上下文
|
||||
context = Column(JSON, default=dict)
|
||||
# 创建时间
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import uuid
|
||||
from typing import Callable, Any, Optional
|
||||
|
||||
from cf_clearance import sync_cf_retry, sync_stealth
|
||||
from playwright.sync_api import sync_playwright, Page
|
||||
from playwright.sync_api import BrowserContext, Page
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
@@ -10,17 +9,33 @@ from app.utils.http import RequestUtils, cookie_parse
|
||||
|
||||
|
||||
class PlaywrightHelper:
|
||||
def __init__(self, browser_type=settings.PLAYWRIGHT_BROWSER_TYPE):
|
||||
self.browser_type = browser_type
|
||||
def __init__(self, browser_type: Optional[str] = None, *args, **kwargs):
|
||||
"""
|
||||
兼容旧的 PlaywrightHelper(browser_type=...) 构造方式。
|
||||
"""
|
||||
self.browser_type = browser_type or settings.PLAYWRIGHT_BROWSER_TYPE
|
||||
|
||||
@staticmethod
|
||||
def __pass_cloudflare(url: str, page: Page) -> bool:
|
||||
def __browser_emulation() -> str:
|
||||
"""
|
||||
尝试跳过cloudfare验证
|
||||
当前浏览器仿真类型。
|
||||
"""
|
||||
sync_stealth(page, pure=True)
|
||||
page.goto(url)
|
||||
return sync_cf_retry(page)[0]
|
||||
return (settings.BROWSER_EMULATION or "cloakbrowser").lower()
|
||||
|
||||
@staticmethod
|
||||
def __launch_cloakbrowser_context(headless: bool,
|
||||
user_agent: Optional[str] = None,
|
||||
proxies: Optional[dict] = None) -> BrowserContext:
|
||||
"""
|
||||
启动 CloakBrowser 上下文。
|
||||
"""
|
||||
from cloakbrowser import launch_context
|
||||
|
||||
return launch_context(headless=headless,
|
||||
proxy=proxies,
|
||||
user_agent=user_agent,
|
||||
humanize=settings.CLOAKBROWSER_HUMANIZE,
|
||||
human_preset=settings.CLOAKBROWSER_HUMAN_PRESET)
|
||||
|
||||
@staticmethod
|
||||
def __fs_cookie_str(cookies: list) -> str:
|
||||
@@ -148,51 +163,44 @@ class PlaywrightHelper:
|
||||
"""
|
||||
result = None
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = None
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
# 如果配置使用 FlareSolverr,先通过其获取清除后的 cookies 与 UA
|
||||
fs_cookie_header = None
|
||||
fs_ua = None
|
||||
if settings.BROWSER_EMULATION == "flaresolverr":
|
||||
solution = self.__flaresolverr_request(url=url, cookies=cookies,
|
||||
proxy_config=proxies, timeout=timeout)
|
||||
if solution:
|
||||
fs_cookie_header = self.__fs_cookie_str(solution.get("cookies", []))
|
||||
fs_ua = solution.get("userAgent")
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
# 如果配置使用 FlareSolverr,先通过其获取清除后的 cookies 与 UA
|
||||
fs_cookie_header = None
|
||||
fs_ua = None
|
||||
if self.__browser_emulation() == "flaresolverr":
|
||||
solution = self.__flaresolverr_request(url=url, cookies=cookies,
|
||||
proxy_config=proxies, timeout=timeout)
|
||||
if solution:
|
||||
fs_cookie_header = self.__fs_cookie_str(solution.get("cookies", []))
|
||||
fs_ua = solution.get("userAgent")
|
||||
|
||||
browser = playwright[self.browser_type].launch(headless=headless)
|
||||
context = browser.new_context(user_agent=fs_ua or ua, proxy=proxies)
|
||||
page = context.new_page()
|
||||
context = self.__launch_cloakbrowser_context(headless=headless,
|
||||
user_agent=fs_ua or ua,
|
||||
proxies=proxies)
|
||||
page = context.new_page()
|
||||
|
||||
# 优先使用 FlareSolverr 返回,其次使用入参
|
||||
merged_cookie = fs_cookie_header or cookies
|
||||
if merged_cookie:
|
||||
page.set_extra_http_headers({"cookie": merged_cookie})
|
||||
# 优先使用 FlareSolverr 返回,其次使用入参
|
||||
merged_cookie = fs_cookie_header or cookies
|
||||
if merged_cookie:
|
||||
page.set_extra_http_headers({"cookie": merged_cookie})
|
||||
|
||||
if settings.BROWSER_EMULATION == "playwright":
|
||||
if not self.__pass_cloudflare(url, page):
|
||||
logger.warn("cloudflare challenge fail!")
|
||||
else:
|
||||
page.goto(url)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
page.goto(url)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
|
||||
# 回调函数
|
||||
result = callback(page)
|
||||
# 回调函数
|
||||
result = callback(page)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"网页操作失败: {str(e)}")
|
||||
finally:
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
if browser:
|
||||
browser.close()
|
||||
except Exception as e:
|
||||
logger.error(f"网页操作失败: {str(e)}")
|
||||
finally:
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Playwright初始化失败: {str(e)}")
|
||||
logger.error(f"CloakBrowser初始化失败: {str(e)}")
|
||||
|
||||
return result
|
||||
|
||||
@@ -213,7 +221,7 @@ class PlaywrightHelper:
|
||||
"""
|
||||
source = None
|
||||
# 如果配置为 FlareSolverr,则直接调用获取页面源码
|
||||
if settings.BROWSER_EMULATION == "flaresolverr":
|
||||
if self.__browser_emulation() == "flaresolverr":
|
||||
try:
|
||||
solution = self.__flaresolverr_request(url=url, cookies=cookies,
|
||||
proxy_config=proxies, timeout=timeout)
|
||||
@@ -222,36 +230,32 @@ class PlaywrightHelper:
|
||||
except Exception as e:
|
||||
logger.error(f"FlareSolverr 获取源码失败: {str(e)}")
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = None
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
browser = playwright[self.browser_type].launch(headless=headless)
|
||||
context = browser.new_context(user_agent=ua, proxy=proxies)
|
||||
page = context.new_page()
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
context = self.__launch_cloakbrowser_context(headless=headless,
|
||||
user_agent=ua,
|
||||
proxies=proxies)
|
||||
page = context.new_page()
|
||||
|
||||
if cookies:
|
||||
page.set_extra_http_headers({"cookie": cookies})
|
||||
if cookies:
|
||||
page.set_extra_http_headers({"cookie": cookies})
|
||||
|
||||
if not self.__pass_cloudflare(url, page):
|
||||
logger.warn("cloudflare challenge fail!")
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
page.goto(url)
|
||||
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
|
||||
|
||||
source = page.content()
|
||||
source = page.content()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取网页源码失败: {str(e)}")
|
||||
source = None
|
||||
finally:
|
||||
# 确保资源被正确清理
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
if browser:
|
||||
browser.close()
|
||||
except Exception as e:
|
||||
logger.error(f"获取网页源码失败: {str(e)}")
|
||||
source = None
|
||||
finally:
|
||||
# 确保资源被正确清理
|
||||
if page:
|
||||
page.close()
|
||||
if context:
|
||||
context.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Playwright初始化失败: {str(e)}")
|
||||
logger.error(f"CloakBrowser初始化失败: {str(e)}")
|
||||
|
||||
return source
|
||||
|
||||
@@ -94,7 +94,8 @@ class SkillHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
返回系统默认的技能市场列表,用于区分内置源和用户追加源。
|
||||
"""
|
||||
default_value = type(settings).model_fields["SKILL_MARKET"].default
|
||||
skill_market_field = type(settings).model_fields.get("SKILL_MARKET")
|
||||
default_value = skill_market_field.default if skill_market_field else None
|
||||
if not default_value:
|
||||
return []
|
||||
return [item.strip() for item in str(default_value).split(",") if item.strip()]
|
||||
|
||||
@@ -13,6 +13,35 @@ from app.utils.string import StringUtils
|
||||
class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
schema = SiteSchema.NexusAudiences
|
||||
|
||||
def _parse_message_unread(self, html_text):
|
||||
"""
|
||||
解析 Audiences 新版顶部用户栏中的未读消息数。
|
||||
"""
|
||||
super()._parse_message_unread(html_text)
|
||||
if self.message_unread:
|
||||
return
|
||||
|
||||
html = etree.HTML(html_text)
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
|
||||
message_tools = html.xpath(
|
||||
'//a[contains(@class, "site-userbar__compact-tool") and contains(@href, "messages.php") '
|
||||
'and (contains(@class, "site-userbar__compact-tool--has-unread") '
|
||||
'or .//*[contains(@class, "site-userbar__compact-tool-badge--unread")])]'
|
||||
'|//a[contains(@href, "messages.php") '
|
||||
'and (contains(@title, "收件箱") or contains(@aria-label, "收件箱"))]'
|
||||
)
|
||||
for message_link in message_tools:
|
||||
unread = self.__parse_inbox_unread(message_link)
|
||||
if unread is not None:
|
||||
self.message_unread = unread
|
||||
return
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_traffic_info(self, html_text):
|
||||
"""
|
||||
解析用户流量信息
|
||||
@@ -128,6 +157,47 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
self.seeding = StringUtils.str_int(active_match.group(1))
|
||||
self.leeching = StringUtils.str_int(active_match.group(2))
|
||||
|
||||
def __parse_inbox_unread(self, message_link):
|
||||
"""
|
||||
从 Audiences 收件箱入口提取未读数。
|
||||
"""
|
||||
inbox_texts = [
|
||||
message_link.get("title"),
|
||||
message_link.get("aria-label"),
|
||||
*message_link.xpath(
|
||||
'.//*[contains(@class, "site-userbar__compact-tool-badge--unread") '
|
||||
'or contains(@class, "site-userbar__compact-tool-badge")]/text()'
|
||||
)
|
||||
]
|
||||
|
||||
for inbox_text in inbox_texts:
|
||||
unread = self.__extract_inbox_unread(inbox_text)
|
||||
if unread is not None:
|
||||
return unread
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __extract_inbox_unread(text: str):
|
||||
"""
|
||||
Audiences 收件箱角标格式为 总数/未读数,例如 1749/172。
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
text = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
inbox_count = re.search(r"(?:收件箱\s*)?(\d[\d,]*)\s*/\s*(\d[\d,]*)", text)
|
||||
if inbox_count:
|
||||
return StringUtils.str_int(inbox_count.group(2))
|
||||
|
||||
single_count = re.search(r"收件箱\s*(\d[\d,]*)", text)
|
||||
if single_count:
|
||||
return StringUtils.str_int(single_count.group(1))
|
||||
return None
|
||||
|
||||
def _parse_seeding_pages(self):
|
||||
if not self._torrent_seeding_page:
|
||||
return
|
||||
|
||||
@@ -3,7 +3,7 @@ import re
|
||||
import traceback
|
||||
from typing import Any, Optional
|
||||
from typing import List
|
||||
from urllib.parse import quote, urlencode, urlparse, parse_qs
|
||||
from urllib.parse import quote, urlparse, parse_qs
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from jinja2 import Template
|
||||
@@ -14,6 +14,7 @@ from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
from app.utils.http import RequestUtils, AsyncRequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.url import UrlUtils
|
||||
|
||||
|
||||
class SiteSpider:
|
||||
@@ -120,14 +121,15 @@ class SiteSpider:
|
||||
search_word = self.keyword
|
||||
# 查询模式与
|
||||
search_mode = "0"
|
||||
is_imdbid_search = isinstance(self.keyword, str) and re.fullmatch(r"tt\d+", self.keyword)
|
||||
search_word = self.__format_search_word(search_word)
|
||||
|
||||
# 搜索URL
|
||||
indexer_params = self.search.get("params", {}).copy()
|
||||
if indexer_params:
|
||||
search_area = indexer_params.get('search_area')
|
||||
# search_area非0表示支持imdbid搜索
|
||||
if (search_area and
|
||||
(not self.keyword or not self.keyword.startswith('tt'))):
|
||||
if search_area and not is_imdbid_search:
|
||||
# 支持imdbid搜索,但关键字不是imdbid时,不启用imdbid搜索
|
||||
indexer_params.pop('search_area')
|
||||
# 变量字典
|
||||
@@ -168,7 +170,7 @@ class SiteSpider:
|
||||
params.update({
|
||||
"cat%s" % cat.get("id"): 1
|
||||
})
|
||||
searchurl = self.domain + torrentspath + "?" + urlencode(params)
|
||||
searchurl = UrlUtils.combine_url(self.domain, torrentspath, params)
|
||||
else:
|
||||
# 变量字典
|
||||
inputs_dict = {
|
||||
@@ -200,6 +202,22 @@ class SiteSpider:
|
||||
|
||||
return searchurl
|
||||
|
||||
def __format_search_word(self, search_word: str) -> str:
|
||||
"""
|
||||
按站点配置转换搜索关键字,用于兼容站点特殊的 IMDb ID 查询格式。
|
||||
"""
|
||||
if not search_word or not isinstance(search_word, str):
|
||||
return search_word
|
||||
if re.fullmatch(r"tt\d+", search_word):
|
||||
imdbid_format = self.search.get("imdbid_format")
|
||||
if imdbid_format:
|
||||
return str(imdbid_format).format(
|
||||
keyword=search_word,
|
||||
imdbid=search_word,
|
||||
imdbid_num=search_word[2:]
|
||||
)
|
||||
return search_word
|
||||
|
||||
def get_torrents(self) -> List[dict]:
|
||||
"""
|
||||
开始请求
|
||||
|
||||
@@ -583,6 +583,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
轮询监控(改进版)
|
||||
"""
|
||||
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
|
||||
with snapshot_lock:
|
||||
try:
|
||||
# 加载上次快照数据
|
||||
@@ -650,10 +651,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
logger.info(f"{storage}:{mon_path} 监控间隔已调整为 {new_interval} 分钟")
|
||||
logger.info(f"{storage}:{monitor_scope} 监控间隔已调整为 {new_interval} 分钟")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询监控 {storage}:{mon_path} 出现错误:{e}")
|
||||
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}")
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
|
||||
@@ -109,9 +109,8 @@ COPY --from=prepare_venv --chmod=777 ${VENV_PATH} ${VENV_PATH}
|
||||
COPY --from=prepare_venv /usr/local/bin/uv /usr/local/bin/uv
|
||||
COPY --from=prepare_venv /usr/local/bin/uv-pip-compat /usr/local/bin/uv-pip-compat
|
||||
|
||||
# playwright 环境
|
||||
# 浏览器运行依赖
|
||||
RUN playwright install-deps chromium \
|
||||
&& playwright install-deps firefox \
|
||||
&& apt-get autoremove -y \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
|
||||
@@ -46,6 +46,7 @@ function load_config_from_app_env() {
|
||||
["PROXY_HOST"]=""
|
||||
["GITHUB_TOKEN"]=""
|
||||
["MOVIEPILOT_AUTO_UPDATE"]="release"
|
||||
["BROWSER_EMULATION"]="cloakbrowser"
|
||||
|
||||
# database
|
||||
["DB_TYPE"]="sqlite"
|
||||
@@ -220,7 +221,7 @@ function graceful_exit() {
|
||||
# 插件依赖和主程序共用同一套 venv 时,历史安装记录可能已经污染环境,
|
||||
# 这里优先在真正拉起后端前做一次自愈,避免容器反复起不来。
|
||||
function ensure_backend_runtime_dependencies() {
|
||||
local probe_code="import alembic, fastapi, pydantic, pydantic_core, pydantic_settings, sqlalchemy, starlette, uvicorn; from pydantic import BaseModel, Field"
|
||||
local probe_code="import alembic, cloakbrowser, fastapi, pydantic, pydantic_core, pydantic_settings, sqlalchemy, starlette, uvicorn; from pydantic import BaseModel, Field"
|
||||
|
||||
INFO "→ 启动前检查后端核心依赖..."
|
||||
if "${VENV_PATH}/bin/python3" -c "${probe_code}" >/dev/null 2>&1; then
|
||||
@@ -327,12 +328,28 @@ chown -R moviepilot:moviepilot \
|
||||
/var/log/nginx
|
||||
chown moviepilot:moviepilot /etc/hosts /tmp
|
||||
|
||||
# 启动前优先确认主运行环境仍然健康,避免插件依赖污染导致服务直接起不来。
|
||||
ensure_backend_runtime_dependencies
|
||||
|
||||
# 下载浏览器内核
|
||||
if [[ "$HTTPS_PROXY" =~ ^https?:// ]] || [[ "$HTTPS_PROXY" =~ ^https?:// ]] || [[ "$PROXY_HOST" =~ ^https?:// ]]; then
|
||||
HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$PROXY_HOST}}" gosu moviepilot:moviepilot playwright install ${PLAYWRIGHT_BROWSER_TYPE:-chromium}
|
||||
else
|
||||
gosu moviepilot:moviepilot playwright install ${PLAYWRIGHT_BROWSER_TYPE:-chromium}
|
||||
fi
|
||||
function install_browser_kernel() {
|
||||
local emulation="${BROWSER_EMULATION:-cloakbrowser}"
|
||||
emulation="${emulation,,}"
|
||||
local proxy="${HTTPS_PROXY:-${https_proxy:-$PROXY_HOST}}"
|
||||
|
||||
if [ "${emulation}" != "cloakbrowser" ] && [ "${emulation}" != "flaresolverr" ] && [ -n "${emulation}" ]; then
|
||||
WARN "浏览器仿真类型 ${emulation} 已按 CloakBrowser 处理。"
|
||||
fi
|
||||
|
||||
INFO "下载 CloakBrowser 浏览器内核"
|
||||
if [[ "$proxy" =~ ^https?:// ]]; then
|
||||
HTTPS_PROXY="$proxy" gosu moviepilot:moviepilot python -m cloakbrowser install
|
||||
else
|
||||
gosu moviepilot:moviepilot python -m cloakbrowser install
|
||||
fi
|
||||
}
|
||||
|
||||
install_browser_kernel
|
||||
|
||||
# 证书管理
|
||||
source /app/docker/cert.sh
|
||||
@@ -358,9 +375,6 @@ fi
|
||||
# 设置后端服务权限掩码
|
||||
umask "${UMASK}"
|
||||
|
||||
# 启动前优先确认主运行环境仍然健康,避免插件依赖污染导致服务直接起不来。
|
||||
ensure_backend_runtime_dependencies
|
||||
|
||||
# 清除非系统环境导入的变量,保证转移到 dumb-init 的时候,不会带入不必要的环境变量
|
||||
INFO "准备为 Python 应用清理的非系统环境导入的变量..."
|
||||
if [ ${#VARS_SET_BY_SCRIPT[@]} -gt 0 ]; then
|
||||
|
||||
@@ -29,7 +29,7 @@ location /cookiecloud {
|
||||
}
|
||||
|
||||
# SSE特殊配置
|
||||
location ~ ^/api/v1/system/(message|progress/) {
|
||||
location ~ ^/api/v1/(system/(message|progress/|logging)|search/.*/stream$) {
|
||||
# SSE MIME类型设置
|
||||
default_type text/event-stream;
|
||||
|
||||
|
||||
@@ -80,6 +80,10 @@ function install_backend_and_download_resources() {
|
||||
cp /tmp/requirements.txt.backup /app/requirements.txt
|
||||
return 1
|
||||
fi
|
||||
INFO "正在更新 CloakBrowser 浏览器内核"
|
||||
if ! ${VENV_PATH}/bin/python -m cloakbrowser install; then
|
||||
WARN "CloakBrowser 浏览器内核更新失败,后续首次使用时可能重新下载"
|
||||
fi
|
||||
INFO "依赖更新成功"
|
||||
else
|
||||
INFO "依赖无变化,跳过依赖更新"
|
||||
|
||||
@@ -38,8 +38,8 @@ pillow~=12.1.1
|
||||
pillow-avif-plugin~=1.5.2
|
||||
pyTelegramBotAPI~=4.27.0
|
||||
telegramify-markdown~=0.5.2
|
||||
cloakbrowser~=0.3.28
|
||||
playwright~=1.53.0
|
||||
cf_clearance~=0.31.0
|
||||
torrentool~=1.2.0
|
||||
slack-bolt~=1.23.0
|
||||
slack-sdk~=3.35.0
|
||||
|
||||
@@ -2653,9 +2653,18 @@ def install_deps(*, python_bin: str, venv_dir: Path, recreate: bool) -> Path:
|
||||
|
||||
print_step("安装项目依赖")
|
||||
run([str(venv_pip), "install", "-r", str(ROOT / "requirements.txt")])
|
||||
install_browser_runtime(venv_python)
|
||||
return venv_python
|
||||
|
||||
|
||||
def install_browser_runtime(venv_python: Path) -> None:
|
||||
"""
|
||||
预下载 CloakBrowser 浏览器内核,避免首次仿真登录时才拉取大文件。
|
||||
"""
|
||||
print_step("安装 CloakBrowser 浏览器内核")
|
||||
run([str(venv_python), "-m", "cloakbrowser", "install"])
|
||||
|
||||
|
||||
def _startup_platform_name() -> str:
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
|
||||
95
tests/test_browser_helper.py
Normal file
95
tests/test_browser_helper.py
Normal file
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.helper.browser import PlaywrightHelper
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self) -> None:
|
||||
self.headers = None
|
||||
self.loaded_url = None
|
||||
self.closed = False
|
||||
|
||||
def set_extra_http_headers(self, headers: dict[str, str]) -> None:
|
||||
self.headers = headers
|
||||
|
||||
def goto(self, url: str) -> None:
|
||||
self.loaded_url = url
|
||||
|
||||
def wait_for_load_state(self, _state: str, timeout: int) -> None:
|
||||
self.timeout = timeout
|
||||
|
||||
def content(self) -> str:
|
||||
return "<html>ok</html>"
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeContext:
|
||||
def __init__(self, page: _FakePage) -> None:
|
||||
self.page = page
|
||||
self.closed = False
|
||||
|
||||
def new_page(self) -> _FakePage:
|
||||
return self.page
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class BrowserHelperTests(unittest.TestCase):
|
||||
def _assert_get_page_source_uses_cloakbrowser(self, emulation: str) -> None:
|
||||
page = _FakePage()
|
||||
context = _FakeContext(page)
|
||||
|
||||
with patch("app.helper.browser.settings.BROWSER_EMULATION", emulation), \
|
||||
patch.object(
|
||||
PlaywrightHelper,
|
||||
"_PlaywrightHelper__launch_cloakbrowser_context",
|
||||
return_value=context,
|
||||
) as launch_context:
|
||||
source = PlaywrightHelper().get_page_source(
|
||||
url="https://example.com",
|
||||
cookies="uid=1",
|
||||
ua="UA",
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
self.assertEqual(source, "<html>ok</html>")
|
||||
launch_context.assert_called_once_with(
|
||||
headless=False,
|
||||
user_agent="UA",
|
||||
proxies=None,
|
||||
)
|
||||
self.assertEqual(page.headers, {"cookie": "uid=1"})
|
||||
self.assertEqual(page.loaded_url, "https://example.com")
|
||||
self.assertTrue(page.closed)
|
||||
self.assertTrue(context.closed)
|
||||
|
||||
def test_default_emulation_uses_cloakbrowser_context(self):
|
||||
self._assert_get_page_source_uses_cloakbrowser("cloakbrowser")
|
||||
|
||||
def test_legacy_playwright_emulation_uses_cloakbrowser_context(self):
|
||||
self._assert_get_page_source_uses_cloakbrowser("Playwright")
|
||||
|
||||
def test_legacy_browser_type_constructor_is_accepted(self):
|
||||
page = _FakePage()
|
||||
context = _FakeContext(page)
|
||||
|
||||
with patch.object(
|
||||
PlaywrightHelper,
|
||||
"_PlaywrightHelper__launch_cloakbrowser_context",
|
||||
return_value=context,
|
||||
):
|
||||
source = PlaywrightHelper(browser_type="firefox").get_page_source(
|
||||
url="https://example.com"
|
||||
)
|
||||
|
||||
self.assertEqual(source, "<html>ok</html>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
97
tests/test_download_chain.py
Normal file
97
tests/test_download_chain.py
Normal file
@@ -0,0 +1,97 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import app.chain.download as download_module
|
||||
from app.chain.download import DownloadChain
|
||||
from app.core.context import Context, MediaInfo, TorrentInfo
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
class _FakeDownloadHistoryOper:
|
||||
"""
|
||||
避免单元测试写入真实下载历史,只验证下载链路的控制流。
|
||||
"""
|
||||
|
||||
def add(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def add_files(self, _files):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeTorrentHelper:
|
||||
"""
|
||||
避免解析真实种子内容,让测试聚焦下载成功后的后台处理。
|
||||
"""
|
||||
|
||||
def get_fileinfo_from_torrent_content(self, _torrent_content):
|
||||
return "", []
|
||||
|
||||
|
||||
class _FakeThreadHelper:
|
||||
"""
|
||||
捕获提交到线程池的任务,测试中手动触发以避免真正启动后台线程。
|
||||
"""
|
||||
|
||||
submitted = []
|
||||
|
||||
def submit(self, func, *args, **kwargs):
|
||||
self.submitted.append((func, args, kwargs))
|
||||
|
||||
|
||||
def test_download_single_submits_download_added_to_background(monkeypatch):
|
||||
"""
|
||||
添加下载成功后,站点字幕等后处理应提交到后台,不能阻塞下载接口返回。
|
||||
"""
|
||||
_FakeThreadHelper.submitted = []
|
||||
monkeypatch.setattr(download_module, "ThreadHelper", _FakeThreadHelper)
|
||||
monkeypatch.setattr(download_module, "DownloadHistoryOper", _FakeDownloadHistoryOper)
|
||||
monkeypatch.setattr(download_module, "TorrentHelper", _FakeTorrentHelper)
|
||||
|
||||
chain = DownloadChain.__new__(DownloadChain)
|
||||
chain.download = MagicMock(return_value=("qb", "hash123", "Original", "添加下载成功"))
|
||||
chain.download_added = MagicMock()
|
||||
chain.eventmanager = MagicMock()
|
||||
chain.eventmanager.send_event.return_value = None
|
||||
chain.post_message = MagicMock()
|
||||
|
||||
context = Context(
|
||||
meta_info=MetaInfo("Demo Movie 2024"),
|
||||
media_info=MediaInfo(
|
||||
type=MediaType.MOVIE,
|
||||
title="Demo Movie",
|
||||
year="2024",
|
||||
tmdb_id=1,
|
||||
genre_ids=[18],
|
||||
),
|
||||
torrent_info=TorrentInfo(
|
||||
title="Demo Movie 2024",
|
||||
enclosure="https://example.com/demo.torrent",
|
||||
site_cookie="uid=1",
|
||||
site_name="TestSite",
|
||||
),
|
||||
)
|
||||
|
||||
result = chain.download_single(
|
||||
context=context,
|
||||
torrent_content=b"torrent-content",
|
||||
save_path="/downloads",
|
||||
username="tester",
|
||||
)
|
||||
|
||||
assert result == "hash123"
|
||||
chain.download_added.assert_not_called()
|
||||
assert len(_FakeThreadHelper.submitted) == 1
|
||||
|
||||
task, args, kwargs = _FakeThreadHelper.submitted[0]
|
||||
assert args == ()
|
||||
assert kwargs == {}
|
||||
|
||||
task()
|
||||
|
||||
chain.download_added.assert_called_once_with(
|
||||
context=context,
|
||||
download_dir=Path("/downloads"),
|
||||
torrent_content=b"torrent-content",
|
||||
)
|
||||
156
tests/test_indexer_spider_search_url.py
Normal file
156
tests/test_indexer_spider_search_url.py
Normal file
@@ -0,0 +1,156 @@
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.modules.indexer.spider import SiteSpider
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
def _build_indexer(**kwargs):
|
||||
"""
|
||||
构造 SiteSpider 生成搜索 URL 所需的最小站点配置。
|
||||
"""
|
||||
indexer = {
|
||||
"id": "test",
|
||||
"name": "测试站点",
|
||||
"domain": "https://example.com/",
|
||||
"search": {
|
||||
"paths": [{"path": "torrents.php"}],
|
||||
"params": {"search": "{keyword}"},
|
||||
},
|
||||
"torrents": {"list": {}, "fields": {}},
|
||||
}
|
||||
indexer.update(kwargs)
|
||||
return indexer
|
||||
|
||||
|
||||
def _get_search_url(indexer: dict, keyword: str | list[str], mtype: MediaType = None) -> str:
|
||||
"""
|
||||
调用 SiteSpider 私有 URL 构造逻辑,避免真实请求站点。
|
||||
"""
|
||||
spider = SiteSpider(indexer=indexer, keyword=keyword, mtype=mtype)
|
||||
return spider._SiteSpider__get_search_url()
|
||||
|
||||
|
||||
def test_eastgame_imdb_search_uses_imdb_area():
|
||||
"""
|
||||
TLF 支持 IMDb ID 搜索时应使用站点配置的 IMDb 搜索区域。
|
||||
"""
|
||||
indexer = _build_indexer(
|
||||
id="eastgame",
|
||||
domain="https://pt.eastgame.org/",
|
||||
search={
|
||||
"paths": [{"path": "torrents.php"}],
|
||||
"params": {
|
||||
"search_area": 4,
|
||||
"search": "{keyword}",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
parsed_url = urlparse(_get_search_url(indexer, "tt16311594"))
|
||||
query = parse_qs(parsed_url.query)
|
||||
|
||||
assert parsed_url.geturl().startswith("https://pt.eastgame.org/torrents.php?")
|
||||
assert query["search"] == ["tt16311594"]
|
||||
assert query["search_area"] == ["4"]
|
||||
|
||||
|
||||
def test_eastgame_title_search_keeps_title_area():
|
||||
"""
|
||||
TLF 普通标题搜索不应误用 IMDb 搜索区域。
|
||||
"""
|
||||
indexer = _build_indexer(
|
||||
id="eastgame",
|
||||
domain="https://pt.eastgame.org/",
|
||||
search={
|
||||
"paths": [{"path": "torrents.php"}],
|
||||
"params": {
|
||||
"search_area": 4,
|
||||
"search": "{keyword}",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
query = parse_qs(urlparse(_get_search_url(indexer, "普通标题")).query)
|
||||
|
||||
assert query["search"] == ["普通标题"]
|
||||
assert query["search_area"] == ["0"]
|
||||
|
||||
|
||||
def test_eastgame_batch_search_keeps_title_area():
|
||||
"""
|
||||
TLF 批量搜索不是单个 IMDb ID,不能触发 IMDb 搜索区域。
|
||||
"""
|
||||
indexer = _build_indexer(
|
||||
id="eastgame",
|
||||
domain="https://pt.eastgame.org/",
|
||||
search={
|
||||
"paths": [{"path": "torrents.php"}],
|
||||
"params": {
|
||||
"search_area": 4,
|
||||
"search": "{keyword}",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
query = parse_qs(urlparse(_get_search_url(indexer, ["tt1234567", "tt7654321"])).query)
|
||||
|
||||
assert query["search"] == ["tt1234567 tt7654321"]
|
||||
assert query["search_mode"] == ["1"]
|
||||
assert query["search_area"] == ["0"]
|
||||
|
||||
|
||||
def test_ttg_imdb_search_formats_keyword_and_keeps_existing_query():
|
||||
"""
|
||||
TTG 的 IMDb 搜索需要 tt 前缀转换,并且路径自带查询参数不能生成双问号。
|
||||
"""
|
||||
indexer = _build_indexer(
|
||||
id="ttg",
|
||||
domain="https://totheglory.im/",
|
||||
search={
|
||||
"paths": [{"path": "browse.php?c=M"}],
|
||||
"params": {
|
||||
"search_field": "{keyword}",
|
||||
"c": "M",
|
||||
},
|
||||
"imdbid_format": "imdb{imdbid_num}",
|
||||
},
|
||||
category={
|
||||
"field": "search_field",
|
||||
"delimiter": " 分类:",
|
||||
"movie": [{"id": "电影DVDRip", "cat": "Movies/SD"}],
|
||||
},
|
||||
)
|
||||
|
||||
search_url = _get_search_url(indexer, "tt0049406", MediaType.MOVIE)
|
||||
query = parse_qs(urlparse(search_url).query)
|
||||
|
||||
assert search_url.count("?") == 1
|
||||
assert query["c"] == ["M"]
|
||||
assert query["search_field"] == ["imdb0049406 分类:电影DVDRip"]
|
||||
|
||||
|
||||
def test_ttg_title_search_does_not_format_keyword():
|
||||
"""
|
||||
TTG 普通标题搜索不能被 IMDb ID 格式化规则影响。
|
||||
"""
|
||||
indexer = _build_indexer(
|
||||
id="ttg",
|
||||
domain="https://totheglory.im/",
|
||||
search={
|
||||
"paths": [{"path": "browse.php?c=M"}],
|
||||
"params": {
|
||||
"search_field": "{keyword}",
|
||||
"c": "M",
|
||||
},
|
||||
"imdbid_format": "imdb{imdbid_num}",
|
||||
},
|
||||
category={
|
||||
"field": "search_field",
|
||||
"delimiter": " 分类:",
|
||||
"movie": [{"id": "电影DVDRip", "cat": "Movies/SD"}],
|
||||
},
|
||||
)
|
||||
|
||||
query = parse_qs(urlparse(_get_search_url(indexer, "The Movie", MediaType.MOVIE)).query)
|
||||
|
||||
assert query["search_field"] == ["The Movie 分类:电影DVDRip"]
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
@@ -58,6 +59,35 @@ class LocalSetupConfigDirTests(unittest.TestCase):
|
||||
self.assertIsNone(result)
|
||||
prompt_mock.assert_not_called()
|
||||
|
||||
def test_install_deps_installs_browser_runtime(self):
|
||||
module = load_local_setup_module()
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
venv_dir = (Path(temp_dir) / "venv").resolve()
|
||||
venv_python = venv_dir / "bin" / "python"
|
||||
venv_pip = venv_dir / "bin" / "pip"
|
||||
|
||||
with patch.object(module, "ensure_supported_python"), \
|
||||
patch.object(
|
||||
module,
|
||||
"configure_venv_pip_compat",
|
||||
return_value=venv_pip,
|
||||
), \
|
||||
patch.object(module, "run") as run_mock, \
|
||||
patch.object(module, "install_browser_runtime") as install_browser:
|
||||
result = module.install_deps(
|
||||
python_bin="python3",
|
||||
venv_dir=venv_dir,
|
||||
recreate=False,
|
||||
)
|
||||
|
||||
self.assertEqual(result, venv_python)
|
||||
run_mock.assert_any_call(["python3", "-m", "venv", str(venv_dir)])
|
||||
run_mock.assert_any_call(
|
||||
[str(venv_pip), "install", "-r", str(module.ROOT / "requirements.txt")]
|
||||
)
|
||||
install_browser.assert_called_once_with(venv_python)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -44,3 +44,89 @@ def test_audiences_userbar_metrics_override_generic_nexus_regex():
|
||||
assert parser.bonus == 1973896.2
|
||||
assert parser.seeding == 355
|
||||
assert parser.leeching == 7
|
||||
|
||||
|
||||
def test_audiences_inbox_total_unread_badge_uses_unread_part():
|
||||
parser = NexusAudiencesSiteUserInfo(
|
||||
site_name="Audiences",
|
||||
url="https://audiences.me/",
|
||||
site_cookie="",
|
||||
apikey=None,
|
||||
token=None,
|
||||
)
|
||||
html_text = """
|
||||
<html>
|
||||
<body>
|
||||
<div class="site-userbar__compact-actions">
|
||||
<a class="site-userbar__compact-tool site-userbar__compact-tool--has-unread"
|
||||
href="messages.php"
|
||||
title="收件箱 1749/172"
|
||||
aria-label="收件箱 1749/172">
|
||||
<i class="fas fa-inbox" aria-hidden="true"></i>
|
||||
<strong>收件箱</strong>
|
||||
<span class="site-userbar__compact-tool-badge site-userbar__compact-tool-badge--unread">1749/172</span>
|
||||
</a>
|
||||
<a class="site-userbar__compact-tool"
|
||||
href="messages.php?action=viewmailbox&box=-1"
|
||||
title="发件箱 0"
|
||||
aria-label="发件箱 0">
|
||||
<strong>发件箱</strong>
|
||||
<span class="site-userbar__compact-tool-badge">0</span>
|
||||
</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
parser._parse_message_unread(html_text)
|
||||
|
||||
assert parser.message_unread == 172
|
||||
|
||||
|
||||
def test_audiences_table_unread_links_ignore_content_rows():
|
||||
parser = NexusAudiencesSiteUserInfo(
|
||||
site_name="Audiences",
|
||||
url="https://audiences.me/",
|
||||
site_cookie="",
|
||||
apikey=None,
|
||||
token=None,
|
||||
)
|
||||
html_text = """
|
||||
<html>
|
||||
<body>
|
||||
<table>
|
||||
<tr>
|
||||
<td class="rowfollow" align="center">
|
||||
<img class="unreadpm" src="pic/trans.gif" alt="Unread" title="未读">
|
||||
</td>
|
||||
<td class="rowfollow" align="left">
|
||||
<a href="messages.php?action=viewmessage&id=4318225">种子被删除</a>
|
||||
</td>
|
||||
<td class="rowfollow" align="left">系统</td>
|
||||
<td class="rowfollow" nowrap=""><span title="2026-05-07 23:01:58">8天17时前</span></td>
|
||||
<td class="rowfollow"><input class="checkbox" type="checkbox" name="messages[]" value="4318225"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="5" style="padding: 8px;">消息摘要内容</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rowfollow" align="center">
|
||||
<img class="readpm" src="pic/trans.gif" alt="Read" title="已读">
|
||||
</td>
|
||||
<td class="rowfollow" align="left">
|
||||
<a href="messages.php?action=viewmessage&id=4318000">已读消息</a>
|
||||
</td>
|
||||
<td class="rowfollow" align="left">系统</td>
|
||||
<td class="rowfollow" nowrap=""><span title="2026-05-07 23:01:58">8天17时前</span></td>
|
||||
<td class="rowfollow"><input class="checkbox" type="checkbox" name="messages[]" value="4318000"></td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
msg_links = []
|
||||
|
||||
next_page = parser._parse_message_unread_links(html_text, msg_links)
|
||||
|
||||
assert msg_links == ["messages.php?action=viewmessage&id=4318225"]
|
||||
assert next_page is None
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from app.core.config import settings
|
||||
from app.chain.transfer import JobManager, TransferChain
|
||||
from app.schemas import FileItem, TransferInfo, TransferTask
|
||||
from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask
|
||||
from app.schemas.types import EventType, MediaType
|
||||
|
||||
|
||||
@@ -126,6 +127,49 @@ def migrate_to_media_job(jobview: JobManager, task: TransferTask):
|
||||
|
||||
|
||||
class TransferJobManagerTest(unittest.TestCase):
|
||||
def test_manual_episode_offset_applies_once(self):
|
||||
chain = make_transfer_chain()
|
||||
source_fileitem = make_fileitem("/downloads/Test.Show.2026.S01E14.mkv")
|
||||
planned_episodes = []
|
||||
|
||||
chain._TransferChain__get_trans_fileitems = lambda fileitem, predicate: [
|
||||
(source_fileitem, False)
|
||||
]
|
||||
chain._TransferChain__put_to_jobview = lambda task: True
|
||||
chain._TransferChain__register_scrape_batch_task = lambda task: None
|
||||
chain._TransferChain__close_scrape_batch = lambda batch_id: None
|
||||
|
||||
def fake_handle_transfer(task, callback=None):
|
||||
planned_episodes.append(task.meta.begin_episode)
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = fake_handle_transfer
|
||||
|
||||
transfer_history_oper = SimpleNamespace(get_by_src=lambda src, storage=None: None)
|
||||
download_history_oper = SimpleNamespace(
|
||||
get_by_hash=lambda download_hash: None,
|
||||
get_file_by_fullpath=lambda fullpath: None,
|
||||
get_files_by_savepath=lambda savepath: [],
|
||||
get_by_path=lambda path: None,
|
||||
)
|
||||
system_config_oper = SimpleNamespace(get=lambda key: None)
|
||||
|
||||
with patch("app.chain.transfer.TransferHistoryOper", return_value=transfer_history_oper), \
|
||||
patch("app.chain.transfer.DownloadHistoryOper", return_value=download_history_oper), \
|
||||
patch("app.chain.transfer.SystemConfigOper", return_value=system_config_oper), \
|
||||
patch("app.chain.transfer.MetaInfoPath", lambda *args, **kwargs: FakeMeta(14)):
|
||||
state, errmsg = chain.do_transfer(
|
||||
fileitem=source_fileitem,
|
||||
mediainfo=FakeMedia(),
|
||||
target_path=Path("/library"),
|
||||
epformat=EpisodeFormat(offset="-1"),
|
||||
background=False,
|
||||
)
|
||||
|
||||
self.assertTrue(state, errmsg)
|
||||
# 手动集数偏移只能应用一次,避免 E14 + (-1) 被二次处理成 E12。
|
||||
self.assertEqual([13], planned_episodes)
|
||||
|
||||
def test_completed_media_job_is_removed_after_last_meta_task_fails(self):
|
||||
jobview = JobManager()
|
||||
tasks = [make_task(episode) for episode in range(1, 4)]
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
APP_VERSION = 'v2.11.4'
|
||||
FRONTEND_VERSION = 'v2.11.4'
|
||||
APP_VERSION = 'v2.12.0'
|
||||
FRONTEND_VERSION = 'v2.12.0'
|
||||
|
||||
Reference in New Issue
Block a user