Compare commits

...

8 Commits

Author SHA1 Message Date
jxxghp
3653164924 chore: bump version to v2.12.0 2026-05-17 15:12:01 +08:00
jxxghp
ca0127cc87 fix: adapt site imdb search urls 2026-05-17 11:43:50 +08:00
jxxghp
092666f9d2 fix: avoid double episode offset in manual transfer 2026-05-17 08:59:17 +08:00
jxxghp
7b97e2039f fix(nginx): expand SSE configuration to include logging and search stream endpoints 2026-05-17 08:29:57 +08:00
jxxghp
e168e31a8f fix: offload subtitle download after add task 2026-05-17 08:26:13 +08:00
jxxghp
3ee601574c fix: reduce low-risk pylint issues 2026-05-17 08:01:39 +08:00
jxxghp
0ee9fec1d2 feat(browser): migrate to CloakBrowser for browser emulation and streamline dependency management
- Replace Playwright-based browser emulation with CloakBrowser as default
- Update config to support CloakBrowser options and humanization presets
- Refactor browser helper to use CloakBrowser context and remove cf_clearance dependency
- Update Dockerfile, entrypoint, and update scripts to install CloakBrowser runtime
- Ensure CloakBrowser kernel is pre-installed during local setup and dependency updates
- Add tests for CloakBrowser integration and legacy compatibility
2026-05-16 20:51:38 +08:00
jxxghp
9069dccb2a fix: parse Audiences unread messages 2026-05-16 16:41:11 +08:00
31 changed files with 845 additions and 185 deletions

View File

@@ -5,38 +5,30 @@ init-hook='import sys; sys.path.append(".")'
# 忽略的文件和目录
ignore=.git,__pycache__,.venv,build,dist,tests,docs
# 通过 `pylint app/` 检查主程序时不扫描内置插件目录,
# 插件依赖和动态模型较多,容易产生与主程序无关的误报。
ignore-paths=^app/plugins(/|$)
# 并行作业数量
jobs=0
[MESSAGES CONTROL]
# 只关注错误级别的问题,禁用警告、约定和重构建议
# E = Error (错误) - 会导致构建失败
# W = Warning (警告) - 仅显示,不会失败
# R = Refactor (重构建议) - 仅显示,不会失败
# C = Convention (约定) - 仅显示,不会失败
# I = Information (信息) - 仅显示,不会失败
# 禁用大部分警告、约定和重构建议,只保留错误和重要警告
# 只启用确定性较强的严重问题检查,避免 SQLAlchemy、FastAPI 依赖注入、
# 第三方 SDK 等动态对象被 Pylint 推断成误报。
disable=all
enable=E,
syntax-error,
enable=syntax-error,
undefined-variable,
used-before-assignment,
possibly-used-before-assignment,
unreachable,
return-outside-function,
yield-outside-function,
continue-in-finally,
nonlocal-without-binding,
undefined-loop-variable,
redefined-builtin,
not-callable,
assignment-from-no-return,
no-value-for-parameter,
too-many-function-args,
unexpected-keyword-arg,
redundant-keyword-arg,
import-error,
relative-beyond-top-level
relative-beyond-top-level,
no-name-in-module
[REPORTS]
# 设置报告格式
@@ -80,4 +72,6 @@ ignore-imports=yes
[TYPECHECK]
# 生成缺失成员提示的类列表
generated-members=requests.packages.urllib3
generated-members=requests.packages.urllib3
# app.helper.sites 会主动隐藏模块属性枚举,避免误报 no-name-in-module
ignored-modules=app.helper.sites

View File

@@ -157,7 +157,7 @@ def _parse_skill_metadata( # noqa: C901
MAX_SKILL_COMPATIBILITY_LENGTH,
skill_path,
)
compatibility_str = compatibility_str[:MAX_SKILL_COMPATIBILITY_LENGTH]
compatibility_str = str(compatibility_str)[:MAX_SKILL_COMPATIBILITY_LENGTH]
# 版本号,默认为 0表示未设置版本
raw_version = frontmatter_data.get("version")

View File

@@ -236,7 +236,8 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
Returns:
str: 友好的提示消息,如果返回 None 或空字符串则使用 explanation
"""
return None
explanation = kwargs.get("explanation")
return str(explanation) if explanation else None
@abstractmethod
async def run(self, **kwargs) -> str:

View File

@@ -26,9 +26,11 @@ class UserChoiceOptionInput(BaseModel):
@model_validator(mode="after")
def validate_option(self):
if not self.label.strip():
label = str(self.label)
value = str(self.value)
if not label.strip():
raise ValueError("label 不能为空")
if not self.value.strip():
if not value.strip():
raise ValueError("value 不能为空")
return self
@@ -55,7 +57,8 @@ class AskUserChoiceInput(BaseModel):
@model_validator(mode="after")
def validate_payload(self):
if not self.message.strip():
message = str(self.message)
if not message.strip():
raise ValueError("message 不能为空")
if not self.options:
raise ValueError("options 至少需要提供一个")

View File

@@ -198,68 +198,62 @@ class BrowseWebpageTool(MoviePilotTool):
cookies: Optional[str],
user_agent: Optional[str],
) -> str:
"""在同步上下文中执行 Playwright 浏览器操作"""
from playwright.sync_api import sync_playwright
"""在同步上下文中执行 CloakBrowser 浏览器操作"""
from cloakbrowser import launch_context
try:
with sync_playwright() as playwright:
browser = None
context = None
page = None
try:
# 启动浏览器
browser_type = settings.PLAYWRIGHT_BROWSER_TYPE or "chromium"
browser = playwright[browser_type].launch(headless=True)
# 创建上下文
context_kwargs = {}
if user_agent:
context_kwargs["user_agent"] = user_agent
# 设置视口大小
context_kwargs["viewport"] = {
context = None
page = None
try:
context_kwargs = {
"viewport": {
"width": SCREENSHOT_MAX_WIDTH,
"height": SCREENSHOT_MAX_HEIGHT,
}
}
if user_agent:
context_kwargs["user_agent"] = user_agent
context = browser.new_context(**context_kwargs)
page = context.new_page()
page.set_default_timeout(timeout * 1000)
context = launch_context(
headless=True,
humanize=settings.CLOAKBROWSER_HUMANIZE,
human_preset=settings.CLOAKBROWSER_HUMAN_PRESET,
**context_kwargs,
)
page = context.new_page()
page.set_default_timeout(timeout * 1000)
# 设置 cookies
if cookies:
page.set_extra_http_headers({"cookie": cookies})
# 设置 cookies
if cookies:
page.set_extra_http_headers({"cookie": cookies})
# 对于非 goto 操作,如果提供了 url 先导航
if url and browser_action != BrowserAction.GOTO:
page.goto(
url, wait_until="domcontentloaded", timeout=timeout * 1000
)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
# 对于非 goto 操作,如果提供了 url 先导航
if url and browser_action != BrowserAction.GOTO:
page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
# 执行具体操作
result = self._do_action(
page,
browser_action,
url,
selector,
value,
script,
content_type,
timeout,
)
return result
# 执行具体操作
result = self._do_action(
page,
browser_action,
url,
selector,
value,
script,
content_type,
timeout,
)
return result
finally:
if page:
page.close()
if context:
context.close()
if browser:
browser.close()
finally:
if page:
page.close()
if context:
context.close()
except Exception as e:
logger.error(f"Playwright 执行失败: {e}", exc_info=True)
return f"Playwright 执行失败: {str(e)}"
logger.error(f"CloakBrowser 执行失败: {e}", exc_info=True)
return f"CloakBrowser 执行失败: {str(e)}"
def _do_action(
self,

View File

@@ -6,7 +6,7 @@ import signal
import subprocess
from dataclasses import dataclass, field
from tempfile import NamedTemporaryFile
from typing import Optional, TextIO, Type
from typing import Any, Optional, TextIO, Type
from pydantic import BaseModel, Field
@@ -188,7 +188,7 @@ class ExecuteCommandTool(MoviePilotTool):
output.append(stream_name, chunk.decode("utf-8", errors="replace"))
@staticmethod
def _terminate_process(process: asyncio.subprocess.Process, sig: int):
def _terminate_process(process: Any, sig: int):
"""向进程组发送终止信号;不支持进程组的平台回退为单进程终止。"""
try:
if os.name == "posix":
@@ -203,7 +203,7 @@ class ExecuteCommandTool(MoviePilotTool):
@classmethod
async def _cleanup_process(
cls,
process: asyncio.subprocess.Process,
process: Any,
wait_task: asyncio.Task,
) -> None:
"""先温和终止,失败后强杀,避免超时 shell 遗留子进程。"""

View File

@@ -159,7 +159,7 @@ class ChainBase(metaclass=ABCMeta):
处理插件模块执行错误
"""
if kwargs.get("raise_exception"):
raise
raise err
logger.error(
f"运行插件 {plugin_id} 模块 {method} 出错:{str(err)}\n{traceback.format_exc()}"
)
@@ -185,7 +185,7 @@ class ChainBase(metaclass=ABCMeta):
处理系统模块执行错误
"""
if kwargs.get("raise_exception"):
raise
raise err
logger.error(
f"运行模块 {module_id}.{method} 出错:{str(err)}\n{traceback.format_exc()}"
)

View File

@@ -17,6 +17,7 @@ from app.core.metainfo import MetaInfo
from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.mediaserver_oper import MediaServerOper
from app.helper.directory import DirectoryHelper
from app.helper.thread import ThreadHelper
from app.helper.torrent import TorrentHelper
from app.log import logger
from app.schemas import ExistMediaInfo, FileURI, NotExistMediaInfo, DownloadingTorrent, Notification, ResourceSelectionEventData, \
@@ -32,6 +33,31 @@ class DownloadChain(ChainBase):
下载处理链
"""
def _submit_download_added_task(
self,
context: Context,
download_dir: Path,
torrent_content: Union[str, bytes],
) -> None:
"""
后台执行下载成功后的附加处理,避免站点字幕下载阻塞添加下载响应。
"""
def _run_download_added() -> None:
try:
self.download_added(
context=context,
download_dir=download_dir,
torrent_content=torrent_content,
)
except Exception as err:
logger.error(f"执行下载成功后处理失败:{str(err)}")
try:
ThreadHelper().submit(_run_download_added)
except Exception as err:
logger.error(f"提交下载成功后处理后台任务失败:{str(err)}")
def download_torrent(self, torrent: TorrentInfo,
channel: MessageChannel = None,
source: Optional[str] = None,
@@ -371,7 +397,11 @@ class DownloadChain(ChainBase):
username=username,
)
# 下载成功后处理
self.download_added(context=context, download_dir=download_dir, torrent_content=torrent_content)
self._submit_download_added_task(
context=context,
download_dir=download_dir,
torrent_content=torrent_content,
)
# 广播事件
self.eventmanager.send_event(EventType.DownloadAdded, {
"hash": _hash,

View File

@@ -788,7 +788,7 @@ class SkillsChain(ChainBase):
if skill.source_type == "registry":
text_lines.append("社区源,安装前请自行甄别安全性")
if any(skill.source_type == "registry" for skill in page_items):
if any(skill.source_type == "registry" for skill in items):
text_lines.extend(
[
"",

View File

@@ -2245,6 +2245,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
)
if not built_meta:
return None
if not meta:
# _build_path_meta 已经应用过手动季集/自定义格式覆盖;
# 这里避免再次偏移集数,导致手动整理的集数偏移翻倍。
return built_meta
return _apply_meta_overrides(built_meta, source_path)
def _build_path_meta(

View File

@@ -998,7 +998,8 @@ def config_list(show_secrets: bool) -> None:
@click.argument("key")
def config_get(key: str) -> None:
"""读取单个配置项"""
if key not in Settings.model_fields and not hasattr(settings, key):
setting_fields = Settings.model_fields.keys()
if key not in setting_fields and not hasattr(settings, key):
raise click.ClickException(f"配置项不存在:{key}")
click.echo(_format_value(getattr(settings, key)))

View File

@@ -331,8 +331,12 @@ class ConfigModel(BaseModel):
NO_CACHE_SITE_KEY: str = "m-team"
# OCR服务器地址用于识别站点验证码
OCR_HOST: str = "https://movie-pilot.org"
# 仿真类型:playwright 或 flaresolverr
BROWSER_EMULATION: str = "playwright"
# 仿真类型:cloakbrowser 或 flaresolverr,其他值按 cloakbrowser 处理
BROWSER_EMULATION: str = "cloakbrowser"
# CloakBrowser 是否启用拟人化输入
CLOAKBROWSER_HUMANIZE: bool = True
# CloakBrowser 拟人化输入预设default 或 careful
CLOAKBROWSER_HUMAN_PRESET: str = "default"
# FlareSolverr 服务地址,例如 http://127.0.0.1:8191
FLARESOLVERR_URL: Optional[str] = None
@@ -526,7 +530,7 @@ class ConfigModel(BaseModel):
# ==================== Docker配置 ====================
# Docker Client API地址
DOCKER_CLIENT_API: Optional[str] = "tcp://127.0.0.1:38379"
# Playwright浏览器类型chromium/firefox
# Playwright浏览器类型供智能体浏览器工具和插件直接使用 Playwright 时读取
PLAYWRIGHT_BROWSER_TYPE: str = "chromium"
# ==================== AI智能体配置 ====================

View File

@@ -1,4 +1,5 @@
from datetime import datetime
from builtins import list as builtin_list
from typing import Optional
from sqlalchemy import Column, Integer, JSON, String, Index, and_, or_, select
@@ -34,9 +35,9 @@ class Workflow(Base):
# 已执行次数
run_count = Column(Integer, default=0)
# 任务列表
actions = Column(JSON, default=list)
actions = Column(JSON, default=builtin_list)
# 任务流
flows = Column(JSON, default=list)
flows = Column(JSON, default=builtin_list)
# 执行上下文
context = Column(JSON, default=dict)
# 创建时间

View File

@@ -1,8 +1,7 @@
import uuid
from typing import Callable, Any, Optional
from cf_clearance import sync_cf_retry, sync_stealth
from playwright.sync_api import sync_playwright, Page
from playwright.sync_api import BrowserContext, Page
from app.core.config import settings
from app.log import logger
@@ -10,17 +9,33 @@ from app.utils.http import RequestUtils, cookie_parse
class PlaywrightHelper:
def __init__(self, browser_type=settings.PLAYWRIGHT_BROWSER_TYPE):
self.browser_type = browser_type
def __init__(self, browser_type: Optional[str] = None, *args, **kwargs):
"""
兼容旧的 PlaywrightHelper(browser_type=...) 构造方式。
"""
self.browser_type = browser_type or settings.PLAYWRIGHT_BROWSER_TYPE
@staticmethod
def __pass_cloudflare(url: str, page: Page) -> bool:
def __browser_emulation() -> str:
"""
尝试跳过cloudfare验证
当前浏览器仿真类型。
"""
sync_stealth(page, pure=True)
page.goto(url)
return sync_cf_retry(page)[0]
return (settings.BROWSER_EMULATION or "cloakbrowser").lower()
@staticmethod
def __launch_cloakbrowser_context(headless: bool,
user_agent: Optional[str] = None,
proxies: Optional[dict] = None) -> BrowserContext:
"""
启动 CloakBrowser 上下文。
"""
from cloakbrowser import launch_context
return launch_context(headless=headless,
proxy=proxies,
user_agent=user_agent,
humanize=settings.CLOAKBROWSER_HUMANIZE,
human_preset=settings.CLOAKBROWSER_HUMAN_PRESET)
@staticmethod
def __fs_cookie_str(cookies: list) -> str:
@@ -148,51 +163,44 @@ class PlaywrightHelper:
"""
result = None
try:
with sync_playwright() as playwright:
browser = None
context = None
page = None
try:
# 如果配置使用 FlareSolverr先通过其获取清除后的 cookies 与 UA
fs_cookie_header = None
fs_ua = None
if settings.BROWSER_EMULATION == "flaresolverr":
solution = self.__flaresolverr_request(url=url, cookies=cookies,
proxy_config=proxies, timeout=timeout)
if solution:
fs_cookie_header = self.__fs_cookie_str(solution.get("cookies", []))
fs_ua = solution.get("userAgent")
context = None
page = None
try:
# 如果配置使用 FlareSolverr先通过其获取清除后的 cookies 与 UA
fs_cookie_header = None
fs_ua = None
if self.__browser_emulation() == "flaresolverr":
solution = self.__flaresolverr_request(url=url, cookies=cookies,
proxy_config=proxies, timeout=timeout)
if solution:
fs_cookie_header = self.__fs_cookie_str(solution.get("cookies", []))
fs_ua = solution.get("userAgent")
browser = playwright[self.browser_type].launch(headless=headless)
context = browser.new_context(user_agent=fs_ua or ua, proxy=proxies)
page = context.new_page()
context = self.__launch_cloakbrowser_context(headless=headless,
user_agent=fs_ua or ua,
proxies=proxies)
page = context.new_page()
# 优先使用 FlareSolverr 返回,其次使用入参
merged_cookie = fs_cookie_header or cookies
if merged_cookie:
page.set_extra_http_headers({"cookie": merged_cookie})
# 优先使用 FlareSolverr 返回,其次使用入参
merged_cookie = fs_cookie_header or cookies
if merged_cookie:
page.set_extra_http_headers({"cookie": merged_cookie})
if settings.BROWSER_EMULATION == "playwright":
if not self.__pass_cloudflare(url, page):
logger.warn("cloudflare challenge fail")
else:
page.goto(url)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
page.goto(url)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
# 回调函数
result = callback(page)
# 回调函数
result = callback(page)
except Exception as e:
logger.error(f"网页操作失败: {str(e)}")
finally:
if page:
page.close()
if context:
context.close()
if browser:
browser.close()
except Exception as e:
logger.error(f"网页操作失败: {str(e)}")
finally:
if page:
page.close()
if context:
context.close()
except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}")
logger.error(f"CloakBrowser初始化失败: {str(e)}")
return result
@@ -213,7 +221,7 @@ class PlaywrightHelper:
"""
source = None
# 如果配置为 FlareSolverr则直接调用获取页面源码
if settings.BROWSER_EMULATION == "flaresolverr":
if self.__browser_emulation() == "flaresolverr":
try:
solution = self.__flaresolverr_request(url=url, cookies=cookies,
proxy_config=proxies, timeout=timeout)
@@ -222,36 +230,32 @@ class PlaywrightHelper:
except Exception as e:
logger.error(f"FlareSolverr 获取源码失败: {str(e)}")
try:
with sync_playwright() as playwright:
browser = None
context = None
page = None
try:
browser = playwright[self.browser_type].launch(headless=headless)
context = browser.new_context(user_agent=ua, proxy=proxies)
page = context.new_page()
context = None
page = None
try:
context = self.__launch_cloakbrowser_context(headless=headless,
user_agent=ua,
proxies=proxies)
page = context.new_page()
if cookies:
page.set_extra_http_headers({"cookie": cookies})
if cookies:
page.set_extra_http_headers({"cookie": cookies})
if not self.__pass_cloudflare(url, page):
logger.warn("cloudflare challenge fail")
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
page.goto(url)
page.wait_for_load_state("networkidle", timeout=timeout * 1000)
source = page.content()
source = page.content()
except Exception as e:
logger.error(f"获取网页源码失败: {str(e)}")
source = None
finally:
# 确保资源被正确清理
if page:
page.close()
if context:
context.close()
if browser:
browser.close()
except Exception as e:
logger.error(f"获取网页源码失败: {str(e)}")
source = None
finally:
# 确保资源被正确清理
if page:
page.close()
if context:
context.close()
except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}")
logger.error(f"CloakBrowser初始化失败: {str(e)}")
return source

View File

@@ -94,7 +94,8 @@ class SkillHelper(metaclass=WeakSingleton):
"""
返回系统默认的技能市场列表,用于区分内置源和用户追加源。
"""
default_value = type(settings).model_fields["SKILL_MARKET"].default
skill_market_field = type(settings).model_fields.get("SKILL_MARKET")
default_value = skill_market_field.default if skill_market_field else None
if not default_value:
return []
return [item.strip() for item in str(default_value).split(",") if item.strip()]

View File

@@ -13,6 +13,35 @@ from app.utils.string import StringUtils
class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
schema = SiteSchema.NexusAudiences
def _parse_message_unread(self, html_text):
"""
解析 Audiences 新版顶部用户栏中的未读消息数。
"""
super()._parse_message_unread(html_text)
if self.message_unread:
return
html = etree.HTML(html_text)
try:
if not StringUtils.is_valid_html_element(html):
return
message_tools = html.xpath(
'//a[contains(@class, "site-userbar__compact-tool") and contains(@href, "messages.php") '
'and (contains(@class, "site-userbar__compact-tool--has-unread") '
'or .//*[contains(@class, "site-userbar__compact-tool-badge--unread")])]'
'|//a[contains(@href, "messages.php") '
'and (contains(@title, "收件箱") or contains(@aria-label, "收件箱"))]'
)
for message_link in message_tools:
unread = self.__parse_inbox_unread(message_link)
if unread is not None:
self.message_unread = unread
return
finally:
if html is not None:
del html
def _parse_user_traffic_info(self, html_text):
"""
解析用户流量信息
@@ -128,6 +157,47 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
self.seeding = StringUtils.str_int(active_match.group(1))
self.leeching = StringUtils.str_int(active_match.group(2))
def __parse_inbox_unread(self, message_link):
"""
从 Audiences 收件箱入口提取未读数。
"""
inbox_texts = [
message_link.get("title"),
message_link.get("aria-label"),
*message_link.xpath(
'.//*[contains(@class, "site-userbar__compact-tool-badge--unread") '
'or contains(@class, "site-userbar__compact-tool-badge")]/text()'
)
]
for inbox_text in inbox_texts:
unread = self.__extract_inbox_unread(inbox_text)
if unread is not None:
return unread
return None
@staticmethod
def __extract_inbox_unread(text: str):
"""
Audiences 收件箱角标格式为 总数/未读数,例如 1749/172。
"""
if not text:
return None
text = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
if not text:
return None
inbox_count = re.search(r"(?:收件箱\s*)?(\d[\d,]*)\s*/\s*(\d[\d,]*)", text)
if inbox_count:
return StringUtils.str_int(inbox_count.group(2))
single_count = re.search(r"收件箱\s*(\d[\d,]*)", text)
if single_count:
return StringUtils.str_int(single_count.group(1))
return None
def _parse_seeding_pages(self):
if not self._torrent_seeding_page:
return

View File

@@ -3,7 +3,7 @@ import re
import traceback
from typing import Any, Optional
from typing import List
from urllib.parse import quote, urlencode, urlparse, parse_qs
from urllib.parse import quote, urlparse, parse_qs
from fastapi.concurrency import run_in_threadpool
from jinja2 import Template
@@ -14,6 +14,7 @@ from app.log import logger
from app.schemas.types import MediaType
from app.utils.http import RequestUtils, AsyncRequestUtils
from app.utils.string import StringUtils
from app.utils.url import UrlUtils
class SiteSpider:
@@ -120,14 +121,15 @@ class SiteSpider:
search_word = self.keyword
# 查询模式与
search_mode = "0"
is_imdbid_search = isinstance(self.keyword, str) and re.fullmatch(r"tt\d+", self.keyword)
search_word = self.__format_search_word(search_word)
# 搜索URL
indexer_params = self.search.get("params", {}).copy()
if indexer_params:
search_area = indexer_params.get('search_area')
# search_area非0表示支持imdbid搜索
if (search_area and
(not self.keyword or not self.keyword.startswith('tt'))):
if search_area and not is_imdbid_search:
# 支持imdbid搜索但关键字不是imdbid时不启用imdbid搜索
indexer_params.pop('search_area')
# 变量字典
@@ -168,7 +170,7 @@ class SiteSpider:
params.update({
"cat%s" % cat.get("id"): 1
})
searchurl = self.domain + torrentspath + "?" + urlencode(params)
searchurl = UrlUtils.combine_url(self.domain, torrentspath, params)
else:
# 变量字典
inputs_dict = {
@@ -200,6 +202,22 @@ class SiteSpider:
return searchurl
def __format_search_word(self, search_word: str) -> str:
"""
按站点配置转换搜索关键字,用于兼容站点特殊的 IMDb ID 查询格式。
"""
if not search_word or not isinstance(search_word, str):
return search_word
if re.fullmatch(r"tt\d+", search_word):
imdbid_format = self.search.get("imdbid_format")
if imdbid_format:
return str(imdbid_format).format(
keyword=search_word,
imdbid=search_word,
imdbid_num=search_word[2:]
)
return search_word
def get_torrents(self) -> List[dict]:
"""
开始请求

View File

@@ -583,6 +583,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
轮询监控(改进版)
"""
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
with snapshot_lock:
try:
# 加载上次快照数据
@@ -650,10 +651,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
trigger='interval',
minutes=new_interval
)
logger.info(f"{storage}:{mon_path} 监控间隔已调整为 {new_interval} 分钟")
logger.info(f"{storage}:{monitor_scope} 监控间隔已调整为 {new_interval} 分钟")
except Exception as e:
logger.error(f"轮询监控 {storage}:{mon_path} 出现错误:{e}")
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}")
logger.debug(traceback.format_exc())
def event_handler(self, event, text: str, event_path: str, file_size: float = None):

View File

@@ -109,9 +109,8 @@ COPY --from=prepare_venv --chmod=777 ${VENV_PATH} ${VENV_PATH}
COPY --from=prepare_venv /usr/local/bin/uv /usr/local/bin/uv
COPY --from=prepare_venv /usr/local/bin/uv-pip-compat /usr/local/bin/uv-pip-compat
# playwright 环境
# 浏览器运行依赖
RUN playwright install-deps chromium \
&& playwright install-deps firefox \
&& apt-get autoremove -y \
&& apt-get clean \
&& rm -rf \

View File

@@ -46,6 +46,7 @@ function load_config_from_app_env() {
["PROXY_HOST"]=""
["GITHUB_TOKEN"]=""
["MOVIEPILOT_AUTO_UPDATE"]="release"
["BROWSER_EMULATION"]="cloakbrowser"
# database
["DB_TYPE"]="sqlite"
@@ -220,7 +221,7 @@ function graceful_exit() {
# 插件依赖和主程序共用同一套 venv 时,历史安装记录可能已经污染环境,
# 这里优先在真正拉起后端前做一次自愈,避免容器反复起不来。
function ensure_backend_runtime_dependencies() {
local probe_code="import alembic, fastapi, pydantic, pydantic_core, pydantic_settings, sqlalchemy, starlette, uvicorn; from pydantic import BaseModel, Field"
local probe_code="import alembic, cloakbrowser, fastapi, pydantic, pydantic_core, pydantic_settings, sqlalchemy, starlette, uvicorn; from pydantic import BaseModel, Field"
INFO "→ 启动前检查后端核心依赖..."
if "${VENV_PATH}/bin/python3" -c "${probe_code}" >/dev/null 2>&1; then
@@ -327,12 +328,28 @@ chown -R moviepilot:moviepilot \
/var/log/nginx
chown moviepilot:moviepilot /etc/hosts /tmp
# 启动前优先确认主运行环境仍然健康,避免插件依赖污染导致服务直接起不来。
ensure_backend_runtime_dependencies
# 下载浏览器内核
if [[ "$HTTPS_PROXY" =~ ^https?:// ]] || [[ "$HTTPS_PROXY" =~ ^https?:// ]] || [[ "$PROXY_HOST" =~ ^https?:// ]]; then
HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$PROXY_HOST}}" gosu moviepilot:moviepilot playwright install ${PLAYWRIGHT_BROWSER_TYPE:-chromium}
else
gosu moviepilot:moviepilot playwright install ${PLAYWRIGHT_BROWSER_TYPE:-chromium}
fi
function install_browser_kernel() {
local emulation="${BROWSER_EMULATION:-cloakbrowser}"
emulation="${emulation,,}"
local proxy="${HTTPS_PROXY:-${https_proxy:-$PROXY_HOST}}"
if [ "${emulation}" != "cloakbrowser" ] && [ "${emulation}" != "flaresolverr" ] && [ -n "${emulation}" ]; then
WARN "浏览器仿真类型 ${emulation} 已按 CloakBrowser 处理。"
fi
INFO "下载 CloakBrowser 浏览器内核"
if [[ "$proxy" =~ ^https?:// ]]; then
HTTPS_PROXY="$proxy" gosu moviepilot:moviepilot python -m cloakbrowser install
else
gosu moviepilot:moviepilot python -m cloakbrowser install
fi
}
install_browser_kernel
# 证书管理
source /app/docker/cert.sh
@@ -358,9 +375,6 @@ fi
# 设置后端服务权限掩码
umask "${UMASK}"
# 启动前优先确认主运行环境仍然健康,避免插件依赖污染导致服务直接起不来。
ensure_backend_runtime_dependencies
# 清除非系统环境导入的变量,保证转移到 dumb-init 的时候,不会带入不必要的环境变量
INFO "准备为 Python 应用清理的非系统环境导入的变量..."
if [ ${#VARS_SET_BY_SCRIPT[@]} -gt 0 ]; then

View File

@@ -29,7 +29,7 @@ location /cookiecloud {
}
# SSE特殊配置
location ~ ^/api/v1/system/(message|progress/) {
location ~ ^/api/v1/(system/(message|progress/|logging)|search/.*/stream$) {
# SSE MIME类型设置
default_type text/event-stream;

View File

@@ -80,6 +80,10 @@ function install_backend_and_download_resources() {
cp /tmp/requirements.txt.backup /app/requirements.txt
return 1
fi
INFO "正在更新 CloakBrowser 浏览器内核"
if ! ${VENV_PATH}/bin/python -m cloakbrowser install; then
WARN "CloakBrowser 浏览器内核更新失败,后续首次使用时可能重新下载"
fi
INFO "依赖更新成功"
else
INFO "依赖无变化,跳过依赖更新"

View File

@@ -38,8 +38,8 @@ pillow~=12.1.1
pillow-avif-plugin~=1.5.2
pyTelegramBotAPI~=4.27.0
telegramify-markdown~=0.5.2
cloakbrowser~=0.3.28
playwright~=1.53.0
cf_clearance~=0.31.0
torrentool~=1.2.0
slack-bolt~=1.23.0
slack-sdk~=3.35.0

View File

@@ -2653,9 +2653,18 @@ def install_deps(*, python_bin: str, venv_dir: Path, recreate: bool) -> Path:
print_step("安装项目依赖")
run([str(venv_pip), "install", "-r", str(ROOT / "requirements.txt")])
install_browser_runtime(venv_python)
return venv_python
def install_browser_runtime(venv_python: Path) -> None:
"""
预下载 CloakBrowser 浏览器内核,避免首次仿真登录时才拉取大文件。
"""
print_step("安装 CloakBrowser 浏览器内核")
run([str(venv_python), "-m", "cloakbrowser", "install"])
def _startup_platform_name() -> str:
system = platform.system()
if system == "Darwin":

View File

@@ -0,0 +1,95 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.helper.browser import PlaywrightHelper
class _FakePage:
def __init__(self) -> None:
self.headers = None
self.loaded_url = None
self.closed = False
def set_extra_http_headers(self, headers: dict[str, str]) -> None:
self.headers = headers
def goto(self, url: str) -> None:
self.loaded_url = url
def wait_for_load_state(self, _state: str, timeout: int) -> None:
self.timeout = timeout
def content(self) -> str:
return "<html>ok</html>"
def close(self) -> None:
self.closed = True
class _FakeContext:
def __init__(self, page: _FakePage) -> None:
self.page = page
self.closed = False
def new_page(self) -> _FakePage:
return self.page
def close(self) -> None:
self.closed = True
class BrowserHelperTests(unittest.TestCase):
def _assert_get_page_source_uses_cloakbrowser(self, emulation: str) -> None:
page = _FakePage()
context = _FakeContext(page)
with patch("app.helper.browser.settings.BROWSER_EMULATION", emulation), \
patch.object(
PlaywrightHelper,
"_PlaywrightHelper__launch_cloakbrowser_context",
return_value=context,
) as launch_context:
source = PlaywrightHelper().get_page_source(
url="https://example.com",
cookies="uid=1",
ua="UA",
timeout=3,
)
self.assertEqual(source, "<html>ok</html>")
launch_context.assert_called_once_with(
headless=False,
user_agent="UA",
proxies=None,
)
self.assertEqual(page.headers, {"cookie": "uid=1"})
self.assertEqual(page.loaded_url, "https://example.com")
self.assertTrue(page.closed)
self.assertTrue(context.closed)
def test_default_emulation_uses_cloakbrowser_context(self):
self._assert_get_page_source_uses_cloakbrowser("cloakbrowser")
def test_legacy_playwright_emulation_uses_cloakbrowser_context(self):
self._assert_get_page_source_uses_cloakbrowser("Playwright")
def test_legacy_browser_type_constructor_is_accepted(self):
page = _FakePage()
context = _FakeContext(page)
with patch.object(
PlaywrightHelper,
"_PlaywrightHelper__launch_cloakbrowser_context",
return_value=context,
):
source = PlaywrightHelper(browser_type="firefox").get_page_source(
url="https://example.com"
)
self.assertEqual(source, "<html>ok</html>")
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,97 @@
from pathlib import Path
from unittest.mock import MagicMock
import app.chain.download as download_module
from app.chain.download import DownloadChain
from app.core.context import Context, MediaInfo, TorrentInfo
from app.core.metainfo import MetaInfo
from app.schemas.types import MediaType
class _FakeDownloadHistoryOper:
"""
避免单元测试写入真实下载历史,只验证下载链路的控制流。
"""
def add(self, **_kwargs):
pass
def add_files(self, _files):
pass
class _FakeTorrentHelper:
"""
避免解析真实种子内容,让测试聚焦下载成功后的后台处理。
"""
def get_fileinfo_from_torrent_content(self, _torrent_content):
return "", []
class _FakeThreadHelper:
"""
捕获提交到线程池的任务,测试中手动触发以避免真正启动后台线程。
"""
submitted = []
def submit(self, func, *args, **kwargs):
self.submitted.append((func, args, kwargs))
def test_download_single_submits_download_added_to_background(monkeypatch):
"""
添加下载成功后,站点字幕等后处理应提交到后台,不能阻塞下载接口返回。
"""
_FakeThreadHelper.submitted = []
monkeypatch.setattr(download_module, "ThreadHelper", _FakeThreadHelper)
monkeypatch.setattr(download_module, "DownloadHistoryOper", _FakeDownloadHistoryOper)
monkeypatch.setattr(download_module, "TorrentHelper", _FakeTorrentHelper)
chain = DownloadChain.__new__(DownloadChain)
chain.download = MagicMock(return_value=("qb", "hash123", "Original", "添加下载成功"))
chain.download_added = MagicMock()
chain.eventmanager = MagicMock()
chain.eventmanager.send_event.return_value = None
chain.post_message = MagicMock()
context = Context(
meta_info=MetaInfo("Demo Movie 2024"),
media_info=MediaInfo(
type=MediaType.MOVIE,
title="Demo Movie",
year="2024",
tmdb_id=1,
genre_ids=[18],
),
torrent_info=TorrentInfo(
title="Demo Movie 2024",
enclosure="https://example.com/demo.torrent",
site_cookie="uid=1",
site_name="TestSite",
),
)
result = chain.download_single(
context=context,
torrent_content=b"torrent-content",
save_path="/downloads",
username="tester",
)
assert result == "hash123"
chain.download_added.assert_not_called()
assert len(_FakeThreadHelper.submitted) == 1
task, args, kwargs = _FakeThreadHelper.submitted[0]
assert args == ()
assert kwargs == {}
task()
chain.download_added.assert_called_once_with(
context=context,
download_dir=Path("/downloads"),
torrent_content=b"torrent-content",
)

View File

@@ -0,0 +1,156 @@
from urllib.parse import parse_qs, urlparse
from app.modules.indexer.spider import SiteSpider
from app.schemas.types import MediaType
def _build_indexer(**kwargs):
"""
构造 SiteSpider 生成搜索 URL 所需的最小站点配置。
"""
indexer = {
"id": "test",
"name": "测试站点",
"domain": "https://example.com/",
"search": {
"paths": [{"path": "torrents.php"}],
"params": {"search": "{keyword}"},
},
"torrents": {"list": {}, "fields": {}},
}
indexer.update(kwargs)
return indexer
def _get_search_url(indexer: dict, keyword: str | list[str], mtype: MediaType = None) -> str:
"""
调用 SiteSpider 私有 URL 构造逻辑,避免真实请求站点。
"""
spider = SiteSpider(indexer=indexer, keyword=keyword, mtype=mtype)
return spider._SiteSpider__get_search_url()
def test_eastgame_imdb_search_uses_imdb_area():
"""
TLF 支持 IMDb ID 搜索时应使用站点配置的 IMDb 搜索区域。
"""
indexer = _build_indexer(
id="eastgame",
domain="https://pt.eastgame.org/",
search={
"paths": [{"path": "torrents.php"}],
"params": {
"search_area": 4,
"search": "{keyword}",
},
},
)
parsed_url = urlparse(_get_search_url(indexer, "tt16311594"))
query = parse_qs(parsed_url.query)
assert parsed_url.geturl().startswith("https://pt.eastgame.org/torrents.php?")
assert query["search"] == ["tt16311594"]
assert query["search_area"] == ["4"]
def test_eastgame_title_search_keeps_title_area():
"""
TLF 普通标题搜索不应误用 IMDb 搜索区域。
"""
indexer = _build_indexer(
id="eastgame",
domain="https://pt.eastgame.org/",
search={
"paths": [{"path": "torrents.php"}],
"params": {
"search_area": 4,
"search": "{keyword}",
},
},
)
query = parse_qs(urlparse(_get_search_url(indexer, "普通标题")).query)
assert query["search"] == ["普通标题"]
assert query["search_area"] == ["0"]
def test_eastgame_batch_search_keeps_title_area():
"""
TLF 批量搜索不是单个 IMDb ID不能触发 IMDb 搜索区域。
"""
indexer = _build_indexer(
id="eastgame",
domain="https://pt.eastgame.org/",
search={
"paths": [{"path": "torrents.php"}],
"params": {
"search_area": 4,
"search": "{keyword}",
},
},
)
query = parse_qs(urlparse(_get_search_url(indexer, ["tt1234567", "tt7654321"])).query)
assert query["search"] == ["tt1234567 tt7654321"]
assert query["search_mode"] == ["1"]
assert query["search_area"] == ["0"]
def test_ttg_imdb_search_formats_keyword_and_keeps_existing_query():
"""
TTG 的 IMDb 搜索需要 tt 前缀转换,并且路径自带查询参数不能生成双问号。
"""
indexer = _build_indexer(
id="ttg",
domain="https://totheglory.im/",
search={
"paths": [{"path": "browse.php?c=M"}],
"params": {
"search_field": "{keyword}",
"c": "M",
},
"imdbid_format": "imdb{imdbid_num}",
},
category={
"field": "search_field",
"delimiter": " 分类:",
"movie": [{"id": "电影DVDRip", "cat": "Movies/SD"}],
},
)
search_url = _get_search_url(indexer, "tt0049406", MediaType.MOVIE)
query = parse_qs(urlparse(search_url).query)
assert search_url.count("?") == 1
assert query["c"] == ["M"]
assert query["search_field"] == ["imdb0049406 分类:电影DVDRip"]
def test_ttg_title_search_does_not_format_keyword():
"""
TTG 普通标题搜索不能被 IMDb ID 格式化规则影响。
"""
indexer = _build_indexer(
id="ttg",
domain="https://totheglory.im/",
search={
"paths": [{"path": "browse.php?c=M"}],
"params": {
"search_field": "{keyword}",
"c": "M",
},
"imdbid_format": "imdb{imdbid_num}",
},
category={
"field": "search_field",
"delimiter": " 分类:",
"movie": [{"id": "电影DVDRip", "cat": "Movies/SD"}],
},
)
query = parse_qs(urlparse(_get_search_url(indexer, "The Movie", MediaType.MOVIE)).query)
assert query["search_field"] == ["The Movie 分类:电影DVDRip"]

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import importlib.util
import tempfile
import unittest
import uuid
from pathlib import Path
@@ -58,6 +59,35 @@ class LocalSetupConfigDirTests(unittest.TestCase):
self.assertIsNone(result)
prompt_mock.assert_not_called()
def test_install_deps_installs_browser_runtime(self):
module = load_local_setup_module()
with tempfile.TemporaryDirectory() as temp_dir:
venv_dir = (Path(temp_dir) / "venv").resolve()
venv_python = venv_dir / "bin" / "python"
venv_pip = venv_dir / "bin" / "pip"
with patch.object(module, "ensure_supported_python"), \
patch.object(
module,
"configure_venv_pip_compat",
return_value=venv_pip,
), \
patch.object(module, "run") as run_mock, \
patch.object(module, "install_browser_runtime") as install_browser:
result = module.install_deps(
python_bin="python3",
venv_dir=venv_dir,
recreate=False,
)
self.assertEqual(result, venv_python)
run_mock.assert_any_call(["python3", "-m", "venv", str(venv_dir)])
run_mock.assert_any_call(
[str(venv_pip), "install", "-r", str(module.ROOT / "requirements.txt")]
)
install_browser.assert_called_once_with(venv_python)
if __name__ == "__main__":
unittest.main()

View File

@@ -44,3 +44,89 @@ def test_audiences_userbar_metrics_override_generic_nexus_regex():
assert parser.bonus == 1973896.2
assert parser.seeding == 355
assert parser.leeching == 7
def test_audiences_inbox_total_unread_badge_uses_unread_part():
parser = NexusAudiencesSiteUserInfo(
site_name="Audiences",
url="https://audiences.me/",
site_cookie="",
apikey=None,
token=None,
)
html_text = """
<html>
<body>
<div class="site-userbar__compact-actions">
<a class="site-userbar__compact-tool site-userbar__compact-tool--has-unread"
href="messages.php"
title="收件箱 1749/172"
aria-label="收件箱 1749/172">
<i class="fas fa-inbox" aria-hidden="true"></i>
<strong>收件箱</strong>
<span class="site-userbar__compact-tool-badge site-userbar__compact-tool-badge--unread">1749/172</span>
</a>
<a class="site-userbar__compact-tool"
href="messages.php?action=viewmailbox&amp;box=-1"
title="发件箱 0"
aria-label="发件箱 0">
<strong>发件箱</strong>
<span class="site-userbar__compact-tool-badge">0</span>
</a>
</div>
</body>
</html>
"""
parser._parse_message_unread(html_text)
assert parser.message_unread == 172
def test_audiences_table_unread_links_ignore_content_rows():
parser = NexusAudiencesSiteUserInfo(
site_name="Audiences",
url="https://audiences.me/",
site_cookie="",
apikey=None,
token=None,
)
html_text = """
<html>
<body>
<table>
<tr>
<td class="rowfollow" align="center">
<img class="unreadpm" src="pic/trans.gif" alt="Unread" title="未读">
</td>
<td class="rowfollow" align="left">
<a href="messages.php?action=viewmessage&amp;id=4318225">种子被删除</a>
</td>
<td class="rowfollow" align="left">系统</td>
<td class="rowfollow" nowrap=""><span title="2026-05-07 23:01:58">8天17时前</span></td>
<td class="rowfollow"><input class="checkbox" type="checkbox" name="messages[]" value="4318225"></td>
</tr>
<tr>
<td colspan="5" style="padding: 8px;">消息摘要内容</td>
</tr>
<tr>
<td class="rowfollow" align="center">
<img class="readpm" src="pic/trans.gif" alt="Read" title="已读">
</td>
<td class="rowfollow" align="left">
<a href="messages.php?action=viewmessage&amp;id=4318000">已读消息</a>
</td>
<td class="rowfollow" align="left">系统</td>
<td class="rowfollow" nowrap=""><span title="2026-05-07 23:01:58">8天17时前</span></td>
<td class="rowfollow"><input class="checkbox" type="checkbox" name="messages[]" value="4318000"></td>
</tr>
</table>
</body>
</html>
"""
msg_links = []
next_page = parser._parse_message_unread_links(html_text, msg_links)
assert msg_links == ["messages.php?action=viewmessage&id=4318225"]
assert next_page is None

View File

@@ -1,10 +1,11 @@
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch, MagicMock
from app.core.config import settings
from app.chain.transfer import JobManager, TransferChain
from app.schemas import FileItem, TransferInfo, TransferTask
from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask
from app.schemas.types import EventType, MediaType
@@ -126,6 +127,49 @@ def migrate_to_media_job(jobview: JobManager, task: TransferTask):
class TransferJobManagerTest(unittest.TestCase):
def test_manual_episode_offset_applies_once(self):
chain = make_transfer_chain()
source_fileitem = make_fileitem("/downloads/Test.Show.2026.S01E14.mkv")
planned_episodes = []
chain._TransferChain__get_trans_fileitems = lambda fileitem, predicate: [
(source_fileitem, False)
]
chain._TransferChain__put_to_jobview = lambda task: True
chain._TransferChain__register_scrape_batch_task = lambda task: None
chain._TransferChain__close_scrape_batch = lambda batch_id: None
def fake_handle_transfer(task, callback=None):
planned_episodes.append(task.meta.begin_episode)
return True, ""
chain._TransferChain__handle_transfer = fake_handle_transfer
transfer_history_oper = SimpleNamespace(get_by_src=lambda src, storage=None: None)
download_history_oper = SimpleNamespace(
get_by_hash=lambda download_hash: None,
get_file_by_fullpath=lambda fullpath: None,
get_files_by_savepath=lambda savepath: [],
get_by_path=lambda path: None,
)
system_config_oper = SimpleNamespace(get=lambda key: None)
with patch("app.chain.transfer.TransferHistoryOper", return_value=transfer_history_oper), \
patch("app.chain.transfer.DownloadHistoryOper", return_value=download_history_oper), \
patch("app.chain.transfer.SystemConfigOper", return_value=system_config_oper), \
patch("app.chain.transfer.MetaInfoPath", lambda *args, **kwargs: FakeMeta(14)):
state, errmsg = chain.do_transfer(
fileitem=source_fileitem,
mediainfo=FakeMedia(),
target_path=Path("/library"),
epformat=EpisodeFormat(offset="-1"),
background=False,
)
self.assertTrue(state, errmsg)
# 手动集数偏移只能应用一次,避免 E14 + (-1) 被二次处理成 E12。
self.assertEqual([13], planned_episodes)
def test_completed_media_job_is_removed_after_last_meta_task_fails(self):
jobview = JobManager()
tasks = [make_task(episode) for episode in range(1, 4)]

View File

@@ -1,2 +1,2 @@
APP_VERSION = 'v2.11.4'
FRONTEND_VERSION = 'v2.11.4'
APP_VERSION = 'v2.12.0'
FRONTEND_VERSION = 'v2.12.0'