Compare commits

...
19 Commits
Author SHA1 Message Date
jxxghp 93761fe7e4 优化 Agent 代码编辑工具 (#6218) 2026-08-01 09:02:05 +08:00
jxxghp 593139faac fix(agent): 移除插件源码写入限制 2026-07-31 22:40:50 +08:00
cyt-666 6c89f1eb4b fix: 修复 MCP 动态加载插件工具 (#6214) 2026-07-31 22:31:46 +08:00
jxxghp 2310a3a456 fix(plugin): translate rating validation error 2026-07-31 13:50:17 +08:00
jxxghp 48852350a0 feat(plugin): add centralized rating APIs 2026-07-31 13:41:29 +08:00
jxxghp a23fce1491 feat: 统计共享媒体识别命中次数 2026-07-30 17:50:16 +08:00
jxxghp c976741574 feat(transfer): control mounted directory cleanup 2026-07-30 17:45:13 +08:00
jxxghp 04facef64d chore(version): update app and frontend version to v2.15.2 2026-07-30 13:45:29 +08:00
jxxghp 33a97eb2c8 feat: 增加 MoviePilot 选项 (#6212) 2026-07-30 13:33:08 +08:00
Aqr-K cf80b551f9 fix(monitor): 目录监控自愈、快照语义修正与覆盖保护闭环 (#6210) 2026-07-30 06:53:52 +08:00
jxxghp e011b20210 add start-local.sh 2026-07-29 16:54:00 +08:00
jxxghpandClaude Opus 5 bdf395f494 feat(trimemedia): 支持飞牛影视访问码
飞牛开启访问码后,需先请求 /c/{访问码} 获得会话凭证,
否则登录接口与图片等资源均返回 404。

- Api 增加 access_code 参数及 verify_access_code(),登录前先校验
- 创建 API 连接时校验访问码,失败则关闭会话
- 图片 Cookies 附带访问码校验凭证,避免代理图片 404

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-28 19:09:10 +08:00
jxxghp 68686bc23a fix https://github.com/jxxghp/MoviePilot/issues/6206
fix https://github.com/jxxghp/MoviePilot/issues/6205
2026-07-28 19:01:55 +08:00
Aqr-K 1a528c7803 fix(http): AsyncRequestUtils 代理请求按 host 熔断降级 h2 隧道 (#6208) 2026-07-28 18:54:16 +08:00
jxxghpandtraeagent 6b4a255f26 feat: 重命名 auth_bridge.py 为 auth.py (#6207)
Co-authored-by: traeagent <traeagent@users.noreply.github.com>
2026-07-28 14:56:33 +08:00
InfinityPacer 1a3c1b8b39 fix(system): derive image proxy MIME from content (#6202) 2026-07-27 14:56:46 +08:00
InfinityPacer 8788dae34b revert(login): remove anonymous wallpaper proxy (#6201) 2026-07-27 13:48:52 +08:00
InfinityPacer bb00814d7a feat(login): 提供无状态签名同源壁纸代理 (#6200) 2026-07-27 12:13:38 +08:00
jxxghp 3d55d44457 fix: ensure media_id is populated in popular subscribes API
- Add media_id field assignment in popular_subscribes endpoint
- Fixes duplicate display issue in frontend by providing complete media identity data

Closes #6199
2026-07-27 09:42:54 +08:00
82 changed files with 4979 additions and 1165 deletions
+6
View File
@@ -729,6 +729,7 @@ class MoviePilotAgent:
user_agent=settings.LLM_USER_AGENT,
use_proxy=settings.LLM_USE_PROXY,
thinking_level=settings.LLM_THINKING_LEVEL,
api_protocol=settings.LLM_API_PROTOCOL,
)
selected_event = await eventmanager.async_send_event(
ChainEventType.AgentLLMProvider,
@@ -769,6 +770,9 @@ class MoviePilotAgent:
)
or settings.LLM_THINKING_LEVEL
)
api_protocol = self._clean_optional_text(
self._get_event_value(resolved_data, "api_protocol")
) or settings.LLM_API_PROTOCOL
selected_provider_id = self._clean_optional_text(
self._get_event_value(resolved_data, "selected_provider_id")
)
@@ -794,6 +798,7 @@ class MoviePilotAgent:
"user_agent": user_agent,
"use_proxy": bool(use_proxy),
"thinking_level": thinking_level,
"api_protocol": api_protocol,
}
return self._llm_runtime_config
@@ -1029,6 +1034,7 @@ class MoviePilotAgent:
runtime_config.get("user_agent"),
bool(runtime_config.get("use_proxy")),
runtime_config.get("thinking_level"),
runtime_config.get("api_protocol"),
)
async def _agent_bundle_signature(self, streaming: bool) -> tuple[Any, ...]:
+38 -5
View File
@@ -846,19 +846,31 @@ class LLMHelper:
provider: str,
model: str | None,
runtime: dict[str, Any],
api_protocol: str | None = None,
) -> bool | None:
"""
判断官方 ChatGPT API Key 模式是否应使用 Responses API。
判断本次 OpenAI 兼容调用是否应使用 Responses API。
GPT-5/o 系推理模型在 Chat Completions 中组合 function tools 与
reasoning_effort 时会被官方端点拒绝,因此 ChatGPT 官方 API Key
模式需要显式切到 Responses API;通用 OpenAI-compatible 入口保持
provider 目录解析出的默认行为,避免误伤第三方兼容服务。
优先级:
1. 运行时显式要求(ChatGPT Plus/Pro OAuth、Codex 等端点契约),始终保留;
2. 用户通过 ``LLM_API_PROTOCOL`` 显式指定 ``responses`` / ``chat_completions``
3. ``auto``(默认)保持原有 ChatGPT 官方 API Key + GPT-5/o 系推理模型
自动切换逻辑,通用 OpenAI 兼容入口仍走 Chat Completions
避免误伤第三方兼容服务。
:param api_protocol: 显式传入的 API 协议,未传入时读取 ``LLM_API_PROTOCOL``
:return: True/False 强制指定协议;None 表示交由 LangChain 默认行为
"""
runtime_use_responses_api = runtime.get("use_responses_api")
if runtime_use_responses_api is not None:
return bool(runtime_use_responses_api)
protocol = cls._normalize_api_protocol(api_protocol)
if protocol == "responses":
return True
if protocol == "chat_completions":
return False
provider_name = (provider or "").strip().lower()
if provider_name != "chatgpt":
return None
@@ -872,6 +884,18 @@ class LLMHelper:
return True
return None
@staticmethod
def _normalize_api_protocol(api_protocol: str | None) -> str:
"""
规范化 API 协议配置,未知值统一回退为 ``auto`` 以保持兼容。
"""
normalized = str(api_protocol or settings.LLM_API_PROTOCOL or "").strip().lower()
if normalized in {"auto", "chat_completions", "responses"}:
return normalized
if normalized:
logger.warning(f"忽略不支持的 LLM_API_PROTOCOL 配置: {api_protocol}")
return "auto"
@staticmethod
def _attach_runtime_metadata(model: Any, runtime: dict[str, Any]) -> None:
"""
@@ -954,6 +978,7 @@ class LLMHelper:
user_agent: str | None = None,
temperature: Optional[float] = None,
use_proxy: bool | None = None,
api_protocol: str | None = None,
):
"""
获取LLM实例
@@ -970,6 +995,10 @@ class LLMHelper:
:param user_agent: OpenAI兼容接口请求 User-Agent。未显式传入时使用配置项 LLM_USER_AGENT。
:param temperature: LLM 温度参数。未显式传入时使用配置项 LLM_TEMPERATURE。
:param use_proxy: 是否为本次 LLM 调用使用系统代理。未显式传入时使用配置项 LLM_USE_PROXY。
:param api_protocol: OpenAI 兼容接口 API 协议
auto/chat_completions/responses)。未显式传入时使用配置项 LLM_API_PROTOCOL。
仅对 OpenAI 兼容运行时生效;``responses`` 强制走 Responses API
``chat_completions`` 强制走 Chat Completions``auto`` 保持原有自动判断。
:return: LLM实例
"""
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).lower()
@@ -1021,6 +1050,7 @@ class LLMHelper:
provider=provider_name,
model=model_name,
runtime=runtime,
api_protocol=api_protocol,
)
llm_proxy = _resolve_llm_proxy(use_proxy)
@@ -1210,11 +1240,13 @@ class LLMHelper:
user_agent: str | None = None,
temperature: Optional[float] = None,
use_proxy: bool | None = None,
api_protocol: str | None = None,
) -> dict:
"""
使用当前配置或显式传入的临时配置执行一次最小 LLM 调用。
:param temperature: LLM 温度参数。未显式传入时沿用已保存配置。
:param api_protocol: OpenAI 兼容接口 API 协议,未显式传入时沿用已保存配置。
"""
provider_name = provider if provider is not None else settings.LLM_PROVIDER
model_name = model if model is not None else settings.LLM_MODEL
@@ -1229,6 +1261,7 @@ class LLMHelper:
"base_url_preset": base_url_preset,
"user_agent": user_agent,
"use_proxy": use_proxy,
"api_protocol": api_protocol,
}
if temperature is not None:
llm_kwargs["temperature"] = temperature
+5 -2
View File
@@ -17,7 +17,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
- Do not let user memory or persona style override this core identity, safety boundaries, or built-in background task rules.
- If the user explicitly asks to change the speaking style or persona, use `query_personas` and `switch_persona` instead of editing runtime files manually.
- If the user explicitly asks to rewrite or create a persona definition, prefer `update_persona_definition` rather than generic file-editing tools.
- Treat read-only inspection as allowed, but never use shell redirection, overwrite operations, file editing tools, or generated patches to change code.
</non_negotiable_boundaries>
<confirmation_policy>
@@ -66,7 +65,11 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
- If `search_media` fails, fall back to `search_web` or `recognize_media`. Only ask the user when automated paths are exhausted.
- If torrent search yields no useful result, check site scope, site health, and recognition quality before concluding that the resource is unavailable.
- Reuse the latest torrent search cache for `get_search_results` and `add_download_tasks` instead of re-running the same search unnecessarily.
- Use `execute_command` only for diagnostics, read-only inspection, or commands the user explicitly asked to run. Its default `action=start` starts a managed background session and returns `session_id`, `status`, `last_seq`, and `output_until_seq`; call the same tool again with `action=read`, `action=wait`, `action=write`, or `action=kill` to poll output, wait in short segments, send stdin, or stop the process.
- For administrator code discovery across local files, use `execute_command(action="run")` with `rg` and narrow globs or paths. Use `list_directory` to inspect one known directory or a supported remote storage backend, and use `read_file` when the exact local file is known.
- Read the relevant file before changing it. Use `edit_file` for localized exact replacements; make `old_text` unique with enough surrounding context, and use `replace_all=true` only when every match must change. Use `write_file` for new files; set `overwrite=true` only for an intentional full rewrite, and use `read_file(include_metadata=true)` plus `expected_sha256` when preserving the previously read version matters.
- When implementation depends on a Python or Node.js API, first identify the installed or locked dependency version from environment metadata, requirements, package manifests, lockfiles, local source, and type declarations. Use `rg` against the relevant package directory, `.venv`, or `node_modules` instead of scanning the entire project without bounds. If local evidence is insufficient, use `search_web` and then `browse_webpage` to read the matching version of the official documentation. Do not guess signatures from memory, mix examples from incompatible versions, or install a package only to inspect its API.
- Use structured file tools for source edits because they enforce file access boundaries and conflict checks. Never use shell redirection, inline scripts, or another tool to bypass a file-tool permission denial.
- Use `execute_command` for administrator-only multi-file diagnostics, tests, Git, service operations, SSH, or an exact command the user requested. Use `action=run` for short bounded commands. Use `action=start` for long-running or interactive commands, including SSH; then continue with `read`, `wait`, `write`, or `kill` using the returned `session_id`. Do not start a background session for a short command that can finish within `action=run`.
</tool_strategy>
<media_rules>
+53
View File
@@ -0,0 +1,53 @@
"""Agent 文件写入工具的共享辅助函数。"""
import hashlib
import os
import tempfile
from pathlib import Path
class FileVersionConflictError(RuntimeError):
"""目标文件在准备写入期间发生变化。"""
def calculate_file_sha256(path: Path) -> str:
"""计算文件原始字节的 SHA-256,用于检测陈旧写入。"""
digest = hashlib.sha256()
with path.open("rb") as file_handle:
for chunk in iter(lambda: file_handle.read(64 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def atomic_write_text(
path: Path,
content: str,
expected_sha256: str | None = None,
) -> None:
"""校验目标版本后,在同目录写入临时文件并原子替换文本。"""
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temp_name = tempfile.mkstemp(
dir=path.parent,
prefix=f".{path.name}.",
suffix=".tmp",
)
temp_path = Path(temp_name)
try:
with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as file_handle:
file_handle.write(content)
file_handle.flush()
os.fsync(file_handle.fileno())
if expected_sha256:
if (
not path.is_file()
or calculate_file_sha256(path).casefold()
!= expected_sha256.casefold()
):
raise FileVersionConflictError(str(path))
if path.exists():
os.chmod(temp_path, path.stat().st_mode)
os.replace(temp_path, path)
finally:
if temp_path.exists():
temp_path.unlink()
+105 -28
View File
@@ -1,4 +1,4 @@
"""文件编辑工具"""
"""文件精确编辑工具"""
from pathlib import Path
from typing import Optional, Type
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool
from app.agent.tools.impl._file_write_utils import (
FileVersionConflictError,
atomic_write_text,
calculate_file_sha256,
)
from app.agent.tools.tags import ToolTag
from app.log import logger
@@ -15,18 +20,44 @@ class EditFileInput(BaseModel):
"""文件编辑工具的输入参数模型。"""
file_path: str = Field(..., description="The absolute path of the file to edit")
old_text: str = Field(..., description="The exact old text to be replaced")
old_text: str = Field(
...,
description=(
"The exact old text to replace. It must be non-empty and uniquely "
"identify one location unless replace_all is true."
),
)
new_text: str = Field(..., description="The new text to replace with")
replace_all: bool = Field(
False,
description=(
"Replace every exact match. Keep false for normal code edits so an "
"ambiguous match fails instead of changing multiple locations."
),
)
expected_sha256: Optional[str] = Field(
None,
pattern=r"^[0-9a-fA-F]{64}$",
description=(
"Optional SHA-256 returned by read_file(include_metadata=true). The "
"edit fails if the file changed after it was read."
),
)
class EditFileTool(MoviePilotTool):
"""使用精确文本匹配安全编辑本地文件。"""
name: str = "edit_file"
tags: list[str] = [
ToolTag.Write,
ToolTag.File,
]
description: str = (
"Edit a local text file by replacing specific old text with new text. "
"Edit an existing local text file using an exact text match. By default "
"the match must occur exactly once; use replace_all only for intentional "
"bulk replacement. old_text cannot be empty, and new files must be "
"created with write_file. Supports an optional SHA-256 conflict check. "
"Non-admin users can only edit files inside the MoviePilot Agent config "
"directory."
)
@@ -38,7 +69,16 @@ class EditFileTool(MoviePilotTool):
file_name = Path(file_path).name if file_path else "未知文件"
return f"编辑文件: {file_name}"
async def run(self, file_path: str, old_text: str, new_text: str, **kwargs) -> str:
async def run(
self,
file_path: str,
old_text: str,
new_text: str,
replace_all: bool = False,
expected_sha256: Optional[str] = None,
**kwargs,
) -> str:
"""校验精确匹配和可选文件版本后,以原子方式写入编辑结果。"""
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
try:
@@ -48,37 +88,74 @@ class EditFileTool(MoviePilotTool):
if access_error:
return access_error
path = AsyncPath(resolved_path)
# 校验逻辑:如果要替换特定文本,文件必须存在且包含该文本
if not await path.exists():
# 如果 old_text 为空,可能用户想直接创建文件,但通常 edit_file 需要匹配旧内容
if old_text:
return f"错误:文件 {resolved_path} 不存在,无法进行内容替换。"
if not old_text:
return "错误:old_text 不能为空;创建或完整写入文件请使用 write_file。"
if await path.exists() and not await path.is_file():
path = AsyncPath(resolved_path)
if not await path.exists():
return f"错误:文件 {resolved_path} 不存在;创建文件请使用 write_file。"
if not await path.is_file():
return f"错误:{resolved_path} 不是一个文件"
if await path.exists():
content = await path.read_text(encoding="utf-8", errors="replace")
if old_text not in content:
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
return f"错误:在文件 {resolved_path} 中未找到指定的旧文本。请确保包含所有的空格、缩进 and 换行符。"
occurrences = content.count(old_text)
new_content = content.replace(old_text, new_text)
else:
# 文件不存在且 old_text 为空的情形(初始化新文件)
new_content = new_text
occurrences = 1
local_path = Path(resolved_path)
current_sha256 = await self.run_blocking(
"default", calculate_file_sha256, local_path
)
if (
expected_sha256
and current_sha256.casefold() != expected_sha256.casefold()
):
return (
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
"请重新读取文件并基于最新内容编辑。"
)
# 自动创建父目录
await path.parent.mkdir(parents=True, exist_ok=True)
content = await path.read_text(encoding="utf-8", errors="strict")
occurrences = content.count(old_text)
if occurrences == 0:
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
return (
f"错误:在文件 {resolved_path} 中未找到指定的旧文本。"
"请重新读取文件并确认空格、缩进和换行。"
)
if occurrences > 1 and not replace_all:
return (
f"错误:old_text 在文件 {resolved_path} 中匹配到 {occurrences} 处,"
"为避免误改已拒绝编辑。请提供更多上下文使其唯一,或明确设置 "
"replace_all=true。"
)
# 写入文件
await path.write_text(new_content, encoding="utf-8")
replacement_count = occurrences if replace_all else 1
new_content = content.replace(
old_text,
new_text,
-1 if replace_all else 1,
)
await self.run_blocking(
"default",
atomic_write_text,
local_path,
new_content,
current_sha256,
)
new_sha256 = await self.run_blocking(
"default", calculate_file_sha256, local_path
)
logger.info(f"成功编辑文件 {resolved_path},替换了 {occurrences} 处内容")
return f"成功编辑文件 {resolved_path} (替换了 {occurrences}匹配内容)"
logger.info(
f"成功编辑文件 {resolved_path}替换了 {replacement_count} 处内容"
)
return (
f"成功编辑文件 {resolved_path}(替换了 {replacement_count} 处匹配内容,"
f"sha256={new_sha256}"
)
except FileVersionConflictError:
return (
f"错误:文件 {file_path} 在编辑期间发生变化,拒绝覆盖。"
"请重新读取文件并再次编辑。"
)
except PermissionError:
return f"错误:没有访问/修改 {file_path} 的权限"
except UnicodeDecodeError:
+38 -3
View File
@@ -1,5 +1,7 @@
"""文件读取工具"""
import hashlib
import json
from pathlib import Path
from typing import Optional, Type
@@ -16,12 +18,22 @@ MAX_READ_SIZE = 50 * 1024
class ReadFileInput(BaseModel):
"""文件读取工具的输入参数模型。"""
file_path: str = Field(..., description="The absolute path of the file to read")
start_line: Optional[int] = Field(None, description="The starting line number (1-based, inclusive). If not provided, reading starts from the beginning of the file.")
end_line: Optional[int] = Field(None, description="The ending line number (1-based, inclusive). If not provided, reading goes until the end of the file.")
include_metadata: bool = Field(
False,
description=(
"Return structured JSON containing content, size, truncation state, "
"and SHA-256. Use before a guarded full-file overwrite."
),
)
class ReadFileTool(MoviePilotTool):
"""按行范围读取本地文本文件,并可返回文件版本元数据。"""
name: str = "read_file"
tags: list[str] = [
ToolTag.Read,
@@ -36,8 +48,15 @@ class ReadFileTool(MoviePilotTool):
file_name = Path(file_path).name if file_path else "未知文件"
return f"读取文件: {file_name}"
async def run(self, file_path: str, start_line: Optional[int] = None,
end_line: Optional[int] = None, **kwargs) -> str:
async def run(
self,
file_path: str,
start_line: Optional[int] = None,
end_line: Optional[int] = None,
include_metadata: bool = False,
**kwargs,
) -> str:
"""读取指定文本范围,必要时附带完整文件的 SHA-256 元数据。"""
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}, start_line={start_line}, end_line={end_line}")
try:
@@ -55,7 +74,8 @@ class ReadFileTool(MoviePilotTool):
if not await path.is_file():
return f"错误:{resolved_path} 不是一个文件"
content = await path.read_text(encoding="utf-8", errors="replace")
raw_content = await path.read_bytes()
content = raw_content.decode("utf-8", errors="replace")
truncated = False
if start_line is not None or end_line is not None:
@@ -78,6 +98,21 @@ class ReadFileTool(MoviePilotTool):
content = content_bytes[:MAX_READ_SIZE].decode("utf-8", errors="replace")
truncated = True
if include_metadata:
return json.dumps(
{
"file_path": str(resolved_path),
"sha256": hashlib.sha256(raw_content).hexdigest(),
"size_bytes": len(raw_content),
"start_line": start_line,
"end_line": end_line,
"truncated": truncated,
"content": content,
},
ensure_ascii=False,
indent=2,
)
if truncated:
return f"{content}\n\n[警告:文件内容已超过50KB限制,以上内容已被截断。请使用 start_line/end_line 参数分段读取。]"
+75 -9
View File
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool
from app.agent.tools.impl._file_write_utils import (
FileVersionConflictError,
atomic_write_text,
calculate_file_sha256,
)
from app.agent.tools.tags import ToolTag
from app.log import logger
@@ -16,17 +21,36 @@ class WriteFileInput(BaseModel):
file_path: str = Field(..., description="The absolute path of the file to write")
content: str = Field(..., description="The content to write into the file")
overwrite: bool = Field(
False,
description=(
"Allow replacing an existing file in full. Keep false when creating a "
"new file; prefer edit_file for localized changes."
),
)
expected_sha256: Optional[str] = Field(
None,
pattern=r"^[0-9a-fA-F]{64}$",
description=(
"Optional SHA-256 returned by read_file(include_metadata=true). When "
"overwriting, fail if the existing file no longer has this hash."
),
)
class WriteFileTool(MoviePilotTool):
"""创建本地文本文件,或在显式允许后完整覆盖已有文件。"""
name: str = "write_file"
tags: list[str] = [
ToolTag.Write,
ToolTag.File,
]
description: str = (
"Write full content to a local text file. Non-admin users can only write "
"inside the MoviePilot Agent config directory."
"Create a local text file with complete content. Existing files are "
"protected unless overwrite=true; localized changes should use edit_file. "
"Supports an optional SHA-256 conflict check and writes atomically. "
"Non-admin users can only write inside the MoviePilot Agent config directory."
)
args_schema: Type[BaseModel] = WriteFileInput
@@ -36,7 +60,15 @@ class WriteFileTool(MoviePilotTool):
file_name = Path(file_path).name if file_path else "未知文件"
return f"写入文件: {file_name}"
async def run(self, file_path: str, content: str, **kwargs) -> str:
async def run(
self,
file_path: str,
content: str,
overwrite: bool = False,
expected_sha256: Optional[str] = None,
**kwargs,
) -> str:
"""创建或显式覆盖文件,并通过可选哈希阻止陈旧写入。"""
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
try:
@@ -48,18 +80,52 @@ class WriteFileTool(MoviePilotTool):
path = AsyncPath(resolved_path)
if await path.exists() and not await path.is_file():
exists = await path.exists()
if exists and not await path.is_file():
return f"错误:{resolved_path} 路径已存在但不是一个文件"
if exists and not overwrite:
return (
f"错误:文件 {resolved_path} 已存在,拒绝完整覆盖。"
"局部修改请使用 edit_file;确需重写时设置 overwrite=true。"
)
if expected_sha256 and not exists:
return (
f"错误:文件 {resolved_path} 不存在,无法校验 expected_sha256。"
"请确认路径和最新文件状态。"
)
# 自动创建父目录
await path.parent.mkdir(parents=True, exist_ok=True)
local_path = Path(resolved_path)
current_sha256 = None
if exists:
current_sha256 = await self.run_blocking(
"default", calculate_file_sha256, local_path
)
if expected_sha256:
if current_sha256.casefold() != expected_sha256.casefold():
return (
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
"请重新读取文件并基于最新内容写入。"
)
# 写入文件
await path.write_text(content, encoding="utf-8")
await self.run_blocking(
"default",
atomic_write_text,
local_path,
content,
current_sha256,
)
new_sha256 = await self.run_blocking(
"default", calculate_file_sha256, local_path
)
logger.info(f"成功写入文件 {resolved_path}")
return f"成功写入文件 {resolved_path}"
return f"成功写入文件 {resolved_path}sha256={new_sha256}"
except FileVersionConflictError:
return (
f"错误:文件 {file_path} 在写入期间发生变化,拒绝覆盖。"
"请重新读取文件并再次写入。"
)
except PermissionError:
return f"错误:没有权限写入 {file_path}"
except Exception as e:
+47 -11
View File
@@ -1,9 +1,11 @@
import json
import threading
import uuid
from typing import Any, Dict, List, Optional
from app.agent.tools.base import ToolExecutionTimeoutError, format_tool_result_for_agent
from app.agent.tools.factory import MoviePilotToolFactory
from app.core.plugin import PluginManager
from app.log import logger
@@ -40,27 +42,59 @@ class MoviePilotToolsManager:
self.session_id = session_id
self.is_admin = is_admin
self.tools: List[Any] = []
self._tools_lock = threading.Lock()
self._plugin_agent_tools_revision = -1
self._load_tools()
def _load_tools(self):
def _load_tools(self) -> None:
"""
加载所有MoviePilot工具
"""
try:
# 创建工具实例
self.tools = MoviePilotToolFactory.create_tools(
session_id=self.session_id,
user_id=self.user_id,
channel=None,
source="api",
username="API Client",
stream_handler=None,
agent_context={"is_admin": self.is_admin},
)
plugin_manager = PluginManager()
while True:
plugin_tools_revision = (
plugin_manager.get_plugin_agent_tools_revision()
)
tools = MoviePilotToolFactory.create_tools(
session_id=self.session_id,
user_id=self.user_id,
channel=None,
source="api",
username="API Client",
stream_handler=None,
agent_context={"is_admin": self.is_admin},
)
if (
plugin_tools_revision
== plugin_manager.get_plugin_agent_tools_revision()
):
break
self.tools = tools
self._plugin_agent_tools_revision = plugin_tools_revision
logger.info(f"成功加载 {len(self.tools)} 个工具")
except Exception as e:
logger.error(f"加载工具失败: {e}", exc_info=True)
self.tools = []
self._plugin_agent_tools_revision = -1
def _ensure_tools_current(self) -> None:
"""
在插件工具注册表变化后惰性刷新工具实例。
"""
plugin_manager = PluginManager()
if (
self._plugin_agent_tools_revision
== plugin_manager.get_plugin_agent_tools_revision()
):
return
with self._tools_lock:
if (
self._plugin_agent_tools_revision
== plugin_manager.get_plugin_agent_tools_revision()
):
return
self._load_tools()
def list_tools(self) -> List[ToolDefinition]:
"""
@@ -69,6 +103,7 @@ class MoviePilotToolsManager:
Returns:
工具定义列表
"""
self._ensure_tools_current()
tools_list = []
for tool in self.tools:
if getattr(tool, "_require_admin", False) and not self.is_admin:
@@ -102,6 +137,7 @@ class MoviePilotToolsManager:
Returns:
工具实例,如果未找到返回None
"""
self._ensure_tools_current()
for tool in self.tools:
if tool.name == tool_name:
return tool
+1 -1
View File
@@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from app import schemas
from app.core.auth_bridge import build_token_response, consume_plugin_auth_ticket
from app.core.auth import build_token_response, consume_plugin_auth_ticket
from app.core.plugin import PluginManager
from app.db.models.passkey import PassKey
from app.db.models.user import User
+7
View File
@@ -4,12 +4,15 @@ from fastapi import APIRouter, Depends
from app import schemas
from app.chain.douban import DoubanChain
from app.core.config import settings
from app.core.context import MediaInfo
from app.core.security import verify_token
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import get_current_active_superuser_async
from app.modules.douban.douban_cache import DoubanCache
from app.schemas import MediaType
from app.schemas.types import SystemConfigKey
router = APIRouter()
@@ -29,6 +32,10 @@ async def douban_recognition_cache(
"count": len(cache_items),
"recognized": recognized_count,
"unrecognized": len(cache_items) - recognized_count,
"shared_recognized": SystemConfigOper().get(
SystemConfigKey.MediaRecognizeShareCount
) or 0,
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
"data": cache_items,
},
)
+3
View File
@@ -38,6 +38,7 @@ class LlmTestRequest(BaseModel):
user_agent: Optional[str] = None
temperature: Optional[float] = None
use_proxy: Optional[bool] = None
api_protocol: Optional[str] = None
class LlmProviderAuthStartRequest(BaseModel):
@@ -269,6 +270,7 @@ async def llm_test(
base_url_preset=settings.LLM_BASE_URL_PRESET,
user_agent=settings.LLM_USER_AGENT,
use_proxy=settings.LLM_USE_PROXY,
api_protocol=settings.LLM_API_PROTOCOL,
)
if not payload.provider:
@@ -302,6 +304,7 @@ async def llm_test(
"base_url_preset": payload.base_url_preset,
"user_agent": payload.user_agent,
"use_proxy": payload.use_proxy,
"api_protocol": payload.api_protocol,
}
if payload.temperature is not None:
test_kwargs["temperature"] = payload.temperature
+66 -1
View File
@@ -1,7 +1,7 @@
import asyncio
import mimetypes
import shutil
from typing import Annotated, Any, List, Optional
from typing import Annotated, Any, Dict, List, Optional
import aiofiles
from anyio import Path as AsyncPath
@@ -476,6 +476,71 @@ async def statistic(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
return await MoviePilotServerHelper.async_get_plugin_statistic()
@router.get(
"/rating",
summary="批量查询插件评分",
response_model=Dict[str, schemas.PluginRating],
)
async def plugin_ratings(
plugin_ids: Optional[str] = None,
_: User = Depends(get_current_active_superuser_async),
) -> Dict[str, schemas.PluginRating]:
"""
批量查询插件平均分、评分人数和当前安装实例评分。
"""
requested_ids = plugin_ids.split(",") if plugin_ids is not None else None
ratings = await MoviePilotServerHelper.async_get_plugin_ratings(requested_ids)
return {
plugin_id: schemas.PluginRating.model_validate(rating)
for plugin_id, rating in ratings.items()
}
@router.get(
"/rating/{plugin_id}",
summary="查询插件评分",
response_model=schemas.PluginRating,
)
async def plugin_rating(
plugin_id: str,
_: User = Depends(get_current_active_superuser_async),
) -> schemas.PluginRating:
"""
查询单个插件平均分、评分人数和当前安装实例评分。
"""
rating = await MoviePilotServerHelper.async_get_plugin_rating(plugin_id)
return schemas.PluginRating.model_validate(rating)
@router.post(
"/rating/{plugin_id}",
summary="提交插件评分",
response_model=schemas.Response,
)
async def rate_plugin(
plugin_id: str,
payload: schemas.PluginRatingRequest,
_: User = Depends(get_current_active_superuser_async),
) -> schemas.Response:
"""
为已安装插件新增或更新当前安装实例评分。
"""
installed_plugins = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
if plugin_id not in installed_plugins:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"插件 {plugin_id} 未安装,无法评分",
)
rating = await MoviePilotServerHelper.async_submit_plugin_rating(
plugin_id,
payload.rating,
)
if rating is None:
return schemas.Response(success=False, message="连接MoviePilot服务器失败")
return schemas.Response(success=True, data=rating)
@router.get(
"/reload/{plugin_id}", summary="重新加载插件", response_model=schemas.Response
)
+1
View File
@@ -630,6 +630,7 @@ async def popular_subscribes(
media.bangumi_id = sub.get("bangumiid")
media.anilist_id = sub.get("anilistid")
media.source = sub.get("media_source")
media.media_id = sub.get("media_id")
media.tvdb_id = sub.get("tvdbid")
media.imdb_id = sub.get("imdbid")
media.season = sub.get("season")
+7 -3
View File
@@ -582,23 +582,27 @@ async def fetch_image(
):
return None
content = await ImageHelper().async_fetch_image(
image_result = await ImageHelper().async_fetch_image_with_mime_type(
url=fetch_url,
proxy=proxy,
use_cache=use_cache,
cookies=cookies,
)
if content:
if image_result:
content, media_type = image_result
# 检查 If-None-Match
etag = HashUtils.md5(content)
headers = RequestUtils.generate_cache_headers(etag, max_age=86400 * 7)
headers["Content-Type"] = media_type
headers["X-Content-Type-Options"] = "nosniff"
if if_none_match == etag:
return Response(status_code=304, headers=headers)
# 返回缓存图片
return Response(
content=content,
media_type=UrlUtils.get_mime_type(fetch_url, "image/jpeg"),
media_type=media_type,
headers=headers,
)
return None
+7 -1
View File
@@ -4,11 +4,13 @@ from fastapi import APIRouter, Depends
from app import schemas
from app.chain.tmdb import TmdbChain
from app.core.config import settings
from app.core.security import verify_token
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import get_current_active_superuser_async
from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.schemas.types import MediaType
from app.schemas.types import MediaType, SystemConfigKey
router = APIRouter()
@@ -28,6 +30,10 @@ async def tmdb_recognition_cache(
"count": len(cache_items),
"recognized": recognized_count,
"unrecognized": len(cache_items) - recognized_count,
"shared_recognized": SystemConfigOper().get(
SystemConfigKey.MediaRecognizeShareCount
) or 0,
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
"data": cache_items,
},
)
+12
View File
@@ -20,6 +20,7 @@ from app.core.meta import MetaBase
from app.core.module import ModuleManager
from app.core.plugin import PluginManager
from app.db.message_oper import MessageOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import UserOper
from app.helper.message import MessageHelper, MessageQueueManager, MessageTemplateHelper
from app.helper.server import MoviePilotServerHelper
@@ -49,6 +50,7 @@ from app.schemas.types import (
MediaImageType,
EventType,
MessageChannel,
SystemConfigKey,
)
from app.utils.object import ObjectUtils
@@ -572,6 +574,14 @@ class ChainBase(metaclass=ABCMeta):
mediainfo=mediainfo,
)
@staticmethod
def _record_media_recognize_share_hit() -> None:
"""记录一次共享媒体识别成功命中,统计失败不影响识别结果。"""
try:
SystemConfigOper().increment(SystemConfigKey.MediaRecognizeShareCount)
except Exception as err:
logger.error(f"记录共享媒体识别命中次数失败:{str(err)}")
@staticmethod
def _resolve_media_source_params(
source: Optional[str] = None,
@@ -730,6 +740,7 @@ class ChainBase(metaclass=ABCMeta):
)
if mediainfo:
self._update_local_recognize_cache(shared_cache_meta, mediainfo)
self._record_media_recognize_share_hit()
return mediainfo
return None
@@ -839,6 +850,7 @@ class ChainBase(metaclass=ABCMeta):
)
if mediainfo:
await self._async_update_local_recognize_cache(shared_cache_meta, mediainfo)
await run_in_threadpool(self._record_media_recognize_share_hit)
return mediainfo
return None
+49 -3
View File
@@ -928,6 +928,36 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
or "/@eaDir" in normalized_path
)
@staticmethod
def __should_delete_empty_source_directories(
task: TransferTask,
delete_mounted_local_disk_empty_dirs: bool,
mounted_filesystem_cache: Dict[Path, bool],
) -> bool:
"""
判断移动整理后是否应删除源空目录
仅在关闭挂载盘空目录清理且源存储为本地时检测文件系统
避免默认流程产生额外系统调用
"""
if delete_mounted_local_disk_empty_dirs:
return True
if task.fileitem.storage != "local":
return True
source_directory = (
Path(task.target_directory.download_path)
if task.target_directory and task.target_directory.download_path
else Path(task.fileitem.path).parent
)
if source_directory not in mounted_filesystem_cache:
mounted_filesystem_cache[source_directory] = (
SystemUtils.is_network_filesystem(
source_directory, include_local_fuse=True
)
)
return not mounted_filesystem_cache[source_directory]
def __default_callback(
self, task: TransferTask, transferinfo: TransferInfo, /
) -> Tuple[bool, str]:
@@ -1189,10 +1219,16 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
tasks = self.jobview.success_tasks(
task.mediainfo, task.meta.begin_season
)
system_config_oper = SystemConfigOper()
# 获取整理屏蔽词
transfer_exclude_words = SystemConfigOper().get(
transfer_exclude_words = system_config_oper.get(
SystemConfigKey.TransferExcludeWords
)
# 挂载盘空目录清理默认开启
delete_mounted_local_disk_empty_dirs = system_config_oper.get(
SystemConfigKey.MountedLocalDiskDeleteEmptyDirs
) is not False
mounted_filesystem_cache: Dict[Path, bool] = {}
processed_hashes = set()
for t in tasks:
if t.download_hash and t.download_hash not in processed_hashes:
@@ -1209,7 +1245,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
logger.info(
f"移动模式删除种子成功:{t.download_hash}"
)
if not t.download_hash and t.fileitem:
if (
not t.download_hash
and t.fileitem
and self.__should_delete_empty_source_directories(
t,
delete_mounted_local_disk_empty_dirs,
mounted_filesystem_cache,
)
):
# 删除剩余空目录
StorageChain().delete_media_file(t.fileitem, delete_self=False)
@@ -1565,7 +1609,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
logger.info(__end_msg)
self._progress.update(value=100, text=__end_msg)
self._progress.end()
# 重置计数
# 重置计数_total_num 一并归零,否则会作为历史最大值一直
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
self._total_num = 0
self._processed_num = 0
self._fail_num = 0
+2
View File
@@ -569,6 +569,8 @@ class ConfigModel(BaseModel):
LLM_MODEL: str = "deepseek-chat"
# 思考模式/深度配置:off/auto/minimal/low/medium/high/max/xhigh
LLM_THINKING_LEVEL: Optional[str] = "off"
# OpenAI兼容接口API协议:auto(自动)/ chat_completions / responses
LLM_API_PROTOCOL: str = "auto"
# LLM是否支持图片输入,开启后消息图片会按多模态输入发送给模型
LLM_SUPPORT_IMAGE_INPUT: bool = True
# 是否启用音频输入,开启后用户语音会先转写为文本再进入 Agent
+44 -28
View File
@@ -58,6 +58,7 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
# 插件智能体工具注册表缓存,插件启停或配置生效时主动失效。
self._plugin_agent_tools_cache: Dict[str, List[Dict[str, Any]]] = {}
self._plugin_agent_tools_cache_lock = threading.Lock()
self._plugin_agent_tools_revision: int = 0
# 开发者模式监测插件修改
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
self.__start_monitor()
@@ -143,6 +144,14 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""
with self._plugin_agent_tools_cache_lock:
self._plugin_agent_tools_cache.clear()
self._plugin_agent_tools_revision += 1
def get_plugin_agent_tools_revision(self) -> int:
"""
获取插件智能体工具注册表版本号
"""
with self._plugin_agent_tools_cache_lock:
return self._plugin_agent_tools_revision
def stop(self, pid: Optional[str] = None):
"""
@@ -1002,35 +1011,42 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
}]
"""
cache_key = pid or "__all__"
with self._plugin_agent_tools_cache_lock:
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
if cached_tools is not None:
return self._copy_plugin_agent_tools(cached_tools)
while True:
with self._plugin_agent_tools_cache_lock:
cache_revision = self._plugin_agent_tools_revision
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
if cached_tools is not None:
return self._copy_plugin_agent_tools(cached_tools)
ret_tools = []
# 创建字典快照避免并发修改
running_plugins_snapshot = dict(self._running_plugins)
for plugin_id, plugin in running_plugins_snapshot.items():
if pid and pid != plugin_id:
continue
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(plugin.get_agent_tools):
try:
if not plugin.get_state():
continue
tools = plugin.get_agent_tools()
if tools:
ret_tools.append({
"plugin_id": plugin_id,
"plugin_name": plugin.plugin_name,
"tools": tools
})
except Exception as e:
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
with self._plugin_agent_tools_cache_lock:
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
ret_tools
)
return ret_tools
ret_tools = []
# 创建字典快照避免并发修改
running_plugins_snapshot = dict(self._running_plugins)
for plugin_id, plugin in running_plugins_snapshot.items():
if pid and pid != plugin_id:
continue
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(
plugin.get_agent_tools
):
try:
if not plugin.get_state():
continue
tools = plugin.get_agent_tools()
if tools:
ret_tools.append({
"plugin_id": plugin_id,
"plugin_name": plugin.plugin_name,
"tools": tools
})
except Exception as e:
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
with self._plugin_agent_tools_cache_lock:
if cache_revision != self._plugin_agent_tools_revision:
# 插件状态在注册表构建期间发生变化,重新读取以避免写回过期快照。
continue
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
ret_tools
)
return ret_tools
@staticmethod
def get_plugin_remote_entry(plugin_id: str, dist_path: str) -> str:
+13
View File
@@ -101,6 +101,19 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
# 避免将__SYSTEMCONF内的值引用出去,会导致set时误判没有变动
return copy.deepcopy(self.__SYSTEMCONF.get(key))
def increment(self, key: SystemConfigKey, step: int = 1) -> int:
"""
原子递增整数系统设置
:param key: 配置键
:param step: 递增步长
:return: 递增后的整数值
"""
with self._rlock:
value = int(self.get(key) or 0) + step
self.set(key, value)
return value
def all(self):
"""
获取所有系统设置
+75 -16
View File
@@ -188,16 +188,31 @@ class ImageHelper(metaclass=Singleton):
return cache_path.as_posix()
@staticmethod
def _validate_image(content: bytes) -> bool:
"""验证图片"""
def get_image_mime_type(content: bytes, verify: bool = True) -> Optional[str]:
"""
根据图片内容返回 Pillow 识别的图片 MIME 类型
外部响应在写入缓存前需要完整校验已校验的缓存只需读取格式头
非图片或可脚本化的 MIME 类型不作为图片代理响应
"""
if not content:
return False
return None
try:
Image.open(io.BytesIO(content)).verify()
return True
except Exception as e:
logger.warn(f"Invalid image format: {e}")
return False
with Image.open(io.BytesIO(content)) as image:
image_format = (image.format or "").upper()
if verify:
image.verify()
mime_type = Image.MIME.get(image_format)
if (
not mime_type
or not mime_type.startswith("image/")
or mime_type == "image/svg+xml"
):
return None
return mime_type
except Exception as err:
logger.warning(f"Invalid image format: {err}")
return None
@staticmethod
def _get_request_params(url: str, proxy: Optional[bool], cookies: Optional[str | dict]) -> dict:
@@ -224,6 +239,26 @@ class ImageHelper(metaclass=Singleton):
"""
获取图片同步版本
"""
result = self.fetch_image_with_mime_type(
url=url,
proxy=proxy,
use_cache=use_cache,
cookies=cookies,
)
return result[0] if result else None
def fetch_image_with_mime_type(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None,
) -> Optional[tuple[bytes, str]]:
"""
同步获取图片及其内容识别 MIME 类型
网络响应在写入缓存前完整验证一次缓存命中仅重新识别格式头
"""
if not url:
return None
@@ -233,7 +268,9 @@ class ImageHelper(metaclass=Singleton):
if use_cache:
content = self.file_cache.get(cache_path, region="images")
if content:
return content
mime_type = self.get_image_mime_type(content, verify=False)
if mime_type:
return content, mime_type
# 请求远程图片
params = self._get_request_params(url, proxy, cookies)
@@ -243,13 +280,13 @@ class ImageHelper(metaclass=Singleton):
return None
content = response.content
# 验证图片
if not self._validate_image(content):
mime_type = self.get_image_mime_type(content)
if not mime_type:
return None
# 保存缓存
self.file_cache.set(cache_path, content, region="images")
return content
return content, mime_type
async def async_fetch_image(
self,
@@ -260,6 +297,26 @@ class ImageHelper(metaclass=Singleton):
"""
获取图片异步版本
"""
result = await self.async_fetch_image_with_mime_type(
url=url,
proxy=proxy,
use_cache=use_cache,
cookies=cookies,
)
return result[0] if result else None
async def async_fetch_image_with_mime_type(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None,
) -> Optional[tuple[bytes, str]]:
"""
异步获取图片及其内容识别 MIME 类型
网络响应在写入缓存前完整验证一次缓存命中仅重新识别格式头
"""
if not url:
return None
@@ -269,7 +326,9 @@ class ImageHelper(metaclass=Singleton):
if use_cache:
content = await self.async_file_cache.get(cache_path, region="images")
if content:
return content
mime_type = self.get_image_mime_type(content, verify=False)
if mime_type:
return content, mime_type
# 请求远程图片
params = self._get_request_params(url, proxy, cookies)
@@ -279,10 +338,10 @@ class ImageHelper(metaclass=Singleton):
return None
content = response.content
# 验证图片
if not self._validate_image(content):
mime_type = self.get_image_mime_type(content)
if not mime_type:
return None
# 保存缓存
await self.async_file_cache.set(cache_path, content, region="images")
return content
return content, mime_type
+86
View File
@@ -29,6 +29,7 @@ class MoviePilotServerHelper:
_USAGE_REPORT_PATH = "/usage/report"
_USAGE_STATISTIC_PATH = "/usage/statistic"
_PLUGIN_INSTALL_PATH = "/plugin/install"
_PLUGIN_RATING_PATH = "/plugin/rating"
_PLUGIN_STATISTIC_PATH = "/plugin/statistic"
_SUBSCRIBE_ADD_PATH = "/subscribe/add"
_SUBSCRIBE_DONE_PATH = "/subscribe/done"
@@ -398,6 +399,39 @@ class MoviePilotServerHelper:
"""
return await cls._async_get(cls._server_url(cls._PLUGIN_STATISTIC_PATH), timeout=10)
@classmethod
async def async_plugin_ratings(cls, plugin_ids: Optional[List[str]] = None):
"""
异步批量查询中心端插件评分
"""
params = {"plugin_ids": ",".join(plugin_ids)} if plugin_ids is not None else None
return await cls._async_get(
cls._server_url(cls._PLUGIN_RATING_PATH),
params=params,
timeout=10,
)
@classmethod
async def async_plugin_rating(cls, plugin_id: str):
"""
异步查询中心端单个插件评分
"""
return await cls._async_get(
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
timeout=10,
)
@classmethod
async def async_rate_plugin(cls, plugin_id: str, rating: float):
"""
异步提交当前安装实例的插件评分
"""
return await cls._async_post_json(
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
{"rating": rating},
timeout=10,
)
@classmethod
def plugin_install(cls, plugin_id: str, payload: Dict[str, Any]):
"""
@@ -459,6 +493,58 @@ class MoviePilotServerHelper:
return res.json()
return {}
@classmethod
async def async_get_plugin_ratings(
cls,
plugin_ids: Optional[List[str]] = None,
) -> Dict[str, Dict[str, Any]]:
"""
批量获取插件评分中心端不可用时返回空结果
"""
try:
res = await cls.async_plugin_ratings(plugin_ids)
if res is not None and res.status_code == 200:
return res.json()
except Exception as err:
logger.debug(f"批量获取插件评分失败:{str(err)}")
return {}
@classmethod
async def async_get_plugin_rating(cls, plugin_id: str) -> Dict[str, Any]:
"""
获取单个插件评分中心端不可用时返回零评分
"""
empty_rating = {
"plugin_id": plugin_id,
"average_rating": 0.0,
"rating_count": 0,
"user_rating": None,
}
try:
res = await cls.async_plugin_rating(plugin_id)
if res is not None and res.status_code == 200:
return res.json()
except Exception as err:
logger.debug(f"获取插件 {plugin_id} 评分失败:{str(err)}")
return empty_rating
@classmethod
async def async_submit_plugin_rating(
cls,
plugin_id: str,
rating: float,
) -> Optional[Dict[str, Any]]:
"""
提交插件评分成功时返回最新评分结果
"""
try:
res = await cls.async_rate_plugin(plugin_id, rating)
if res is not None and res.status_code == 200:
return res.json()
except Exception as err:
logger.debug(f"提交插件 {plugin_id} 评分失败:{str(err)}")
return None
@classmethod
def install_plugin_reg(cls, plugin_id: str, repo_url: Optional[str] = None) -> bool:
"""
+15 -2
View File
@@ -1,7 +1,10 @@
from typing import Dict, List, Optional, Type, TypeVar, Generic, Iterator
from pydantic import ValidationError
from app.core.module import ModuleManager
from app.db.systemconfig_oper import SystemConfigOper
from app.log import logger
from app.schemas import DownloaderConf, MediaServerConf, NotificationConf, NotificationSwitchConf, ServiceInfo
from app.schemas.types import NotificationType, SystemConfigKey, ModuleType
@@ -25,8 +28,18 @@ class ServiceConfigHelper:
config_data = SystemConfigOper().get(config_key)
if not config_data:
return []
# 直接使用 conf_type 来实例化配置对象
return [conf_type(**conf) for conf in config_data]
configs = []
for conf in config_data:
if not isinstance(conf, dict):
logger.warn(f"{config_key.value} 配置格式不正确,已跳过:{conf}")
continue
try:
# 直接使用 conf_type 来实例化配置对象
configs.append(conf_type(**conf))
except ValidationError as e:
# 单条配置存在非法值时跳过,避免影响其它服务的初始化
logger.error(f"{config_key.value} 配置 {conf.get('name')} 校验失败,已跳过:{e}")
return configs
@staticmethod
def get_downloader_configs() -> List[DownloaderConf]:
+4
View File
@@ -422,6 +422,10 @@
"source": "插件 {plugin} 不存在或未安装",
"target": "Plugin {plugin} does not exist or is not installed"
},
{
"source": "插件 {plugin} 未安装,无法评分",
"target": "Plugin {plugin} is not installed and cannot be rated"
},
{
"source": "插件 {plugin} 不存在或未加载",
"target": "Plugin {plugin} does not exist or is not loaded"
+4
View File
@@ -422,6 +422,10 @@
"source": "插件 {plugin} 不存在或未安装",
"target": "插件 {plugin} 不存在或未安裝"
},
{
"source": "插件 {plugin} 未安装,无法评分",
"target": "插件 {plugin} 未安裝,無法評分"
},
{
"source": "插件 {plugin} 不存在或未加载",
"target": "插件 {plugin} 不存在或未載入"
@@ -174,6 +174,13 @@ class StorageBase(metaclass=ABCMeta):
"""
pass
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError
默认实现不区分不存在查询失败由具体存储按需覆写
"""
return self.get_item(path)
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取父目录
+39 -15
View File
@@ -13,6 +13,7 @@ from app.core.config import settings, global_vars
from app.log import logger
from app.modules.filemanager import StorageBase
from app.modules.filemanager.storages import transfer_process
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.utils.http import RequestUtils
from app.utils.singleton import WeakSingleton
@@ -834,30 +835,53 @@ class AliPan(StorageBase, metaclass=WeakSingleton):
return False
return True
def __get_by_path_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
"""
按路径查询文件/目录项无法确认状态时抛出 StorageQueryError
NotFound 系列错误码表示确认不存在其余错误网络失败限流
权限或未知业务错误均无法确认目标状态
"""
resp = self._request_api(
"POST",
"/adrive/v1.0/openFile/get_by_path",
json={
"drive_id": drive_id or self._default_drive_id,
"file_path": path.as_posix(),
},
no_error_log=True,
)
if resp is None:
raise StorageQueryError(f"【阿里云盘】无法确认文件状态(请求失败): {path}")
code = resp.get("code")
if code:
if "NotFound" in str(code):
# 明确的不存在错误码,确认目标不存在
return None
raise StorageQueryError(
f"【阿里云盘】查询文件信息出错: {path} - {code} {resp.get('message')}")
return self.__get_fileitem(resp, parent=str(path.parent))
def get_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件/目录项
"""
try:
resp = self._request_api(
"POST",
"/adrive/v1.0/openFile/get_by_path",
json={
"drive_id": drive_id or self._default_drive_id,
"file_path": path.as_posix(),
},
no_error_log=True,
)
if not resp:
return None
if resp.get("code"):
logger.debug(f"【阿里云盘】获取文件信息失败: {resp.get('message')}")
return None
return self.__get_fileitem(resp, parent=str(path.parent))
return self.__get_by_path_item(path, drive_id=drive_id)
except Exception as e:
logger.debug(f"【阿里云盘】获取文件信息失败: {str(e)}")
return None
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件/目录项无法确认状态时抛出 StorageQueryError
"""
try:
return self.__get_by_path_item(path)
except StorageQueryError:
raise
except Exception as e:
raise StorageQueryError(f"【阿里云盘】查询文件信息失败: {path} - {e}") from e
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件夹如不存在则创建
+18
View File
@@ -8,6 +8,7 @@ from app.core.config import global_vars, settings
from app.helper.directory import DirectoryHelper
from app.log import logger
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.utils.system import SystemUtils
@@ -148,6 +149,23 @@ class LocalStorage(StorageBase):
return self.__get_fileitem(path)
return self.__get_diritem(path)
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取文件或目录无法确认状态时抛出 StorageQueryError
Path.exists() 会把部分 errno EBADF/ELOOP归入不存在
网络/FUSE 挂载抖动时会误判这里用 stat 显式区分
"""
try:
path.stat()
except (FileNotFoundError, NotADirectoryError):
return None
except OSError as e:
raise StorageQueryError(f"【本地】读取文件状态失败: {path} - {e}") from e
try:
return self.get_item(path)
except OSError as e:
raise StorageQueryError(f"【本地】读取文件信息失败: {path} - {e}") from e
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取文件详情
+46 -23
View File
@@ -17,6 +17,7 @@ from app.core.config import settings, global_vars
from app.log import logger
from app.modules.filemanager import StorageBase
from app.modules.filemanager.storages import transfer_process
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.utils.singleton import WeakSingleton
from app.utils.string import StringUtils
@@ -906,38 +907,60 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
return True
return False
def __get_info_item(self, path: Path) -> Optional[schemas.FileItem]:
"""
查询指定路径的文件/目录项无法确认状态时抛出 StorageQueryError
接口业务码 20004记录不存在 0 一样视为确认结果其余错误
网络失败限流重试用尽未知业务错误均无法确认目标状态
"""
resp = self._request_api(
"POST",
"/open/folder/get_info",
data={"path": path.as_posix()},
no_error_log=True,
)
if resp is None:
raise StorageQueryError(f"【115】无法确认文件状态(请求失败或接口错误): {path}")
data = resp.get("data") if isinstance(resp, dict) else None
if not data or not data.get("file_id"):
# code 20004(记录不存在)等场景,确认目标不存在
return None
return schemas.FileItem(
storage=self.schema.value,
fileid=str(data["file_id"]),
path=path.as_posix() + ("/" if data["file_category"] == "0" else ""),
type="file" if data["file_category"] == "1" else "dir",
name=data["file_name"],
basename=Path(data["file_name"]).stem,
extension=Path(data["file_name"]).suffix[1:]
if data["file_category"] == "1"
else None,
pickcode=data["pick_code"],
size=data["size_byte"] if data["file_category"] == "1" else None,
modify_time=data["utime"],
)
def get_item(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件/目录项
"""
try:
resp = self._request_api(
"POST",
"/open/folder/get_info",
"data",
data={"path": path.as_posix()},
no_error_log=True,
)
if not resp:
return None
return schemas.FileItem(
storage=self.schema.value,
fileid=str(resp["file_id"]),
path=path.as_posix() + ("/" if resp["file_category"] == "0" else ""),
type="file" if resp["file_category"] == "1" else "dir",
name=resp["file_name"],
basename=Path(resp["file_name"]).stem,
extension=Path(resp["file_name"]).suffix[1:]
if resp["file_category"] == "1"
else None,
pickcode=resp["pick_code"],
size=resp["size_byte"] if resp["file_category"] == "1" else None,
modify_time=resp["utime"],
)
return self.__get_info_item(path)
except Exception as e:
logger.debug(f"【115】获取文件信息失败: {str(e)}")
return None
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件/目录项无法确认状态时抛出 StorageQueryError
"""
try:
return self.__get_info_item(path)
except StorageQueryError:
raise
except Exception as e:
raise StorageQueryError(f"【115】查询文件信息失败: {path} - {e}") from e
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取指定路径的文件夹如不存在则创建
+18 -2
View File
@@ -23,6 +23,7 @@ from app.schemas import (
TransferRenameBuildEventData,
TransferRenameEventData,
)
from app.schemas.exception import StorageQueryError
from app.schemas.types import MediaType, ChainEventType
from app.utils.system import SystemUtils
@@ -405,8 +406,23 @@ class TransHandler:
# 判断是否要覆盖,附加文件强制覆盖
overflag = False
if not __is_extra_file(fileitem):
# 目标文件
target_item = target_oper.get_item(new_file)
# 目标文件(严格查询:无法确认状态时拒绝覆盖,避免已有文件被误覆盖)
try:
target_item = target_oper.get_item_strict(new_file)
except StorageQueryError as query_err:
errmsg = f"无法确认目标文件状态,已跳过整理以避免误覆盖:{new_file} - {query_err}"
logger.warn(errmsg)
self.__update_result(
result=result,
success=False,
message=errmsg,
fileitem=fileitem,
target_diritem=target_diritem,
fail_list=[fileitem.path],
transfer_type=transfer_type,
need_notify=need_notify,
)
return result
if target_item:
# 目标文件已存在
target_file = new_file
+39 -1
View File
@@ -5,6 +5,7 @@ import time
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union
from urllib.parse import quote
from app.core.config import settings
from app.log import logger
@@ -108,6 +109,7 @@ class Api:
"_host",
"_token",
"_apikey",
"_access_code",
"_api_path",
"_request_utils",
"_version",
@@ -130,18 +132,51 @@ class Api:
def version(self) -> Optional[Version]:
return self._version
def __init__(self, host: str, apikey: str):
@property
def cookies(self) -> dict:
"""
当前会话的Cookies开启访问码后包含访问码校验凭证
"""
return self._session.cookies.get_dict()
def __init__(self, host: str, apikey: str, access_code: Optional[str] = None):
"""
:param host: 飞牛服务端地址如http://127.0.0.1:5666/v
:param access_code: 访问码未开启时为空
"""
self._api_path = "/api/v1"
self._host = host.rstrip("/")
self._apikey = apikey
self._access_code = access_code
self._token: Optional[str] = None
self._version: Optional[Version] = None
self._session = requests.Session()
self._request_utils = RequestUtils(session=self._session, timeout=10)
def verify_access_code(self) -> bool:
"""
校验访问码通过后会话获得访问凭证否则无法访问登录页和各应用接口
:return: 未配置访问码或校验通过返回True
"""
if not self._access_code:
return True
# 访问码校验地址位于设备根路径,不在/v下
root = self._host[: -len("/v")] if self._host.endswith("/v") else self._host
url = f"{root}/c/{quote(self._access_code, safe='')}"
res = self._request_utils.get_res(url, allow_redirects=True)
if res is None:
logger.error(f"校验飞牛访问码失败,无法访问 {url}")
return False
if res.status_code == 404:
# 访问码错误或校验失败时返回404
logger.error("飞牛访问码校验失败,请检查访问码是否正确")
return False
if not res.ok:
logger.error(f"飞牛访问码校验失败,状态码:{res.status_code}")
return False
return True
def sys_version(self) -> Optional[Version]:
"""
飞牛影视版本号
@@ -161,6 +196,9 @@ class Api:
:return: 成功返回token 否则返回None
"""
# 开启访问码后需先通过访问码校验,否则无法访问登录接口
if not self.verify_access_code():
return None
if (
res := self.request(
"/login",
+21 -8
View File
@@ -12,6 +12,7 @@ from app.utils.url import UrlUtils
class TrimeMedia:
_username: Optional[str] = None
_password: Optional[str] = None
_access_code: Optional[str] = None
_userinfo: Optional[fnapi.User] = None
_host: Optional[str] = None
@@ -28,6 +29,7 @@ class TrimeMedia:
host: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
access_code: Optional[str] = None,
play_host: Optional[str] = None,
sync_libraries: Optional[list] = None,
**kwargs,
@@ -37,13 +39,14 @@ class TrimeMedia:
return
self._username = username
self._password = password
self._access_code = access_code
self._host = host
self._sync_libraries = sync_libraries or []
if not self.reconnect():
logger.error(f"请检查服务端地址 {host}")
return
if result := self.__create_api(play_host):
if result := self.__create_api(play_host, access_code):
self._playhost = result.api.host
result.api.close()
elif play_host:
@@ -69,11 +72,14 @@ class TrimeMedia:
version: fnapi.Version
@staticmethod
def __create_api(host: Optional[str]) -> Optional["TrimeMedia._ApiCreateResult"]:
def __create_api(
host: Optional[str], access_code: Optional[str] = None
) -> Optional["TrimeMedia._ApiCreateResult"]:
"""
创建一个飞牛API
:param host: 服务端地址
:param access_code: 访问码未开启时为空
:return: 如果地址无效不可访问则返回None
"""
@@ -85,16 +91,19 @@ class TrimeMedia:
if not host.endswith("/v"):
# 尝试补上结尾的/v 测试能否正常访问
res = TrimeMedia._ApiCreateResult()
res.api = fnapi.Api(host + "/v", api_key)
if fnver := res.api.sys_version():
res.api = fnapi.Api(host + "/v", api_key, access_code)
# 开启访问码后,需先校验才能访问各应用接口
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
res.version = fnver
return res
res.api.close()
# 测试用户配置的地址
res = TrimeMedia._ApiCreateResult()
res.api = fnapi.Api(host, api_key)
if fnver := res.api.sys_version():
res.api = fnapi.Api(host, api_key, access_code)
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
res.version = fnver
return res
res.api.close()
return None
def close(self):
@@ -130,7 +139,7 @@ class TrimeMedia:
if not self.is_configured():
return False
self.disconnect()
if result := self.__create_api(self._host):
if result := self.__create_api(self._host, self._access_code):
self._api = result.api
self._version = result.version
# 版本号:0.8.53, 服务版本:0.8.23
@@ -683,4 +692,8 @@ class TrimeMedia:
image_url, [self._api.host], strict=True
):
return None
return {"Trim-MC-token": self._api.token}
cookies = {"Trim-MC-token": self._api.token}
if self._access_code:
# 开启访问码后,图片请求也需要携带访问码校验凭证
cookies.update(self._api.cookies)
return cookies
-954
View File
@@ -1,954 +0,0 @@
import json
import platform
import re
import threading
import time
import traceback
from dataclasses import dataclass
from pathlib import Path
from threading import Lock
from typing import Any, Optional, Dict, List
from apscheduler.schedulers.background import BackgroundScheduler
from watchfiles import Change, DefaultFilter, watch
from app.chain import ChainBase
from app.chain.storage import StorageChain
from app.chain.transfer import TransferChain
from app.core.cache import TTLCache, FileCache
from app.core.config import settings
from app.db.transferhistory_oper import TransferHistoryOper
from app.helper.directory import DirectoryHelper
from app.helper.message import MessageHelper
from app.log import logger
from app.schemas import FileItem
from app.schemas.types import SystemConfigKey
from app.utils.mixins import ConfigReloadMixin
from app.utils.singleton import SingletonClass
from app.utils.system import SystemUtils
lock = Lock()
snapshot_lock = Lock()
class MonitorChain(ChainBase):
pass
@dataclass(frozen=True)
class DirectoryChangeEvent:
"""
目录文件变化事件隔离底层 watchfiles 事件结构
"""
change_type: Change
src_path: str
is_directory: bool
class LocalDirectoryWatcher:
"""
基于 watchfiles 的本地目录监控线程
"""
_HANDLE_CHANGES = {Change.added, Change.modified}
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None):
"""
初始化本地目录监控
:param mon_path: 监控目录
:param callback: 目录变化回调对象
:param force_polling: 是否强制使用轮询模式None 表示由 watchfiles 自动选择
"""
self._watch_path = mon_path
self._callback = callback
self._force_polling = force_polling
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
self._watch_filter = DefaultFilter()
@property
def watch_path(self) -> Path:
"""
获取监控目录
:return: 监控目录
"""
return self._watch_path
def start(self):
"""
启动本地目录监控线程
"""
if not self._watch_path.exists():
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
if not self._watch_path.is_dir():
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
if self.is_alive():
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
return
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run,
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
daemon=True
)
self._thread.start()
def stop(self):
"""
请求停止本地目录监控线程
"""
self._stop_event.set()
def join(self, timeout: Optional[float] = None):
"""
等待本地目录监控线程退出
:param timeout: 最长等待秒数
"""
if self._thread:
self._thread.join(timeout=timeout)
def is_alive(self) -> bool:
"""
判断监控线程是否仍在运行
:return: 线程存活状态
"""
return bool(self._thread and self._thread.is_alive())
def _run(self):
"""
运行 watchfiles 主循环并在快速模式不可用时回退到轮询
"""
try:
self._run_watch(force_polling=self._force_polling)
except Exception as err:
if self._stop_event.is_set():
return
if self._force_polling is True:
logger.error(f"本地目录监控发生错误: {self._watch_path} - {err}")
logger.debug(traceback.format_exc())
return
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
try:
self._run_watch(force_polling=True)
except Exception as fallback_err:
if not self._stop_event.is_set():
logger.error(f"兼容模式监控 {self._watch_path} 仍然失败: {fallback_err}")
logger.debug(traceback.format_exc())
def _run_watch(self, force_polling: Optional[bool]):
"""
执行一次 watchfiles 监控循环
:param force_polling: 是否强制轮询
"""
for changes in watch(
str(self._watch_path),
watch_filter=self._watch_filter,
stop_event=self._stop_event,
rust_timeout=1000,
yield_on_timeout=True,
force_polling=force_polling,
recursive=True,
ignore_permission_denied=True):
if self._stop_event.is_set():
break
if not changes:
continue
self._handle_changes(changes)
def _handle_changes(self, changes: set[tuple[Change, str]]):
"""
watchfiles 原始变更转换为目录监控事件
:param changes: watchfiles 返回的变更集合
"""
changes = self._expand_added_directories(changes)
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
if change_type not in self._HANDLE_CHANGES:
continue
event_path = Path(path_str)
event = self._build_event(change_type=change_type, event_path=event_path)
if not event or event.is_directory:
continue
file_size = self._get_file_size(event_path)
if file_size is None:
continue
text = self._change_text(change_type)
try:
self._callback.event_handler(
event=event,
text=text,
event_path=path_str,
file_size=file_size
)
except Exception as err:
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
"""
将整体移入监控范围的新增目录展开为内部文件事件
:param changes: watchfiles 返回的变更集合
:return: 包含目录内新增文件的变更集合
"""
expanded_changes = set(changes)
for change_type, path_str in changes:
if change_type != Change.added:
continue
event_path = Path(path_str)
try:
if not event_path.is_dir():
continue
for nested_path in event_path.rglob("*"):
if not nested_path.is_file():
continue
nested_path_str = nested_path.as_posix()
if self._watch_filter(Change.added, nested_path_str):
expanded_changes.add((Change.added, nested_path_str))
except OSError as err:
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
return expanded_changes
@staticmethod
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
"""
构建目录变化事件路径已不存在时忽略
:param change_type: watchfiles 变化类型
:param event_path: 变化路径
:return: 目录变化事件
"""
try:
is_directory = event_path.is_dir()
except OSError as err:
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
return None
if not event_path.exists():
return None
return DirectoryChangeEvent(
change_type=change_type,
src_path=event_path.as_posix(),
is_directory=is_directory
)
@staticmethod
def _get_file_size(event_path: Path) -> Optional[int]:
"""
读取事件文件大小文件已消失时返回 None
:param event_path: 事件文件路径
:return: 文件大小
"""
try:
return event_path.stat().st_size
except OSError as err:
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
return None
@staticmethod
def _change_text(change_type: Change) -> str:
"""
转换 watchfiles 事件类型为日志文案
:param change_type: watchfiles 变化类型
:return: 事件描述
"""
if change_type == Change.modified:
return "修改"
return "新增"
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
目录监控处理链单例模式
"""
CONFIG_WATCH = {SystemConfigKey.Directories.value}
def __init__(self):
super().__init__()
# 本地目录监控服务
self._watchers = []
# 定时服务
self._scheduler = None
# 存储过照间隔(分钟)
self._snapshot_interval = 5
# TTL缓存,10秒钟有效
self._cache = TTLCache(region="monitor", maxsize=1024, ttl=10)
# 快照文件缓存
self._snapshot_cache = FileCache(base=settings.CACHE_PATH / "snapshots")
# 监控的文件扩展名
self.all_exts = settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT
# 启动目录监控和文件整理
self.init()
def on_config_changed(self):
self.init()
def get_reload_name(self):
return "目录监控"
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
last_snapshot_time: Optional[float] = None):
"""
保存快照到文件缓存
:param storage: 存储名称
:param snapshot: 快照数据
:param last_snapshot_time: 上次快照时间戳
:param file_count: 文件数量用于调整监控间隔
"""
try:
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
if snapshot_time is None:
snapshot_time = last_snapshot_time or time.time()
snapshot_data = {
'timestamp': snapshot_time,
'file_count': file_count,
'snapshot': snapshot
}
# 使用FileCache保存快照数据
cache_key = f"{storage}_snapshot"
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
self._snapshot_cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
logger.debug(f"快照已保存到缓存: {storage}")
except Exception as e:
logger.error(f"保存快照失败: {e}")
def reset_snapshot(self, storage: str) -> bool:
"""
重置快照强制下次扫描时重新建立基准
:param storage: 存储名称
:return: 是否成功
"""
try:
cache_key = f"{storage}_snapshot"
if self._snapshot_cache.exists(cache_key, region="snapshots"):
self._snapshot_cache.delete(cache_key, region="snapshots")
logger.info(f"快照已重置: {storage}")
return True
logger.debug(f"快照文件不存在,无需重置: {storage}")
return True
except Exception as e:
logger.error(f"重置快照失败: {storage} - {e}")
return False
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
"""
强制全量扫描并处理所有文件包括已存在的文件
:param storage: 存储名称
:param mon_path: 监控路径
:return: 是否成功
"""
try:
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
# 生成快照
new_snapshot = StorageChain().snapshot_storage(
storage=storage,
path=mon_path,
last_snapshot_time=0 # 全量扫描,不使用增量
)
if new_snapshot is None:
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
return False
file_count = len(new_snapshot)
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
# 处理所有文件
processed_count = 0
for file_path, file_info in new_snapshot.items():
try:
if not self.__is_transfer_candidate_path(Path(file_path)):
continue
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self.__handle_file(storage=storage, event_path=Path(file_path), file_size=file_size):
processed_count += 1
except Exception as e:
logger.error(f"处理文件 {file_path} 失败: {e}")
continue
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
# 保存快照
self.save_snapshot(storage, new_snapshot, file_count)
return True
except Exception as e:
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
return False
def load_snapshot(self, storage: str) -> Optional[Dict]:
"""
从文件缓存加载快照
:param storage: 存储名称
:return: 快照数据或None
"""
try:
cache_key = f"{storage}_snapshot"
snapshot_data = self._snapshot_cache.get(cache_key, region="snapshots")
if snapshot_data:
data = json.loads(snapshot_data.decode('utf-8'))
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
return data
logger.debug(f"快照文件不存在: {storage}")
return None
except Exception as e:
logger.error(f"加载快照失败: {e}")
return None
@staticmethod
def adjust_monitor_interval(file_count: int) -> int:
"""
根据文件数量动态调整监控间隔
:param file_count: 文件数量
:return: 监控间隔分钟
"""
if file_count < 100:
return 5 # 5分钟
elif file_count < 500:
return 10 # 10分钟
elif file_count < 1000:
return 15 # 15分钟
else:
return 30 # 30分钟
@staticmethod
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
"""
比对快照找出变化的文件只处理新增和修改不处理删除
:param old_snapshot: 旧快照
:param new_snapshot: 新快照
:return: 变化信息
"""
changes = {
'added': [],
'modified': []
}
old_files = set(old_snapshot.keys())
new_files = set(new_snapshot.keys())
# 新增文件
changes['added'] = list(new_files - old_files)
# 修改文件(大小或时间变化)
for file_path in old_files & new_files:
old_info = old_snapshot[file_path]
new_info = new_snapshot[file_path]
# 检查文件大小变化
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
# 检查修改时间变化(如果有的话)
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
if old_size != new_size or (old_time and new_time and old_time != new_time):
changes['modified'].append(file_path)
return changes
@staticmethod
def __is_bluray_sub(_path: Path) -> bool:
"""
判断是否蓝光原盘目录内的媒体流文件
"""
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
@staticmethod
def __get_bluray_dir(_path: Path) -> Optional[Path]:
"""
获取蓝光原盘BDMV目录的上级目录
"""
for p in _path.parents:
if p.name == "BDMV":
return p.parent
return None
@staticmethod
def __has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
"""
判断路径后缀是否命中给定扩展名列表
"""
if not file_path.suffix:
return False
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
def __is_transfer_candidate_path(self, file_path: Path) -> bool:
"""
判断监控事件路径是否需要进入整理链
"""
if self.__has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
return False
return self.__has_suffix_in(file_path, self.all_exts)
@staticmethod
def __build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
"""
生成整理记录使用的源路径
"""
if is_bluray_folder:
return f"{event_path.as_posix()}/"
return event_path.as_posix()
@staticmethod
def __has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
"""
判断源文件是否已经存在整理记录
"""
try:
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
except Exception as err:
logger.error(f"查询整理历史失败: {src_path} - {err}")
return None
@staticmethod
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
"""
统计目录下的文件数量用于检测是否超过系统限制
:param directory: 目录路径
:param max_check: 最大检查数量避免长时间阻塞
:return: 文件数量
"""
try:
count = 0
import os
for root, dirs, files in os.walk(str(directory)):
count += len(files)
if count > max_check:
return count
return count
except Exception as err:
logger.debug(f"统计目录文件数量失败: {err}")
return 0
@staticmethod
def check_system_limits() -> Dict[str, Any]:
"""
检查系统限制
:return: 系统限制信息
"""
limits = {
'max_user_watches': 0,
'max_user_instances': 0,
'current_watches': 0,
'warnings': []
}
try:
system = platform.system()
if system == 'Linux':
# 检查 inotify 限制
try:
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
limits['max_user_watches'] = int(f.read().strip())
except Exception as e:
logger.debug(f"读取 inotify 限制失败: {e}")
limits['max_user_watches'] = 8192 # 默认值
try:
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
limits['max_user_instances'] = int(f.read().strip())
except Exception as e:
logger.debug(f"读取 inotify 实例限制失败: {e}")
# 检查当前使用的watches
try:
import subprocess
result = subprocess.run(['find', '/proc/*/fd', '-lname', 'anon_inode:inotify', '-printf', '%h\n'],
capture_output=True, text=True, timeout=5)
if result.returncode == 0:
limits['current_watches'] = len(result.stdout.strip().split('\n'))
except Exception as e:
logger.debug(f"检查当前 inotify 使用失败: {e}")
except Exception as e:
limits['warnings'].append(f"检查系统限制时出错: {e}")
return limits
@staticmethod
def get_system_optimization_tips() -> List[str]:
"""
获取系统优化建议
:return: 优化建议列表
"""
tips = []
system = platform.system()
if system == 'Linux':
tips.extend([
"增加 inotify 监控数量限制:",
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
"sudo sysctl -p",
"",
"如果在Docker中运行,请在宿主机上执行以上命令"
])
elif system == 'Darwin':
tips.extend([
"macOS 系统优化建议:",
"sudo sysctl kern.maxfiles=65536",
"sudo sysctl kern.maxfilesperproc=32768",
"ulimit -n 32768"
])
elif system == 'Windows':
tips.extend([
"Windows 系统优化建议:",
"1. 关闭不必要的实时保护软件对监控目录的扫描",
"2. 将监控目录添加到Windows Defender排除列表",
"3. 确保有足够的可用内存"
])
return tips
@staticmethod
def should_use_polling(directory: Path, monitor_mode: str,
file_count: int, limits: dict) -> tuple[bool, str]:
"""
判断是否应该使用轮询模式
:param directory: 监控目录
:param monitor_mode: 配置的监控模式
:param file_count: 目录文件数量
:param limits: 系统限制信息
:return: (是否使用轮询, 原因)
"""
if monitor_mode == "compatibility":
return True, "用户配置为兼容模式"
# 检查网络文件系统
if SystemUtils.is_network_filesystem(directory):
return True, "检测到网络文件系统,建议使用兼容模式"
max_watches = limits.get('max_user_watches')
if max_watches and file_count > max_watches * 0.8:
return True, f"目录文件数量({file_count})接近系统限制({max_watches})"
return False, "使用快速模式"
def init(self):
"""
启动监控
"""
# 停止现有任务
self.stop()
# 读取目录配置
monitor_dirs = DirectoryHelper().get_download_dirs()
if not monitor_dirs:
logger.info("未找到任何目录监控配置")
return
# 按下载目录去重
monitor_dirs = list({f"{d.storage}_{d.download_path}": d for d in monitor_dirs}.values())
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
# 启动定时服务进程
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
messagehelper = MessageHelper()
mon_storages = {}
for mon_dir in monitor_dirs:
if not mon_dir.library_path:
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
continue
if mon_dir.monitor_type != "monitor":
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
continue
# 检查媒体库目录是不是下载目录的子目录
mon_path = Path(mon_dir.download_path)
target_path = Path(mon_dir.library_path)
if target_path.is_relative_to(mon_path):
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
continue
# 启动监控
if mon_dir.storage == "local":
# 本地目录监控
logger.info(f"正在启动本地目录监控: {mon_path}")
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
try:
# 统计文件数量并给出提示
file_count = self.count_directory_files(mon_path)
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
# 检查系统限制
limits = self.check_system_limits()
# 检查是否需要使用轮询模式
use_polling, reason = self.should_use_polling(mon_path,
monitor_mode=mon_dir.monitor_mode,
file_count=file_count,
limits=limits)
logger.info(f"监控模式决策: {reason}")
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
logger.info(f"使用{mode_name}监控 {mon_path}")
if not use_polling:
if limits['warnings']:
for warning in limits['warnings']:
logger.warn(f"系统限制警告: {warning}")
if limits['max_user_watches'] > 0:
usage_percent = (file_count / limits['max_user_watches']) * 100
logger.info(
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
watcher = LocalDirectoryWatcher(
mon_path=mon_path,
callback=self,
force_polling=True if use_polling else None
)
self._watchers.append(watcher)
watcher.start()
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
except Exception as e:
err_msg = str(e)
logger.error(f"启动本地目录监控失败: {mon_path}")
logger.error(f"错误详情: {err_msg}")
if "inotify" in err_msg.lower():
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
logger.error("解决方案:")
tips = self.get_system_optimization_tips()
for tip in tips:
logger.error(f" {tip}")
logger.error("执行上述命令后重启 MoviePilot")
elif "permission" in err_msg.lower():
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
else:
logger.error("建议尝试使用兼容模式进行监控")
messagehelper.put(f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}", title="目录监控")
else:
if not mon_storages.get(mon_dir.storage):
mon_storages[mon_dir.storage] = []
mon_storages[mon_dir.storage].append(mon_path)
for storage, paths in mon_storages.items():
# 远程目录监控 - 使用智能间隔
# 先尝试加载已有快照获取文件数量
snapshot_data = self.load_snapshot(storage)
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
interval = self.adjust_monitor_interval(file_count)
for path in paths:
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
self._scheduler.add_job(
self.polling_observer,
'interval',
minutes=interval,
kwargs={
'storage': storage,
'mon_paths': paths
},
id=f"monitor_{storage}",
replace_existing=True
)
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
# 启动定时服务
if self._scheduler.get_jobs():
self._scheduler.print_jobs()
self._scheduler.start()
logger.info("定时监控服务已启动")
# 输出监控总结
local_count = len([d for d in monitor_dirs if d.storage == "local" and d.monitor_type == "monitor"])
remote_count = len([d for d in monitor_dirs if d.storage != "local" and d.monitor_type == "monitor"])
logger.info(f"目录监控启动完成: 本地监控 {local_count} 个,远程监控 {remote_count}")
def polling_observer(self, storage: str, mon_paths: List[Path]):
"""
轮询监控改进版
"""
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
with snapshot_lock:
try:
# 加载上次快照数据
old_snapshot_data = self.load_snapshot(storage)
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
# 判断是否为首次快照:检查快照文件是否存在且有效
is_first_snapshot = old_snapshot_data is None
new_snapshot = {}
for mon_path in mon_paths:
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
# 生成新快照(增量模式)
snapshot = StorageChain().snapshot_storage(
storage=storage,
path=mon_path,
last_snapshot_time=last_snapshot_time
)
if snapshot is None:
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
continue
new_snapshot.update(snapshot)
file_count = len(snapshot)
logger.info(f"{storage}:{mon_path} 快照完成,发现 {file_count} 个文件")
file_count = len(new_snapshot)
if not is_first_snapshot:
# 比较快照找出变化
changes = self.compare_snapshots(old_snapshot, new_snapshot)
added_files = [
file_path
for file_path in changes['added']
if self.__is_transfer_candidate_path(Path(file_path))
]
modified_files = [
file_path
for file_path in changes['modified']
if self.__is_transfer_candidate_path(Path(file_path))
]
# 处理新增文件
handled_added_count = 0
for new_file in added_files:
file_info = new_snapshot.get(new_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self.__handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
handled_added_count += 1
# 处理修改文件
handled_modified_count = 0
for modified_file in modified_files:
file_info = new_snapshot.get(modified_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self.__handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
handled_modified_count += 1
if handled_added_count or handled_modified_count:
logger.info(
f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
else:
logger.debug(f"{storage} 无文件变化")
else:
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
# 保存新快照
self.save_snapshot(storage, new_snapshot, file_count, last_snapshot_time)
# 动态调整监控间隔
new_interval = self.adjust_monitor_interval(file_count)
current_job = self._scheduler.get_job(f"monitor_{storage}")
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
# 重新安排任务
self._scheduler.modify_job(
f"monitor_{storage}",
trigger='interval',
minutes=new_interval
)
logger.info(f"{storage}:{monitor_scope} 监控间隔已调整为 {new_interval} 分钟")
except Exception as e:
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}")
logger.debug(traceback.format_exc())
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
"""
处理文件变化
:param event: 事件
:param text: 事件描述
:param event_path: 事件文件路径
:param file_size: 文件大小
"""
if not event.is_directory:
if not self.__is_transfer_candidate_path(Path(event_path)):
return
# 整理文件
self.__handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
def __handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
"""
整理一个文件
:param storage: 存储
:param event_path: 事件文件路径
:param file_size: 文件大小
:return: 是否进入整理链
"""
# 全程加锁
with lock:
is_bluray_folder = False
# 蓝光原盘文件处理
if self.__is_bluray_sub(event_path):
event_path = self.__get_bluray_dir(event_path)
if not event_path:
return False
is_bluray_folder = True
elif not self.__is_transfer_candidate_path(event_path):
return False
# TTL缓存控重
if self._cache.get(str(event_path)):
return False
self._cache[str(event_path)] = True
src_path = self.__build_transfer_src_path(
event_path=event_path,
is_bluray_folder=is_bluray_folder,
)
has_transfer_history = self.__has_transfer_history(
storage=storage,
src_path=src_path,
)
if has_transfer_history is not False:
return False
try:
if is_bluray_folder:
logger.info(f"开始整理蓝光原盘: {event_path}")
else:
logger.info(f"开始整理文件: {event_path}")
# 开始整理
TransferChain().do_transfer(
fileitem=FileItem(
storage=storage,
path=src_path,
type="file" if not is_bluray_folder else "dir",
name=event_path.name,
basename=event_path.stem,
extension=event_path.suffix[1:],
size=file_size
)
)
return True
except Exception as e:
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
return False
def stop(self):
"""
退出监控
"""
if self._watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in self._watchers:
try:
watcher.stop()
watcher.join(timeout=5)
if watcher.is_alive():
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
else:
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
except Exception as e:
logger.error(f"停止目录监控服务出现了错误:{e}")
self._watchers = []
logger.info("本地目录监控服务已停止")
if self._scheduler:
self._scheduler.remove_all_jobs()
if self._scheduler.running:
try:
self._scheduler.shutdown()
logger.info("定时监控服务已停止")
except Exception as e:
logger.error(f"停止定时服务出现了错误:{e}")
self._scheduler = None
if self._cache:
self._cache.close()
if self._snapshot_cache:
self._snapshot_cache.close()
+14
View File
@@ -0,0 +1,14 @@
"""
目录监控包
- watcher.py 本地目录监控线程watchfiles
- syslimits.py 系统限制探测与监控模式决策
- snapshot.py 远程快照存取与比对
- dispatcher.py 监控事件到整理链的分发
- poller.py 远程目录轮询监控
- monitor.py Monitor 门面装配生命周期与健康检查
"""
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
from app.monitor.monitor import Monitor
__all__ = ["DirectoryChangeEvent", "LocalDirectoryWatcher", "Monitor"]
+210
View File
@@ -0,0 +1,210 @@
import re
import traceback
from pathlib import Path
from threading import Lock
from typing import Any, Dict, List, Optional
from app.chain.transfer import TransferChain
from app.core.cache import TTLCache
from app.core.config import settings
from app.db.transferhistory_oper import TransferHistoryOper
from app.log import logger
from app.schemas import FileItem
class TransferDispatcher:
"""
将监控事件分发到整理链候选判定TTL 去重整理历史查重与整理触发
"""
# 历史查询失败待重试队列上限,防止长时间故障期间无限增长
MAX_PENDING_RETRIES = 1000
# 单个文件的最大重试次数(按健康检查周期计,60 次约 1 小时)
MAX_RETRY_ATTEMPTS = 60
def __init__(self, all_exts: Optional[List[str]] = None, cache: Optional[Any] = None):
"""
初始化整理分发器
:param all_exts: 监控的文件扩展名默认取系统配置
:param cache: 去重缓存默认使用 10 TTL 缓存
"""
self.all_exts = all_exts if all_exts is not None else (
settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT)
self._cache = cache if cache is not None else TTLCache(region="monitor", maxsize=1024, ttl=10)
self._lock = Lock()
# 历史查询失败待重试的文件
self._pending_retries: Dict[str, Dict[str, Any]] = {}
self._pending_guard = Lock()
@staticmethod
def _is_bluray_sub(_path: Path) -> bool:
"""
判断是否蓝光原盘目录内的媒体流文件
"""
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
@staticmethod
def _get_bluray_dir(_path: Path) -> Optional[Path]:
"""
获取蓝光原盘BDMV目录的上级目录
"""
for p in _path.parents:
if p.name == "BDMV":
return p.parent
return None
@staticmethod
def _has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
"""
判断路径后缀是否命中给定扩展名列表
"""
if not file_path.suffix:
return False
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
def is_transfer_candidate_path(self, file_path: Path) -> bool:
"""
判断监控事件路径是否需要进入整理链
"""
if self._has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
return False
return self._has_suffix_in(file_path, self.all_exts)
@staticmethod
def _build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
"""
生成整理记录使用的源路径
"""
if is_bluray_folder:
return f"{event_path.as_posix()}/"
return event_path.as_posix()
@staticmethod
def _has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
"""
判断源文件是否已经存在整理记录
:return: True/False 查询成功None 查询失败
"""
try:
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
except Exception as err:
logger.error(f"查询整理历史失败: {src_path} - {err}")
return None
@staticmethod
def _pending_key(storage: str, event_path: Path) -> str:
"""
生成待重试文件的唯一键
"""
return f"{storage}:{Path(event_path).as_posix()}"
def _register_pending(self, storage: str, event_path: Path, file_size: float = None):
"""
登记历史查询失败的文件待重试重复失败累计次数超限后放弃
:param storage: 存储
:param event_path: 原始事件路径
:param file_size: 文件大小
"""
key = self._pending_key(storage, event_path)
with self._pending_guard:
entry = self._pending_retries.get(key)
if entry:
entry["attempts"] += 1
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
self._pending_retries.pop(key, None)
logger.error(f"整理历史查询持续失败,已放弃重试: {key}")
return
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
logger.error(f"整理重试队列已满,丢弃: {key}")
return
self._pending_retries[key] = {
"storage": storage,
"event_path": event_path,
"file_size": file_size,
"attempts": 1
}
logger.warn(f"整理历史查询失败,已登记待重试: {key}")
def _discard_pending(self, storage: str, event_path: Path):
"""
历史查询已得到确定结果移除待重试登记
:param storage: 存储
:param event_path: 原始事件路径
"""
with self._pending_guard:
self._pending_retries.pop(self._pending_key(storage, event_path), None)
def retry_pending(self):
"""
重试历史查询失败的文件由健康检查周期驱动
成功或得到确定结果的条目在 handle_file 内部自动移除
"""
with self._pending_guard:
items = list(self._pending_retries.values())
for item in items:
logger.info(f"重试整理: {item['storage']}:{item['event_path']}")
self.handle_file(storage=item["storage"], event_path=item["event_path"],
file_size=item["file_size"])
def handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
"""
整理一个文件
:param storage: 存储
:param event_path: 事件文件路径
:param file_size: 文件大小
:return: 是否进入整理链
"""
with self._lock:
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
origin_path = event_path
is_bluray_folder = False
# 蓝光原盘文件处理
if self._is_bluray_sub(event_path):
event_path = self._get_bluray_dir(event_path)
if not event_path:
return False
is_bluray_folder = True
elif not self.is_transfer_candidate_path(event_path):
return False
# TTL缓存控重
if self._cache.get(str(event_path)):
return False
self._cache[str(event_path)] = True
src_path = self._build_transfer_src_path(
event_path=event_path,
is_bluray_folder=is_bluray_folder,
)
has_transfer_history = self._has_transfer_history(
storage=storage,
src_path=src_path,
)
if has_transfer_history is None:
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
self._register_pending(storage=storage, event_path=origin_path, file_size=file_size)
return False
self._discard_pending(storage=storage, event_path=origin_path)
if has_transfer_history:
return False
try:
if is_bluray_folder:
logger.info(f"开始整理蓝光原盘: {event_path}")
else:
logger.info(f"开始整理文件: {event_path}")
# 开始整理
TransferChain().do_transfer(
fileitem=FileItem(
storage=storage,
path=src_path,
type="file" if not is_bluray_folder else "dir",
name=event_path.name,
basename=event_path.stem,
extension=event_path.suffix[1:],
size=file_size
)
)
return True
except Exception as e:
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
return False
+508
View File
@@ -0,0 +1,508 @@
import traceback
from pathlib import Path
from threading import Lock
from typing import Any, Dict, List, Optional
from apscheduler.schedulers.background import BackgroundScheduler
from app.core.config import settings
from app.helper.directory import DirectoryHelper
from app.helper.message import MessageHelper
from app.log import logger
from app.monitor.dispatcher import TransferDispatcher
from app.monitor.poller import RemotePoller
from app.monitor.snapshot import SnapshotStore
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
from app.monitor.watcher import LocalDirectoryWatcher
from app.schemas.types import SystemConfigKey
from app.utils.mixins import ConfigReloadMixin
from app.utils.singleton import SingletonClass
from app.utils.system import SystemUtils
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
目录监控门面单例模式装配本地/远程监控维护生命周期与健康检查
"""
CONFIG_WATCH = {SystemConfigKey.Directories.value}
# 目录监控健康检查间隔(秒)
WATCHDOG_INTERVAL = 60
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
RECOVERY_STABLE_CYCLES = 5
def __init__(self):
super().__init__()
# 本地目录监控服务
self._watchers = []
# 本地目录监控列表读写锁
self._watcher_lock = Lock()
# 启动失败待重试的本地监控配置
self._pending_locals: List[Dict[str, Any]] = []
# 已告警的监控目录,避免重复推送
self._alerted_paths: set = set()
# 各监控目录已告警过的自动重启次数
self._restart_marks: Dict[str, int] = {}
# 各监控目录连续稳定的健康检查周期数
self._stable_cycles: Dict[str, int] = {}
# 定时服务
self._scheduler = None
# 整理分发器
self._dispatcher = TransferDispatcher()
# 快照存储
self._store = SnapshotStore()
# 远程轮询监控
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
alert_cb=self.__poller_alert)
# 启动目录监控和文件整理
self.init()
def on_config_changed(self):
self.init()
def get_reload_name(self):
return "目录监控"
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
last_snapshot_time: Optional[float] = None):
"""
保存快照到文件缓存
"""
self._store.save(storage, snapshot, file_count=file_count, last_snapshot_time=last_snapshot_time)
def load_snapshot(self, storage: str) -> Optional[Dict]:
"""
从文件缓存加载快照
"""
return self._store.load(storage)
def reset_snapshot(self, storage: str) -> bool:
"""
重置快照强制下次扫描时重新建立基准
"""
return self._store.reset(storage)
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
"""
强制全量扫描并处理所有文件包括已存在的文件
"""
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
@staticmethod
def adjust_monitor_interval(file_count: int) -> int:
"""
根据文件数量动态调整监控间隔
"""
return SnapshotStore.adjust_interval(file_count)
@staticmethod
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
"""
比对快照找出变化的文件
"""
return SnapshotStore.compare(old_snapshot, new_snapshot)
def init(self):
"""
启动监控
"""
# 停止现有任务
self.stop()
# 读取目录配置
monitor_dirs = DirectoryHelper().get_download_dirs()
if not monitor_dirs:
logger.info("未找到任何目录监控配置")
return
messagehelper = MessageHelper()
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
valid_dirs = []
for mon_dir in monitor_dirs:
if not mon_dir.library_path:
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
continue
if mon_dir.monitor_type != "monitor":
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
continue
valid_dirs.append(mon_dir)
deduped: Dict[str, Any] = {}
for mon_dir in valid_dirs:
key = f"{mon_dir.storage}_{mon_dir.download_path}"
if key in deduped:
logger.warn(f"监控配置重复,忽略后一条: {mon_dir.download_path}"
f"(媒体库 {mon_dir.library_path}")
continue
deduped[key] = mon_dir
monitor_dirs = list(deduped.values())
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
# 启动定时服务进程
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
mon_storages: Dict[str, List[Path]] = {}
# 本地监控启动结果计数,用于输出真实的启动总结
local_started = 0
local_failed = 0
for mon_dir in monitor_dirs:
# 检查媒体库目录是不是下载目录的子目录
mon_path = Path(mon_dir.download_path)
target_path = Path(mon_dir.library_path)
if target_path.is_relative_to(mon_path):
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
continue
# 启动监控
if mon_dir.storage == "local":
if self.__start_local_monitor(mon_path=mon_path, monitor_mode=mon_dir.monitor_mode):
local_started += 1
else:
local_failed += 1
else:
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
for storage, paths in mon_storages.items():
# 远程目录监控 - 使用智能间隔
# 先尝试加载已有快照获取文件数量
snapshot_data = self._store.load(storage)
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
interval = SnapshotStore.adjust_interval(file_count)
for path in paths:
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
self._scheduler.add_job(
self.polling_observer,
'interval',
minutes=interval,
kwargs={
'storage': storage,
'mon_paths': paths
},
id=f"monitor_{storage}",
replace_existing=True
)
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
# 监控健康检查:重建异常监控线程、重试启动失败目录、重试历史查询失败的文件
if local_started or local_failed or mon_storages:
self._scheduler.add_job(
self.watchdog,
'interval',
seconds=self.WATCHDOG_INTERVAL,
id="monitor_watchdog",
replace_existing=True
)
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
# 启动定时服务
if self._scheduler.get_jobs():
self._scheduler.print_jobs()
self._scheduler.start()
logger.info("定时监控服务已启动")
# 输出监控总结,报告实际启动成功数而不是配置数
remote_count = sum(len(paths) for paths in mon_storages.values())
summary = f"目录监控启动完成: 本地监控 {local_started} 个成功"
if local_failed:
summary += f"{local_failed} 个失败(将自动退避重试)"
summary += f",远程监控 {remote_count}"
if local_failed:
logger.warn(summary)
else:
logger.info(summary)
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
"""
启动单个本地目录监控失败时登记待重试
:param mon_path: 监控目录
:param monitor_mode: 配置的监控模式
:return: 是否启动成功
"""
logger.info(f"正在启动本地目录监控: {mon_path}")
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
try:
# 检查是否需要使用轮询模式(兼容模式/网络存储不做启动期目录遍历)
use_polling, reason, limits, file_count = decide_monitor_mode(mon_path, monitor_mode)
logger.info(f"监控模式决策: {reason}")
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
logger.info(f"使用{mode_name}监控 {mon_path}")
if file_count is not None:
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
if not use_polling and limits:
if limits['warnings']:
for warning in limits['warnings']:
logger.warn(f"系统限制警告: {warning}")
if limits['max_user_watches'] > 0 and file_count is not None:
usage_percent = (file_count / limits['max_user_watches']) * 100
logger.info(
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
poll_delay_ms = None
if use_polling and SystemUtils.is_network_filesystem(mon_path):
poll_delay_ms = LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
watcher = LocalDirectoryWatcher(
mon_path=mon_path,
callback=self,
force_polling=True if use_polling else None,
poll_delay_ms=poll_delay_ms
)
# 启动成功后再登记,避免失败的监控残留在列表中
watcher.start()
with self._watcher_lock:
self._watchers.append(watcher)
self._pending_locals = [
pending for pending in self._pending_locals
if pending["mon_path"] != mon_path
]
self.__clear_alert(mon_path, f"本地目录监控已恢复: {mon_path} [{mode_name}]")
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
return True
except Exception as e:
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
return False
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
"""
处理本地目录监控启动失败登记待重试并按需告警
:param mon_path: 监控目录
:param monitor_mode: 配置的监控模式
:param err: 启动异常
"""
err_msg = str(err)
logger.error(f"启动本地目录监控失败: {mon_path}")
logger.error(f"错误详情: {err_msg}")
if "inotify" in err_msg.lower():
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
logger.error("解决方案:")
for tip in get_system_optimization_tips():
logger.error(f" {tip}")
logger.error("执行上述命令后重启 MoviePilot")
elif "permission" in err_msg.lower():
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
elif isinstance(err, (FileNotFoundError, NotADirectoryError)):
logger.error("监控目录当前不可用,网络存储/FUSE 挂载可能尚未就绪,将自动重试")
elif monitor_mode != "compatibility":
logger.error("建议尝试使用兼容模式进行监控")
with self._watcher_lock:
if all(pending["mon_path"] != mon_path for pending in self._pending_locals):
self._pending_locals.append({
"mon_path": mon_path,
"monitor_mode": monitor_mode
})
self.__send_alert(mon_path,
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
f"将自动退避重试")
def watchdog(self):
"""
目录监控健康检查重建崩溃或静默失效的监控线程并重试启动失败的监控目录
"""
try:
self.__check_watchers()
self.__retry_pending_locals()
self._dispatcher.retry_pending()
except Exception as e:
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
def __check_watchers(self):
"""
检查本地目录监控线程状态异常时重建
"""
with self._watcher_lock:
watchers = list(self._watchers)
for watcher in watchers:
key = str(watcher.watch_path)
if watcher.is_stalled():
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
elif not watcher.is_alive():
reason = "监控线程已退出"
else:
# 线程已自愈,但崩溃过就要告警,避免自动重启把故障变成新的静默
if watcher.restart_count > self._restart_marks.get(key, 0):
self._restart_marks[key] = watcher.restart_count
self._stable_cycles[key] = 0
self.__send_alert(watcher.watch_path,
f"目录监控发生错误并已自动重启"
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
else:
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
if self._stable_cycles[key] >= self.RECOVERY_STABLE_CYCLES:
self.__clear_alert(watcher.watch_path, f"目录监控已恢复正常: {watcher.watch_path}")
continue
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
self.__send_alert(watcher.watch_path,
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
self.__rebuild_watcher(watcher)
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
"""
重建一个本地目录监控线程
:param watcher: 需要重建的监控
"""
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
watcher.stop()
new_watcher = LocalDirectoryWatcher(
mon_path=watcher.watch_path,
callback=self,
force_polling=watcher.force_polling,
poll_delay_ms=watcher.poll_delay_ms
)
try:
new_watcher.start()
except Exception as e:
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
with self._watcher_lock:
self._watchers = [item for item in self._watchers if item is not watcher]
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
self._pending_locals.append({
"mon_path": watcher.watch_path,
# 重建沿用原监控模式,force_polling 为 True 即兼容模式
"monitor_mode": "compatibility" if watcher.force_polling else "fast"
})
return
with self._watcher_lock:
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
# 新监控的重启计数从零开始,同步重置告警基准
self._restart_marks.pop(str(watcher.watch_path), None)
self._stable_cycles.pop(str(watcher.watch_path), None)
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
def __retry_pending_locals(self):
"""
重试启动失败的本地目录监控给网络存储/FUSE 挂载留出就绪时间
"""
with self._watcher_lock:
pending = list(self._pending_locals)
for item in pending:
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
if item.get("skip_cycles", 0) > 0:
item["skip_cycles"] -= 1
continue
logger.info(f"重试启动本地目录监控: {item['mon_path']}")
if not self.__start_local_monitor(mon_path=item["mon_path"], monitor_mode=item["monitor_mode"]):
item["attempts"] = item.get("attempts", 0) + 1
item["skip_cycles"] = min(item["attempts"], 10)
def __send_alert(self, mon_path: Path, message: str):
"""
推送目录监控异常告警同一目录仅在状态变化时推送一次
:param mon_path: 监控目录
:param message: 告警内容
"""
key = str(mon_path)
with self._watcher_lock:
if key in self._alerted_paths:
return
self._alerted_paths.add(key)
MessageHelper().put(message, title="目录监控")
@staticmethod
def __poller_alert(storage: str, message: str):
"""
远程轮询监控告警回调复用消息渠道推送
:param storage: 存储名称
:param message: 告警内容
"""
logger.warn(f"[{storage}] {message}")
MessageHelper().put(message, title="目录监控")
def __clear_alert(self, mon_path: Path, message: str):
"""
清除目录监控异常告警状态并在此前告警过时推送恢复消息
:param mon_path: 监控目录
:param message: 恢复内容
"""
key = str(mon_path)
with self._watcher_lock:
if key not in self._alerted_paths:
return
self._alerted_paths.discard(key)
logger.info(message)
MessageHelper().put(message, title="目录监控")
def polling_observer(self, storage: str, mon_paths: List[Path]):
"""
轮询监控执行一轮快照并按结果动态调整监控间隔
"""
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
if file_count is None or not self._scheduler:
return
# 动态调整监控间隔
new_interval = SnapshotStore.adjust_interval(file_count)
try:
current_job = self._scheduler.get_job(f"monitor_{storage}")
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
self._scheduler.modify_job(
f"monitor_{storage}",
trigger='interval',
minutes=new_interval
)
logger.info(f"{storage} 监控间隔已调整为 {new_interval} 分钟")
except Exception as e:
logger.error(f"调整监控间隔失败: {storage} - {e}")
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
"""
处理文件变化
:param event: 事件
:param text: 事件描述
:param event_path: 事件文件路径
:param file_size: 文件大小
"""
if event.is_directory:
return
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
return
# 整理文件
self._dispatcher.handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
def stop(self):
"""
退出监控
"""
# 先停定时服务,避免健康检查在停止过程中重建监控线程
if self._scheduler:
self._scheduler.remove_all_jobs()
if self._scheduler.running:
try:
self._scheduler.shutdown()
logger.info("定时监控服务已停止")
except Exception as e:
logger.error(f"停止定时服务出现了错误:{e}")
self._scheduler = None
with self._watcher_lock:
watchers = self._watchers
self._watchers = []
self._pending_locals = []
self._alerted_paths = set()
self._restart_marks = {}
self._stable_cycles = {}
if watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in watchers:
try:
watcher.stop()
watcher.join(timeout=5)
if watcher.is_alive():
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
else:
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
except Exception as e:
logger.error(f"停止目录监控服务出现了错误:{e}")
logger.info("本地目录监控服务已停止")
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
+228
View File
@@ -0,0 +1,228 @@
import traceback
from pathlib import Path
from threading import Lock
from typing import Callable, Dict, List, Optional
from app.chain.storage import StorageChain
from app.log import logger
from app.monitor.dispatcher import TransferDispatcher
from app.monitor.snapshot import SnapshotStore
class RemotePoller:
"""
远程目录轮询监控快照比对并分发变化文件
"""
# 同一存储连续异常达到该次数后推送告警
FAILURE_ALERT_THRESHOLD = 3
def __init__(self, store: SnapshotStore, dispatcher: TransferDispatcher,
alert_cb: Optional[Callable[[str, str], None]] = None):
"""
初始化远程轮询监控
:param store: 快照存储
:param dispatcher: 整理分发器
:param alert_cb: 告警回调 (storage, message)
"""
self._store = store
self._dispatcher = dispatcher
self._alert_cb = alert_cb
# 快照锁按存储隔离,避免一个慢存储阻塞其他存储的轮询
self._locks: Dict[str, Lock] = {}
self._locks_guard = Lock()
# 各存储连续异常次数
self._failure_counts: Dict[str, int] = {}
def _get_lock(self, storage: str) -> Lock:
"""
获取指定存储的快照锁
:param storage: 存储名称
:return: 快照锁
"""
with self._locks_guard:
return self._locks.setdefault(storage, Lock())
def _note_failure(self, storage: str, reason: str):
"""
记录一次轮询异常连续异常达到阈值时推送告警
:param storage: 存储名称
:param reason: 异常原因
"""
count = self._failure_counts.get(storage, 0) + 1
self._failure_counts[storage] = count
logger.warn(f"远程目录监控异常(连续第 {count} 次): {storage} - {reason}")
if count == self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
self._alert_cb(storage,
f"远程目录监控连续 {count} 次异常: {storage}\n原因: {reason}\n将继续按周期重试")
def _note_success(self, storage: str):
"""
记录一次轮询成功此前告警过时推送恢复消息
:param storage: 存储名称
"""
if self._failure_counts.get(storage, 0) >= self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
self._alert_cb(storage, f"远程目录监控已恢复: {storage}")
self._failure_counts[storage] = 0
def poll(self, storage: str, mon_paths: List[Path]) -> Optional[int]:
"""
执行一轮轮询监控
:param storage: 存储名称
:param mon_paths: 监控路径列表
:return: 基线文件数量本轮无有效结果时返回 None
"""
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
with self._get_lock(storage):
try:
# 加载上次快照数据,读取失败不能当作首次快照,否则会丢弃已有基线
old_snapshot_data, load_ok = self._store.load_checked(storage)
if not load_ok:
self._note_failure(storage, "读取快照基线失败,跳过本轮")
return None
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
is_first_snapshot = old_snapshot_data is None
new_snapshot = {}
failed_paths = []
for mon_path in mon_paths:
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
# 生成新快照(增量模式)
snapshot = StorageChain().snapshot_storage(
storage=storage,
path=mon_path,
last_snapshot_time=last_snapshot_time
)
if snapshot is None:
failed_paths.append(str(mon_path))
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
continue
new_snapshot.update(snapshot)
logger.info(f"{storage}:{mon_path} 快照完成,发现 {len(snapshot)} 个文件")
if failed_paths and (is_first_snapshot or len(failed_paths) == len(mon_paths)):
# 首次基线必须完整建立;全部路径失败时本轮没有有效数据,均不落盘
self._note_failure(storage, f"快照失败: {','.join(failed_paths)}")
return None
# 增量快照只包含变化子树,必须与基线合并才是完整视图;
# 直接把增量当基线会导致下一轮把未扫到的旧文件全部误判为新增
merged_snapshot = {**old_snapshot, **new_snapshot}
file_count = len(merged_snapshot)
if not is_first_snapshot:
self._handle_changes(storage, old_snapshot, new_snapshot)
else:
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
# 保存合并后的基线
if not self._store.save(storage, merged_snapshot, file_count, last_snapshot_time):
self._note_failure(storage, "保存快照基线失败")
return None
if failed_paths:
# 部分路径失败:成功路径已合并,失败路径保留旧基线,下轮重试
self._note_failure(storage, f"部分路径快照失败: {','.join(failed_paths)}")
else:
self._note_success(storage)
return file_count
except Exception as e:
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}\n{traceback.format_exc()}")
self._note_failure(storage, str(e))
return None
def _handle_changes(self, storage: str, old_snapshot: dict, new_snapshot: dict):
"""
比对快照并把变化文件送入整理链
:param storage: 存储名称
:param old_snapshot: 旧基线
:param new_snapshot: 本轮增量快照
"""
changes = SnapshotStore.compare(old_snapshot, new_snapshot)
added_files = [
file_path
for file_path in changes['added']
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
]
modified_files = [
file_path
for file_path in changes['modified']
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
]
# 处理新增文件
handled_added_count = 0
for new_file in added_files:
file_info = new_snapshot.get(new_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
handled_added_count += 1
# 处理修改文件
handled_modified_count = 0
for modified_file in modified_files:
file_info = new_snapshot.get(modified_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
handled_modified_count += 1
if handled_added_count or handled_modified_count:
logger.info(f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
else:
logger.debug(f"{storage} 无文件变化")
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
"""
强制全量扫描并处理所有文件包括已存在的文件
:param storage: 存储名称
:param mon_path: 监控路径
:return: 是否成功
"""
try:
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
# 生成快照
new_snapshot = StorageChain().snapshot_storage(
storage=storage,
path=mon_path,
last_snapshot_time=0 # 全量扫描,不使用增量
)
if new_snapshot is None:
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
return False
file_count = len(new_snapshot)
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
# 处理所有文件
processed_count = 0
for file_path, file_info in new_snapshot.items():
try:
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
continue
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(file_path),
file_size=file_size):
processed_count += 1
except Exception as e:
logger.error(f"处理文件 {file_path} 失败: {e}")
continue
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
# 全量扫描覆盖单个路径,与已有基线合并后落盘,避免覆盖其他监控路径的基线
old_snapshot_data, load_ok = self._store.load_checked(storage)
old_snapshot = old_snapshot_data.get('snapshot', {}) if (load_ok and old_snapshot_data) else {}
merged_snapshot = {**old_snapshot, **new_snapshot}
self._store.save(storage, merged_snapshot, len(merged_snapshot))
return True
except Exception as e:
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
return False
+148
View File
@@ -0,0 +1,148 @@
import json
import time
from typing import Dict, List, Optional, Tuple
from app.core.cache import FileCache
from app.core.config import settings
from app.log import logger
class SnapshotStore:
"""
远程目录监控快照的存取与比对
"""
def __init__(self, cache: Optional[FileCache] = None):
"""
初始化快照存储
:param cache: 快照文件缓存默认使用 CACHE_PATH/snapshots
"""
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
def save(self, storage: str, snapshot: Dict, file_count: int = 0,
last_snapshot_time: Optional[float] = None) -> bool:
"""
保存快照到文件缓存
:param storage: 存储名称
:param snapshot: 快照数据
:param file_count: 文件数量用于调整监控间隔
:param last_snapshot_time: 上次快照时间戳
:return: 是否保存成功
"""
try:
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
if snapshot_time is None:
snapshot_time = last_snapshot_time or time.time()
snapshot_data = {
'timestamp': snapshot_time,
'file_count': file_count,
'snapshot': snapshot
}
cache_key = f"{storage}_snapshot"
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
self._cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
logger.debug(f"快照已保存到缓存: {storage}")
return True
except Exception as e:
logger.error(f"保存快照失败: {e}")
return False
def load_checked(self, storage: str) -> Tuple[Optional[Dict], bool]:
"""
从文件缓存加载快照并区分快照不存在读取失败
读取失败时不能当作首次快照处理否则会静默丢弃已有基线
:param storage: 存储名称
:return: (快照数据或None, 是否读取成功)
"""
try:
cache_key = f"{storage}_snapshot"
snapshot_data = self._cache.get(cache_key, region="snapshots")
if snapshot_data:
data = json.loads(snapshot_data.decode('utf-8'))
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
return data, True
logger.debug(f"快照文件不存在: {storage}")
return None, True
except Exception as e:
logger.error(f"加载快照失败: {e}")
return None, False
def load(self, storage: str) -> Optional[Dict]:
"""
从文件缓存加载快照
:param storage: 存储名称
:return: 快照数据或None
"""
data, _ = self.load_checked(storage)
return data
def reset(self, storage: str) -> bool:
"""
重置快照强制下次扫描时重新建立基准
:param storage: 存储名称
:return: 是否成功
"""
try:
cache_key = f"{storage}_snapshot"
if self._cache.exists(cache_key, region="snapshots"):
self._cache.delete(cache_key, region="snapshots")
logger.info(f"快照已重置: {storage}")
return True
logger.debug(f"快照文件不存在,无需重置: {storage}")
return True
except Exception as e:
logger.error(f"重置快照失败: {storage} - {e}")
return False
@staticmethod
def compare(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
"""
比对快照找出变化的文件只处理新增和修改不处理删除
:param old_snapshot: 旧快照
:param new_snapshot: 新快照
:return: 变化信息
"""
changes = {
'added': [],
'modified': []
}
old_files = set(old_snapshot.keys())
new_files = set(new_snapshot.keys())
# 新增文件
changes['added'] = list(new_files - old_files)
# 修改文件(大小或时间变化)
for file_path in old_files & new_files:
old_info = old_snapshot[file_path]
new_info = new_snapshot[file_path]
# 检查文件大小变化
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
# 检查修改时间变化(如果有的话)
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
if old_size != new_size or (old_time and new_time and old_time != new_time):
changes['modified'].append(file_path)
return changes
@staticmethod
def adjust_interval(file_count: int) -> int:
"""
根据文件数量动态调整监控间隔
:param file_count: 文件数量
:return: 监控间隔分钟
"""
if file_count < 100:
return 5 # 5分钟
elif file_count < 500:
return 10 # 10分钟
elif file_count < 1000:
return 15 # 15分钟
else:
return 30 # 30分钟
+134
View File
@@ -0,0 +1,134 @@
import os
import platform
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from app.log import logger
from app.utils.system import SystemUtils
def count_directory_entries(directory: Path, max_check: int = 10000) -> Tuple[int, int]:
"""
统计目录下的文件与子目录数量用于检测是否超过系统限制
:param directory: 目录路径
:param max_check: 最大检查文件数量避免长时间阻塞
:return: (文件数量, 目录数量)
"""
file_count = 0
dir_count = 0
try:
for _, dirs, files in os.walk(str(directory)):
file_count += len(files)
dir_count += len(dirs)
if file_count > max_check:
break
except Exception as err:
logger.debug(f"统计目录规模失败: {err}")
return file_count, dir_count
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
"""
统计目录下的文件数量
:param directory: 目录路径
:param max_check: 最大检查数量避免长时间阻塞
:return: 文件数量
"""
file_count, _ = count_directory_entries(directory, max_check=max_check)
return file_count
def check_system_limits() -> Dict[str, Any]:
"""
检查系统监控相关限制
:return: 系统限制信息
"""
limits = {
'max_user_watches': 0,
'max_user_instances': 0,
'warnings': []
}
try:
if platform.system() == 'Linux':
# 检查 inotify 限制
try:
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
limits['max_user_watches'] = int(f.read().strip())
except Exception as e:
logger.debug(f"读取 inotify 限制失败: {e}")
limits['max_user_watches'] = 8192 # 默认值
try:
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
limits['max_user_instances'] = int(f.read().strip())
except Exception as e:
logger.debug(f"读取 inotify 实例限制失败: {e}")
except Exception as e:
limits['warnings'].append(f"检查系统限制时出错: {e}")
return limits
def get_system_optimization_tips() -> List[str]:
"""
获取系统优化建议
:return: 优化建议列表
"""
tips = []
system = platform.system()
if system == 'Linux':
tips.extend([
"增加 inotify 监控数量限制:",
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
"sudo sysctl -p",
"",
"如果在Docker中运行,请在宿主机上执行以上命令"
])
elif system == 'Darwin':
tips.extend([
"macOS 系统优化建议:",
"sudo sysctl kern.maxfiles=65536",
"sudo sysctl kern.maxfilesperproc=32768",
"ulimit -n 32768"
])
elif system == 'Windows':
tips.extend([
"Windows 系统优化建议:",
"1. 关闭不必要的实时保护软件对监控目录的扫描",
"2. 将监控目录添加到Windows Defender排除列表",
"3. 确保有足够的可用内存"
])
return tips
def decide_monitor_mode(directory: Path,
monitor_mode: str) -> Tuple[bool, str, Optional[Dict[str, Any]], Optional[int]]:
"""
决策监控模式兼容模式与网络文件系统直接短路只有快速模式候选才统计
目录规模与系统限制避免启动期对网络挂载做无谓的全量遍历
inotify max_user_watches 按监视点目录计数因此用目录数量而不是
文件数量与上限比较
:param directory: 监控目录
:param monitor_mode: 配置的监控模式
:return: (是否使用轮询, 原因, 系统限制信息或None, 文件数量或None)
"""
if monitor_mode == "compatibility":
return True, "用户配置为兼容模式", None, None
# 检查网络文件系统
if SystemUtils.is_network_filesystem(directory):
return True, "检测到网络文件系统,建议使用兼容模式", None, None
limits = check_system_limits()
file_count, dir_count = count_directory_entries(directory)
max_watches = limits.get('max_user_watches')
if max_watches and dir_count > max_watches * 0.8:
return (True, f"目录数量({dir_count})接近 inotify 监控上限({max_watches})",
limits, file_count)
return False, "使用快速模式", limits, file_count
+303
View File
@@ -0,0 +1,303 @@
import threading
import time
import traceback
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional
from watchfiles import Change, DefaultFilter, watch
from app.log import logger
@dataclass(frozen=True)
class DirectoryChangeEvent:
"""
目录文件变化事件隔离底层 watchfiles 事件结构
"""
change_type: Change
src_path: str
is_directory: bool
class LocalDirectoryWatcher:
"""
基于 watchfiles 的本地目录监控线程
"""
_HANDLE_CHANGES = {Change.added, Change.modified}
# 监控循环异常退出后的重启退避秒数,网络存储/FUSE 挂载抖动通常是暂时的
RESTART_BACKOFF = (5, 15, 30, 60, 120, 300)
# 单次监控循环存活超过该秒数视为已恢复,重置退避
HEALTHY_UPTIME = 60
# 超过该秒数监控循环没有任何活动,判定为静默失效
STALL_TIMEOUT = 600
# 轮询模式目录扫描间隔(毫秒):本地磁盘用 watchfiles 默认值
POLL_DELAY_LOCAL_MS = 300
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
POLL_DELAY_NETWORK_MS = 5000
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
poll_delay_ms: Optional[int] = None):
"""
初始化本地目录监控
:param mon_path: 监控目录
:param callback: 目录变化回调对象
:param force_polling: 是否强制使用轮询模式None 表示由 watchfiles 自动选择
:param poll_delay_ms: 轮询模式目录扫描间隔毫秒仅轮询时生效
"""
self._watch_path = mon_path
self._callback = callback
self._force_polling = force_polling
self._poll_delay_ms = poll_delay_ms or self.POLL_DELAY_LOCAL_MS
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
self._watch_filter = DefaultFilter()
# 最近一次监控循环活动时间(monotonic),用于检测静默失效
self._last_activity: float = 0.0
# 累计自动重启次数
self._restart_count: int = 0
@property
def watch_path(self) -> Path:
"""
获取监控目录
:return: 监控目录
"""
return self._watch_path
@property
def force_polling(self) -> Optional[bool]:
"""
获取监控模式配置重建监控线程时沿用
:return: 是否强制轮询
"""
return self._force_polling
@property
def restart_count(self) -> int:
"""
获取累计自动重启次数
:return: 自动重启次数
"""
return self._restart_count
@property
def poll_delay_ms(self) -> int:
"""
获取轮询模式目录扫描间隔毫秒重建监控线程时沿用
:return: 扫描间隔
"""
return self._poll_delay_ms
def start(self):
"""
启动本地目录监控线程
"""
if not self._watch_path.exists():
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
if not self._watch_path.is_dir():
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
if self.is_alive():
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
return
self._stop_event.clear()
self._mark_activity()
self._thread = threading.Thread(
target=self._run,
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
daemon=True
)
self._thread.start()
def stop(self):
"""
请求停止本地目录监控线程
"""
self._stop_event.set()
def join(self, timeout: Optional[float] = None):
"""
等待本地目录监控线程退出
:param timeout: 最长等待秒数
"""
if self._thread:
self._thread.join(timeout=timeout)
def is_alive(self) -> bool:
"""
判断监控线程是否仍在运行
:return: 线程存活状态
"""
return bool(self._thread and self._thread.is_alive())
def is_stalled(self) -> bool:
"""
判断监控线程是否已静默失效线程存活但监控循环长时间无任何活动
:return: 是否静默失效
"""
if self._stop_event.is_set() or not self.is_alive():
return False
if not self._last_activity:
return False
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
def _mark_activity(self):
"""
记录一次监控循环活动时间作为静默失效检测的心跳
"""
self._last_activity = time.monotonic()
def _run(self):
"""
运行 watchfiles 主循环异常时退避重启避免一次故障导致监控永久停摆
"""
# 快速模式失败后降级为轮询,降级后的失败一律走退避重启
force_polling = self._force_polling
attempt = 0
while not self._stop_event.is_set():
started_at = time.monotonic()
try:
self._mark_activity()
self._run_watch(force_polling=force_polling)
# 正常返回表示收到停止信号
return
except Exception as err:
if self._stop_event.is_set():
return
# 崩溃堆栈按 ERROR 级输出,生产环境 LOG_LEVEL=ERROR 时也能落盘
logger.error(f"本地目录监控异常堆栈: {self._watch_path}\n{traceback.format_exc()}")
if force_polling is not True:
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
force_polling = True
continue
if time.monotonic() - started_at >= self.HEALTHY_UPTIME:
# 上一轮监控已稳定运行过,重新从最短间隔开始退避
attempt = 0
delay = self.RESTART_BACKOFF[min(attempt, len(self.RESTART_BACKOFF) - 1)]
attempt += 1
self._restart_count += 1
logger.error(f"本地目录监控发生错误,{delay} 秒后自动重启"
f"(累计第 {self._restart_count} 次): {self._watch_path} - {err}")
if self._stop_event.wait(timeout=delay):
return
def _run_watch(self, force_polling: Optional[bool]):
"""
执行一次 watchfiles 监控循环
:param force_polling: 是否强制轮询
"""
for changes in watch(
str(self._watch_path),
watch_filter=self._watch_filter,
stop_event=self._stop_event,
rust_timeout=1000,
yield_on_timeout=True,
force_polling=force_polling,
poll_delay_ms=self._poll_delay_ms,
recursive=True,
ignore_permission_denied=True):
self._mark_activity()
if self._stop_event.is_set():
break
if not changes:
continue
self._handle_changes(changes)
self._mark_activity()
def _handle_changes(self, changes: set[tuple[Change, str]]):
"""
watchfiles 原始变更转换为目录监控事件
:param changes: watchfiles 返回的变更集合
"""
changes = self._expand_added_directories(changes)
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
self._mark_activity()
if change_type not in self._HANDLE_CHANGES:
continue
event_path = Path(path_str)
event = self._build_event(change_type=change_type, event_path=event_path)
if not event or event.is_directory:
continue
file_size = self._get_file_size(event_path)
if file_size is None:
continue
text = self._change_text(change_type)
try:
self._callback.event_handler(
event=event,
text=text,
event_path=path_str,
file_size=file_size
)
except Exception as err:
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
"""
将整体移入监控范围的新增目录展开为内部文件事件
:param changes: watchfiles 返回的变更集合
:return: 包含目录内新增文件的变更集合
"""
expanded_changes = set(changes)
for change_type, path_str in changes:
if change_type != Change.added:
continue
event_path = Path(path_str)
try:
if not event_path.is_dir():
continue
for nested_path in event_path.rglob("*"):
if not nested_path.is_file():
continue
nested_path_str = nested_path.as_posix()
if self._watch_filter(Change.added, nested_path_str):
expanded_changes.add((Change.added, nested_path_str))
except OSError as err:
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
return expanded_changes
@staticmethod
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
"""
构建目录变化事件路径已不存在时忽略
:param change_type: watchfiles 变化类型
:param event_path: 变化路径
:return: 目录变化事件
"""
try:
is_directory = event_path.is_dir()
except OSError as err:
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
return None
if not event_path.exists():
return None
return DirectoryChangeEvent(
change_type=change_type,
src_path=event_path.as_posix(),
is_directory=is_directory
)
@staticmethod
def _get_file_size(event_path: Path) -> Optional[int]:
"""
读取事件文件大小文件已消失时返回 None
:param event_path: 事件文件路径
:return: 文件大小
"""
try:
return event_path.stat().st_size
except OSError as err:
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
return None
@staticmethod
def _change_text(change_type: Change) -> str:
"""
转换 watchfiles 事件类型为日志文案
:param change_type: watchfiles 变化类型
:return: 事件描述
"""
if change_type == Change.modified:
return "修改"
return "新增"
+1
View File
@@ -94,6 +94,7 @@ class AgentLLMProviderEventData(ChainEventData):
user_agent: Optional[str] = Field(default=None, description="OpenAI兼容接口User-Agent")
use_proxy: Optional[bool] = Field(default=None, description="是否使用系统代理")
thinking_level: Optional[str] = Field(default=None, description="思考模式级别")
api_protocol: Optional[str] = Field(default=None, description="OpenAI兼容接口API协议:auto/chat_completions/responses")
selected_provider_id: Optional[str] = Field(default=None, description="插件侧供应商ID")
selected_provider_name: Optional[str] = Field(default=None, description="插件侧供应商名称")
source: Optional[str] = Field(default=None, description="选择来源")
+9
View File
@@ -36,3 +36,12 @@ class OperationInterrupted(KeyboardInterrupt):
用于表示操作被中断
"""
pass
class StorageQueryError(Exception):
"""
用于表示存储查询无法确认结果的异常类
当文件信息查询因网络限流或接口错误失败区别于确认不存在时抛出
调用方不应把该状态当作文件不存在处理
"""
pass
+16 -2
View File
@@ -1,9 +1,13 @@
import re
from typing import Optional
from pathlib import Path
from pydantic import BaseModel, Field
from app.schemas.types import StorageSchema
# Windows 盘符绝对路径,如 Z:/Downloads 或 Z:\Downloads
WINDOWS_DRIVE_PATTERN = re.compile(r"^[A-Za-z]:[\\/]")
class FileURI(BaseModel):
# 文件路径
@@ -13,10 +17,19 @@ class FileURI(BaseModel):
@property
def uri(self) -> str:
"""
文件 URI本地存储直接返回路径其他存储带上存储前缀
"""
return self.path if self.storage == "local" else f"{self.storage}:{self.path}"
@classmethod
def from_uri(cls, uri: str) -> "FileURI":
"""
解析文件 URI 为存储类型和路径
:param uri: 文件 URI /media/movieu115:/media/movie Windows 盘符路径 Z:/media
:return: FileURI 对象
"""
storage, path = 'local', uri
for s in StorageSchema:
protocol = f"{s.value}:"
@@ -24,11 +37,13 @@ class FileURI(BaseModel):
path = uri[len(protocol):]
storage = s.value
break
if not path.startswith("/"):
# Windows 盘符路径本身就是绝对路径,补上根斜杠会得到 /Z:/xxx 这样的非法路径
if not path.startswith("/") and not WINDOWS_DRIVE_PATTERN.match(path):
path = "/" + path
path = Path(path).as_posix()
return cls(storage=storage, path=path)
class FileItem(FileURI):
# 类型 dir/file
type: Optional[str] = None
@@ -68,4 +83,3 @@ class StorageUsage(BaseModel):
class StorageTransType(BaseModel):
# 传输类型
transtype: Optional[dict] = Field(default_factory=dict)
+20
View File
@@ -95,6 +95,26 @@ class PluginSidebarNavItem(BaseModel):
order: int = Field(default=0, description="同组内排序,越小越靠前")
class PluginRatingRequest(BaseModel):
"""插件评分请求"""
rating: float = Field(
ge=0.1,
le=5.0,
multiple_of=0.1,
description="评分,范围 0.1 至 5.0,精确到 0.1",
)
class PluginRating(BaseModel):
"""插件评分结果"""
plugin_id: str = Field(description="插件 ID")
average_rating: float = Field(default=0.0, description="平均评分")
rating_count: int = Field(default=0, description="评分人数")
user_rating: Optional[float] = Field(default=None, description="当前安装实例评分")
class PluginMemoryInfo(BaseModel):
"""插件内存信息"""
plugin_id: str = Field(description="插件ID")
+21 -1
View File
@@ -1,7 +1,7 @@
from dataclasses import dataclass
from typing import Optional, Any
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
@dataclass
@@ -40,6 +40,26 @@ class MediaServerConf(BaseModel):
# 自动同步间隔(小时),未设置时使用旧全局配置
sync_interval: Optional[int] = None
@field_validator("sync_interval", mode="before")
@classmethod
def validate_sync_interval(cls, value: Any) -> Optional[int]:
"""
兼容前端清空输入框后残留的空字符串等非法值避免历史配置导致模块初始化失败
:param value: 原始配置值
:return: 合法的间隔小时数无法解析时返回 None
"""
if value is None:
return None
if isinstance(value, str):
value = value.strip()
if not value:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
class DownloaderConf(BaseModel):
"""
+4
View File
@@ -215,6 +215,8 @@ class SystemConfigKey(Enum):
NotificationSwitchs = "NotificationSwitchs"
# 目录配置
Directories = "Directories"
# 挂载型本地盘是否删除空目录
MountedLocalDiskDeleteEmptyDirs = "MountedLocalDiskDeleteEmptyDirs"
# 存储配置
Storages = "Storages"
# 搜索站点范围
@@ -279,6 +281,8 @@ class SystemConfigKey(Enum):
SetupWizardState = "SetupWizardState"
# 绿联影视登录会话缓存
UgreenSessionCache = "UgreenSessionCache"
# 共享媒体识别成功次数
MediaRecognizeShareCount = "MediaRecognizeShareCount"
# 处理进度Key字典
+86 -3
View File
@@ -3,10 +3,11 @@ import collections
import re
import sys
import threading
import time
import weakref
from contextlib import AsyncExitStack, contextmanager, asynccontextmanager
from pathlib import Path
from typing import Any, Optional, Tuple, Union
from typing import Any, Dict, Optional, Tuple, Union
import chardet
import httpx
@@ -72,6 +73,53 @@ _DEFAULT_MAX_CONNECTIONS = 40
_DEFAULT_KEEPALIVE_EXPIRY = 30
# 同步 requests.Session 复用连接时,遇到对端或代理关闭 keep-alive 后允许重试的方法
_REQUESTS_RETRY_IDEMPOTENT_METHODS = ("GET", "HEAD", "OPTIONS")
# 代理走 CONNECT 隧道时,httpx 默认开启的 HTTP/2 多路复用会把并发请求叠加到极少数隧道上;
# 隧道被代理节点切换或空闲回收打断后,复用其上的所有请求会同时失败。按 (proxy, host) 熔断:
# 命中一次连接层失败就记录下次允许再尝试 h2 的时间戳(time.monotonic 基准),冷却期内该
# (proxy, host) 的请求直接退化为 http1.1;冷却期结束后自动恢复尝试 h2。
_H2_PROXY_BREAKER_COOLDOWN = 1800 # 30 分钟
_h2_proxy_breaker_lock = threading.Lock()
_h2_proxy_retry_at: Dict[Tuple[str, str], float] = {}
# 只有这些错误是"h2 隧道被打断"的特征(对应实测日志里的 SEND_HEADERS in CLOSED、
# EndOfStream 等);超时、连接失败、代理不可达等错误换 h1 一样会发生,
# 不应触发熔断,也不值得付出一次注定同样失败的 h1 重试
_H2_TUNNEL_BREAK_ERRORS = (
httpx.RemoteProtocolError,
httpx.LocalProtocolError,
httpx.ReadError,
httpx.WriteError,
httpx.CloseError,
)
def _h2_proxy_breaker_key(proxy: str, url: str) -> Tuple[str, str]:
try:
host = httpx.URL(url).host or ""
except Exception:
host = url
return proxy, host
def _h2_proxy_allowed(proxy: Optional[str], url: str) -> bool:
"""判断给定代理 + 目标 host 当前是否允许尝试 h2(未处于熔断冷却期)"""
if not proxy:
return True
with _h2_proxy_breaker_lock:
retry_at = _h2_proxy_retry_at.get(_h2_proxy_breaker_key(proxy, url), 0.0)
return time.monotonic() >= retry_at
def _trip_h2_proxy_breaker(proxy: str, url: str) -> None:
"""记录一次 h2 连接层失败,熔断该 (proxy, host) 冷却期内的 h2 尝试"""
now = time.monotonic()
with _h2_proxy_breaker_lock:
# 顺手清掉已过冷却期的条目,防止长期运行下字典无限增长
for key in [k for k, retry_at in _h2_proxy_retry_at.items() if now >= retry_at]:
del _h2_proxy_retry_at[key]
_h2_proxy_retry_at[_h2_proxy_breaker_key(proxy, url)] = now + _H2_PROXY_BREAKER_COOLDOWN
# 持有 LRU 淘汰后正在异步关闭的 transport task,避免 fire-and-forget 被 GC 警告
_pending_eviction_tasks: set[asyncio.Task] = set()
@@ -1087,13 +1135,48 @@ class AsyncRequestUtils:
self._client, method, url, raise_exception, **kwargs
)
# 代理走 CONNECT 隧道时 h2 多路复用容易被隧道打断放大成批量失败(见
# _h2_proxy_allowed 处注释);仅对幂等方法做"h2 失败就地降级 h1 重试",
# 避免非幂等请求在服务端可能已收到数据的情况下重复产生副作用
http2 = self._http2 and _h2_proxy_allowed(self._proxies, url)
if not (http2 and self._proxies and method.upper() in _REQUESTS_RETRY_IDEMPOTENT_METHODS):
return await self._dispatch_request(
http2, cookies_dict, method, url, raise_exception, **kwargs
)
try:
return await self._dispatch_request(
True, cookies_dict, method, url, True, **kwargs
)
except _H2_TUNNEL_BREAK_ERRORS as e:
logger.debug(f"h2 代理连接层失败,熔断 {url} 所在 host 并降级 h1 重试: {e!r}")
_trip_h2_proxy_breaker(self._proxies, url)
return await self._dispatch_request(
False, cookies_dict, method, url, raise_exception, **kwargs
)
except httpx.RequestError as e:
# 与 h2 隧道无关的失败(超时、连接失败等):不熔断也不重试,
# 恢复调用方原本的 raise_exception 语义
if raise_exception:
raise
error_msg = str(e) or f"未知网络错误 (URL: {url}, Method: {method.upper()})"
logger.debug(f"异步请求失败: {error_msg}")
return None
async def _dispatch_request(
self, http2: bool, cookies_dict: Optional[dict], method: str, url: str,
raise_exception: bool, **kwargs
) -> Optional[httpx.Response]:
"""
按给定 http2 开关构建/复用底层连接并发起请求 request() h2/h1 熔断切换复用
"""
# 共享底层 transport(连接池+TLS 复用),每次请求创建轻量 AsyncClient。
# AsyncClient 持有的 cookie jar 仅存活于本次请求 lifecycle
# 既复用握手又彻底避免 jar 跨调用累积。
transport = _get_shared_async_transport(
proxy=self._proxies,
verify=self._verify,
http2=self._http2,
http2=http2,
max_keepalive_connections=self._max_keepalive_connections,
max_connections=self._max_connections,
keepalive_expiry=self._keepalive_expiry,
@@ -1113,7 +1196,7 @@ class AsyncRequestUtils:
# 兜底:没有运行中的事件循环时,临时客户端走完即关
async with httpx.AsyncClient(
http2=self._http2,
http2=http2,
proxy=self._proxies,
timeout=self._timeout,
verify=self._verify,
+13 -3
View File
@@ -828,10 +828,13 @@ class SystemUtils:
return False
@staticmethod
def is_network_filesystem(directory: Path) -> bool:
def is_network_filesystem(
directory: Path, include_local_fuse: bool = False
) -> bool:
"""
检测是否为网络文件系统
:param directory: 目录路径
:param include_local_fuse: 是否将本地 FUSE 挂载视为挂载文件系统
:return: 是否为网络文件系统
"""
try:
@@ -849,7 +852,10 @@ class SystemUtils:
"fuseblk",
# TBD
]
if any(fs in output for fs in local_fs):
if (
not include_local_fuse
and any(fs in output for fs in local_fs)
):
return False
network_fs = ['nfs', 'cifs', 'smbfs', 'fuse', 'sshfs', 'ftpfs']
return any(fs in output for fs in network_fs)
@@ -859,7 +865,11 @@ class SystemUtils:
capture_output=True, text=True, timeout=5)
if result.returncode == 0:
output = result.stdout.lower()
return 'nfs' in output or 'smbfs' in output
return (
'nfs' in output
or 'smbfs' in output
or (include_local_fuse and 'fuse' in output)
)
elif system == 'Windows':
# Windows 检查网络驱动器
return str(directory).startswith('\\\\')
+3
View File
@@ -481,6 +481,9 @@ moviepilot tool run search_torrents media_type=movie tmdb_id=12345
- `tool list` 用于动态发现当前服务可调用的工具
- `tool show` 会输出参数名、类型和描述
- `tool run` 参数格式固定为 `key=value`
- `read_file``write_file``edit_file``execute_command`
属于内置 Agent 的本地敏感能力,不通过 MCP/`moviepilot tool` 暴露;插件开发时
由 Agent 按当前用户权限直接调用这些工具。
## Scheduler 命令
+37
View File
@@ -55,6 +55,43 @@ pip install -r requirements.txt
pip install -r requirements-dev.in
```
### 2.1 本地启动脚本
不需要打开 IDE 时,可以直接使用仓库内的启动脚本。脚本会自动定位项目根目录和虚拟环境,并以模块方式启动后端,避免 `ModuleNotFoundError: No module named 'app'`
```bash
# 默认启动后端开发服务,前台运行,按 Ctrl+C 停止
./scripts/start-local.sh
./scripts/start-local.sh backend
# 如果已经安装前端发布包,可启动完整的前后端服务
./scripts/start-local.sh service start
# 管理完整服务
./scripts/start-local.sh stop
./scripts/start-local.sh restart
./scripts/start-local.sh status
./scripts/start-local.sh logs --follow
```
默认会使用 `DEBUG=true``DEV=true`,与 IDE 开发启动保持一致;如果不需要热重载,可以这样启动以降低资源占用:
```bash
DEV=false ./scripts/start-local.sh
```
脚本会优先使用 `CONFIG_DIR`,其次使用 `MOVIEPILOT_CONFIG_DIR`,再检测 `~/Documents/moviepilot`,最后回退到仓库内的 `config` 目录。需要使用其他配置目录时,可以这样运行:
```bash
MOVIEPILOT_CONFIG_DIR=/path/to/moviepilot-config ./scripts/start-local.sh
```
首次使用前如果脚本没有执行权限,运行:
```bash
chmod +x scripts/start-local.sh
```
### 3. 修改主程序依赖
新增或升级依赖时,先确认依赖属于哪个层级:
+30 -2
View File
@@ -31,6 +31,12 @@ MCP 使用系统配置中的 `API_TOKEN` 作为认证密钥,文档中的 API K
- `tools/call`: 调用特定工具。
- `ping`: 连接存活检测。
### 动态插件工具
`tools/list` 会同时返回 MoviePilot 内置工具和已启用插件通过 `get_agent_tools()` 声明的工具。插件启动、停止、重载或配置生效后,MCP 工具管理器会在下一次列出或调用工具时按注册表版本惰性刷新,避免继续暴露已移除的工具或遗漏新工具。
MCP 当前不会主动发送工具列表变更通知(`listChanged=false`)。如果客户端缓存了工具列表,插件状态变化后需要让客户端重新请求 `tools/list`;无法手动刷新的客户端应重新连接 MCP 服务或新建会话。
---
## 4. 客户端配置示例
@@ -197,9 +203,15 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
| 方法 | 路径 | 说明 |
| :--- | :--- | :--- |
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存及识别成功、失败条目统计 |
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存统计、共享识别累计成功命中次数及开关状态 |
| DELETE | `/api/v1/tmdb/cache/{cache_key}` | 按缓存键删除单条 TheMovieDb 识别缓存,缓存键需要进行 URL 编码 |
| DELETE | `/api/v1/tmdb/cache` | 清空全部 TheMovieDb 识别缓存 |
| GET | `/api/v1/douban/cache` | 查询豆瓣识别缓存统计、共享识别累计成功命中次数及开关状态 |
| DELETE | `/api/v1/douban/cache/{cache_key}` | 按缓存键删除单条豆瓣识别缓存,缓存键需要进行 URL 编码 |
| DELETE | `/api/v1/douban/cache` | 清空全部豆瓣识别缓存 |
缓存查询响应的 `data` 包含 `count``recognized``unrecognized``data`,以及共享识别统计字段
`shared_recognized` 和开关字段 `shared_recognize_enabled`。共享命中次数仅在共享结果驱动的二次媒体识别成功后累计。
### 插件补充接口
@@ -207,13 +219,29 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
按需读取指定已安装插件的最新远端更新说明。该接口用于前端在用户点击“查看更新说明”时再实时访问插件仓库,避免加载已安装插件列表时批量请求网络。
**GET** `/api/v1/plugin/rating?plugin_ids={plugin_id,...}`
批量查询插件平均分、评分人数和当前安装实例评分。`plugin_ids` 省略时查询中心端已有的全部插件评分。
**GET** `/api/v1/plugin/rating/{plugin_id}`
查询单个插件平均分、评分人数和当前安装实例评分。中心端暂不可用时返回该插件的零评分结果。
**POST** `/api/v1/plugin/rating/{plugin_id}`
为已安装插件提交当前安装实例评分,请求体为 `{"rating": 4.5}`。评分范围为 `0.1``5.0`,精确到 `0.1`;同一安装实例再次提交会更新原评分。
### 1. 列出所有工具
**GET** `/api/v1/mcp/tools`
获取所有可用的MCP工具列表。
工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。
内置工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。插件工具的参数结构由插件自身声明。
内置 Agent 的本地文件与命令工具 `read_file``write_file``edit_file`
`execute_command` 不通过 MCP 暴露。这些工具在 Agent 运行时执行独立的
用户权限与路径边界检查;MCP 隐藏列表只负责收敛接口暴露面,不替代权限控制。
媒体相关 MCP 工具(如 `query_media_detail``search_torrents``query_library_exists``add_subscribe``transfer_file`)接受 `tmdb_id`/`tmdbid``douban_id`/`doubanid``bangumi_id`/`bangumiid``anilist_id`/`anilistid`,也接受 `media_source` + `media_id`。工具返回的媒体、订阅、下载和整理记录会同步带回可用的四种专用 ID 及通用主身份。
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
MOVIEPILOT_BIN="$PROJECT_ROOT/moviepilot"
VENV_PYTHON="$PROJECT_ROOT/venv/bin/python"
show_usage() {
cat <<'EOF'
用法:
./scripts/start-local.sh 启动后端开发服务(前台运行)
./scripts/start-local.sh backend 启动后端开发服务(前台运行)
./scripts/start-local.sh service start 启动后端和已安装的前端服务
./scripts/start-local.sh service start --safe 以安全模式启动完整服务
./scripts/start-local.sh stop|restart|status 管理后端和前端服务
./scripts/start-local.sh logs [OPTIONS] 查看后端日志
./scripts/start-local.sh help 显示本帮助
EOF
}
if [[ ! -x "$MOVIEPILOT_BIN" ]]; then
printf '未找到本地 CLI%s\n' "$MOVIEPILOT_BIN" >&2
exit 1
fi
if [[ ! -x "$VENV_PYTHON" ]]; then
printf '未找到项目虚拟环境:%s\n请先执行:%s install deps\n' "$VENV_PYTHON" "$MOVIEPILOT_BIN" >&2
exit 1
fi
# 显式传入配置目录,避免被仓库中的临时 .moviepilot.env 覆盖。
if [[ -z "${CONFIG_DIR:-}" ]]; then
if [[ -n "${MOVIEPILOT_CONFIG_DIR:-}" ]]; then
CONFIG_DIR="$MOVIEPILOT_CONFIG_DIR"
elif [[ -d "${HOME:-}/Documents/moviepilot" ]]; then
CONFIG_DIR="${HOME}/Documents/moviepilot"
else
CONFIG_DIR="$PROJECT_ROOT/config"
fi
fi
export CONFIG_DIR
export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
export DEBUG="${DEBUG:-true}"
export DEV="${DEV:-true}"
cd "$PROJECT_ROOT"
if [[ "$#" -eq 0 ]]; then
set -- backend
fi
command_name="$1"
shift
case "$command_name" in
backend|start)
if [[ "$#" -gt 0 ]]; then
printf '后端模块启动不接受额外参数;完整服务请使用:%s service start [OPTIONS]\n' "$0" >&2
exit 2
fi
exec "$VENV_PYTHON" -m app.main
;;
service)
if [[ "$#" -eq 0 ]]; then
set -- start
fi
exec "$MOVIEPILOT_BIN" "$@"
;;
stop|restart|status|logs|doctor|config|version)
exec "$MOVIEPILOT_BIN" "$command_name" "$@"
;;
help|--help|-h)
show_usage
;;
*)
printf '未知命令:%s\n\n' "$command_name" >&2
show_usage >&2
exit 2
;;
esac
+29 -2
View File
@@ -1,6 +1,6 @@
---
name: create-moviepilot-plugin
version: 2
version: 3
description: >-
Use this skill when the user asks to create, modify, debug, validate, or
scaffold a MoviePilot local plugin. Covers MoviePilot V2 plugin development,
@@ -11,7 +11,7 @@ description: >-
sidebar pages, commands, services, workflow actions, agent tools, and local
install/reload flows. Also use for Chinese requests mentioning 编写插件、本地插件源,
插件开发, V2插件, 插件市场, 本地安装插件, 插件热加载, 前端联邦, 侧栏入口, Vue插件页面.
allowed-tools: list_directory read_file write_file edit_file execute_command query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
allowed-tools: list_directory read_file write_file edit_file execute_command search_web browse_webpage query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
---
# Create MoviePilot Plugin
@@ -33,6 +33,33 @@ a local plugin source and installed into the running MoviePilot instance.
- When working in or from `MoviePilot-Plugins`, read its `README.md`,
`docs/Repository_Guide.md`, and `docs/V2_Plugin_Development.md`. For
scenario-specific extensions, read the matching `docs/faq/*.md`.
## Code Tool Workflow
- Use `execute_command(action="run")` with `rg` and narrow globs or paths to
locate plugin classes, extension points, tests, and package entries. Use
`list_directory` only when inspecting one known folder or a configured remote
storage backend.
- Read the relevant implementation and adjacent example before editing.
- Before using a Python or Node.js dependency API, determine the exact installed
or locked version from requirements, package manifests, lockfiles, local
package source, and `.pyi`/`.d.ts` declarations. If those are insufficient,
use `search_web` with the official documentation domain and `browse_webpage`
to read the matching version. Do not guess API signatures from memory or mix
examples from different major versions. Search the relevant package directory,
`.venv`, or `node_modules` directly with `rg` instead of scanning the entire
project without bounds.
- Use `edit_file` for localized changes. Its `old_text` must identify one exact
location by default; add surrounding context instead of enabling
`replace_all` unless every match intentionally changes.
- Use `write_file` for new files. Existing files require `overwrite=true` for a
full rewrite; first call `read_file(include_metadata=true)` and pass its
`sha256` as `expected_sha256` when replacing previously read content.
- Use `execute_command(action="run")` for short validation, Git, and diagnostic
commands. Use `action="start"` only for interactive or long-running commands,
then continue through the returned session ID.
- Do not use shell redirection or inline scripts to perform source edits or to
bypass a file-tool permission error.
- When the plugin uses Vue federation, also read
`MoviePilot-Frontend/docs/module-federation-guide.md`,
`MoviePilot-Frontend/docs/federation-troubleshooting.md`,
+23 -1
View File
@@ -324,13 +324,16 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
| GET | `/api/v1/dashboard/network` | Network traffic |
| GET | `/api/v1/dashboard/network2` | Network traffic (API_TOKEN) |
### Plugin (22 endpoints)
### Plugin (25 endpoints)
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/v1/plugin/` | List plugins. Params: `state` (installed/market/all), `force` |
| GET | `/api/v1/plugin/installed` | List installed plugins |
| GET | `/api/v1/plugin/statistic` | Plugin install statistics |
| GET | `/api/v1/plugin/rating` | Batch plugin ratings. Params: comma-separated `plugin_ids` |
| GET | `/api/v1/plugin/rating/{plugin_id}` | Get average rating, rating count, and this installation's rating |
| POST | `/api/v1/plugin/rating/{plugin_id}` | Rate an installed plugin. Body: `{"rating": 4.5}`; range 0.1-5.0 |
| GET | `/api/v1/plugin/install/{plugin_id}` | Install plugin. Params: `repo_url`, `force` |
| GET | `/api/v1/plugin/reload/{plugin_id}` | Reload plugin |
| GET | `/api/v1/plugin/reset/{plugin_id}` | Reset plugin config & data |
@@ -441,6 +444,20 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
| POST | `/api/v1/torrent/cache/refresh` | Refresh torrent cache |
| POST | `/api/v1/torrent/cache/reidentify/{domain}/{torrent_hash}` | Re-identify torrent. Params: `tmdbid`, `doubanid` |
### Recognition Cache (6 endpoints)
The two list endpoints return local cache totals plus `shared_recognized` and
`shared_recognize_enabled` for the persisted successful shared-recognition count.
| Method | Path | Description |
|--------|------|-------------|
| GET | `/api/v1/tmdb/cache` | Get TheMovieDb recognition cache statistics |
| DELETE | `/api/v1/tmdb/cache/{cache_key}` | Delete one URL-encoded TheMovieDb recognition cache key |
| DELETE | `/api/v1/tmdb/cache` | Clear TheMovieDb recognition cache |
| GET | `/api/v1/douban/cache` | Get Douban recognition cache statistics |
| DELETE | `/api/v1/douban/cache/{cache_key}` | Delete one URL-encoded Douban recognition cache key |
| DELETE | `/api/v1/douban/cache` | Clear Douban recognition cache |
### Message (8 endpoints)
| Method | Path | Description |
@@ -488,6 +505,11 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
| GET | `/api/v1/mcp/tools/{tool_name}` | Get tool definition |
| GET | `/api/v1/mcp/tools/{tool_name}/schema` | Get tool input schema |
The exposed tool list is dynamic: it includes tools declared by enabled plugins
and is refreshed lazily after plugin startup, shutdown, reload, or configuration
activation. Clients that cache MCP metadata must request `tools/list` again or
reconnect after a plugin lifecycle change.
### Agent MCP Client (3 endpoints)
| Method | Path | Description |
+4
View File
@@ -45,6 +45,10 @@ List all available commands: `moviepilot tool list`
Show parameters and usage for a specific command: `moviepilot tool show <command>`
The tool list includes tools declared by enabled plugins. Re-run `tool list` and
`tool show` after a plugin is enabled, disabled, reloaded, or reconfigured so the
command selection uses the refreshed runtime registry.
Always run `show <command>` before calling a command — parameter names are not inferable, do not guess.
## Command Groups
+131
View File
@@ -0,0 +1,131 @@
"""Agent 本地文件搜索与安全编辑工具测试。"""
import asyncio
import hashlib
import json
from app.agent.tools.impl.edit_file import EditFileTool
from app.agent.tools.impl.read_file import ReadFileTool
from app.agent.tools.impl.write_file import WriteFileTool
def _make_admin_tool(tool_class):
"""创建带管理员上下文的文件工具实例。"""
tool = tool_class(session_id="session-1", user_id="admin")
tool.set_agent_context({"is_admin": True})
return tool
def test_edit_file_rejects_ambiguous_match_by_default(tmp_path):
"""精确编辑默认应拒绝多处匹配,避免静默批量修改代码。"""
file_path = tmp_path / "plugin.py"
file_path.write_text("enabled = False\nenabled = False\n", encoding="utf-8")
tool = _make_admin_tool(EditFileTool)
result = asyncio.run(
tool.run(str(file_path), "enabled = False", "enabled = True")
)
assert "匹配到 2 处" in result
assert "replace_all=true" in result
assert file_path.read_text(encoding="utf-8") == (
"enabled = False\nenabled = False\n"
)
def test_edit_file_replace_all_requires_explicit_flag(tmp_path):
"""显式开启 replace_all 后才应替换全部精确匹配。"""
file_path = tmp_path / "plugin.py"
file_path.write_text("old\nold\n", encoding="utf-8")
tool = _make_admin_tool(EditFileTool)
result = asyncio.run(
tool.run(str(file_path), "old", "new", replace_all=True)
)
assert "替换了 2 处" in result
assert file_path.read_text(encoding="utf-8") == "new\nnew\n"
def test_edit_file_rejects_empty_match_and_missing_file(tmp_path):
"""编辑工具不应再通过空匹配隐式创建文件。"""
file_path = tmp_path / "missing.py"
tool = _make_admin_tool(EditFileTool)
empty_result = asyncio.run(tool.run(str(file_path), "", "content"))
missing_result = asyncio.run(tool.run(str(file_path), "old", "new"))
assert "old_text 不能为空" in empty_result
assert "不存在" in missing_result
assert "write_file" in missing_result
assert not file_path.exists()
def test_edit_file_rejects_stale_sha256(tmp_path):
"""文件在读取后变化时,哈希保护应拒绝基于旧版本编辑。"""
file_path = tmp_path / "plugin.py"
file_path.write_text("before", encoding="utf-8")
old_sha256 = hashlib.sha256(b"before").hexdigest()
file_path.write_text("changed elsewhere", encoding="utf-8")
tool = _make_admin_tool(EditFileTool)
result = asyncio.run(
tool.run(
str(file_path),
"changed elsewhere",
"agent change",
expected_sha256=old_sha256,
)
)
assert "已在读取后发生变化" in result
assert file_path.read_text(encoding="utf-8") == "changed elsewhere"
def test_write_file_protects_existing_file_and_supports_guarded_overwrite(tmp_path):
"""完整写入应默认保护已有文件,并允许带版本校验的显式覆盖。"""
file_path = tmp_path / "plugin.py"
file_path.write_text("before", encoding="utf-8")
expected_sha256 = hashlib.sha256(b"before").hexdigest()
tool = _make_admin_tool(WriteFileTool)
refused_result = asyncio.run(tool.run(str(file_path), "unexpected"))
written_result = asyncio.run(
tool.run(
str(file_path),
"after",
overwrite=True,
expected_sha256=expected_sha256,
)
)
stale_result = asyncio.run(
tool.run(
str(file_path),
"stale write",
overwrite=True,
expected_sha256=expected_sha256,
)
)
assert "拒绝完整覆盖" in refused_result
assert "成功写入文件" in written_result
assert "sha256=" in written_result
assert "已在读取后发生变化" in stale_result
assert file_path.read_text(encoding="utf-8") == "after"
def test_read_file_can_return_sha256_metadata(tmp_path):
"""读取工具应能返回供后续冲突检查使用的文件哈希。"""
file_path = tmp_path / "plugin.py"
file_path.write_text("插件内容", encoding="utf-8")
tool = _make_admin_tool(ReadFileTool)
result = asyncio.run(tool.run(str(file_path), include_metadata=True))
payload = json.loads(result)
assert payload["content"] == "插件内容"
assert payload["size_bytes"] == len("插件内容".encode("utf-8"))
assert payload["sha256"] == hashlib.sha256(
"插件内容".encode("utf-8")
).hexdigest()
assert payload["truncated"] is False
+1
View File
@@ -78,6 +78,7 @@ async def test_agent_bundle_signature_changes_with_temperature(monkeypatch) -> N
"user_agent": None,
"use_proxy": False,
"thinking_level": "off",
"api_protocol": "auto",
}
with patch.object(
+39
View File
@@ -45,3 +45,42 @@ def test_resolve_llm_runtime_config_prefers_plugin_thinking_level(monkeypatch) -
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
assert runtime_config["thinking_level"] == "high"
def test_resolve_llm_runtime_config_uses_system_api_protocol(monkeypatch) -> None:
"""插件未提供 API 协议时应使用系统配置。"""
monkeypatch.setattr(settings, "LLM_API_PROTOCOL", "responses")
agent = MoviePilotAgent(session_id="api-protocol-default", user_id="user-1")
async def return_empty_config(event_type, event_data):
"""模拟插件未返回有效运行时配置。"""
assert event_type == ChainEventType.AgentLLMProvider
assert event_data.api_protocol == "responses"
return SimpleNamespace(event_data=AgentLLMProviderEventData())
with patch(
"app.agent.eventmanager.async_send_event",
new=AsyncMock(side_effect=return_empty_config),
):
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
assert runtime_config["api_protocol"] == "responses"
def test_resolve_llm_runtime_config_prefers_plugin_api_protocol(monkeypatch) -> None:
"""插件显式覆盖 API 协议时应优先使用插件值。"""
monkeypatch.setattr(settings, "LLM_API_PROTOCOL", "responses")
agent = MoviePilotAgent(session_id="api-protocol-plugin", user_id="user-1")
async def override_api_protocol(_event_type, event_data):
"""模拟插件覆盖 API 协议。"""
event_data.api_protocol = "chat_completions"
return SimpleNamespace(event_data=event_data)
with patch(
"app.agent.eventmanager.async_send_event",
new=AsyncMock(side_effect=override_api_protocol),
):
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
assert runtime_config["api_protocol"] == "chat_completions"
@@ -49,7 +49,12 @@ def test_non_admin_manager_exposes_restricted_file_tools():
manager = MoviePilotToolsManager(is_admin=False)
tool_names = {tool.name for tool in manager.list_tools()}
assert {"read_file", "write_file", "edit_file", "list_directory"} <= tool_names
assert {
"read_file",
"write_file",
"edit_file",
"list_directory",
} <= tool_names
def test_non_admin_manager_hides_admin_only_send_local_file_tool():
+1
View File
@@ -87,6 +87,7 @@ def test_initialize_llm_uses_chain_event_selection(monkeypatch) -> None:
user_agent="AgentTokens-UA/1.0",
use_proxy=True,
thinking_level="xhigh",
api_protocol="auto",
)
assert agent._llm_provider_selection["selected_provider_id"] == "provider-1"
+156
View File
@@ -0,0 +1,156 @@
import asyncio
import time
import httpx
import pytest
from app.utils import http as http_module
from app.utils.http import AsyncRequestUtils
PROXY = "http://proxy.example:7890"
URL = "https://raw.githubusercontent.com/demo/repo/main/package.json"
@pytest.fixture(autouse=True)
def _reset_h2_proxy_breaker():
"""每个用例前后清空熔断状态,避免跨用例污染进程级熔断字典。"""
http_module._h2_proxy_retry_at.clear()
yield
http_module._h2_proxy_retry_at.clear()
def _fake_dispatch(calls, fail_when):
async def fake(_self, http2, _cookies_dict, _method, _url, raise_exception, **_kwargs):
calls.append(http2)
if fail_when(http2):
if raise_exception:
raise httpx.RemoteProtocolError("tunnel closed")
return None
return "ok"
return fake
def test_get_downgrades_to_h1_and_trips_breaker_on_h2_failure(monkeypatch):
"""
走代理的幂等请求 h2 遇到连接层失败时就地降级 h1 重试一次并触发该 (proxy, host) 的熔断
"""
calls = []
monkeypatch.setattr(
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: http2)
)
utils = AsyncRequestUtils(proxies={"https": PROXY})
result = asyncio.run(utils.request("get", URL))
assert result == "ok"
assert calls == [True, False]
assert http_module._h2_proxy_allowed(PROXY, URL) is False
def test_get_skips_h2_while_breaker_is_tripped(monkeypatch):
"""
熔断冷却期内走代理的请求直接使用 h1不再尝试 h2
"""
http_module._trip_h2_proxy_breaker(PROXY, URL)
calls = []
monkeypatch.setattr(
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
)
utils = AsyncRequestUtils(proxies={"https": PROXY})
result = asyncio.run(utils.request("get", URL))
assert result == "ok"
assert calls == [False]
def test_get_retries_h2_after_cooldown_expires(monkeypatch):
"""
熔断冷却期结束后代理请求恢复尝试 h2
"""
http_module._h2_proxy_retry_at[
http_module._h2_proxy_breaker_key(PROXY, URL)
] = time.monotonic() - 1
calls = []
monkeypatch.setattr(
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
)
utils = AsyncRequestUtils(proxies={"https": PROXY})
result = asyncio.run(utils.request("get", URL))
assert result == "ok"
assert calls == [True]
def test_timeout_does_not_trip_breaker_or_retry(monkeypatch):
"""
超时等与 h2 隧道无关的错误不触发熔断不做 h1 重试且保持 raise_exception=False 返回 None 的语义
"""
calls = []
async def fake(_self, http2, _cookies_dict, _method, _url, _raise_exception, **_kwargs):
calls.append(http2)
raise httpx.ConnectTimeout("proxy slow")
monkeypatch.setattr(AsyncRequestUtils, "_dispatch_request", fake)
utils = AsyncRequestUtils(proxies={"https": PROXY})
result = asyncio.run(utils.request("get", URL))
assert result is None
assert calls == [True]
assert http_module._h2_proxy_allowed(PROXY, URL) is True
def test_timeout_still_raises_when_raise_exception_enabled(monkeypatch):
"""
h2 隧道无关的错误在 raise_exception=True 时按原语义抛出
"""
calls = []
async def fake(_self, http2, _cookies_dict, _method, _url, _raise_exception, **_kwargs):
calls.append(http2)
raise httpx.ConnectTimeout("proxy slow")
monkeypatch.setattr(AsyncRequestUtils, "_dispatch_request", fake)
utils = AsyncRequestUtils(proxies={"https": PROXY})
with pytest.raises(httpx.ConnectTimeout):
asyncio.run(utils.request("get", URL, raise_exception=True))
assert calls == [True]
assert http_module._h2_proxy_allowed(PROXY, URL) is True
def test_post_does_not_downgrade_on_h2_failure(monkeypatch):
"""
非幂等方法 h2 失败时不做 h1 降级重试避免服务端可能已收到数据时重复产生副作用
"""
calls = []
monkeypatch.setattr(
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: True)
)
utils = AsyncRequestUtils(proxies={"https": PROXY})
result = asyncio.run(utils.request("post", URL))
assert result is None
assert calls == [True]
def test_no_proxy_configured_skips_breaker_logic(monkeypatch):
"""
未配置代理时不触发熔断判断行为与原实现一致仅走一次 h2
"""
calls = []
monkeypatch.setattr(
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
)
utils = AsyncRequestUtils()
result = asyncio.run(utils.request("get", URL))
assert result == "ok"
assert calls == [True]
+28
View File
@@ -3,6 +3,7 @@ from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
SKILLS_ROOT = PROJECT_ROOT / "skills"
CORE_PROMPT_PATH = PROJECT_ROOT / "app/agent/prompt/System Core Prompt.txt"
def _read_skill(skill_name: str) -> str:
@@ -26,6 +27,7 @@ def test_modified_builtin_skills_have_incremented_versions() -> None:
"moviepilot-cli": "6",
"moviepilot-update": "3",
"transfer-failed-retry": "2",
"create-moviepilot-plugin": "3",
}
for skill_name, expected_version in expected_versions.items():
@@ -64,3 +66,29 @@ def test_api_and_database_skills_declare_fallback_boundaries() -> None:
assert "INSERT" in db_content
assert "UPDATE" in db_content
assert "DELETE" in db_content
def test_agent_core_prompt_does_not_block_plugin_source_edits() -> None:
"""核心提示词不应禁止插件开发技能写入源码。"""
core_prompt = CORE_PROMPT_PATH.read_text(encoding="utf-8")
plugin_skill = _read_skill("create-moviepilot-plugin")
allowed_tools = _frontmatter_value(plugin_skill, "allowed-tools")
assert "file editing tools, or generated patches to change code" not in core_prompt
assert "write_file" in allowed_tools
assert "edit_file" in allowed_tools
assert "search_web" in allowed_tools
assert "browse_webpage" in allowed_tools
def test_agent_core_prompt_routes_code_tools_safely() -> None:
"""核心提示词应区分代码搜索、精确编辑和交互式命令场景。"""
core_prompt = CORE_PROMPT_PATH.read_text(encoding="utf-8")
assert '`execute_command(action="run")` with `rg`' in core_prompt
assert "`replace_all=true` only when every match must change" in core_prompt
assert "Use `action=run` for short bounded commands" in core_prompt
assert "including SSH" in core_prompt
assert "Never use shell redirection" in core_prompt
assert "matching version of the official documentation" in core_prompt
assert "Do not guess signatures from memory" in core_prompt
+14 -1
View File
@@ -1,10 +1,11 @@
import asyncio
import inspect
from unittest.mock import Mock
from app.api.endpoints import douban as douban_endpoint
from app.db.user_oper import get_current_active_superuser_async
from app.modules.douban.douban_cache import DoubanCache
from app.schemas.types import MediaType
from app.schemas.types import MediaType, SystemConfigKey
class _MemoryCacheStub:
@@ -116,7 +117,14 @@ def test_douban_cache_endpoint_returns_management_statistics(monkeypatch):
"recognized": {"id": "1", "title": "Alpha", "type": MediaType.MOVIE},
"unrecognized": {"id": 0},
})
get_system_config = Mock(return_value=None)
monkeypatch.setattr(douban_endpoint, "DoubanCache", lambda: cache)
monkeypatch.setattr(
douban_endpoint,
"SystemConfigOper",
lambda: type("SystemConfigStub", (), {"get": get_system_config})(),
)
monkeypatch.setattr(douban_endpoint.settings, "MEDIA_RECOGNIZE_SHARE", False)
response = asyncio.run(douban_endpoint.douban_recognition_cache(None))
@@ -124,6 +132,11 @@ def test_douban_cache_endpoint_returns_management_statistics(monkeypatch):
assert response.data["count"] == 2
assert response.data["recognized"] == 1
assert response.data["unrecognized"] == 1
assert response.data["shared_recognized"] == 0
assert response.data["shared_recognize_enabled"] is False
get_system_config.assert_called_once_with(
SystemConfigKey.MediaRecognizeShareCount
)
def test_douban_cache_delete_endpoint_reports_missing_item(monkeypatch):
+50
View File
@@ -0,0 +1,50 @@
from app.schemas.file import FileURI
def test_from_uri_keeps_windows_drive_path() -> None:
"""Windows 盘符路径已是绝对路径,不能再补根斜杠,否则映射网络驱动器整理会报 WinError 123。"""
file_uri = FileURI.from_uri("Z:/Downloads/电视剧/国产剧")
assert file_uri.storage == "local"
assert file_uri.path == "Z:/Downloads/电视剧/国产剧"
assert file_uri.uri == "Z:/Downloads/电视剧/国产剧"
def test_from_uri_keeps_windows_drive_path_with_backslash() -> None:
"""反斜杠写法的盘符路径同样不能补根斜杠。"""
file_uri = FileURI.from_uri("Z:\\Downloads\\电视剧")
assert file_uri.storage == "local"
assert not file_uri.path.startswith("/")
def test_from_uri_keeps_posix_absolute_path() -> None:
"""POSIX 绝对路径保持原样。"""
file_uri = FileURI.from_uri("/downloads/movies")
assert file_uri.storage == "local"
assert file_uri.path == "/downloads/movies"
def test_from_uri_adds_root_for_relative_path() -> None:
"""无前导斜杠的相对路径仍补全为绝对路径。"""
file_uri = FileURI.from_uri("downloads/movies")
assert file_uri.path == "/downloads/movies"
def test_from_uri_parses_storage_prefix() -> None:
"""带存储前缀的 URI 应拆分出存储类型并保留 POSIX 路径。"""
file_uri = FileURI.from_uri("u115:/media/anime")
assert file_uri.storage == "u115"
assert file_uri.path == "/media/anime"
assert file_uri.uri == "u115:/media/anime"
def test_from_uri_storage_prefix_with_relative_path() -> None:
"""远端存储的无前导斜杠路径补全为绝对路径。"""
file_uri = FileURI.from_uri("rclone:media/anime")
assert file_uri.storage == "rclone"
assert file_uri.path == "/media/anime"
+177
View File
@@ -130,6 +130,7 @@ _config_stub.settings = SimpleNamespace(
LLM_BASE_URL_PRESET=None,
LLM_USER_AGENT=None,
LLM_THINKING_LEVEL=None,
LLM_API_PROTOCOL="auto",
LLM_TEMPERATURE=0.1,
LLM_MAX_CONTEXT_TOKENS=64,
LLM_USE_PROXY=True,
@@ -243,6 +244,7 @@ class LlmHelperTestCallTest(unittest.TestCase):
base_url_preset="deepseek-default",
user_agent=None,
use_proxy=None,
api_protocol=None,
)
self.assertEqual(result["provider"], "deepseek")
self.assertEqual(result["model"], "deepseek-chat")
@@ -870,3 +872,178 @@ class LlmHelperTestCallTest(unittest.TestCase):
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].get("thinking_level"), "high")
self.assertFalse(calls[0].get("include_thoughts"))
def test_get_llm_responses_protocol_forces_responses_api(self):
"""显式 responses 协议应让通用 OpenAI 兼容入口走 Responses API。"""
calls = []
class _FakeChatOpenAI:
def __init__(self, **kwargs):
calls.append(kwargs)
self.model = kwargs["model"]
self.profile = None
with patch.dict(
sys.modules,
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
):
asyncio.run(
llm_module.LLMHelper.get_llm(
provider="openai",
model="gpt-5.6-terra",
api_key="sk-test",
base_url="https://example.com/v1",
api_protocol="responses",
)
)
self.assertEqual(len(calls), 1)
self.assertTrue(calls[0].get("use_responses_api"))
def test_get_llm_chat_completions_protocol_overrides_chatgpt_auto(self):
"""显式 chat_completions 应覆盖 ChatGPT 官方推理模型的自动 Responses 切换。"""
calls = []
class _FakeChatOpenAI:
def __init__(self, **kwargs):
calls.append(kwargs)
self.model = kwargs["model"]
self.profile = None
with patch.dict(
sys.modules,
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
):
asyncio.run(
llm_module.LLMHelper.get_llm(
provider="chatgpt",
model="gpt-5.4",
api_key="sk-test",
base_url="https://api.openai.com/v1",
api_protocol="chat_completions",
)
)
self.assertEqual(len(calls), 1)
self.assertFalse(calls[0].get("use_responses_api"))
def test_get_llm_auto_protocol_keeps_chat_completions_for_compatible(self):
"""auto 协议下通用 OpenAI 兼容入口应保持默认 Chat CompletionsNone)。"""
calls = []
class _FakeChatOpenAI:
def __init__(self, **kwargs):
calls.append(kwargs)
self.model = kwargs["model"]
self.profile = None
with patch.dict(
sys.modules,
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
):
asyncio.run(
llm_module.LLMHelper.get_llm(
provider="openai",
model="gpt-4o",
api_key="sk-test",
base_url="https://example.com/v1",
api_protocol="auto",
)
)
self.assertEqual(len(calls), 1)
self.assertIsNone(calls[0].get("use_responses_api"))
def test_get_llm_runtime_override_beats_chat_completions_protocol(self):
"""运行时强制 ResponsesOAuth/Codex)应优先于用户 chat_completions 设置。"""
calls = []
class _FakeProviderManager:
async def resolve_runtime(self, **kwargs):
return {
"provider_id": kwargs["provider_id"],
"runtime": "openai_compatible",
"model_id": kwargs["model"],
"api_key": kwargs["api_key"],
"base_url": kwargs["base_url"],
"default_headers": None,
"use_responses_api": True,
"model_record": None,
"model_metadata": None,
}
class _FakeChatOpenAI:
def __init__(self, **kwargs):
calls.append(kwargs)
self.model = kwargs["model"]
self.profile = None
provider_module = ModuleType("app.agent.llm.provider")
provider_module.LLMProviderManager = _FakeProviderManager
with patch.dict(
sys.modules,
{
"app.agent.llm.provider": provider_module,
"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI),
},
):
asyncio.run(
llm_module.LLMHelper.get_llm(
provider="chatgpt",
model="gpt-5.4",
api_key="sk-test",
base_url="https://api.openai.com/v1",
api_protocol="chat_completions",
)
)
self.assertEqual(len(calls), 1)
self.assertTrue(calls[0].get("use_responses_api"))
def test_get_llm_reads_api_protocol_from_settings_when_omitted(self):
"""未显式传入协议时应读取 LLM_API_PROTOCOL 配置。"""
calls = []
class _FakeChatOpenAI:
def __init__(self, **kwargs):
calls.append(kwargs)
self.model = kwargs["model"]
self.profile = None
with patch.object(
llm_module.settings, "LLM_API_PROTOCOL", "responses"
), patch.dict(
sys.modules,
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
):
asyncio.run(
llm_module.LLMHelper.get_llm(
provider="openai",
model="gpt-5.6-terra",
api_key="sk-test",
base_url="https://example.com/v1",
)
)
self.assertEqual(len(calls), 1)
self.assertTrue(calls[0].get("use_responses_api"))
def test_normalize_api_protocol_accepts_known_and_falls_back(self):
"""_normalize_api_protocol 应大小写不敏感识别已知值,未知值回退 auto。"""
self.assertEqual(
llm_module.LLMHelper._normalize_api_protocol("Responses"), "responses"
)
self.assertEqual(
llm_module.LLMHelper._normalize_api_protocol("CHAT_COMPLETIONS"),
"chat_completions",
)
self.assertEqual(
llm_module.LLMHelper._normalize_api_protocol("auto"), "auto"
)
self.assertEqual(
llm_module.LLMHelper._normalize_api_protocol(None), "auto"
)
self.assertEqual(
llm_module.LLMHelper._normalize_api_protocol("weird"), "auto"
)
+98
View File
@@ -0,0 +1,98 @@
import asyncio
import json
from types import SimpleNamespace
from typing import Iterator
from unittest.mock import patch
import pytest
from app.agent.tools.base import MoviePilotTool
from app.agent.tools.factory import MoviePilotToolFactory
from app.agent.tools.manager import MoviePilotToolsManager
from app.api.endpoints import mcp
from app.core.plugin import PluginManager
from app.utils.singleton import Singleton
class DemoPluginTool(MoviePilotTool):
"""测试用插件 MCP 工具。"""
name: str = "demo_plugin_tool"
description: str = "测试插件动态注册的 MCP 工具"
async def run(self, **kwargs) -> str:
"""返回固定测试结果。"""
return "plugin-ok"
@pytest.fixture
def plugin_manager() -> Iterator[PluginManager]:
"""构造隔离的插件管理器并在测试后恢复原单例。"""
singleton_key = (PluginManager, (), frozenset())
previous_instance = Singleton._instances.pop(singleton_key, None)
manager = PluginManager()
yield manager
Singleton._instances.pop(singleton_key, None)
if previous_instance is not None:
Singleton._instances[singleton_key] = previous_instance
def _build_plugin() -> SimpleNamespace:
"""构造声明一个 Agent 工具的已启用插件。"""
return SimpleNamespace(
plugin_name="Demo Plugin",
get_state=lambda: True,
get_agent_tools=lambda: [DemoPluginTool],
)
def test_mcp_refreshes_tools_after_plugin_lifecycle_change(
plugin_manager: PluginManager,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""MCP 管理器应发现初始化后新增的插件工具,并在插件移除后停止暴露。"""
with patch.object(
MoviePilotToolFactory,
"_get_builtin_tool_classes",
return_value=[],
):
tool_manager = MoviePilotToolsManager(
session_id="mcp-plugin-test",
user_id="api_user",
)
monkeypatch.setattr(mcp, "moviepilot_tool_manager", tool_manager)
assert asyncio.run(mcp.handle_tools_list()) == {"tools": []}
plugin_manager.running_plugins["DemoPlugin"] = _build_plugin()
plugin_manager.clear_plugin_agent_tools_cache()
listed_tools = asyncio.run(mcp.handle_tools_list())["tools"]
assert [tool["name"] for tool in listed_tools] == ["demo_plugin_tool"]
call_result = asyncio.run(
mcp.handle_tools_call(
{
"name": "demo_plugin_tool",
"arguments": {},
}
)
)
assert call_result == {
"content": [{"type": "text", "text": "plugin-ok"}]
}
plugin_manager.running_plugins.pop("DemoPlugin")
plugin_manager.clear_plugin_agent_tools_cache()
assert asyncio.run(mcp.handle_tools_list()) == {"tools": []}
missing_result = asyncio.run(
mcp.handle_tools_call(
{
"name": "demo_plugin_tool",
"arguments": {},
}
)
)
missing_payload = json.loads(missing_result["content"][0]["text"])
assert "未找到" in missing_payload["error"]
@@ -0,0 +1,134 @@
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from app.chain import ChainBase
from app.core.context import MediaInfo
from app.core.meta import MetaBase
from app.helper.server import MoviePilotServerHelper
from app.schemas.types import MediaType, SystemConfigKey
def _build_meta(name: str) -> MetaBase:
"""构造共享识别统计测试所需的媒体元数据。"""
meta = MetaBase(name)
meta.name = name
meta.type = MediaType.UNKNOWN
return meta
def _shared_params(tmdb_id: int) -> dict:
"""构造共享识别结果转换后的模块参数。"""
return {
"mtype": MediaType.MOVIE,
"source": "themoviedb",
"mediaid": str(tmdb_id),
"tmdbid": tmdb_id,
"doubanid": None,
"bangumiid": None,
"anilistid": None,
}
def _mock_counter(monkeypatch) -> Mock:
"""替换系统配置持久化入口并返回递增调用桩。"""
increment = Mock()
monkeypatch.setattr(
"app.chain.SystemConfigOper",
lambda: SimpleNamespace(increment=increment),
)
return increment
def test_sync_shared_recognize_success_increments_persisted_count(monkeypatch):
"""同步共享识别二次识别成功后应累计一次命中。"""
chain = object.__new__(ChainBase)
meta = _build_meta("共享识别电影")
media = MediaInfo(
title="共享识别电影",
year="2026",
tmdb_id=101,
type=MediaType.MOVIE,
)
increment = _mock_counter(monkeypatch)
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, media]))
monkeypatch.setattr(chain, "_update_local_recognize_cache", Mock())
monkeypatch.setattr(
MoviePilotServerHelper,
"query_recognize_share",
Mock(return_value={"type": "movie", "tmdbid": 101}),
)
monkeypatch.setattr(
MoviePilotServerHelper,
"to_recognize_params",
Mock(return_value=_shared_params(101)),
)
result = chain.recognize_media(meta=meta, cache=False)
assert result is media
increment.assert_called_once_with(SystemConfigKey.MediaRecognizeShareCount)
def test_sync_shared_result_without_local_match_does_not_increment(monkeypatch):
"""共享接口返回数据但二次识别失败时不应累计命中。"""
chain = object.__new__(ChainBase)
meta = _build_meta("共享识别失败电影")
increment = _mock_counter(monkeypatch)
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, None]))
monkeypatch.setattr(
MoviePilotServerHelper,
"query_recognize_share",
Mock(return_value={"type": "movie", "tmdbid": 102}),
)
monkeypatch.setattr(
MoviePilotServerHelper,
"to_recognize_params",
Mock(return_value=_shared_params(102)),
)
result = chain.recognize_media(meta=meta, cache=False)
assert result is None
increment.assert_not_called()
def test_async_shared_recognize_success_increments_persisted_count(monkeypatch):
"""异步共享识别二次识别成功后应累计一次命中。"""
chain = object.__new__(ChainBase)
meta = _build_meta("异步共享识别电影")
media = MediaInfo(
title="异步共享识别电影",
year="2026",
tmdb_id=103,
type=MediaType.MOVIE,
)
increment = _mock_counter(monkeypatch)
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
monkeypatch.setattr(
chain,
"async_run_module",
AsyncMock(side_effect=[None, media]),
)
monkeypatch.setattr(
chain,
"_async_update_local_recognize_cache",
AsyncMock(),
)
monkeypatch.setattr(
MoviePilotServerHelper,
"async_query_recognize_share",
AsyncMock(return_value={"type": "movie", "tmdbid": 103}),
)
monkeypatch.setattr(
MoviePilotServerHelper,
"to_recognize_params",
Mock(return_value=_shared_params(103)),
)
result = asyncio.run(chain.async_recognize_media(meta=meta, cache=False))
assert result is media
increment.assert_called_once_with(SystemConfigKey.MediaRecognizeShareCount)
@@ -0,0 +1,29 @@
from app.helper.service import ServiceConfigHelper
from app.schemas.system import MediaServerConf
from app.schemas.types import SystemConfigKey
def test_mediaserver_conf_tolerates_blank_sync_interval():
"""自动同步间隔为空字符串等非法值时应回退为 None 而不是抛出校验错误。"""
assert MediaServerConf(name="blank", sync_interval="").sync_interval is None
assert MediaServerConf(name="spaces", sync_interval=" ").sync_interval is None
assert MediaServerConf(name="invalid", sync_interval="abc").sync_interval is None
assert MediaServerConf(name="text", sync_interval="12").sync_interval == 12
assert MediaServerConf(name="number", sync_interval=6).sync_interval == 6
assert MediaServerConf(name="none", sync_interval=None).sync_interval is None
def test_get_configs_skips_invalid_entries(monkeypatch):
"""单条配置校验失败时应跳过该条,不影响其它服务配置的加载。"""
monkeypatch.setattr(
"app.helper.service.SystemConfigOper.get",
lambda self, key: [
{"name": "good", "type": "emby", "enabled": True},
"bad-format",
{"name": "bad-type", "type": "plex", "enabled": "maybe"},
],
)
configs = ServiceConfigHelper.get_configs(SystemConfigKey.MediaServers, MediaServerConf)
assert [conf.name for conf in configs] == ["good"]
+273
View File
@@ -0,0 +1,273 @@
from pathlib import Path
from unittest.mock import MagicMock
from app.monitor import LocalDirectoryWatcher, Monitor
def _build_watcher(tmp_path, force_polling):
"""
构造测试用目录监控
:param tmp_path: 监控目录
:param force_polling: 是否强制轮询
:return: 目录监控
"""
return LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=force_polling)
def test_run_retries_with_backoff_in_compatibility_mode(tmp_path, monkeypatch):
"""
兼容模式下监控循环抛异常后应退避重启而不是直接结束线程
"""
monkeypatch.setattr(LocalDirectoryWatcher, "RESTART_BACKOFF", (0,))
watcher = _build_watcher(tmp_path, force_polling=True)
calls = []
def fake_run_watch(force_polling):
"""
模拟底层监控循环持续抛出 FUSE 错误
"""
calls.append(force_polling)
if len(calls) >= 3:
watcher.stop()
raise OSError(131, "State not recoverable")
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
watcher._run()
assert calls == [True, True, True]
assert watcher.restart_count == 2
def test_run_falls_back_to_polling_before_backoff(tmp_path, monkeypatch):
"""
快速模式失败应先降级为兼容模式重试且降级不计入退避重启次数
"""
monkeypatch.setattr(LocalDirectoryWatcher, "RESTART_BACKOFF", (0,))
watcher = _build_watcher(tmp_path, force_polling=None)
calls = []
def fake_run_watch(force_polling):
"""
模拟快速模式与兼容模式先后失败
"""
calls.append(force_polling)
if len(calls) >= 2:
watcher.stop()
raise OSError("inotify watch limit reached")
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
watcher._run()
assert calls == [None, True]
assert watcher.restart_count == 0
def test_run_returns_when_stop_requested(tmp_path, monkeypatch):
"""
收到停止信号后监控循环正常返回不应触发重启
"""
watcher = _build_watcher(tmp_path, force_polling=True)
calls = []
def fake_run_watch(force_polling):
"""
模拟收到停止信号后正常退出的监控循环
"""
calls.append(force_polling)
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
watcher._run()
assert calls == [True]
assert watcher.restart_count == 0
def test_is_stalled_detects_silent_failure(tmp_path):
"""
监控线程存活但长时间无活动时应判定为静默失效
"""
watcher = _build_watcher(tmp_path, force_polling=True)
# 线程未启动时不做判定
assert watcher.is_stalled() is False
thread = MagicMock()
thread.is_alive.return_value = True
watcher._thread = thread
watcher._mark_activity()
assert watcher.is_stalled() is False
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
assert watcher.is_stalled() is True
def test_is_stalled_ignores_stopped_watcher(tmp_path):
"""
已请求停止的监控不应再被判定为静默失效
"""
watcher = _build_watcher(tmp_path, force_polling=True)
thread = MagicMock()
thread.is_alive.return_value = True
watcher._thread = thread
watcher._mark_activity()
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
watcher.stop()
assert watcher.is_stalled() is False
def _build_monitor(monkeypatch, put_recorder):
"""
构造测试用 Monitor 骨架绕过单例初始化
:param monkeypatch: pytest monkeypatch
:param put_recorder: 消息推送记录器
:return: Monitor 骨架
"""
from threading import Lock
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
monitor = object.__new__(Monitor)
monitor._watchers = []
monitor._watcher_lock = Lock()
monitor._pending_locals = []
monitor._alerted_paths = set()
monitor._restart_marks = {}
monitor._stable_cycles = {}
return monitor
def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
"""
构造测试用监控线程替身
:param mon_path: 监控目录
:param alive: 线程是否存活
:param stalled: 是否静默失效
:param restart_count: 自动重启次数
:return: 监控线程替身
"""
watcher = MagicMock()
watcher.watch_path = mon_path
watcher.is_alive.return_value = alive
watcher.is_stalled.return_value = stalled
watcher.restart_count = restart_count
return watcher
def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch):
"""
监控线程退出后健康检查应重建线程并告警
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
watcher = _fake_watcher(tmp_path, alive=False)
monitor._watchers = [watcher]
rebuild = MagicMock()
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
monitor._Monitor__check_watchers()
rebuild.assert_called_once_with(watcher)
put_recorder.put.assert_called_once()
def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch):
"""
静默失效的监控线程也应被健康检查重建
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
rebuild = MagicMock()
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
monitor._Monitor__check_watchers()
rebuild.assert_called_once_with(watcher)
def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch):
"""
自动重启应触发一次告警恢复消息需等满稳定窗口避免来回刷屏
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
watcher = _fake_watcher(tmp_path, alive=True, stalled=False, restart_count=1)
monitor._watchers = [watcher]
monitor._Monitor__check_watchers()
assert str(tmp_path) in monitor._alerted_paths
assert put_recorder.put.call_count == 1
for _ in range(Monitor.RECOVERY_STABLE_CYCLES - 1):
monitor._Monitor__check_watchers()
assert str(tmp_path) in monitor._alerted_paths
monitor._Monitor__check_watchers()
assert str(tmp_path) not in monitor._alerted_paths
assert put_recorder.put.call_count == 2
def test_retry_pending_locals_backs_off(tmp_path, monkeypatch):
"""
启动失败的监控重试应按失败次数退避避免持续故障时刷屏
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
start = MagicMock(return_value=False)
setattr(monitor, "_Monitor__start_local_monitor", start)
for _ in range(6):
monitor._Monitor__retry_pending_locals()
assert start.call_count == 3
def test_dispatcher_retries_after_history_query_failure(monkeypatch):
"""
整理历史查询失败应登记待重试重试成功后进入整理链并清除登记
"""
from app.monitor.dispatcher import TransferDispatcher
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv")
history = MagicMock(side_effect=[None, False])
monkeypatch.setattr(dispatcher, "_has_transfer_history", history)
transfer_chain_instance = MagicMock()
monkeypatch.setattr("app.monitor.dispatcher.TransferChain",
MagicMock(return_value=transfer_chain_instance))
# 首次查询失败:不整理,登记待重试
assert dispatcher.handle_file(storage="local", event_path=event_path, file_size=1) is False
assert len(dispatcher._pending_retries) == 1
transfer_chain_instance.do_transfer.assert_not_called()
# 模拟 TTL 缓存过期后由健康检查驱动重试
dispatcher._cache.clear()
dispatcher.retry_pending()
transfer_chain_instance.do_transfer.assert_called_once()
assert dispatcher._pending_retries == {}
def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
"""
历史查询持续失败达到上限后应放弃重试避免队列无限累积
"""
from app.monitor.dispatcher import TransferDispatcher
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv")
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=None))
dispatcher.handle_file(storage="local", event_path=event_path, file_size=1)
key = f"local:{event_path.as_posix()}"
assert key in dispatcher._pending_retries
dispatcher._pending_retries[key]["attempts"] = TransferDispatcher.MAX_RETRY_ATTEMPTS - 1
dispatcher._cache.clear()
dispatcher.retry_pending()
assert dispatcher._pending_retries == {}
+177
View File
@@ -0,0 +1,177 @@
from pathlib import Path
from unittest.mock import MagicMock
from app.monitor.poller import RemotePoller
from app.monitor.watcher import LocalDirectoryWatcher
def _build_poller(alert_cb=None):
"""
构造测试用远程轮询监控
:param alert_cb: 告警回调替身
:return: (poller, store, dispatcher)
"""
store = MagicMock()
store.save.return_value = True
dispatcher = MagicMock()
dispatcher.is_transfer_candidate_path.return_value = True
dispatcher.handle_file.return_value = True
poller = RemotePoller(store=store, dispatcher=dispatcher, alert_cb=alert_cb)
return poller, store, dispatcher
def _mock_storage_chain(monkeypatch, side_effect):
"""
替换 StorageChain 的快照返回
:param monkeypatch: pytest monkeypatch
:param side_effect: snapshot_storage 的返回序列
:return: StorageChain 实例替身
"""
chain_instance = MagicMock()
chain_instance.snapshot_storage.side_effect = side_effect
monkeypatch.setattr("app.monitor.poller.StorageChain", MagicMock(return_value=chain_instance))
return chain_instance
BASELINE = {
'timestamp': 100,
'file_count': 1,
'snapshot': {'/mon/a.mkv': {'size': 1, 'modify_time': 100}}
}
def test_poll_merges_incremental_into_baseline(monkeypatch):
"""
增量快照应与基线合并落盘未扫到的旧文件不能从基线消失
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}])
file_count = poller.poll("u115", [Path("/mon")])
assert file_count == 2
saved_snapshot = store.save.call_args.args[1]
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon/b.mkv'}
dispatcher.handle_file.assert_called_once()
assert dispatcher.handle_file.call_args.kwargs["event_path"] == Path('/mon/b.mkv')
def test_poll_detects_modified_files(monkeypatch):
"""
增量中已有文件的大小变化应作为修改事件分发并更新基线
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 5, 'modify_time': 300}}])
file_count = poller.poll("u115", [Path("/mon")])
assert file_count == 1
saved_snapshot = store.save.call_args.args[1]
assert saved_snapshot['/mon/a.mkv']['size'] == 5
dispatcher.handle_file.assert_called_once()
def test_poll_partial_failure_merges_success_and_keeps_baseline(monkeypatch):
"""
部分路径快照失败时成功路径合并落盘失败路径保留旧基线
"""
alert_cb = MagicMock()
poller, store, dispatcher = _build_poller(alert_cb)
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [None, {'/mon2/b.mkv': {'size': 2, 'modify_time': 200}}])
file_count = poller.poll("u115", [Path("/mon"), Path("/mon2")])
assert file_count == 2
saved_snapshot = store.save.call_args.args[1]
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon2/b.mkv'}
# 单次失败未达告警阈值
alert_cb.assert_not_called()
def test_poll_all_paths_failed_skips_save(monkeypatch):
"""
全部路径快照失败时本轮不落盘基线保持不变
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [None])
assert poller.poll("u115", [Path("/mon")]) is None
store.save.assert_not_called()
dispatcher.handle_file.assert_not_called()
def test_poll_first_snapshot_failure_builds_no_empty_baseline(monkeypatch):
"""
首次快照失败时不得落盘空基线否则下一轮会把全部存量当作新增
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (None, True)
_mock_storage_chain(monkeypatch, [None])
assert poller.poll("u115", [Path("/mon")]) is None
store.save.assert_not_called()
def test_poll_first_snapshot_success_saves_baseline_without_dispatch(monkeypatch):
"""
首次快照成功仅建立基准不应处理存量文件
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (None, True)
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
assert poller.poll("u115", [Path("/mon")]) == 1
store.save.assert_called_once()
dispatcher.handle_file.assert_not_called()
def test_poll_load_error_skips_round(monkeypatch):
"""
基线读取失败不能当作首次快照应跳过本轮避免丢弃已有基线
"""
poller, store, dispatcher = _build_poller()
store.load_checked.return_value = (None, False)
chain = _mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
assert poller.poll("u115", [Path("/mon")]) is None
chain.snapshot_storage.assert_not_called()
store.save.assert_not_called()
def test_poll_failure_alert_threshold_and_recovery(monkeypatch):
"""
连续异常达到阈值只告警一次恢复后推送恢复消息
"""
alert_cb = MagicMock()
poller, store, dispatcher = _build_poller(alert_cb)
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(
monkeypatch,
[None] * RemotePoller.FAILURE_ALERT_THRESHOLD + [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}]
)
for _ in range(RemotePoller.FAILURE_ALERT_THRESHOLD):
poller.poll("u115", [Path("/mon")])
assert alert_cb.call_count == 1
poller.poll("u115", [Path("/mon")])
assert alert_cb.call_count == 2
assert "已恢复" in alert_cb.call_args.args[1]
def test_watcher_poll_delay_defaults_and_override(tmp_path):
"""
轮询扫描间隔默认取本地值显式传入网络值时生效
"""
default_watcher = LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=True)
assert default_watcher.poll_delay_ms == LocalDirectoryWatcher.POLL_DELAY_LOCAL_MS
network_watcher = LocalDirectoryWatcher(
tmp_path, callback=MagicMock(), force_polling=True,
poll_delay_ms=LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
)
assert network_watcher.poll_delay_ms == LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
+29 -26
View File
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
from watchfiles import Change
from app.monitor import DirectoryChangeEvent, LocalDirectoryWatcher, Monitor
from app.monitor.dispatcher import TransferDispatcher
class CallbackRecorder:
@@ -28,6 +29,20 @@ class CallbackRecorder:
self.events.append((event, text, event_path, file_size))
def _build_monitor_with_dispatcher(handle_file: MagicMock = None):
"""
构造带分发器的测试用 Monitor 骨架
:param handle_file: 替换分发器 handle_file 的替身
:return: (Monitor 骨架, 分发器)
"""
monitor = object.__new__(Monitor)
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
if handle_file is not None:
dispatcher.handle_file = handle_file
monitor._dispatcher = dispatcher
return monitor, dispatcher
def test_handle_changes_dispatches_added_and_modified_files(tmp_path):
"""
新增和修改文件应转换成目录监控整理回调
@@ -120,10 +135,8 @@ def test_event_handler_routes_file_events_to_transfer_handler():
"""
文件事件应继续按 local 存储交给整理流程
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
handle_file = MagicMock()
setattr(monitor, "_Monitor__handle_file", handle_file)
monitor, _ = _build_monitor_with_dispatcher(handle_file)
event_path = Path("/downloads/movie.mkv")
event = DirectoryChangeEvent(
change_type=Change.added,
@@ -149,10 +162,8 @@ def test_event_handler_ignores_directory_events():
"""
目录事件不应进入文件整理流程
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
handle_file = MagicMock()
setattr(monitor, "_Monitor__handle_file", handle_file)
monitor, _ = _build_monitor_with_dispatcher(handle_file)
event_path = Path("/downloads/folder")
event = DirectoryChangeEvent(
change_type=Change.added,
@@ -173,10 +184,8 @@ def test_event_handler_ignores_download_temp_files():
"""
下载器临时文件不应进入整理流程
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
handle_file = MagicMock()
setattr(monitor, "_Monitor__handle_file", handle_file)
monitor, _ = _build_monitor_with_dispatcher(handle_file)
event_path = Path("/downloads/movie.mkv.!qB")
event = DirectoryChangeEvent(
change_type=Change.modified,
@@ -198,10 +207,8 @@ def test_event_handler_ignores_non_transferable_files():
"""
非可整理后缀文件不应进入整理流程
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
handle_file = MagicMock()
setattr(monitor, "_Monitor__handle_file", handle_file)
monitor, _ = _build_monitor_with_dispatcher(handle_file)
event_path = Path("/downloads/movie.nfo")
event = DirectoryChangeEvent(
change_type=Change.added,
@@ -223,9 +230,7 @@ def test_handle_file_skips_transfer_when_history_exists(monkeypatch):
"""
已有整理记录的源文件不应再次进入整理链
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
monitor._cache = {}
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv")
lookups = []
@@ -244,12 +249,12 @@ def test_handle_file_skips_transfer_when_history_exists(monkeypatch):
transfer_chain = MagicMock()
logger_info = MagicMock()
logger_debug = MagicMock()
monkeypatch.setattr("app.monitor.TransferHistoryOper", FakeTransferHistoryOper)
monkeypatch.setattr("app.monitor.TransferChain", transfer_chain)
monkeypatch.setattr("app.monitor.logger.info", logger_info)
monkeypatch.setattr("app.monitor.logger.debug", logger_debug)
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", FakeTransferHistoryOper)
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", transfer_chain)
monkeypatch.setattr("app.monitor.dispatcher.logger.info", logger_info)
monkeypatch.setattr("app.monitor.dispatcher.logger.debug", logger_debug)
handled = monitor._Monitor__handle_file(
handled = dispatcher.handle_file(
storage="local",
event_path=event_path,
file_size=1024,
@@ -266,9 +271,7 @@ def test_handle_file_invokes_transfer_when_history_missing(monkeypatch):
"""
没有整理记录的源文件应继续进入整理链
"""
monitor = object.__new__(Monitor)
monitor.all_exts = [".mkv"]
monitor._cache = {}
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv")
class FakeTransferHistoryOper:
@@ -284,10 +287,10 @@ def test_handle_file_invokes_transfer_when_history_missing(monkeypatch):
transfer_chain_instance = MagicMock()
transfer_chain = MagicMock(return_value=transfer_chain_instance)
monkeypatch.setattr("app.monitor.TransferHistoryOper", FakeTransferHistoryOper)
monkeypatch.setattr("app.monitor.TransferChain", transfer_chain)
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", FakeTransferHistoryOper)
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", transfer_chain)
handled = monitor._Monitor__handle_file(
handled = dispatcher.handle_file(
storage="local",
event_path=event_path,
file_size=1024,
+127
View File
@@ -0,0 +1,127 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from app import schemas
from app.api.endpoints.plugin import plugin_rating, plugin_ratings, rate_plugin
from app.helper.server import MoviePilotServerHelper
def test_server_helper_uses_plugin_rating_endpoints() -> None:
"""评分辅助方法应使用独立中心端路径并传递评分载荷。"""
async def run_scenario() -> None:
with (
patch("app.helper.server.settings.MP_SERVER_HOST", "https://movie-pilot.org"),
patch.object(
MoviePilotServerHelper,
"_async_get",
new=AsyncMock(),
) as get_request,
patch.object(
MoviePilotServerHelper,
"_async_post_json",
new=AsyncMock(),
) as post_request,
):
await MoviePilotServerHelper.async_plugin_ratings(["DemoPlugin", "OtherPlugin"])
await MoviePilotServerHelper.async_plugin_rating("Demo Plugin")
await MoviePilotServerHelper.async_rate_plugin("Demo Plugin", 4.5)
assert get_request.await_args_list[0].args == (
"https://movie-pilot.org/plugin/rating",
)
assert get_request.await_args_list[0].kwargs == {
"params": {"plugin_ids": "DemoPlugin,OtherPlugin"},
"timeout": 10,
}
assert get_request.await_args_list[1].args == (
"https://movie-pilot.org/plugin/rating/Demo%20Plugin",
)
assert post_request.await_args.args == (
"https://movie-pilot.org/plugin/rating/Demo%20Plugin",
{"rating": 4.5},
)
asyncio.run(run_scenario())
def test_plugin_rating_endpoints_return_center_results() -> None:
"""评分查询和提交接口应返回中心端结果并校验安装状态。"""
async def run_scenario() -> None:
rating_result = {
"plugin_id": "DemoPlugin",
"average_rating": 4.3,
"rating_count": 12,
"user_rating": 4.5,
}
with patch.object(
MoviePilotServerHelper,
"async_get_plugin_ratings",
new=AsyncMock(return_value={"DemoPlugin": rating_result}),
) as batch_query:
batch = await plugin_ratings("DemoPlugin", None)
assert batch["DemoPlugin"].average_rating == 4.3
batch_query.assert_awaited_once_with(["DemoPlugin"])
with patch.object(
MoviePilotServerHelper,
"async_get_plugin_rating",
new=AsyncMock(return_value=rating_result),
):
single = await plugin_rating("DemoPlugin", None)
assert single.user_rating == 4.5
system_config = MagicMock()
system_config.get.return_value = ["DemoPlugin"]
with (
patch("app.api.endpoints.plugin.SystemConfigOper", return_value=system_config),
patch.object(
MoviePilotServerHelper,
"async_submit_plugin_rating",
new=AsyncMock(return_value=rating_result),
) as submit_rating,
):
response = await rate_plugin(
"DemoPlugin",
schemas.PluginRatingRequest(rating=4.5),
None,
)
assert response.success is True
assert response.data == rating_result
submit_rating.assert_awaited_once_with("DemoPlugin", 4.5)
asyncio.run(run_scenario())
def test_plugin_rating_rejects_uninstalled_plugin() -> None:
"""未安装插件不能借助 MoviePilot 接口向中心端提交评分。"""
async def run_scenario() -> None:
system_config = MagicMock()
system_config.get.return_value = []
with (
patch("app.api.endpoints.plugin.SystemConfigOper", return_value=system_config),
patch.object(
MoviePilotServerHelper,
"async_submit_plugin_rating",
new=AsyncMock(),
) as submit_rating,
):
with pytest.raises(HTTPException) as error:
await rate_plugin(
"DemoPlugin",
schemas.PluginRatingRequest(rating=4.5),
None,
)
assert error.value.status_code == 400
submit_rating.assert_not_awaited()
asyncio.run(run_scenario())
+220
View File
@@ -0,0 +1,220 @@
import asyncio
import io
from unittest.mock import AsyncMock, Mock, patch
import pytest
from PIL import Image
from app.api.endpoints import system as system_endpoint
from app.helper.image import ImageHelper
def _image_bytes(image_format: str, trailing: bytes = b"") -> bytes:
buffer = io.BytesIO()
Image.new("RGB", (2, 2), color=(32, 96, 160)).save(buffer, format=image_format)
return buffer.getvalue() + trailing
@pytest.mark.parametrize(
("image_format", "expected_mime"),
[
("PNG", "image/png"),
("JPEG", "image/jpeg"),
("GIF", "image/gif"),
("WEBP", "image/webp"),
("PCX", "image/x-pcx"),
("PPM", "image/x-portable-anymap"),
],
)
def test_get_image_mime_type_uses_pillow_detected_format(
image_format: str,
expected_mime: str,
):
assert ImageHelper.get_image_mime_type(_image_bytes(image_format)) == expected_mime
def test_get_image_mime_type_rejects_non_image_pillow_mime():
assert ImageHelper.get_image_mime_type(_image_bytes("EPS")) is None
def test_get_image_mime_type_rejects_scriptable_svg_mime():
with patch.dict(Image.MIME, {"PNG": "image/svg+xml"}):
assert ImageHelper.get_image_mime_type(_image_bytes("PNG")) is None
def test_fetch_image_with_mime_type_only_reads_cached_format_header():
content = _image_bytes("PNG")
image_helper = ImageHelper()
with patch.object(
image_helper.file_cache,
"get",
return_value=content,
), patch.object(
image_helper,
"get_image_mime_type",
return_value="image/png",
) as get_mime_type:
result = image_helper.fetch_image_with_mime_type(
"https://images.example/wallpaper.png"
)
assert result == (content, "image/png")
get_mime_type.assert_called_once_with(content, verify=False)
def test_fetch_image_with_mime_type_validates_network_content_once():
content = _image_bytes("PNG")
image_helper = ImageHelper()
response = Mock(status_code=200, content=content)
request = Mock()
request.get_res.return_value = response
with patch.object(
image_helper.file_cache,
"get",
return_value=None,
), patch.object(
image_helper.file_cache,
"set",
), patch(
"app.helper.image.RequestUtils",
return_value=request,
), patch.object(
image_helper,
"get_image_mime_type",
return_value="image/png",
) as get_mime_type:
result = image_helper.fetch_image_with_mime_type(
"https://images.example/wallpaper.png"
)
assert result == (content, "image/png")
get_mime_type.assert_called_once_with(content)
def test_async_fetch_image_with_mime_type_only_reads_cached_format_header():
content = _image_bytes("PNG")
image_helper = ImageHelper()
with patch.object(
image_helper.async_file_cache,
"get",
new=AsyncMock(return_value=content),
), patch.object(
image_helper,
"get_image_mime_type",
return_value="image/png",
) as get_mime_type:
result = asyncio.run(
image_helper.async_fetch_image_with_mime_type(
"https://images.example/wallpaper.png"
)
)
assert result == (content, "image/png")
get_mime_type.assert_called_once_with(content, verify=False)
def test_async_fetch_image_with_mime_type_validates_network_content_once():
content = _image_bytes("PNG")
image_helper = ImageHelper()
response = Mock(status_code=200, content=content)
request = Mock()
request.get_res = AsyncMock(return_value=response)
with patch.object(
image_helper.async_file_cache,
"get",
new=AsyncMock(return_value=None),
), patch.object(
image_helper.async_file_cache,
"set",
new=AsyncMock(),
), patch(
"app.helper.image.AsyncRequestUtils",
return_value=request,
), patch.object(
image_helper,
"get_image_mime_type",
return_value="image/png",
) as get_mime_type:
result = asyncio.run(
image_helper.async_fetch_image_with_mime_type(
"https://images.example/wallpaper.png"
)
)
assert result == (content, "image/png")
get_mime_type.assert_called_once_with(content)
def test_fetch_image_does_not_trust_active_url_suffix():
content = _image_bytes("PNG", b"<script>window.xss = true</script>")
image_helper = Mock()
image_helper.async_fetch_image_with_mime_type = AsyncMock(
return_value=(content, "image/png")
)
with patch.object(
system_endpoint.SecurityUtils,
"is_safe_image_url_async",
new=AsyncMock(return_value=True),
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
response = asyncio.run(
system_endpoint.fetch_image(
url="https://images.example/wallpaper.html",
allowed_domains={"images.example"},
)
)
assert response is not None
assert response.headers["content-type"] == "image/png"
assert response.headers["x-content-type-options"] == "nosniff"
assert response.body == content
def test_fetch_image_rejects_unverified_content():
image_helper = Mock()
image_helper.async_fetch_image_with_mime_type = AsyncMock(return_value=None)
with patch.object(
system_endpoint.SecurityUtils,
"is_safe_image_url_async",
new=AsyncMock(return_value=True),
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
response = asyncio.run(
system_endpoint.fetch_image(
url="https://images.example/wallpaper.png",
allowed_domains={"images.example"},
)
)
assert response is None
def test_fetch_image_adds_nosniff_to_not_modified_response():
content = _image_bytes("JPEG")
image_helper = Mock()
image_helper.async_fetch_image_with_mime_type = AsyncMock(
return_value=(content, "image/jpeg")
)
etag = system_endpoint.HashUtils.md5(content)
with patch.object(
system_endpoint.SecurityUtils,
"is_safe_image_url_async",
new=AsyncMock(return_value=True),
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
response = asyncio.run(
system_endpoint.fetch_image(
url="https://images.example/wallpaper.jpg",
if_none_match=etag,
allowed_domains={"images.example"},
)
)
assert response is not None
assert response.status_code == 304
assert response.headers["content-type"] == "image/jpeg"
assert response.headers["x-content-type-options"] == "nosniff"
+5
View File
@@ -118,6 +118,8 @@ class LlmTestEndpointTest(unittest.TestCase):
system_endpoint.settings, "LLM_USER_AGENT", "MoviePilot-Test/1.0"
), patch.object(
system_endpoint.settings, "LLM_USE_PROXY", True
), patch.object(
system_endpoint.settings, "LLM_API_PROTOCOL", "responses"
), patch.object(
system_endpoint.LLMHelper,
"test_current_settings",
@@ -135,6 +137,7 @@ class LlmTestEndpointTest(unittest.TestCase):
base_url_preset="deepseek-default",
user_agent="MoviePilot-Test/1.0",
use_proxy=True,
api_protocol="responses",
)
self.assertTrue(resp.success)
self.assertEqual(resp.data["provider"], "deepseek")
@@ -186,6 +189,7 @@ class LlmTestEndpointTest(unittest.TestCase):
base_url_preset="openai-default",
user_agent="MoviePilot-Custom/1.0",
use_proxy=False,
api_protocol=None,
)
self.assertTrue(resp.success)
self.assertEqual(resp.data["provider"], "openai")
@@ -228,6 +232,7 @@ class LlmTestEndpointTest(unittest.TestCase):
base_url_preset="deepseek-default",
user_agent=None,
use_proxy=None,
api_protocol=None,
)
self.assertTrue(resp.success)
+8 -4
View File
@@ -88,7 +88,9 @@ class NettestSecurityTest(unittest.TestCase):
image_url = "http://192.168.1.50:8096/System/Info/Public"
signed_url = system_endpoint.SecurityUtils.sign_url(image_url)
image_helper = Mock()
image_helper.async_fetch_image = AsyncMock(return_value=b"image-bytes")
image_helper.async_fetch_image_with_mime_type = AsyncMock(
return_value=(b"image-bytes", "image/jpeg")
)
with patch.object(system_endpoint, "ImageHelper", return_value=image_helper), patch.object(
system_endpoint.HashUtils, "md5", return_value="etag", create=True
@@ -103,7 +105,7 @@ class NettestSecurityTest(unittest.TestCase):
)
self.assertEqual(resp.status_code, 200)
image_helper.async_fetch_image.assert_awaited_once_with(
image_helper.async_fetch_image_with_mime_type.assert_awaited_once_with(
url=image_url,
proxy=None,
use_cache=False,
@@ -133,7 +135,9 @@ class NettestSecurityTest(unittest.TestCase):
图片代理在域名白名单命中后可按配置放行指定非公网解析网段
"""
image_helper = Mock()
image_helper.async_fetch_image = AsyncMock(return_value=b"image-bytes")
image_helper.async_fetch_image_with_mime_type = AsyncMock(
return_value=(b"image-bytes", "image/jpeg")
)
with patch.object(system_endpoint, "ImageHelper", return_value=image_helper), patch.object(
system_endpoint.HashUtils, "md5", return_value="etag", create=True
@@ -161,7 +165,7 @@ class NettestSecurityTest(unittest.TestCase):
)
self.assertEqual(resp.status_code, 200)
image_helper.async_fetch_image.assert_awaited_once_with(
image_helper.async_fetch_image_with_mime_type.assert_awaited_once_with(
url="https://img1.doubanio.com/poster.webp",
proxy=None,
use_cache=False,
+51
View File
@@ -0,0 +1,51 @@
import threading
from concurrent.futures import ThreadPoolExecutor
from app.db.systemconfig_oper import SystemConfigOper
from app.schemas.types import SystemConfigKey
def test_increment_serializes_concurrent_counter_updates(monkeypatch):
"""并发递增系统计数时不应丢失更新。"""
oper = object.__new__(SystemConfigOper)
oper._rlock = threading.RLock()
stored_value = {"value": 0}
monkeypatch.setattr(oper, "get", lambda _key: stored_value["value"])
monkeypatch.setattr(
oper,
"set",
lambda _key, value: stored_value.update(value=value),
)
with ThreadPoolExecutor(max_workers=8) as executor:
results = list(
executor.map(
lambda _index: oper.increment(
SystemConfigKey.MediaRecognizeShareCount
),
range(100),
)
)
assert sorted(results) == list(range(1, 101))
assert stored_value["value"] == 100
def test_increment_supports_custom_step(monkeypatch):
"""整数系统计数应支持指定递增步长。"""
oper = object.__new__(SystemConfigOper)
oper._rlock = threading.RLock()
stored_value = {"value": 4}
monkeypatch.setattr(oper, "get", lambda _key: stored_value["value"])
monkeypatch.setattr(
oper,
"set",
lambda _key, value: stored_value.update(value=value),
)
result = oper.increment(SystemConfigKey.MediaRecognizeShareCount, step=3)
assert result == 7
assert stored_value["value"] == 7
+14 -1
View File
@@ -1,10 +1,11 @@
import asyncio
import inspect
from unittest.mock import Mock
from app.api.endpoints import tmdb as tmdb_endpoint
from app.db.user_oper import get_current_active_superuser_async
from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.schemas.types import MediaType
from app.schemas.types import MediaType, SystemConfigKey
class _MemoryCacheStub:
@@ -97,7 +98,14 @@ def test_tmdb_cache_endpoint_returns_management_statistics(monkeypatch):
"recognized": {"id": 1, "title": "Alpha", "type": MediaType.MOVIE},
"unrecognized": {"id": 0},
})
get_system_config = Mock(return_value=7)
monkeypatch.setattr(tmdb_endpoint, "TmdbCache", lambda: cache)
monkeypatch.setattr(
tmdb_endpoint,
"SystemConfigOper",
lambda: type("SystemConfigStub", (), {"get": get_system_config})(),
)
monkeypatch.setattr(tmdb_endpoint.settings, "MEDIA_RECOGNIZE_SHARE", True)
response = asyncio.run(tmdb_endpoint.tmdb_recognition_cache(None))
@@ -105,6 +113,11 @@ def test_tmdb_cache_endpoint_returns_management_statistics(monkeypatch):
assert response.data["count"] == 2
assert response.data["recognized"] == 1
assert response.data["unrecognized"] == 1
assert response.data["shared_recognized"] == 7
assert response.data["shared_recognize_enabled"] is True
get_system_config.assert_called_once_with(
SystemConfigKey.MediaRecognizeShareCount
)
def test_tmdb_cache_delete_endpoint_reports_missing_item(monkeypatch):
+3
View File
@@ -183,6 +183,7 @@ class TransferJobManagerTest(unittest.TestCase):
target_oper = SimpleNamespace(
get_folder=lambda path: target_folder,
get_item=lambda path: None,
get_item_strict=lambda path: None,
)
new_item, errmsg = TransHandler._TransHandler__transfer_command(
@@ -243,6 +244,7 @@ class TransferJobManagerTest(unittest.TestCase):
target_oper = SimpleNamespace(
get_folder=lambda path: target_folder,
get_item=lambda path: None,
get_item_strict=lambda path: None,
)
with patch.object(
@@ -313,6 +315,7 @@ class TransferJobManagerTest(unittest.TestCase):
target_oper = SimpleNamespace(
get_folder=lambda path: target_folder,
get_item=lambda path: None,
get_item_strict=lambda path: None,
)
in_meta = MetaVideo("Test.Show.S02E03")
+146
View File
@@ -0,0 +1,146 @@
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from app.chain.transfer import TransferChain
from app.schemas import FileItem, TransferDirectoryConf, TransferTask
from app.utils.system import SystemUtils
def _make_task(
storage: str = "local",
download_path: str = "/mnt/clouddrive/downloads",
) -> TransferTask:
return TransferTask(
fileitem=FileItem(
storage=storage,
path=f"{download_path}/Test.Show.S01E01.mkv",
type="file",
name="Test.Show.S01E01.mkv",
),
target_directory=TransferDirectoryConf(
storage=storage,
download_path=download_path,
),
)
def test_enabled_cleanup_skips_filesystem_detection():
"""
开关开启时应保持旧行为且不产生额外文件系统检测
"""
with patch(
"app.chain.transfer.SystemUtils.is_network_filesystem"
) as is_network_filesystem:
should_delete = (
TransferChain._TransferChain__should_delete_empty_source_directories(
_make_task(),
True,
{},
)
)
assert should_delete is True
is_network_filesystem.assert_not_called()
def test_disabled_cleanup_keeps_mounted_local_source_directories():
"""
开关关闭时应保留网络或 FUSE 挂载的本地源目录
"""
with patch(
"app.chain.transfer.SystemUtils.is_network_filesystem",
return_value=True,
) as is_network_filesystem:
should_delete = (
TransferChain._TransferChain__should_delete_empty_source_directories(
_make_task(),
False,
{},
)
)
assert should_delete is False
is_network_filesystem.assert_called_once_with(
Path("/mnt/clouddrive/downloads"), include_local_fuse=True
)
def test_disabled_cleanup_still_deletes_ordinary_local_source_directories():
"""
开关关闭时普通本地文件系统仍应删除空目录
"""
with patch(
"app.chain.transfer.SystemUtils.is_network_filesystem",
return_value=False,
):
should_delete = (
TransferChain._TransferChain__should_delete_empty_source_directories(
_make_task(download_path="/downloads"),
False,
{},
)
)
assert should_delete is True
def test_disabled_cleanup_does_not_change_remote_storage_cleanup():
"""
开关关闭时非本地存储仍应执行原有空目录清理
"""
with patch(
"app.chain.transfer.SystemUtils.is_network_filesystem"
) as is_network_filesystem:
should_delete = (
TransferChain._TransferChain__should_delete_empty_source_directories(
_make_task(storage="alist", download_path="/downloads"),
False,
{},
)
)
assert should_delete is True
is_network_filesystem.assert_not_called()
def test_mounted_filesystem_detection_is_cached_by_source_directory():
"""
同一源根目录的批量任务应只检测一次文件系统
"""
mounted_filesystem_cache = {}
with patch(
"app.chain.transfer.SystemUtils.is_network_filesystem",
return_value=True,
) as is_network_filesystem:
for _ in range(2):
should_delete = (
TransferChain._TransferChain__should_delete_empty_source_directories(
_make_task(),
False,
mounted_filesystem_cache,
)
)
assert should_delete is False
is_network_filesystem.assert_called_once_with(
Path("/mnt/clouddrive/downloads"), include_local_fuse=True
)
def test_cleanup_detection_includes_local_fuse_mounts():
"""
空目录清理场景应将原本排除的本地 FUSE 文件系统视为挂载盘
"""
df_result = SimpleNamespace(
returncode=0,
stdout="Filesystem Type 1K-blocks Used Available Use% Mounted on\n"
"shfs fuse.shfs 1 1 1 1% /mnt/user\n",
)
with patch("app.utils.system.platform.system", return_value="Linux"), patch(
"app.utils.system.subprocess.run", return_value=df_result
):
assert SystemUtils.is_network_filesystem(Path("/mnt/user")) is False
assert SystemUtils.is_network_filesystem(
Path("/mnt/user"), include_local_fuse=True
) is True
+186
View File
@@ -0,0 +1,186 @@
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from app.modules.filemanager.storages.alipan import AliPan
from app.modules.filemanager.storages.local import LocalStorage
from app.modules.filemanager.storages.rclone import Rclone
from app.modules.filemanager.storages.u115 import U115Pan
from app.schemas.exception import StorageQueryError
def _local() -> LocalStorage:
"""
构造本地存储实例跳过初始化
"""
return object.__new__(LocalStorage)
def _u115() -> U115Pan:
"""
构造 115 存储实例跳过初始化
"""
return object.__new__(U115Pan)
def _alipan(monkeypatch) -> AliPan:
"""
构造阿里云盘存储实例跳过初始化_default_drive_id 为只读属性需在类级替换
"""
monkeypatch.setattr(AliPan, "_default_drive_id", "drive-1", raising=False)
return object.__new__(AliPan)
def test_local_strict_missing_file_returns_none(tmp_path):
"""
目标文件确实不存在时应确认为不存在允许正常整理
"""
assert _local().get_item_strict(tmp_path / "missing.mkv") is None
def test_local_strict_existing_file_returns_item(tmp_path):
"""
目标文件存在时应返回文件项
"""
target = tmp_path / "movie.mkv"
target.write_bytes(b"movie")
item = _local().get_item_strict(target)
assert item is not None
assert item.path == target.as_posix()
def test_local_strict_broken_symlink_returns_none(tmp_path):
"""
失效软链接视为目标不存在不应阻断整理
"""
target = tmp_path / "movie.mkv"
target.symlink_to(tmp_path / "gone.mkv")
assert _local().get_item_strict(target) is None
def test_local_strict_raises_on_stat_error(tmp_path, monkeypatch):
"""
FUSE 挂载抖动导致 stat 失败时应抛出 StorageQueryError拒绝覆盖
"""
target = tmp_path / "movie.mkv"
def raise_stat_error(self, *args, **kwargs):
"""
模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE
"""
raise OSError(131, "State not recoverable")
monkeypatch.setattr(Path, "stat", raise_stat_error)
with pytest.raises(StorageQueryError):
_local().get_item_strict(target)
def test_u115_strict_transport_failure_raises():
"""
115 请求失败网络/限流重试用尽时应抛出 StorageQueryError
"""
storage = _u115()
storage._request_api = MagicMock(return_value=None)
with pytest.raises(StorageQueryError):
storage.get_item_strict(Path("/movie.mkv"))
def test_u115_get_item_keeps_swallowing_transport_failure():
"""
宽松版 get_item 行为保持兼容请求失败仍返回 None
"""
storage = _u115()
storage._request_api = MagicMock(return_value=None)
assert storage.get_item(Path("/movie.mkv")) is None
def test_u115_strict_confirmed_absent_returns_none():
"""
115 业务码返回记录不存在data 为空时应确认为不存在
"""
storage = _u115()
storage._request_api = MagicMock(return_value={"state": True, "code": 20004, "data": {}})
assert storage.get_item_strict(Path("/movie.mkv")) is None
def test_u115_strict_returns_item():
"""
115 返回有效文件数据时应构造文件项
"""
storage = _u115()
storage._request_api = MagicMock(return_value={"state": True, "code": 0, "data": {
"file_id": 123,
"file_category": "1",
"file_name": "movie.mkv",
"pick_code": "abc",
"size_byte": 1024,
"utime": 100,
}})
item = storage.get_item_strict(Path("/movie.mkv"))
assert item is not None
assert item.fileid == "123"
assert item.size == 1024
def test_alipan_strict_notfound_returns_none(monkeypatch):
"""
阿里云盘 NotFound 系列错误码应确认为不存在
"""
storage = _alipan(monkeypatch)
storage._request_api = MagicMock(return_value={"code": "NotFound.File", "message": "not found"})
assert storage.get_item_strict(Path("/movie.mkv")) is None
def test_alipan_strict_other_error_raises(monkeypatch):
"""
阿里云盘非 NotFound 的业务错误如限流应抛出 StorageQueryError
"""
storage = _alipan(monkeypatch)
storage._request_api = MagicMock(return_value={"code": "TooManyRequests", "message": "limit"})
with pytest.raises(StorageQueryError):
storage.get_item_strict(Path("/movie.mkv"))
def test_alipan_strict_transport_failure_raises(monkeypatch):
"""
阿里云盘请求失败时应抛出 StorageQueryError
"""
storage = _alipan(monkeypatch)
storage._request_api = MagicMock(return_value=None)
with pytest.raises(StorageQueryError):
storage.get_item_strict(Path("/movie.mkv"))
def test_alipan_strict_returns_item(monkeypatch):
"""
阿里云盘返回有效数据时应构造文件项
"""
storage = _alipan(monkeypatch)
storage._request_api = MagicMock(return_value={"file_id": "f1", "name": "movie.mkv"})
setattr(storage, "_AliPan__get_fileitem", MagicMock(return_value="ITEM"))
assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM"
def test_storage_base_strict_defaults_to_get_item():
"""
未覆写的存储沿用 get_item 判定行为不变
"""
storage = object.__new__(Rclone)
storage.get_item = MagicMock(return_value=None)
assert storage.get_item_strict(Path("/movie.mkv")) is None
storage.get_item.assert_called_once()
+2 -2
View File
@@ -1,2 +1,2 @@
APP_VERSION = 'v2.15.1'
FRONTEND_VERSION = 'v2.15.1'
APP_VERSION = 'v2.15.2'
FRONTEND_VERSION = 'v2.15.2'