mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-28 03:27:31 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93761fe7e4 | ||
|
|
593139faac | ||
|
|
6c89f1eb4b | ||
|
|
2310a3a456 | ||
|
|
48852350a0 | ||
|
|
a23fce1491 | ||
|
|
c976741574 | ||
|
|
04facef64d | ||
|
|
33a97eb2c8 | ||
|
|
cf80b551f9 | ||
|
|
e011b20210 | ||
|
|
bdf395f494 | ||
|
|
68686bc23a | ||
|
|
1a528c7803 | ||
|
|
6b4a255f26 | ||
|
|
1a3c1b8b39 | ||
|
|
8788dae34b | ||
|
|
bb00814d7a | ||
|
|
3d55d44457 |
@@ -729,6 +729,7 @@ class MoviePilotAgent:
|
||||
user_agent=settings.LLM_USER_AGENT,
|
||||
use_proxy=settings.LLM_USE_PROXY,
|
||||
thinking_level=settings.LLM_THINKING_LEVEL,
|
||||
api_protocol=settings.LLM_API_PROTOCOL,
|
||||
)
|
||||
selected_event = await eventmanager.async_send_event(
|
||||
ChainEventType.AgentLLMProvider,
|
||||
@@ -769,6 +770,9 @@ class MoviePilotAgent:
|
||||
)
|
||||
or settings.LLM_THINKING_LEVEL
|
||||
)
|
||||
api_protocol = self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "api_protocol")
|
||||
) or settings.LLM_API_PROTOCOL
|
||||
selected_provider_id = self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "selected_provider_id")
|
||||
)
|
||||
@@ -794,6 +798,7 @@ class MoviePilotAgent:
|
||||
"user_agent": user_agent,
|
||||
"use_proxy": bool(use_proxy),
|
||||
"thinking_level": thinking_level,
|
||||
"api_protocol": api_protocol,
|
||||
}
|
||||
return self._llm_runtime_config
|
||||
|
||||
@@ -1029,6 +1034,7 @@ class MoviePilotAgent:
|
||||
runtime_config.get("user_agent"),
|
||||
bool(runtime_config.get("use_proxy")),
|
||||
runtime_config.get("thinking_level"),
|
||||
runtime_config.get("api_protocol"),
|
||||
)
|
||||
|
||||
async def _agent_bundle_signature(self, streaming: bool) -> tuple[Any, ...]:
|
||||
|
||||
+38
-5
@@ -846,19 +846,31 @@ class LLMHelper:
|
||||
provider: str,
|
||||
model: str | None,
|
||||
runtime: dict[str, Any],
|
||||
api_protocol: str | None = None,
|
||||
) -> bool | None:
|
||||
"""
|
||||
判断官方 ChatGPT API Key 模式是否应使用 Responses API。
|
||||
判断本次 OpenAI 兼容调用是否应使用 Responses API。
|
||||
|
||||
GPT-5/o 系推理模型在 Chat Completions 中组合 function tools 与
|
||||
reasoning_effort 时会被官方端点拒绝,因此 ChatGPT 官方 API Key
|
||||
模式需要显式切到 Responses API;通用 OpenAI-compatible 入口保持
|
||||
provider 目录解析出的默认行为,避免误伤第三方兼容服务。
|
||||
优先级:
|
||||
1. 运行时显式要求(ChatGPT Plus/Pro OAuth、Codex 等端点契约),始终保留;
|
||||
2. 用户通过 ``LLM_API_PROTOCOL`` 显式指定 ``responses`` / ``chat_completions``;
|
||||
3. ``auto``(默认)保持原有 ChatGPT 官方 API Key + GPT-5/o 系推理模型
|
||||
自动切换逻辑,通用 OpenAI 兼容入口仍走 Chat Completions,
|
||||
避免误伤第三方兼容服务。
|
||||
|
||||
:param api_protocol: 显式传入的 API 协议,未传入时读取 ``LLM_API_PROTOCOL``
|
||||
:return: True/False 强制指定协议;None 表示交由 LangChain 默认行为
|
||||
"""
|
||||
runtime_use_responses_api = runtime.get("use_responses_api")
|
||||
if runtime_use_responses_api is not None:
|
||||
return bool(runtime_use_responses_api)
|
||||
|
||||
protocol = cls._normalize_api_protocol(api_protocol)
|
||||
if protocol == "responses":
|
||||
return True
|
||||
if protocol == "chat_completions":
|
||||
return False
|
||||
|
||||
provider_name = (provider or "").strip().lower()
|
||||
if provider_name != "chatgpt":
|
||||
return None
|
||||
@@ -872,6 +884,18 @@ class LLMHelper:
|
||||
return True
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_api_protocol(api_protocol: str | None) -> str:
|
||||
"""
|
||||
规范化 API 协议配置,未知值统一回退为 ``auto`` 以保持兼容。
|
||||
"""
|
||||
normalized = str(api_protocol or settings.LLM_API_PROTOCOL or "").strip().lower()
|
||||
if normalized in {"auto", "chat_completions", "responses"}:
|
||||
return normalized
|
||||
if normalized:
|
||||
logger.warning(f"忽略不支持的 LLM_API_PROTOCOL 配置: {api_protocol}")
|
||||
return "auto"
|
||||
|
||||
@staticmethod
|
||||
def _attach_runtime_metadata(model: Any, runtime: dict[str, Any]) -> None:
|
||||
"""
|
||||
@@ -954,6 +978,7 @@ class LLMHelper:
|
||||
user_agent: str | None = None,
|
||||
temperature: Optional[float] = None,
|
||||
use_proxy: bool | None = None,
|
||||
api_protocol: str | None = None,
|
||||
):
|
||||
"""
|
||||
获取LLM实例
|
||||
@@ -970,6 +995,10 @@ class LLMHelper:
|
||||
:param user_agent: OpenAI兼容接口请求 User-Agent。未显式传入时使用配置项 LLM_USER_AGENT。
|
||||
:param temperature: LLM 温度参数。未显式传入时使用配置项 LLM_TEMPERATURE。
|
||||
:param use_proxy: 是否为本次 LLM 调用使用系统代理。未显式传入时使用配置项 LLM_USE_PROXY。
|
||||
:param api_protocol: OpenAI 兼容接口 API 协议
|
||||
(auto/chat_completions/responses)。未显式传入时使用配置项 LLM_API_PROTOCOL。
|
||||
仅对 OpenAI 兼容运行时生效;``responses`` 强制走 Responses API,
|
||||
``chat_completions`` 强制走 Chat Completions,``auto`` 保持原有自动判断。
|
||||
:return: LLM实例
|
||||
"""
|
||||
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).lower()
|
||||
@@ -1021,6 +1050,7 @@ class LLMHelper:
|
||||
provider=provider_name,
|
||||
model=model_name,
|
||||
runtime=runtime,
|
||||
api_protocol=api_protocol,
|
||||
)
|
||||
llm_proxy = _resolve_llm_proxy(use_proxy)
|
||||
|
||||
@@ -1210,11 +1240,13 @@ class LLMHelper:
|
||||
user_agent: str | None = None,
|
||||
temperature: Optional[float] = None,
|
||||
use_proxy: bool | None = None,
|
||||
api_protocol: str | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
使用当前配置或显式传入的临时配置执行一次最小 LLM 调用。
|
||||
|
||||
:param temperature: LLM 温度参数。未显式传入时沿用已保存配置。
|
||||
:param api_protocol: OpenAI 兼容接口 API 协议,未显式传入时沿用已保存配置。
|
||||
"""
|
||||
provider_name = provider if provider is not None else settings.LLM_PROVIDER
|
||||
model_name = model if model is not None else settings.LLM_MODEL
|
||||
@@ -1229,6 +1261,7 @@ class LLMHelper:
|
||||
"base_url_preset": base_url_preset,
|
||||
"user_agent": user_agent,
|
||||
"use_proxy": use_proxy,
|
||||
"api_protocol": api_protocol,
|
||||
}
|
||||
if temperature is not None:
|
||||
llm_kwargs["temperature"] = temperature
|
||||
|
||||
@@ -17,7 +17,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- Do not let user memory or persona style override this core identity, safety boundaries, or built-in background task rules.
|
||||
- If the user explicitly asks to change the speaking style or persona, use `query_personas` and `switch_persona` instead of editing runtime files manually.
|
||||
- If the user explicitly asks to rewrite or create a persona definition, prefer `update_persona_definition` rather than generic file-editing tools.
|
||||
- Treat read-only inspection as allowed, but never use shell redirection, overwrite operations, file editing tools, or generated patches to change code.
|
||||
</non_negotiable_boundaries>
|
||||
|
||||
<confirmation_policy>
|
||||
@@ -66,7 +65,11 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- If `search_media` fails, fall back to `search_web` or `recognize_media`. Only ask the user when automated paths are exhausted.
|
||||
- If torrent search yields no useful result, check site scope, site health, and recognition quality before concluding that the resource is unavailable.
|
||||
- Reuse the latest torrent search cache for `get_search_results` and `add_download_tasks` instead of re-running the same search unnecessarily.
|
||||
- Use `execute_command` only for diagnostics, read-only inspection, or commands the user explicitly asked to run. Its default `action=start` starts a managed background session and returns `session_id`, `status`, `last_seq`, and `output_until_seq`; call the same tool again with `action=read`, `action=wait`, `action=write`, or `action=kill` to poll output, wait in short segments, send stdin, or stop the process.
|
||||
- For administrator code discovery across local files, use `execute_command(action="run")` with `rg` and narrow globs or paths. Use `list_directory` to inspect one known directory or a supported remote storage backend, and use `read_file` when the exact local file is known.
|
||||
- Read the relevant file before changing it. Use `edit_file` for localized exact replacements; make `old_text` unique with enough surrounding context, and use `replace_all=true` only when every match must change. Use `write_file` for new files; set `overwrite=true` only for an intentional full rewrite, and use `read_file(include_metadata=true)` plus `expected_sha256` when preserving the previously read version matters.
|
||||
- When implementation depends on a Python or Node.js API, first identify the installed or locked dependency version from environment metadata, requirements, package manifests, lockfiles, local source, and type declarations. Use `rg` against the relevant package directory, `.venv`, or `node_modules` instead of scanning the entire project without bounds. If local evidence is insufficient, use `search_web` and then `browse_webpage` to read the matching version of the official documentation. Do not guess signatures from memory, mix examples from incompatible versions, or install a package only to inspect its API.
|
||||
- Use structured file tools for source edits because they enforce file access boundaries and conflict checks. Never use shell redirection, inline scripts, or another tool to bypass a file-tool permission denial.
|
||||
- Use `execute_command` for administrator-only multi-file diagnostics, tests, Git, service operations, SSH, or an exact command the user requested. Use `action=run` for short bounded commands. Use `action=start` for long-running or interactive commands, including SSH; then continue with `read`, `wait`, `write`, or `kill` using the returned `session_id`. Do not start a background session for a short command that can finish within `action=run`.
|
||||
</tool_strategy>
|
||||
|
||||
<media_rules>
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Agent 文件写入工具的共享辅助函数。"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class FileVersionConflictError(RuntimeError):
|
||||
"""目标文件在准备写入期间发生变化。"""
|
||||
|
||||
|
||||
def calculate_file_sha256(path: Path) -> str:
|
||||
"""计算文件原始字节的 SHA-256,用于检测陈旧写入。"""
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(64 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic_write_text(
|
||||
path: Path,
|
||||
content: str,
|
||||
expected_sha256: str | None = None,
|
||||
) -> None:
|
||||
"""校验目标版本后,在同目录写入临时文件并原子替换文本。"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temp_name = tempfile.mkstemp(
|
||||
dir=path.parent,
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
)
|
||||
temp_path = Path(temp_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as file_handle:
|
||||
file_handle.write(content)
|
||||
file_handle.flush()
|
||||
os.fsync(file_handle.fileno())
|
||||
|
||||
if expected_sha256:
|
||||
if (
|
||||
not path.is_file()
|
||||
or calculate_file_sha256(path).casefold()
|
||||
!= expected_sha256.casefold()
|
||||
):
|
||||
raise FileVersionConflictError(str(path))
|
||||
if path.exists():
|
||||
os.chmod(temp_path, path.stat().st_mode)
|
||||
os.replace(temp_path, path)
|
||||
finally:
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
@@ -1,4 +1,4 @@
|
||||
"""文件编辑工具"""
|
||||
"""文件精确编辑工具。"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.impl._file_write_utils import (
|
||||
FileVersionConflictError,
|
||||
atomic_write_text,
|
||||
calculate_file_sha256,
|
||||
)
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.log import logger
|
||||
|
||||
@@ -15,18 +20,44 @@ class EditFileInput(BaseModel):
|
||||
"""文件编辑工具的输入参数模型。"""
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to edit")
|
||||
old_text: str = Field(..., description="The exact old text to be replaced")
|
||||
old_text: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The exact old text to replace. It must be non-empty and uniquely "
|
||||
"identify one location unless replace_all is true."
|
||||
),
|
||||
)
|
||||
new_text: str = Field(..., description="The new text to replace with")
|
||||
replace_all: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Replace every exact match. Keep false for normal code edits so an "
|
||||
"ambiguous match fails instead of changing multiple locations."
|
||||
),
|
||||
)
|
||||
expected_sha256: Optional[str] = Field(
|
||||
None,
|
||||
pattern=r"^[0-9a-fA-F]{64}$",
|
||||
description=(
|
||||
"Optional SHA-256 returned by read_file(include_metadata=true). The "
|
||||
"edit fails if the file changed after it was read."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class EditFileTool(MoviePilotTool):
|
||||
"""使用精确文本匹配安全编辑本地文件。"""
|
||||
|
||||
name: str = "edit_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Write,
|
||||
ToolTag.File,
|
||||
]
|
||||
description: str = (
|
||||
"Edit a local text file by replacing specific old text with new text. "
|
||||
"Edit an existing local text file using an exact text match. By default "
|
||||
"the match must occur exactly once; use replace_all only for intentional "
|
||||
"bulk replacement. old_text cannot be empty, and new files must be "
|
||||
"created with write_file. Supports an optional SHA-256 conflict check. "
|
||||
"Non-admin users can only edit files inside the MoviePilot Agent config "
|
||||
"directory."
|
||||
)
|
||||
@@ -38,7 +69,16 @@ class EditFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"编辑文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, old_text: str, new_text: str, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
old_text: str,
|
||||
new_text: str,
|
||||
replace_all: bool = False,
|
||||
expected_sha256: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""校验精确匹配和可选文件版本后,以原子方式写入编辑结果。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
|
||||
|
||||
try:
|
||||
@@ -48,37 +88,74 @@ class EditFileTool(MoviePilotTool):
|
||||
if access_error:
|
||||
return access_error
|
||||
|
||||
path = AsyncPath(resolved_path)
|
||||
# 校验逻辑:如果要替换特定文本,文件必须存在且包含该文本
|
||||
if not await path.exists():
|
||||
# 如果 old_text 为空,可能用户想直接创建文件,但通常 edit_file 需要匹配旧内容
|
||||
if old_text:
|
||||
return f"错误:文件 {resolved_path} 不存在,无法进行内容替换。"
|
||||
if not old_text:
|
||||
return "错误:old_text 不能为空;创建或完整写入文件请使用 write_file。"
|
||||
|
||||
if await path.exists() and not await path.is_file():
|
||||
path = AsyncPath(resolved_path)
|
||||
if not await path.exists():
|
||||
return f"错误:文件 {resolved_path} 不存在;创建文件请使用 write_file。"
|
||||
|
||||
if not await path.is_file():
|
||||
return f"错误:{resolved_path} 不是一个文件"
|
||||
|
||||
if await path.exists():
|
||||
content = await path.read_text(encoding="utf-8", errors="replace")
|
||||
if old_text not in content:
|
||||
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
|
||||
return f"错误:在文件 {resolved_path} 中未找到指定的旧文本。请确保包含所有的空格、缩进 and 换行符。"
|
||||
occurrences = content.count(old_text)
|
||||
new_content = content.replace(old_text, new_text)
|
||||
else:
|
||||
# 文件不存在且 old_text 为空的情形(初始化新文件)
|
||||
new_content = new_text
|
||||
occurrences = 1
|
||||
local_path = Path(resolved_path)
|
||||
current_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
if (
|
||||
expected_sha256
|
||||
and current_sha256.casefold() != expected_sha256.casefold()
|
||||
):
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并基于最新内容编辑。"
|
||||
)
|
||||
|
||||
# 自动创建父目录
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = await path.read_text(encoding="utf-8", errors="strict")
|
||||
occurrences = content.count(old_text)
|
||||
if occurrences == 0:
|
||||
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
|
||||
return (
|
||||
f"错误:在文件 {resolved_path} 中未找到指定的旧文本。"
|
||||
"请重新读取文件并确认空格、缩进和换行。"
|
||||
)
|
||||
if occurrences > 1 and not replace_all:
|
||||
return (
|
||||
f"错误:old_text 在文件 {resolved_path} 中匹配到 {occurrences} 处,"
|
||||
"为避免误改已拒绝编辑。请提供更多上下文使其唯一,或明确设置 "
|
||||
"replace_all=true。"
|
||||
)
|
||||
|
||||
# 写入文件
|
||||
await path.write_text(new_content, encoding="utf-8")
|
||||
replacement_count = occurrences if replace_all else 1
|
||||
new_content = content.replace(
|
||||
old_text,
|
||||
new_text,
|
||||
-1 if replace_all else 1,
|
||||
)
|
||||
await self.run_blocking(
|
||||
"default",
|
||||
atomic_write_text,
|
||||
local_path,
|
||||
new_content,
|
||||
current_sha256,
|
||||
)
|
||||
new_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
|
||||
logger.info(f"成功编辑文件 {resolved_path},替换了 {occurrences} 处内容")
|
||||
return f"成功编辑文件 {resolved_path} (替换了 {occurrences} 处匹配内容)"
|
||||
logger.info(
|
||||
f"成功编辑文件 {resolved_path},替换了 {replacement_count} 处内容"
|
||||
)
|
||||
return (
|
||||
f"成功编辑文件 {resolved_path}(替换了 {replacement_count} 处匹配内容,"
|
||||
f"sha256={new_sha256})"
|
||||
)
|
||||
|
||||
except FileVersionConflictError:
|
||||
return (
|
||||
f"错误:文件 {file_path} 在编辑期间发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并再次编辑。"
|
||||
)
|
||||
except PermissionError:
|
||||
return f"错误:没有访问/修改 {file_path} 的权限"
|
||||
except UnicodeDecodeError:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""文件读取工具"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
|
||||
@@ -16,12 +18,22 @@ MAX_READ_SIZE = 50 * 1024
|
||||
|
||||
class ReadFileInput(BaseModel):
|
||||
"""文件读取工具的输入参数模型。"""
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to read")
|
||||
start_line: Optional[int] = Field(None, description="The starting line number (1-based, inclusive). If not provided, reading starts from the beginning of the file.")
|
||||
end_line: Optional[int] = Field(None, description="The ending line number (1-based, inclusive). If not provided, reading goes until the end of the file.")
|
||||
include_metadata: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Return structured JSON containing content, size, truncation state, "
|
||||
"and SHA-256. Use before a guarded full-file overwrite."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ReadFileTool(MoviePilotTool):
|
||||
"""按行范围读取本地文本文件,并可返回文件版本元数据。"""
|
||||
|
||||
name: str = "read_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Read,
|
||||
@@ -36,8 +48,15 @@ class ReadFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"读取文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, start_line: Optional[int] = None,
|
||||
end_line: Optional[int] = None, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
start_line: Optional[int] = None,
|
||||
end_line: Optional[int] = None,
|
||||
include_metadata: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""读取指定文本范围,必要时附带完整文件的 SHA-256 元数据。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}, start_line={start_line}, end_line={end_line}")
|
||||
|
||||
try:
|
||||
@@ -55,7 +74,8 @@ class ReadFileTool(MoviePilotTool):
|
||||
if not await path.is_file():
|
||||
return f"错误:{resolved_path} 不是一个文件"
|
||||
|
||||
content = await path.read_text(encoding="utf-8", errors="replace")
|
||||
raw_content = await path.read_bytes()
|
||||
content = raw_content.decode("utf-8", errors="replace")
|
||||
truncated = False
|
||||
|
||||
if start_line is not None or end_line is not None:
|
||||
@@ -78,6 +98,21 @@ class ReadFileTool(MoviePilotTool):
|
||||
content = content_bytes[:MAX_READ_SIZE].decode("utf-8", errors="replace")
|
||||
truncated = True
|
||||
|
||||
if include_metadata:
|
||||
return json.dumps(
|
||||
{
|
||||
"file_path": str(resolved_path),
|
||||
"sha256": hashlib.sha256(raw_content).hexdigest(),
|
||||
"size_bytes": len(raw_content),
|
||||
"start_line": start_line,
|
||||
"end_line": end_line,
|
||||
"truncated": truncated,
|
||||
"content": content,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if truncated:
|
||||
return f"{content}\n\n[警告:文件内容已超过50KB限制,以上内容已被截断。请使用 start_line/end_line 参数分段读取。]"
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.impl._file_write_utils import (
|
||||
FileVersionConflictError,
|
||||
atomic_write_text,
|
||||
calculate_file_sha256,
|
||||
)
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.log import logger
|
||||
|
||||
@@ -16,17 +21,36 @@ class WriteFileInput(BaseModel):
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to write")
|
||||
content: str = Field(..., description="The content to write into the file")
|
||||
overwrite: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Allow replacing an existing file in full. Keep false when creating a "
|
||||
"new file; prefer edit_file for localized changes."
|
||||
),
|
||||
)
|
||||
expected_sha256: Optional[str] = Field(
|
||||
None,
|
||||
pattern=r"^[0-9a-fA-F]{64}$",
|
||||
description=(
|
||||
"Optional SHA-256 returned by read_file(include_metadata=true). When "
|
||||
"overwriting, fail if the existing file no longer has this hash."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WriteFileTool(MoviePilotTool):
|
||||
"""创建本地文本文件,或在显式允许后完整覆盖已有文件。"""
|
||||
|
||||
name: str = "write_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Write,
|
||||
ToolTag.File,
|
||||
]
|
||||
description: str = (
|
||||
"Write full content to a local text file. Non-admin users can only write "
|
||||
"inside the MoviePilot Agent config directory."
|
||||
"Create a local text file with complete content. Existing files are "
|
||||
"protected unless overwrite=true; localized changes should use edit_file. "
|
||||
"Supports an optional SHA-256 conflict check and writes atomically. "
|
||||
"Non-admin users can only write inside the MoviePilot Agent config directory."
|
||||
)
|
||||
args_schema: Type[BaseModel] = WriteFileInput
|
||||
|
||||
@@ -36,7 +60,15 @@ class WriteFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"写入文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, content: str, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
content: str,
|
||||
overwrite: bool = False,
|
||||
expected_sha256: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""创建或显式覆盖文件,并通过可选哈希阻止陈旧写入。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
|
||||
|
||||
try:
|
||||
@@ -48,18 +80,52 @@ class WriteFileTool(MoviePilotTool):
|
||||
|
||||
path = AsyncPath(resolved_path)
|
||||
|
||||
if await path.exists() and not await path.is_file():
|
||||
exists = await path.exists()
|
||||
if exists and not await path.is_file():
|
||||
return f"错误:{resolved_path} 路径已存在但不是一个文件"
|
||||
if exists and not overwrite:
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已存在,拒绝完整覆盖。"
|
||||
"局部修改请使用 edit_file;确需重写时设置 overwrite=true。"
|
||||
)
|
||||
if expected_sha256 and not exists:
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 不存在,无法校验 expected_sha256。"
|
||||
"请确认路径和最新文件状态。"
|
||||
)
|
||||
|
||||
# 自动创建父目录
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
local_path = Path(resolved_path)
|
||||
current_sha256 = None
|
||||
if exists:
|
||||
current_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
if expected_sha256:
|
||||
if current_sha256.casefold() != expected_sha256.casefold():
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并基于最新内容写入。"
|
||||
)
|
||||
|
||||
# 写入文件
|
||||
await path.write_text(content, encoding="utf-8")
|
||||
await self.run_blocking(
|
||||
"default",
|
||||
atomic_write_text,
|
||||
local_path,
|
||||
content,
|
||||
current_sha256,
|
||||
)
|
||||
new_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
|
||||
logger.info(f"成功写入文件 {resolved_path}")
|
||||
return f"成功写入文件 {resolved_path}"
|
||||
return f"成功写入文件 {resolved_path}(sha256={new_sha256})"
|
||||
|
||||
except FileVersionConflictError:
|
||||
return (
|
||||
f"错误:文件 {file_path} 在写入期间发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并再次写入。"
|
||||
)
|
||||
except PermissionError:
|
||||
return f"错误:没有权限写入 {file_path}"
|
||||
except Exception as e:
|
||||
|
||||
+47
-11
@@ -1,9 +1,11 @@
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.agent.tools.base import ToolExecutionTimeoutError, format_tool_result_for_agent
|
||||
from app.agent.tools.factory import MoviePilotToolFactory
|
||||
from app.core.plugin import PluginManager
|
||||
from app.log import logger
|
||||
|
||||
|
||||
@@ -40,27 +42,59 @@ class MoviePilotToolsManager:
|
||||
self.session_id = session_id
|
||||
self.is_admin = is_admin
|
||||
self.tools: List[Any] = []
|
||||
self._tools_lock = threading.Lock()
|
||||
self._plugin_agent_tools_revision = -1
|
||||
self._load_tools()
|
||||
|
||||
def _load_tools(self):
|
||||
def _load_tools(self) -> None:
|
||||
"""
|
||||
加载所有MoviePilot工具
|
||||
"""
|
||||
try:
|
||||
# 创建工具实例
|
||||
self.tools = MoviePilotToolFactory.create_tools(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
channel=None,
|
||||
source="api",
|
||||
username="API Client",
|
||||
stream_handler=None,
|
||||
agent_context={"is_admin": self.is_admin},
|
||||
)
|
||||
plugin_manager = PluginManager()
|
||||
while True:
|
||||
plugin_tools_revision = (
|
||||
plugin_manager.get_plugin_agent_tools_revision()
|
||||
)
|
||||
tools = MoviePilotToolFactory.create_tools(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
channel=None,
|
||||
source="api",
|
||||
username="API Client",
|
||||
stream_handler=None,
|
||||
agent_context={"is_admin": self.is_admin},
|
||||
)
|
||||
if (
|
||||
plugin_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
break
|
||||
self.tools = tools
|
||||
self._plugin_agent_tools_revision = plugin_tools_revision
|
||||
logger.info(f"成功加载 {len(self.tools)} 个工具")
|
||||
except Exception as e:
|
||||
logger.error(f"加载工具失败: {e}", exc_info=True)
|
||||
self.tools = []
|
||||
self._plugin_agent_tools_revision = -1
|
||||
|
||||
def _ensure_tools_current(self) -> None:
|
||||
"""
|
||||
在插件工具注册表变化后惰性刷新工具实例。
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
if (
|
||||
self._plugin_agent_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
return
|
||||
with self._tools_lock:
|
||||
if (
|
||||
self._plugin_agent_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
return
|
||||
self._load_tools()
|
||||
|
||||
def list_tools(self) -> List[ToolDefinition]:
|
||||
"""
|
||||
@@ -69,6 +103,7 @@ class MoviePilotToolsManager:
|
||||
Returns:
|
||||
工具定义列表
|
||||
"""
|
||||
self._ensure_tools_current()
|
||||
tools_list = []
|
||||
for tool in self.tools:
|
||||
if getattr(tool, "_require_admin", False) and not self.is_admin:
|
||||
@@ -102,6 +137,7 @@ class MoviePilotToolsManager:
|
||||
Returns:
|
||||
工具实例,如果未找到返回None
|
||||
"""
|
||||
self._ensure_tools_current()
|
||||
for tool in self.tools:
|
||||
if tool.name == tool_name:
|
||||
return tool
|
||||
|
||||
@@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app import schemas
|
||||
from app.core.auth_bridge import build_token_response, consume_plugin_auth_ticket
|
||||
from app.core.auth import build_token_response, consume_plugin_auth_ticket
|
||||
from app.core.plugin import PluginManager
|
||||
from app.db.models.passkey import PassKey
|
||||
from app.db.models.user import User
|
||||
|
||||
@@ -4,12 +4,15 @@ from fastapi import APIRouter, Depends
|
||||
|
||||
from app import schemas
|
||||
from app.chain.douban import DoubanChain
|
||||
from app.core.config import settings
|
||||
from app.core.context import MediaInfo
|
||||
from app.core.security import verify_token
|
||||
from app.db.models.user import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.douban.douban_cache import DoubanCache
|
||||
from app.schemas import MediaType
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -29,6 +32,10 @@ async def douban_recognition_cache(
|
||||
"count": len(cache_items),
|
||||
"recognized": recognized_count,
|
||||
"unrecognized": len(cache_items) - recognized_count,
|
||||
"shared_recognized": SystemConfigOper().get(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
) or 0,
|
||||
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
|
||||
"data": cache_items,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -38,6 +38,7 @@ class LlmTestRequest(BaseModel):
|
||||
user_agent: Optional[str] = None
|
||||
temperature: Optional[float] = None
|
||||
use_proxy: Optional[bool] = None
|
||||
api_protocol: Optional[str] = None
|
||||
|
||||
|
||||
class LlmProviderAuthStartRequest(BaseModel):
|
||||
@@ -269,6 +270,7 @@ async def llm_test(
|
||||
base_url_preset=settings.LLM_BASE_URL_PRESET,
|
||||
user_agent=settings.LLM_USER_AGENT,
|
||||
use_proxy=settings.LLM_USE_PROXY,
|
||||
api_protocol=settings.LLM_API_PROTOCOL,
|
||||
)
|
||||
|
||||
if not payload.provider:
|
||||
@@ -302,6 +304,7 @@ async def llm_test(
|
||||
"base_url_preset": payload.base_url_preset,
|
||||
"user_agent": payload.user_agent,
|
||||
"use_proxy": payload.use_proxy,
|
||||
"api_protocol": payload.api_protocol,
|
||||
}
|
||||
if payload.temperature is not None:
|
||||
test_kwargs["temperature"] = payload.temperature
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import shutil
|
||||
from typing import Annotated, Any, List, Optional
|
||||
from typing import Annotated, Any, Dict, List, Optional
|
||||
|
||||
import aiofiles
|
||||
from anyio import Path as AsyncPath
|
||||
@@ -476,6 +476,71 @@ async def statistic(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
return await MoviePilotServerHelper.async_get_plugin_statistic()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/rating",
|
||||
summary="批量查询插件评分",
|
||||
response_model=Dict[str, schemas.PluginRating],
|
||||
)
|
||||
async def plugin_ratings(
|
||||
plugin_ids: Optional[str] = None,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> Dict[str, schemas.PluginRating]:
|
||||
"""
|
||||
批量查询插件平均分、评分人数和当前安装实例评分。
|
||||
"""
|
||||
requested_ids = plugin_ids.split(",") if plugin_ids is not None else None
|
||||
ratings = await MoviePilotServerHelper.async_get_plugin_ratings(requested_ids)
|
||||
return {
|
||||
plugin_id: schemas.PluginRating.model_validate(rating)
|
||||
for plugin_id, rating in ratings.items()
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/rating/{plugin_id}",
|
||||
summary="查询插件评分",
|
||||
response_model=schemas.PluginRating,
|
||||
)
|
||||
async def plugin_rating(
|
||||
plugin_id: str,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> schemas.PluginRating:
|
||||
"""
|
||||
查询单个插件平均分、评分人数和当前安装实例评分。
|
||||
"""
|
||||
rating = await MoviePilotServerHelper.async_get_plugin_rating(plugin_id)
|
||||
return schemas.PluginRating.model_validate(rating)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/rating/{plugin_id}",
|
||||
summary="提交插件评分",
|
||||
response_model=schemas.Response,
|
||||
)
|
||||
async def rate_plugin(
|
||||
plugin_id: str,
|
||||
payload: schemas.PluginRatingRequest,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> schemas.Response:
|
||||
"""
|
||||
为已安装插件新增或更新当前安装实例评分。
|
||||
"""
|
||||
installed_plugins = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id not in installed_plugins:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"插件 {plugin_id} 未安装,无法评分",
|
||||
)
|
||||
|
||||
rating = await MoviePilotServerHelper.async_submit_plugin_rating(
|
||||
plugin_id,
|
||||
payload.rating,
|
||||
)
|
||||
if rating is None:
|
||||
return schemas.Response(success=False, message="连接MoviePilot服务器失败")
|
||||
return schemas.Response(success=True, data=rating)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/reload/{plugin_id}", summary="重新加载插件", response_model=schemas.Response
|
||||
)
|
||||
|
||||
@@ -630,6 +630,7 @@ async def popular_subscribes(
|
||||
media.bangumi_id = sub.get("bangumiid")
|
||||
media.anilist_id = sub.get("anilistid")
|
||||
media.source = sub.get("media_source")
|
||||
media.media_id = sub.get("media_id")
|
||||
media.tvdb_id = sub.get("tvdbid")
|
||||
media.imdb_id = sub.get("imdbid")
|
||||
media.season = sub.get("season")
|
||||
|
||||
@@ -582,23 +582,27 @@ async def fetch_image(
|
||||
):
|
||||
return None
|
||||
|
||||
content = await ImageHelper().async_fetch_image(
|
||||
image_result = await ImageHelper().async_fetch_image_with_mime_type(
|
||||
url=fetch_url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if content:
|
||||
if image_result:
|
||||
content, media_type = image_result
|
||||
|
||||
# 检查 If-None-Match
|
||||
etag = HashUtils.md5(content)
|
||||
headers = RequestUtils.generate_cache_headers(etag, max_age=86400 * 7)
|
||||
headers["Content-Type"] = media_type
|
||||
headers["X-Content-Type-Options"] = "nosniff"
|
||||
if if_none_match == etag:
|
||||
return Response(status_code=304, headers=headers)
|
||||
# 返回缓存图片
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=UrlUtils.get_mime_type(fetch_url, "image/jpeg"),
|
||||
media_type=media_type,
|
||||
headers=headers,
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -4,11 +4,13 @@ from fastapi import APIRouter, Depends
|
||||
|
||||
from app import schemas
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.core.config import settings
|
||||
from app.core.security import verify_token
|
||||
from app.db.models.user import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||
from app.schemas.types import MediaType
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -28,6 +30,10 @@ async def tmdb_recognition_cache(
|
||||
"count": len(cache_items),
|
||||
"recognized": recognized_count,
|
||||
"unrecognized": len(cache_items) - recognized_count,
|
||||
"shared_recognized": SystemConfigOper().get(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
) or 0,
|
||||
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
|
||||
"data": cache_items,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.core.meta import MetaBase
|
||||
from app.core.module import ModuleManager
|
||||
from app.core.plugin import PluginManager
|
||||
from app.db.message_oper import MessageOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import UserOper
|
||||
from app.helper.message import MessageHelper, MessageQueueManager, MessageTemplateHelper
|
||||
from app.helper.server import MoviePilotServerHelper
|
||||
@@ -49,6 +50,7 @@ from app.schemas.types import (
|
||||
MediaImageType,
|
||||
EventType,
|
||||
MessageChannel,
|
||||
SystemConfigKey,
|
||||
)
|
||||
from app.utils.object import ObjectUtils
|
||||
|
||||
@@ -572,6 +574,14 @@ class ChainBase(metaclass=ABCMeta):
|
||||
mediainfo=mediainfo,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_media_recognize_share_hit() -> None:
|
||||
"""记录一次共享媒体识别成功命中,统计失败不影响识别结果。"""
|
||||
try:
|
||||
SystemConfigOper().increment(SystemConfigKey.MediaRecognizeShareCount)
|
||||
except Exception as err:
|
||||
logger.error(f"记录共享媒体识别命中次数失败:{str(err)}")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_media_source_params(
|
||||
source: Optional[str] = None,
|
||||
@@ -730,6 +740,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
)
|
||||
if mediainfo:
|
||||
self._update_local_recognize_cache(shared_cache_meta, mediainfo)
|
||||
self._record_media_recognize_share_hit()
|
||||
return mediainfo
|
||||
return None
|
||||
|
||||
@@ -839,6 +850,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
)
|
||||
if mediainfo:
|
||||
await self._async_update_local_recognize_cache(shared_cache_meta, mediainfo)
|
||||
await run_in_threadpool(self._record_media_recognize_share_hit)
|
||||
return mediainfo
|
||||
return None
|
||||
|
||||
|
||||
+49
-3
@@ -928,6 +928,36 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
or "/@eaDir" in normalized_path
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __should_delete_empty_source_directories(
|
||||
task: TransferTask,
|
||||
delete_mounted_local_disk_empty_dirs: bool,
|
||||
mounted_filesystem_cache: Dict[Path, bool],
|
||||
) -> bool:
|
||||
"""
|
||||
判断移动整理后是否应删除源空目录。
|
||||
|
||||
仅在关闭挂载盘空目录清理且源存储为本地时检测文件系统,
|
||||
避免默认流程产生额外系统调用。
|
||||
"""
|
||||
if delete_mounted_local_disk_empty_dirs:
|
||||
return True
|
||||
if task.fileitem.storage != "local":
|
||||
return True
|
||||
|
||||
source_directory = (
|
||||
Path(task.target_directory.download_path)
|
||||
if task.target_directory and task.target_directory.download_path
|
||||
else Path(task.fileitem.path).parent
|
||||
)
|
||||
if source_directory not in mounted_filesystem_cache:
|
||||
mounted_filesystem_cache[source_directory] = (
|
||||
SystemUtils.is_network_filesystem(
|
||||
source_directory, include_local_fuse=True
|
||||
)
|
||||
)
|
||||
return not mounted_filesystem_cache[source_directory]
|
||||
|
||||
def __default_callback(
|
||||
self, task: TransferTask, transferinfo: TransferInfo, /
|
||||
) -> Tuple[bool, str]:
|
||||
@@ -1189,10 +1219,16 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
tasks = self.jobview.success_tasks(
|
||||
task.mediainfo, task.meta.begin_season
|
||||
)
|
||||
system_config_oper = SystemConfigOper()
|
||||
# 获取整理屏蔽词
|
||||
transfer_exclude_words = SystemConfigOper().get(
|
||||
transfer_exclude_words = system_config_oper.get(
|
||||
SystemConfigKey.TransferExcludeWords
|
||||
)
|
||||
# 挂载盘空目录清理默认开启
|
||||
delete_mounted_local_disk_empty_dirs = system_config_oper.get(
|
||||
SystemConfigKey.MountedLocalDiskDeleteEmptyDirs
|
||||
) is not False
|
||||
mounted_filesystem_cache: Dict[Path, bool] = {}
|
||||
processed_hashes = set()
|
||||
for t in tasks:
|
||||
if t.download_hash and t.download_hash not in processed_hashes:
|
||||
@@ -1209,7 +1245,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
logger.info(
|
||||
f"移动模式删除种子成功:{t.download_hash}"
|
||||
)
|
||||
if not t.download_hash and t.fileitem:
|
||||
if (
|
||||
not t.download_hash
|
||||
and t.fileitem
|
||||
and self.__should_delete_empty_source_directories(
|
||||
t,
|
||||
delete_mounted_local_disk_empty_dirs,
|
||||
mounted_filesystem_cache,
|
||||
)
|
||||
):
|
||||
# 删除剩余空目录
|
||||
StorageChain().delete_media_file(t.fileitem, delete_self=False)
|
||||
|
||||
@@ -1565,7 +1609,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
# 重置计数
|
||||
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
|
||||
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
|
||||
|
||||
@@ -569,6 +569,8 @@ class ConfigModel(BaseModel):
|
||||
LLM_MODEL: str = "deepseek-chat"
|
||||
# 思考模式/深度配置:off/auto/minimal/low/medium/high/max/xhigh
|
||||
LLM_THINKING_LEVEL: Optional[str] = "off"
|
||||
# OpenAI兼容接口API协议:auto(自动)/ chat_completions / responses
|
||||
LLM_API_PROTOCOL: str = "auto"
|
||||
# LLM是否支持图片输入,开启后消息图片会按多模态输入发送给模型
|
||||
LLM_SUPPORT_IMAGE_INPUT: bool = True
|
||||
# 是否启用音频输入,开启后用户语音会先转写为文本再进入 Agent
|
||||
|
||||
+44
-28
@@ -58,6 +58,7 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
# 插件智能体工具注册表缓存,插件启停或配置生效时主动失效。
|
||||
self._plugin_agent_tools_cache: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self._plugin_agent_tools_cache_lock = threading.Lock()
|
||||
self._plugin_agent_tools_revision: int = 0
|
||||
# 开发者模式监测插件修改
|
||||
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
|
||||
self.__start_monitor()
|
||||
@@ -143,6 +144,14 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
self._plugin_agent_tools_cache.clear()
|
||||
self._plugin_agent_tools_revision += 1
|
||||
|
||||
def get_plugin_agent_tools_revision(self) -> int:
|
||||
"""
|
||||
获取插件智能体工具注册表版本号。
|
||||
"""
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
return self._plugin_agent_tools_revision
|
||||
|
||||
def stop(self, pid: Optional[str] = None):
|
||||
"""
|
||||
@@ -1002,35 +1011,42 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
}]
|
||||
"""
|
||||
cache_key = pid or "__all__"
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
|
||||
if cached_tools is not None:
|
||||
return self._copy_plugin_agent_tools(cached_tools)
|
||||
while True:
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
cache_revision = self._plugin_agent_tools_revision
|
||||
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
|
||||
if cached_tools is not None:
|
||||
return self._copy_plugin_agent_tools(cached_tools)
|
||||
|
||||
ret_tools = []
|
||||
# 创建字典快照避免并发修改
|
||||
running_plugins_snapshot = dict(self._running_plugins)
|
||||
for plugin_id, plugin in running_plugins_snapshot.items():
|
||||
if pid and pid != plugin_id:
|
||||
continue
|
||||
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(plugin.get_agent_tools):
|
||||
try:
|
||||
if not plugin.get_state():
|
||||
continue
|
||||
tools = plugin.get_agent_tools()
|
||||
if tools:
|
||||
ret_tools.append({
|
||||
"plugin_id": plugin_id,
|
||||
"plugin_name": plugin.plugin_name,
|
||||
"tools": tools
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
|
||||
ret_tools
|
||||
)
|
||||
return ret_tools
|
||||
ret_tools = []
|
||||
# 创建字典快照避免并发修改
|
||||
running_plugins_snapshot = dict(self._running_plugins)
|
||||
for plugin_id, plugin in running_plugins_snapshot.items():
|
||||
if pid and pid != plugin_id:
|
||||
continue
|
||||
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(
|
||||
plugin.get_agent_tools
|
||||
):
|
||||
try:
|
||||
if not plugin.get_state():
|
||||
continue
|
||||
tools = plugin.get_agent_tools()
|
||||
if tools:
|
||||
ret_tools.append({
|
||||
"plugin_id": plugin_id,
|
||||
"plugin_name": plugin.plugin_name,
|
||||
"tools": tools
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
if cache_revision != self._plugin_agent_tools_revision:
|
||||
# 插件状态在注册表构建期间发生变化,重新读取以避免写回过期快照。
|
||||
continue
|
||||
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
|
||||
ret_tools
|
||||
)
|
||||
return ret_tools
|
||||
|
||||
@staticmethod
|
||||
def get_plugin_remote_entry(plugin_id: str, dist_path: str) -> str:
|
||||
|
||||
@@ -101,6 +101,19 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
# 避免将__SYSTEMCONF内的值引用出去,会导致set时误判没有变动
|
||||
return copy.deepcopy(self.__SYSTEMCONF.get(key))
|
||||
|
||||
def increment(self, key: SystemConfigKey, step: int = 1) -> int:
|
||||
"""
|
||||
原子递增整数系统设置
|
||||
|
||||
:param key: 配置键
|
||||
:param step: 递增步长
|
||||
:return: 递增后的整数值
|
||||
"""
|
||||
with self._rlock:
|
||||
value = int(self.get(key) or 0) + step
|
||||
self.set(key, value)
|
||||
return value
|
||||
|
||||
def all(self):
|
||||
"""
|
||||
获取所有系统设置
|
||||
|
||||
+75
-16
@@ -188,16 +188,31 @@ class ImageHelper(metaclass=Singleton):
|
||||
return cache_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _validate_image(content: bytes) -> bool:
|
||||
"""验证图片"""
|
||||
def get_image_mime_type(content: bytes, verify: bool = True) -> Optional[str]:
|
||||
"""
|
||||
根据图片内容返回 Pillow 识别的图片 MIME 类型。
|
||||
|
||||
外部响应在写入缓存前需要完整校验;已校验的缓存只需读取格式头。
|
||||
非图片或可脚本化的 MIME 类型不作为图片代理响应。
|
||||
"""
|
||||
if not content:
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
Image.open(io.BytesIO(content)).verify()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn(f"Invalid image format: {e}")
|
||||
return False
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
image_format = (image.format or "").upper()
|
||||
if verify:
|
||||
image.verify()
|
||||
mime_type = Image.MIME.get(image_format)
|
||||
if (
|
||||
not mime_type
|
||||
or not mime_type.startswith("image/")
|
||||
or mime_type == "image/svg+xml"
|
||||
):
|
||||
return None
|
||||
return mime_type
|
||||
except Exception as err:
|
||||
logger.warning(f"Invalid image format: {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_request_params(url: str, proxy: Optional[bool], cookies: Optional[str | dict]) -> dict:
|
||||
@@ -224,6 +239,26 @@ class ImageHelper(metaclass=Singleton):
|
||||
"""
|
||||
获取图片(同步版本)
|
||||
"""
|
||||
result = self.fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
def fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
同步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
@@ -233,7 +268,9 @@ class ImageHelper(metaclass=Singleton):
|
||||
if use_cache:
|
||||
content = self.file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
return content
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
@@ -243,13 +280,13 @@ class ImageHelper(metaclass=Singleton):
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
# 验证图片
|
||||
if not self._validate_image(content):
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
self.file_cache.set(cache_path, content, region="images")
|
||||
return content
|
||||
return content, mime_type
|
||||
|
||||
async def async_fetch_image(
|
||||
self,
|
||||
@@ -260,6 +297,26 @@ class ImageHelper(metaclass=Singleton):
|
||||
"""
|
||||
获取图片(异步版本)
|
||||
"""
|
||||
result = await self.async_fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
async def async_fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
异步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
@@ -269,7 +326,9 @@ class ImageHelper(metaclass=Singleton):
|
||||
if use_cache:
|
||||
content = await self.async_file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
return content
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
@@ -279,10 +338,10 @@ class ImageHelper(metaclass=Singleton):
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
# 验证图片
|
||||
if not self._validate_image(content):
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
await self.async_file_cache.set(cache_path, content, region="images")
|
||||
return content
|
||||
return content, mime_type
|
||||
|
||||
@@ -29,6 +29,7 @@ class MoviePilotServerHelper:
|
||||
_USAGE_REPORT_PATH = "/usage/report"
|
||||
_USAGE_STATISTIC_PATH = "/usage/statistic"
|
||||
_PLUGIN_INSTALL_PATH = "/plugin/install"
|
||||
_PLUGIN_RATING_PATH = "/plugin/rating"
|
||||
_PLUGIN_STATISTIC_PATH = "/plugin/statistic"
|
||||
_SUBSCRIBE_ADD_PATH = "/subscribe/add"
|
||||
_SUBSCRIBE_DONE_PATH = "/subscribe/done"
|
||||
@@ -398,6 +399,39 @@ class MoviePilotServerHelper:
|
||||
"""
|
||||
return await cls._async_get(cls._server_url(cls._PLUGIN_STATISTIC_PATH), timeout=10)
|
||||
|
||||
@classmethod
|
||||
async def async_plugin_ratings(cls, plugin_ids: Optional[List[str]] = None):
|
||||
"""
|
||||
异步批量查询中心端插件评分。
|
||||
"""
|
||||
params = {"plugin_ids": ",".join(plugin_ids)} if plugin_ids is not None else None
|
||||
return await cls._async_get(
|
||||
cls._server_url(cls._PLUGIN_RATING_PATH),
|
||||
params=params,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def async_plugin_rating(cls, plugin_id: str):
|
||||
"""
|
||||
异步查询中心端单个插件评分。
|
||||
"""
|
||||
return await cls._async_get(
|
||||
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def async_rate_plugin(cls, plugin_id: str, rating: float):
|
||||
"""
|
||||
异步提交当前安装实例的插件评分。
|
||||
"""
|
||||
return await cls._async_post_json(
|
||||
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
|
||||
{"rating": rating},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def plugin_install(cls, plugin_id: str, payload: Dict[str, Any]):
|
||||
"""
|
||||
@@ -459,6 +493,58 @@ class MoviePilotServerHelper:
|
||||
return res.json()
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
async def async_get_plugin_ratings(
|
||||
cls,
|
||||
plugin_ids: Optional[List[str]] = None,
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
批量获取插件评分,中心端不可用时返回空结果。
|
||||
"""
|
||||
try:
|
||||
res = await cls.async_plugin_ratings(plugin_ids)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"批量获取插件评分失败:{str(err)}")
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
async def async_get_plugin_rating(cls, plugin_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取单个插件评分,中心端不可用时返回零评分。
|
||||
"""
|
||||
empty_rating = {
|
||||
"plugin_id": plugin_id,
|
||||
"average_rating": 0.0,
|
||||
"rating_count": 0,
|
||||
"user_rating": None,
|
||||
}
|
||||
try:
|
||||
res = await cls.async_plugin_rating(plugin_id)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"获取插件 {plugin_id} 评分失败:{str(err)}")
|
||||
return empty_rating
|
||||
|
||||
@classmethod
|
||||
async def async_submit_plugin_rating(
|
||||
cls,
|
||||
plugin_id: str,
|
||||
rating: float,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
提交插件评分,成功时返回最新评分结果。
|
||||
"""
|
||||
try:
|
||||
res = await cls.async_rate_plugin(plugin_id, rating)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"提交插件 {plugin_id} 评分失败:{str(err)}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def install_plugin_reg(cls, plugin_id: str, repo_url: Optional[str] = None) -> bool:
|
||||
"""
|
||||
|
||||
+15
-2
@@ -1,7 +1,10 @@
|
||||
from typing import Dict, List, Optional, Type, TypeVar, Generic, Iterator
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.module import ModuleManager
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas import DownloaderConf, MediaServerConf, NotificationConf, NotificationSwitchConf, ServiceInfo
|
||||
from app.schemas.types import NotificationType, SystemConfigKey, ModuleType
|
||||
|
||||
@@ -25,8 +28,18 @@ class ServiceConfigHelper:
|
||||
config_data = SystemConfigOper().get(config_key)
|
||||
if not config_data:
|
||||
return []
|
||||
# 直接使用 conf_type 来实例化配置对象
|
||||
return [conf_type(**conf) for conf in config_data]
|
||||
configs = []
|
||||
for conf in config_data:
|
||||
if not isinstance(conf, dict):
|
||||
logger.warn(f"{config_key.value} 配置格式不正确,已跳过:{conf}")
|
||||
continue
|
||||
try:
|
||||
# 直接使用 conf_type 来实例化配置对象
|
||||
configs.append(conf_type(**conf))
|
||||
except ValidationError as e:
|
||||
# 单条配置存在非法值时跳过,避免影响其它服务的初始化
|
||||
logger.error(f"{config_key.value} 配置 {conf.get('name')} 校验失败,已跳过:{e}")
|
||||
return configs
|
||||
|
||||
@staticmethod
|
||||
def get_downloader_configs() -> List[DownloaderConf]:
|
||||
|
||||
@@ -422,6 +422,10 @@
|
||||
"source": "插件 {plugin} 不存在或未安装",
|
||||
"target": "Plugin {plugin} does not exist or is not installed"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 未安装,无法评分",
|
||||
"target": "Plugin {plugin} is not installed and cannot be rated"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 不存在或未加载",
|
||||
"target": "Plugin {plugin} does not exist or is not loaded"
|
||||
|
||||
@@ -422,6 +422,10 @@
|
||||
"source": "插件 {plugin} 不存在或未安装",
|
||||
"target": "插件 {plugin} 不存在或未安裝"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 未安装,无法评分",
|
||||
"target": "插件 {plugin} 未安裝,無法評分"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 不存在或未加载",
|
||||
"target": "插件 {plugin} 不存在或未載入"
|
||||
|
||||
@@ -174,6 +174,13 @@ class StorageBase(metaclass=ABCMeta):
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
默认实现不区分「不存在」与「查询失败」,由具体存储按需覆写。
|
||||
"""
|
||||
return self.get_item(path)
|
||||
|
||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取父目录
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.modules.filemanager.storages import transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import WeakSingleton
|
||||
@@ -834,30 +835,53 @@ class AliPan(StorageBase, metaclass=WeakSingleton):
|
||||
return False
|
||||
return True
|
||||
|
||||
def __get_by_path_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
按路径查询文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
NotFound 系列错误码表示确认不存在,其余错误(网络失败、限流、
|
||||
权限或未知业务错误)均无法确认目标状态。
|
||||
"""
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/adrive/v1.0/openFile/get_by_path",
|
||||
json={
|
||||
"drive_id": drive_id or self._default_drive_id,
|
||||
"file_path": path.as_posix(),
|
||||
},
|
||||
no_error_log=True,
|
||||
)
|
||||
if resp is None:
|
||||
raise StorageQueryError(f"【阿里云盘】无法确认文件状态(请求失败): {path}")
|
||||
code = resp.get("code")
|
||||
if code:
|
||||
if "NotFound" in str(code):
|
||||
# 明确的不存在错误码,确认目标不存在
|
||||
return None
|
||||
raise StorageQueryError(
|
||||
f"【阿里云盘】查询文件信息出错: {path} - {code} {resp.get('message')}")
|
||||
return self.__get_fileitem(resp, parent=str(path.parent))
|
||||
|
||||
def get_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项
|
||||
"""
|
||||
try:
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/adrive/v1.0/openFile/get_by_path",
|
||||
json={
|
||||
"drive_id": drive_id or self._default_drive_id,
|
||||
"file_path": path.as_posix(),
|
||||
},
|
||||
no_error_log=True,
|
||||
)
|
||||
if not resp:
|
||||
return None
|
||||
if resp.get("code"):
|
||||
logger.debug(f"【阿里云盘】获取文件信息失败: {resp.get('message')}")
|
||||
return None
|
||||
return self.__get_fileitem(resp, parent=str(path.parent))
|
||||
return self.__get_by_path_item(path, drive_id=drive_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"【阿里云盘】获取文件信息失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
"""
|
||||
try:
|
||||
return self.__get_by_path_item(path)
|
||||
except StorageQueryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise StorageQueryError(f"【阿里云盘】查询文件信息失败: {path} - {e}") from e
|
||||
|
||||
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件夹,如不存在则创建
|
||||
|
||||
@@ -8,6 +8,7 @@ from app.core.config import global_vars, settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
@@ -148,6 +149,23 @@ class LocalStorage(StorageBase):
|
||||
return self.__get_fileitem(path)
|
||||
return self.__get_diritem(path)
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,无法确认状态时抛出 StorageQueryError。
|
||||
Path.exists() 会把部分 errno(如 EBADF/ELOOP)归入「不存在」,
|
||||
网络/FUSE 挂载抖动时会误判,这里用 stat 显式区分。
|
||||
"""
|
||||
try:
|
||||
path.stat()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as e:
|
||||
raise StorageQueryError(f"【本地】读取文件状态失败: {path} - {e}") from e
|
||||
try:
|
||||
return self.get_item(path)
|
||||
except OSError as e:
|
||||
raise StorageQueryError(f"【本地】读取文件信息失败: {path} - {e}") from e
|
||||
|
||||
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件详情
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.modules.filemanager.storages import transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
@@ -906,38 +907,60 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
return True
|
||||
return False
|
||||
|
||||
def __get_info_item(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
查询指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
接口业务码 20004(记录不存在)与 0 一样视为确认结果,其余错误
|
||||
(网络失败、限流重试用尽、未知业务错误)均无法确认目标状态。
|
||||
"""
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/open/folder/get_info",
|
||||
data={"path": path.as_posix()},
|
||||
no_error_log=True,
|
||||
)
|
||||
if resp is None:
|
||||
raise StorageQueryError(f"【115】无法确认文件状态(请求失败或接口错误): {path}")
|
||||
data = resp.get("data") if isinstance(resp, dict) else None
|
||||
if not data or not data.get("file_id"):
|
||||
# code 20004(记录不存在)等场景,确认目标不存在
|
||||
return None
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
fileid=str(data["file_id"]),
|
||||
path=path.as_posix() + ("/" if data["file_category"] == "0" else ""),
|
||||
type="file" if data["file_category"] == "1" else "dir",
|
||||
name=data["file_name"],
|
||||
basename=Path(data["file_name"]).stem,
|
||||
extension=Path(data["file_name"]).suffix[1:]
|
||||
if data["file_category"] == "1"
|
||||
else None,
|
||||
pickcode=data["pick_code"],
|
||||
size=data["size_byte"] if data["file_category"] == "1" else None,
|
||||
modify_time=data["utime"],
|
||||
)
|
||||
|
||||
def get_item(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项
|
||||
"""
|
||||
try:
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/open/folder/get_info",
|
||||
"data",
|
||||
data={"path": path.as_posix()},
|
||||
no_error_log=True,
|
||||
)
|
||||
if not resp:
|
||||
return None
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
fileid=str(resp["file_id"]),
|
||||
path=path.as_posix() + ("/" if resp["file_category"] == "0" else ""),
|
||||
type="file" if resp["file_category"] == "1" else "dir",
|
||||
name=resp["file_name"],
|
||||
basename=Path(resp["file_name"]).stem,
|
||||
extension=Path(resp["file_name"]).suffix[1:]
|
||||
if resp["file_category"] == "1"
|
||||
else None,
|
||||
pickcode=resp["pick_code"],
|
||||
size=resp["size_byte"] if resp["file_category"] == "1" else None,
|
||||
modify_time=resp["utime"],
|
||||
)
|
||||
return self.__get_info_item(path)
|
||||
except Exception as e:
|
||||
logger.debug(f"【115】获取文件信息失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
"""
|
||||
try:
|
||||
return self.__get_info_item(path)
|
||||
except StorageQueryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise StorageQueryError(f"【115】查询文件信息失败: {path} - {e}") from e
|
||||
|
||||
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件夹,如不存在则创建
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.schemas import (
|
||||
TransferRenameBuildEventData,
|
||||
TransferRenameEventData,
|
||||
)
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import MediaType, ChainEventType
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
@@ -405,8 +406,23 @@ class TransHandler:
|
||||
# 判断是否要覆盖,附加文件强制覆盖
|
||||
overflag = False
|
||||
if not __is_extra_file(fileitem):
|
||||
# 目标文件
|
||||
target_item = target_oper.get_item(new_file)
|
||||
# 目标文件(严格查询:无法确认状态时拒绝覆盖,避免已有文件被误覆盖)
|
||||
try:
|
||||
target_item = target_oper.get_item_strict(new_file)
|
||||
except StorageQueryError as query_err:
|
||||
errmsg = f"无法确认目标文件状态,已跳过整理以避免误覆盖:{new_file} - {query_err}"
|
||||
logger.warn(errmsg)
|
||||
self.__update_result(
|
||||
result=result,
|
||||
success=False,
|
||||
message=errmsg,
|
||||
fileitem=fileitem,
|
||||
target_diritem=target_diritem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
)
|
||||
return result
|
||||
if target_item:
|
||||
# 目标文件已存在
|
||||
target_file = new_file
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Union
|
||||
from urllib.parse import quote
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
@@ -108,6 +109,7 @@ class Api:
|
||||
"_host",
|
||||
"_token",
|
||||
"_apikey",
|
||||
"_access_code",
|
||||
"_api_path",
|
||||
"_request_utils",
|
||||
"_version",
|
||||
@@ -130,18 +132,51 @@ class Api:
|
||||
def version(self) -> Optional[Version]:
|
||||
return self._version
|
||||
|
||||
def __init__(self, host: str, apikey: str):
|
||||
@property
|
||||
def cookies(self) -> dict:
|
||||
"""
|
||||
当前会话的Cookies,开启访问码后包含访问码校验凭证
|
||||
"""
|
||||
return self._session.cookies.get_dict()
|
||||
|
||||
def __init__(self, host: str, apikey: str, access_code: Optional[str] = None):
|
||||
"""
|
||||
:param host: 飞牛服务端地址,如http://127.0.0.1:5666/v
|
||||
:param access_code: 访问码,未开启时为空
|
||||
"""
|
||||
self._api_path = "/api/v1"
|
||||
self._host = host.rstrip("/")
|
||||
self._apikey = apikey
|
||||
self._access_code = access_code
|
||||
self._token: Optional[str] = None
|
||||
self._version: Optional[Version] = None
|
||||
self._session = requests.Session()
|
||||
self._request_utils = RequestUtils(session=self._session, timeout=10)
|
||||
|
||||
def verify_access_code(self) -> bool:
|
||||
"""
|
||||
校验访问码,通过后会话获得访问凭证,否则无法访问登录页和各应用接口
|
||||
|
||||
:return: 未配置访问码或校验通过返回True
|
||||
"""
|
||||
if not self._access_code:
|
||||
return True
|
||||
# 访问码校验地址位于设备根路径,不在/v下
|
||||
root = self._host[: -len("/v")] if self._host.endswith("/v") else self._host
|
||||
url = f"{root}/c/{quote(self._access_code, safe='')}"
|
||||
res = self._request_utils.get_res(url, allow_redirects=True)
|
||||
if res is None:
|
||||
logger.error(f"校验飞牛访问码失败,无法访问 {url}")
|
||||
return False
|
||||
if res.status_code == 404:
|
||||
# 访问码错误或校验失败时返回404
|
||||
logger.error("飞牛访问码校验失败,请检查访问码是否正确")
|
||||
return False
|
||||
if not res.ok:
|
||||
logger.error(f"飞牛访问码校验失败,状态码:{res.status_code}")
|
||||
return False
|
||||
return True
|
||||
|
||||
def sys_version(self) -> Optional[Version]:
|
||||
"""
|
||||
飞牛影视版本号
|
||||
@@ -161,6 +196,9 @@ class Api:
|
||||
|
||||
:return: 成功返回token 否则返回None
|
||||
"""
|
||||
# 开启访问码后需先通过访问码校验,否则无法访问登录接口
|
||||
if not self.verify_access_code():
|
||||
return None
|
||||
if (
|
||||
res := self.request(
|
||||
"/login",
|
||||
|
||||
@@ -12,6 +12,7 @@ from app.utils.url import UrlUtils
|
||||
class TrimeMedia:
|
||||
_username: Optional[str] = None
|
||||
_password: Optional[str] = None
|
||||
_access_code: Optional[str] = None
|
||||
|
||||
_userinfo: Optional[fnapi.User] = None
|
||||
_host: Optional[str] = None
|
||||
@@ -28,6 +29,7 @@ class TrimeMedia:
|
||||
host: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
access_code: Optional[str] = None,
|
||||
play_host: Optional[str] = None,
|
||||
sync_libraries: Optional[list] = None,
|
||||
**kwargs,
|
||||
@@ -37,13 +39,14 @@ class TrimeMedia:
|
||||
return
|
||||
self._username = username
|
||||
self._password = password
|
||||
self._access_code = access_code
|
||||
self._host = host
|
||||
self._sync_libraries = sync_libraries or []
|
||||
|
||||
if not self.reconnect():
|
||||
logger.error(f"请检查服务端地址 {host}")
|
||||
return
|
||||
if result := self.__create_api(play_host):
|
||||
if result := self.__create_api(play_host, access_code):
|
||||
self._playhost = result.api.host
|
||||
result.api.close()
|
||||
elif play_host:
|
||||
@@ -69,11 +72,14 @@ class TrimeMedia:
|
||||
version: fnapi.Version
|
||||
|
||||
@staticmethod
|
||||
def __create_api(host: Optional[str]) -> Optional["TrimeMedia._ApiCreateResult"]:
|
||||
def __create_api(
|
||||
host: Optional[str], access_code: Optional[str] = None
|
||||
) -> Optional["TrimeMedia._ApiCreateResult"]:
|
||||
"""
|
||||
创建一个飞牛API
|
||||
|
||||
:param host: 服务端地址
|
||||
:param access_code: 访问码,未开启时为空
|
||||
:return: 如果地址无效、不可访问则返回None
|
||||
"""
|
||||
|
||||
@@ -85,16 +91,19 @@ class TrimeMedia:
|
||||
if not host.endswith("/v"):
|
||||
# 尝试补上结尾的/v 测试能否正常访问
|
||||
res = TrimeMedia._ApiCreateResult()
|
||||
res.api = fnapi.Api(host + "/v", api_key)
|
||||
if fnver := res.api.sys_version():
|
||||
res.api = fnapi.Api(host + "/v", api_key, access_code)
|
||||
# 开启访问码后,需先校验才能访问各应用接口
|
||||
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
|
||||
res.version = fnver
|
||||
return res
|
||||
res.api.close()
|
||||
# 测试用户配置的地址
|
||||
res = TrimeMedia._ApiCreateResult()
|
||||
res.api = fnapi.Api(host, api_key)
|
||||
if fnver := res.api.sys_version():
|
||||
res.api = fnapi.Api(host, api_key, access_code)
|
||||
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
|
||||
res.version = fnver
|
||||
return res
|
||||
res.api.close()
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
@@ -130,7 +139,7 @@ class TrimeMedia:
|
||||
if not self.is_configured():
|
||||
return False
|
||||
self.disconnect()
|
||||
if result := self.__create_api(self._host):
|
||||
if result := self.__create_api(self._host, self._access_code):
|
||||
self._api = result.api
|
||||
self._version = result.version
|
||||
# 版本号:0.8.53, 服务版本:0.8.23
|
||||
@@ -683,4 +692,8 @@ class TrimeMedia:
|
||||
image_url, [self._api.host], strict=True
|
||||
):
|
||||
return None
|
||||
return {"Trim-MC-token": self._api.token}
|
||||
cookies = {"Trim-MC-token": self._api.token}
|
||||
if self._access_code:
|
||||
# 开启访问码后,图片请求也需要携带访问码校验凭证
|
||||
cookies.update(self._api.cookies)
|
||||
return cookies
|
||||
|
||||
-954
@@ -1,954 +0,0 @@
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Optional, Dict, List
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from watchfiles import Change, DefaultFilter, watch
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.chain.storage import StorageChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.cache import TTLCache, FileCache
|
||||
from app.core.config import settings
|
||||
from app.db.transferhistory_oper import TransferHistoryOper
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.log import logger
|
||||
from app.schemas import FileItem
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.mixins import ConfigReloadMixin
|
||||
from app.utils.singleton import SingletonClass
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
lock = Lock()
|
||||
snapshot_lock = Lock()
|
||||
|
||||
|
||||
class MonitorChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryChangeEvent:
|
||||
"""
|
||||
目录文件变化事件,隔离底层 watchfiles 事件结构。
|
||||
"""
|
||||
change_type: Change
|
||||
src_path: str
|
||||
is_directory: bool
|
||||
|
||||
|
||||
class LocalDirectoryWatcher:
|
||||
"""
|
||||
基于 watchfiles 的本地目录监控线程。
|
||||
"""
|
||||
_HANDLE_CHANGES = {Change.added, Change.modified}
|
||||
|
||||
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None):
|
||||
"""
|
||||
初始化本地目录监控。
|
||||
:param mon_path: 监控目录
|
||||
:param callback: 目录变化回调对象
|
||||
:param force_polling: 是否强制使用轮询模式,None 表示由 watchfiles 自动选择
|
||||
"""
|
||||
self._watch_path = mon_path
|
||||
self._callback = callback
|
||||
self._force_polling = force_polling
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._watch_filter = DefaultFilter()
|
||||
|
||||
@property
|
||||
def watch_path(self) -> Path:
|
||||
"""
|
||||
获取监控目录。
|
||||
:return: 监控目录
|
||||
"""
|
||||
return self._watch_path
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
启动本地目录监控线程。
|
||||
"""
|
||||
if not self._watch_path.exists():
|
||||
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
|
||||
if not self._watch_path.is_dir():
|
||||
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
|
||||
if self.is_alive():
|
||||
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
请求停止本地目录监控线程。
|
||||
"""
|
||||
self._stop_event.set()
|
||||
|
||||
def join(self, timeout: Optional[float] = None):
|
||||
"""
|
||||
等待本地目录监控线程退出。
|
||||
:param timeout: 最长等待秒数
|
||||
"""
|
||||
if self._thread:
|
||||
self._thread.join(timeout=timeout)
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否仍在运行。
|
||||
:return: 线程存活状态
|
||||
"""
|
||||
return bool(self._thread and self._thread.is_alive())
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
运行 watchfiles 主循环,并在快速模式不可用时回退到轮询。
|
||||
"""
|
||||
try:
|
||||
self._run_watch(force_polling=self._force_polling)
|
||||
except Exception as err:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if self._force_polling is True:
|
||||
logger.error(f"本地目录监控发生错误: {self._watch_path} - {err}")
|
||||
logger.debug(traceback.format_exc())
|
||||
return
|
||||
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
|
||||
try:
|
||||
self._run_watch(force_polling=True)
|
||||
except Exception as fallback_err:
|
||||
if not self._stop_event.is_set():
|
||||
logger.error(f"兼容模式监控 {self._watch_path} 仍然失败: {fallback_err}")
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def _run_watch(self, force_polling: Optional[bool]):
|
||||
"""
|
||||
执行一次 watchfiles 监控循环。
|
||||
:param force_polling: 是否强制轮询
|
||||
"""
|
||||
for changes in watch(
|
||||
str(self._watch_path),
|
||||
watch_filter=self._watch_filter,
|
||||
stop_event=self._stop_event,
|
||||
rust_timeout=1000,
|
||||
yield_on_timeout=True,
|
||||
force_polling=force_polling,
|
||||
recursive=True,
|
||||
ignore_permission_denied=True):
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
if not changes:
|
||||
continue
|
||||
self._handle_changes(changes)
|
||||
|
||||
def _handle_changes(self, changes: set[tuple[Change, str]]):
|
||||
"""
|
||||
将 watchfiles 原始变更转换为目录监控事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
"""
|
||||
changes = self._expand_added_directories(changes)
|
||||
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
||||
if change_type not in self._HANDLE_CHANGES:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
event = self._build_event(change_type=change_type, event_path=event_path)
|
||||
if not event or event.is_directory:
|
||||
continue
|
||||
file_size = self._get_file_size(event_path)
|
||||
if file_size is None:
|
||||
continue
|
||||
text = self._change_text(change_type)
|
||||
try:
|
||||
self._callback.event_handler(
|
||||
event=event,
|
||||
text=text,
|
||||
event_path=path_str,
|
||||
file_size=file_size
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
||||
|
||||
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
||||
"""
|
||||
将整体移入监控范围的新增目录展开为内部文件事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
:return: 包含目录内新增文件的变更集合
|
||||
"""
|
||||
expanded_changes = set(changes)
|
||||
for change_type, path_str in changes:
|
||||
if change_type != Change.added:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
try:
|
||||
if not event_path.is_dir():
|
||||
continue
|
||||
for nested_path in event_path.rglob("*"):
|
||||
if not nested_path.is_file():
|
||||
continue
|
||||
nested_path_str = nested_path.as_posix()
|
||||
if self._watch_filter(Change.added, nested_path_str):
|
||||
expanded_changes.add((Change.added, nested_path_str))
|
||||
except OSError as err:
|
||||
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
|
||||
return expanded_changes
|
||||
|
||||
@staticmethod
|
||||
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
||||
"""
|
||||
构建目录变化事件,路径已不存在时忽略。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:param event_path: 变化路径
|
||||
:return: 目录变化事件
|
||||
"""
|
||||
try:
|
||||
is_directory = event_path.is_dir()
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
|
||||
return None
|
||||
if not event_path.exists():
|
||||
return None
|
||||
return DirectoryChangeEvent(
|
||||
change_type=change_type,
|
||||
src_path=event_path.as_posix(),
|
||||
is_directory=is_directory
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_file_size(event_path: Path) -> Optional[int]:
|
||||
"""
|
||||
读取事件文件大小,文件已消失时返回 None。
|
||||
:param event_path: 事件文件路径
|
||||
:return: 文件大小
|
||||
"""
|
||||
try:
|
||||
return event_path.stat().st_size
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _change_text(change_type: Change) -> str:
|
||||
"""
|
||||
转换 watchfiles 事件类型为日志文案。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:return: 事件描述
|
||||
"""
|
||||
if change_type == Change.modified:
|
||||
return "修改"
|
||||
return "新增"
|
||||
|
||||
|
||||
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控处理链,单例模式
|
||||
"""
|
||||
CONFIG_WATCH = {SystemConfigKey.Directories.value}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 存储过照间隔(分钟)
|
||||
self._snapshot_interval = 5
|
||||
# TTL缓存,10秒钟有效
|
||||
self._cache = TTLCache(region="monitor", maxsize=1024, ttl=10)
|
||||
# 快照文件缓存
|
||||
self._snapshot_cache = FileCache(base=settings.CACHE_PATH / "snapshots")
|
||||
# 监控的文件扩展名
|
||||
self.all_exts = settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
|
||||
def on_config_changed(self):
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
return "目录监控"
|
||||
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None):
|
||||
"""
|
||||
保存快照到文件缓存
|
||||
:param storage: 存储名称
|
||||
:param snapshot: 快照数据
|
||||
:param last_snapshot_time: 上次快照时间戳
|
||||
:param file_count: 文件数量,用于调整监控间隔
|
||||
"""
|
||||
try:
|
||||
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
|
||||
if snapshot_time is None:
|
||||
snapshot_time = last_snapshot_time or time.time()
|
||||
snapshot_data = {
|
||||
'timestamp': snapshot_time,
|
||||
'file_count': file_count,
|
||||
'snapshot': snapshot
|
||||
}
|
||||
# 使用FileCache保存快照数据
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
|
||||
self._snapshot_cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
|
||||
logger.debug(f"快照已保存到缓存: {storage}")
|
||||
except Exception as e:
|
||||
logger.error(f"保存快照失败: {e}")
|
||||
|
||||
def reset_snapshot(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准
|
||||
:param storage: 存储名称
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
if self._snapshot_cache.exists(cache_key, region="snapshots"):
|
||||
self._snapshot_cache.delete(cache_key, region="snapshots")
|
||||
logger.info(f"快照已重置: {storage}")
|
||||
return True
|
||||
logger.debug(f"快照文件不存在,无需重置: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"重置快照失败: {storage} - {e}")
|
||||
return False
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)
|
||||
:param storage: 存储名称
|
||||
:param mon_path: 监控路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
|
||||
|
||||
# 生成快照
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=0 # 全量扫描,不使用增量
|
||||
)
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return False
|
||||
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
|
||||
|
||||
# 处理所有文件
|
||||
processed_count = 0
|
||||
for file_path, file_info in new_snapshot.items():
|
||||
try:
|
||||
if not self.__is_transfer_candidate_path(Path(file_path)):
|
||||
continue
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(file_path), file_size=file_size):
|
||||
processed_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||
|
||||
# 保存快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
|
||||
return False
|
||||
|
||||
def load_snapshot(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照
|
||||
:param storage: 存储名称
|
||||
:return: 快照数据或None
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_data = self._snapshot_cache.get(cache_key, region="snapshots")
|
||||
if snapshot_data:
|
||||
data = json.loads(snapshot_data.decode('utf-8'))
|
||||
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
|
||||
return data
|
||||
logger.debug(f"快照文件不存在: {storage}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"加载快照失败: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def adjust_monitor_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔
|
||||
:param file_count: 文件数量
|
||||
:return: 监控间隔(分钟)
|
||||
"""
|
||||
if file_count < 100:
|
||||
return 5 # 5分钟
|
||||
elif file_count < 500:
|
||||
return 10 # 10分钟
|
||||
elif file_count < 1000:
|
||||
return 15 # 15分钟
|
||||
else:
|
||||
return 30 # 30分钟
|
||||
|
||||
@staticmethod
|
||||
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件(只处理新增和修改,不处理删除)
|
||||
:param old_snapshot: 旧快照
|
||||
:param new_snapshot: 新快照
|
||||
:return: 变化信息
|
||||
"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_files = set(old_snapshot.keys())
|
||||
new_files = set(new_snapshot.keys())
|
||||
|
||||
# 新增文件
|
||||
changes['added'] = list(new_files - old_files)
|
||||
|
||||
# 修改文件(大小或时间变化)
|
||||
for file_path in old_files & new_files:
|
||||
old_info = old_snapshot[file_path]
|
||||
new_info = new_snapshot[file_path]
|
||||
|
||||
# 检查文件大小变化
|
||||
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
|
||||
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
|
||||
|
||||
# 检查修改时间变化(如果有的话)
|
||||
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
||||
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
||||
|
||||
if old_size != new_size or (old_time and new_time and old_time != new_time):
|
||||
changes['modified'].append(file_path)
|
||||
|
||||
return changes
|
||||
|
||||
@staticmethod
|
||||
def __is_bluray_sub(_path: Path) -> bool:
|
||||
"""
|
||||
判断是否蓝光原盘目录内的媒体流文件。
|
||||
"""
|
||||
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
|
||||
|
||||
@staticmethod
|
||||
def __get_bluray_dir(_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
获取蓝光原盘BDMV目录的上级目录。
|
||||
"""
|
||||
for p in _path.parents:
|
||||
if p.name == "BDMV":
|
||||
return p.parent
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
|
||||
"""
|
||||
判断路径后缀是否命中给定扩展名列表。
|
||||
"""
|
||||
if not file_path.suffix:
|
||||
return False
|
||||
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
|
||||
|
||||
def __is_transfer_candidate_path(self, file_path: Path) -> bool:
|
||||
"""
|
||||
判断监控事件路径是否需要进入整理链。
|
||||
"""
|
||||
if self.__has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
|
||||
return False
|
||||
return self.__has_suffix_in(file_path, self.all_exts)
|
||||
|
||||
@staticmethod
|
||||
def __build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
|
||||
"""
|
||||
生成整理记录使用的源路径。
|
||||
"""
|
||||
if is_bluray_folder:
|
||||
return f"{event_path.as_posix()}/"
|
||||
return event_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def __has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
|
||||
"""
|
||||
判断源文件是否已经存在整理记录。
|
||||
"""
|
||||
try:
|
||||
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
|
||||
except Exception as err:
|
||||
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
|
||||
"""
|
||||
统计目录下的文件数量(用于检测是否超过系统限制)
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查数量,避免长时间阻塞
|
||||
:return: 文件数量
|
||||
"""
|
||||
try:
|
||||
count = 0
|
||||
import os
|
||||
for root, dirs, files in os.walk(str(directory)):
|
||||
count += len(files)
|
||||
if count > max_check:
|
||||
return count
|
||||
return count
|
||||
except Exception as err:
|
||||
logger.debug(f"统计目录文件数量失败: {err}")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def check_system_limits() -> Dict[str, Any]:
|
||||
"""
|
||||
检查系统限制
|
||||
:return: 系统限制信息
|
||||
"""
|
||||
limits = {
|
||||
'max_user_watches': 0,
|
||||
'max_user_instances': 0,
|
||||
'current_watches': 0,
|
||||
'warnings': []
|
||||
}
|
||||
|
||||
try:
|
||||
system = platform.system()
|
||||
if system == 'Linux':
|
||||
# 检查 inotify 限制
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_watches'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 限制失败: {e}")
|
||||
limits['max_user_watches'] = 8192 # 默认值
|
||||
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_instances'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 实例限制失败: {e}")
|
||||
|
||||
# 检查当前使用的watches
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(['find', '/proc/*/fd', '-lname', 'anon_inode:inotify', '-printf', '%h\n'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
limits['current_watches'] = len(result.stdout.strip().split('\n'))
|
||||
except Exception as e:
|
||||
logger.debug(f"检查当前 inotify 使用失败: {e}")
|
||||
|
||||
except Exception as e:
|
||||
limits['warnings'].append(f"检查系统限制时出错: {e}")
|
||||
|
||||
return limits
|
||||
|
||||
@staticmethod
|
||||
def get_system_optimization_tips() -> List[str]:
|
||||
"""
|
||||
获取系统优化建议
|
||||
:return: 优化建议列表
|
||||
"""
|
||||
tips = []
|
||||
system = platform.system()
|
||||
|
||||
if system == 'Linux':
|
||||
tips.extend([
|
||||
"增加 inotify 监控数量限制:",
|
||||
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"sudo sysctl -p",
|
||||
"",
|
||||
"如果在Docker中运行,请在宿主机上执行以上命令"
|
||||
])
|
||||
elif system == 'Darwin':
|
||||
tips.extend([
|
||||
"macOS 系统优化建议:",
|
||||
"sudo sysctl kern.maxfiles=65536",
|
||||
"sudo sysctl kern.maxfilesperproc=32768",
|
||||
"ulimit -n 32768"
|
||||
])
|
||||
elif system == 'Windows':
|
||||
tips.extend([
|
||||
"Windows 系统优化建议:",
|
||||
"1. 关闭不必要的实时保护软件对监控目录的扫描",
|
||||
"2. 将监控目录添加到Windows Defender排除列表",
|
||||
"3. 确保有足够的可用内存"
|
||||
])
|
||||
|
||||
return tips
|
||||
|
||||
@staticmethod
|
||||
def should_use_polling(directory: Path, monitor_mode: str,
|
||||
file_count: int, limits: dict) -> tuple[bool, str]:
|
||||
"""
|
||||
判断是否应该使用轮询模式
|
||||
:param directory: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param file_count: 目录文件数量
|
||||
:param limits: 系统限制信息
|
||||
:return: (是否使用轮询, 原因)
|
||||
"""
|
||||
if monitor_mode == "compatibility":
|
||||
return True, "用户配置为兼容模式"
|
||||
|
||||
# 检查网络文件系统
|
||||
if SystemUtils.is_network_filesystem(directory):
|
||||
return True, "检测到网络文件系统,建议使用兼容模式"
|
||||
|
||||
max_watches = limits.get('max_user_watches')
|
||||
if max_watches and file_count > max_watches * 0.8:
|
||||
return True, f"目录文件数量({file_count})接近系统限制({max_watches})"
|
||||
return False, "使用快速模式"
|
||||
|
||||
def init(self):
|
||||
"""
|
||||
启动监控
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
|
||||
# 按下载目录去重
|
||||
monitor_dirs = list({f"{d.storage}_{d.download_path}": d for d in monitor_dirs}.values())
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
mon_storages = {}
|
||||
for mon_dir in monitor_dirs:
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
if mon_dir.monitor_type != "monitor":
|
||||
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
|
||||
continue
|
||||
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
if target_path.is_relative_to(mon_path):
|
||||
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
|
||||
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
|
||||
continue
|
||||
|
||||
# 启动监控
|
||||
if mon_dir.storage == "local":
|
||||
# 本地目录监控
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
try:
|
||||
# 统计文件数量并给出提示
|
||||
file_count = self.count_directory_files(mon_path)
|
||||
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
|
||||
|
||||
# 检查系统限制
|
||||
limits = self.check_system_limits()
|
||||
|
||||
# 检查是否需要使用轮询模式
|
||||
use_polling, reason = self.should_use_polling(mon_path,
|
||||
monitor_mode=mon_dir.monitor_mode,
|
||||
file_count=file_count,
|
||||
limits=limits)
|
||||
logger.info(f"监控模式决策: {reason}")
|
||||
|
||||
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
|
||||
logger.info(f"使用{mode_name}监控 {mon_path}")
|
||||
if not use_polling:
|
||||
if limits['warnings']:
|
||||
for warning in limits['warnings']:
|
||||
logger.warn(f"系统限制警告: {warning}")
|
||||
if limits['max_user_watches'] > 0:
|
||||
usage_percent = (file_count / limits['max_user_watches']) * 100
|
||||
logger.info(
|
||||
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
|
||||
|
||||
watcher = LocalDirectoryWatcher(
|
||||
mon_path=mon_path,
|
||||
callback=self,
|
||||
force_polling=True if use_polling else None
|
||||
)
|
||||
self._watchers.append(watcher)
|
||||
watcher.start()
|
||||
|
||||
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
|
||||
|
||||
except Exception as e:
|
||||
err_msg = str(e)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
|
||||
if "inotify" in err_msg.lower():
|
||||
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
|
||||
logger.error("解决方案:")
|
||||
tips = self.get_system_optimization_tips()
|
||||
for tip in tips:
|
||||
logger.error(f" {tip}")
|
||||
logger.error("执行上述命令后重启 MoviePilot")
|
||||
elif "permission" in err_msg.lower():
|
||||
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
|
||||
else:
|
||||
logger.error("建议尝试使用兼容模式进行监控")
|
||||
|
||||
messagehelper.put(f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}", title="目录监控")
|
||||
else:
|
||||
if not mon_storages.get(mon_dir.storage):
|
||||
mon_storages[mon_dir.storage] = []
|
||||
mon_storages[mon_dir.storage].append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self.load_snapshot(storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = self.adjust_monitor_interval(file_count)
|
||||
for path in paths:
|
||||
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': storage,
|
||||
'mon_paths': paths
|
||||
},
|
||||
id=f"monitor_{storage}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
|
||||
|
||||
# 启动定时服务
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
logger.info("定时监控服务已启动")
|
||||
|
||||
# 输出监控总结
|
||||
local_count = len([d for d in monitor_dirs if d.storage == "local" and d.monitor_type == "monitor"])
|
||||
remote_count = len([d for d in monitor_dirs if d.storage != "local" and d.monitor_type == "monitor"])
|
||||
logger.info(f"目录监控启动完成: 本地监控 {local_count} 个,远程监控 {remote_count} 个")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
"""
|
||||
轮询监控(改进版)
|
||||
"""
|
||||
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
|
||||
with snapshot_lock:
|
||||
try:
|
||||
# 加载上次快照数据
|
||||
old_snapshot_data = self.load_snapshot(storage)
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
|
||||
|
||||
# 判断是否为首次快照:检查快照文件是否存在且有效
|
||||
is_first_snapshot = old_snapshot_data is None
|
||||
new_snapshot = {}
|
||||
for mon_path in mon_paths:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
# 生成新快照(增量模式)
|
||||
snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
|
||||
if snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
continue
|
||||
new_snapshot.update(snapshot)
|
||||
file_count = len(snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {file_count} 个文件")
|
||||
file_count = len(new_snapshot)
|
||||
if not is_first_snapshot:
|
||||
# 比较快照找出变化
|
||||
changes = self.compare_snapshots(old_snapshot, new_snapshot)
|
||||
added_files = [
|
||||
file_path
|
||||
for file_path in changes['added']
|
||||
if self.__is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
modified_files = [
|
||||
file_path
|
||||
for file_path in changes['modified']
|
||||
if self.__is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
|
||||
# 处理新增文件
|
||||
handled_added_count = 0
|
||||
for new_file in added_files:
|
||||
file_info = new_snapshot.get(new_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
|
||||
handled_added_count += 1
|
||||
|
||||
# 处理修改文件
|
||||
handled_modified_count = 0
|
||||
for modified_file in modified_files:
|
||||
file_info = new_snapshot.get(modified_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
|
||||
handled_modified_count += 1
|
||||
|
||||
if handled_added_count or handled_modified_count:
|
||||
logger.info(
|
||||
f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
|
||||
else:
|
||||
logger.debug(f"{storage} 无文件变化")
|
||||
else:
|
||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||
|
||||
# 保存新快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count, last_snapshot_time)
|
||||
|
||||
# 动态调整监控间隔
|
||||
new_interval = self.adjust_monitor_interval(file_count)
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}")
|
||||
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
|
||||
# 重新安排任务
|
||||
self._scheduler.modify_job(
|
||||
f"monitor_{storage}",
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
logger.info(f"{storage}:{monitor_scope} 监控间隔已调整为 {new_interval} 分钟")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}")
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
"""
|
||||
处理文件变化
|
||||
:param event: 事件
|
||||
:param text: 事件描述
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if not event.is_directory:
|
||||
if not self.__is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
# 整理文件
|
||||
self.__handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
|
||||
|
||||
def __handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
|
||||
"""
|
||||
整理一个文件
|
||||
:param storage: 存储
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
:return: 是否进入整理链
|
||||
"""
|
||||
# 全程加锁
|
||||
with lock:
|
||||
is_bluray_folder = False
|
||||
# 蓝光原盘文件处理
|
||||
if self.__is_bluray_sub(event_path):
|
||||
event_path = self.__get_bluray_dir(event_path)
|
||||
if not event_path:
|
||||
return False
|
||||
is_bluray_folder = True
|
||||
elif not self.__is_transfer_candidate_path(event_path):
|
||||
return False
|
||||
|
||||
# TTL缓存控重
|
||||
if self._cache.get(str(event_path)):
|
||||
return False
|
||||
self._cache[str(event_path)] = True
|
||||
|
||||
src_path = self.__build_transfer_src_path(
|
||||
event_path=event_path,
|
||||
is_bluray_folder=is_bluray_folder,
|
||||
)
|
||||
has_transfer_history = self.__has_transfer_history(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if has_transfer_history is not False:
|
||||
return False
|
||||
|
||||
try:
|
||||
if is_bluray_folder:
|
||||
logger.info(f"开始整理蓝光原盘: {event_path}")
|
||||
else:
|
||||
logger.info(f"开始整理文件: {event_path}")
|
||||
# 开始整理
|
||||
TransferChain().do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=src_path,
|
||||
type="file" if not is_bluray_folder else "dir",
|
||||
name=event_path.name,
|
||||
basename=event_path.stem,
|
||||
extension=event_path.suffix[1:],
|
||||
size=file_size
|
||||
)
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
||||
return False
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
退出监控
|
||||
"""
|
||||
if self._watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in self._watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
self._watchers = []
|
||||
logger.info("本地目录监控服务已停止")
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
if self._cache:
|
||||
self._cache.close()
|
||||
if self._snapshot_cache:
|
||||
self._snapshot_cache.close()
|
||||
@@ -0,0 +1,14 @@
|
||||
"""
|
||||
目录监控包。
|
||||
|
||||
- watcher.py 本地目录监控线程(watchfiles)
|
||||
- syslimits.py 系统限制探测与监控模式决策
|
||||
- snapshot.py 远程快照存取与比对
|
||||
- dispatcher.py 监控事件到整理链的分发
|
||||
- poller.py 远程目录轮询监控
|
||||
- monitor.py Monitor 门面:装配、生命周期与健康检查
|
||||
"""
|
||||
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
|
||||
from app.monitor.monitor import Monitor
|
||||
|
||||
__all__ = ["DirectoryChangeEvent", "LocalDirectoryWatcher", "Monitor"]
|
||||
@@ -0,0 +1,210 @@
|
||||
import re
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import settings
|
||||
from app.db.transferhistory_oper import TransferHistoryOper
|
||||
from app.log import logger
|
||||
from app.schemas import FileItem
|
||||
|
||||
|
||||
class TransferDispatcher:
|
||||
"""
|
||||
将监控事件分发到整理链:候选判定、TTL 去重、整理历史查重与整理触发。
|
||||
"""
|
||||
# 历史查询失败待重试队列上限,防止长时间故障期间无限增长
|
||||
MAX_PENDING_RETRIES = 1000
|
||||
# 单个文件的最大重试次数(按健康检查周期计,60 次约 1 小时)
|
||||
MAX_RETRY_ATTEMPTS = 60
|
||||
|
||||
def __init__(self, all_exts: Optional[List[str]] = None, cache: Optional[Any] = None):
|
||||
"""
|
||||
初始化整理分发器。
|
||||
:param all_exts: 监控的文件扩展名,默认取系统配置
|
||||
:param cache: 去重缓存,默认使用 10 秒 TTL 缓存
|
||||
"""
|
||||
self.all_exts = all_exts if all_exts is not None else (
|
||||
settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT)
|
||||
self._cache = cache if cache is not None else TTLCache(region="monitor", maxsize=1024, ttl=10)
|
||||
self._lock = Lock()
|
||||
# 历史查询失败待重试的文件
|
||||
self._pending_retries: Dict[str, Dict[str, Any]] = {}
|
||||
self._pending_guard = Lock()
|
||||
|
||||
@staticmethod
|
||||
def _is_bluray_sub(_path: Path) -> bool:
|
||||
"""
|
||||
判断是否蓝光原盘目录内的媒体流文件。
|
||||
"""
|
||||
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
|
||||
|
||||
@staticmethod
|
||||
def _get_bluray_dir(_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
获取蓝光原盘BDMV目录的上级目录。
|
||||
"""
|
||||
for p in _path.parents:
|
||||
if p.name == "BDMV":
|
||||
return p.parent
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
|
||||
"""
|
||||
判断路径后缀是否命中给定扩展名列表。
|
||||
"""
|
||||
if not file_path.suffix:
|
||||
return False
|
||||
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
|
||||
|
||||
def is_transfer_candidate_path(self, file_path: Path) -> bool:
|
||||
"""
|
||||
判断监控事件路径是否需要进入整理链。
|
||||
"""
|
||||
if self._has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
|
||||
return False
|
||||
return self._has_suffix_in(file_path, self.all_exts)
|
||||
|
||||
@staticmethod
|
||||
def _build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
|
||||
"""
|
||||
生成整理记录使用的源路径。
|
||||
"""
|
||||
if is_bluray_folder:
|
||||
return f"{event_path.as_posix()}/"
|
||||
return event_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
|
||||
"""
|
||||
判断源文件是否已经存在整理记录。
|
||||
:return: True/False 查询成功,None 查询失败
|
||||
"""
|
||||
try:
|
||||
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
|
||||
except Exception as err:
|
||||
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pending_key(storage: str, event_path: Path) -> str:
|
||||
"""
|
||||
生成待重试文件的唯一键。
|
||||
"""
|
||||
return f"{storage}:{Path(event_path).as_posix()}"
|
||||
|
||||
def _register_pending(self, storage: str, event_path: Path, file_size: float = None):
|
||||
"""
|
||||
登记历史查询失败的文件待重试,重复失败累计次数,超限后放弃。
|
||||
:param storage: 存储
|
||||
:param event_path: 原始事件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
key = self._pending_key(storage, event_path)
|
||||
with self._pending_guard:
|
||||
entry = self._pending_retries.get(key)
|
||||
if entry:
|
||||
entry["attempts"] += 1
|
||||
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
|
||||
self._pending_retries.pop(key, None)
|
||||
logger.error(f"整理历史查询持续失败,已放弃重试: {key}")
|
||||
return
|
||||
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
|
||||
logger.error(f"整理重试队列已满,丢弃: {key}")
|
||||
return
|
||||
self._pending_retries[key] = {
|
||||
"storage": storage,
|
||||
"event_path": event_path,
|
||||
"file_size": file_size,
|
||||
"attempts": 1
|
||||
}
|
||||
logger.warn(f"整理历史查询失败,已登记待重试: {key}")
|
||||
|
||||
def _discard_pending(self, storage: str, event_path: Path):
|
||||
"""
|
||||
历史查询已得到确定结果,移除待重试登记。
|
||||
:param storage: 存储
|
||||
:param event_path: 原始事件路径
|
||||
"""
|
||||
with self._pending_guard:
|
||||
self._pending_retries.pop(self._pending_key(storage, event_path), None)
|
||||
|
||||
def retry_pending(self):
|
||||
"""
|
||||
重试历史查询失败的文件,由健康检查周期驱动。
|
||||
成功或得到确定结果的条目在 handle_file 内部自动移除。
|
||||
"""
|
||||
with self._pending_guard:
|
||||
items = list(self._pending_retries.values())
|
||||
for item in items:
|
||||
logger.info(f"重试整理: {item['storage']}:{item['event_path']}")
|
||||
self.handle_file(storage=item["storage"], event_path=item["event_path"],
|
||||
file_size=item["file_size"])
|
||||
|
||||
def handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
|
||||
"""
|
||||
整理一个文件。
|
||||
:param storage: 存储
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
:return: 是否进入整理链
|
||||
"""
|
||||
with self._lock:
|
||||
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
|
||||
origin_path = event_path
|
||||
is_bluray_folder = False
|
||||
# 蓝光原盘文件处理
|
||||
if self._is_bluray_sub(event_path):
|
||||
event_path = self._get_bluray_dir(event_path)
|
||||
if not event_path:
|
||||
return False
|
||||
is_bluray_folder = True
|
||||
elif not self.is_transfer_candidate_path(event_path):
|
||||
return False
|
||||
|
||||
# TTL缓存控重
|
||||
if self._cache.get(str(event_path)):
|
||||
return False
|
||||
self._cache[str(event_path)] = True
|
||||
|
||||
src_path = self._build_transfer_src_path(
|
||||
event_path=event_path,
|
||||
is_bluray_folder=is_bluray_folder,
|
||||
)
|
||||
has_transfer_history = self._has_transfer_history(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if has_transfer_history is None:
|
||||
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
|
||||
self._register_pending(storage=storage, event_path=origin_path, file_size=file_size)
|
||||
return False
|
||||
self._discard_pending(storage=storage, event_path=origin_path)
|
||||
if has_transfer_history:
|
||||
return False
|
||||
|
||||
try:
|
||||
if is_bluray_folder:
|
||||
logger.info(f"开始整理蓝光原盘: {event_path}")
|
||||
else:
|
||||
logger.info(f"开始整理文件: {event_path}")
|
||||
# 开始整理
|
||||
TransferChain().do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=src_path,
|
||||
type="file" if not is_bluray_folder else "dir",
|
||||
name=event_path.name,
|
||||
basename=event_path.stem,
|
||||
extension=event_path.suffix[1:],
|
||||
size=file_size
|
||||
)
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
||||
return False
|
||||
@@ -0,0 +1,508 @@
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from app.core.config import settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.log import logger
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.monitor.poller import RemotePoller
|
||||
from app.monitor.snapshot import SnapshotStore
|
||||
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.mixins import ConfigReloadMixin
|
||||
from app.utils.singleton import SingletonClass
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
||||
"""
|
||||
CONFIG_WATCH = {SystemConfigKey.Directories.value}
|
||||
# 目录监控健康检查间隔(秒)
|
||||
WATCHDOG_INTERVAL = 60
|
||||
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
|
||||
RECOVERY_STABLE_CYCLES = 5
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 本地目录监控列表读写锁
|
||||
self._watcher_lock = Lock()
|
||||
# 启动失败待重试的本地监控配置
|
||||
self._pending_locals: List[Dict[str, Any]] = []
|
||||
# 已告警的监控目录,避免重复推送
|
||||
self._alerted_paths: set = set()
|
||||
# 各监控目录已告警过的自动重启次数
|
||||
self._restart_marks: Dict[str, int] = {}
|
||||
# 各监控目录连续稳定的健康检查周期数
|
||||
self._stable_cycles: Dict[str, int] = {}
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 整理分发器
|
||||
self._dispatcher = TransferDispatcher()
|
||||
# 快照存储
|
||||
self._store = SnapshotStore()
|
||||
# 远程轮询监控
|
||||
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
|
||||
alert_cb=self.__poller_alert)
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
|
||||
def on_config_changed(self):
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
return "目录监控"
|
||||
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None):
|
||||
"""
|
||||
保存快照到文件缓存。
|
||||
"""
|
||||
self._store.save(storage, snapshot, file_count=file_count, last_snapshot_time=last_snapshot_time)
|
||||
|
||||
def load_snapshot(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照。
|
||||
"""
|
||||
return self._store.load(storage)
|
||||
|
||||
def reset_snapshot(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准。
|
||||
"""
|
||||
return self._store.reset(storage)
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
"""
|
||||
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
|
||||
|
||||
@staticmethod
|
||||
def adjust_monitor_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔。
|
||||
"""
|
||||
return SnapshotStore.adjust_interval(file_count)
|
||||
|
||||
@staticmethod
|
||||
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件。
|
||||
"""
|
||||
return SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
|
||||
def init(self):
|
||||
"""
|
||||
启动监控
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
|
||||
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
|
||||
valid_dirs = []
|
||||
for mon_dir in monitor_dirs:
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
if mon_dir.monitor_type != "monitor":
|
||||
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
|
||||
continue
|
||||
valid_dirs.append(mon_dir)
|
||||
|
||||
deduped: Dict[str, Any] = {}
|
||||
for mon_dir in valid_dirs:
|
||||
key = f"{mon_dir.storage}_{mon_dir.download_path}"
|
||||
if key in deduped:
|
||||
logger.warn(f"监控配置重复,忽略后一条: {mon_dir.download_path}"
|
||||
f"(媒体库 {mon_dir.library_path})")
|
||||
continue
|
||||
deduped[key] = mon_dir
|
||||
monitor_dirs = list(deduped.values())
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
|
||||
|
||||
mon_storages: Dict[str, List[Path]] = {}
|
||||
# 本地监控启动结果计数,用于输出真实的启动总结
|
||||
local_started = 0
|
||||
local_failed = 0
|
||||
for mon_dir in monitor_dirs:
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
if target_path.is_relative_to(mon_path):
|
||||
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
|
||||
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
|
||||
continue
|
||||
|
||||
# 启动监控
|
||||
if mon_dir.storage == "local":
|
||||
if self.__start_local_monitor(mon_path=mon_path, monitor_mode=mon_dir.monitor_mode):
|
||||
local_started += 1
|
||||
else:
|
||||
local_failed += 1
|
||||
else:
|
||||
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self._store.load(storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = SnapshotStore.adjust_interval(file_count)
|
||||
for path in paths:
|
||||
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': storage,
|
||||
'mon_paths': paths
|
||||
},
|
||||
id=f"monitor_{storage}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
|
||||
|
||||
# 监控健康检查:重建异常监控线程、重试启动失败目录、重试历史查询失败的文件
|
||||
if local_started or local_failed or mon_storages:
|
||||
self._scheduler.add_job(
|
||||
self.watchdog,
|
||||
'interval',
|
||||
seconds=self.WATCHDOG_INTERVAL,
|
||||
id="monitor_watchdog",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
|
||||
|
||||
# 启动定时服务
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
logger.info("定时监控服务已启动")
|
||||
|
||||
# 输出监控总结,报告实际启动成功数而不是配置数
|
||||
remote_count = sum(len(paths) for paths in mon_storages.values())
|
||||
summary = f"目录监控启动完成: 本地监控 {local_started} 个成功"
|
||||
if local_failed:
|
||||
summary += f"、{local_failed} 个失败(将自动退避重试)"
|
||||
summary += f",远程监控 {remote_count} 个"
|
||||
if local_failed:
|
||||
logger.warn(summary)
|
||||
else:
|
||||
logger.info(summary)
|
||||
|
||||
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
|
||||
"""
|
||||
启动单个本地目录监控,失败时登记待重试。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: 是否启动成功
|
||||
"""
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
try:
|
||||
# 检查是否需要使用轮询模式(兼容模式/网络存储不做启动期目录遍历)
|
||||
use_polling, reason, limits, file_count = decide_monitor_mode(mon_path, monitor_mode)
|
||||
logger.info(f"监控模式决策: {reason}")
|
||||
|
||||
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
|
||||
logger.info(f"使用{mode_name}监控 {mon_path}")
|
||||
if file_count is not None:
|
||||
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
|
||||
if not use_polling and limits:
|
||||
if limits['warnings']:
|
||||
for warning in limits['warnings']:
|
||||
logger.warn(f"系统限制警告: {warning}")
|
||||
if limits['max_user_watches'] > 0 and file_count is not None:
|
||||
usage_percent = (file_count / limits['max_user_watches']) * 100
|
||||
logger.info(
|
||||
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
|
||||
|
||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||
poll_delay_ms = None
|
||||
if use_polling and SystemUtils.is_network_filesystem(mon_path):
|
||||
poll_delay_ms = LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
|
||||
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
|
||||
|
||||
watcher = LocalDirectoryWatcher(
|
||||
mon_path=mon_path,
|
||||
callback=self,
|
||||
force_polling=True if use_polling else None,
|
||||
poll_delay_ms=poll_delay_ms
|
||||
)
|
||||
# 启动成功后再登记,避免失败的监控残留在列表中
|
||||
watcher.start()
|
||||
with self._watcher_lock:
|
||||
self._watchers.append(watcher)
|
||||
self._pending_locals = [
|
||||
pending for pending in self._pending_locals
|
||||
if pending["mon_path"] != mon_path
|
||||
]
|
||||
self.__clear_alert(mon_path, f"本地目录监控已恢复: {mon_path} [{mode_name}]")
|
||||
|
||||
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
|
||||
return False
|
||||
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
|
||||
"""
|
||||
处理本地目录监控启动失败,登记待重试并按需告警。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param err: 启动异常
|
||||
"""
|
||||
err_msg = str(err)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
|
||||
if "inotify" in err_msg.lower():
|
||||
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
|
||||
logger.error("解决方案:")
|
||||
for tip in get_system_optimization_tips():
|
||||
logger.error(f" {tip}")
|
||||
logger.error("执行上述命令后重启 MoviePilot")
|
||||
elif "permission" in err_msg.lower():
|
||||
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
|
||||
elif isinstance(err, (FileNotFoundError, NotADirectoryError)):
|
||||
logger.error("监控目录当前不可用,网络存储/FUSE 挂载可能尚未就绪,将自动重试")
|
||||
elif monitor_mode != "compatibility":
|
||||
logger.error("建议尝试使用兼容模式进行监控")
|
||||
|
||||
with self._watcher_lock:
|
||||
if all(pending["mon_path"] != mon_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": mon_path,
|
||||
"monitor_mode": monitor_mode
|
||||
})
|
||||
self.__send_alert(mon_path,
|
||||
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
|
||||
f"将自动退避重试")
|
||||
|
||||
def watchdog(self):
|
||||
"""
|
||||
目录监控健康检查:重建崩溃或静默失效的监控线程,并重试启动失败的监控目录。
|
||||
"""
|
||||
try:
|
||||
self.__check_watchers()
|
||||
self.__retry_pending_locals()
|
||||
self._dispatcher.retry_pending()
|
||||
except Exception as e:
|
||||
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
||||
|
||||
def __check_watchers(self):
|
||||
"""
|
||||
检查本地目录监控线程状态,异常时重建。
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
watchers = list(self._watchers)
|
||||
for watcher in watchers:
|
||||
key = str(watcher.watch_path)
|
||||
if watcher.is_stalled():
|
||||
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
|
||||
elif not watcher.is_alive():
|
||||
reason = "监控线程已退出"
|
||||
else:
|
||||
# 线程已自愈,但崩溃过就要告警,避免自动重启把故障变成新的静默
|
||||
if watcher.restart_count > self._restart_marks.get(key, 0):
|
||||
self._restart_marks[key] = watcher.restart_count
|
||||
self._stable_cycles[key] = 0
|
||||
self.__send_alert(watcher.watch_path,
|
||||
f"目录监控发生错误并已自动重启"
|
||||
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
|
||||
else:
|
||||
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
|
||||
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
|
||||
if self._stable_cycles[key] >= self.RECOVERY_STABLE_CYCLES:
|
||||
self.__clear_alert(watcher.watch_path, f"目录监控已恢复正常: {watcher.watch_path}")
|
||||
continue
|
||||
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
|
||||
self.__send_alert(watcher.watch_path,
|
||||
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
|
||||
self.__rebuild_watcher(watcher)
|
||||
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
||||
"""
|
||||
重建一个本地目录监控线程。
|
||||
:param watcher: 需要重建的监控
|
||||
"""
|
||||
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
|
||||
watcher.stop()
|
||||
new_watcher = LocalDirectoryWatcher(
|
||||
mon_path=watcher.watch_path,
|
||||
callback=self,
|
||||
force_polling=watcher.force_polling,
|
||||
poll_delay_ms=watcher.poll_delay_ms
|
||||
)
|
||||
try:
|
||||
new_watcher.start()
|
||||
except Exception as e:
|
||||
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
|
||||
with self._watcher_lock:
|
||||
self._watchers = [item for item in self._watchers if item is not watcher]
|
||||
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": watcher.watch_path,
|
||||
# 重建沿用原监控模式,force_polling 为 True 即兼容模式
|
||||
"monitor_mode": "compatibility" if watcher.force_polling else "fast"
|
||||
})
|
||||
return
|
||||
with self._watcher_lock:
|
||||
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
||||
# 新监控的重启计数从零开始,同步重置告警基准
|
||||
self._restart_marks.pop(str(watcher.watch_path), None)
|
||||
self._stable_cycles.pop(str(watcher.watch_path), None)
|
||||
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
|
||||
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
|
||||
|
||||
def __retry_pending_locals(self):
|
||||
"""
|
||||
重试启动失败的本地目录监控,给网络存储/FUSE 挂载留出就绪时间。
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
pending = list(self._pending_locals)
|
||||
for item in pending:
|
||||
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
|
||||
if item.get("skip_cycles", 0) > 0:
|
||||
item["skip_cycles"] -= 1
|
||||
continue
|
||||
logger.info(f"重试启动本地目录监控: {item['mon_path']}")
|
||||
if not self.__start_local_monitor(mon_path=item["mon_path"], monitor_mode=item["monitor_mode"]):
|
||||
item["attempts"] = item.get("attempts", 0) + 1
|
||||
item["skip_cycles"] = min(item["attempts"], 10)
|
||||
|
||||
def __send_alert(self, mon_path: Path, message: str):
|
||||
"""
|
||||
推送目录监控异常告警,同一目录仅在状态变化时推送一次。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 告警内容
|
||||
"""
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key in self._alerted_paths:
|
||||
return
|
||||
self._alerted_paths.add(key)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
@staticmethod
|
||||
def __poller_alert(storage: str, message: str):
|
||||
"""
|
||||
远程轮询监控告警回调,复用消息渠道推送。
|
||||
:param storage: 存储名称
|
||||
:param message: 告警内容
|
||||
"""
|
||||
logger.warn(f"[{storage}] {message}")
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def __clear_alert(self, mon_path: Path, message: str):
|
||||
"""
|
||||
清除目录监控异常告警状态,并在此前告警过时推送恢复消息。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 恢复内容
|
||||
"""
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key not in self._alerted_paths:
|
||||
return
|
||||
self._alerted_paths.discard(key)
|
||||
logger.info(message)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
"""
|
||||
轮询监控:执行一轮快照并按结果动态调整监控间隔。
|
||||
"""
|
||||
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
|
||||
if file_count is None or not self._scheduler:
|
||||
return
|
||||
# 动态调整监控间隔
|
||||
new_interval = SnapshotStore.adjust_interval(file_count)
|
||||
try:
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}")
|
||||
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
|
||||
self._scheduler.modify_job(
|
||||
f"monitor_{storage}",
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
logger.info(f"{storage} 监控间隔已调整为 {new_interval} 分钟")
|
||||
except Exception as e:
|
||||
logger.error(f"调整监控间隔失败: {storage} - {e}")
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
"""
|
||||
处理文件变化。
|
||||
:param event: 事件
|
||||
:param text: 事件描述
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if event.is_directory:
|
||||
return
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
# 整理文件
|
||||
self._dispatcher.handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
退出监控
|
||||
"""
|
||||
# 先停定时服务,避免健康检查在停止过程中重建监控线程
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
with self._watcher_lock:
|
||||
watchers = self._watchers
|
||||
self._watchers = []
|
||||
self._pending_locals = []
|
||||
self._alerted_paths = set()
|
||||
self._restart_marks = {}
|
||||
self._stable_cycles = {}
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
logger.info("本地目录监控服务已停止")
|
||||
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
|
||||
@@ -0,0 +1,228 @@
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from app.chain.storage import StorageChain
|
||||
from app.log import logger
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.monitor.snapshot import SnapshotStore
|
||||
|
||||
|
||||
class RemotePoller:
|
||||
"""
|
||||
远程目录轮询监控:快照、比对并分发变化文件。
|
||||
"""
|
||||
# 同一存储连续异常达到该次数后推送告警
|
||||
FAILURE_ALERT_THRESHOLD = 3
|
||||
|
||||
def __init__(self, store: SnapshotStore, dispatcher: TransferDispatcher,
|
||||
alert_cb: Optional[Callable[[str, str], None]] = None):
|
||||
"""
|
||||
初始化远程轮询监控。
|
||||
:param store: 快照存储
|
||||
:param dispatcher: 整理分发器
|
||||
:param alert_cb: 告警回调 (storage, message)
|
||||
"""
|
||||
self._store = store
|
||||
self._dispatcher = dispatcher
|
||||
self._alert_cb = alert_cb
|
||||
# 快照锁按存储隔离,避免一个慢存储阻塞其他存储的轮询
|
||||
self._locks: Dict[str, Lock] = {}
|
||||
self._locks_guard = Lock()
|
||||
# 各存储连续异常次数
|
||||
self._failure_counts: Dict[str, int] = {}
|
||||
|
||||
def _get_lock(self, storage: str) -> Lock:
|
||||
"""
|
||||
获取指定存储的快照锁。
|
||||
:param storage: 存储名称
|
||||
:return: 快照锁
|
||||
"""
|
||||
with self._locks_guard:
|
||||
return self._locks.setdefault(storage, Lock())
|
||||
|
||||
def _note_failure(self, storage: str, reason: str):
|
||||
"""
|
||||
记录一次轮询异常,连续异常达到阈值时推送告警。
|
||||
:param storage: 存储名称
|
||||
:param reason: 异常原因
|
||||
"""
|
||||
count = self._failure_counts.get(storage, 0) + 1
|
||||
self._failure_counts[storage] = count
|
||||
logger.warn(f"远程目录监控异常(连续第 {count} 次): {storage} - {reason}")
|
||||
if count == self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
|
||||
self._alert_cb(storage,
|
||||
f"远程目录监控连续 {count} 次异常: {storage}\n原因: {reason}\n将继续按周期重试")
|
||||
|
||||
def _note_success(self, storage: str):
|
||||
"""
|
||||
记录一次轮询成功,此前告警过时推送恢复消息。
|
||||
:param storage: 存储名称
|
||||
"""
|
||||
if self._failure_counts.get(storage, 0) >= self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
|
||||
self._alert_cb(storage, f"远程目录监控已恢复: {storage}")
|
||||
self._failure_counts[storage] = 0
|
||||
|
||||
def poll(self, storage: str, mon_paths: List[Path]) -> Optional[int]:
|
||||
"""
|
||||
执行一轮轮询监控。
|
||||
:param storage: 存储名称
|
||||
:param mon_paths: 监控路径列表
|
||||
:return: 基线文件数量,本轮无有效结果时返回 None
|
||||
"""
|
||||
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
|
||||
with self._get_lock(storage):
|
||||
try:
|
||||
# 加载上次快照数据,读取失败不能当作首次快照,否则会丢弃已有基线
|
||||
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
||||
if not load_ok:
|
||||
self._note_failure(storage, "读取快照基线失败,跳过本轮")
|
||||
return None
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
|
||||
is_first_snapshot = old_snapshot_data is None
|
||||
|
||||
new_snapshot = {}
|
||||
failed_paths = []
|
||||
for mon_path in mon_paths:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
# 生成新快照(增量模式)
|
||||
snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
|
||||
if snapshot is None:
|
||||
failed_paths.append(str(mon_path))
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
continue
|
||||
new_snapshot.update(snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {len(snapshot)} 个文件")
|
||||
|
||||
if failed_paths and (is_first_snapshot or len(failed_paths) == len(mon_paths)):
|
||||
# 首次基线必须完整建立;全部路径失败时本轮没有有效数据,均不落盘
|
||||
self._note_failure(storage, f"快照失败: {','.join(failed_paths)}")
|
||||
return None
|
||||
|
||||
# 增量快照只包含变化子树,必须与基线合并才是完整视图;
|
||||
# 直接把增量当基线会导致下一轮把未扫到的旧文件全部误判为新增
|
||||
merged_snapshot = {**old_snapshot, **new_snapshot}
|
||||
file_count = len(merged_snapshot)
|
||||
|
||||
if not is_first_snapshot:
|
||||
self._handle_changes(storage, old_snapshot, new_snapshot)
|
||||
else:
|
||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||
|
||||
# 保存合并后的基线
|
||||
if not self._store.save(storage, merged_snapshot, file_count, last_snapshot_time):
|
||||
self._note_failure(storage, "保存快照基线失败")
|
||||
return None
|
||||
|
||||
if failed_paths:
|
||||
# 部分路径失败:成功路径已合并,失败路径保留旧基线,下轮重试
|
||||
self._note_failure(storage, f"部分路径快照失败: {','.join(failed_paths)}")
|
||||
else:
|
||||
self._note_success(storage)
|
||||
return file_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}\n{traceback.format_exc()}")
|
||||
self._note_failure(storage, str(e))
|
||||
return None
|
||||
|
||||
def _handle_changes(self, storage: str, old_snapshot: dict, new_snapshot: dict):
|
||||
"""
|
||||
比对快照并把变化文件送入整理链。
|
||||
:param storage: 存储名称
|
||||
:param old_snapshot: 旧基线
|
||||
:param new_snapshot: 本轮增量快照
|
||||
"""
|
||||
changes = SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
added_files = [
|
||||
file_path
|
||||
for file_path in changes['added']
|
||||
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
modified_files = [
|
||||
file_path
|
||||
for file_path in changes['modified']
|
||||
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
|
||||
# 处理新增文件
|
||||
handled_added_count = 0
|
||||
for new_file in added_files:
|
||||
file_info = new_snapshot.get(new_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
|
||||
handled_added_count += 1
|
||||
|
||||
# 处理修改文件
|
||||
handled_modified_count = 0
|
||||
for modified_file in modified_files:
|
||||
file_info = new_snapshot.get(modified_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
|
||||
handled_modified_count += 1
|
||||
|
||||
if handled_added_count or handled_modified_count:
|
||||
logger.info(f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
|
||||
else:
|
||||
logger.debug(f"{storage} 无文件变化")
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
:param storage: 存储名称
|
||||
:param mon_path: 监控路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
|
||||
|
||||
# 生成快照
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=0 # 全量扫描,不使用增量
|
||||
)
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return False
|
||||
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
|
||||
|
||||
# 处理所有文件
|
||||
processed_count = 0
|
||||
for file_path, file_info in new_snapshot.items():
|
||||
try:
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
|
||||
continue
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(file_path),
|
||||
file_size=file_size):
|
||||
processed_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||
|
||||
# 全量扫描覆盖单个路径,与已有基线合并后落盘,避免覆盖其他监控路径的基线
|
||||
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if (load_ok and old_snapshot_data) else {}
|
||||
merged_snapshot = {**old_snapshot, **new_snapshot}
|
||||
self._store.save(storage, merged_snapshot, len(merged_snapshot))
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
|
||||
return False
|
||||
@@ -0,0 +1,148 @@
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.cache import FileCache
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class SnapshotStore:
|
||||
"""
|
||||
远程目录监控快照的存取与比对。
|
||||
"""
|
||||
|
||||
def __init__(self, cache: Optional[FileCache] = None):
|
||||
"""
|
||||
初始化快照存储。
|
||||
:param cache: 快照文件缓存,默认使用 CACHE_PATH/snapshots
|
||||
"""
|
||||
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
|
||||
|
||||
def save(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None) -> bool:
|
||||
"""
|
||||
保存快照到文件缓存。
|
||||
:param storage: 存储名称
|
||||
:param snapshot: 快照数据
|
||||
:param file_count: 文件数量,用于调整监控间隔
|
||||
:param last_snapshot_time: 上次快照时间戳
|
||||
:return: 是否保存成功
|
||||
"""
|
||||
try:
|
||||
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
|
||||
if snapshot_time is None:
|
||||
snapshot_time = last_snapshot_time or time.time()
|
||||
snapshot_data = {
|
||||
'timestamp': snapshot_time,
|
||||
'file_count': file_count,
|
||||
'snapshot': snapshot
|
||||
}
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
|
||||
self._cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
|
||||
logger.debug(f"快照已保存到缓存: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存快照失败: {e}")
|
||||
return False
|
||||
|
||||
def load_checked(self, storage: str) -> Tuple[Optional[Dict], bool]:
|
||||
"""
|
||||
从文件缓存加载快照,并区分「快照不存在」与「读取失败」。
|
||||
读取失败时不能当作首次快照处理,否则会静默丢弃已有基线。
|
||||
:param storage: 存储名称
|
||||
:return: (快照数据或None, 是否读取成功)
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_data = self._cache.get(cache_key, region="snapshots")
|
||||
if snapshot_data:
|
||||
data = json.loads(snapshot_data.decode('utf-8'))
|
||||
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
|
||||
return data, True
|
||||
logger.debug(f"快照文件不存在: {storage}")
|
||||
return None, True
|
||||
except Exception as e:
|
||||
logger.error(f"加载快照失败: {e}")
|
||||
return None, False
|
||||
|
||||
def load(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照。
|
||||
:param storage: 存储名称
|
||||
:return: 快照数据或None
|
||||
"""
|
||||
data, _ = self.load_checked(storage)
|
||||
return data
|
||||
|
||||
def reset(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准。
|
||||
:param storage: 存储名称
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
if self._cache.exists(cache_key, region="snapshots"):
|
||||
self._cache.delete(cache_key, region="snapshots")
|
||||
logger.info(f"快照已重置: {storage}")
|
||||
return True
|
||||
logger.debug(f"快照文件不存在,无需重置: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"重置快照失败: {storage} - {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def compare(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件(只处理新增和修改,不处理删除)。
|
||||
:param old_snapshot: 旧快照
|
||||
:param new_snapshot: 新快照
|
||||
:return: 变化信息
|
||||
"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_files = set(old_snapshot.keys())
|
||||
new_files = set(new_snapshot.keys())
|
||||
|
||||
# 新增文件
|
||||
changes['added'] = list(new_files - old_files)
|
||||
|
||||
# 修改文件(大小或时间变化)
|
||||
for file_path in old_files & new_files:
|
||||
old_info = old_snapshot[file_path]
|
||||
new_info = new_snapshot[file_path]
|
||||
|
||||
# 检查文件大小变化
|
||||
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
|
||||
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
|
||||
|
||||
# 检查修改时间变化(如果有的话)
|
||||
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
||||
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
||||
|
||||
if old_size != new_size or (old_time and new_time and old_time != new_time):
|
||||
changes['modified'].append(file_path)
|
||||
|
||||
return changes
|
||||
|
||||
@staticmethod
|
||||
def adjust_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔。
|
||||
:param file_count: 文件数量
|
||||
:return: 监控间隔(分钟)
|
||||
"""
|
||||
if file_count < 100:
|
||||
return 5 # 5分钟
|
||||
elif file_count < 500:
|
||||
return 10 # 10分钟
|
||||
elif file_count < 1000:
|
||||
return 15 # 15分钟
|
||||
else:
|
||||
return 30 # 30分钟
|
||||
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.log import logger
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
def count_directory_entries(directory: Path, max_check: int = 10000) -> Tuple[int, int]:
|
||||
"""
|
||||
统计目录下的文件与子目录数量(用于检测是否超过系统限制)。
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查文件数量,避免长时间阻塞
|
||||
:return: (文件数量, 目录数量)
|
||||
"""
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
try:
|
||||
for _, dirs, files in os.walk(str(directory)):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > max_check:
|
||||
break
|
||||
except Exception as err:
|
||||
logger.debug(f"统计目录规模失败: {err}")
|
||||
return file_count, dir_count
|
||||
|
||||
|
||||
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
|
||||
"""
|
||||
统计目录下的文件数量。
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查数量,避免长时间阻塞
|
||||
:return: 文件数量
|
||||
"""
|
||||
file_count, _ = count_directory_entries(directory, max_check=max_check)
|
||||
return file_count
|
||||
|
||||
|
||||
def check_system_limits() -> Dict[str, Any]:
|
||||
"""
|
||||
检查系统监控相关限制。
|
||||
:return: 系统限制信息
|
||||
"""
|
||||
limits = {
|
||||
'max_user_watches': 0,
|
||||
'max_user_instances': 0,
|
||||
'warnings': []
|
||||
}
|
||||
|
||||
try:
|
||||
if platform.system() == 'Linux':
|
||||
# 检查 inotify 限制
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_watches'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 限制失败: {e}")
|
||||
limits['max_user_watches'] = 8192 # 默认值
|
||||
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_instances'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 实例限制失败: {e}")
|
||||
except Exception as e:
|
||||
limits['warnings'].append(f"检查系统限制时出错: {e}")
|
||||
|
||||
return limits
|
||||
|
||||
|
||||
def get_system_optimization_tips() -> List[str]:
|
||||
"""
|
||||
获取系统优化建议。
|
||||
:return: 优化建议列表
|
||||
"""
|
||||
tips = []
|
||||
system = platform.system()
|
||||
|
||||
if system == 'Linux':
|
||||
tips.extend([
|
||||
"增加 inotify 监控数量限制:",
|
||||
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"sudo sysctl -p",
|
||||
"",
|
||||
"如果在Docker中运行,请在宿主机上执行以上命令"
|
||||
])
|
||||
elif system == 'Darwin':
|
||||
tips.extend([
|
||||
"macOS 系统优化建议:",
|
||||
"sudo sysctl kern.maxfiles=65536",
|
||||
"sudo sysctl kern.maxfilesperproc=32768",
|
||||
"ulimit -n 32768"
|
||||
])
|
||||
elif system == 'Windows':
|
||||
tips.extend([
|
||||
"Windows 系统优化建议:",
|
||||
"1. 关闭不必要的实时保护软件对监控目录的扫描",
|
||||
"2. 将监控目录添加到Windows Defender排除列表",
|
||||
"3. 确保有足够的可用内存"
|
||||
])
|
||||
|
||||
return tips
|
||||
|
||||
|
||||
def decide_monitor_mode(directory: Path,
|
||||
monitor_mode: str) -> Tuple[bool, str, Optional[Dict[str, Any]], Optional[int]]:
|
||||
"""
|
||||
决策监控模式。兼容模式与网络文件系统直接短路,只有快速模式候选才统计
|
||||
目录规模与系统限制,避免启动期对网络挂载做无谓的全量遍历。
|
||||
|
||||
inotify 的 max_user_watches 按监视点(目录)计数,因此用目录数量而不是
|
||||
文件数量与上限比较。
|
||||
|
||||
:param directory: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: (是否使用轮询, 原因, 系统限制信息或None, 文件数量或None)
|
||||
"""
|
||||
if monitor_mode == "compatibility":
|
||||
return True, "用户配置为兼容模式", None, None
|
||||
|
||||
# 检查网络文件系统
|
||||
if SystemUtils.is_network_filesystem(directory):
|
||||
return True, "检测到网络文件系统,建议使用兼容模式", None, None
|
||||
|
||||
limits = check_system_limits()
|
||||
file_count, dir_count = count_directory_entries(directory)
|
||||
max_watches = limits.get('max_user_watches')
|
||||
if max_watches and dir_count > max_watches * 0.8:
|
||||
return (True, f"目录数量({dir_count})接近 inotify 监控上限({max_watches})",
|
||||
limits, file_count)
|
||||
return False, "使用快速模式", limits, file_count
|
||||
@@ -0,0 +1,303 @@
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from watchfiles import Change, DefaultFilter, watch
|
||||
|
||||
from app.log import logger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryChangeEvent:
|
||||
"""
|
||||
目录文件变化事件,隔离底层 watchfiles 事件结构。
|
||||
"""
|
||||
change_type: Change
|
||||
src_path: str
|
||||
is_directory: bool
|
||||
|
||||
|
||||
class LocalDirectoryWatcher:
|
||||
"""
|
||||
基于 watchfiles 的本地目录监控线程。
|
||||
"""
|
||||
_HANDLE_CHANGES = {Change.added, Change.modified}
|
||||
# 监控循环异常退出后的重启退避秒数,网络存储/FUSE 挂载抖动通常是暂时的
|
||||
RESTART_BACKOFF = (5, 15, 30, 60, 120, 300)
|
||||
# 单次监控循环存活超过该秒数视为已恢复,重置退避
|
||||
HEALTHY_UPTIME = 60
|
||||
# 超过该秒数监控循环没有任何活动,判定为静默失效
|
||||
STALL_TIMEOUT = 600
|
||||
# 轮询模式目录扫描间隔(毫秒):本地磁盘用 watchfiles 默认值
|
||||
POLL_DELAY_LOCAL_MS = 300
|
||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||
POLL_DELAY_NETWORK_MS = 5000
|
||||
|
||||
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
|
||||
poll_delay_ms: Optional[int] = None):
|
||||
"""
|
||||
初始化本地目录监控。
|
||||
:param mon_path: 监控目录
|
||||
:param callback: 目录变化回调对象
|
||||
:param force_polling: 是否强制使用轮询模式,None 表示由 watchfiles 自动选择
|
||||
:param poll_delay_ms: 轮询模式目录扫描间隔(毫秒),仅轮询时生效
|
||||
"""
|
||||
self._watch_path = mon_path
|
||||
self._callback = callback
|
||||
self._force_polling = force_polling
|
||||
self._poll_delay_ms = poll_delay_ms or self.POLL_DELAY_LOCAL_MS
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._watch_filter = DefaultFilter()
|
||||
# 最近一次监控循环活动时间(monotonic),用于检测静默失效
|
||||
self._last_activity: float = 0.0
|
||||
# 累计自动重启次数
|
||||
self._restart_count: int = 0
|
||||
|
||||
@property
|
||||
def watch_path(self) -> Path:
|
||||
"""
|
||||
获取监控目录。
|
||||
:return: 监控目录
|
||||
"""
|
||||
return self._watch_path
|
||||
|
||||
@property
|
||||
def force_polling(self) -> Optional[bool]:
|
||||
"""
|
||||
获取监控模式配置,重建监控线程时沿用。
|
||||
:return: 是否强制轮询
|
||||
"""
|
||||
return self._force_polling
|
||||
|
||||
@property
|
||||
def restart_count(self) -> int:
|
||||
"""
|
||||
获取累计自动重启次数。
|
||||
:return: 自动重启次数
|
||||
"""
|
||||
return self._restart_count
|
||||
|
||||
@property
|
||||
def poll_delay_ms(self) -> int:
|
||||
"""
|
||||
获取轮询模式目录扫描间隔(毫秒),重建监控线程时沿用。
|
||||
:return: 扫描间隔
|
||||
"""
|
||||
return self._poll_delay_ms
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
启动本地目录监控线程。
|
||||
"""
|
||||
if not self._watch_path.exists():
|
||||
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
|
||||
if not self._watch_path.is_dir():
|
||||
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
|
||||
if self.is_alive():
|
||||
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._mark_activity()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
请求停止本地目录监控线程。
|
||||
"""
|
||||
self._stop_event.set()
|
||||
|
||||
def join(self, timeout: Optional[float] = None):
|
||||
"""
|
||||
等待本地目录监控线程退出。
|
||||
:param timeout: 最长等待秒数
|
||||
"""
|
||||
if self._thread:
|
||||
self._thread.join(timeout=timeout)
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否仍在运行。
|
||||
:return: 线程存活状态
|
||||
"""
|
||||
return bool(self._thread and self._thread.is_alive())
|
||||
|
||||
def is_stalled(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否已静默失效(线程存活但监控循环长时间无任何活动)。
|
||||
:return: 是否静默失效
|
||||
"""
|
||||
if self._stop_event.is_set() or not self.is_alive():
|
||||
return False
|
||||
if not self._last_activity:
|
||||
return False
|
||||
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
|
||||
|
||||
def _mark_activity(self):
|
||||
"""
|
||||
记录一次监控循环活动时间,作为静默失效检测的心跳。
|
||||
"""
|
||||
self._last_activity = time.monotonic()
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
运行 watchfiles 主循环,异常时退避重启,避免一次故障导致监控永久停摆。
|
||||
"""
|
||||
# 快速模式失败后降级为轮询,降级后的失败一律走退避重启
|
||||
force_polling = self._force_polling
|
||||
attempt = 0
|
||||
while not self._stop_event.is_set():
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
self._mark_activity()
|
||||
self._run_watch(force_polling=force_polling)
|
||||
# 正常返回表示收到停止信号
|
||||
return
|
||||
except Exception as err:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
# 崩溃堆栈按 ERROR 级输出,生产环境 LOG_LEVEL=ERROR 时也能落盘
|
||||
logger.error(f"本地目录监控异常堆栈: {self._watch_path}\n{traceback.format_exc()}")
|
||||
if force_polling is not True:
|
||||
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
|
||||
force_polling = True
|
||||
continue
|
||||
if time.monotonic() - started_at >= self.HEALTHY_UPTIME:
|
||||
# 上一轮监控已稳定运行过,重新从最短间隔开始退避
|
||||
attempt = 0
|
||||
delay = self.RESTART_BACKOFF[min(attempt, len(self.RESTART_BACKOFF) - 1)]
|
||||
attempt += 1
|
||||
self._restart_count += 1
|
||||
logger.error(f"本地目录监控发生错误,{delay} 秒后自动重启"
|
||||
f"(累计第 {self._restart_count} 次): {self._watch_path} - {err}")
|
||||
if self._stop_event.wait(timeout=delay):
|
||||
return
|
||||
|
||||
def _run_watch(self, force_polling: Optional[bool]):
|
||||
"""
|
||||
执行一次 watchfiles 监控循环。
|
||||
:param force_polling: 是否强制轮询
|
||||
"""
|
||||
for changes in watch(
|
||||
str(self._watch_path),
|
||||
watch_filter=self._watch_filter,
|
||||
stop_event=self._stop_event,
|
||||
rust_timeout=1000,
|
||||
yield_on_timeout=True,
|
||||
force_polling=force_polling,
|
||||
poll_delay_ms=self._poll_delay_ms,
|
||||
recursive=True,
|
||||
ignore_permission_denied=True):
|
||||
self._mark_activity()
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
if not changes:
|
||||
continue
|
||||
self._handle_changes(changes)
|
||||
self._mark_activity()
|
||||
|
||||
def _handle_changes(self, changes: set[tuple[Change, str]]):
|
||||
"""
|
||||
将 watchfiles 原始变更转换为目录监控事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
"""
|
||||
changes = self._expand_added_directories(changes)
|
||||
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
||||
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
|
||||
self._mark_activity()
|
||||
if change_type not in self._HANDLE_CHANGES:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
event = self._build_event(change_type=change_type, event_path=event_path)
|
||||
if not event or event.is_directory:
|
||||
continue
|
||||
file_size = self._get_file_size(event_path)
|
||||
if file_size is None:
|
||||
continue
|
||||
text = self._change_text(change_type)
|
||||
try:
|
||||
self._callback.event_handler(
|
||||
event=event,
|
||||
text=text,
|
||||
event_path=path_str,
|
||||
file_size=file_size
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
||||
|
||||
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
||||
"""
|
||||
将整体移入监控范围的新增目录展开为内部文件事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
:return: 包含目录内新增文件的变更集合
|
||||
"""
|
||||
expanded_changes = set(changes)
|
||||
for change_type, path_str in changes:
|
||||
if change_type != Change.added:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
try:
|
||||
if not event_path.is_dir():
|
||||
continue
|
||||
for nested_path in event_path.rglob("*"):
|
||||
if not nested_path.is_file():
|
||||
continue
|
||||
nested_path_str = nested_path.as_posix()
|
||||
if self._watch_filter(Change.added, nested_path_str):
|
||||
expanded_changes.add((Change.added, nested_path_str))
|
||||
except OSError as err:
|
||||
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
|
||||
return expanded_changes
|
||||
|
||||
@staticmethod
|
||||
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
||||
"""
|
||||
构建目录变化事件,路径已不存在时忽略。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:param event_path: 变化路径
|
||||
:return: 目录变化事件
|
||||
"""
|
||||
try:
|
||||
is_directory = event_path.is_dir()
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
|
||||
return None
|
||||
if not event_path.exists():
|
||||
return None
|
||||
return DirectoryChangeEvent(
|
||||
change_type=change_type,
|
||||
src_path=event_path.as_posix(),
|
||||
is_directory=is_directory
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_file_size(event_path: Path) -> Optional[int]:
|
||||
"""
|
||||
读取事件文件大小,文件已消失时返回 None。
|
||||
:param event_path: 事件文件路径
|
||||
:return: 文件大小
|
||||
"""
|
||||
try:
|
||||
return event_path.stat().st_size
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _change_text(change_type: Change) -> str:
|
||||
"""
|
||||
转换 watchfiles 事件类型为日志文案。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:return: 事件描述
|
||||
"""
|
||||
if change_type == Change.modified:
|
||||
return "修改"
|
||||
return "新增"
|
||||
@@ -94,6 +94,7 @@ class AgentLLMProviderEventData(ChainEventData):
|
||||
user_agent: Optional[str] = Field(default=None, description="OpenAI兼容接口User-Agent")
|
||||
use_proxy: Optional[bool] = Field(default=None, description="是否使用系统代理")
|
||||
thinking_level: Optional[str] = Field(default=None, description="思考模式级别")
|
||||
api_protocol: Optional[str] = Field(default=None, description="OpenAI兼容接口API协议:auto/chat_completions/responses")
|
||||
selected_provider_id: Optional[str] = Field(default=None, description="插件侧供应商ID")
|
||||
selected_provider_name: Optional[str] = Field(default=None, description="插件侧供应商名称")
|
||||
source: Optional[str] = Field(default=None, description="选择来源")
|
||||
|
||||
@@ -36,3 +36,12 @@ class OperationInterrupted(KeyboardInterrupt):
|
||||
用于表示操作被中断
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class StorageQueryError(Exception):
|
||||
"""
|
||||
用于表示存储查询无法确认结果的异常类。
|
||||
当文件信息查询因网络、限流或接口错误失败(区别于「确认不存在」)时抛出,
|
||||
调用方不应把该状态当作文件不存在处理。
|
||||
"""
|
||||
pass
|
||||
|
||||
+16
-2
@@ -1,9 +1,13 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field
|
||||
from app.schemas.types import StorageSchema
|
||||
|
||||
# Windows 盘符绝对路径,如 Z:/Downloads 或 Z:\Downloads
|
||||
WINDOWS_DRIVE_PATTERN = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
class FileURI(BaseModel):
|
||||
# 文件路径
|
||||
@@ -13,10 +17,19 @@ class FileURI(BaseModel):
|
||||
|
||||
@property
|
||||
def uri(self) -> str:
|
||||
"""
|
||||
文件 URI,本地存储直接返回路径,其他存储带上存储前缀
|
||||
"""
|
||||
return self.path if self.storage == "local" else f"{self.storage}:{self.path}"
|
||||
|
||||
@classmethod
|
||||
def from_uri(cls, uri: str) -> "FileURI":
|
||||
"""
|
||||
解析文件 URI 为存储类型和路径
|
||||
|
||||
:param uri: 文件 URI,如 /media/movie、u115:/media/movie 或 Windows 盘符路径 Z:/media
|
||||
:return: FileURI 对象
|
||||
"""
|
||||
storage, path = 'local', uri
|
||||
for s in StorageSchema:
|
||||
protocol = f"{s.value}:"
|
||||
@@ -24,11 +37,13 @@ class FileURI(BaseModel):
|
||||
path = uri[len(protocol):]
|
||||
storage = s.value
|
||||
break
|
||||
if not path.startswith("/"):
|
||||
# Windows 盘符路径本身就是绝对路径,补上根斜杠会得到 /Z:/xxx 这样的非法路径
|
||||
if not path.startswith("/") and not WINDOWS_DRIVE_PATTERN.match(path):
|
||||
path = "/" + path
|
||||
path = Path(path).as_posix()
|
||||
return cls(storage=storage, path=path)
|
||||
|
||||
|
||||
class FileItem(FileURI):
|
||||
# 类型 dir/file
|
||||
type: Optional[str] = None
|
||||
@@ -68,4 +83,3 @@ class StorageUsage(BaseModel):
|
||||
class StorageTransType(BaseModel):
|
||||
# 传输类型
|
||||
transtype: Optional[dict] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@@ -95,6 +95,26 @@ class PluginSidebarNavItem(BaseModel):
|
||||
order: int = Field(default=0, description="同组内排序,越小越靠前")
|
||||
|
||||
|
||||
class PluginRatingRequest(BaseModel):
|
||||
"""插件评分请求"""
|
||||
|
||||
rating: float = Field(
|
||||
ge=0.1,
|
||||
le=5.0,
|
||||
multiple_of=0.1,
|
||||
description="评分,范围 0.1 至 5.0,精确到 0.1",
|
||||
)
|
||||
|
||||
|
||||
class PluginRating(BaseModel):
|
||||
"""插件评分结果"""
|
||||
|
||||
plugin_id: str = Field(description="插件 ID")
|
||||
average_rating: float = Field(default=0.0, description="平均评分")
|
||||
rating_count: int = Field(default=0, description="评分人数")
|
||||
user_rating: Optional[float] = Field(default=None, description="当前安装实例评分")
|
||||
|
||||
|
||||
class PluginMemoryInfo(BaseModel):
|
||||
"""插件内存信息"""
|
||||
plugin_id: str = Field(description="插件ID")
|
||||
|
||||
+21
-1
@@ -1,7 +1,7 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +40,26 @@ class MediaServerConf(BaseModel):
|
||||
# 自动同步间隔(小时),未设置时使用旧全局配置
|
||||
sync_interval: Optional[int] = None
|
||||
|
||||
@field_validator("sync_interval", mode="before")
|
||||
@classmethod
|
||||
def validate_sync_interval(cls, value: Any) -> Optional[int]:
|
||||
"""
|
||||
兼容前端清空输入框后残留的空字符串等非法值,避免历史配置导致模块初始化失败
|
||||
|
||||
:param value: 原始配置值
|
||||
:return: 合法的间隔小时数,无法解析时返回 None
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class DownloaderConf(BaseModel):
|
||||
"""
|
||||
|
||||
@@ -215,6 +215,8 @@ class SystemConfigKey(Enum):
|
||||
NotificationSwitchs = "NotificationSwitchs"
|
||||
# 目录配置
|
||||
Directories = "Directories"
|
||||
# 挂载型本地盘是否删除空目录
|
||||
MountedLocalDiskDeleteEmptyDirs = "MountedLocalDiskDeleteEmptyDirs"
|
||||
# 存储配置
|
||||
Storages = "Storages"
|
||||
# 搜索站点范围
|
||||
@@ -279,6 +281,8 @@ class SystemConfigKey(Enum):
|
||||
SetupWizardState = "SetupWizardState"
|
||||
# 绿联影视登录会话缓存
|
||||
UgreenSessionCache = "UgreenSessionCache"
|
||||
# 共享媒体识别成功次数
|
||||
MediaRecognizeShareCount = "MediaRecognizeShareCount"
|
||||
|
||||
|
||||
# 处理进度Key字典
|
||||
|
||||
+86
-3
@@ -3,10 +3,11 @@ import collections
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from contextlib import AsyncExitStack, contextmanager, asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
|
||||
import chardet
|
||||
import httpx
|
||||
@@ -72,6 +73,53 @@ _DEFAULT_MAX_CONNECTIONS = 40
|
||||
_DEFAULT_KEEPALIVE_EXPIRY = 30
|
||||
# 同步 requests.Session 复用连接时,遇到对端或代理关闭 keep-alive 后允许重试的方法
|
||||
_REQUESTS_RETRY_IDEMPOTENT_METHODS = ("GET", "HEAD", "OPTIONS")
|
||||
|
||||
# 代理走 CONNECT 隧道时,httpx 默认开启的 HTTP/2 多路复用会把并发请求叠加到极少数隧道上;
|
||||
# 隧道被代理节点切换或空闲回收打断后,复用其上的所有请求会同时失败。按 (proxy, host) 熔断:
|
||||
# 命中一次连接层失败就记录下次允许再尝试 h2 的时间戳(time.monotonic 基准),冷却期内该
|
||||
# (proxy, host) 的请求直接退化为 http1.1;冷却期结束后自动恢复尝试 h2。
|
||||
_H2_PROXY_BREAKER_COOLDOWN = 1800 # 30 分钟
|
||||
_h2_proxy_breaker_lock = threading.Lock()
|
||||
_h2_proxy_retry_at: Dict[Tuple[str, str], float] = {}
|
||||
# 只有这些错误是"h2 隧道被打断"的特征(对应实测日志里的 SEND_HEADERS in CLOSED、
|
||||
# EndOfStream 等);超时、连接失败、代理不可达等错误换 h1 一样会发生,
|
||||
# 不应触发熔断,也不值得付出一次注定同样失败的 h1 重试
|
||||
_H2_TUNNEL_BREAK_ERRORS = (
|
||||
httpx.RemoteProtocolError,
|
||||
httpx.LocalProtocolError,
|
||||
httpx.ReadError,
|
||||
httpx.WriteError,
|
||||
httpx.CloseError,
|
||||
)
|
||||
|
||||
|
||||
def _h2_proxy_breaker_key(proxy: str, url: str) -> Tuple[str, str]:
|
||||
try:
|
||||
host = httpx.URL(url).host or ""
|
||||
except Exception:
|
||||
host = url
|
||||
return proxy, host
|
||||
|
||||
|
||||
def _h2_proxy_allowed(proxy: Optional[str], url: str) -> bool:
|
||||
"""判断给定代理 + 目标 host 当前是否允许尝试 h2(未处于熔断冷却期)"""
|
||||
if not proxy:
|
||||
return True
|
||||
with _h2_proxy_breaker_lock:
|
||||
retry_at = _h2_proxy_retry_at.get(_h2_proxy_breaker_key(proxy, url), 0.0)
|
||||
return time.monotonic() >= retry_at
|
||||
|
||||
|
||||
def _trip_h2_proxy_breaker(proxy: str, url: str) -> None:
|
||||
"""记录一次 h2 连接层失败,熔断该 (proxy, host) 冷却期内的 h2 尝试"""
|
||||
now = time.monotonic()
|
||||
with _h2_proxy_breaker_lock:
|
||||
# 顺手清掉已过冷却期的条目,防止长期运行下字典无限增长
|
||||
for key in [k for k, retry_at in _h2_proxy_retry_at.items() if now >= retry_at]:
|
||||
del _h2_proxy_retry_at[key]
|
||||
_h2_proxy_retry_at[_h2_proxy_breaker_key(proxy, url)] = now + _H2_PROXY_BREAKER_COOLDOWN
|
||||
|
||||
|
||||
# 持有 LRU 淘汰后正在异步关闭的 transport task,避免 fire-and-forget 被 GC 警告
|
||||
_pending_eviction_tasks: set[asyncio.Task] = set()
|
||||
|
||||
@@ -1087,13 +1135,48 @@ class AsyncRequestUtils:
|
||||
self._client, method, url, raise_exception, **kwargs
|
||||
)
|
||||
|
||||
# 代理走 CONNECT 隧道时 h2 多路复用容易被隧道打断放大成批量失败(见
|
||||
# _h2_proxy_allowed 处注释);仅对幂等方法做"h2 失败就地降级 h1 重试",
|
||||
# 避免非幂等请求在服务端可能已收到数据的情况下重复产生副作用
|
||||
http2 = self._http2 and _h2_proxy_allowed(self._proxies, url)
|
||||
if not (http2 and self._proxies and method.upper() in _REQUESTS_RETRY_IDEMPOTENT_METHODS):
|
||||
return await self._dispatch_request(
|
||||
http2, cookies_dict, method, url, raise_exception, **kwargs
|
||||
)
|
||||
|
||||
try:
|
||||
return await self._dispatch_request(
|
||||
True, cookies_dict, method, url, True, **kwargs
|
||||
)
|
||||
except _H2_TUNNEL_BREAK_ERRORS as e:
|
||||
logger.debug(f"h2 代理连接层失败,熔断 {url} 所在 host 并降级 h1 重试: {e!r}")
|
||||
_trip_h2_proxy_breaker(self._proxies, url)
|
||||
return await self._dispatch_request(
|
||||
False, cookies_dict, method, url, raise_exception, **kwargs
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
# 与 h2 隧道无关的失败(超时、连接失败等):不熔断也不重试,
|
||||
# 恢复调用方原本的 raise_exception 语义
|
||||
if raise_exception:
|
||||
raise
|
||||
error_msg = str(e) or f"未知网络错误 (URL: {url}, Method: {method.upper()})"
|
||||
logger.debug(f"异步请求失败: {error_msg}")
|
||||
return None
|
||||
|
||||
async def _dispatch_request(
|
||||
self, http2: bool, cookies_dict: Optional[dict], method: str, url: str,
|
||||
raise_exception: bool, **kwargs
|
||||
) -> Optional[httpx.Response]:
|
||||
"""
|
||||
按给定 http2 开关构建/复用底层连接并发起请求,供 request() 的 h2/h1 熔断切换复用
|
||||
"""
|
||||
# 共享底层 transport(连接池+TLS 复用),每次请求创建轻量 AsyncClient。
|
||||
# AsyncClient 持有的 cookie jar 仅存活于本次请求 lifecycle,
|
||||
# 既复用握手又彻底避免 jar 跨调用累积。
|
||||
transport = _get_shared_async_transport(
|
||||
proxy=self._proxies,
|
||||
verify=self._verify,
|
||||
http2=self._http2,
|
||||
http2=http2,
|
||||
max_keepalive_connections=self._max_keepalive_connections,
|
||||
max_connections=self._max_connections,
|
||||
keepalive_expiry=self._keepalive_expiry,
|
||||
@@ -1113,7 +1196,7 @@ class AsyncRequestUtils:
|
||||
|
||||
# 兜底:没有运行中的事件循环时,临时客户端走完即关
|
||||
async with httpx.AsyncClient(
|
||||
http2=self._http2,
|
||||
http2=http2,
|
||||
proxy=self._proxies,
|
||||
timeout=self._timeout,
|
||||
verify=self._verify,
|
||||
|
||||
+13
-3
@@ -828,10 +828,13 @@ class SystemUtils:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_network_filesystem(directory: Path) -> bool:
|
||||
def is_network_filesystem(
|
||||
directory: Path, include_local_fuse: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
检测是否为网络文件系统
|
||||
:param directory: 目录路径
|
||||
:param include_local_fuse: 是否将本地 FUSE 挂载视为挂载文件系统
|
||||
:return: 是否为网络文件系统
|
||||
"""
|
||||
try:
|
||||
@@ -849,7 +852,10 @@ class SystemUtils:
|
||||
"fuseblk",
|
||||
# TBD
|
||||
]
|
||||
if any(fs in output for fs in local_fs):
|
||||
if (
|
||||
not include_local_fuse
|
||||
and any(fs in output for fs in local_fs)
|
||||
):
|
||||
return False
|
||||
network_fs = ['nfs', 'cifs', 'smbfs', 'fuse', 'sshfs', 'ftpfs']
|
||||
return any(fs in output for fs in network_fs)
|
||||
@@ -859,7 +865,11 @@ class SystemUtils:
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.lower()
|
||||
return 'nfs' in output or 'smbfs' in output
|
||||
return (
|
||||
'nfs' in output
|
||||
or 'smbfs' in output
|
||||
or (include_local_fuse and 'fuse' in output)
|
||||
)
|
||||
elif system == 'Windows':
|
||||
# Windows 检查网络驱动器
|
||||
return str(directory).startswith('\\\\')
|
||||
|
||||
@@ -481,6 +481,9 @@ moviepilot tool run search_torrents media_type=movie tmdb_id=12345
|
||||
- `tool list` 用于动态发现当前服务可调用的工具
|
||||
- `tool show` 会输出参数名、类型和描述
|
||||
- `tool run` 参数格式固定为 `key=value`
|
||||
- `read_file`、`write_file`、`edit_file` 和 `execute_command`
|
||||
属于内置 Agent 的本地敏感能力,不通过 MCP/`moviepilot tool` 暴露;插件开发时
|
||||
由 Agent 按当前用户权限直接调用这些工具。
|
||||
|
||||
## Scheduler 命令
|
||||
|
||||
|
||||
@@ -55,6 +55,43 @@ pip install -r requirements.txt
|
||||
pip install -r requirements-dev.in
|
||||
```
|
||||
|
||||
### 2.1 本地启动脚本
|
||||
|
||||
不需要打开 IDE 时,可以直接使用仓库内的启动脚本。脚本会自动定位项目根目录和虚拟环境,并以模块方式启动后端,避免 `ModuleNotFoundError: No module named 'app'`。
|
||||
|
||||
```bash
|
||||
# 默认启动后端开发服务,前台运行,按 Ctrl+C 停止
|
||||
./scripts/start-local.sh
|
||||
./scripts/start-local.sh backend
|
||||
|
||||
# 如果已经安装前端发布包,可启动完整的前后端服务
|
||||
./scripts/start-local.sh service start
|
||||
|
||||
# 管理完整服务
|
||||
./scripts/start-local.sh stop
|
||||
./scripts/start-local.sh restart
|
||||
./scripts/start-local.sh status
|
||||
./scripts/start-local.sh logs --follow
|
||||
```
|
||||
|
||||
默认会使用 `DEBUG=true` 和 `DEV=true`,与 IDE 开发启动保持一致;如果不需要热重载,可以这样启动以降低资源占用:
|
||||
|
||||
```bash
|
||||
DEV=false ./scripts/start-local.sh
|
||||
```
|
||||
|
||||
脚本会优先使用 `CONFIG_DIR`,其次使用 `MOVIEPILOT_CONFIG_DIR`,再检测 `~/Documents/moviepilot`,最后回退到仓库内的 `config` 目录。需要使用其他配置目录时,可以这样运行:
|
||||
|
||||
```bash
|
||||
MOVIEPILOT_CONFIG_DIR=/path/to/moviepilot-config ./scripts/start-local.sh
|
||||
```
|
||||
|
||||
首次使用前如果脚本没有执行权限,运行:
|
||||
|
||||
```bash
|
||||
chmod +x scripts/start-local.sh
|
||||
```
|
||||
|
||||
### 3. 修改主程序依赖
|
||||
|
||||
新增或升级依赖时,先确认依赖属于哪个层级:
|
||||
|
||||
+30
-2
@@ -31,6 +31,12 @@ MCP 使用系统配置中的 `API_TOKEN` 作为认证密钥,文档中的 API K
|
||||
- `tools/call`: 调用特定工具。
|
||||
- `ping`: 连接存活检测。
|
||||
|
||||
### 动态插件工具
|
||||
|
||||
`tools/list` 会同时返回 MoviePilot 内置工具和已启用插件通过 `get_agent_tools()` 声明的工具。插件启动、停止、重载或配置生效后,MCP 工具管理器会在下一次列出或调用工具时按注册表版本惰性刷新,避免继续暴露已移除的工具或遗漏新工具。
|
||||
|
||||
MCP 当前不会主动发送工具列表变更通知(`listChanged=false`)。如果客户端缓存了工具列表,插件状态变化后需要让客户端重新请求 `tools/list`;无法手动刷新的客户端应重新连接 MCP 服务或新建会话。
|
||||
|
||||
---
|
||||
|
||||
## 4. 客户端配置示例
|
||||
@@ -197,9 +203,15 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| :--- | :--- | :--- |
|
||||
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存及识别成功、失败条目统计 |
|
||||
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存统计、共享识别累计成功命中次数及开关状态 |
|
||||
| DELETE | `/api/v1/tmdb/cache/{cache_key}` | 按缓存键删除单条 TheMovieDb 识别缓存,缓存键需要进行 URL 编码 |
|
||||
| DELETE | `/api/v1/tmdb/cache` | 清空全部 TheMovieDb 识别缓存 |
|
||||
| GET | `/api/v1/douban/cache` | 查询豆瓣识别缓存统计、共享识别累计成功命中次数及开关状态 |
|
||||
| DELETE | `/api/v1/douban/cache/{cache_key}` | 按缓存键删除单条豆瓣识别缓存,缓存键需要进行 URL 编码 |
|
||||
| DELETE | `/api/v1/douban/cache` | 清空全部豆瓣识别缓存 |
|
||||
|
||||
缓存查询响应的 `data` 包含 `count`、`recognized`、`unrecognized`、`data`,以及共享识别统计字段
|
||||
`shared_recognized` 和开关字段 `shared_recognize_enabled`。共享命中次数仅在共享结果驱动的二次媒体识别成功后累计。
|
||||
|
||||
### 插件补充接口
|
||||
|
||||
@@ -207,13 +219,29 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
|
||||
|
||||
按需读取指定已安装插件的最新远端更新说明。该接口用于前端在用户点击“查看更新说明”时再实时访问插件仓库,避免加载已安装插件列表时批量请求网络。
|
||||
|
||||
**GET** `/api/v1/plugin/rating?plugin_ids={plugin_id,...}`
|
||||
|
||||
批量查询插件平均分、评分人数和当前安装实例评分。`plugin_ids` 省略时查询中心端已有的全部插件评分。
|
||||
|
||||
**GET** `/api/v1/plugin/rating/{plugin_id}`
|
||||
|
||||
查询单个插件平均分、评分人数和当前安装实例评分。中心端暂不可用时返回该插件的零评分结果。
|
||||
|
||||
**POST** `/api/v1/plugin/rating/{plugin_id}`
|
||||
|
||||
为已安装插件提交当前安装实例评分,请求体为 `{"rating": 4.5}`。评分范围为 `0.1` 至 `5.0`,精确到 `0.1`;同一安装实例再次提交会更新原评分。
|
||||
|
||||
### 1. 列出所有工具
|
||||
|
||||
**GET** `/api/v1/mcp/tools`
|
||||
|
||||
获取所有可用的MCP工具列表。
|
||||
|
||||
工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。
|
||||
内置工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。插件工具的参数结构由插件自身声明。
|
||||
|
||||
内置 Agent 的本地文件与命令工具 `read_file`、`write_file`、`edit_file`、
|
||||
`execute_command` 不通过 MCP 暴露。这些工具在 Agent 运行时执行独立的
|
||||
用户权限与路径边界检查;MCP 隐藏列表只负责收敛接口暴露面,不替代权限控制。
|
||||
|
||||
媒体相关 MCP 工具(如 `query_media_detail`、`search_torrents`、`query_library_exists`、`add_subscribe`、`transfer_file`)接受 `tmdb_id`/`tmdbid`、`douban_id`/`doubanid`、`bangumi_id`/`bangumiid`、`anilist_id`/`anilistid`,也接受 `media_source` + `media_id`。工具返回的媒体、订阅、下载和整理记录会同步带回可用的四种专用 ID 及通用主身份。
|
||||
|
||||
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MOVIEPILOT_BIN="$PROJECT_ROOT/moviepilot"
|
||||
VENV_PYTHON="$PROJECT_ROOT/venv/bin/python"
|
||||
|
||||
show_usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/start-local.sh 启动后端开发服务(前台运行)
|
||||
./scripts/start-local.sh backend 启动后端开发服务(前台运行)
|
||||
./scripts/start-local.sh service start 启动后端和已安装的前端服务
|
||||
./scripts/start-local.sh service start --safe 以安全模式启动完整服务
|
||||
./scripts/start-local.sh stop|restart|status 管理后端和前端服务
|
||||
./scripts/start-local.sh logs [OPTIONS] 查看后端日志
|
||||
./scripts/start-local.sh help 显示本帮助
|
||||
EOF
|
||||
}
|
||||
|
||||
if [[ ! -x "$MOVIEPILOT_BIN" ]]; then
|
||||
printf '未找到本地 CLI:%s\n' "$MOVIEPILOT_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -x "$VENV_PYTHON" ]]; then
|
||||
printf '未找到项目虚拟环境:%s\n请先执行:%s install deps\n' "$VENV_PYTHON" "$MOVIEPILOT_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 显式传入配置目录,避免被仓库中的临时 .moviepilot.env 覆盖。
|
||||
if [[ -z "${CONFIG_DIR:-}" ]]; then
|
||||
if [[ -n "${MOVIEPILOT_CONFIG_DIR:-}" ]]; then
|
||||
CONFIG_DIR="$MOVIEPILOT_CONFIG_DIR"
|
||||
elif [[ -d "${HOME:-}/Documents/moviepilot" ]]; then
|
||||
CONFIG_DIR="${HOME}/Documents/moviepilot"
|
||||
else
|
||||
CONFIG_DIR="$PROJECT_ROOT/config"
|
||||
fi
|
||||
fi
|
||||
export CONFIG_DIR
|
||||
export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
|
||||
export DEBUG="${DEBUG:-true}"
|
||||
export DEV="${DEV:-true}"
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
set -- backend
|
||||
fi
|
||||
|
||||
command_name="$1"
|
||||
shift
|
||||
|
||||
case "$command_name" in
|
||||
backend|start)
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
printf '后端模块启动不接受额外参数;完整服务请使用:%s service start [OPTIONS]\n' "$0" >&2
|
||||
exit 2
|
||||
fi
|
||||
exec "$VENV_PYTHON" -m app.main
|
||||
;;
|
||||
service)
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
set -- start
|
||||
fi
|
||||
exec "$MOVIEPILOT_BIN" "$@"
|
||||
;;
|
||||
stop|restart|status|logs|doctor|config|version)
|
||||
exec "$MOVIEPILOT_BIN" "$command_name" "$@"
|
||||
;;
|
||||
help|--help|-h)
|
||||
show_usage
|
||||
;;
|
||||
*)
|
||||
printf '未知命令:%s\n\n' "$command_name" >&2
|
||||
show_usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: create-moviepilot-plugin
|
||||
version: 2
|
||||
version: 3
|
||||
description: >-
|
||||
Use this skill when the user asks to create, modify, debug, validate, or
|
||||
scaffold a MoviePilot local plugin. Covers MoviePilot V2 plugin development,
|
||||
@@ -11,7 +11,7 @@ description: >-
|
||||
sidebar pages, commands, services, workflow actions, agent tools, and local
|
||||
install/reload flows. Also use for Chinese requests mentioning 编写插件、本地插件源,
|
||||
插件开发, V2插件, 插件市场, 本地安装插件, 插件热加载, 前端联邦, 侧栏入口, Vue插件页面.
|
||||
allowed-tools: list_directory read_file write_file edit_file execute_command query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
|
||||
allowed-tools: list_directory read_file write_file edit_file execute_command search_web browse_webpage query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
|
||||
---
|
||||
|
||||
# Create MoviePilot Plugin
|
||||
@@ -33,6 +33,33 @@ a local plugin source and installed into the running MoviePilot instance.
|
||||
- When working in or from `MoviePilot-Plugins`, read its `README.md`,
|
||||
`docs/Repository_Guide.md`, and `docs/V2_Plugin_Development.md`. For
|
||||
scenario-specific extensions, read the matching `docs/faq/*.md`.
|
||||
|
||||
## Code Tool Workflow
|
||||
|
||||
- Use `execute_command(action="run")` with `rg` and narrow globs or paths to
|
||||
locate plugin classes, extension points, tests, and package entries. Use
|
||||
`list_directory` only when inspecting one known folder or a configured remote
|
||||
storage backend.
|
||||
- Read the relevant implementation and adjacent example before editing.
|
||||
- Before using a Python or Node.js dependency API, determine the exact installed
|
||||
or locked version from requirements, package manifests, lockfiles, local
|
||||
package source, and `.pyi`/`.d.ts` declarations. If those are insufficient,
|
||||
use `search_web` with the official documentation domain and `browse_webpage`
|
||||
to read the matching version. Do not guess API signatures from memory or mix
|
||||
examples from different major versions. Search the relevant package directory,
|
||||
`.venv`, or `node_modules` directly with `rg` instead of scanning the entire
|
||||
project without bounds.
|
||||
- Use `edit_file` for localized changes. Its `old_text` must identify one exact
|
||||
location by default; add surrounding context instead of enabling
|
||||
`replace_all` unless every match intentionally changes.
|
||||
- Use `write_file` for new files. Existing files require `overwrite=true` for a
|
||||
full rewrite; first call `read_file(include_metadata=true)` and pass its
|
||||
`sha256` as `expected_sha256` when replacing previously read content.
|
||||
- Use `execute_command(action="run")` for short validation, Git, and diagnostic
|
||||
commands. Use `action="start"` only for interactive or long-running commands,
|
||||
then continue through the returned session ID.
|
||||
- Do not use shell redirection or inline scripts to perform source edits or to
|
||||
bypass a file-tool permission error.
|
||||
- When the plugin uses Vue federation, also read
|
||||
`MoviePilot-Frontend/docs/module-federation-guide.md`,
|
||||
`MoviePilot-Frontend/docs/federation-troubleshooting.md`,
|
||||
|
||||
@@ -324,13 +324,16 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
|
||||
| GET | `/api/v1/dashboard/network` | Network traffic |
|
||||
| GET | `/api/v1/dashboard/network2` | Network traffic (API_TOKEN) |
|
||||
|
||||
### Plugin (22 endpoints)
|
||||
### Plugin (25 endpoints)
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/api/v1/plugin/` | List plugins. Params: `state` (installed/market/all), `force` |
|
||||
| GET | `/api/v1/plugin/installed` | List installed plugins |
|
||||
| GET | `/api/v1/plugin/statistic` | Plugin install statistics |
|
||||
| GET | `/api/v1/plugin/rating` | Batch plugin ratings. Params: comma-separated `plugin_ids` |
|
||||
| GET | `/api/v1/plugin/rating/{plugin_id}` | Get average rating, rating count, and this installation's rating |
|
||||
| POST | `/api/v1/plugin/rating/{plugin_id}` | Rate an installed plugin. Body: `{"rating": 4.5}`; range 0.1-5.0 |
|
||||
| GET | `/api/v1/plugin/install/{plugin_id}` | Install plugin. Params: `repo_url`, `force` |
|
||||
| GET | `/api/v1/plugin/reload/{plugin_id}` | Reload plugin |
|
||||
| GET | `/api/v1/plugin/reset/{plugin_id}` | Reset plugin config & data |
|
||||
@@ -441,6 +444,20 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
|
||||
| POST | `/api/v1/torrent/cache/refresh` | Refresh torrent cache |
|
||||
| POST | `/api/v1/torrent/cache/reidentify/{domain}/{torrent_hash}` | Re-identify torrent. Params: `tmdbid`, `doubanid` |
|
||||
|
||||
### Recognition Cache (6 endpoints)
|
||||
|
||||
The two list endpoints return local cache totals plus `shared_recognized` and
|
||||
`shared_recognize_enabled` for the persisted successful shared-recognition count.
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/api/v1/tmdb/cache` | Get TheMovieDb recognition cache statistics |
|
||||
| DELETE | `/api/v1/tmdb/cache/{cache_key}` | Delete one URL-encoded TheMovieDb recognition cache key |
|
||||
| DELETE | `/api/v1/tmdb/cache` | Clear TheMovieDb recognition cache |
|
||||
| GET | `/api/v1/douban/cache` | Get Douban recognition cache statistics |
|
||||
| DELETE | `/api/v1/douban/cache/{cache_key}` | Delete one URL-encoded Douban recognition cache key |
|
||||
| DELETE | `/api/v1/douban/cache` | Clear Douban recognition cache |
|
||||
|
||||
### Message (8 endpoints)
|
||||
|
||||
| Method | Path | Description |
|
||||
@@ -488,6 +505,11 @@ Streaming search sends `{"type":"heartbeat"}` every 15 seconds without business
|
||||
| GET | `/api/v1/mcp/tools/{tool_name}` | Get tool definition |
|
||||
| GET | `/api/v1/mcp/tools/{tool_name}/schema` | Get tool input schema |
|
||||
|
||||
The exposed tool list is dynamic: it includes tools declared by enabled plugins
|
||||
and is refreshed lazily after plugin startup, shutdown, reload, or configuration
|
||||
activation. Clients that cache MCP metadata must request `tools/list` again or
|
||||
reconnect after a plugin lifecycle change.
|
||||
|
||||
### Agent MCP Client (3 endpoints)
|
||||
|
||||
| Method | Path | Description |
|
||||
|
||||
@@ -45,6 +45,10 @@ List all available commands: `moviepilot tool list`
|
||||
|
||||
Show parameters and usage for a specific command: `moviepilot tool show <command>`
|
||||
|
||||
The tool list includes tools declared by enabled plugins. Re-run `tool list` and
|
||||
`tool show` after a plugin is enabled, disabled, reloaded, or reconfigured so the
|
||||
command selection uses the refreshed runtime registry.
|
||||
|
||||
Always run `show <command>` before calling a command — parameter names are not inferable, do not guess.
|
||||
|
||||
## Command Groups
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Agent 本地文件搜索与安全编辑工具测试。"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from app.agent.tools.impl.edit_file import EditFileTool
|
||||
from app.agent.tools.impl.read_file import ReadFileTool
|
||||
from app.agent.tools.impl.write_file import WriteFileTool
|
||||
|
||||
|
||||
def _make_admin_tool(tool_class):
|
||||
"""创建带管理员上下文的文件工具实例。"""
|
||||
tool = tool_class(session_id="session-1", user_id="admin")
|
||||
tool.set_agent_context({"is_admin": True})
|
||||
return tool
|
||||
|
||||
|
||||
def test_edit_file_rejects_ambiguous_match_by_default(tmp_path):
|
||||
"""精确编辑默认应拒绝多处匹配,避免静默批量修改代码。"""
|
||||
file_path = tmp_path / "plugin.py"
|
||||
file_path.write_text("enabled = False\nenabled = False\n", encoding="utf-8")
|
||||
tool = _make_admin_tool(EditFileTool)
|
||||
|
||||
result = asyncio.run(
|
||||
tool.run(str(file_path), "enabled = False", "enabled = True")
|
||||
)
|
||||
|
||||
assert "匹配到 2 处" in result
|
||||
assert "replace_all=true" in result
|
||||
assert file_path.read_text(encoding="utf-8") == (
|
||||
"enabled = False\nenabled = False\n"
|
||||
)
|
||||
|
||||
|
||||
def test_edit_file_replace_all_requires_explicit_flag(tmp_path):
|
||||
"""显式开启 replace_all 后才应替换全部精确匹配。"""
|
||||
file_path = tmp_path / "plugin.py"
|
||||
file_path.write_text("old\nold\n", encoding="utf-8")
|
||||
tool = _make_admin_tool(EditFileTool)
|
||||
|
||||
result = asyncio.run(
|
||||
tool.run(str(file_path), "old", "new", replace_all=True)
|
||||
)
|
||||
|
||||
assert "替换了 2 处" in result
|
||||
assert file_path.read_text(encoding="utf-8") == "new\nnew\n"
|
||||
|
||||
|
||||
def test_edit_file_rejects_empty_match_and_missing_file(tmp_path):
|
||||
"""编辑工具不应再通过空匹配隐式创建文件。"""
|
||||
file_path = tmp_path / "missing.py"
|
||||
tool = _make_admin_tool(EditFileTool)
|
||||
|
||||
empty_result = asyncio.run(tool.run(str(file_path), "", "content"))
|
||||
missing_result = asyncio.run(tool.run(str(file_path), "old", "new"))
|
||||
|
||||
assert "old_text 不能为空" in empty_result
|
||||
assert "不存在" in missing_result
|
||||
assert "write_file" in missing_result
|
||||
assert not file_path.exists()
|
||||
|
||||
|
||||
def test_edit_file_rejects_stale_sha256(tmp_path):
|
||||
"""文件在读取后变化时,哈希保护应拒绝基于旧版本编辑。"""
|
||||
file_path = tmp_path / "plugin.py"
|
||||
file_path.write_text("before", encoding="utf-8")
|
||||
old_sha256 = hashlib.sha256(b"before").hexdigest()
|
||||
file_path.write_text("changed elsewhere", encoding="utf-8")
|
||||
tool = _make_admin_tool(EditFileTool)
|
||||
|
||||
result = asyncio.run(
|
||||
tool.run(
|
||||
str(file_path),
|
||||
"changed elsewhere",
|
||||
"agent change",
|
||||
expected_sha256=old_sha256,
|
||||
)
|
||||
)
|
||||
|
||||
assert "已在读取后发生变化" in result
|
||||
assert file_path.read_text(encoding="utf-8") == "changed elsewhere"
|
||||
|
||||
|
||||
def test_write_file_protects_existing_file_and_supports_guarded_overwrite(tmp_path):
|
||||
"""完整写入应默认保护已有文件,并允许带版本校验的显式覆盖。"""
|
||||
file_path = tmp_path / "plugin.py"
|
||||
file_path.write_text("before", encoding="utf-8")
|
||||
expected_sha256 = hashlib.sha256(b"before").hexdigest()
|
||||
tool = _make_admin_tool(WriteFileTool)
|
||||
|
||||
refused_result = asyncio.run(tool.run(str(file_path), "unexpected"))
|
||||
written_result = asyncio.run(
|
||||
tool.run(
|
||||
str(file_path),
|
||||
"after",
|
||||
overwrite=True,
|
||||
expected_sha256=expected_sha256,
|
||||
)
|
||||
)
|
||||
stale_result = asyncio.run(
|
||||
tool.run(
|
||||
str(file_path),
|
||||
"stale write",
|
||||
overwrite=True,
|
||||
expected_sha256=expected_sha256,
|
||||
)
|
||||
)
|
||||
|
||||
assert "拒绝完整覆盖" in refused_result
|
||||
assert "成功写入文件" in written_result
|
||||
assert "sha256=" in written_result
|
||||
assert "已在读取后发生变化" in stale_result
|
||||
assert file_path.read_text(encoding="utf-8") == "after"
|
||||
|
||||
|
||||
def test_read_file_can_return_sha256_metadata(tmp_path):
|
||||
"""读取工具应能返回供后续冲突检查使用的文件哈希。"""
|
||||
file_path = tmp_path / "plugin.py"
|
||||
file_path.write_text("插件内容", encoding="utf-8")
|
||||
tool = _make_admin_tool(ReadFileTool)
|
||||
|
||||
result = asyncio.run(tool.run(str(file_path), include_metadata=True))
|
||||
payload = json.loads(result)
|
||||
|
||||
assert payload["content"] == "插件内容"
|
||||
assert payload["size_bytes"] == len("插件内容".encode("utf-8"))
|
||||
assert payload["sha256"] == hashlib.sha256(
|
||||
"插件内容".encode("utf-8")
|
||||
).hexdigest()
|
||||
assert payload["truncated"] is False
|
||||
@@ -78,6 +78,7 @@ async def test_agent_bundle_signature_changes_with_temperature(monkeypatch) -> N
|
||||
"user_agent": None,
|
||||
"use_proxy": False,
|
||||
"thinking_level": "off",
|
||||
"api_protocol": "auto",
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
|
||||
@@ -45,3 +45,42 @@ def test_resolve_llm_runtime_config_prefers_plugin_thinking_level(monkeypatch) -
|
||||
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
|
||||
|
||||
assert runtime_config["thinking_level"] == "high"
|
||||
|
||||
|
||||
def test_resolve_llm_runtime_config_uses_system_api_protocol(monkeypatch) -> None:
|
||||
"""插件未提供 API 协议时应使用系统配置。"""
|
||||
monkeypatch.setattr(settings, "LLM_API_PROTOCOL", "responses")
|
||||
agent = MoviePilotAgent(session_id="api-protocol-default", user_id="user-1")
|
||||
|
||||
async def return_empty_config(event_type, event_data):
|
||||
"""模拟插件未返回有效运行时配置。"""
|
||||
assert event_type == ChainEventType.AgentLLMProvider
|
||||
assert event_data.api_protocol == "responses"
|
||||
return SimpleNamespace(event_data=AgentLLMProviderEventData())
|
||||
|
||||
with patch(
|
||||
"app.agent.eventmanager.async_send_event",
|
||||
new=AsyncMock(side_effect=return_empty_config),
|
||||
):
|
||||
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
|
||||
|
||||
assert runtime_config["api_protocol"] == "responses"
|
||||
|
||||
|
||||
def test_resolve_llm_runtime_config_prefers_plugin_api_protocol(monkeypatch) -> None:
|
||||
"""插件显式覆盖 API 协议时应优先使用插件值。"""
|
||||
monkeypatch.setattr(settings, "LLM_API_PROTOCOL", "responses")
|
||||
agent = MoviePilotAgent(session_id="api-protocol-plugin", user_id="user-1")
|
||||
|
||||
async def override_api_protocol(_event_type, event_data):
|
||||
"""模拟插件覆盖 API 协议。"""
|
||||
event_data.api_protocol = "chat_completions"
|
||||
return SimpleNamespace(event_data=event_data)
|
||||
|
||||
with patch(
|
||||
"app.agent.eventmanager.async_send_event",
|
||||
new=AsyncMock(side_effect=override_api_protocol),
|
||||
):
|
||||
runtime_config = asyncio.run(agent._resolve_llm_runtime_config())
|
||||
|
||||
assert runtime_config["api_protocol"] == "chat_completions"
|
||||
|
||||
@@ -49,7 +49,12 @@ def test_non_admin_manager_exposes_restricted_file_tools():
|
||||
manager = MoviePilotToolsManager(is_admin=False)
|
||||
|
||||
tool_names = {tool.name for tool in manager.list_tools()}
|
||||
assert {"read_file", "write_file", "edit_file", "list_directory"} <= tool_names
|
||||
assert {
|
||||
"read_file",
|
||||
"write_file",
|
||||
"edit_file",
|
||||
"list_directory",
|
||||
} <= tool_names
|
||||
|
||||
|
||||
def test_non_admin_manager_hides_admin_only_send_local_file_tool():
|
||||
|
||||
@@ -87,6 +87,7 @@ def test_initialize_llm_uses_chain_event_selection(monkeypatch) -> None:
|
||||
user_agent="AgentTokens-UA/1.0",
|
||||
use_proxy=True,
|
||||
thinking_level="xhigh",
|
||||
api_protocol="auto",
|
||||
)
|
||||
assert agent._llm_provider_selection["selected_provider_id"] == "provider-1"
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.utils import http as http_module
|
||||
from app.utils.http import AsyncRequestUtils
|
||||
|
||||
PROXY = "http://proxy.example:7890"
|
||||
URL = "https://raw.githubusercontent.com/demo/repo/main/package.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_h2_proxy_breaker():
|
||||
"""每个用例前后清空熔断状态,避免跨用例污染进程级熔断字典。"""
|
||||
http_module._h2_proxy_retry_at.clear()
|
||||
yield
|
||||
http_module._h2_proxy_retry_at.clear()
|
||||
|
||||
|
||||
def _fake_dispatch(calls, fail_when):
|
||||
async def fake(_self, http2, _cookies_dict, _method, _url, raise_exception, **_kwargs):
|
||||
calls.append(http2)
|
||||
if fail_when(http2):
|
||||
if raise_exception:
|
||||
raise httpx.RemoteProtocolError("tunnel closed")
|
||||
return None
|
||||
return "ok"
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def test_get_downgrades_to_h1_and_trips_breaker_on_h2_failure(monkeypatch):
|
||||
"""
|
||||
走代理的幂等请求 h2 遇到连接层失败时,就地降级 h1 重试一次并触发该 (proxy, host) 的熔断。
|
||||
"""
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: http2)
|
||||
)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
result = asyncio.run(utils.request("get", URL))
|
||||
|
||||
assert result == "ok"
|
||||
assert calls == [True, False]
|
||||
assert http_module._h2_proxy_allowed(PROXY, URL) is False
|
||||
|
||||
|
||||
def test_get_skips_h2_while_breaker_is_tripped(monkeypatch):
|
||||
"""
|
||||
熔断冷却期内,走代理的请求直接使用 h1,不再尝试 h2。
|
||||
"""
|
||||
http_module._trip_h2_proxy_breaker(PROXY, URL)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
|
||||
)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
result = asyncio.run(utils.request("get", URL))
|
||||
|
||||
assert result == "ok"
|
||||
assert calls == [False]
|
||||
|
||||
|
||||
def test_get_retries_h2_after_cooldown_expires(monkeypatch):
|
||||
"""
|
||||
熔断冷却期结束后,代理请求恢复尝试 h2。
|
||||
"""
|
||||
http_module._h2_proxy_retry_at[
|
||||
http_module._h2_proxy_breaker_key(PROXY, URL)
|
||||
] = time.monotonic() - 1
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
|
||||
)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
result = asyncio.run(utils.request("get", URL))
|
||||
|
||||
assert result == "ok"
|
||||
assert calls == [True]
|
||||
|
||||
|
||||
def test_timeout_does_not_trip_breaker_or_retry(monkeypatch):
|
||||
"""
|
||||
超时等与 h2 隧道无关的错误不触发熔断、不做 h1 重试,且保持 raise_exception=False 返回 None 的语义。
|
||||
"""
|
||||
calls = []
|
||||
|
||||
async def fake(_self, http2, _cookies_dict, _method, _url, _raise_exception, **_kwargs):
|
||||
calls.append(http2)
|
||||
raise httpx.ConnectTimeout("proxy slow")
|
||||
|
||||
monkeypatch.setattr(AsyncRequestUtils, "_dispatch_request", fake)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
result = asyncio.run(utils.request("get", URL))
|
||||
|
||||
assert result is None
|
||||
assert calls == [True]
|
||||
assert http_module._h2_proxy_allowed(PROXY, URL) is True
|
||||
|
||||
|
||||
def test_timeout_still_raises_when_raise_exception_enabled(monkeypatch):
|
||||
"""
|
||||
与 h2 隧道无关的错误在 raise_exception=True 时按原语义抛出。
|
||||
"""
|
||||
calls = []
|
||||
|
||||
async def fake(_self, http2, _cookies_dict, _method, _url, _raise_exception, **_kwargs):
|
||||
calls.append(http2)
|
||||
raise httpx.ConnectTimeout("proxy slow")
|
||||
|
||||
monkeypatch.setattr(AsyncRequestUtils, "_dispatch_request", fake)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
with pytest.raises(httpx.ConnectTimeout):
|
||||
asyncio.run(utils.request("get", URL, raise_exception=True))
|
||||
|
||||
assert calls == [True]
|
||||
assert http_module._h2_proxy_allowed(PROXY, URL) is True
|
||||
|
||||
|
||||
def test_post_does_not_downgrade_on_h2_failure(monkeypatch):
|
||||
"""
|
||||
非幂等方法 h2 失败时不做 h1 降级重试,避免服务端可能已收到数据时重复产生副作用。
|
||||
"""
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: True)
|
||||
)
|
||||
|
||||
utils = AsyncRequestUtils(proxies={"https": PROXY})
|
||||
result = asyncio.run(utils.request("post", URL))
|
||||
|
||||
assert result is None
|
||||
assert calls == [True]
|
||||
|
||||
|
||||
def test_no_proxy_configured_skips_breaker_logic(monkeypatch):
|
||||
"""
|
||||
未配置代理时不触发熔断判断,行为与原实现一致(仅走一次 h2)。
|
||||
"""
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AsyncRequestUtils, "_dispatch_request", _fake_dispatch(calls, fail_when=lambda http2: False)
|
||||
)
|
||||
|
||||
utils = AsyncRequestUtils()
|
||||
result = asyncio.run(utils.request("get", URL))
|
||||
|
||||
assert result == "ok"
|
||||
assert calls == [True]
|
||||
@@ -3,6 +3,7 @@ from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILLS_ROOT = PROJECT_ROOT / "skills"
|
||||
CORE_PROMPT_PATH = PROJECT_ROOT / "app/agent/prompt/System Core Prompt.txt"
|
||||
|
||||
|
||||
def _read_skill(skill_name: str) -> str:
|
||||
@@ -26,6 +27,7 @@ def test_modified_builtin_skills_have_incremented_versions() -> None:
|
||||
"moviepilot-cli": "6",
|
||||
"moviepilot-update": "3",
|
||||
"transfer-failed-retry": "2",
|
||||
"create-moviepilot-plugin": "3",
|
||||
}
|
||||
|
||||
for skill_name, expected_version in expected_versions.items():
|
||||
@@ -64,3 +66,29 @@ def test_api_and_database_skills_declare_fallback_boundaries() -> None:
|
||||
assert "INSERT" in db_content
|
||||
assert "UPDATE" in db_content
|
||||
assert "DELETE" in db_content
|
||||
|
||||
|
||||
def test_agent_core_prompt_does_not_block_plugin_source_edits() -> None:
|
||||
"""核心提示词不应禁止插件开发技能写入源码。"""
|
||||
core_prompt = CORE_PROMPT_PATH.read_text(encoding="utf-8")
|
||||
plugin_skill = _read_skill("create-moviepilot-plugin")
|
||||
allowed_tools = _frontmatter_value(plugin_skill, "allowed-tools")
|
||||
|
||||
assert "file editing tools, or generated patches to change code" not in core_prompt
|
||||
assert "write_file" in allowed_tools
|
||||
assert "edit_file" in allowed_tools
|
||||
assert "search_web" in allowed_tools
|
||||
assert "browse_webpage" in allowed_tools
|
||||
|
||||
|
||||
def test_agent_core_prompt_routes_code_tools_safely() -> None:
|
||||
"""核心提示词应区分代码搜索、精确编辑和交互式命令场景。"""
|
||||
core_prompt = CORE_PROMPT_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert '`execute_command(action="run")` with `rg`' in core_prompt
|
||||
assert "`replace_all=true` only when every match must change" in core_prompt
|
||||
assert "Use `action=run` for short bounded commands" in core_prompt
|
||||
assert "including SSH" in core_prompt
|
||||
assert "Never use shell redirection" in core_prompt
|
||||
assert "matching version of the official documentation" in core_prompt
|
||||
assert "Do not guess signatures from memory" in core_prompt
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
from unittest.mock import Mock
|
||||
|
||||
from app.api.endpoints import douban as douban_endpoint
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.douban.douban_cache import DoubanCache
|
||||
from app.schemas.types import MediaType
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
|
||||
|
||||
class _MemoryCacheStub:
|
||||
@@ -116,7 +117,14 @@ def test_douban_cache_endpoint_returns_management_statistics(monkeypatch):
|
||||
"recognized": {"id": "1", "title": "Alpha", "type": MediaType.MOVIE},
|
||||
"unrecognized": {"id": 0},
|
||||
})
|
||||
get_system_config = Mock(return_value=None)
|
||||
monkeypatch.setattr(douban_endpoint, "DoubanCache", lambda: cache)
|
||||
monkeypatch.setattr(
|
||||
douban_endpoint,
|
||||
"SystemConfigOper",
|
||||
lambda: type("SystemConfigStub", (), {"get": get_system_config})(),
|
||||
)
|
||||
monkeypatch.setattr(douban_endpoint.settings, "MEDIA_RECOGNIZE_SHARE", False)
|
||||
|
||||
response = asyncio.run(douban_endpoint.douban_recognition_cache(None))
|
||||
|
||||
@@ -124,6 +132,11 @@ def test_douban_cache_endpoint_returns_management_statistics(monkeypatch):
|
||||
assert response.data["count"] == 2
|
||||
assert response.data["recognized"] == 1
|
||||
assert response.data["unrecognized"] == 1
|
||||
assert response.data["shared_recognized"] == 0
|
||||
assert response.data["shared_recognize_enabled"] is False
|
||||
get_system_config.assert_called_once_with(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
)
|
||||
|
||||
|
||||
def test_douban_cache_delete_endpoint_reports_missing_item(monkeypatch):
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
from app.schemas.file import FileURI
|
||||
|
||||
|
||||
def test_from_uri_keeps_windows_drive_path() -> None:
|
||||
"""Windows 盘符路径已是绝对路径,不能再补根斜杠,否则映射网络驱动器整理会报 WinError 123。"""
|
||||
file_uri = FileURI.from_uri("Z:/Downloads/电视剧/国产剧")
|
||||
|
||||
assert file_uri.storage == "local"
|
||||
assert file_uri.path == "Z:/Downloads/电视剧/国产剧"
|
||||
assert file_uri.uri == "Z:/Downloads/电视剧/国产剧"
|
||||
|
||||
|
||||
def test_from_uri_keeps_windows_drive_path_with_backslash() -> None:
|
||||
"""反斜杠写法的盘符路径同样不能补根斜杠。"""
|
||||
file_uri = FileURI.from_uri("Z:\\Downloads\\电视剧")
|
||||
|
||||
assert file_uri.storage == "local"
|
||||
assert not file_uri.path.startswith("/")
|
||||
|
||||
|
||||
def test_from_uri_keeps_posix_absolute_path() -> None:
|
||||
"""POSIX 绝对路径保持原样。"""
|
||||
file_uri = FileURI.from_uri("/downloads/movies")
|
||||
|
||||
assert file_uri.storage == "local"
|
||||
assert file_uri.path == "/downloads/movies"
|
||||
|
||||
|
||||
def test_from_uri_adds_root_for_relative_path() -> None:
|
||||
"""无前导斜杠的相对路径仍补全为绝对路径。"""
|
||||
file_uri = FileURI.from_uri("downloads/movies")
|
||||
|
||||
assert file_uri.path == "/downloads/movies"
|
||||
|
||||
|
||||
def test_from_uri_parses_storage_prefix() -> None:
|
||||
"""带存储前缀的 URI 应拆分出存储类型并保留 POSIX 路径。"""
|
||||
file_uri = FileURI.from_uri("u115:/media/anime")
|
||||
|
||||
assert file_uri.storage == "u115"
|
||||
assert file_uri.path == "/media/anime"
|
||||
assert file_uri.uri == "u115:/media/anime"
|
||||
|
||||
|
||||
def test_from_uri_storage_prefix_with_relative_path() -> None:
|
||||
"""远端存储的无前导斜杠路径补全为绝对路径。"""
|
||||
file_uri = FileURI.from_uri("rclone:media/anime")
|
||||
|
||||
assert file_uri.storage == "rclone"
|
||||
assert file_uri.path == "/media/anime"
|
||||
@@ -130,6 +130,7 @@ _config_stub.settings = SimpleNamespace(
|
||||
LLM_BASE_URL_PRESET=None,
|
||||
LLM_USER_AGENT=None,
|
||||
LLM_THINKING_LEVEL=None,
|
||||
LLM_API_PROTOCOL="auto",
|
||||
LLM_TEMPERATURE=0.1,
|
||||
LLM_MAX_CONTEXT_TOKENS=64,
|
||||
LLM_USE_PROXY=True,
|
||||
@@ -243,6 +244,7 @@ class LlmHelperTestCallTest(unittest.TestCase):
|
||||
base_url_preset="deepseek-default",
|
||||
user_agent=None,
|
||||
use_proxy=None,
|
||||
api_protocol=None,
|
||||
)
|
||||
self.assertEqual(result["provider"], "deepseek")
|
||||
self.assertEqual(result["model"], "deepseek-chat")
|
||||
@@ -870,3 +872,178 @@ class LlmHelperTestCallTest(unittest.TestCase):
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0].get("thinking_level"), "high")
|
||||
self.assertFalse(calls[0].get("include_thoughts"))
|
||||
|
||||
def test_get_llm_responses_protocol_forces_responses_api(self):
|
||||
"""显式 responses 协议应让通用 OpenAI 兼容入口走 Responses API。"""
|
||||
calls = []
|
||||
|
||||
class _FakeChatOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
self.model = kwargs["model"]
|
||||
self.profile = None
|
||||
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
|
||||
):
|
||||
asyncio.run(
|
||||
llm_module.LLMHelper.get_llm(
|
||||
provider="openai",
|
||||
model="gpt-5.6-terra",
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
api_protocol="responses",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertTrue(calls[0].get("use_responses_api"))
|
||||
|
||||
def test_get_llm_chat_completions_protocol_overrides_chatgpt_auto(self):
|
||||
"""显式 chat_completions 应覆盖 ChatGPT 官方推理模型的自动 Responses 切换。"""
|
||||
calls = []
|
||||
|
||||
class _FakeChatOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
self.model = kwargs["model"]
|
||||
self.profile = None
|
||||
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
|
||||
):
|
||||
asyncio.run(
|
||||
llm_module.LLMHelper.get_llm(
|
||||
provider="chatgpt",
|
||||
model="gpt-5.4",
|
||||
api_key="sk-test",
|
||||
base_url="https://api.openai.com/v1",
|
||||
api_protocol="chat_completions",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertFalse(calls[0].get("use_responses_api"))
|
||||
|
||||
def test_get_llm_auto_protocol_keeps_chat_completions_for_compatible(self):
|
||||
"""auto 协议下通用 OpenAI 兼容入口应保持默认 Chat Completions(None)。"""
|
||||
calls = []
|
||||
|
||||
class _FakeChatOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
self.model = kwargs["model"]
|
||||
self.profile = None
|
||||
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
|
||||
):
|
||||
asyncio.run(
|
||||
llm_module.LLMHelper.get_llm(
|
||||
provider="openai",
|
||||
model="gpt-4o",
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
api_protocol="auto",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertIsNone(calls[0].get("use_responses_api"))
|
||||
|
||||
def test_get_llm_runtime_override_beats_chat_completions_protocol(self):
|
||||
"""运行时强制 Responses(OAuth/Codex)应优先于用户 chat_completions 设置。"""
|
||||
calls = []
|
||||
|
||||
class _FakeProviderManager:
|
||||
async def resolve_runtime(self, **kwargs):
|
||||
return {
|
||||
"provider_id": kwargs["provider_id"],
|
||||
"runtime": "openai_compatible",
|
||||
"model_id": kwargs["model"],
|
||||
"api_key": kwargs["api_key"],
|
||||
"base_url": kwargs["base_url"],
|
||||
"default_headers": None,
|
||||
"use_responses_api": True,
|
||||
"model_record": None,
|
||||
"model_metadata": None,
|
||||
}
|
||||
|
||||
class _FakeChatOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
self.model = kwargs["model"]
|
||||
self.profile = None
|
||||
|
||||
provider_module = ModuleType("app.agent.llm.provider")
|
||||
provider_module.LLMProviderManager = _FakeProviderManager
|
||||
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"app.agent.llm.provider": provider_module,
|
||||
"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI),
|
||||
},
|
||||
):
|
||||
asyncio.run(
|
||||
llm_module.LLMHelper.get_llm(
|
||||
provider="chatgpt",
|
||||
model="gpt-5.4",
|
||||
api_key="sk-test",
|
||||
base_url="https://api.openai.com/v1",
|
||||
api_protocol="chat_completions",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertTrue(calls[0].get("use_responses_api"))
|
||||
|
||||
def test_get_llm_reads_api_protocol_from_settings_when_omitted(self):
|
||||
"""未显式传入协议时应读取 LLM_API_PROTOCOL 配置。"""
|
||||
calls = []
|
||||
|
||||
class _FakeChatOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
calls.append(kwargs)
|
||||
self.model = kwargs["model"]
|
||||
self.profile = None
|
||||
|
||||
with patch.object(
|
||||
llm_module.settings, "LLM_API_PROTOCOL", "responses"
|
||||
), patch.dict(
|
||||
sys.modules,
|
||||
{"langchain_openai": SimpleNamespace(ChatOpenAI=_FakeChatOpenAI)},
|
||||
):
|
||||
asyncio.run(
|
||||
llm_module.LLMHelper.get_llm(
|
||||
provider="openai",
|
||||
model="gpt-5.6-terra",
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertTrue(calls[0].get("use_responses_api"))
|
||||
|
||||
def test_normalize_api_protocol_accepts_known_and_falls_back(self):
|
||||
"""_normalize_api_protocol 应大小写不敏感识别已知值,未知值回退 auto。"""
|
||||
self.assertEqual(
|
||||
llm_module.LLMHelper._normalize_api_protocol("Responses"), "responses"
|
||||
)
|
||||
self.assertEqual(
|
||||
llm_module.LLMHelper._normalize_api_protocol("CHAT_COMPLETIONS"),
|
||||
"chat_completions",
|
||||
)
|
||||
self.assertEqual(
|
||||
llm_module.LLMHelper._normalize_api_protocol("auto"), "auto"
|
||||
)
|
||||
self.assertEqual(
|
||||
llm_module.LLMHelper._normalize_api_protocol(None), "auto"
|
||||
)
|
||||
self.assertEqual(
|
||||
llm_module.LLMHelper._normalize_api_protocol("weird"), "auto"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import Iterator
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.factory import MoviePilotToolFactory
|
||||
from app.agent.tools.manager import MoviePilotToolsManager
|
||||
from app.api.endpoints import mcp
|
||||
from app.core.plugin import PluginManager
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
|
||||
class DemoPluginTool(MoviePilotTool):
|
||||
"""测试用插件 MCP 工具。"""
|
||||
|
||||
name: str = "demo_plugin_tool"
|
||||
description: str = "测试插件动态注册的 MCP 工具"
|
||||
|
||||
async def run(self, **kwargs) -> str:
|
||||
"""返回固定测试结果。"""
|
||||
return "plugin-ok"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_manager() -> Iterator[PluginManager]:
|
||||
"""构造隔离的插件管理器并在测试后恢复原单例。"""
|
||||
singleton_key = (PluginManager, (), frozenset())
|
||||
previous_instance = Singleton._instances.pop(singleton_key, None)
|
||||
manager = PluginManager()
|
||||
yield manager
|
||||
Singleton._instances.pop(singleton_key, None)
|
||||
if previous_instance is not None:
|
||||
Singleton._instances[singleton_key] = previous_instance
|
||||
|
||||
|
||||
def _build_plugin() -> SimpleNamespace:
|
||||
"""构造声明一个 Agent 工具的已启用插件。"""
|
||||
return SimpleNamespace(
|
||||
plugin_name="Demo Plugin",
|
||||
get_state=lambda: True,
|
||||
get_agent_tools=lambda: [DemoPluginTool],
|
||||
)
|
||||
|
||||
|
||||
def test_mcp_refreshes_tools_after_plugin_lifecycle_change(
|
||||
plugin_manager: PluginManager,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""MCP 管理器应发现初始化后新增的插件工具,并在插件移除后停止暴露。"""
|
||||
with patch.object(
|
||||
MoviePilotToolFactory,
|
||||
"_get_builtin_tool_classes",
|
||||
return_value=[],
|
||||
):
|
||||
tool_manager = MoviePilotToolsManager(
|
||||
session_id="mcp-plugin-test",
|
||||
user_id="api_user",
|
||||
)
|
||||
monkeypatch.setattr(mcp, "moviepilot_tool_manager", tool_manager)
|
||||
|
||||
assert asyncio.run(mcp.handle_tools_list()) == {"tools": []}
|
||||
|
||||
plugin_manager.running_plugins["DemoPlugin"] = _build_plugin()
|
||||
plugin_manager.clear_plugin_agent_tools_cache()
|
||||
|
||||
listed_tools = asyncio.run(mcp.handle_tools_list())["tools"]
|
||||
assert [tool["name"] for tool in listed_tools] == ["demo_plugin_tool"]
|
||||
|
||||
call_result = asyncio.run(
|
||||
mcp.handle_tools_call(
|
||||
{
|
||||
"name": "demo_plugin_tool",
|
||||
"arguments": {},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert call_result == {
|
||||
"content": [{"type": "text", "text": "plugin-ok"}]
|
||||
}
|
||||
|
||||
plugin_manager.running_plugins.pop("DemoPlugin")
|
||||
plugin_manager.clear_plugin_agent_tools_cache()
|
||||
|
||||
assert asyncio.run(mcp.handle_tools_list()) == {"tools": []}
|
||||
missing_result = asyncio.run(
|
||||
mcp.handle_tools_call(
|
||||
{
|
||||
"name": "demo_plugin_tool",
|
||||
"arguments": {},
|
||||
}
|
||||
)
|
||||
)
|
||||
missing_payload = json.loads(missing_result["content"][0]["text"])
|
||||
assert "未找到" in missing_payload["error"]
|
||||
@@ -0,0 +1,134 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.core.context import MediaInfo
|
||||
from app.core.meta import MetaBase
|
||||
from app.helper.server import MoviePilotServerHelper
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
|
||||
|
||||
def _build_meta(name: str) -> MetaBase:
|
||||
"""构造共享识别统计测试所需的媒体元数据。"""
|
||||
meta = MetaBase(name)
|
||||
meta.name = name
|
||||
meta.type = MediaType.UNKNOWN
|
||||
return meta
|
||||
|
||||
|
||||
def _shared_params(tmdb_id: int) -> dict:
|
||||
"""构造共享识别结果转换后的模块参数。"""
|
||||
return {
|
||||
"mtype": MediaType.MOVIE,
|
||||
"source": "themoviedb",
|
||||
"mediaid": str(tmdb_id),
|
||||
"tmdbid": tmdb_id,
|
||||
"doubanid": None,
|
||||
"bangumiid": None,
|
||||
"anilistid": None,
|
||||
}
|
||||
|
||||
|
||||
def _mock_counter(monkeypatch) -> Mock:
|
||||
"""替换系统配置持久化入口并返回递增调用桩。"""
|
||||
increment = Mock()
|
||||
monkeypatch.setattr(
|
||||
"app.chain.SystemConfigOper",
|
||||
lambda: SimpleNamespace(increment=increment),
|
||||
)
|
||||
return increment
|
||||
|
||||
|
||||
def test_sync_shared_recognize_success_increments_persisted_count(monkeypatch):
|
||||
"""同步共享识别二次识别成功后应累计一次命中。"""
|
||||
chain = object.__new__(ChainBase)
|
||||
meta = _build_meta("共享识别电影")
|
||||
media = MediaInfo(
|
||||
title="共享识别电影",
|
||||
year="2026",
|
||||
tmdb_id=101,
|
||||
type=MediaType.MOVIE,
|
||||
)
|
||||
increment = _mock_counter(monkeypatch)
|
||||
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
|
||||
monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, media]))
|
||||
monkeypatch.setattr(chain, "_update_local_recognize_cache", Mock())
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"query_recognize_share",
|
||||
Mock(return_value={"type": "movie", "tmdbid": 101}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"to_recognize_params",
|
||||
Mock(return_value=_shared_params(101)),
|
||||
)
|
||||
|
||||
result = chain.recognize_media(meta=meta, cache=False)
|
||||
|
||||
assert result is media
|
||||
increment.assert_called_once_with(SystemConfigKey.MediaRecognizeShareCount)
|
||||
|
||||
|
||||
def test_sync_shared_result_without_local_match_does_not_increment(monkeypatch):
|
||||
"""共享接口返回数据但二次识别失败时不应累计命中。"""
|
||||
chain = object.__new__(ChainBase)
|
||||
meta = _build_meta("共享识别失败电影")
|
||||
increment = _mock_counter(monkeypatch)
|
||||
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
|
||||
monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, None]))
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"query_recognize_share",
|
||||
Mock(return_value={"type": "movie", "tmdbid": 102}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"to_recognize_params",
|
||||
Mock(return_value=_shared_params(102)),
|
||||
)
|
||||
|
||||
result = chain.recognize_media(meta=meta, cache=False)
|
||||
|
||||
assert result is None
|
||||
increment.assert_not_called()
|
||||
|
||||
|
||||
def test_async_shared_recognize_success_increments_persisted_count(monkeypatch):
|
||||
"""异步共享识别二次识别成功后应累计一次命中。"""
|
||||
chain = object.__new__(ChainBase)
|
||||
meta = _build_meta("异步共享识别电影")
|
||||
media = MediaInfo(
|
||||
title="异步共享识别电影",
|
||||
year="2026",
|
||||
tmdb_id=103,
|
||||
type=MediaType.MOVIE,
|
||||
)
|
||||
increment = _mock_counter(monkeypatch)
|
||||
monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True)
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"async_run_module",
|
||||
AsyncMock(side_effect=[None, media]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"_async_update_local_recognize_cache",
|
||||
AsyncMock(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"async_query_recognize_share",
|
||||
AsyncMock(return_value={"type": "movie", "tmdbid": 103}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MoviePilotServerHelper,
|
||||
"to_recognize_params",
|
||||
Mock(return_value=_shared_params(103)),
|
||||
)
|
||||
|
||||
result = asyncio.run(chain.async_recognize_media(meta=meta, cache=False))
|
||||
|
||||
assert result is media
|
||||
increment.assert_called_once_with(SystemConfigKey.MediaRecognizeShareCount)
|
||||
@@ -0,0 +1,29 @@
|
||||
from app.helper.service import ServiceConfigHelper
|
||||
from app.schemas.system import MediaServerConf
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
def test_mediaserver_conf_tolerates_blank_sync_interval():
|
||||
"""自动同步间隔为空字符串等非法值时应回退为 None 而不是抛出校验错误。"""
|
||||
assert MediaServerConf(name="blank", sync_interval="").sync_interval is None
|
||||
assert MediaServerConf(name="spaces", sync_interval=" ").sync_interval is None
|
||||
assert MediaServerConf(name="invalid", sync_interval="abc").sync_interval is None
|
||||
assert MediaServerConf(name="text", sync_interval="12").sync_interval == 12
|
||||
assert MediaServerConf(name="number", sync_interval=6).sync_interval == 6
|
||||
assert MediaServerConf(name="none", sync_interval=None).sync_interval is None
|
||||
|
||||
|
||||
def test_get_configs_skips_invalid_entries(monkeypatch):
|
||||
"""单条配置校验失败时应跳过该条,不影响其它服务配置的加载。"""
|
||||
monkeypatch.setattr(
|
||||
"app.helper.service.SystemConfigOper.get",
|
||||
lambda self, key: [
|
||||
{"name": "good", "type": "emby", "enabled": True},
|
||||
"bad-format",
|
||||
{"name": "bad-type", "type": "plex", "enabled": "maybe"},
|
||||
],
|
||||
)
|
||||
|
||||
configs = ServiceConfigHelper.get_configs(SystemConfigKey.MediaServers, MediaServerConf)
|
||||
|
||||
assert [conf.name for conf in configs] == ["good"]
|
||||
@@ -0,0 +1,273 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||
|
||||
|
||||
def _build_watcher(tmp_path, force_polling):
|
||||
"""
|
||||
构造测试用目录监控。
|
||||
:param tmp_path: 监控目录
|
||||
:param force_polling: 是否强制轮询
|
||||
:return: 目录监控
|
||||
"""
|
||||
return LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=force_polling)
|
||||
|
||||
|
||||
def test_run_retries_with_backoff_in_compatibility_mode(tmp_path, monkeypatch):
|
||||
"""
|
||||
兼容模式下监控循环抛异常后应退避重启,而不是直接结束线程。
|
||||
"""
|
||||
monkeypatch.setattr(LocalDirectoryWatcher, "RESTART_BACKOFF", (0,))
|
||||
watcher = _build_watcher(tmp_path, force_polling=True)
|
||||
calls = []
|
||||
|
||||
def fake_run_watch(force_polling):
|
||||
"""
|
||||
模拟底层监控循环持续抛出 FUSE 错误。
|
||||
"""
|
||||
calls.append(force_polling)
|
||||
if len(calls) >= 3:
|
||||
watcher.stop()
|
||||
raise OSError(131, "State not recoverable")
|
||||
|
||||
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
|
||||
|
||||
watcher._run()
|
||||
|
||||
assert calls == [True, True, True]
|
||||
assert watcher.restart_count == 2
|
||||
|
||||
|
||||
def test_run_falls_back_to_polling_before_backoff(tmp_path, monkeypatch):
|
||||
"""
|
||||
快速模式失败应先降级为兼容模式重试,且降级不计入退避重启次数。
|
||||
"""
|
||||
monkeypatch.setattr(LocalDirectoryWatcher, "RESTART_BACKOFF", (0,))
|
||||
watcher = _build_watcher(tmp_path, force_polling=None)
|
||||
calls = []
|
||||
|
||||
def fake_run_watch(force_polling):
|
||||
"""
|
||||
模拟快速模式与兼容模式先后失败。
|
||||
"""
|
||||
calls.append(force_polling)
|
||||
if len(calls) >= 2:
|
||||
watcher.stop()
|
||||
raise OSError("inotify watch limit reached")
|
||||
|
||||
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
|
||||
|
||||
watcher._run()
|
||||
|
||||
assert calls == [None, True]
|
||||
assert watcher.restart_count == 0
|
||||
|
||||
|
||||
def test_run_returns_when_stop_requested(tmp_path, monkeypatch):
|
||||
"""
|
||||
收到停止信号后监控循环正常返回,不应触发重启。
|
||||
"""
|
||||
watcher = _build_watcher(tmp_path, force_polling=True)
|
||||
calls = []
|
||||
|
||||
def fake_run_watch(force_polling):
|
||||
"""
|
||||
模拟收到停止信号后正常退出的监控循环。
|
||||
"""
|
||||
calls.append(force_polling)
|
||||
|
||||
monkeypatch.setattr(watcher, "_run_watch", fake_run_watch)
|
||||
|
||||
watcher._run()
|
||||
|
||||
assert calls == [True]
|
||||
assert watcher.restart_count == 0
|
||||
|
||||
|
||||
def test_is_stalled_detects_silent_failure(tmp_path):
|
||||
"""
|
||||
监控线程存活但长时间无活动时应判定为静默失效。
|
||||
"""
|
||||
watcher = _build_watcher(tmp_path, force_polling=True)
|
||||
|
||||
# 线程未启动时不做判定
|
||||
assert watcher.is_stalled() is False
|
||||
|
||||
thread = MagicMock()
|
||||
thread.is_alive.return_value = True
|
||||
watcher._thread = thread
|
||||
watcher._mark_activity()
|
||||
assert watcher.is_stalled() is False
|
||||
|
||||
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
|
||||
assert watcher.is_stalled() is True
|
||||
|
||||
|
||||
def test_is_stalled_ignores_stopped_watcher(tmp_path):
|
||||
"""
|
||||
已请求停止的监控不应再被判定为静默失效。
|
||||
"""
|
||||
watcher = _build_watcher(tmp_path, force_polling=True)
|
||||
thread = MagicMock()
|
||||
thread.is_alive.return_value = True
|
||||
watcher._thread = thread
|
||||
watcher._mark_activity()
|
||||
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert watcher.is_stalled() is False
|
||||
|
||||
|
||||
def _build_monitor(monkeypatch, put_recorder):
|
||||
"""
|
||||
构造测试用 Monitor 骨架,绕过单例初始化。
|
||||
:param monkeypatch: pytest monkeypatch
|
||||
:param put_recorder: 消息推送记录器
|
||||
:return: Monitor 骨架
|
||||
"""
|
||||
from threading import Lock
|
||||
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor._watchers = []
|
||||
monitor._watcher_lock = Lock()
|
||||
monitor._pending_locals = []
|
||||
monitor._alerted_paths = set()
|
||||
monitor._restart_marks = {}
|
||||
monitor._stable_cycles = {}
|
||||
return monitor
|
||||
|
||||
|
||||
def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
|
||||
"""
|
||||
构造测试用监控线程替身。
|
||||
:param mon_path: 监控目录
|
||||
:param alive: 线程是否存活
|
||||
:param stalled: 是否静默失效
|
||||
:param restart_count: 自动重启次数
|
||||
:return: 监控线程替身
|
||||
"""
|
||||
watcher = MagicMock()
|
||||
watcher.watch_path = mon_path
|
||||
watcher.is_alive.return_value = alive
|
||||
watcher.is_stalled.return_value = stalled
|
||||
watcher.restart_count = restart_count
|
||||
return watcher
|
||||
|
||||
|
||||
def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch):
|
||||
"""
|
||||
监控线程退出后健康检查应重建线程并告警。
|
||||
"""
|
||||
put_recorder = MagicMock()
|
||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||
watcher = _fake_watcher(tmp_path, alive=False)
|
||||
monitor._watchers = [watcher]
|
||||
rebuild = MagicMock()
|
||||
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
|
||||
|
||||
monitor._Monitor__check_watchers()
|
||||
|
||||
rebuild.assert_called_once_with(watcher)
|
||||
put_recorder.put.assert_called_once()
|
||||
|
||||
|
||||
def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch):
|
||||
"""
|
||||
静默失效的监控线程也应被健康检查重建。
|
||||
"""
|
||||
put_recorder = MagicMock()
|
||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||
monitor._watchers = [watcher]
|
||||
rebuild = MagicMock()
|
||||
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
|
||||
|
||||
monitor._Monitor__check_watchers()
|
||||
|
||||
rebuild.assert_called_once_with(watcher)
|
||||
|
||||
|
||||
def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch):
|
||||
"""
|
||||
自动重启应触发一次告警,恢复消息需等满稳定窗口,避免来回刷屏。
|
||||
"""
|
||||
put_recorder = MagicMock()
|
||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||
watcher = _fake_watcher(tmp_path, alive=True, stalled=False, restart_count=1)
|
||||
monitor._watchers = [watcher]
|
||||
|
||||
monitor._Monitor__check_watchers()
|
||||
assert str(tmp_path) in monitor._alerted_paths
|
||||
assert put_recorder.put.call_count == 1
|
||||
|
||||
for _ in range(Monitor.RECOVERY_STABLE_CYCLES - 1):
|
||||
monitor._Monitor__check_watchers()
|
||||
assert str(tmp_path) in monitor._alerted_paths
|
||||
|
||||
monitor._Monitor__check_watchers()
|
||||
assert str(tmp_path) not in monitor._alerted_paths
|
||||
assert put_recorder.put.call_count == 2
|
||||
|
||||
|
||||
def test_retry_pending_locals_backs_off(tmp_path, monkeypatch):
|
||||
"""
|
||||
启动失败的监控重试应按失败次数退避,避免持续故障时刷屏。
|
||||
"""
|
||||
put_recorder = MagicMock()
|
||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
|
||||
start = MagicMock(return_value=False)
|
||||
setattr(monitor, "_Monitor__start_local_monitor", start)
|
||||
|
||||
for _ in range(6):
|
||||
monitor._Monitor__retry_pending_locals()
|
||||
|
||||
assert start.call_count == 3
|
||||
|
||||
|
||||
def test_dispatcher_retries_after_history_query_failure(monkeypatch):
|
||||
"""
|
||||
整理历史查询失败应登记待重试,重试成功后进入整理链并清除登记。
|
||||
"""
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||
event_path = Path("/downloads/movie.mkv")
|
||||
history = MagicMock(side_effect=[None, False])
|
||||
monkeypatch.setattr(dispatcher, "_has_transfer_history", history)
|
||||
transfer_chain_instance = MagicMock()
|
||||
monkeypatch.setattr("app.monitor.dispatcher.TransferChain",
|
||||
MagicMock(return_value=transfer_chain_instance))
|
||||
|
||||
# 首次查询失败:不整理,登记待重试
|
||||
assert dispatcher.handle_file(storage="local", event_path=event_path, file_size=1) is False
|
||||
assert len(dispatcher._pending_retries) == 1
|
||||
transfer_chain_instance.do_transfer.assert_not_called()
|
||||
|
||||
# 模拟 TTL 缓存过期后由健康检查驱动重试
|
||||
dispatcher._cache.clear()
|
||||
dispatcher.retry_pending()
|
||||
|
||||
transfer_chain_instance.do_transfer.assert_called_once()
|
||||
assert dispatcher._pending_retries == {}
|
||||
|
||||
|
||||
def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
|
||||
"""
|
||||
历史查询持续失败达到上限后应放弃重试,避免队列无限累积。
|
||||
"""
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||
event_path = Path("/downloads/movie.mkv")
|
||||
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=None))
|
||||
|
||||
dispatcher.handle_file(storage="local", event_path=event_path, file_size=1)
|
||||
key = f"local:{event_path.as_posix()}"
|
||||
assert key in dispatcher._pending_retries
|
||||
dispatcher._pending_retries[key]["attempts"] = TransferDispatcher.MAX_RETRY_ATTEMPTS - 1
|
||||
|
||||
dispatcher._cache.clear()
|
||||
dispatcher.retry_pending()
|
||||
|
||||
assert dispatcher._pending_retries == {}
|
||||
@@ -0,0 +1,177 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.monitor.poller import RemotePoller
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
|
||||
|
||||
def _build_poller(alert_cb=None):
|
||||
"""
|
||||
构造测试用远程轮询监控。
|
||||
:param alert_cb: 告警回调替身
|
||||
:return: (poller, store, dispatcher)
|
||||
"""
|
||||
store = MagicMock()
|
||||
store.save.return_value = True
|
||||
dispatcher = MagicMock()
|
||||
dispatcher.is_transfer_candidate_path.return_value = True
|
||||
dispatcher.handle_file.return_value = True
|
||||
poller = RemotePoller(store=store, dispatcher=dispatcher, alert_cb=alert_cb)
|
||||
return poller, store, dispatcher
|
||||
|
||||
|
||||
def _mock_storage_chain(monkeypatch, side_effect):
|
||||
"""
|
||||
替换 StorageChain 的快照返回。
|
||||
:param monkeypatch: pytest monkeypatch
|
||||
:param side_effect: snapshot_storage 的返回序列
|
||||
:return: StorageChain 实例替身
|
||||
"""
|
||||
chain_instance = MagicMock()
|
||||
chain_instance.snapshot_storage.side_effect = side_effect
|
||||
monkeypatch.setattr("app.monitor.poller.StorageChain", MagicMock(return_value=chain_instance))
|
||||
return chain_instance
|
||||
|
||||
|
||||
BASELINE = {
|
||||
'timestamp': 100,
|
||||
'file_count': 1,
|
||||
'snapshot': {'/mon/a.mkv': {'size': 1, 'modify_time': 100}}
|
||||
}
|
||||
|
||||
|
||||
def test_poll_merges_incremental_into_baseline(monkeypatch):
|
||||
"""
|
||||
增量快照应与基线合并落盘,未扫到的旧文件不能从基线消失。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (dict(BASELINE), True)
|
||||
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}])
|
||||
|
||||
file_count = poller.poll("u115", [Path("/mon")])
|
||||
|
||||
assert file_count == 2
|
||||
saved_snapshot = store.save.call_args.args[1]
|
||||
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon/b.mkv'}
|
||||
dispatcher.handle_file.assert_called_once()
|
||||
assert dispatcher.handle_file.call_args.kwargs["event_path"] == Path('/mon/b.mkv')
|
||||
|
||||
|
||||
def test_poll_detects_modified_files(monkeypatch):
|
||||
"""
|
||||
增量中已有文件的大小变化应作为修改事件分发,并更新基线。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (dict(BASELINE), True)
|
||||
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 5, 'modify_time': 300}}])
|
||||
|
||||
file_count = poller.poll("u115", [Path("/mon")])
|
||||
|
||||
assert file_count == 1
|
||||
saved_snapshot = store.save.call_args.args[1]
|
||||
assert saved_snapshot['/mon/a.mkv']['size'] == 5
|
||||
dispatcher.handle_file.assert_called_once()
|
||||
|
||||
|
||||
def test_poll_partial_failure_merges_success_and_keeps_baseline(monkeypatch):
|
||||
"""
|
||||
部分路径快照失败时,成功路径合并落盘,失败路径保留旧基线。
|
||||
"""
|
||||
alert_cb = MagicMock()
|
||||
poller, store, dispatcher = _build_poller(alert_cb)
|
||||
store.load_checked.return_value = (dict(BASELINE), True)
|
||||
_mock_storage_chain(monkeypatch, [None, {'/mon2/b.mkv': {'size': 2, 'modify_time': 200}}])
|
||||
|
||||
file_count = poller.poll("u115", [Path("/mon"), Path("/mon2")])
|
||||
|
||||
assert file_count == 2
|
||||
saved_snapshot = store.save.call_args.args[1]
|
||||
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon2/b.mkv'}
|
||||
# 单次失败未达告警阈值
|
||||
alert_cb.assert_not_called()
|
||||
|
||||
|
||||
def test_poll_all_paths_failed_skips_save(monkeypatch):
|
||||
"""
|
||||
全部路径快照失败时本轮不落盘,基线保持不变。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (dict(BASELINE), True)
|
||||
_mock_storage_chain(monkeypatch, [None])
|
||||
|
||||
assert poller.poll("u115", [Path("/mon")]) is None
|
||||
store.save.assert_not_called()
|
||||
dispatcher.handle_file.assert_not_called()
|
||||
|
||||
|
||||
def test_poll_first_snapshot_failure_builds_no_empty_baseline(monkeypatch):
|
||||
"""
|
||||
首次快照失败时不得落盘空基线,否则下一轮会把全部存量当作新增。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (None, True)
|
||||
_mock_storage_chain(monkeypatch, [None])
|
||||
|
||||
assert poller.poll("u115", [Path("/mon")]) is None
|
||||
store.save.assert_not_called()
|
||||
|
||||
|
||||
def test_poll_first_snapshot_success_saves_baseline_without_dispatch(monkeypatch):
|
||||
"""
|
||||
首次快照成功仅建立基准,不应处理存量文件。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (None, True)
|
||||
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
|
||||
|
||||
assert poller.poll("u115", [Path("/mon")]) == 1
|
||||
store.save.assert_called_once()
|
||||
dispatcher.handle_file.assert_not_called()
|
||||
|
||||
|
||||
def test_poll_load_error_skips_round(monkeypatch):
|
||||
"""
|
||||
基线读取失败不能当作首次快照,应跳过本轮避免丢弃已有基线。
|
||||
"""
|
||||
poller, store, dispatcher = _build_poller()
|
||||
store.load_checked.return_value = (None, False)
|
||||
chain = _mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
|
||||
|
||||
assert poller.poll("u115", [Path("/mon")]) is None
|
||||
chain.snapshot_storage.assert_not_called()
|
||||
store.save.assert_not_called()
|
||||
|
||||
|
||||
def test_poll_failure_alert_threshold_and_recovery(monkeypatch):
|
||||
"""
|
||||
连续异常达到阈值只告警一次,恢复后推送恢复消息。
|
||||
"""
|
||||
alert_cb = MagicMock()
|
||||
poller, store, dispatcher = _build_poller(alert_cb)
|
||||
store.load_checked.return_value = (dict(BASELINE), True)
|
||||
_mock_storage_chain(
|
||||
monkeypatch,
|
||||
[None] * RemotePoller.FAILURE_ALERT_THRESHOLD + [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}]
|
||||
)
|
||||
|
||||
for _ in range(RemotePoller.FAILURE_ALERT_THRESHOLD):
|
||||
poller.poll("u115", [Path("/mon")])
|
||||
assert alert_cb.call_count == 1
|
||||
|
||||
poller.poll("u115", [Path("/mon")])
|
||||
assert alert_cb.call_count == 2
|
||||
assert "已恢复" in alert_cb.call_args.args[1]
|
||||
|
||||
|
||||
def test_watcher_poll_delay_defaults_and_override(tmp_path):
|
||||
"""
|
||||
轮询扫描间隔默认取本地值,显式传入网络值时生效。
|
||||
"""
|
||||
default_watcher = LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=True)
|
||||
assert default_watcher.poll_delay_ms == LocalDirectoryWatcher.POLL_DELAY_LOCAL_MS
|
||||
|
||||
network_watcher = LocalDirectoryWatcher(
|
||||
tmp_path, callback=MagicMock(), force_polling=True,
|
||||
poll_delay_ms=LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
|
||||
)
|
||||
assert network_watcher.poll_delay_ms == LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
|
||||
from watchfiles import Change
|
||||
|
||||
from app.monitor import DirectoryChangeEvent, LocalDirectoryWatcher, Monitor
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
|
||||
|
||||
class CallbackRecorder:
|
||||
@@ -28,6 +29,20 @@ class CallbackRecorder:
|
||||
self.events.append((event, text, event_path, file_size))
|
||||
|
||||
|
||||
def _build_monitor_with_dispatcher(handle_file: MagicMock = None):
|
||||
"""
|
||||
构造带分发器的测试用 Monitor 骨架。
|
||||
:param handle_file: 替换分发器 handle_file 的替身
|
||||
:return: (Monitor 骨架, 分发器)
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||
if handle_file is not None:
|
||||
dispatcher.handle_file = handle_file
|
||||
monitor._dispatcher = dispatcher
|
||||
return monitor, dispatcher
|
||||
|
||||
|
||||
def test_handle_changes_dispatches_added_and_modified_files(tmp_path):
|
||||
"""
|
||||
新增和修改文件应转换成目录监控整理回调。
|
||||
@@ -120,10 +135,8 @@ def test_event_handler_routes_file_events_to_transfer_handler():
|
||||
"""
|
||||
文件事件应继续按 local 存储交给整理流程。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
handle_file = MagicMock()
|
||||
setattr(monitor, "_Monitor__handle_file", handle_file)
|
||||
monitor, _ = _build_monitor_with_dispatcher(handle_file)
|
||||
event_path = Path("/downloads/movie.mkv")
|
||||
event = DirectoryChangeEvent(
|
||||
change_type=Change.added,
|
||||
@@ -149,10 +162,8 @@ def test_event_handler_ignores_directory_events():
|
||||
"""
|
||||
目录事件不应进入文件整理流程。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
handle_file = MagicMock()
|
||||
setattr(monitor, "_Monitor__handle_file", handle_file)
|
||||
monitor, _ = _build_monitor_with_dispatcher(handle_file)
|
||||
event_path = Path("/downloads/folder")
|
||||
event = DirectoryChangeEvent(
|
||||
change_type=Change.added,
|
||||
@@ -173,10 +184,8 @@ def test_event_handler_ignores_download_temp_files():
|
||||
"""
|
||||
下载器临时文件不应进入整理流程。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
handle_file = MagicMock()
|
||||
setattr(monitor, "_Monitor__handle_file", handle_file)
|
||||
monitor, _ = _build_monitor_with_dispatcher(handle_file)
|
||||
event_path = Path("/downloads/movie.mkv.!qB")
|
||||
event = DirectoryChangeEvent(
|
||||
change_type=Change.modified,
|
||||
@@ -198,10 +207,8 @@ def test_event_handler_ignores_non_transferable_files():
|
||||
"""
|
||||
非可整理后缀文件不应进入整理流程。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
handle_file = MagicMock()
|
||||
setattr(monitor, "_Monitor__handle_file", handle_file)
|
||||
monitor, _ = _build_monitor_with_dispatcher(handle_file)
|
||||
event_path = Path("/downloads/movie.nfo")
|
||||
event = DirectoryChangeEvent(
|
||||
change_type=Change.added,
|
||||
@@ -223,9 +230,7 @@ def test_handle_file_skips_transfer_when_history_exists(monkeypatch):
|
||||
"""
|
||||
已有整理记录的源文件不应再次进入整理链。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
monitor._cache = {}
|
||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||
event_path = Path("/downloads/movie.mkv")
|
||||
lookups = []
|
||||
|
||||
@@ -244,12 +249,12 @@ def test_handle_file_skips_transfer_when_history_exists(monkeypatch):
|
||||
transfer_chain = MagicMock()
|
||||
logger_info = MagicMock()
|
||||
logger_debug = MagicMock()
|
||||
monkeypatch.setattr("app.monitor.TransferHistoryOper", FakeTransferHistoryOper)
|
||||
monkeypatch.setattr("app.monitor.TransferChain", transfer_chain)
|
||||
monkeypatch.setattr("app.monitor.logger.info", logger_info)
|
||||
monkeypatch.setattr("app.monitor.logger.debug", logger_debug)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", FakeTransferHistoryOper)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", transfer_chain)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.logger.info", logger_info)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.logger.debug", logger_debug)
|
||||
|
||||
handled = monitor._Monitor__handle_file(
|
||||
handled = dispatcher.handle_file(
|
||||
storage="local",
|
||||
event_path=event_path,
|
||||
file_size=1024,
|
||||
@@ -266,9 +271,7 @@ def test_handle_file_invokes_transfer_when_history_missing(monkeypatch):
|
||||
"""
|
||||
没有整理记录的源文件应继续进入整理链。
|
||||
"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor.all_exts = [".mkv"]
|
||||
monitor._cache = {}
|
||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||
event_path = Path("/downloads/movie.mkv")
|
||||
|
||||
class FakeTransferHistoryOper:
|
||||
@@ -284,10 +287,10 @@ def test_handle_file_invokes_transfer_when_history_missing(monkeypatch):
|
||||
|
||||
transfer_chain_instance = MagicMock()
|
||||
transfer_chain = MagicMock(return_value=transfer_chain_instance)
|
||||
monkeypatch.setattr("app.monitor.TransferHistoryOper", FakeTransferHistoryOper)
|
||||
monkeypatch.setattr("app.monitor.TransferChain", transfer_chain)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", FakeTransferHistoryOper)
|
||||
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", transfer_chain)
|
||||
|
||||
handled = monitor._Monitor__handle_file(
|
||||
handled = dispatcher.handle_file(
|
||||
storage="local",
|
||||
event_path=event_path,
|
||||
file_size=1024,
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app import schemas
|
||||
from app.api.endpoints.plugin import plugin_rating, plugin_ratings, rate_plugin
|
||||
from app.helper.server import MoviePilotServerHelper
|
||||
|
||||
|
||||
def test_server_helper_uses_plugin_rating_endpoints() -> None:
|
||||
"""评分辅助方法应使用独立中心端路径并传递评分载荷。"""
|
||||
|
||||
async def run_scenario() -> None:
|
||||
with (
|
||||
patch("app.helper.server.settings.MP_SERVER_HOST", "https://movie-pilot.org"),
|
||||
patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"_async_get",
|
||||
new=AsyncMock(),
|
||||
) as get_request,
|
||||
patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"_async_post_json",
|
||||
new=AsyncMock(),
|
||||
) as post_request,
|
||||
):
|
||||
await MoviePilotServerHelper.async_plugin_ratings(["DemoPlugin", "OtherPlugin"])
|
||||
await MoviePilotServerHelper.async_plugin_rating("Demo Plugin")
|
||||
await MoviePilotServerHelper.async_rate_plugin("Demo Plugin", 4.5)
|
||||
|
||||
assert get_request.await_args_list[0].args == (
|
||||
"https://movie-pilot.org/plugin/rating",
|
||||
)
|
||||
assert get_request.await_args_list[0].kwargs == {
|
||||
"params": {"plugin_ids": "DemoPlugin,OtherPlugin"},
|
||||
"timeout": 10,
|
||||
}
|
||||
assert get_request.await_args_list[1].args == (
|
||||
"https://movie-pilot.org/plugin/rating/Demo%20Plugin",
|
||||
)
|
||||
assert post_request.await_args.args == (
|
||||
"https://movie-pilot.org/plugin/rating/Demo%20Plugin",
|
||||
{"rating": 4.5},
|
||||
)
|
||||
|
||||
asyncio.run(run_scenario())
|
||||
|
||||
|
||||
def test_plugin_rating_endpoints_return_center_results() -> None:
|
||||
"""评分查询和提交接口应返回中心端结果并校验安装状态。"""
|
||||
|
||||
async def run_scenario() -> None:
|
||||
rating_result = {
|
||||
"plugin_id": "DemoPlugin",
|
||||
"average_rating": 4.3,
|
||||
"rating_count": 12,
|
||||
"user_rating": 4.5,
|
||||
}
|
||||
with patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"async_get_plugin_ratings",
|
||||
new=AsyncMock(return_value={"DemoPlugin": rating_result}),
|
||||
) as batch_query:
|
||||
batch = await plugin_ratings("DemoPlugin", None)
|
||||
|
||||
assert batch["DemoPlugin"].average_rating == 4.3
|
||||
batch_query.assert_awaited_once_with(["DemoPlugin"])
|
||||
|
||||
with patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"async_get_plugin_rating",
|
||||
new=AsyncMock(return_value=rating_result),
|
||||
):
|
||||
single = await plugin_rating("DemoPlugin", None)
|
||||
|
||||
assert single.user_rating == 4.5
|
||||
|
||||
system_config = MagicMock()
|
||||
system_config.get.return_value = ["DemoPlugin"]
|
||||
with (
|
||||
patch("app.api.endpoints.plugin.SystemConfigOper", return_value=system_config),
|
||||
patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"async_submit_plugin_rating",
|
||||
new=AsyncMock(return_value=rating_result),
|
||||
) as submit_rating,
|
||||
):
|
||||
response = await rate_plugin(
|
||||
"DemoPlugin",
|
||||
schemas.PluginRatingRequest(rating=4.5),
|
||||
None,
|
||||
)
|
||||
|
||||
assert response.success is True
|
||||
assert response.data == rating_result
|
||||
submit_rating.assert_awaited_once_with("DemoPlugin", 4.5)
|
||||
|
||||
asyncio.run(run_scenario())
|
||||
|
||||
|
||||
def test_plugin_rating_rejects_uninstalled_plugin() -> None:
|
||||
"""未安装插件不能借助 MoviePilot 接口向中心端提交评分。"""
|
||||
|
||||
async def run_scenario() -> None:
|
||||
system_config = MagicMock()
|
||||
system_config.get.return_value = []
|
||||
with (
|
||||
patch("app.api.endpoints.plugin.SystemConfigOper", return_value=system_config),
|
||||
patch.object(
|
||||
MoviePilotServerHelper,
|
||||
"async_submit_plugin_rating",
|
||||
new=AsyncMock(),
|
||||
) as submit_rating,
|
||||
):
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await rate_plugin(
|
||||
"DemoPlugin",
|
||||
schemas.PluginRatingRequest(rating=4.5),
|
||||
None,
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
submit_rating.assert_not_awaited()
|
||||
|
||||
asyncio.run(run_scenario())
|
||||
@@ -0,0 +1,220 @@
|
||||
import asyncio
|
||||
import io
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from app.api.endpoints import system as system_endpoint
|
||||
from app.helper.image import ImageHelper
|
||||
|
||||
|
||||
def _image_bytes(image_format: str, trailing: bytes = b"") -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (2, 2), color=(32, 96, 160)).save(buffer, format=image_format)
|
||||
return buffer.getvalue() + trailing
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_format", "expected_mime"),
|
||||
[
|
||||
("PNG", "image/png"),
|
||||
("JPEG", "image/jpeg"),
|
||||
("GIF", "image/gif"),
|
||||
("WEBP", "image/webp"),
|
||||
("PCX", "image/x-pcx"),
|
||||
("PPM", "image/x-portable-anymap"),
|
||||
],
|
||||
)
|
||||
def test_get_image_mime_type_uses_pillow_detected_format(
|
||||
image_format: str,
|
||||
expected_mime: str,
|
||||
):
|
||||
assert ImageHelper.get_image_mime_type(_image_bytes(image_format)) == expected_mime
|
||||
|
||||
|
||||
def test_get_image_mime_type_rejects_non_image_pillow_mime():
|
||||
assert ImageHelper.get_image_mime_type(_image_bytes("EPS")) is None
|
||||
|
||||
|
||||
def test_get_image_mime_type_rejects_scriptable_svg_mime():
|
||||
with patch.dict(Image.MIME, {"PNG": "image/svg+xml"}):
|
||||
assert ImageHelper.get_image_mime_type(_image_bytes("PNG")) is None
|
||||
|
||||
|
||||
def test_fetch_image_with_mime_type_only_reads_cached_format_header():
|
||||
content = _image_bytes("PNG")
|
||||
image_helper = ImageHelper()
|
||||
|
||||
with patch.object(
|
||||
image_helper.file_cache,
|
||||
"get",
|
||||
return_value=content,
|
||||
), patch.object(
|
||||
image_helper,
|
||||
"get_image_mime_type",
|
||||
return_value="image/png",
|
||||
) as get_mime_type:
|
||||
result = image_helper.fetch_image_with_mime_type(
|
||||
"https://images.example/wallpaper.png"
|
||||
)
|
||||
|
||||
assert result == (content, "image/png")
|
||||
get_mime_type.assert_called_once_with(content, verify=False)
|
||||
|
||||
|
||||
def test_fetch_image_with_mime_type_validates_network_content_once():
|
||||
content = _image_bytes("PNG")
|
||||
image_helper = ImageHelper()
|
||||
response = Mock(status_code=200, content=content)
|
||||
request = Mock()
|
||||
request.get_res.return_value = response
|
||||
|
||||
with patch.object(
|
||||
image_helper.file_cache,
|
||||
"get",
|
||||
return_value=None,
|
||||
), patch.object(
|
||||
image_helper.file_cache,
|
||||
"set",
|
||||
), patch(
|
||||
"app.helper.image.RequestUtils",
|
||||
return_value=request,
|
||||
), patch.object(
|
||||
image_helper,
|
||||
"get_image_mime_type",
|
||||
return_value="image/png",
|
||||
) as get_mime_type:
|
||||
result = image_helper.fetch_image_with_mime_type(
|
||||
"https://images.example/wallpaper.png"
|
||||
)
|
||||
|
||||
assert result == (content, "image/png")
|
||||
get_mime_type.assert_called_once_with(content)
|
||||
|
||||
|
||||
def test_async_fetch_image_with_mime_type_only_reads_cached_format_header():
|
||||
content = _image_bytes("PNG")
|
||||
image_helper = ImageHelper()
|
||||
|
||||
with patch.object(
|
||||
image_helper.async_file_cache,
|
||||
"get",
|
||||
new=AsyncMock(return_value=content),
|
||||
), patch.object(
|
||||
image_helper,
|
||||
"get_image_mime_type",
|
||||
return_value="image/png",
|
||||
) as get_mime_type:
|
||||
result = asyncio.run(
|
||||
image_helper.async_fetch_image_with_mime_type(
|
||||
"https://images.example/wallpaper.png"
|
||||
)
|
||||
)
|
||||
|
||||
assert result == (content, "image/png")
|
||||
get_mime_type.assert_called_once_with(content, verify=False)
|
||||
|
||||
|
||||
def test_async_fetch_image_with_mime_type_validates_network_content_once():
|
||||
content = _image_bytes("PNG")
|
||||
image_helper = ImageHelper()
|
||||
response = Mock(status_code=200, content=content)
|
||||
request = Mock()
|
||||
request.get_res = AsyncMock(return_value=response)
|
||||
|
||||
with patch.object(
|
||||
image_helper.async_file_cache,
|
||||
"get",
|
||||
new=AsyncMock(return_value=None),
|
||||
), patch.object(
|
||||
image_helper.async_file_cache,
|
||||
"set",
|
||||
new=AsyncMock(),
|
||||
), patch(
|
||||
"app.helper.image.AsyncRequestUtils",
|
||||
return_value=request,
|
||||
), patch.object(
|
||||
image_helper,
|
||||
"get_image_mime_type",
|
||||
return_value="image/png",
|
||||
) as get_mime_type:
|
||||
result = asyncio.run(
|
||||
image_helper.async_fetch_image_with_mime_type(
|
||||
"https://images.example/wallpaper.png"
|
||||
)
|
||||
)
|
||||
|
||||
assert result == (content, "image/png")
|
||||
get_mime_type.assert_called_once_with(content)
|
||||
|
||||
|
||||
def test_fetch_image_does_not_trust_active_url_suffix():
|
||||
content = _image_bytes("PNG", b"<script>window.xss = true</script>")
|
||||
image_helper = Mock()
|
||||
image_helper.async_fetch_image_with_mime_type = AsyncMock(
|
||||
return_value=(content, "image/png")
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
system_endpoint.SecurityUtils,
|
||||
"is_safe_image_url_async",
|
||||
new=AsyncMock(return_value=True),
|
||||
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
|
||||
response = asyncio.run(
|
||||
system_endpoint.fetch_image(
|
||||
url="https://images.example/wallpaper.html",
|
||||
allowed_domains={"images.example"},
|
||||
)
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.headers["content-type"] == "image/png"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
assert response.body == content
|
||||
|
||||
|
||||
def test_fetch_image_rejects_unverified_content():
|
||||
image_helper = Mock()
|
||||
image_helper.async_fetch_image_with_mime_type = AsyncMock(return_value=None)
|
||||
|
||||
with patch.object(
|
||||
system_endpoint.SecurityUtils,
|
||||
"is_safe_image_url_async",
|
||||
new=AsyncMock(return_value=True),
|
||||
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
|
||||
response = asyncio.run(
|
||||
system_endpoint.fetch_image(
|
||||
url="https://images.example/wallpaper.png",
|
||||
allowed_domains={"images.example"},
|
||||
)
|
||||
)
|
||||
|
||||
assert response is None
|
||||
|
||||
|
||||
def test_fetch_image_adds_nosniff_to_not_modified_response():
|
||||
content = _image_bytes("JPEG")
|
||||
image_helper = Mock()
|
||||
image_helper.async_fetch_image_with_mime_type = AsyncMock(
|
||||
return_value=(content, "image/jpeg")
|
||||
)
|
||||
etag = system_endpoint.HashUtils.md5(content)
|
||||
|
||||
with patch.object(
|
||||
system_endpoint.SecurityUtils,
|
||||
"is_safe_image_url_async",
|
||||
new=AsyncMock(return_value=True),
|
||||
), patch.object(system_endpoint, "ImageHelper", return_value=image_helper):
|
||||
response = asyncio.run(
|
||||
system_endpoint.fetch_image(
|
||||
url="https://images.example/wallpaper.jpg",
|
||||
if_none_match=etag,
|
||||
allowed_domains={"images.example"},
|
||||
)
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.status_code == 304
|
||||
assert response.headers["content-type"] == "image/jpeg"
|
||||
assert response.headers["x-content-type-options"] == "nosniff"
|
||||
@@ -118,6 +118,8 @@ class LlmTestEndpointTest(unittest.TestCase):
|
||||
system_endpoint.settings, "LLM_USER_AGENT", "MoviePilot-Test/1.0"
|
||||
), patch.object(
|
||||
system_endpoint.settings, "LLM_USE_PROXY", True
|
||||
), patch.object(
|
||||
system_endpoint.settings, "LLM_API_PROTOCOL", "responses"
|
||||
), patch.object(
|
||||
system_endpoint.LLMHelper,
|
||||
"test_current_settings",
|
||||
@@ -135,6 +137,7 @@ class LlmTestEndpointTest(unittest.TestCase):
|
||||
base_url_preset="deepseek-default",
|
||||
user_agent="MoviePilot-Test/1.0",
|
||||
use_proxy=True,
|
||||
api_protocol="responses",
|
||||
)
|
||||
self.assertTrue(resp.success)
|
||||
self.assertEqual(resp.data["provider"], "deepseek")
|
||||
@@ -186,6 +189,7 @@ class LlmTestEndpointTest(unittest.TestCase):
|
||||
base_url_preset="openai-default",
|
||||
user_agent="MoviePilot-Custom/1.0",
|
||||
use_proxy=False,
|
||||
api_protocol=None,
|
||||
)
|
||||
self.assertTrue(resp.success)
|
||||
self.assertEqual(resp.data["provider"], "openai")
|
||||
@@ -228,6 +232,7 @@ class LlmTestEndpointTest(unittest.TestCase):
|
||||
base_url_preset="deepseek-default",
|
||||
user_agent=None,
|
||||
use_proxy=None,
|
||||
api_protocol=None,
|
||||
)
|
||||
self.assertTrue(resp.success)
|
||||
|
||||
|
||||
@@ -88,7 +88,9 @@ class NettestSecurityTest(unittest.TestCase):
|
||||
image_url = "http://192.168.1.50:8096/System/Info/Public"
|
||||
signed_url = system_endpoint.SecurityUtils.sign_url(image_url)
|
||||
image_helper = Mock()
|
||||
image_helper.async_fetch_image = AsyncMock(return_value=b"image-bytes")
|
||||
image_helper.async_fetch_image_with_mime_type = AsyncMock(
|
||||
return_value=(b"image-bytes", "image/jpeg")
|
||||
)
|
||||
|
||||
with patch.object(system_endpoint, "ImageHelper", return_value=image_helper), patch.object(
|
||||
system_endpoint.HashUtils, "md5", return_value="etag", create=True
|
||||
@@ -103,7 +105,7 @@ class NettestSecurityTest(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
image_helper.async_fetch_image.assert_awaited_once_with(
|
||||
image_helper.async_fetch_image_with_mime_type.assert_awaited_once_with(
|
||||
url=image_url,
|
||||
proxy=None,
|
||||
use_cache=False,
|
||||
@@ -133,7 +135,9 @@ class NettestSecurityTest(unittest.TestCase):
|
||||
图片代理在域名白名单命中后,可按配置放行指定非公网解析网段。
|
||||
"""
|
||||
image_helper = Mock()
|
||||
image_helper.async_fetch_image = AsyncMock(return_value=b"image-bytes")
|
||||
image_helper.async_fetch_image_with_mime_type = AsyncMock(
|
||||
return_value=(b"image-bytes", "image/jpeg")
|
||||
)
|
||||
|
||||
with patch.object(system_endpoint, "ImageHelper", return_value=image_helper), patch.object(
|
||||
system_endpoint.HashUtils, "md5", return_value="etag", create=True
|
||||
@@ -161,7 +165,7 @@ class NettestSecurityTest(unittest.TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
image_helper.async_fetch_image.assert_awaited_once_with(
|
||||
image_helper.async_fetch_image_with_mime_type.assert_awaited_once_with(
|
||||
url="https://img1.doubanio.com/poster.webp",
|
||||
proxy=None,
|
||||
use_cache=False,
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
def test_increment_serializes_concurrent_counter_updates(monkeypatch):
|
||||
"""并发递增系统计数时不应丢失更新。"""
|
||||
oper = object.__new__(SystemConfigOper)
|
||||
oper._rlock = threading.RLock()
|
||||
stored_value = {"value": 0}
|
||||
|
||||
monkeypatch.setattr(oper, "get", lambda _key: stored_value["value"])
|
||||
monkeypatch.setattr(
|
||||
oper,
|
||||
"set",
|
||||
lambda _key, value: stored_value.update(value=value),
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
results = list(
|
||||
executor.map(
|
||||
lambda _index: oper.increment(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
),
|
||||
range(100),
|
||||
)
|
||||
)
|
||||
|
||||
assert sorted(results) == list(range(1, 101))
|
||||
assert stored_value["value"] == 100
|
||||
|
||||
|
||||
def test_increment_supports_custom_step(monkeypatch):
|
||||
"""整数系统计数应支持指定递增步长。"""
|
||||
oper = object.__new__(SystemConfigOper)
|
||||
oper._rlock = threading.RLock()
|
||||
stored_value = {"value": 4}
|
||||
|
||||
monkeypatch.setattr(oper, "get", lambda _key: stored_value["value"])
|
||||
monkeypatch.setattr(
|
||||
oper,
|
||||
"set",
|
||||
lambda _key, value: stored_value.update(value=value),
|
||||
)
|
||||
|
||||
result = oper.increment(SystemConfigKey.MediaRecognizeShareCount, step=3)
|
||||
|
||||
assert result == 7
|
||||
assert stored_value["value"] == 7
|
||||
@@ -1,10 +1,11 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
from unittest.mock import Mock
|
||||
|
||||
from app.api.endpoints import tmdb as tmdb_endpoint
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||
from app.schemas.types import MediaType
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
|
||||
|
||||
class _MemoryCacheStub:
|
||||
@@ -97,7 +98,14 @@ def test_tmdb_cache_endpoint_returns_management_statistics(monkeypatch):
|
||||
"recognized": {"id": 1, "title": "Alpha", "type": MediaType.MOVIE},
|
||||
"unrecognized": {"id": 0},
|
||||
})
|
||||
get_system_config = Mock(return_value=7)
|
||||
monkeypatch.setattr(tmdb_endpoint, "TmdbCache", lambda: cache)
|
||||
monkeypatch.setattr(
|
||||
tmdb_endpoint,
|
||||
"SystemConfigOper",
|
||||
lambda: type("SystemConfigStub", (), {"get": get_system_config})(),
|
||||
)
|
||||
monkeypatch.setattr(tmdb_endpoint.settings, "MEDIA_RECOGNIZE_SHARE", True)
|
||||
|
||||
response = asyncio.run(tmdb_endpoint.tmdb_recognition_cache(None))
|
||||
|
||||
@@ -105,6 +113,11 @@ def test_tmdb_cache_endpoint_returns_management_statistics(monkeypatch):
|
||||
assert response.data["count"] == 2
|
||||
assert response.data["recognized"] == 1
|
||||
assert response.data["unrecognized"] == 1
|
||||
assert response.data["shared_recognized"] == 7
|
||||
assert response.data["shared_recognize_enabled"] is True
|
||||
get_system_config.assert_called_once_with(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
)
|
||||
|
||||
|
||||
def test_tmdb_cache_delete_endpoint_reports_missing_item(monkeypatch):
|
||||
|
||||
@@ -183,6 +183,7 @@ class TransferJobManagerTest(unittest.TestCase):
|
||||
target_oper = SimpleNamespace(
|
||||
get_folder=lambda path: target_folder,
|
||||
get_item=lambda path: None,
|
||||
get_item_strict=lambda path: None,
|
||||
)
|
||||
|
||||
new_item, errmsg = TransHandler._TransHandler__transfer_command(
|
||||
@@ -243,6 +244,7 @@ class TransferJobManagerTest(unittest.TestCase):
|
||||
target_oper = SimpleNamespace(
|
||||
get_folder=lambda path: target_folder,
|
||||
get_item=lambda path: None,
|
||||
get_item_strict=lambda path: None,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
@@ -313,6 +315,7 @@ class TransferJobManagerTest(unittest.TestCase):
|
||||
target_oper = SimpleNamespace(
|
||||
get_folder=lambda path: target_folder,
|
||||
get_item=lambda path: None,
|
||||
get_item_strict=lambda path: None,
|
||||
)
|
||||
in_meta = MetaVideo("Test.Show.S02E03")
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.schemas import FileItem, TransferDirectoryConf, TransferTask
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
def _make_task(
|
||||
storage: str = "local",
|
||||
download_path: str = "/mnt/clouddrive/downloads",
|
||||
) -> TransferTask:
|
||||
return TransferTask(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=f"{download_path}/Test.Show.S01E01.mkv",
|
||||
type="file",
|
||||
name="Test.Show.S01E01.mkv",
|
||||
),
|
||||
target_directory=TransferDirectoryConf(
|
||||
storage=storage,
|
||||
download_path=download_path,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_enabled_cleanup_skips_filesystem_detection():
|
||||
"""
|
||||
开关开启时应保持旧行为,且不产生额外文件系统检测。
|
||||
"""
|
||||
with patch(
|
||||
"app.chain.transfer.SystemUtils.is_network_filesystem"
|
||||
) as is_network_filesystem:
|
||||
should_delete = (
|
||||
TransferChain._TransferChain__should_delete_empty_source_directories(
|
||||
_make_task(),
|
||||
True,
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
assert should_delete is True
|
||||
is_network_filesystem.assert_not_called()
|
||||
|
||||
|
||||
def test_disabled_cleanup_keeps_mounted_local_source_directories():
|
||||
"""
|
||||
开关关闭时应保留网络或 FUSE 挂载的本地源目录。
|
||||
"""
|
||||
with patch(
|
||||
"app.chain.transfer.SystemUtils.is_network_filesystem",
|
||||
return_value=True,
|
||||
) as is_network_filesystem:
|
||||
should_delete = (
|
||||
TransferChain._TransferChain__should_delete_empty_source_directories(
|
||||
_make_task(),
|
||||
False,
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
assert should_delete is False
|
||||
is_network_filesystem.assert_called_once_with(
|
||||
Path("/mnt/clouddrive/downloads"), include_local_fuse=True
|
||||
)
|
||||
|
||||
|
||||
def test_disabled_cleanup_still_deletes_ordinary_local_source_directories():
|
||||
"""
|
||||
开关关闭时普通本地文件系统仍应删除空目录。
|
||||
"""
|
||||
with patch(
|
||||
"app.chain.transfer.SystemUtils.is_network_filesystem",
|
||||
return_value=False,
|
||||
):
|
||||
should_delete = (
|
||||
TransferChain._TransferChain__should_delete_empty_source_directories(
|
||||
_make_task(download_path="/downloads"),
|
||||
False,
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
assert should_delete is True
|
||||
|
||||
|
||||
def test_disabled_cleanup_does_not_change_remote_storage_cleanup():
|
||||
"""
|
||||
开关关闭时非本地存储仍应执行原有空目录清理。
|
||||
"""
|
||||
with patch(
|
||||
"app.chain.transfer.SystemUtils.is_network_filesystem"
|
||||
) as is_network_filesystem:
|
||||
should_delete = (
|
||||
TransferChain._TransferChain__should_delete_empty_source_directories(
|
||||
_make_task(storage="alist", download_path="/downloads"),
|
||||
False,
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
assert should_delete is True
|
||||
is_network_filesystem.assert_not_called()
|
||||
|
||||
|
||||
def test_mounted_filesystem_detection_is_cached_by_source_directory():
|
||||
"""
|
||||
同一源根目录的批量任务应只检测一次文件系统。
|
||||
"""
|
||||
mounted_filesystem_cache = {}
|
||||
with patch(
|
||||
"app.chain.transfer.SystemUtils.is_network_filesystem",
|
||||
return_value=True,
|
||||
) as is_network_filesystem:
|
||||
for _ in range(2):
|
||||
should_delete = (
|
||||
TransferChain._TransferChain__should_delete_empty_source_directories(
|
||||
_make_task(),
|
||||
False,
|
||||
mounted_filesystem_cache,
|
||||
)
|
||||
)
|
||||
assert should_delete is False
|
||||
|
||||
is_network_filesystem.assert_called_once_with(
|
||||
Path("/mnt/clouddrive/downloads"), include_local_fuse=True
|
||||
)
|
||||
|
||||
|
||||
def test_cleanup_detection_includes_local_fuse_mounts():
|
||||
"""
|
||||
空目录清理场景应将原本排除的本地 FUSE 文件系统视为挂载盘。
|
||||
"""
|
||||
df_result = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="Filesystem Type 1K-blocks Used Available Use% Mounted on\n"
|
||||
"shfs fuse.shfs 1 1 1 1% /mnt/user\n",
|
||||
)
|
||||
with patch("app.utils.system.platform.system", return_value="Linux"), patch(
|
||||
"app.utils.system.subprocess.run", return_value=df_result
|
||||
):
|
||||
assert SystemUtils.is_network_filesystem(Path("/mnt/user")) is False
|
||||
assert SystemUtils.is_network_filesystem(
|
||||
Path("/mnt/user"), include_local_fuse=True
|
||||
) is True
|
||||
@@ -0,0 +1,186 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.modules.filemanager.storages.alipan import AliPan
|
||||
from app.modules.filemanager.storages.local import LocalStorage
|
||||
from app.modules.filemanager.storages.rclone import Rclone
|
||||
from app.modules.filemanager.storages.u115 import U115Pan
|
||||
from app.schemas.exception import StorageQueryError
|
||||
|
||||
|
||||
def _local() -> LocalStorage:
|
||||
"""
|
||||
构造本地存储实例(跳过初始化)。
|
||||
"""
|
||||
return object.__new__(LocalStorage)
|
||||
|
||||
|
||||
def _u115() -> U115Pan:
|
||||
"""
|
||||
构造 115 存储实例(跳过初始化)。
|
||||
"""
|
||||
return object.__new__(U115Pan)
|
||||
|
||||
|
||||
def _alipan(monkeypatch) -> AliPan:
|
||||
"""
|
||||
构造阿里云盘存储实例(跳过初始化,_default_drive_id 为只读属性需在类级替换)。
|
||||
"""
|
||||
monkeypatch.setattr(AliPan, "_default_drive_id", "drive-1", raising=False)
|
||||
return object.__new__(AliPan)
|
||||
|
||||
|
||||
def test_local_strict_missing_file_returns_none(tmp_path):
|
||||
"""
|
||||
目标文件确实不存在时应确认为不存在,允许正常整理。
|
||||
"""
|
||||
assert _local().get_item_strict(tmp_path / "missing.mkv") is None
|
||||
|
||||
|
||||
def test_local_strict_existing_file_returns_item(tmp_path):
|
||||
"""
|
||||
目标文件存在时应返回文件项。
|
||||
"""
|
||||
target = tmp_path / "movie.mkv"
|
||||
target.write_bytes(b"movie")
|
||||
|
||||
item = _local().get_item_strict(target)
|
||||
|
||||
assert item is not None
|
||||
assert item.path == target.as_posix()
|
||||
|
||||
|
||||
def test_local_strict_broken_symlink_returns_none(tmp_path):
|
||||
"""
|
||||
失效软链接视为目标不存在,不应阻断整理。
|
||||
"""
|
||||
target = tmp_path / "movie.mkv"
|
||||
target.symlink_to(tmp_path / "gone.mkv")
|
||||
|
||||
assert _local().get_item_strict(target) is None
|
||||
|
||||
|
||||
def test_local_strict_raises_on_stat_error(tmp_path, monkeypatch):
|
||||
"""
|
||||
FUSE 挂载抖动导致 stat 失败时应抛出 StorageQueryError,拒绝覆盖。
|
||||
"""
|
||||
target = tmp_path / "movie.mkv"
|
||||
|
||||
def raise_stat_error(self, *args, **kwargs):
|
||||
"""
|
||||
模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE。
|
||||
"""
|
||||
raise OSError(131, "State not recoverable")
|
||||
|
||||
monkeypatch.setattr(Path, "stat", raise_stat_error)
|
||||
|
||||
with pytest.raises(StorageQueryError):
|
||||
_local().get_item_strict(target)
|
||||
|
||||
|
||||
def test_u115_strict_transport_failure_raises():
|
||||
"""
|
||||
115 请求失败(网络/限流重试用尽)时应抛出 StorageQueryError。
|
||||
"""
|
||||
storage = _u115()
|
||||
storage._request_api = MagicMock(return_value=None)
|
||||
|
||||
with pytest.raises(StorageQueryError):
|
||||
storage.get_item_strict(Path("/movie.mkv"))
|
||||
|
||||
|
||||
def test_u115_get_item_keeps_swallowing_transport_failure():
|
||||
"""
|
||||
宽松版 get_item 行为保持兼容:请求失败仍返回 None。
|
||||
"""
|
||||
storage = _u115()
|
||||
storage._request_api = MagicMock(return_value=None)
|
||||
|
||||
assert storage.get_item(Path("/movie.mkv")) is None
|
||||
|
||||
|
||||
def test_u115_strict_confirmed_absent_returns_none():
|
||||
"""
|
||||
115 业务码返回记录不存在(data 为空)时应确认为不存在。
|
||||
"""
|
||||
storage = _u115()
|
||||
storage._request_api = MagicMock(return_value={"state": True, "code": 20004, "data": {}})
|
||||
|
||||
assert storage.get_item_strict(Path("/movie.mkv")) is None
|
||||
|
||||
|
||||
def test_u115_strict_returns_item():
|
||||
"""
|
||||
115 返回有效文件数据时应构造文件项。
|
||||
"""
|
||||
storage = _u115()
|
||||
storage._request_api = MagicMock(return_value={"state": True, "code": 0, "data": {
|
||||
"file_id": 123,
|
||||
"file_category": "1",
|
||||
"file_name": "movie.mkv",
|
||||
"pick_code": "abc",
|
||||
"size_byte": 1024,
|
||||
"utime": 100,
|
||||
}})
|
||||
|
||||
item = storage.get_item_strict(Path("/movie.mkv"))
|
||||
|
||||
assert item is not None
|
||||
assert item.fileid == "123"
|
||||
assert item.size == 1024
|
||||
|
||||
|
||||
def test_alipan_strict_notfound_returns_none(monkeypatch):
|
||||
"""
|
||||
阿里云盘 NotFound 系列错误码应确认为不存在。
|
||||
"""
|
||||
storage = _alipan(monkeypatch)
|
||||
storage._request_api = MagicMock(return_value={"code": "NotFound.File", "message": "not found"})
|
||||
|
||||
assert storage.get_item_strict(Path("/movie.mkv")) is None
|
||||
|
||||
|
||||
def test_alipan_strict_other_error_raises(monkeypatch):
|
||||
"""
|
||||
阿里云盘非 NotFound 的业务错误(如限流)应抛出 StorageQueryError。
|
||||
"""
|
||||
storage = _alipan(monkeypatch)
|
||||
storage._request_api = MagicMock(return_value={"code": "TooManyRequests", "message": "limit"})
|
||||
|
||||
with pytest.raises(StorageQueryError):
|
||||
storage.get_item_strict(Path("/movie.mkv"))
|
||||
|
||||
|
||||
def test_alipan_strict_transport_failure_raises(monkeypatch):
|
||||
"""
|
||||
阿里云盘请求失败时应抛出 StorageQueryError。
|
||||
"""
|
||||
storage = _alipan(monkeypatch)
|
||||
storage._request_api = MagicMock(return_value=None)
|
||||
|
||||
with pytest.raises(StorageQueryError):
|
||||
storage.get_item_strict(Path("/movie.mkv"))
|
||||
|
||||
|
||||
def test_alipan_strict_returns_item(monkeypatch):
|
||||
"""
|
||||
阿里云盘返回有效数据时应构造文件项。
|
||||
"""
|
||||
storage = _alipan(monkeypatch)
|
||||
storage._request_api = MagicMock(return_value={"file_id": "f1", "name": "movie.mkv"})
|
||||
setattr(storage, "_AliPan__get_fileitem", MagicMock(return_value="ITEM"))
|
||||
|
||||
assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM"
|
||||
|
||||
|
||||
def test_storage_base_strict_defaults_to_get_item():
|
||||
"""
|
||||
未覆写的存储沿用 get_item 判定,行为不变。
|
||||
"""
|
||||
storage = object.__new__(Rclone)
|
||||
storage.get_item = MagicMock(return_value=None)
|
||||
|
||||
assert storage.get_item_strict(Path("/movie.mkv")) is None
|
||||
storage.get_item.assert_called_once()
|
||||
+2
-2
@@ -1,2 +1,2 @@
|
||||
APP_VERSION = 'v2.15.1'
|
||||
FRONTEND_VERSION = 'v2.15.1'
|
||||
APP_VERSION = 'v2.15.2'
|
||||
FRONTEND_VERSION = 'v2.15.2'
|
||||
|
||||
Reference in New Issue
Block a user