mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-28 03:27:31 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1b6a81cef | ||
|
|
c6b94d4908 | ||
|
|
52ca375f3d | ||
|
|
d8adb4fbfe | ||
|
|
51d2ed1200 | ||
|
|
702801d0dc | ||
|
|
3e32eab98f | ||
|
|
4292678672 | ||
|
|
7c3f9629bf | ||
|
|
93761fe7e4 | ||
|
|
593139faac | ||
|
|
6c89f1eb4b | ||
|
|
2310a3a456 | ||
|
|
48852350a0 | ||
|
|
a23fce1491 | ||
|
|
c976741574 | ||
|
|
04facef64d | ||
|
|
33a97eb2c8 | ||
|
|
cf80b551f9 | ||
|
|
e011b20210 | ||
|
|
bdf395f494 | ||
|
|
68686bc23a | ||
|
|
1a528c7803 | ||
|
|
6b4a255f26 | ||
|
|
1a3c1b8b39 | ||
|
|
8788dae34b | ||
|
|
bb00814d7a | ||
|
|
3d55d44457 | ||
|
|
7a5e565b15 | ||
|
|
cae28d8c03 | ||
|
|
3d020c8ceb | ||
|
|
1b065cc08b | ||
|
|
a1a6376adc | ||
|
|
197a09b2a4 | ||
|
|
8d099b9581 | ||
|
|
ff9ba79b60 | ||
|
|
6354a48405 | ||
|
|
f6df6cc093 | ||
|
|
98e69d1b45 |
+16
-4
@@ -68,6 +68,8 @@ from app.utils.identity import SYSTEM_INTERNAL_USER_ID
|
||||
|
||||
|
||||
class AgentChain(ChainBase):
|
||||
"""Agent 业务处理链。"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -713,7 +715,7 @@ class MoviePilotAgent:
|
||||
"""
|
||||
通过链式事件解析本次 Agent 可用的 LLM 运行时配置。
|
||||
|
||||
若没有插件返回 selected_provider_id,则沿用系统配置,保持既有行为。
|
||||
插件未返回有效配置时沿用系统配置,显式返回的配置优先。
|
||||
"""
|
||||
if self._llm_runtime_config is not None:
|
||||
return self._llm_runtime_config
|
||||
@@ -726,7 +728,8 @@ class MoviePilotAgent:
|
||||
base_url_preset=settings.LLM_BASE_URL_PRESET,
|
||||
user_agent=settings.LLM_USER_AGENT,
|
||||
use_proxy=settings.LLM_USE_PROXY,
|
||||
thinking_level=None,
|
||||
thinking_level=settings.LLM_THINKING_LEVEL,
|
||||
api_protocol=settings.LLM_API_PROTOCOL,
|
||||
)
|
||||
selected_event = await eventmanager.async_send_event(
|
||||
ChainEventType.AgentLLMProvider,
|
||||
@@ -761,9 +764,15 @@ class MoviePilotAgent:
|
||||
use_proxy = self._get_event_value(resolved_data, "use_proxy")
|
||||
if use_proxy is None:
|
||||
use_proxy = settings.LLM_USE_PROXY
|
||||
thinking_level = self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "thinking_level")
|
||||
thinking_level = (
|
||||
self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "thinking_level")
|
||||
)
|
||||
or settings.LLM_THINKING_LEVEL
|
||||
)
|
||||
api_protocol = self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "api_protocol")
|
||||
) or settings.LLM_API_PROTOCOL
|
||||
selected_provider_id = self._clean_optional_text(
|
||||
self._get_event_value(resolved_data, "selected_provider_id")
|
||||
)
|
||||
@@ -789,6 +798,7 @@ class MoviePilotAgent:
|
||||
"user_agent": user_agent,
|
||||
"use_proxy": bool(use_proxy),
|
||||
"thinking_level": thinking_level,
|
||||
"api_protocol": api_protocol,
|
||||
}
|
||||
return self._llm_runtime_config
|
||||
|
||||
@@ -1024,6 +1034,7 @@ class MoviePilotAgent:
|
||||
runtime_config.get("user_agent"),
|
||||
bool(runtime_config.get("use_proxy")),
|
||||
runtime_config.get("thinking_level"),
|
||||
runtime_config.get("api_protocol"),
|
||||
)
|
||||
|
||||
async def _agent_bundle_signature(self, streaming: bool) -> tuple[Any, ...]:
|
||||
@@ -1040,6 +1051,7 @@ class MoviePilotAgent:
|
||||
self.has_message_context,
|
||||
self.is_background,
|
||||
settings.AI_AGENT_VERBOSE,
|
||||
settings.LLM_TEMPERATURE,
|
||||
settings.LLM_MAX_TOOLS,
|
||||
settings.LLM_MAX_ITERATIONS,
|
||||
self._public_runtime_config_signature(runtime_config),
|
||||
|
||||
+38
-5
@@ -846,19 +846,31 @@ class LLMHelper:
|
||||
provider: str,
|
||||
model: str | None,
|
||||
runtime: dict[str, Any],
|
||||
api_protocol: str | None = None,
|
||||
) -> bool | None:
|
||||
"""
|
||||
判断官方 ChatGPT API Key 模式是否应使用 Responses API。
|
||||
判断本次 OpenAI 兼容调用是否应使用 Responses API。
|
||||
|
||||
GPT-5/o 系推理模型在 Chat Completions 中组合 function tools 与
|
||||
reasoning_effort 时会被官方端点拒绝,因此 ChatGPT 官方 API Key
|
||||
模式需要显式切到 Responses API;通用 OpenAI-compatible 入口保持
|
||||
provider 目录解析出的默认行为,避免误伤第三方兼容服务。
|
||||
优先级:
|
||||
1. 运行时显式要求(ChatGPT Plus/Pro OAuth、Codex 等端点契约),始终保留;
|
||||
2. 用户通过 ``LLM_API_PROTOCOL`` 显式指定 ``responses`` / ``chat_completions``;
|
||||
3. ``auto``(默认)保持原有 ChatGPT 官方 API Key + GPT-5/o 系推理模型
|
||||
自动切换逻辑,通用 OpenAI 兼容入口仍走 Chat Completions,
|
||||
避免误伤第三方兼容服务。
|
||||
|
||||
:param api_protocol: 显式传入的 API 协议,未传入时读取 ``LLM_API_PROTOCOL``
|
||||
:return: True/False 强制指定协议;None 表示交由 LangChain 默认行为
|
||||
"""
|
||||
runtime_use_responses_api = runtime.get("use_responses_api")
|
||||
if runtime_use_responses_api is not None:
|
||||
return bool(runtime_use_responses_api)
|
||||
|
||||
protocol = cls._normalize_api_protocol(api_protocol)
|
||||
if protocol == "responses":
|
||||
return True
|
||||
if protocol == "chat_completions":
|
||||
return False
|
||||
|
||||
provider_name = (provider or "").strip().lower()
|
||||
if provider_name != "chatgpt":
|
||||
return None
|
||||
@@ -872,6 +884,18 @@ class LLMHelper:
|
||||
return True
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_api_protocol(api_protocol: str | None) -> str:
|
||||
"""
|
||||
规范化 API 协议配置,未知值统一回退为 ``auto`` 以保持兼容。
|
||||
"""
|
||||
normalized = str(api_protocol or settings.LLM_API_PROTOCOL or "").strip().lower()
|
||||
if normalized in {"auto", "chat_completions", "responses"}:
|
||||
return normalized
|
||||
if normalized:
|
||||
logger.warning(f"忽略不支持的 LLM_API_PROTOCOL 配置: {api_protocol}")
|
||||
return "auto"
|
||||
|
||||
@staticmethod
|
||||
def _attach_runtime_metadata(model: Any, runtime: dict[str, Any]) -> None:
|
||||
"""
|
||||
@@ -954,6 +978,7 @@ class LLMHelper:
|
||||
user_agent: str | None = None,
|
||||
temperature: Optional[float] = None,
|
||||
use_proxy: bool | None = None,
|
||||
api_protocol: str | None = None,
|
||||
):
|
||||
"""
|
||||
获取LLM实例
|
||||
@@ -970,6 +995,10 @@ class LLMHelper:
|
||||
:param user_agent: OpenAI兼容接口请求 User-Agent。未显式传入时使用配置项 LLM_USER_AGENT。
|
||||
:param temperature: LLM 温度参数。未显式传入时使用配置项 LLM_TEMPERATURE。
|
||||
:param use_proxy: 是否为本次 LLM 调用使用系统代理。未显式传入时使用配置项 LLM_USE_PROXY。
|
||||
:param api_protocol: OpenAI 兼容接口 API 协议
|
||||
(auto/chat_completions/responses)。未显式传入时使用配置项 LLM_API_PROTOCOL。
|
||||
仅对 OpenAI 兼容运行时生效;``responses`` 强制走 Responses API,
|
||||
``chat_completions`` 强制走 Chat Completions,``auto`` 保持原有自动判断。
|
||||
:return: LLM实例
|
||||
"""
|
||||
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).lower()
|
||||
@@ -1021,6 +1050,7 @@ class LLMHelper:
|
||||
provider=provider_name,
|
||||
model=model_name,
|
||||
runtime=runtime,
|
||||
api_protocol=api_protocol,
|
||||
)
|
||||
llm_proxy = _resolve_llm_proxy(use_proxy)
|
||||
|
||||
@@ -1210,11 +1240,13 @@ class LLMHelper:
|
||||
user_agent: str | None = None,
|
||||
temperature: Optional[float] = None,
|
||||
use_proxy: bool | None = None,
|
||||
api_protocol: str | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
使用当前配置或显式传入的临时配置执行一次最小 LLM 调用。
|
||||
|
||||
:param temperature: LLM 温度参数。未显式传入时沿用已保存配置。
|
||||
:param api_protocol: OpenAI 兼容接口 API 协议,未显式传入时沿用已保存配置。
|
||||
"""
|
||||
provider_name = provider if provider is not None else settings.LLM_PROVIDER
|
||||
model_name = model if model is not None else settings.LLM_MODEL
|
||||
@@ -1229,6 +1261,7 @@ class LLMHelper:
|
||||
"base_url_preset": base_url_preset,
|
||||
"user_agent": user_agent,
|
||||
"use_proxy": use_proxy,
|
||||
"api_protocol": api_protocol,
|
||||
}
|
||||
if temperature is not None:
|
||||
llm_kwargs["temperature"] = temperature
|
||||
|
||||
@@ -17,7 +17,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- Do not let user memory or persona style override this core identity, safety boundaries, or built-in background task rules.
|
||||
- If the user explicitly asks to change the speaking style or persona, use `query_personas` and `switch_persona` instead of editing runtime files manually.
|
||||
- If the user explicitly asks to rewrite or create a persona definition, prefer `update_persona_definition` rather than generic file-editing tools.
|
||||
- Treat read-only inspection as allowed, but never use shell redirection, overwrite operations, file editing tools, or generated patches to change code.
|
||||
</non_negotiable_boundaries>
|
||||
|
||||
<confirmation_policy>
|
||||
@@ -66,7 +65,11 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- If `search_media` fails, fall back to `search_web` or `recognize_media`. Only ask the user when automated paths are exhausted.
|
||||
- If torrent search yields no useful result, check site scope, site health, and recognition quality before concluding that the resource is unavailable.
|
||||
- Reuse the latest torrent search cache for `get_search_results` and `add_download_tasks` instead of re-running the same search unnecessarily.
|
||||
- Use `execute_command` only for diagnostics, read-only inspection, or commands the user explicitly asked to run. Its default `action=start` starts a managed background session and returns `session_id`, `status`, `last_seq`, and `output_until_seq`; call the same tool again with `action=read`, `action=wait`, `action=write`, or `action=kill` to poll output, wait in short segments, send stdin, or stop the process.
|
||||
- For administrator code discovery across local files, use `execute_command(action="run")` with `rg` and narrow globs or paths. Use `list_directory` to inspect one known directory or a supported remote storage backend, and use `read_file` when the exact local file is known.
|
||||
- Read the relevant file before changing it. Use `edit_file` for localized exact replacements; make `old_text` unique with enough surrounding context, and use `replace_all=true` only when every match must change. Use `write_file` for new files; set `overwrite=true` only for an intentional full rewrite, and use `read_file(include_metadata=true)` plus `expected_sha256` when preserving the previously read version matters.
|
||||
- When implementation depends on a Python or Node.js API, first identify the installed or locked dependency version from environment metadata, requirements, package manifests, lockfiles, local source, and type declarations. Use `rg` against the relevant package directory, `.venv`, or `node_modules` instead of scanning the entire project without bounds. If local evidence is insufficient, use `search_web` and then `browse_webpage` to read the matching version of the official documentation. Do not guess signatures from memory, mix examples from incompatible versions, or install a package only to inspect its API.
|
||||
- Use structured file tools for source edits because they enforce file access boundaries and conflict checks. Never use shell redirection, inline scripts, or another tool to bypass a file-tool permission denial.
|
||||
- Use `execute_command` for administrator-only multi-file diagnostics, tests, Git, service operations, SSH, or an exact command the user requested. Use `action=run` for short bounded commands. Use `action=start` for long-running or interactive commands, including SSH; then continue with `read`, `wait`, `write`, or `kill` using the returned `session_id`. Do not start a background session for a short command that can finish within `action=run`.
|
||||
</tool_strategy>
|
||||
|
||||
<media_rules>
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Agent 文件写入工具的共享辅助函数。"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class FileVersionConflictError(RuntimeError):
|
||||
"""目标文件在准备写入期间发生变化。"""
|
||||
|
||||
|
||||
def calculate_file_sha256(path: Path) -> str:
|
||||
"""计算文件原始字节的 SHA-256,用于检测陈旧写入。"""
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(64 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic_write_text(
|
||||
path: Path,
|
||||
content: str,
|
||||
expected_sha256: str | None = None,
|
||||
) -> None:
|
||||
"""校验目标版本后,在同目录写入临时文件并原子替换文本。"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temp_name = tempfile.mkstemp(
|
||||
dir=path.parent,
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
)
|
||||
temp_path = Path(temp_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as file_handle:
|
||||
file_handle.write(content)
|
||||
file_handle.flush()
|
||||
os.fsync(file_handle.fileno())
|
||||
|
||||
if expected_sha256:
|
||||
if (
|
||||
not path.is_file()
|
||||
or calculate_file_sha256(path).casefold()
|
||||
!= expected_sha256.casefold()
|
||||
):
|
||||
raise FileVersionConflictError(str(path))
|
||||
if path.exists():
|
||||
os.chmod(temp_path, path.stat().st_mode)
|
||||
os.replace(temp_path, path)
|
||||
finally:
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
@@ -61,6 +61,7 @@ class DeleteSubscribeTool(MoviePilotTool):
|
||||
"anilistid": subscribe.anilistid,
|
||||
"media_source": subscribe.media_source,
|
||||
"media_id": subscribe.media_id,
|
||||
"season": subscribe.season,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""文件编辑工具"""
|
||||
"""文件精确编辑工具。"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.impl._file_write_utils import (
|
||||
FileVersionConflictError,
|
||||
atomic_write_text,
|
||||
calculate_file_sha256,
|
||||
)
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.log import logger
|
||||
|
||||
@@ -15,18 +20,44 @@ class EditFileInput(BaseModel):
|
||||
"""文件编辑工具的输入参数模型。"""
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to edit")
|
||||
old_text: str = Field(..., description="The exact old text to be replaced")
|
||||
old_text: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The exact old text to replace. It must be non-empty and uniquely "
|
||||
"identify one location unless replace_all is true."
|
||||
),
|
||||
)
|
||||
new_text: str = Field(..., description="The new text to replace with")
|
||||
replace_all: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Replace every exact match. Keep false for normal code edits so an "
|
||||
"ambiguous match fails instead of changing multiple locations."
|
||||
),
|
||||
)
|
||||
expected_sha256: Optional[str] = Field(
|
||||
None,
|
||||
pattern=r"^[0-9a-fA-F]{64}$",
|
||||
description=(
|
||||
"Optional SHA-256 returned by read_file(include_metadata=true). The "
|
||||
"edit fails if the file changed after it was read."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class EditFileTool(MoviePilotTool):
|
||||
"""使用精确文本匹配安全编辑本地文件。"""
|
||||
|
||||
name: str = "edit_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Write,
|
||||
ToolTag.File,
|
||||
]
|
||||
description: str = (
|
||||
"Edit a local text file by replacing specific old text with new text. "
|
||||
"Edit an existing local text file using an exact text match. By default "
|
||||
"the match must occur exactly once; use replace_all only for intentional "
|
||||
"bulk replacement. old_text cannot be empty, and new files must be "
|
||||
"created with write_file. Supports an optional SHA-256 conflict check. "
|
||||
"Non-admin users can only edit files inside the MoviePilot Agent config "
|
||||
"directory."
|
||||
)
|
||||
@@ -38,7 +69,16 @@ class EditFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"编辑文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, old_text: str, new_text: str, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
old_text: str,
|
||||
new_text: str,
|
||||
replace_all: bool = False,
|
||||
expected_sha256: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""校验精确匹配和可选文件版本后,以原子方式写入编辑结果。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
|
||||
|
||||
try:
|
||||
@@ -48,37 +88,74 @@ class EditFileTool(MoviePilotTool):
|
||||
if access_error:
|
||||
return access_error
|
||||
|
||||
path = AsyncPath(resolved_path)
|
||||
# 校验逻辑:如果要替换特定文本,文件必须存在且包含该文本
|
||||
if not await path.exists():
|
||||
# 如果 old_text 为空,可能用户想直接创建文件,但通常 edit_file 需要匹配旧内容
|
||||
if old_text:
|
||||
return f"错误:文件 {resolved_path} 不存在,无法进行内容替换。"
|
||||
if not old_text:
|
||||
return "错误:old_text 不能为空;创建或完整写入文件请使用 write_file。"
|
||||
|
||||
if await path.exists() and not await path.is_file():
|
||||
path = AsyncPath(resolved_path)
|
||||
if not await path.exists():
|
||||
return f"错误:文件 {resolved_path} 不存在;创建文件请使用 write_file。"
|
||||
|
||||
if not await path.is_file():
|
||||
return f"错误:{resolved_path} 不是一个文件"
|
||||
|
||||
if await path.exists():
|
||||
content = await path.read_text(encoding="utf-8", errors="replace")
|
||||
if old_text not in content:
|
||||
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
|
||||
return f"错误:在文件 {resolved_path} 中未找到指定的旧文本。请确保包含所有的空格、缩进 and 换行符。"
|
||||
occurrences = content.count(old_text)
|
||||
new_content = content.replace(old_text, new_text)
|
||||
else:
|
||||
# 文件不存在且 old_text 为空的情形(初始化新文件)
|
||||
new_content = new_text
|
||||
occurrences = 1
|
||||
local_path = Path(resolved_path)
|
||||
current_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
if (
|
||||
expected_sha256
|
||||
and current_sha256.casefold() != expected_sha256.casefold()
|
||||
):
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并基于最新内容编辑。"
|
||||
)
|
||||
|
||||
# 自动创建父目录
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = await path.read_text(encoding="utf-8", errors="strict")
|
||||
occurrences = content.count(old_text)
|
||||
if occurrences == 0:
|
||||
logger.warning(f"编辑文件 {resolved_path} 失败:未找到指定的旧文本块")
|
||||
return (
|
||||
f"错误:在文件 {resolved_path} 中未找到指定的旧文本。"
|
||||
"请重新读取文件并确认空格、缩进和换行。"
|
||||
)
|
||||
if occurrences > 1 and not replace_all:
|
||||
return (
|
||||
f"错误:old_text 在文件 {resolved_path} 中匹配到 {occurrences} 处,"
|
||||
"为避免误改已拒绝编辑。请提供更多上下文使其唯一,或明确设置 "
|
||||
"replace_all=true。"
|
||||
)
|
||||
|
||||
# 写入文件
|
||||
await path.write_text(new_content, encoding="utf-8")
|
||||
replacement_count = occurrences if replace_all else 1
|
||||
new_content = content.replace(
|
||||
old_text,
|
||||
new_text,
|
||||
-1 if replace_all else 1,
|
||||
)
|
||||
await self.run_blocking(
|
||||
"default",
|
||||
atomic_write_text,
|
||||
local_path,
|
||||
new_content,
|
||||
current_sha256,
|
||||
)
|
||||
new_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
|
||||
logger.info(f"成功编辑文件 {resolved_path},替换了 {occurrences} 处内容")
|
||||
return f"成功编辑文件 {resolved_path} (替换了 {occurrences} 处匹配内容)"
|
||||
logger.info(
|
||||
f"成功编辑文件 {resolved_path},替换了 {replacement_count} 处内容"
|
||||
)
|
||||
return (
|
||||
f"成功编辑文件 {resolved_path}(替换了 {replacement_count} 处匹配内容,"
|
||||
f"sha256={new_sha256})"
|
||||
)
|
||||
|
||||
except FileVersionConflictError:
|
||||
return (
|
||||
f"错误:文件 {file_path} 在编辑期间发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并再次编辑。"
|
||||
)
|
||||
except PermissionError:
|
||||
return f"错误:没有访问/修改 {file_path} 的权限"
|
||||
except UnicodeDecodeError:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""文件读取工具"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
|
||||
@@ -16,12 +18,22 @@ MAX_READ_SIZE = 50 * 1024
|
||||
|
||||
class ReadFileInput(BaseModel):
|
||||
"""文件读取工具的输入参数模型。"""
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to read")
|
||||
start_line: Optional[int] = Field(None, description="The starting line number (1-based, inclusive). If not provided, reading starts from the beginning of the file.")
|
||||
end_line: Optional[int] = Field(None, description="The ending line number (1-based, inclusive). If not provided, reading goes until the end of the file.")
|
||||
include_metadata: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Return structured JSON containing content, size, truncation state, "
|
||||
"and SHA-256. Use before a guarded full-file overwrite."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ReadFileTool(MoviePilotTool):
|
||||
"""按行范围读取本地文本文件,并可返回文件版本元数据。"""
|
||||
|
||||
name: str = "read_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Read,
|
||||
@@ -36,8 +48,15 @@ class ReadFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"读取文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, start_line: Optional[int] = None,
|
||||
end_line: Optional[int] = None, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
start_line: Optional[int] = None,
|
||||
end_line: Optional[int] = None,
|
||||
include_metadata: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""读取指定文本范围,必要时附带完整文件的 SHA-256 元数据。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}, start_line={start_line}, end_line={end_line}")
|
||||
|
||||
try:
|
||||
@@ -55,7 +74,8 @@ class ReadFileTool(MoviePilotTool):
|
||||
if not await path.is_file():
|
||||
return f"错误:{resolved_path} 不是一个文件"
|
||||
|
||||
content = await path.read_text(encoding="utf-8", errors="replace")
|
||||
raw_content = await path.read_bytes()
|
||||
content = raw_content.decode("utf-8", errors="replace")
|
||||
truncated = False
|
||||
|
||||
if start_line is not None or end_line is not None:
|
||||
@@ -78,6 +98,21 @@ class ReadFileTool(MoviePilotTool):
|
||||
content = content_bytes[:MAX_READ_SIZE].decode("utf-8", errors="replace")
|
||||
truncated = True
|
||||
|
||||
if include_metadata:
|
||||
return json.dumps(
|
||||
{
|
||||
"file_path": str(resolved_path),
|
||||
"sha256": hashlib.sha256(raw_content).hexdigest(),
|
||||
"size_bytes": len(raw_content),
|
||||
"start_line": start_line,
|
||||
"end_line": end_line,
|
||||
"truncated": truncated,
|
||||
"content": content,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if truncated:
|
||||
return f"{content}\n\n[警告:文件内容已超过50KB限制,以上内容已被截断。请使用 start_line/end_line 参数分段读取。]"
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@ from anyio import Path as AsyncPath
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.impl._file_write_utils import (
|
||||
FileVersionConflictError,
|
||||
atomic_write_text,
|
||||
calculate_file_sha256,
|
||||
)
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.log import logger
|
||||
|
||||
@@ -16,17 +21,36 @@ class WriteFileInput(BaseModel):
|
||||
|
||||
file_path: str = Field(..., description="The absolute path of the file to write")
|
||||
content: str = Field(..., description="The content to write into the file")
|
||||
overwrite: bool = Field(
|
||||
False,
|
||||
description=(
|
||||
"Allow replacing an existing file in full. Keep false when creating a "
|
||||
"new file; prefer edit_file for localized changes."
|
||||
),
|
||||
)
|
||||
expected_sha256: Optional[str] = Field(
|
||||
None,
|
||||
pattern=r"^[0-9a-fA-F]{64}$",
|
||||
description=(
|
||||
"Optional SHA-256 returned by read_file(include_metadata=true). When "
|
||||
"overwriting, fail if the existing file no longer has this hash."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WriteFileTool(MoviePilotTool):
|
||||
"""创建本地文本文件,或在显式允许后完整覆盖已有文件。"""
|
||||
|
||||
name: str = "write_file"
|
||||
tags: list[str] = [
|
||||
ToolTag.Write,
|
||||
ToolTag.File,
|
||||
]
|
||||
description: str = (
|
||||
"Write full content to a local text file. Non-admin users can only write "
|
||||
"inside the MoviePilot Agent config directory."
|
||||
"Create a local text file with complete content. Existing files are "
|
||||
"protected unless overwrite=true; localized changes should use edit_file. "
|
||||
"Supports an optional SHA-256 conflict check and writes atomically. "
|
||||
"Non-admin users can only write inside the MoviePilot Agent config directory."
|
||||
)
|
||||
args_schema: Type[BaseModel] = WriteFileInput
|
||||
|
||||
@@ -36,7 +60,15 @@ class WriteFileTool(MoviePilotTool):
|
||||
file_name = Path(file_path).name if file_path else "未知文件"
|
||||
return f"写入文件: {file_name}"
|
||||
|
||||
async def run(self, file_path: str, content: str, **kwargs) -> str:
|
||||
async def run(
|
||||
self,
|
||||
file_path: str,
|
||||
content: str,
|
||||
overwrite: bool = False,
|
||||
expected_sha256: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""创建或显式覆盖文件,并通过可选哈希阻止陈旧写入。"""
|
||||
logger.info(f"执行工具: {self.name}, 参数: file_path={file_path}")
|
||||
|
||||
try:
|
||||
@@ -48,18 +80,52 @@ class WriteFileTool(MoviePilotTool):
|
||||
|
||||
path = AsyncPath(resolved_path)
|
||||
|
||||
if await path.exists() and not await path.is_file():
|
||||
exists = await path.exists()
|
||||
if exists and not await path.is_file():
|
||||
return f"错误:{resolved_path} 路径已存在但不是一个文件"
|
||||
if exists and not overwrite:
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已存在,拒绝完整覆盖。"
|
||||
"局部修改请使用 edit_file;确需重写时设置 overwrite=true。"
|
||||
)
|
||||
if expected_sha256 and not exists:
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 不存在,无法校验 expected_sha256。"
|
||||
"请确认路径和最新文件状态。"
|
||||
)
|
||||
|
||||
# 自动创建父目录
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
local_path = Path(resolved_path)
|
||||
current_sha256 = None
|
||||
if exists:
|
||||
current_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
if expected_sha256:
|
||||
if current_sha256.casefold() != expected_sha256.casefold():
|
||||
return (
|
||||
f"错误:文件 {resolved_path} 已在读取后发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并基于最新内容写入。"
|
||||
)
|
||||
|
||||
# 写入文件
|
||||
await path.write_text(content, encoding="utf-8")
|
||||
await self.run_blocking(
|
||||
"default",
|
||||
atomic_write_text,
|
||||
local_path,
|
||||
content,
|
||||
current_sha256,
|
||||
)
|
||||
new_sha256 = await self.run_blocking(
|
||||
"default", calculate_file_sha256, local_path
|
||||
)
|
||||
|
||||
logger.info(f"成功写入文件 {resolved_path}")
|
||||
return f"成功写入文件 {resolved_path}"
|
||||
return f"成功写入文件 {resolved_path}(sha256={new_sha256})"
|
||||
|
||||
except FileVersionConflictError:
|
||||
return (
|
||||
f"错误:文件 {file_path} 在写入期间发生变化,拒绝覆盖。"
|
||||
"请重新读取文件并再次写入。"
|
||||
)
|
||||
except PermissionError:
|
||||
return f"错误:没有权限写入 {file_path}"
|
||||
except Exception as e:
|
||||
|
||||
+47
-11
@@ -1,9 +1,11 @@
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.agent.tools.base import ToolExecutionTimeoutError, format_tool_result_for_agent
|
||||
from app.agent.tools.factory import MoviePilotToolFactory
|
||||
from app.core.plugin import PluginManager
|
||||
from app.log import logger
|
||||
|
||||
|
||||
@@ -40,27 +42,59 @@ class MoviePilotToolsManager:
|
||||
self.session_id = session_id
|
||||
self.is_admin = is_admin
|
||||
self.tools: List[Any] = []
|
||||
self._tools_lock = threading.Lock()
|
||||
self._plugin_agent_tools_revision = -1
|
||||
self._load_tools()
|
||||
|
||||
def _load_tools(self):
|
||||
def _load_tools(self) -> None:
|
||||
"""
|
||||
加载所有MoviePilot工具
|
||||
"""
|
||||
try:
|
||||
# 创建工具实例
|
||||
self.tools = MoviePilotToolFactory.create_tools(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
channel=None,
|
||||
source="api",
|
||||
username="API Client",
|
||||
stream_handler=None,
|
||||
agent_context={"is_admin": self.is_admin},
|
||||
)
|
||||
plugin_manager = PluginManager()
|
||||
while True:
|
||||
plugin_tools_revision = (
|
||||
plugin_manager.get_plugin_agent_tools_revision()
|
||||
)
|
||||
tools = MoviePilotToolFactory.create_tools(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
channel=None,
|
||||
source="api",
|
||||
username="API Client",
|
||||
stream_handler=None,
|
||||
agent_context={"is_admin": self.is_admin},
|
||||
)
|
||||
if (
|
||||
plugin_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
break
|
||||
self.tools = tools
|
||||
self._plugin_agent_tools_revision = plugin_tools_revision
|
||||
logger.info(f"成功加载 {len(self.tools)} 个工具")
|
||||
except Exception as e:
|
||||
logger.error(f"加载工具失败: {e}", exc_info=True)
|
||||
self.tools = []
|
||||
self._plugin_agent_tools_revision = -1
|
||||
|
||||
def _ensure_tools_current(self) -> None:
|
||||
"""
|
||||
在插件工具注册表变化后惰性刷新工具实例。
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
if (
|
||||
self._plugin_agent_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
return
|
||||
with self._tools_lock:
|
||||
if (
|
||||
self._plugin_agent_tools_revision
|
||||
== plugin_manager.get_plugin_agent_tools_revision()
|
||||
):
|
||||
return
|
||||
self._load_tools()
|
||||
|
||||
def list_tools(self) -> List[ToolDefinition]:
|
||||
"""
|
||||
@@ -69,6 +103,7 @@ class MoviePilotToolsManager:
|
||||
Returns:
|
||||
工具定义列表
|
||||
"""
|
||||
self._ensure_tools_current()
|
||||
tools_list = []
|
||||
for tool in self.tools:
|
||||
if getattr(tool, "_require_admin", False) and not self.is_admin:
|
||||
@@ -102,6 +137,7 @@ class MoviePilotToolsManager:
|
||||
Returns:
|
||||
工具实例,如果未找到返回None
|
||||
"""
|
||||
self._ensure_tools_current()
|
||||
for tool in self.tools:
|
||||
if tool.name == tool_name:
|
||||
return tool
|
||||
|
||||
@@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app import schemas
|
||||
from app.core.auth_bridge import build_token_response, consume_plugin_auth_ticket
|
||||
from app.core.auth import build_token_response, consume_plugin_auth_ticket
|
||||
from app.core.plugin import PluginManager
|
||||
from app.db.models.passkey import PassKey
|
||||
from app.db.models.user import User
|
||||
|
||||
@@ -4,12 +4,15 @@ from fastapi import APIRouter, Depends
|
||||
|
||||
from app import schemas
|
||||
from app.chain.douban import DoubanChain
|
||||
from app.core.config import settings
|
||||
from app.core.context import MediaInfo
|
||||
from app.core.security import verify_token
|
||||
from app.db.models.user import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.douban.douban_cache import DoubanCache
|
||||
from app.schemas import MediaType
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -29,6 +32,10 @@ async def douban_recognition_cache(
|
||||
"count": len(cache_items),
|
||||
"recognized": recognized_count,
|
||||
"unrecognized": len(cache_items) - recognized_count,
|
||||
"shared_recognized": SystemConfigOper().get(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
) or 0,
|
||||
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
|
||||
"data": cache_items,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -140,7 +140,7 @@ async def download_history(
|
||||
_: schemas.TokenPayload = Depends(verify_token),
|
||||
) -> Any:
|
||||
"""
|
||||
查询下载历史记录
|
||||
按下载时间倒序查询下载历史记录
|
||||
"""
|
||||
return await DownloadHistory.async_list_by_page(db, page, count)
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ class LlmTestRequest(BaseModel):
|
||||
user_agent: Optional[str] = None
|
||||
temperature: Optional[float] = None
|
||||
use_proxy: Optional[bool] = None
|
||||
api_protocol: Optional[str] = None
|
||||
|
||||
|
||||
class LlmProviderAuthStartRequest(BaseModel):
|
||||
@@ -269,6 +270,7 @@ async def llm_test(
|
||||
base_url_preset=settings.LLM_BASE_URL_PRESET,
|
||||
user_agent=settings.LLM_USER_AGENT,
|
||||
use_proxy=settings.LLM_USE_PROXY,
|
||||
api_protocol=settings.LLM_API_PROTOCOL,
|
||||
)
|
||||
|
||||
if not payload.provider:
|
||||
@@ -302,6 +304,7 @@ async def llm_test(
|
||||
"base_url_preset": payload.base_url_preset,
|
||||
"user_agent": payload.user_agent,
|
||||
"use_proxy": payload.use_proxy,
|
||||
"api_protocol": payload.api_protocol,
|
||||
}
|
||||
if payload.temperature is not None:
|
||||
test_kwargs["temperature"] = payload.temperature
|
||||
|
||||
@@ -36,7 +36,7 @@ def _build_media_seasons(
|
||||
episode_count=item.get("episode_count"),
|
||||
name=item.get("name"),
|
||||
overview=item.get("overview"),
|
||||
poster_path=item.get("poster_path"),
|
||||
poster_path=item.get("poster_path") or mediainfo.poster_path,
|
||||
season_number=season_number,
|
||||
vote_average=item.get("vote_average"),
|
||||
))
|
||||
@@ -366,6 +366,9 @@ async def seasons(
|
||||
)
|
||||
if mediainfo:
|
||||
return _build_media_seasons(mediainfo, season)
|
||||
# 明确来源的查询不能按标题切换到默认识别源,避免辅助 TMDB 信息替换主身份。
|
||||
if media_source and source_media_id:
|
||||
return []
|
||||
if title:
|
||||
meta = MetaInfo(title)
|
||||
if year:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Any, List, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import schemas
|
||||
@@ -21,6 +21,18 @@ from app.utils.media import build_media_key, resolve_media_identity
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _require_mediaserver_result(result: Optional[List[Any]]) -> List[Any]:
|
||||
"""
|
||||
保留媒体服务器成功空列表,并把提供方失败转换为明确的网关错误。
|
||||
"""
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="媒体服务器请求失败",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/play/{itemid:path}", summary="在线播放")
|
||||
def play_item(
|
||||
itemid: str, _: schemas.TokenPayload = Depends(verify_token)
|
||||
@@ -153,11 +165,12 @@ def latest(
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
return (
|
||||
return _require_mediaserver_result(
|
||||
MediaServerChain().latest(
|
||||
server=server, count=count, username=userinfo.username
|
||||
server=server,
|
||||
count=count,
|
||||
username=userinfo.username,
|
||||
)
|
||||
or []
|
||||
)
|
||||
|
||||
|
||||
@@ -172,11 +185,12 @@ def playing(
|
||||
"""
|
||||
获取媒体服务器正在播放条目
|
||||
"""
|
||||
return (
|
||||
return _require_mediaserver_result(
|
||||
MediaServerChain().playing(
|
||||
server=server, count=count, username=userinfo.username
|
||||
server=server,
|
||||
count=count,
|
||||
username=userinfo.username,
|
||||
)
|
||||
or []
|
||||
)
|
||||
|
||||
|
||||
@@ -191,11 +205,12 @@ def library(
|
||||
"""
|
||||
获取媒体服务器媒体库列表
|
||||
"""
|
||||
return (
|
||||
return _require_mediaserver_result(
|
||||
MediaServerChain().librarys(
|
||||
server=server, username=userinfo.username, hidden=hidden
|
||||
server=server,
|
||||
username=userinfo.username,
|
||||
hidden=hidden,
|
||||
)
|
||||
or []
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,8 +22,8 @@ from app.helper.passkey import (
|
||||
PassKeyHelper,
|
||||
PassKeyRegistrationOriginMismatchError,
|
||||
PassKeyRegistrationVerificationError,
|
||||
PasskeyChallengeStore,
|
||||
)
|
||||
from app.helper.passkey_challenge import PasskeyChallengeStore
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.otp import OtpUtils
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import shutil
|
||||
from typing import Annotated, Any, List, Optional
|
||||
from typing import Annotated, Any, Dict, List, Optional
|
||||
|
||||
import aiofiles
|
||||
from anyio import Path as AsyncPath
|
||||
@@ -476,6 +476,71 @@ async def statistic(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
return await MoviePilotServerHelper.async_get_plugin_statistic()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/rating",
|
||||
summary="批量查询插件评分",
|
||||
response_model=Dict[str, schemas.PluginRating],
|
||||
)
|
||||
async def plugin_ratings(
|
||||
plugin_ids: Optional[str] = None,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> Dict[str, schemas.PluginRating]:
|
||||
"""
|
||||
批量查询插件平均分、评分人数和当前安装实例评分。
|
||||
"""
|
||||
requested_ids = plugin_ids.split(",") if plugin_ids is not None else None
|
||||
ratings = await MoviePilotServerHelper.async_get_plugin_ratings(requested_ids)
|
||||
return {
|
||||
plugin_id: schemas.PluginRating.model_validate(rating)
|
||||
for plugin_id, rating in ratings.items()
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/rating/{plugin_id}",
|
||||
summary="查询插件评分",
|
||||
response_model=schemas.PluginRating,
|
||||
)
|
||||
async def plugin_rating(
|
||||
plugin_id: str,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> schemas.PluginRating:
|
||||
"""
|
||||
查询单个插件平均分、评分人数和当前安装实例评分。
|
||||
"""
|
||||
rating = await MoviePilotServerHelper.async_get_plugin_rating(plugin_id)
|
||||
return schemas.PluginRating.model_validate(rating)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/rating/{plugin_id}",
|
||||
summary="提交插件评分",
|
||||
response_model=schemas.Response,
|
||||
)
|
||||
async def rate_plugin(
|
||||
plugin_id: str,
|
||||
payload: schemas.PluginRatingRequest,
|
||||
_: User = Depends(get_current_active_superuser_async),
|
||||
) -> schemas.Response:
|
||||
"""
|
||||
为已安装插件新增或更新当前安装实例评分。
|
||||
"""
|
||||
installed_plugins = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id not in installed_plugins:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"插件 {plugin_id} 未安装,无法评分",
|
||||
)
|
||||
|
||||
rating = await MoviePilotServerHelper.async_submit_plugin_rating(
|
||||
plugin_id,
|
||||
payload.rating,
|
||||
)
|
||||
if rating is None:
|
||||
return schemas.Response(success=False, message="连接MoviePilot服务器失败")
|
||||
return schemas.Response(success=True, data=rating)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/reload/{plugin_id}", summary="重新加载插件", response_model=schemas.Response
|
||||
)
|
||||
|
||||
@@ -1,17 +1,29 @@
|
||||
from typing import Any, List, Optional
|
||||
from typing import Any, Awaitable, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from app import schemas
|
||||
from app.chain.recommend import RecommendChain
|
||||
from app.core.event import eventmanager
|
||||
from app.core.security import verify_token
|
||||
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbException
|
||||
from app.schemas import RecommendSourceEventData
|
||||
from app.schemas.types import ChainEventType
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
async def _require_tmdb_result(operation: Awaitable[List[Any]]) -> List[Any]:
|
||||
"""保留 TMDB 成功空列表,并把上游请求异常转换为明确的网关错误。"""
|
||||
try:
|
||||
return await operation
|
||||
except TMDbException as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail="TMDB请求失败",
|
||||
) from error
|
||||
|
||||
|
||||
@router.get(
|
||||
"/source",
|
||||
summary="获取推荐数据源",
|
||||
@@ -204,16 +216,19 @@ async def tmdb_movies(
|
||||
"""
|
||||
浏览TMDB电影信息
|
||||
"""
|
||||
return await RecommendChain().async_tmdb_movies(
|
||||
sort_by=sort_by,
|
||||
with_genres=with_genres,
|
||||
with_original_language=with_original_language,
|
||||
with_keywords=with_keywords,
|
||||
with_watch_providers=with_watch_providers,
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page,
|
||||
return await _require_tmdb_result(
|
||||
RecommendChain().async_tmdb_movies(
|
||||
sort_by=sort_by,
|
||||
with_genres=with_genres,
|
||||
with_original_language=with_original_language,
|
||||
with_keywords=with_keywords,
|
||||
with_watch_providers=with_watch_providers,
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page,
|
||||
raise_exception=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -233,16 +248,19 @@ async def tmdb_tvs(
|
||||
"""
|
||||
浏览TMDB剧集信息
|
||||
"""
|
||||
return await RecommendChain().async_tmdb_tvs(
|
||||
sort_by=sort_by,
|
||||
with_genres=with_genres,
|
||||
with_original_language=with_original_language,
|
||||
with_keywords=with_keywords,
|
||||
with_watch_providers=with_watch_providers,
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page,
|
||||
return await _require_tmdb_result(
|
||||
RecommendChain().async_tmdb_tvs(
|
||||
sort_by=sort_by,
|
||||
with_genres=with_genres,
|
||||
with_original_language=with_original_language,
|
||||
with_keywords=with_keywords,
|
||||
with_watch_providers=with_watch_providers,
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page,
|
||||
raise_exception=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -255,4 +273,6 @@ async def tmdb_trending(
|
||||
"""
|
||||
TMDB流行趋势
|
||||
"""
|
||||
return await RecommendChain().async_tmdb_trending(page=page)
|
||||
return await _require_tmdb_result(
|
||||
RecommendChain().async_tmdb_trending(page=page, raise_exception=True)
|
||||
)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import List, Any, Optional, AsyncIterator
|
||||
import time
|
||||
from typing import Any, AsyncIterator, Iterator, List, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, Body, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
@@ -23,6 +25,12 @@ router = APIRouter()
|
||||
|
||||
_SSE_APPEND_FLUSH_INTERVAL = 1
|
||||
_SSE_APPEND_MAX_ITEMS = 48
|
||||
_SSE_HEARTBEAT_INTERVAL = 15
|
||||
_SSE_REPLACE_MAX_ITEMS = 48
|
||||
_SSE_RESPONSE_HEADERS = {
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
|
||||
|
||||
def _parse_site_list(sites: Optional[str]) -> Optional[List[int]]:
|
||||
@@ -180,11 +188,40 @@ def _merge_append_event(pending_event: Optional[dict], event: dict) -> dict:
|
||||
return merged_event
|
||||
|
||||
|
||||
def _iter_replace_event_batches(event: dict) -> Iterator[dict]:
|
||||
"""
|
||||
将超大的最终替换事件拆成有序批次,避免单个 SSE 消息承载全部完整对象。
|
||||
"""
|
||||
items = event.get("items")
|
||||
if (
|
||||
event.get("type") != "replace"
|
||||
or not isinstance(items, list)
|
||||
or len(items) <= _SSE_REPLACE_MAX_ITEMS
|
||||
):
|
||||
yield event
|
||||
return
|
||||
|
||||
batch_count = (len(items) + _SSE_REPLACE_MAX_ITEMS - 1) // _SSE_REPLACE_MAX_ITEMS
|
||||
for batch_index in range(batch_count):
|
||||
start = batch_index * _SSE_REPLACE_MAX_ITEMS
|
||||
batch_event = dict(event)
|
||||
batch_event.update(
|
||||
{
|
||||
"type": "replace" if batch_index == 0 else "append",
|
||||
"items": items[start:start + _SSE_REPLACE_MAX_ITEMS],
|
||||
"replace_batch": True,
|
||||
"batch_index": batch_index,
|
||||
"batch_count": batch_count,
|
||||
}
|
||||
)
|
||||
yield batch_event
|
||||
|
||||
|
||||
async def _iter_batched_search_events(
|
||||
event_source: AsyncIterator[dict],
|
||||
) -> AsyncIterator[dict]:
|
||||
"""
|
||||
对搜索流事件做轻量批处理,避免站点结果集中返回时产生过密 SSE。
|
||||
对搜索流事件做轻量批处理,并在上游长时间静默时发送心跳。
|
||||
"""
|
||||
iterator = event_source.__aiter__()
|
||||
pending_append_event: Optional[dict] = None
|
||||
@@ -195,13 +232,19 @@ async def _iter_batched_search_events(
|
||||
if next_event_task is None:
|
||||
next_event_task = asyncio.create_task(anext(iterator))
|
||||
|
||||
timeout = _SSE_APPEND_FLUSH_INTERVAL if pending_append_event else None
|
||||
timeout = (
|
||||
_SSE_APPEND_FLUSH_INTERVAL
|
||||
if pending_append_event
|
||||
else _SSE_HEARTBEAT_INTERVAL
|
||||
)
|
||||
done, _ = await asyncio.wait({next_event_task}, timeout=timeout)
|
||||
|
||||
if not done:
|
||||
if pending_append_event:
|
||||
yield pending_append_event
|
||||
pending_append_event = None
|
||||
else:
|
||||
yield {"type": "heartbeat"}
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -227,7 +270,8 @@ async def _iter_batched_search_events(
|
||||
yield pending_append_event
|
||||
pending_append_event = None
|
||||
|
||||
yield event
|
||||
for batched_event in _iter_replace_event_batches(event):
|
||||
yield batched_event
|
||||
finally:
|
||||
if next_event_task and not next_event_task.done():
|
||||
next_event_task.cancel()
|
||||
@@ -239,13 +283,29 @@ async def _iter_batched_search_events(
|
||||
|
||||
async def _stream_search_events(request: Request, event_source: AsyncIterator[dict]):
|
||||
"""
|
||||
输出搜索SSE事件
|
||||
输出搜索 SSE 事件,并记录连接生命周期与传输规模。
|
||||
"""
|
||||
locale = LocaleHelper.get_locale_from_request(request)
|
||||
search_id = uuid4().hex[:12]
|
||||
request_path = getattr(getattr(request, "url", None), "path", "unknown")
|
||||
started_at = time.monotonic()
|
||||
event_count = 0
|
||||
transmitted_bytes = 0
|
||||
last_event_type = "none"
|
||||
last_stage = "none"
|
||||
termination_reason = "source_exhausted"
|
||||
logger.info(f"渐进式搜索流已建立,搜索ID:{search_id},路径:{request_path}")
|
||||
try:
|
||||
has_sent_final_replace = False
|
||||
async for event in _iter_batched_search_events(event_source):
|
||||
last_event_type = event.get("type") or "unknown"
|
||||
last_stage = event.get("stage") or last_stage
|
||||
if await request.is_disconnected():
|
||||
termination_reason = "client_disconnected"
|
||||
logger.warning(
|
||||
f"渐进式搜索客户端已断开,搜索ID:{search_id},路径:{request_path},"
|
||||
f"事件:{last_event_type},阶段:{last_stage}"
|
||||
)
|
||||
break
|
||||
# 精确搜索会先发送 replace,再发送 done。done 再带整包 items 只会重复占用带宽和前端内存。
|
||||
if event.get("type") == "replace" and event.get("items"):
|
||||
@@ -257,13 +317,36 @@ async def _stream_search_events(request: Request, event_source: AsyncIterator[di
|
||||
and event.get("items")
|
||||
):
|
||||
event = {key: value for key, value in event.items() if key != "items"}
|
||||
yield _sse_event(event, locale=locale)
|
||||
payload = _sse_event(event, locale=locale)
|
||||
event_count += 1
|
||||
transmitted_bytes += len(payload.encode("utf-8"))
|
||||
if event.get("type") == "done":
|
||||
termination_reason = "completed"
|
||||
yield payload
|
||||
except asyncio.CancelledError:
|
||||
termination_reason = "cancelled"
|
||||
logger.warning(
|
||||
f"渐进式搜索流已取消,搜索ID:{search_id},路径:{request_path},"
|
||||
f"事件:{last_event_type},阶段:{last_stage}"
|
||||
)
|
||||
raise
|
||||
except Exception as err:
|
||||
termination_reason = "error"
|
||||
logger.error(f"渐进式搜索出错:{err}", exc_info=True)
|
||||
yield _sse_event(
|
||||
payload = _sse_event(
|
||||
{"type": "error", "success": False, "message": str(err)},
|
||||
locale=locale,
|
||||
)
|
||||
event_count += 1
|
||||
transmitted_bytes += len(payload.encode("utf-8"))
|
||||
yield payload
|
||||
finally:
|
||||
elapsed = time.monotonic() - started_at
|
||||
logger.info(
|
||||
f"渐进式搜索流结束,搜索ID:{search_id},路径:{request_path},"
|
||||
f"状态:{termination_reason},事件数:{event_count},"
|
||||
f"发送字节:{transmitted_bytes},耗时:{elapsed:.2f}秒"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/last", summary="查询搜索结果", response_model=List[schemas.Context])
|
||||
@@ -319,6 +402,7 @@ async def search_by_id_stream(
|
||||
search_chain = SearchChain()
|
||||
|
||||
async def event_source():
|
||||
"""解析媒体身份并输出精确搜索流事件。"""
|
||||
search_params, message = await _resolve_media_search_params(
|
||||
mediaid=mediaid,
|
||||
media_type=media_type,
|
||||
@@ -341,7 +425,9 @@ async def search_by_id_stream(
|
||||
yield event
|
||||
|
||||
return StreamingResponse(
|
||||
_stream_search_events(request, event_source()), media_type="text/event-stream"
|
||||
_stream_search_events(request, event_source()),
|
||||
media_type="text/event-stream",
|
||||
headers=_SSE_RESPONSE_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
@@ -401,7 +487,9 @@ async def search_by_title_stream(
|
||||
title=keyword, page=page, sites=_parse_site_list(sites), cache_local=True
|
||||
)
|
||||
return StreamingResponse(
|
||||
_stream_search_events(request, event_source), media_type="text/event-stream"
|
||||
_stream_search_events(request, event_source),
|
||||
media_type="text/event-stream",
|
||||
headers=_SSE_RESPONSE_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
@@ -446,6 +534,7 @@ async def search_subtitle_by_title_stream(
|
||||
_iter_signed_subtitle_search_events(event_source),
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers=_SSE_RESPONSE_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
@@ -558,6 +647,7 @@ async def search_subtitle_by_id_stream(
|
||||
_iter_signed_subtitle_search_events(event_source()),
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers=_SSE_RESPONSE_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -504,7 +504,8 @@ async def seerr_subscribe(
|
||||
tmdbid=tmdbId,
|
||||
title=subject,
|
||||
year="",
|
||||
season=0,
|
||||
# 电影不传季号,避免被误判为剧集(S00)并污染通知标题
|
||||
season=None,
|
||||
username=user_name,
|
||||
)
|
||||
else:
|
||||
@@ -630,6 +631,7 @@ async def popular_subscribes(
|
||||
media.bangumi_id = sub.get("bangumiid")
|
||||
media.anilist_id = sub.get("anilistid")
|
||||
media.source = sub.get("media_source")
|
||||
media.media_id = sub.get("media_id")
|
||||
media.tvdb_id = sub.get("tvdbid")
|
||||
media.imdb_id = sub.get("imdbid")
|
||||
media.season = sub.get("season")
|
||||
@@ -863,6 +865,7 @@ async def delete_subscribe(
|
||||
"anilistid": subscribe_info.get("anilistid"),
|
||||
"media_source": subscribe_info.get("media_source"),
|
||||
"media_id": subscribe_info.get("media_id"),
|
||||
"season": subscribe_info.get("season"),
|
||||
}
|
||||
)
|
||||
return schemas.Response(success=True)
|
||||
|
||||
@@ -582,23 +582,27 @@ async def fetch_image(
|
||||
):
|
||||
return None
|
||||
|
||||
content = await ImageHelper().async_fetch_image(
|
||||
image_result = await ImageHelper().async_fetch_image_with_mime_type(
|
||||
url=fetch_url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
|
||||
if content:
|
||||
if image_result:
|
||||
content, media_type = image_result
|
||||
|
||||
# 检查 If-None-Match
|
||||
etag = HashUtils.md5(content)
|
||||
headers = RequestUtils.generate_cache_headers(etag, max_age=86400 * 7)
|
||||
headers["Content-Type"] = media_type
|
||||
headers["X-Content-Type-Options"] = "nosniff"
|
||||
if if_none_match == etag:
|
||||
return Response(status_code=304, headers=headers)
|
||||
# 返回缓存图片
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=UrlUtils.get_mime_type(fetch_url, "image/jpeg"),
|
||||
media_type=media_type,
|
||||
headers=headers,
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -4,11 +4,13 @@ from fastapi import APIRouter, Depends
|
||||
|
||||
from app import schemas
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.core.config import settings
|
||||
from app.core.security import verify_token
|
||||
from app.db.models.user import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser_async
|
||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||
from app.schemas.types import MediaType
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -28,6 +30,10 @@ async def tmdb_recognition_cache(
|
||||
"count": len(cache_items),
|
||||
"recognized": recognized_count,
|
||||
"unrecognized": len(cache_items) - recognized_count,
|
||||
"shared_recognized": SystemConfigOper().get(
|
||||
SystemConfigKey.MediaRecognizeShareCount
|
||||
) or 0,
|
||||
"shared_recognize_enabled": settings.MEDIA_RECOGNIZE_SHARE,
|
||||
"data": cache_items,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -240,6 +240,40 @@ def match_manual_transfer_target_path(
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/manual/history",
|
||||
summary="查询手动转移成功历史",
|
||||
response_model=schemas.Response,
|
||||
)
|
||||
def query_manual_transfer_history(
|
||||
transer_item: ManualTransferItem,
|
||||
db: Session = Depends(get_db),
|
||||
_: User = Depends(get_current_active_manage_user),
|
||||
) -> Any:
|
||||
"""
|
||||
查询文件或目录命中的成功整理记录。
|
||||
|
||||
:param transer_item: 手工整理项
|
||||
:param db: 数据库
|
||||
:param _: Token校验
|
||||
"""
|
||||
src_fileitems, error_message = _resolve_manual_transfer_source_fileitems(
|
||||
transer_item=transer_item,
|
||||
db=db,
|
||||
)
|
||||
if error_message:
|
||||
return schemas.Response(success=False, message=error_message)
|
||||
|
||||
histories = TransferChain().get_manual_transfer_histories(
|
||||
_deduplicate_fileitems(src_fileitems)
|
||||
)
|
||||
history_info = schemas.ManualTransferHistoryInfo(
|
||||
reorganize=bool(histories),
|
||||
history_count=len(histories),
|
||||
)
|
||||
return schemas.Response(success=True, data=history_info.model_dump())
|
||||
|
||||
|
||||
@router.post("/manual", summary="手动转移", response_model=schemas.Response)
|
||||
def manual_transfer(
|
||||
transer_item: ManualTransferItem,
|
||||
@@ -278,7 +312,11 @@ def manual_transfer(
|
||||
else:
|
||||
# 源路径
|
||||
src_fileitems = [FileItem(**history.src_fileitem)]
|
||||
if history.dest_fileitem and not transer_item.preview:
|
||||
if (
|
||||
history.dest_fileitem
|
||||
and not transer_item.preview
|
||||
and not transer_item.reorganize
|
||||
):
|
||||
cleanup_dest_fileitem = FileItem(**history.dest_fileitem)
|
||||
|
||||
# 从历史数据获取信息
|
||||
@@ -435,6 +473,7 @@ def manual_transfer(
|
||||
downloader=downloader,
|
||||
download_hash=download_hash,
|
||||
preview=transer_item.preview,
|
||||
reorganize=transer_item.reorganize,
|
||||
sync_extra_files=False,
|
||||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||||
)
|
||||
@@ -521,6 +560,7 @@ def manual_transfer(
|
||||
downloader=downloader,
|
||||
download_hash=download_hash,
|
||||
preview=transer_item.preview,
|
||||
reorganize=transer_item.reorganize,
|
||||
sync_extra_files=True,
|
||||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||||
)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.core.meta import MetaBase
|
||||
from app.core.module import ModuleManager
|
||||
from app.core.plugin import PluginManager
|
||||
from app.db.message_oper import MessageOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import UserOper
|
||||
from app.helper.message import MessageHelper, MessageQueueManager, MessageTemplateHelper
|
||||
from app.helper.server import MoviePilotServerHelper
|
||||
@@ -49,6 +50,7 @@ from app.schemas.types import (
|
||||
MediaImageType,
|
||||
EventType,
|
||||
MessageChannel,
|
||||
SystemConfigKey,
|
||||
)
|
||||
from app.utils.object import ObjectUtils
|
||||
|
||||
@@ -572,6 +574,14 @@ class ChainBase(metaclass=ABCMeta):
|
||||
mediainfo=mediainfo,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_media_recognize_share_hit() -> None:
|
||||
"""记录一次共享媒体识别成功命中,统计失败不影响识别结果。"""
|
||||
try:
|
||||
SystemConfigOper().increment(SystemConfigKey.MediaRecognizeShareCount)
|
||||
except Exception as err:
|
||||
logger.error(f"记录共享媒体识别命中次数失败:{str(err)}")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_media_source_params(
|
||||
source: Optional[str] = None,
|
||||
@@ -730,6 +740,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
)
|
||||
if mediainfo:
|
||||
self._update_local_recognize_cache(shared_cache_meta, mediainfo)
|
||||
self._record_media_recognize_share_hit()
|
||||
return mediainfo
|
||||
return None
|
||||
|
||||
@@ -839,6 +850,7 @@ class ChainBase(metaclass=ABCMeta):
|
||||
)
|
||||
if mediainfo:
|
||||
await self._async_update_local_recognize_cache(shared_cache_meta, mediainfo)
|
||||
await run_in_threadpool(self._record_media_recognize_share_hit)
|
||||
return mediainfo
|
||||
return None
|
||||
|
||||
|
||||
+18
-10
@@ -7,10 +7,11 @@ import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Set, Dict, Union
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
from urllib.parse import parse_qs, urlencode, urljoin, urlparse
|
||||
|
||||
from app import schemas
|
||||
from app.chain import ChainBase
|
||||
from app.chain.media import MediaChain
|
||||
from app.chain.storage import StorageChain
|
||||
from app.core.cache import FileCache
|
||||
from app.core.config import settings, global_vars
|
||||
@@ -402,6 +403,7 @@ class DownloadChain(ChainBase):
|
||||
)
|
||||
if not mediainfo:
|
||||
return False, "无法识别媒体信息", []
|
||||
mediainfo = MediaChain().supplement_tmdb_info(mediainfo, metainfo)
|
||||
|
||||
storage, target_dir, error_msg = self._resolve_media_download_dir(
|
||||
media_info=mediainfo,
|
||||
@@ -711,10 +713,21 @@ class DownloadChain(ChainBase):
|
||||
return res.text
|
||||
else:
|
||||
data = res.json()
|
||||
success_key = req_params.get('success')
|
||||
if success_key and not data.get(success_key):
|
||||
return None
|
||||
for key in str(req_params.get('result')).split("."):
|
||||
data = data.get(key)
|
||||
if not data:
|
||||
return None
|
||||
result_path = req_params.get('result_path')
|
||||
result_query_param = req_params.get('result_query_param')
|
||||
if result_path and result_query_param:
|
||||
result_url = urljoin(
|
||||
f"{str(req_params.get('result_base_url')).rstrip('/')}/",
|
||||
str(result_path).lstrip('/'),
|
||||
)
|
||||
return f"{result_url}?{urlencode({result_query_param: data})}"
|
||||
data = self._normalize_indirect_download_url(
|
||||
url=data,
|
||||
base_url=req_params.get('result_base_url'),
|
||||
@@ -804,6 +817,10 @@ class DownloadChain(ChainBase):
|
||||
_meta = context.meta_info
|
||||
_site_downloader = _torrent.site_downloader
|
||||
|
||||
# 下载目录和下载器分类依赖 TMDB 辅助分类,但媒体主身份保持不变。
|
||||
_media = MediaChain().supplement_tmdb_info(_media, _meta)
|
||||
context.media_info = _media
|
||||
|
||||
# 发送资源下载事件,允许外部拦截下载
|
||||
event_data = ResourceDownloadEventData(
|
||||
context=context,
|
||||
@@ -839,15 +856,6 @@ class DownloadChain(ChainBase):
|
||||
logger.warn(str(err))
|
||||
return (None, str(err)) if return_detail else None
|
||||
|
||||
# 补充完整的media数据
|
||||
if not _media.genre_ids:
|
||||
new_media = self.recognize_media(mtype=_media.type, tmdbid=_media.tmdb_id,
|
||||
doubanid=_media.douban_id, bangumiid=_media.bangumi_id,
|
||||
anilistid=_media.anilist_id,
|
||||
episode_group=_media.episode_group)
|
||||
if new_media:
|
||||
_media = new_media
|
||||
|
||||
# 实际下载的集数
|
||||
download_episodes = StringUtils.format_ep(list(episodes)) if episodes else None
|
||||
if episodes is not None:
|
||||
|
||||
@@ -614,6 +614,113 @@ class MediaChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
logger.warn(f"{metainfo.title} 未识别到媒体信息")
|
||||
return mediainfo
|
||||
|
||||
@staticmethod
|
||||
def _build_tmdb_supplement_meta(
|
||||
mediainfo: MediaInfo,
|
||||
metainfo: Optional[MetaBase] = None,
|
||||
) -> MetaBase:
|
||||
"""
|
||||
根据主识别结果构造 TMDB 辅助识别参数。
|
||||
|
||||
:param mediainfo: 主识别源返回的媒体信息
|
||||
:param metainfo: 原始标题解析信息
|
||||
:return: 不携带主识别源身份的 TMDB 查询参数
|
||||
"""
|
||||
title = mediainfo.title or getattr(metainfo, "name", None) or ""
|
||||
tmdb_meta = MetaInfo(title)
|
||||
if not tmdb_meta.cn_name and getattr(metainfo, "cn_name", None):
|
||||
tmdb_meta.cn_name = metainfo.cn_name
|
||||
if not tmdb_meta.en_name:
|
||||
tmdb_meta.en_name = mediainfo.en_title or (
|
||||
getattr(metainfo, "en_name", None)
|
||||
)
|
||||
tmdb_meta.type = mediainfo.type or (
|
||||
getattr(metainfo, "type", None) or MediaType.UNKNOWN
|
||||
)
|
||||
season = (
|
||||
mediainfo.season
|
||||
if mediainfo.season is not None
|
||||
else getattr(metainfo, "begin_season", None)
|
||||
)
|
||||
tmdb_meta.begin_season = season
|
||||
season_year = None
|
||||
if season is not None and mediainfo.season_years:
|
||||
season_year = (
|
||||
mediainfo.season_years.get(season)
|
||||
or mediainfo.season_years.get(str(season))
|
||||
)
|
||||
tmdb_meta.year = (
|
||||
season_year
|
||||
or mediainfo.year
|
||||
or getattr(metainfo, "year", None)
|
||||
)
|
||||
return tmdb_meta
|
||||
|
||||
@staticmethod
|
||||
def _merge_tmdb_auxiliary(
|
||||
mediainfo: MediaInfo,
|
||||
tmdb_media: MediaInfo,
|
||||
) -> MediaInfo:
|
||||
"""
|
||||
将 TMDB 兼容字段合并到主识别结果,不改变主数据源身份和展示信息。
|
||||
|
||||
:param mediainfo: 主识别源返回的媒体信息
|
||||
:param tmdb_media: TMDB 辅助识别结果
|
||||
:return: 已补充 TMDB 兼容字段的主媒体信息
|
||||
"""
|
||||
if not tmdb_media or tmdb_media.source != "themoviedb" or not tmdb_media.tmdb_id:
|
||||
return mediainfo
|
||||
|
||||
mediainfo.tmdb_id = tmdb_media.tmdb_id
|
||||
mediainfo.tmdb_info = tmdb_media.tmdb_info or mediainfo.tmdb_info
|
||||
if not mediainfo.category:
|
||||
mediainfo.category = tmdb_media.category
|
||||
if not mediainfo.genre_ids:
|
||||
mediainfo.genre_ids = list(tmdb_media.genre_ids or [])
|
||||
for field in ("imdb_id", "tvdb_id", "collection_id"):
|
||||
if not getattr(mediainfo, field, None):
|
||||
setattr(mediainfo, field, getattr(tmdb_media, field, None))
|
||||
return mediainfo
|
||||
|
||||
def supplement_tmdb_info(
|
||||
self,
|
||||
mediainfo: Optional[MediaInfo],
|
||||
metainfo: Optional[MetaBase] = None,
|
||||
) -> Optional[MediaInfo]:
|
||||
"""
|
||||
为任意主识别源补充 TMDB 辅助信息,同时保留原始媒体身份。
|
||||
|
||||
:param mediainfo: 主识别源返回的媒体信息
|
||||
:param metainfo: 原始标题解析信息
|
||||
:return: 已补充 TMDB 辅助字段的原媒体对象
|
||||
"""
|
||||
if not mediainfo:
|
||||
return None
|
||||
if mediainfo.tmdb_id and mediainfo.tmdb_info and mediainfo.genre_ids:
|
||||
return mediainfo
|
||||
tmdb_meta = self._build_tmdb_supplement_meta(mediainfo, metainfo)
|
||||
tmdb_module = self.modulemanager.get_running_module("TheMovieDbModule")
|
||||
if not tmdb_module:
|
||||
logger.warn("TMDB 模块未启用,无法补充 TMDB 辅助信息")
|
||||
return mediainfo
|
||||
try:
|
||||
tmdb_media = tmdb_module.recognize_media(
|
||||
meta=tmdb_meta,
|
||||
mtype=mediainfo.type,
|
||||
source="themoviedb",
|
||||
mediaid=str(mediainfo.tmdb_id) if mediainfo.tmdb_id else None,
|
||||
tmdbid=mediainfo.tmdb_id,
|
||||
episode_group=mediainfo.episode_group,
|
||||
cache=True,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.warn(f"{mediainfo.title_year} 补充 TMDB 辅助信息失败:{err}")
|
||||
return mediainfo
|
||||
if not tmdb_media:
|
||||
logger.warn(f"{mediainfo.title_year} 未匹配到 TMDB 辅助信息")
|
||||
return mediainfo
|
||||
return self._merge_tmdb_auxiliary(mediainfo, tmdb_media)
|
||||
|
||||
def _recognize_with_fallback_by_meta(
|
||||
self,
|
||||
metainfo: MetaBase,
|
||||
|
||||
+15
-11
@@ -27,11 +27,13 @@ class MediaServerChain(ChainBase):
|
||||
|
||||
def _sign_library_images(
|
||||
self, libraries: Optional[List[MediaServerLibrary]]
|
||||
) -> List[MediaServerLibrary]:
|
||||
) -> Optional[List[MediaServerLibrary]]:
|
||||
"""
|
||||
给媒体库列表中的封面和封面组添加代理签名。
|
||||
给媒体库列表中的封面和封面组添加代理签名,并保留提供方失败状态。
|
||||
"""
|
||||
for library in libraries or []:
|
||||
if libraries is None:
|
||||
return None
|
||||
for library in libraries:
|
||||
if library.image:
|
||||
library.image = self._sign_image_url(library.image)
|
||||
if library.image_list:
|
||||
@@ -40,21 +42,23 @@ class MediaServerChain(ChainBase):
|
||||
for image in library.image_list
|
||||
if image
|
||||
]
|
||||
return libraries or []
|
||||
return libraries
|
||||
|
||||
def _sign_play_item_images(
|
||||
self, items: Optional[List[MediaServerPlayItem]]
|
||||
) -> List[MediaServerPlayItem]:
|
||||
) -> Optional[List[MediaServerPlayItem]]:
|
||||
"""
|
||||
给媒体服务器播放条目中的图片 URL 添加代理签名。
|
||||
给媒体服务器播放条目中的图片 URL 添加代理签名,并保留提供方失败状态。
|
||||
"""
|
||||
for item in items or []:
|
||||
if items is None:
|
||||
return None
|
||||
for item in items:
|
||||
if item.image:
|
||||
item.image = self._sign_image_url(item.image)
|
||||
return items or []
|
||||
return items
|
||||
|
||||
def librarys(self, server: str, username: Optional[str] = None,
|
||||
hidden: bool = False) -> List[MediaServerLibrary]:
|
||||
hidden: bool = False) -> Optional[List[MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库
|
||||
"""
|
||||
@@ -151,7 +155,7 @@ class MediaServerChain(ChainBase):
|
||||
return self.run_module("mediaserver_tv_episodes", server=server, item_id=item_id)
|
||||
|
||||
def playing(self, server: str, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
@@ -165,7 +169,7 @@ class MediaServerChain(ChainBase):
|
||||
)
|
||||
|
||||
def latest(self, server: str, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
|
||||
+16
-6
@@ -313,7 +313,8 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
vote_average: Optional[float] = 0.0,
|
||||
vote_count: Optional[int] = 0,
|
||||
release_date: Optional[str] = "",
|
||||
page: Optional[int] = 1) -> List[dict]:
|
||||
page: Optional[int] = 1,
|
||||
raise_exception: bool = False) -> List[dict]:
|
||||
"""
|
||||
异步TMDB热门电影
|
||||
"""
|
||||
@@ -326,7 +327,8 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page)
|
||||
page=page,
|
||||
raise_exception=raise_exception)
|
||||
return [movie.to_dict() for movie in movies] if movies else []
|
||||
|
||||
@log_execution_time(logger=logger)
|
||||
@@ -339,7 +341,8 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
vote_average: Optional[float] = 0.0,
|
||||
vote_count: Optional[int] = 0,
|
||||
release_date: Optional[str] = "",
|
||||
page: Optional[int] = 1) -> List[dict]:
|
||||
page: Optional[int] = 1,
|
||||
raise_exception: bool = False) -> List[dict]:
|
||||
"""
|
||||
异步TMDB热门电视剧
|
||||
"""
|
||||
@@ -352,16 +355,23 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
vote_average=vote_average,
|
||||
vote_count=vote_count,
|
||||
release_date=release_date,
|
||||
page=page)
|
||||
page=page,
|
||||
raise_exception=raise_exception)
|
||||
return [tv.to_dict() for tv in tvs] if tvs else []
|
||||
|
||||
@log_execution_time(logger=logger)
|
||||
@cached(ttl=recommend_ttl, region=recommend_cache_region, skip_empty=True)
|
||||
async def async_tmdb_trending(self, page: Optional[int] = 1) -> List[dict]:
|
||||
async def async_tmdb_trending(
|
||||
self, page: Optional[int] = 1, raise_exception: bool = False
|
||||
) -> List[dict]:
|
||||
"""
|
||||
异步TMDB流行趋势
|
||||
"""
|
||||
infos = await TmdbChain().async_run_module("async_tmdb_trending", page=page)
|
||||
infos = await TmdbChain().async_run_module(
|
||||
"async_tmdb_trending",
|
||||
page=page,
|
||||
raise_exception=raise_exception,
|
||||
)
|
||||
return [info.to_dict() for info in infos] if infos else []
|
||||
|
||||
@log_execution_time(logger=logger)
|
||||
|
||||
+49
-8
@@ -904,6 +904,16 @@ class SubscribeChain(ChainBase):
|
||||
metainfo.begin_season = season
|
||||
if not media_source and not media_id and mediaid:
|
||||
media_source, media_id = parse_media_key(mediaid)
|
||||
resolved_source, resolved_media_id = resolve_media_identity(
|
||||
source=media_source,
|
||||
media_id=media_id,
|
||||
tmdbid=tmdbid,
|
||||
doubanid=doubanid,
|
||||
bangumiid=bangumiid,
|
||||
anilistid=anilistid,
|
||||
)
|
||||
if resolved_source and resolved_media_id:
|
||||
media_source, media_id = resolved_source, resolved_media_id
|
||||
if any((media_id, tmdbid, doubanid, bangumiid, anilistid)):
|
||||
mediainfo = self.recognize_media(
|
||||
meta=metainfo,
|
||||
@@ -926,10 +936,11 @@ class SubscribeChain(ChainBase):
|
||||
if season is None:
|
||||
season = meta.begin_season
|
||||
|
||||
# 使用名称识别兜底
|
||||
# 明确来源时只允许在同一来源内按名称兜底,不能切换主识别源。
|
||||
if not mediainfo:
|
||||
mediainfo = MediaChain().recognize_by_meta(
|
||||
metainfo,
|
||||
source=media_source,
|
||||
episode_group=episode_group,
|
||||
obtain_images=False,
|
||||
)
|
||||
@@ -1055,7 +1066,7 @@ class SubscribeChain(ChainBase):
|
||||
"anilistid": mediainfo.anilist_id,
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": metainfo.begin_season,
|
||||
"season": season,
|
||||
"poster": mediainfo.get_poster_image(),
|
||||
"backdrop": mediainfo.get_backdrop_image(),
|
||||
"vote": mediainfo.vote_average,
|
||||
@@ -1099,6 +1110,16 @@ class SubscribeChain(ChainBase):
|
||||
metainfo.begin_season = season
|
||||
if not media_source and not media_id and mediaid:
|
||||
media_source, media_id = parse_media_key(mediaid)
|
||||
resolved_source, resolved_media_id = resolve_media_identity(
|
||||
source=media_source,
|
||||
media_id=media_id,
|
||||
tmdbid=tmdbid,
|
||||
doubanid=doubanid,
|
||||
bangumiid=bangumiid,
|
||||
anilistid=anilistid,
|
||||
)
|
||||
if resolved_source and resolved_media_id:
|
||||
media_source, media_id = resolved_source, resolved_media_id
|
||||
if any((media_id, tmdbid, doubanid, bangumiid, anilistid)):
|
||||
mediainfo = await self.async_recognize_media(
|
||||
meta=metainfo,
|
||||
@@ -1121,10 +1142,11 @@ class SubscribeChain(ChainBase):
|
||||
if season is None:
|
||||
season = meta.begin_season
|
||||
|
||||
# 使用名称识别兜底
|
||||
# 明确来源时只允许在同一来源内按名称兜底,不能切换主识别源。
|
||||
if not mediainfo:
|
||||
mediainfo = await MediaChain().async_recognize_by_meta(
|
||||
metainfo,
|
||||
source=media_source,
|
||||
episode_group=episode_group,
|
||||
obtain_images=False,
|
||||
)
|
||||
@@ -1250,7 +1272,7 @@ class SubscribeChain(ChainBase):
|
||||
"anilistid": mediainfo.anilist_id,
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": metainfo.begin_season,
|
||||
"season": season,
|
||||
"poster": mediainfo.get_poster_image(),
|
||||
"backdrop": mediainfo.get_backdrop_image(),
|
||||
"vote": mediainfo.vote_average,
|
||||
@@ -1273,6 +1295,7 @@ class SubscribeChain(ChainBase):
|
||||
media_source=media_source,
|
||||
media_id=media_id,
|
||||
season=meta.begin_season if meta else None,
|
||||
episode_group=mediainfo.episode_group,
|
||||
):
|
||||
return True
|
||||
return False
|
||||
@@ -2266,7 +2289,8 @@ class SubscribeChain(ChainBase):
|
||||
anilistid=share_sub.get("anilistid"),
|
||||
media_source=share_sub.get("media_source"),
|
||||
media_id=share_sub.get("media_id"),
|
||||
season=share_sub.get("season")):
|
||||
season=share_sub.get("season"),
|
||||
episode_group=share_sub.get("episode_group")):
|
||||
continue
|
||||
# 已经订阅过跳过
|
||||
if subscribeoper.exist_history(tmdbid=share_sub.get("tmdbid"),
|
||||
@@ -2275,7 +2299,8 @@ class SubscribeChain(ChainBase):
|
||||
anilistid=share_sub.get("anilistid"),
|
||||
media_source=share_sub.get("media_source"),
|
||||
media_id=share_sub.get("media_id"),
|
||||
season=share_sub.get("season")):
|
||||
season=share_sub.get("season"),
|
||||
episode_group=share_sub.get("episode_group")):
|
||||
continue
|
||||
# 去除无效属性
|
||||
for key in list(share_sub.keys()):
|
||||
@@ -2306,6 +2331,7 @@ class SubscribeChain(ChainBase):
|
||||
year=subscribe_in.year,
|
||||
tmdbid=subscribe_in.tmdbid,
|
||||
season=subscribe_in.season,
|
||||
episode_group=subscribe_in.episode_group,
|
||||
doubanid=subscribe_in.doubanid,
|
||||
bangumiid=subscribe_in.bangumiid,
|
||||
anilistid=subscribe_in.anilistid,
|
||||
@@ -2835,7 +2861,12 @@ class SubscribeChain(ChainBase):
|
||||
# 统计订阅
|
||||
MoviePilotServerHelper.sub_done_async({
|
||||
"tmdbid": mediainfo.tmdb_id,
|
||||
"doubanid": mediainfo.douban_id
|
||||
"doubanid": mediainfo.douban_id,
|
||||
"bangumiid": mediainfo.bangumi_id,
|
||||
"anilistid": mediainfo.anilist_id,
|
||||
"media_source": subscribe.media_source,
|
||||
"media_id": subscribe.media_id,
|
||||
"season": subscribe.season,
|
||||
})
|
||||
|
||||
def remote_list(
|
||||
@@ -3490,6 +3521,11 @@ class SubscribeChain(ChainBase):
|
||||
{
|
||||
"tmdbid": subscribe.tmdbid,
|
||||
"doubanid": subscribe.doubanid,
|
||||
"bangumiid": subscribe.bangumiid,
|
||||
"anilistid": subscribe.anilistid,
|
||||
"media_source": subscribe.media_source,
|
||||
"media_id": subscribe.media_id,
|
||||
"season": subscribe.season,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3537,7 +3573,12 @@ class SubscribeChain(ChainBase):
|
||||
# 统计订阅
|
||||
MoviePilotServerHelper.sub_done_async({
|
||||
"tmdbid": subscribe.tmdbid,
|
||||
"doubanid": subscribe.doubanid
|
||||
"doubanid": subscribe.doubanid,
|
||||
"bangumiid": subscribe.bangumiid,
|
||||
"anilistid": subscribe.anilistid,
|
||||
"media_source": subscribe.media_source,
|
||||
"media_id": subscribe.media_id,
|
||||
"season": subscribe.season,
|
||||
})
|
||||
# 重新发送消息
|
||||
self.remote_list(channel=channel, userid=userid, source=source)
|
||||
|
||||
+224
-24
@@ -55,7 +55,7 @@ from app.schemas.types import (
|
||||
ContentType,
|
||||
)
|
||||
from app.utils.mixins import ConfigReloadMixin
|
||||
from app.utils.media import parse_media_key
|
||||
from app.utils.media import normalize_media_source, parse_media_key, resolve_media_identity
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
@@ -142,19 +142,8 @@ class JobManager:
|
||||
"""
|
||||
if not media:
|
||||
return None, season
|
||||
media_ids = {
|
||||
"themoviedb": media.tmdb_id,
|
||||
"douban": media.douban_id,
|
||||
"bangumi": media.bangumi_id,
|
||||
"anilist": media.anilist_id,
|
||||
}
|
||||
source = media.source
|
||||
if not source or media_ids.get(source) is None:
|
||||
source = next(
|
||||
(name for name, media_id in media_ids.items() if media_id is not None),
|
||||
source,
|
||||
)
|
||||
return (source, media_ids.get(source)), season
|
||||
source, media_id = resolve_media_identity(media=media)
|
||||
return (source, media_id), season
|
||||
|
||||
@staticmethod
|
||||
def __get_file_key(fileitem: FileItem) -> Optional[Tuple[str, str]]:
|
||||
@@ -794,6 +783,23 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
"TRANSFER_THREADS",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _requires_automatic_category(task: TransferTask) -> bool:
|
||||
"""
|
||||
判断当前整理任务是否需要根据媒体识别结果自动创建类别目录。
|
||||
|
||||
:param task: 整理任务
|
||||
:return: 是否必须具备自动分类结果
|
||||
"""
|
||||
target_directory = task.target_directory
|
||||
if target_directory and target_directory.media_category:
|
||||
return False
|
||||
if task.library_category_folder is not None:
|
||||
return bool(task.library_category_folder)
|
||||
return bool(
|
||||
target_directory and target_directory.library_category_folder
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
"""初始化文件整理处理链。"""
|
||||
super().__init__()
|
||||
@@ -922,6 +928,36 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
or "/@eaDir" in normalized_path
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __should_delete_empty_source_directories(
|
||||
task: TransferTask,
|
||||
delete_mounted_local_disk_empty_dirs: bool,
|
||||
mounted_filesystem_cache: Dict[Path, bool],
|
||||
) -> bool:
|
||||
"""
|
||||
判断移动整理后是否应删除源空目录。
|
||||
|
||||
仅在关闭挂载盘空目录清理且源存储为本地时检测文件系统,
|
||||
避免默认流程产生额外系统调用。
|
||||
"""
|
||||
if delete_mounted_local_disk_empty_dirs:
|
||||
return True
|
||||
if task.fileitem.storage != "local":
|
||||
return True
|
||||
|
||||
source_directory = (
|
||||
Path(task.target_directory.download_path)
|
||||
if task.target_directory and task.target_directory.download_path
|
||||
else Path(task.fileitem.path).parent
|
||||
)
|
||||
if source_directory not in mounted_filesystem_cache:
|
||||
mounted_filesystem_cache[source_directory] = (
|
||||
SystemUtils.is_network_filesystem(
|
||||
source_directory, include_local_fuse=True
|
||||
)
|
||||
)
|
||||
return not mounted_filesystem_cache[source_directory]
|
||||
|
||||
def __default_callback(
|
||||
self, task: TransferTask, transferinfo: TransferInfo, /
|
||||
) -> Tuple[bool, str]:
|
||||
@@ -1183,10 +1219,16 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
tasks = self.jobview.success_tasks(
|
||||
task.mediainfo, task.meta.begin_season
|
||||
)
|
||||
system_config_oper = SystemConfigOper()
|
||||
# 获取整理屏蔽词
|
||||
transfer_exclude_words = SystemConfigOper().get(
|
||||
transfer_exclude_words = system_config_oper.get(
|
||||
SystemConfigKey.TransferExcludeWords
|
||||
)
|
||||
# 挂载盘空目录清理默认开启
|
||||
delete_mounted_local_disk_empty_dirs = system_config_oper.get(
|
||||
SystemConfigKey.MountedLocalDiskDeleteEmptyDirs
|
||||
) is not False
|
||||
mounted_filesystem_cache: Dict[Path, bool] = {}
|
||||
processed_hashes = set()
|
||||
for t in tasks:
|
||||
if t.download_hash and t.download_hash not in processed_hashes:
|
||||
@@ -1203,7 +1245,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
logger.info(
|
||||
f"移动模式删除种子成功:{t.download_hash}"
|
||||
)
|
||||
if not t.download_hash and t.fileitem:
|
||||
if (
|
||||
not t.download_hash
|
||||
and t.fileitem
|
||||
and self.__should_delete_empty_source_directories(
|
||||
t,
|
||||
delete_mounted_local_disk_empty_dirs,
|
||||
mounted_filesystem_cache,
|
||||
)
|
||||
):
|
||||
# 删除剩余空目录
|
||||
StorageChain().delete_media_file(t.fileitem, delete_self=False)
|
||||
|
||||
@@ -1559,7 +1609,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
# 重置计数
|
||||
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
|
||||
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
|
||||
@@ -1709,8 +1761,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
|
||||
mediainfo_changed = True
|
||||
|
||||
# 如果未开启新增已入库媒体是否跟随TMDB信息变化则根据tmdbid查询之前的title
|
||||
if not settings.SCRAP_FOLLOW_TMDB:
|
||||
# TMDB 仅作为辅助信息合并,不能改变原识别源的主身份和标题。
|
||||
mediainfo = MediaChain().supplement_tmdb_info(mediainfo, task.meta)
|
||||
task.mediainfo = mediainfo
|
||||
|
||||
# 只有 TMDB 主源沿用历史 TMDB 标题,避免辅助 ID 改写其它识别源标题。
|
||||
if (
|
||||
not settings.SCRAP_FOLLOW_TMDB
|
||||
and normalize_media_source(mediainfo.source) == "themoviedb"
|
||||
):
|
||||
transfer_history = transferhis.get_by_type_tmdbid(
|
||||
tmdbid=mediainfo.tmdb_id, mtype=mediainfo.type.value
|
||||
)
|
||||
@@ -1727,7 +1786,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
return False, f"{task.fileitem.name} 已在整理队列中"
|
||||
|
||||
# 获取集数据
|
||||
if task.mediainfo.type == MediaType.TV and not task.episodes_info:
|
||||
if (
|
||||
task.mediainfo.type == MediaType.TV
|
||||
and task.mediainfo.tmdb_id
|
||||
and not task.episodes_info
|
||||
):
|
||||
# 判断注意season为0的情况
|
||||
season_num = task.mediainfo.season
|
||||
if season_num is None and task.meta.season_seq:
|
||||
@@ -1762,6 +1825,24 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
if not task.target_storage and task.target_directory:
|
||||
task.target_storage = task.target_directory.library_storage
|
||||
|
||||
if self._requires_automatic_category(task) and not task.mediainfo.category:
|
||||
if task.mediainfo.tmdb_id:
|
||||
error_message = "TMDB 信息未匹配到媒体分类,无法按媒体类别整理"
|
||||
else:
|
||||
error_message = "未识别到 TMDB 辅助信息,无法按媒体类别整理"
|
||||
logger.error(f"{task.fileitem.name} {error_message}")
|
||||
if callback:
|
||||
return callback(
|
||||
task,
|
||||
TransferInfo(
|
||||
success=False,
|
||||
fileitem=task.fileitem,
|
||||
transfer_type=task.transfer_type,
|
||||
message=error_message,
|
||||
),
|
||||
)
|
||||
return False, error_message
|
||||
|
||||
# 正在处理
|
||||
self.jobview.running_task(task)
|
||||
|
||||
@@ -2601,6 +2682,97 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
else None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_successful_move_history(history: Optional[TransferHistory]) -> bool:
|
||||
"""判断历史记录是否为已成功完成的移动类整理。"""
|
||||
return bool(
|
||||
history
|
||||
and history.status
|
||||
and history.mode
|
||||
and "move" in history.mode
|
||||
)
|
||||
|
||||
def _get_manual_transfer_history(
|
||||
self,
|
||||
fileitem: FileItem,
|
||||
transfer_history_oper: TransferHistoryOper,
|
||||
include_move_dest: bool = False,
|
||||
) -> Optional[TransferHistory]:
|
||||
"""查询文件源路径历史,并兼容从成功移动后的目标现址重新整理。"""
|
||||
history = transfer_history_oper.get_by_src(
|
||||
fileitem.path,
|
||||
storage=fileitem.storage,
|
||||
)
|
||||
if history or not include_move_dest:
|
||||
return history
|
||||
|
||||
history = transfer_history_oper.get_by_dest(
|
||||
fileitem.path,
|
||||
storage=fileitem.storage,
|
||||
)
|
||||
return history if self._is_successful_move_history(history) else None
|
||||
|
||||
def get_manual_transfer_histories(
|
||||
self,
|
||||
fileitems: List[FileItem],
|
||||
) -> List[TransferHistory]:
|
||||
"""
|
||||
查询文件或目录命中的成功整理记录,供手动整理界面显示重整状态。
|
||||
|
||||
:param fileitems: 待查询的文件或目录项
|
||||
:return: 去重后的成功整理记录
|
||||
"""
|
||||
transfer_history_oper = TransferHistoryOper()
|
||||
histories: Dict[int, TransferHistory] = {}
|
||||
for fileitem in fileitems or []:
|
||||
if not fileitem or not fileitem.path:
|
||||
continue
|
||||
storage = fileitem.storage or "local"
|
||||
if fileitem.type == "dir":
|
||||
matched_histories = transfer_history_oper.list_success_by_src(
|
||||
fileitem.path,
|
||||
storage=storage,
|
||||
recursive=True,
|
||||
)
|
||||
matched_histories.extend(
|
||||
transfer_history_oper.list_success_move_by_dest(
|
||||
fileitem.path,
|
||||
storage=storage,
|
||||
recursive=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
history = self._get_manual_transfer_history(
|
||||
fileitem=fileitem,
|
||||
transfer_history_oper=transfer_history_oper,
|
||||
include_move_dest=True,
|
||||
)
|
||||
matched_histories = [history] if history and history.status else []
|
||||
|
||||
for history in matched_histories:
|
||||
histories[history.id] = history
|
||||
return list(histories.values())
|
||||
|
||||
@staticmethod
|
||||
def _delete_manual_transfer_history(
|
||||
history: TransferHistory,
|
||||
transfer_history_oper: TransferHistoryOper,
|
||||
) -> Tuple[bool, str]:
|
||||
"""删除手动重整历史;非成功移动记录同时清理可能存在的旧目标。"""
|
||||
if (
|
||||
history.dest_fileitem
|
||||
and not TransferChain._is_successful_move_history(history)
|
||||
):
|
||||
dest_fileitem = FileItem(**history.dest_fileitem)
|
||||
storage_chain = StorageChain()
|
||||
if (
|
||||
storage_chain.exists(dest_fileitem)
|
||||
and not storage_chain.delete_media_file(dest_fileitem)
|
||||
):
|
||||
return False, f"{dest_fileitem.path} 删除失败"
|
||||
transfer_history_oper.delete(history.id)
|
||||
return True, ""
|
||||
|
||||
def do_transfer(
|
||||
self,
|
||||
fileitem: FileItem,
|
||||
@@ -2626,6 +2798,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
sync_extra_files: Optional[bool] = False,
|
||||
cleanup_dest_fileitem: Optional[FileItem] = None,
|
||||
continue_callback: Callable = None,
|
||||
reorganize: Optional[bool] = False,
|
||||
) -> Tuple[bool, Union[str, dict]]:
|
||||
"""
|
||||
执行一个复杂目录的整理操作
|
||||
@@ -2649,6 +2822,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
:param background: 是否后台运行
|
||||
:param manual: 是否手动整理
|
||||
:param preview: 是否仅预览
|
||||
:param reorganize: 是否清理已有成功记录后重新整理
|
||||
:param sync_extra_files: 是否在整理主视频文件时同步整理同媒体附加文件
|
||||
:param cleanup_dest_fileitem: 确认存在待整理任务后需要清理的旧目标文件
|
||||
:param continue_callback: 继续处理回调
|
||||
@@ -3079,11 +3253,33 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
raise OperationInterrupted()
|
||||
file_path = Path(file_item.path)
|
||||
|
||||
# 整理成功的不再处理
|
||||
if not force and not preview:
|
||||
transferd = TransferHistoryOper().get_by_src(
|
||||
file_item.path, storage=file_item.storage
|
||||
# 自动整理继续按全部历史去重;手动整理可清理失败记录,或按用户确认清理成功记录。
|
||||
if (not force or reorganize) and not preview:
|
||||
transfer_history_oper = TransferHistoryOper()
|
||||
transferd = self._get_manual_transfer_history(
|
||||
fileitem=file_item,
|
||||
transfer_history_oper=transfer_history_oper,
|
||||
include_move_dest=manual and reorganize,
|
||||
)
|
||||
if transferd:
|
||||
should_reorganize = manual and (
|
||||
reorganize or not transferd.status
|
||||
)
|
||||
if should_reorganize:
|
||||
state, message = self._delete_manual_transfer_history(
|
||||
history=transferd,
|
||||
transfer_history_oper=transfer_history_oper,
|
||||
)
|
||||
if not state:
|
||||
all_success = False
|
||||
logger.error(message)
|
||||
err_msgs.append(message)
|
||||
continue
|
||||
logger.info(
|
||||
f"{file_item.path} 已清理旧整理记录,继续重新整理。"
|
||||
)
|
||||
transferd = None
|
||||
|
||||
if transferd:
|
||||
skipped_history_count += 1
|
||||
if not transferd.status:
|
||||
@@ -3540,6 +3736,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
cleanup_dest_fileitem: Optional[FileItem] = None,
|
||||
bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None,
|
||||
reorganize: Optional[bool] = False,
|
||||
) -> Tuple[bool, Union[str, dict]]:
|
||||
"""
|
||||
手动整理,支持复杂条件,带进度显示
|
||||
@@ -3566,6 +3763,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
:param downloader: 下载器名称
|
||||
:param download_hash: 下载任务哈希
|
||||
:param preview: 是否仅预览
|
||||
:param reorganize: 是否清理已有成功记录后重新整理
|
||||
:param sync_extra_files: 是否同步整理同媒体附加文件
|
||||
:param cleanup_dest_fileitem: 确认存在待整理任务后需要清理的旧目标文件
|
||||
"""
|
||||
@@ -3616,6 +3814,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
downloader=downloader,
|
||||
download_hash=download_hash,
|
||||
preview=preview,
|
||||
reorganize=reorganize,
|
||||
sync_extra_files=sync_extra_files,
|
||||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||||
)
|
||||
@@ -3644,6 +3843,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
||||
downloader=downloader,
|
||||
download_hash=download_hash,
|
||||
preview=preview,
|
||||
reorganize=reorganize,
|
||||
sync_extra_files=sync_extra_files,
|
||||
cleanup_dest_fileitem=cleanup_dest_fileitem,
|
||||
)
|
||||
|
||||
+5
-2
@@ -569,6 +569,8 @@ class ConfigModel(BaseModel):
|
||||
LLM_MODEL: str = "deepseek-chat"
|
||||
# 思考模式/深度配置:off/auto/minimal/low/medium/high/max/xhigh
|
||||
LLM_THINKING_LEVEL: Optional[str] = "off"
|
||||
# OpenAI兼容接口API协议:auto(自动)/ chat_completions / responses
|
||||
LLM_API_PROTOCOL: str = "auto"
|
||||
# LLM是否支持图片输入,开启后消息图片会按多模态输入发送给模型
|
||||
LLM_SUPPORT_IMAGE_INPUT: bool = True
|
||||
# 是否启用音频输入,开启后用户语音会先转写为文本再进入 Agent
|
||||
@@ -749,8 +751,9 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
converted = int(value)
|
||||
return converted, str(converted) != str(original_value)
|
||||
elif expected_type is float:
|
||||
if isinstance(value, float):
|
||||
return value, str(value) != str(original_value)
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
converted = float(value)
|
||||
return converted, str(converted) != str(original_value)
|
||||
if isinstance(value, str):
|
||||
converted = float(value)
|
||||
return converted, str(converted) != str(original_value)
|
||||
|
||||
+44
-28
@@ -58,6 +58,7 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
# 插件智能体工具注册表缓存,插件启停或配置生效时主动失效。
|
||||
self._plugin_agent_tools_cache: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self._plugin_agent_tools_cache_lock = threading.Lock()
|
||||
self._plugin_agent_tools_revision: int = 0
|
||||
# 开发者模式监测插件修改
|
||||
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
|
||||
self.__start_monitor()
|
||||
@@ -143,6 +144,14 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
self._plugin_agent_tools_cache.clear()
|
||||
self._plugin_agent_tools_revision += 1
|
||||
|
||||
def get_plugin_agent_tools_revision(self) -> int:
|
||||
"""
|
||||
获取插件智能体工具注册表版本号。
|
||||
"""
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
return self._plugin_agent_tools_revision
|
||||
|
||||
def stop(self, pid: Optional[str] = None):
|
||||
"""
|
||||
@@ -1002,35 +1011,42 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
}]
|
||||
"""
|
||||
cache_key = pid or "__all__"
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
|
||||
if cached_tools is not None:
|
||||
return self._copy_plugin_agent_tools(cached_tools)
|
||||
while True:
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
cache_revision = self._plugin_agent_tools_revision
|
||||
cached_tools = self._plugin_agent_tools_cache.get(cache_key)
|
||||
if cached_tools is not None:
|
||||
return self._copy_plugin_agent_tools(cached_tools)
|
||||
|
||||
ret_tools = []
|
||||
# 创建字典快照避免并发修改
|
||||
running_plugins_snapshot = dict(self._running_plugins)
|
||||
for plugin_id, plugin in running_plugins_snapshot.items():
|
||||
if pid and pid != plugin_id:
|
||||
continue
|
||||
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(plugin.get_agent_tools):
|
||||
try:
|
||||
if not plugin.get_state():
|
||||
continue
|
||||
tools = plugin.get_agent_tools()
|
||||
if tools:
|
||||
ret_tools.append({
|
||||
"plugin_id": plugin_id,
|
||||
"plugin_name": plugin.plugin_name,
|
||||
"tools": tools
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
|
||||
ret_tools
|
||||
)
|
||||
return ret_tools
|
||||
ret_tools = []
|
||||
# 创建字典快照避免并发修改
|
||||
running_plugins_snapshot = dict(self._running_plugins)
|
||||
for plugin_id, plugin in running_plugins_snapshot.items():
|
||||
if pid and pid != plugin_id:
|
||||
continue
|
||||
if hasattr(plugin, "get_agent_tools") and ObjectUtils.check_method(
|
||||
plugin.get_agent_tools
|
||||
):
|
||||
try:
|
||||
if not plugin.get_state():
|
||||
continue
|
||||
tools = plugin.get_agent_tools()
|
||||
if tools:
|
||||
ret_tools.append({
|
||||
"plugin_id": plugin_id,
|
||||
"plugin_name": plugin.plugin_name,
|
||||
"tools": tools
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取插件 {plugin_id} 智能体工具出错:{str(e)}")
|
||||
with self._plugin_agent_tools_cache_lock:
|
||||
if cache_revision != self._plugin_agent_tools_revision:
|
||||
# 插件状态在注册表构建期间发生变化,重新读取以避免写回过期快照。
|
||||
continue
|
||||
self._plugin_agent_tools_cache[cache_key] = self._copy_plugin_agent_tools(
|
||||
ret_tools
|
||||
)
|
||||
return ret_tools
|
||||
|
||||
@staticmethod
|
||||
def get_plugin_remote_entry(plugin_id: str, dist_path: str) -> str:
|
||||
|
||||
@@ -148,14 +148,25 @@ class DownloadHistory(Base):
|
||||
def list_by_page(
|
||||
cls, db: Session, page: Optional[int] = 1, count: Optional[int] = 30
|
||||
):
|
||||
return db.query(DownloadHistory).offset((page - 1) * count).limit(count).all()
|
||||
return (
|
||||
db.query(DownloadHistory)
|
||||
.order_by(DownloadHistory.date.desc(), DownloadHistory.id.desc())
|
||||
.offset((page - 1) * count)
|
||||
.limit(count)
|
||||
.all()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@async_db_query
|
||||
async def async_list_by_page(
|
||||
cls, db: AsyncSession, page: Optional[int] = 1, count: Optional[int] = 30
|
||||
):
|
||||
result = await db.execute(select(cls).offset((page - 1) * count).limit(count))
|
||||
result = await db.execute(
|
||||
select(cls)
|
||||
.order_by(cls.date.desc(), cls.id.desc())
|
||||
.offset((page - 1) * count)
|
||||
.limit(count)
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -130,8 +130,9 @@ class Subscribe(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""按媒体身份与季号查询已有订阅。"""
|
||||
"""按媒体身份、季号与剧集组查询已有订阅。"""
|
||||
condition = cls._identity_condition(
|
||||
media_source, media_id, tmdbid, doubanid, bangumiid, anilistid
|
||||
)
|
||||
@@ -140,6 +141,7 @@ class Subscribe(Base):
|
||||
query = db.query(cls).filter(condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
return query.first()
|
||||
|
||||
@classmethod
|
||||
@@ -149,8 +151,9 @@ class Subscribe(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""异步按媒体身份与季号查询已有订阅。"""
|
||||
"""异步按媒体身份、季号与剧集组查询已有订阅。"""
|
||||
condition = cls._identity_condition(
|
||||
media_source, media_id, tmdbid, doubanid, bangumiid, anilistid
|
||||
)
|
||||
@@ -159,6 +162,7 @@ class Subscribe(Base):
|
||||
query = select(cls).filter(condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
result = await db.execute(query)
|
||||
return result.scalars().first()
|
||||
|
||||
@@ -169,9 +173,10 @@ class Subscribe(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
按订阅 owner 查询同一媒体的订阅行。
|
||||
按订阅 owner、媒体身份、季号与剧集组查询订阅行。
|
||||
"""
|
||||
if not username:
|
||||
return None
|
||||
@@ -183,6 +188,7 @@ class Subscribe(Base):
|
||||
query = db.query(cls).filter(cls.username == username, condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
return query.first()
|
||||
|
||||
@classmethod
|
||||
@@ -192,9 +198,10 @@ class Subscribe(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
异步按订阅 owner 查询同一媒体的订阅行。
|
||||
异步按订阅 owner、媒体身份、季号与剧集组查询订阅行。
|
||||
"""
|
||||
if not username:
|
||||
return None
|
||||
@@ -206,6 +213,7 @@ class Subscribe(Base):
|
||||
query = select(cls).filter(cls.username == username, condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
result = await db.execute(query)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
@@ -161,8 +161,9 @@ class SubscribeHistory(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""按媒体身份与季号查询订阅历史。"""
|
||||
"""按媒体身份、季号及可选剧集组查询订阅历史。"""
|
||||
condition = cls._identity_condition(
|
||||
media_source, media_id, tmdbid, doubanid, bangumiid, anilistid
|
||||
)
|
||||
@@ -171,6 +172,7 @@ class SubscribeHistory(Base):
|
||||
query = db.query(cls).filter(condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
return query.first()
|
||||
|
||||
@classmethod
|
||||
@@ -180,8 +182,9 @@ class SubscribeHistory(Base):
|
||||
doubanid: Optional[str] = None, bangumiid: Optional[int] = None,
|
||||
anilistid: Optional[int] = None, media_source: Optional[str] = None,
|
||||
media_id: Optional[str] = None, season: Optional[int] = None,
|
||||
episode_group: Optional[str] = None,
|
||||
):
|
||||
"""异步按媒体身份与季号查询订阅历史。"""
|
||||
"""异步按媒体身份、季号及可选剧集组查询订阅历史。"""
|
||||
condition = cls._identity_condition(
|
||||
media_source, media_id, tmdbid, doubanid, bangumiid, anilistid
|
||||
)
|
||||
@@ -190,5 +193,6 @@ class SubscribeHistory(Base):
|
||||
query = select(cls).filter(condition)
|
||||
if season is not None:
|
||||
query = query.filter(cls.season == season)
|
||||
query = query.filter(cls.episode_group == episode_group)
|
||||
result = await db.execute(query)
|
||||
return result.scalars().first()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import Boolean, Column, Index, Integer, JSON, String, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -184,7 +185,17 @@ class TransferHistory(Base):
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def get_by_src(cls, db: Session, src: str, storage: Optional[str] = None):
|
||||
def get_by_src(
|
||||
cls, db: Session, src: str, storage: Optional[str] = None
|
||||
) -> Optional["TransferHistory"]:
|
||||
"""
|
||||
按源路径和存储查询单条整理记录。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param src: 源路径
|
||||
:param storage: 源存储类型
|
||||
:return: 命中的整理记录,未命中时返回 None
|
||||
"""
|
||||
if storage:
|
||||
return db.query(cls).filter(cls.src == src,
|
||||
cls.src_storage == storage).first()
|
||||
@@ -193,8 +204,104 @@ class TransferHistory(Base):
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def get_by_dest(cls, db: Session, dest: str):
|
||||
return db.query(cls).filter(cls.dest == dest).first()
|
||||
def get_by_dest(
|
||||
cls, db: Session, dest: str, storage: Optional[str] = None
|
||||
) -> Optional["TransferHistory"]:
|
||||
"""
|
||||
按目标路径和存储查询单条整理记录。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param dest: 目标路径
|
||||
:param storage: 目标存储类型
|
||||
:return: 命中的整理记录,未命中时返回 None
|
||||
"""
|
||||
query = db.query(cls).filter(cls.dest == dest)
|
||||
if storage:
|
||||
query = query.filter(cls.dest_storage == storage)
|
||||
return query.first()
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def list_success_by_src(
|
||||
cls,
|
||||
db: Session,
|
||||
src: str,
|
||||
storage: Optional[str] = None,
|
||||
recursive: bool = False,
|
||||
) -> List["TransferHistory"]:
|
||||
"""
|
||||
按源路径查询成功整理记录,目录模式仅匹配其直接或间接子项。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param src: 源路径
|
||||
:param storage: 源存储类型
|
||||
:param recursive: 是否递归匹配目录子项
|
||||
:return: 命中的成功整理记录
|
||||
"""
|
||||
normalized_src = (
|
||||
Path(str(src).replace("\\", "/")).as_posix().rstrip("/") or "/"
|
||||
)
|
||||
query = db.query(cls).filter(cls.status.is_(True))
|
||||
if recursive:
|
||||
escaped_src = (
|
||||
normalized_src.replace("\\", "\\\\")
|
||||
.replace("%", "\\%")
|
||||
.replace("_", "\\_")
|
||||
)
|
||||
query = query.filter(
|
||||
or_(
|
||||
cls.src == normalized_src,
|
||||
cls.src.like(f"{escaped_src.rstrip('/')}/%", escape="\\"),
|
||||
)
|
||||
)
|
||||
else:
|
||||
query = query.filter(cls.src == normalized_src)
|
||||
if storage:
|
||||
query = query.filter(cls.src_storage == storage)
|
||||
return query.all()
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
def list_success_move_by_dest(
|
||||
cls,
|
||||
db: Session,
|
||||
dest: str,
|
||||
storage: Optional[str] = None,
|
||||
recursive: bool = False,
|
||||
) -> List["TransferHistory"]:
|
||||
"""
|
||||
按目标路径查询成功移动记录,供从媒体库现址发起重新整理时识别历史。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param dest: 目标路径
|
||||
:param storage: 目标存储类型
|
||||
:param recursive: 是否递归匹配目录子项
|
||||
:return: 命中的成功移动记录
|
||||
"""
|
||||
normalized_dest = (
|
||||
Path(str(dest).replace("\\", "/")).as_posix().rstrip("/") or "/"
|
||||
)
|
||||
query = db.query(cls).filter(
|
||||
cls.status.is_(True),
|
||||
cls.mode.contains("move"),
|
||||
)
|
||||
if recursive:
|
||||
escaped_dest = (
|
||||
normalized_dest.replace("\\", "\\\\")
|
||||
.replace("%", "\\%")
|
||||
.replace("_", "\\_")
|
||||
)
|
||||
query = query.filter(
|
||||
or_(
|
||||
cls.dest == normalized_dest,
|
||||
cls.dest.like(f"{escaped_dest.rstrip('/')}/%", escape="\\"),
|
||||
)
|
||||
)
|
||||
else:
|
||||
query = query.filter(cls.dest == normalized_dest)
|
||||
if storage:
|
||||
query = query.filter(cls.dest_storage == storage)
|
||||
return query.all()
|
||||
|
||||
@classmethod
|
||||
@db_query
|
||||
|
||||
@@ -24,6 +24,24 @@ class PluginDataOper(DbOper):
|
||||
else:
|
||||
PluginData(plugin_id=plugin_id, key=key, value=value).create(self._db)
|
||||
|
||||
async def async_save(self, plugin_id: str, key: str, value: Any) -> None:
|
||||
"""
|
||||
异步保存插件数据
|
||||
|
||||
:param plugin_id: 插件ID
|
||||
:param key: 数据键
|
||||
:param value: 数据值
|
||||
"""
|
||||
plugin = await PluginData.async_get_plugin_data_by_key(
|
||||
self._db, plugin_id, key
|
||||
)
|
||||
if plugin:
|
||||
await plugin.async_update(self._db, {"value": value})
|
||||
else:
|
||||
await PluginData(
|
||||
plugin_id=plugin_id, key=key, value=value
|
||||
).async_create(self._db)
|
||||
|
||||
def get_data(self, plugin_id: str, key: Optional[str] = None) -> Any:
|
||||
"""
|
||||
获取插件数据
|
||||
|
||||
+28
-24
@@ -45,6 +45,7 @@ class SubscribeOper(DbOper):
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": kwargs.get("season"),
|
||||
"episode_group": mediainfo.episode_group,
|
||||
}
|
||||
if username:
|
||||
subscribe = Subscribe.exists_by_username(self._db,
|
||||
@@ -106,6 +107,7 @@ class SubscribeOper(DbOper):
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": kwargs.get("season"),
|
||||
"episode_group": mediainfo.episode_group,
|
||||
}
|
||||
if username:
|
||||
subscribe = await Subscribe.async_exists_by_username(self._db,
|
||||
@@ -152,21 +154,22 @@ class SubscribeOper(DbOper):
|
||||
self, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
bangumiid: Optional[int] = None, anilistid: Optional[int] = None,
|
||||
media_source: Optional[str] = None, media_id: Optional[str] = None,
|
||||
season: Optional[int] = None,
|
||||
season: Optional[int] = None, episode_group: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判断是否存在
|
||||
按媒体身份、季号及可选剧集组判断订阅是否存在。
|
||||
"""
|
||||
return bool(Subscribe.exists(
|
||||
self._db,
|
||||
tmdbid=tmdbid,
|
||||
doubanid=doubanid,
|
||||
bangumiid=bangumiid,
|
||||
anilistid=anilistid,
|
||||
media_source=media_source,
|
||||
media_id=media_id,
|
||||
season=season,
|
||||
))
|
||||
identity_params = {
|
||||
"tmdbid": tmdbid,
|
||||
"doubanid": doubanid,
|
||||
"bangumiid": bangumiid,
|
||||
"anilistid": anilistid,
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": season,
|
||||
"episode_group": episode_group,
|
||||
}
|
||||
return bool(Subscribe.exists(self._db, **identity_params))
|
||||
|
||||
def get(self, sid: int) -> Subscribe:
|
||||
"""
|
||||
@@ -300,18 +303,19 @@ class SubscribeOper(DbOper):
|
||||
self, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,
|
||||
bangumiid: Optional[int] = None, anilistid: Optional[int] = None,
|
||||
media_source: Optional[str] = None, media_id: Optional[str] = None,
|
||||
season: Optional[int] = None,
|
||||
season: Optional[int] = None, episode_group: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判断是否存在订阅历史
|
||||
按媒体身份、季号及可选剧集组判断订阅历史是否存在。
|
||||
"""
|
||||
return bool(SubscribeHistory.exists(
|
||||
self._db,
|
||||
tmdbid=tmdbid,
|
||||
doubanid=doubanid,
|
||||
bangumiid=bangumiid,
|
||||
anilistid=anilistid,
|
||||
media_source=media_source,
|
||||
media_id=media_id,
|
||||
season=season,
|
||||
))
|
||||
identity_params = {
|
||||
"tmdbid": tmdbid,
|
||||
"doubanid": doubanid,
|
||||
"bangumiid": bangumiid,
|
||||
"anilistid": anilistid,
|
||||
"media_source": media_source,
|
||||
"media_id": media_id,
|
||||
"season": season,
|
||||
"episode_group": episode_group,
|
||||
}
|
||||
return bool(SubscribeHistory.exists(self._db, **identity_params))
|
||||
|
||||
@@ -101,6 +101,19 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
# 避免将__SYSTEMCONF内的值引用出去,会导致set时误判没有变动
|
||||
return copy.deepcopy(self.__SYSTEMCONF.get(key))
|
||||
|
||||
def increment(self, key: SystemConfigKey, step: int = 1) -> int:
|
||||
"""
|
||||
原子递增整数系统设置
|
||||
|
||||
:param key: 配置键
|
||||
:param step: 递增步长
|
||||
:return: 递增后的整数值
|
||||
"""
|
||||
with self._rlock:
|
||||
value = int(self.get(key) or 0) + step
|
||||
self.set(key, value)
|
||||
return value
|
||||
|
||||
def all(self):
|
||||
"""
|
||||
获取所有系统设置
|
||||
|
||||
@@ -78,20 +78,68 @@ class TransferHistoryOper(DbOper):
|
||||
"""
|
||||
return TransferHistory.list_by_title(self._db, title)
|
||||
|
||||
def get_by_src(self, src: str, storage: Optional[str] = None) -> TransferHistory:
|
||||
def get_by_src(
|
||||
self, src: str, storage: Optional[str] = None
|
||||
) -> Optional[TransferHistory]:
|
||||
"""
|
||||
按源查询转移记录
|
||||
:param src: 数据key
|
||||
:param storage: 存储类型
|
||||
:return: 命中的整理记录,未命中时返回 None
|
||||
"""
|
||||
return TransferHistory.get_by_src(self._db, src, storage)
|
||||
|
||||
def get_by_dest(self, dest: str) -> TransferHistory:
|
||||
def get_by_dest(
|
||||
self, dest: str, storage: Optional[str] = None
|
||||
) -> Optional[TransferHistory]:
|
||||
"""
|
||||
按转移路径查询转移记录
|
||||
:param dest: 数据key
|
||||
:param storage: 存储类型
|
||||
"""
|
||||
return TransferHistory.get_by_dest(self._db, dest)
|
||||
return TransferHistory.get_by_dest(self._db, dest, storage)
|
||||
|
||||
def list_success_by_src(
|
||||
self,
|
||||
src: str,
|
||||
storage: Optional[str] = None,
|
||||
recursive: bool = False,
|
||||
) -> List[TransferHistory]:
|
||||
"""
|
||||
按源路径查询成功整理记录。
|
||||
|
||||
:param src: 源路径
|
||||
:param storage: 源存储类型
|
||||
:param recursive: 是否递归匹配目录子项
|
||||
:return: 命中的成功整理记录
|
||||
"""
|
||||
return TransferHistory.list_success_by_src(
|
||||
self._db,
|
||||
src=src,
|
||||
storage=storage,
|
||||
recursive=recursive,
|
||||
)
|
||||
|
||||
def list_success_move_by_dest(
|
||||
self,
|
||||
dest: str,
|
||||
storage: Optional[str] = None,
|
||||
recursive: bool = False,
|
||||
) -> List[TransferHistory]:
|
||||
"""
|
||||
按目标路径查询成功移动记录。
|
||||
|
||||
:param dest: 目标路径
|
||||
:param storage: 目标存储类型
|
||||
:param recursive: 是否递归匹配目录子项
|
||||
:return: 命中的成功移动记录
|
||||
"""
|
||||
return TransferHistory.list_success_move_by_dest(
|
||||
self._db,
|
||||
dest=dest,
|
||||
storage=storage,
|
||||
recursive=recursive,
|
||||
)
|
||||
|
||||
def list_by_hash(self, download_hash: str) -> List[TransferHistory]:
|
||||
"""
|
||||
|
||||
+75
-16
@@ -188,16 +188,31 @@ class ImageHelper(metaclass=Singleton):
|
||||
return cache_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _validate_image(content: bytes) -> bool:
|
||||
"""验证图片"""
|
||||
def get_image_mime_type(content: bytes, verify: bool = True) -> Optional[str]:
|
||||
"""
|
||||
根据图片内容返回 Pillow 识别的图片 MIME 类型。
|
||||
|
||||
外部响应在写入缓存前需要完整校验;已校验的缓存只需读取格式头。
|
||||
非图片或可脚本化的 MIME 类型不作为图片代理响应。
|
||||
"""
|
||||
if not content:
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
Image.open(io.BytesIO(content)).verify()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn(f"Invalid image format: {e}")
|
||||
return False
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
image_format = (image.format or "").upper()
|
||||
if verify:
|
||||
image.verify()
|
||||
mime_type = Image.MIME.get(image_format)
|
||||
if (
|
||||
not mime_type
|
||||
or not mime_type.startswith("image/")
|
||||
or mime_type == "image/svg+xml"
|
||||
):
|
||||
return None
|
||||
return mime_type
|
||||
except Exception as err:
|
||||
logger.warning(f"Invalid image format: {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_request_params(url: str, proxy: Optional[bool], cookies: Optional[str | dict]) -> dict:
|
||||
@@ -224,6 +239,26 @@ class ImageHelper(metaclass=Singleton):
|
||||
"""
|
||||
获取图片(同步版本)
|
||||
"""
|
||||
result = self.fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
def fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
同步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
@@ -233,7 +268,9 @@ class ImageHelper(metaclass=Singleton):
|
||||
if use_cache:
|
||||
content = self.file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
return content
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
@@ -243,13 +280,13 @@ class ImageHelper(metaclass=Singleton):
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
# 验证图片
|
||||
if not self._validate_image(content):
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
self.file_cache.set(cache_path, content, region="images")
|
||||
return content
|
||||
return content, mime_type
|
||||
|
||||
async def async_fetch_image(
|
||||
self,
|
||||
@@ -260,6 +297,26 @@ class ImageHelper(metaclass=Singleton):
|
||||
"""
|
||||
获取图片(异步版本)
|
||||
"""
|
||||
result = await self.async_fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
async def async_fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
异步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
@@ -269,7 +326,9 @@ class ImageHelper(metaclass=Singleton):
|
||||
if use_cache:
|
||||
content = await self.async_file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
return content
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
@@ -279,10 +338,10 @@ class ImageHelper(metaclass=Singleton):
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
# 验证图片
|
||||
if not self._validate_image(content):
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
await self.async_file_cache.set(cache_path, content, region="images")
|
||||
return content
|
||||
return content, mime_type
|
||||
|
||||
+79
-2
@@ -2,9 +2,12 @@
|
||||
PassKey WebAuthn 辅助工具类
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import binascii
|
||||
from typing import Optional, Tuple, List, Dict, Any
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from webauthn import (
|
||||
@@ -28,9 +31,83 @@ from webauthn.helpers.structs import (
|
||||
from webauthn.helpers.cose import COSEAlgorithmIdentifier
|
||||
from webauthn.helpers.exceptions import InvalidRegistrationResponse
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import settings
|
||||
from app.helper.redis import RedisHelper
|
||||
from app.log import logger
|
||||
|
||||
PASSKEY_CHALLENGE_TTL_SECONDS = 5 * 60
|
||||
PasskeyChallengePurpose = Literal["authentication", "registration"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PasskeyChallenge:
|
||||
"""服务端保存的一次性 Passkey challenge 及其认证边界。"""
|
||||
|
||||
challenge: str
|
||||
purpose: PasskeyChallengePurpose
|
||||
user_id: Optional[int]
|
||||
|
||||
|
||||
class PasskeyChallengeStore:
|
||||
"""使用当前缓存后端签发并原子消费短时 Passkey challenge。"""
|
||||
|
||||
_cache = TTLCache(
|
||||
region="passkey_challenge",
|
||||
maxsize=4096,
|
||||
ttl=PASSKEY_CHALLENGE_TTL_SECONDS,
|
||||
)
|
||||
_memory_consume_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def issue(
|
||||
cls,
|
||||
*,
|
||||
challenge: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
user_id: Optional[int],
|
||||
) -> str:
|
||||
"""保存 challenge 并返回不携带认证事实的随机事务 token。"""
|
||||
transaction_token = secrets.token_urlsafe(32)
|
||||
cls._cache.set(
|
||||
transaction_token,
|
||||
PasskeyChallenge(
|
||||
challenge=challenge,
|
||||
purpose=purpose,
|
||||
user_id=user_id,
|
||||
),
|
||||
)
|
||||
return transaction_token
|
||||
|
||||
@classmethod
|
||||
def consume(
|
||||
cls,
|
||||
*,
|
||||
transaction_token: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
) -> Optional[PasskeyChallenge]:
|
||||
"""原子领取 challenge;任何完成尝试都会使事务失效。"""
|
||||
if not transaction_token:
|
||||
return None
|
||||
|
||||
if cls._cache.is_redis():
|
||||
challenge = RedisHelper().pop(
|
||||
transaction_token,
|
||||
region="passkey_challenge",
|
||||
)
|
||||
else:
|
||||
with cls._memory_consume_lock:
|
||||
try:
|
||||
challenge = cls._cache.pop(transaction_token)
|
||||
except KeyError:
|
||||
challenge = None
|
||||
|
||||
if not isinstance(challenge, PasskeyChallenge):
|
||||
return None
|
||||
if challenge.purpose != purpose:
|
||||
return None
|
||||
return challenge
|
||||
|
||||
|
||||
class PassKeyRegistrationVerificationError(Exception):
|
||||
"""Passkey 注册响应未通过 WebAuthn 安全校验。"""
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
import secrets
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, Optional
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.helper.redis import RedisHelper
|
||||
|
||||
PASSKEY_CHALLENGE_TTL_SECONDS = 5 * 60
|
||||
PasskeyChallengePurpose = Literal["authentication", "registration"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PasskeyChallenge:
|
||||
"""服务端保存的一次性 Passkey challenge 及其认证边界。"""
|
||||
|
||||
challenge: str
|
||||
purpose: PasskeyChallengePurpose
|
||||
user_id: Optional[int]
|
||||
|
||||
|
||||
class PasskeyChallengeStore:
|
||||
"""使用当前缓存后端签发并原子消费短时 Passkey challenge。"""
|
||||
|
||||
_cache = TTLCache(
|
||||
region="passkey_challenge",
|
||||
maxsize=4096,
|
||||
ttl=PASSKEY_CHALLENGE_TTL_SECONDS,
|
||||
)
|
||||
_memory_consume_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def issue(
|
||||
cls,
|
||||
*,
|
||||
challenge: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
user_id: Optional[int],
|
||||
) -> str:
|
||||
"""保存 challenge 并返回不携带认证事实的随机事务 token。"""
|
||||
transaction_token = secrets.token_urlsafe(32)
|
||||
cls._cache.set(
|
||||
transaction_token,
|
||||
PasskeyChallenge(
|
||||
challenge=challenge,
|
||||
purpose=purpose,
|
||||
user_id=user_id,
|
||||
),
|
||||
)
|
||||
return transaction_token
|
||||
|
||||
@classmethod
|
||||
def consume(
|
||||
cls,
|
||||
*,
|
||||
transaction_token: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
) -> Optional[PasskeyChallenge]:
|
||||
"""原子领取 challenge;任何完成尝试都会使事务失效。"""
|
||||
if not transaction_token:
|
||||
return None
|
||||
|
||||
if cls._cache.is_redis():
|
||||
challenge = RedisHelper().pop(
|
||||
transaction_token,
|
||||
region="passkey_challenge",
|
||||
)
|
||||
else:
|
||||
with cls._memory_consume_lock:
|
||||
try:
|
||||
challenge = cls._cache.pop(transaction_token)
|
||||
except KeyError:
|
||||
challenge = None
|
||||
|
||||
if not isinstance(challenge, PasskeyChallenge):
|
||||
return None
|
||||
if challenge.purpose != purpose:
|
||||
return None
|
||||
return challenge
|
||||
@@ -29,6 +29,7 @@ class MoviePilotServerHelper:
|
||||
_USAGE_REPORT_PATH = "/usage/report"
|
||||
_USAGE_STATISTIC_PATH = "/usage/statistic"
|
||||
_PLUGIN_INSTALL_PATH = "/plugin/install"
|
||||
_PLUGIN_RATING_PATH = "/plugin/rating"
|
||||
_PLUGIN_STATISTIC_PATH = "/plugin/statistic"
|
||||
_SUBSCRIBE_ADD_PATH = "/subscribe/add"
|
||||
_SUBSCRIBE_DONE_PATH = "/subscribe/done"
|
||||
@@ -398,6 +399,39 @@ class MoviePilotServerHelper:
|
||||
"""
|
||||
return await cls._async_get(cls._server_url(cls._PLUGIN_STATISTIC_PATH), timeout=10)
|
||||
|
||||
@classmethod
|
||||
async def async_plugin_ratings(cls, plugin_ids: Optional[List[str]] = None):
|
||||
"""
|
||||
异步批量查询中心端插件评分。
|
||||
"""
|
||||
params = {"plugin_ids": ",".join(plugin_ids)} if plugin_ids is not None else None
|
||||
return await cls._async_get(
|
||||
cls._server_url(cls._PLUGIN_RATING_PATH),
|
||||
params=params,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def async_plugin_rating(cls, plugin_id: str):
|
||||
"""
|
||||
异步查询中心端单个插件评分。
|
||||
"""
|
||||
return await cls._async_get(
|
||||
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def async_rate_plugin(cls, plugin_id: str, rating: float):
|
||||
"""
|
||||
异步提交当前安装实例的插件评分。
|
||||
"""
|
||||
return await cls._async_post_json(
|
||||
f"{cls._server_url(cls._PLUGIN_RATING_PATH)}/{quote(plugin_id, safe='')}",
|
||||
{"rating": rating},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def plugin_install(cls, plugin_id: str, payload: Dict[str, Any]):
|
||||
"""
|
||||
@@ -459,6 +493,58 @@ class MoviePilotServerHelper:
|
||||
return res.json()
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
async def async_get_plugin_ratings(
|
||||
cls,
|
||||
plugin_ids: Optional[List[str]] = None,
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
批量获取插件评分,中心端不可用时返回空结果。
|
||||
"""
|
||||
try:
|
||||
res = await cls.async_plugin_ratings(plugin_ids)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"批量获取插件评分失败:{str(err)}")
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
async def async_get_plugin_rating(cls, plugin_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取单个插件评分,中心端不可用时返回零评分。
|
||||
"""
|
||||
empty_rating = {
|
||||
"plugin_id": plugin_id,
|
||||
"average_rating": 0.0,
|
||||
"rating_count": 0,
|
||||
"user_rating": None,
|
||||
}
|
||||
try:
|
||||
res = await cls.async_plugin_rating(plugin_id)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"获取插件 {plugin_id} 评分失败:{str(err)}")
|
||||
return empty_rating
|
||||
|
||||
@classmethod
|
||||
async def async_submit_plugin_rating(
|
||||
cls,
|
||||
plugin_id: str,
|
||||
rating: float,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
提交插件评分,成功时返回最新评分结果。
|
||||
"""
|
||||
try:
|
||||
res = await cls.async_rate_plugin(plugin_id, rating)
|
||||
if res is not None and res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as err:
|
||||
logger.debug(f"提交插件 {plugin_id} 评分失败:{str(err)}")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def install_plugin_reg(cls, plugin_id: str, repo_url: Optional[str] = None) -> bool:
|
||||
"""
|
||||
|
||||
+15
-2
@@ -1,7 +1,10 @@
|
||||
from typing import Dict, List, Optional, Type, TypeVar, Generic, Iterator
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.module import ModuleManager
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas import DownloaderConf, MediaServerConf, NotificationConf, NotificationSwitchConf, ServiceInfo
|
||||
from app.schemas.types import NotificationType, SystemConfigKey, ModuleType
|
||||
|
||||
@@ -25,8 +28,18 @@ class ServiceConfigHelper:
|
||||
config_data = SystemConfigOper().get(config_key)
|
||||
if not config_data:
|
||||
return []
|
||||
# 直接使用 conf_type 来实例化配置对象
|
||||
return [conf_type(**conf) for conf in config_data]
|
||||
configs = []
|
||||
for conf in config_data:
|
||||
if not isinstance(conf, dict):
|
||||
logger.warn(f"{config_key.value} 配置格式不正确,已跳过:{conf}")
|
||||
continue
|
||||
try:
|
||||
# 直接使用 conf_type 来实例化配置对象
|
||||
configs.append(conf_type(**conf))
|
||||
except ValidationError as e:
|
||||
# 单条配置存在非法值时跳过,避免影响其它服务的初始化
|
||||
logger.error(f"{config_key.value} 配置 {conf.get('name')} 校验失败,已跳过:{e}")
|
||||
return configs
|
||||
|
||||
@staticmethod
|
||||
def get_downloader_configs() -> List[DownloaderConf]:
|
||||
|
||||
@@ -99,6 +99,8 @@
|
||||
"messages": {
|
||||
"模块不支持测试": "Module does not support testing",
|
||||
"网络请求失败": "Network request failed",
|
||||
"TMDB请求失败": "TMDB request failed",
|
||||
"媒体服务器请求失败": "Media server request failed",
|
||||
"附件保存失败": "Failed to save attachment",
|
||||
"该选择已失效,请重新发起选择": "This selection has expired. Please start the selection again",
|
||||
"会话不存在或无权访问": "The conversation does not exist or you do not have access",
|
||||
@@ -420,6 +422,10 @@
|
||||
"source": "插件 {plugin} 不存在或未安装",
|
||||
"target": "Plugin {plugin} does not exist or is not installed"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 未安装,无法评分",
|
||||
"target": "Plugin {plugin} is not installed and cannot be rated"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 不存在或未加载",
|
||||
"target": "Plugin {plugin} does not exist or is not loaded"
|
||||
|
||||
@@ -99,6 +99,8 @@
|
||||
"messages": {
|
||||
"模块不支持测试": "模块不支持测试",
|
||||
"网络请求失败": "网络请求失败",
|
||||
"TMDB请求失败": "TMDB请求失败",
|
||||
"媒体服务器请求失败": "媒体服务器请求失败",
|
||||
"豆瓣网络连接失败": "豆瓣网络连接失败",
|
||||
"Bangumi网络连接失败": "Bangumi网络连接失败",
|
||||
"fanart网络连接失败": "fanart网络连接失败",
|
||||
|
||||
@@ -99,6 +99,8 @@
|
||||
"messages": {
|
||||
"模块不支持测试": "模組不支援測試",
|
||||
"网络请求失败": "網路請求失敗",
|
||||
"TMDB请求失败": "TMDB 請求失敗",
|
||||
"媒体服务器请求失败": "媒體伺服器請求失敗",
|
||||
"附件保存失败": "附件儲存失敗",
|
||||
"该选择已失效,请重新发起选择": "此選擇已失效,請重新發起選擇",
|
||||
"会话不存在或无权访问": "會話不存在或無權存取",
|
||||
@@ -420,6 +422,10 @@
|
||||
"source": "插件 {plugin} 不存在或未安装",
|
||||
"target": "插件 {plugin} 不存在或未安裝"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 未安装,无法评分",
|
||||
"target": "插件 {plugin} 未安裝,無法評分"
|
||||
},
|
||||
{
|
||||
"source": "插件 {plugin} 不存在或未加载",
|
||||
"target": "插件 {plugin} 不存在或未載入"
|
||||
|
||||
@@ -282,13 +282,13 @@ class EmbyModule(_ModuleBase, _MediaServerBase[Emby]):
|
||||
) for season, episodes in seasoninfo.items()]
|
||||
|
||||
def mediaserver_playing(self, server: str, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: Emby = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_resume(num=count, username=username)
|
||||
|
||||
def mediaserver_play_url(self, server: str, item_id: Union[str, int]) -> Optional[str]:
|
||||
@@ -316,13 +316,13 @@ class EmbyModule(_ModuleBase, _MediaServerBase[Emby]):
|
||||
return server_obj.get_season_episode_ids(str(item_id), season)
|
||||
|
||||
def mediaserver_latest(self, server: Optional[str] = None, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: Emby = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_latest(num=count, username=username)
|
||||
|
||||
def mediaserver_latest_images(self,
|
||||
@@ -345,8 +345,7 @@ class EmbyModule(_ModuleBase, _MediaServerBase[Emby]):
|
||||
return []
|
||||
|
||||
links = []
|
||||
items: List[schemas.MediaServerPlayItem] = self.mediaserver_latest(server=server, count=count,
|
||||
username=username)
|
||||
items = self.mediaserver_latest(server=server, count=count, username=username) or []
|
||||
for item in items:
|
||||
if item.BackdropImageTags:
|
||||
image_url = server_obj.get_backdrop_url(item_id=item.id,
|
||||
|
||||
+41
-20
@@ -118,38 +118,47 @@ class Emby:
|
||||
logger.error(f"连接Library/VirtualFolders/Query 出错:" + str(e))
|
||||
return []
|
||||
|
||||
def __get_emby_librarys(self, username: Optional[str] = None) -> List[dict]:
|
||||
def __get_emby_librarys(self, username: Optional[str] = None) -> Optional[List[dict]]:
|
||||
"""
|
||||
获取Emby媒体库列表
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
if username:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return None
|
||||
url = f"{self._host}emby/Users/{user}/Views"
|
||||
params = {"api_key": self._apikey}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
if res:
|
||||
return res.json().get("Items")
|
||||
items = res.json().get("Items")
|
||||
return items if isinstance(items, list) else None
|
||||
else:
|
||||
logger.error(f"User/Views 未获取到返回数据")
|
||||
return []
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"连接User/Views 出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_librarys(self, username: Optional[str] = None, hidden: Optional[bool] = False) -> List[
|
||||
schemas.MediaServerLibrary]:
|
||||
def get_librarys(
|
||||
self,
|
||||
username: Optional[str] = None,
|
||||
hidden: Optional[bool] = False,
|
||||
) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库列表
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
source_libraries = self.__get_emby_librarys(username)
|
||||
if source_libraries is None:
|
||||
return None
|
||||
libraries = []
|
||||
for library in self.__get_emby_librarys(username) or []:
|
||||
for library in source_libraries:
|
||||
if hidden and self._sync_libraries and "all" not in self._sync_libraries \
|
||||
and library.get("Id") not in self._sync_libraries:
|
||||
continue
|
||||
@@ -182,7 +191,12 @@ class Emby:
|
||||
|
||||
def get_user(self, user_name: Optional[str] = None) -> Optional[Union[str, int]]:
|
||||
"""
|
||||
获得管理员用户
|
||||
获取用于查询用户范围数据的用户ID
|
||||
|
||||
优先匹配指定用户名,其次匹配媒体服务器配置用户名,最后回退管理员。
|
||||
|
||||
:param user_name: 优先匹配的用户名
|
||||
:return: 匹配到的用户ID,未找到可用用户时返回None
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return None
|
||||
@@ -194,15 +208,18 @@ class Emby:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
if res:
|
||||
users = res.json()
|
||||
# 先查询是否有与当前用户名称匹配的
|
||||
if user_name:
|
||||
for user in users:
|
||||
if user.get("Name") == user_name:
|
||||
return user.get("Id")
|
||||
candidate_usernames = []
|
||||
for candidate_username in (user_name, self._username):
|
||||
if candidate_username and candidate_username not in candidate_usernames:
|
||||
candidate_usernames.append(candidate_username)
|
||||
for candidate_username in candidate_usernames:
|
||||
for emby_user in users:
|
||||
if emby_user.get("Name") == candidate_username:
|
||||
return emby_user.get("Id")
|
||||
# 查询管理员
|
||||
for user in users:
|
||||
if user.get("Policy", {}).get("IsAdministrator"):
|
||||
return user.get("Id")
|
||||
for emby_user in users:
|
||||
if emby_user.get("Policy", {}).get("IsAdministrator"):
|
||||
return emby_user.get("Id")
|
||||
else:
|
||||
logger.error(f"Users 未获取到返回数据")
|
||||
except Exception as e:
|
||||
@@ -1206,6 +1223,8 @@ class Emby:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return None
|
||||
url = f"{self._host}Users/{user}/Items/Resume"
|
||||
params = {
|
||||
"Limit": 100,
|
||||
@@ -1266,7 +1285,7 @@ class Emby:
|
||||
logger.error(f"Users/Items/Resume 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Items/Resume出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_latest(self, num: Optional[int] = 20, username: Optional[str] = None) -> Optional[
|
||||
List[schemas.MediaServerPlayItem]]:
|
||||
@@ -1279,6 +1298,8 @@ class Emby:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return None
|
||||
url = f"{self._host}Users/{user}/Items/Latest"
|
||||
params = {
|
||||
"Limit": 100,
|
||||
@@ -1323,7 +1344,7 @@ class Emby:
|
||||
logger.error(f"Users/Items/Latest 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Items/Latest出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_user_library_folders(self):
|
||||
"""
|
||||
|
||||
@@ -1909,12 +1909,16 @@ class Feishu:
|
||||
) -> Optional[dict]:
|
||||
"""发送媒体列表消息,复用通知发送链路。"""
|
||||
lines = []
|
||||
image = message.image
|
||||
for index, media in enumerate(medias[:10], start=1):
|
||||
if not image:
|
||||
image = media.get_message_image()
|
||||
title = getattr(media, "title_year", None) or getattr(media, "title", None) or "未知媒体"
|
||||
lines.append(f"{index}. {title}")
|
||||
proxy_message = Notification(
|
||||
title=message.title,
|
||||
text="\n".join(lines),
|
||||
image=image,
|
||||
link=message.link,
|
||||
buttons=message.buttons,
|
||||
userid=message.userid,
|
||||
|
||||
@@ -174,6 +174,13 @@ class StorageBase(metaclass=ABCMeta):
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||
默认实现不区分「不存在」与「查询失败」,由具体存储按需覆写。
|
||||
"""
|
||||
return self.get_item(path)
|
||||
|
||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取父目录
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.modules.filemanager.storages import transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import WeakSingleton
|
||||
@@ -834,30 +835,53 @@ class AliPan(StorageBase, metaclass=WeakSingleton):
|
||||
return False
|
||||
return True
|
||||
|
||||
def __get_by_path_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
按路径查询文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
NotFound 系列错误码表示确认不存在,其余错误(网络失败、限流、
|
||||
权限或未知业务错误)均无法确认目标状态。
|
||||
"""
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/adrive/v1.0/openFile/get_by_path",
|
||||
json={
|
||||
"drive_id": drive_id or self._default_drive_id,
|
||||
"file_path": path.as_posix(),
|
||||
},
|
||||
no_error_log=True,
|
||||
)
|
||||
if resp is None:
|
||||
raise StorageQueryError(f"【阿里云盘】无法确认文件状态(请求失败): {path}")
|
||||
code = resp.get("code")
|
||||
if code:
|
||||
if "NotFound" in str(code):
|
||||
# 明确的不存在错误码,确认目标不存在
|
||||
return None
|
||||
raise StorageQueryError(
|
||||
f"【阿里云盘】查询文件信息出错: {path} - {code} {resp.get('message')}")
|
||||
return self.__get_fileitem(resp, parent=str(path.parent))
|
||||
|
||||
def get_item(self, path: Path, drive_id: str = None) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项
|
||||
"""
|
||||
try:
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/adrive/v1.0/openFile/get_by_path",
|
||||
json={
|
||||
"drive_id": drive_id or self._default_drive_id,
|
||||
"file_path": path.as_posix(),
|
||||
},
|
||||
no_error_log=True,
|
||||
)
|
||||
if not resp:
|
||||
return None
|
||||
if resp.get("code"):
|
||||
logger.debug(f"【阿里云盘】获取文件信息失败: {resp.get('message')}")
|
||||
return None
|
||||
return self.__get_fileitem(resp, parent=str(path.parent))
|
||||
return self.__get_by_path_item(path, drive_id=drive_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"【阿里云盘】获取文件信息失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
"""
|
||||
try:
|
||||
return self.__get_by_path_item(path)
|
||||
except StorageQueryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise StorageQueryError(f"【阿里云盘】查询文件信息失败: {path} - {e}") from e
|
||||
|
||||
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件夹,如不存在则创建
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
@@ -7,6 +8,7 @@ from app.core.config import global_vars, settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
@@ -147,6 +149,23 @@ class LocalStorage(StorageBase):
|
||||
return self.__get_fileitem(path)
|
||||
return self.__get_diritem(path)
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件或目录,无法确认状态时抛出 StorageQueryError。
|
||||
Path.exists() 会把部分 errno(如 EBADF/ELOOP)归入「不存在」,
|
||||
网络/FUSE 挂载抖动时会误判,这里用 stat 显式区分。
|
||||
"""
|
||||
try:
|
||||
path.stat()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as e:
|
||||
raise StorageQueryError(f"【本地】读取文件状态失败: {path} - {e}") from e
|
||||
try:
|
||||
return self.get_item(path)
|
||||
except OSError as e:
|
||||
raise StorageQueryError(f"【本地】读取文件信息失败: {path} - {e}") from e
|
||||
|
||||
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取文件详情
|
||||
@@ -195,11 +214,31 @@ class LocalStorage(StorageBase):
|
||||
"""
|
||||
return Path(fileitem.path)
|
||||
|
||||
def _copy_with_progress(self, src: Path, dest: Path):
|
||||
@staticmethod
|
||||
def _copy_with_target_permissions(src: Path, dest: Path) -> Path:
|
||||
"""
|
||||
复制文件内容和时间戳,并保留目标目录赋予新文件的权限。
|
||||
|
||||
目标目录的默认权限或继承 ACL 应作为媒体库的访问策略,复制完成后不能再用
|
||||
源文件权限覆盖,否则部分文件系统会清除已继承的 ACL。
|
||||
|
||||
:param src: 源文件路径
|
||||
:param dest: 目标文件路径
|
||||
:return: 目标文件路径
|
||||
"""
|
||||
src = Path(src)
|
||||
dest = Path(dest)
|
||||
src_stat = src.stat()
|
||||
shutil.copyfile(src, dest)
|
||||
os.utime(dest, ns=(src_stat.st_atime_ns, src_stat.st_mtime_ns))
|
||||
return dest
|
||||
|
||||
def _copy_with_progress(self, src: Path, dest: Path) -> bool:
|
||||
"""
|
||||
分块复制文件并回调进度
|
||||
"""
|
||||
total_size = src.stat().st_size
|
||||
src_stat = src.stat()
|
||||
total_size = src_stat.st_size
|
||||
copied_size = 0
|
||||
progress_callback = transfer_process(src.as_posix())
|
||||
try:
|
||||
@@ -217,8 +256,7 @@ class LocalStorage(StorageBase):
|
||||
if progress_callback:
|
||||
percent = copied_size / total_size * 100
|
||||
progress_callback(percent)
|
||||
# 保留文件时间戳、权限等信息
|
||||
shutil.copystat(src, dest)
|
||||
os.utime(dest, ns=(src_stat.st_atime_ns, src_stat.st_mtime_ns))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"【本地】复制文件 {src} 失败:{e}")
|
||||
@@ -273,11 +311,8 @@ class LocalStorage(StorageBase):
|
||||
if self._copy_with_progress(src, dest):
|
||||
return True
|
||||
else:
|
||||
code, message = SystemUtils.copy(src, dest)
|
||||
if code == 0:
|
||||
return True
|
||||
else:
|
||||
logger.error(f"【本地】复制文件失败:{message}")
|
||||
self._copy_with_target_permissions(src, dest)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】复制文件失败:{err}")
|
||||
return False
|
||||
@@ -303,11 +338,8 @@ class LocalStorage(StorageBase):
|
||||
src.unlink()
|
||||
return True
|
||||
else:
|
||||
code, message = SystemUtils.move(src, dest)
|
||||
if code == 0:
|
||||
return True
|
||||
else:
|
||||
logger.error(f"【本地】移动文件失败:{message}")
|
||||
shutil.move(src, dest, copy_function=self._copy_with_target_permissions)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"【本地】移动文件失败:{err}")
|
||||
return False
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.core.config import settings, global_vars
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.modules.filemanager.storages import transfer_process
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
@@ -28,6 +29,8 @@ lock = Lock()
|
||||
|
||||
MIN_U115_UPLOAD_PART_SIZE = 1 * 1024 * 1024
|
||||
U115_UPLOAD_PART_COUNT_TARGET = 96
|
||||
U115_DEFAULT_ACCEPTED_CODES = (0, 20004)
|
||||
U115_GET_INFO_ACCEPTED_CODES = (*U115_DEFAULT_ACCEPTED_CODES, 430004)
|
||||
U115_UPLOAD_PART_SIZE_STEPS = (
|
||||
10 * 1024 * 1024,
|
||||
16 * 1024 * 1024,
|
||||
@@ -297,10 +300,18 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
return result.get("data")
|
||||
|
||||
def _request_api(
|
||||
self, method: str, endpoint: str, result_key: Optional[str] = None, **kwargs
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
result_key: Optional[str] = None,
|
||||
*,
|
||||
accepted_codes: Tuple[int, ...] = U115_DEFAULT_ACCEPTED_CODES,
|
||||
**kwargs,
|
||||
) -> Optional[Union[dict, list]]:
|
||||
"""
|
||||
带错误处理和速率限制的API请求
|
||||
|
||||
:param accepted_codes: 当前接口可确认处理的业务码
|
||||
"""
|
||||
# 检查会话
|
||||
self._check_session()
|
||||
@@ -357,7 +368,13 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
time.sleep(self.limit_sleep_seconds)
|
||||
kwargs["retry_limit"] = retry_times - 1
|
||||
kwargs["no_error_log"] = no_error_log
|
||||
return self._request_api(method, endpoint, result_key, **kwargs)
|
||||
return self._request_api(
|
||||
method,
|
||||
endpoint,
|
||||
result_key,
|
||||
accepted_codes=accepted_codes,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# 处理请求错误
|
||||
try:
|
||||
@@ -375,11 +392,17 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
f"【115】{method} 请求 {endpoint} 错误 {e},等待 {sleep_duration} 秒后重试..."
|
||||
)
|
||||
time.sleep(sleep_duration)
|
||||
return self._request_api(method, endpoint, result_key, **kwargs)
|
||||
return self._request_api(
|
||||
method,
|
||||
endpoint,
|
||||
result_key,
|
||||
accepted_codes=accepted_codes,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# 返回数据
|
||||
ret_data = resp.json()
|
||||
if ret_data.get("code") not in (0, 20004):
|
||||
if ret_data.get("code") not in accepted_codes:
|
||||
error_msg = ret_data.get("message", "")
|
||||
if not no_error_log:
|
||||
logger.warn(f"【115】{method} 请求 {endpoint} 出错:{error_msg}")
|
||||
@@ -401,7 +424,13 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
time.sleep(self.limit_sleep_seconds)
|
||||
kwargs["retry_limit"] = retry_times - 1
|
||||
kwargs["no_error_log"] = no_error_log
|
||||
return self._request_api(method, endpoint, result_key, **kwargs)
|
||||
return self._request_api(
|
||||
method,
|
||||
endpoint,
|
||||
result_key,
|
||||
accepted_codes=accepted_codes,
|
||||
**kwargs,
|
||||
)
|
||||
return None
|
||||
|
||||
if result_key:
|
||||
@@ -906,38 +935,62 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
return True
|
||||
return False
|
||||
|
||||
def __get_info_item(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
查询指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
接口业务码 20004(记录不存在)、430004(路径不存在)与 0 一样
|
||||
视为确认结果,其余错误(网络失败、限流重试用尽、未知业务错误)
|
||||
均无法确认目标状态。
|
||||
"""
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/open/folder/get_info",
|
||||
data={"path": path.as_posix()},
|
||||
no_error_log=True,
|
||||
accepted_codes=U115_GET_INFO_ACCEPTED_CODES,
|
||||
)
|
||||
if resp is None:
|
||||
raise StorageQueryError(f"【115】无法确认文件状态(请求失败或接口错误): {path}")
|
||||
data = resp.get("data") if isinstance(resp, dict) else None
|
||||
if not data or not data.get("file_id"):
|
||||
# 115 对记录不存在和路径不存在返回不同业务码,两者都可确认目标不存在
|
||||
return None
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
fileid=str(data["file_id"]),
|
||||
path=path.as_posix() + ("/" if data["file_category"] == "0" else ""),
|
||||
type="file" if data["file_category"] == "1" else "dir",
|
||||
name=data["file_name"],
|
||||
basename=Path(data["file_name"]).stem,
|
||||
extension=Path(data["file_name"]).suffix[1:]
|
||||
if data["file_category"] == "1"
|
||||
else None,
|
||||
pickcode=data["pick_code"],
|
||||
size=data["size_byte"] if data["file_category"] == "1" else None,
|
||||
modify_time=data["utime"],
|
||||
)
|
||||
|
||||
def get_item(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项
|
||||
"""
|
||||
try:
|
||||
resp = self._request_api(
|
||||
"POST",
|
||||
"/open/folder/get_info",
|
||||
"data",
|
||||
data={"path": path.as_posix()},
|
||||
no_error_log=True,
|
||||
)
|
||||
if not resp:
|
||||
return None
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
fileid=str(resp["file_id"]),
|
||||
path=path.as_posix() + ("/" if resp["file_category"] == "0" else ""),
|
||||
type="file" if resp["file_category"] == "1" else "dir",
|
||||
name=resp["file_name"],
|
||||
basename=Path(resp["file_name"]).stem,
|
||||
extension=Path(resp["file_name"]).suffix[1:]
|
||||
if resp["file_category"] == "1"
|
||||
else None,
|
||||
pickcode=resp["pick_code"],
|
||||
size=resp["size_byte"] if resp["file_category"] == "1" else None,
|
||||
modify_time=resp["utime"],
|
||||
)
|
||||
return self.__get_info_item(path)
|
||||
except Exception as e:
|
||||
logger.debug(f"【115】获取文件信息失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件/目录项,无法确认状态时抛出 StorageQueryError。
|
||||
"""
|
||||
try:
|
||||
return self.__get_info_item(path)
|
||||
except StorageQueryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise StorageQueryError(f"【115】查询文件信息失败: {path} - {e}") from e
|
||||
|
||||
def get_folder(self, path: Path) -> Optional[schemas.FileItem]:
|
||||
"""
|
||||
获取指定路径的文件夹,如不存在则创建
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.schemas import (
|
||||
TransferRenameBuildEventData,
|
||||
TransferRenameEventData,
|
||||
)
|
||||
from app.schemas.exception import StorageQueryError
|
||||
from app.schemas.types import MediaType, ChainEventType
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
@@ -405,8 +406,23 @@ class TransHandler:
|
||||
# 判断是否要覆盖,附加文件强制覆盖
|
||||
overflag = False
|
||||
if not __is_extra_file(fileitem):
|
||||
# 目标文件
|
||||
target_item = target_oper.get_item(new_file)
|
||||
# 目标文件(严格查询:无法确认状态时拒绝覆盖,避免已有文件被误覆盖)
|
||||
try:
|
||||
target_item = target_oper.get_item_strict(new_file)
|
||||
except StorageQueryError as query_err:
|
||||
errmsg = f"无法确认目标文件状态,已跳过整理以避免误覆盖:{new_file} - {query_err}"
|
||||
logger.warn(errmsg)
|
||||
self.__update_result(
|
||||
result=result,
|
||||
success=False,
|
||||
message=errmsg,
|
||||
fileitem=fileitem,
|
||||
target_diritem=target_diritem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
)
|
||||
return result
|
||||
if target_item:
|
||||
# 目标文件已存在
|
||||
target_file = new_file
|
||||
|
||||
@@ -129,6 +129,8 @@ class SiteParserBase(metaclass=ABCMeta):
|
||||
self._user_basic_page = None
|
||||
# 用户基础信息参数
|
||||
self._user_basic_params = None
|
||||
# 用户基础信息请求方法
|
||||
self._user_basic_method = None
|
||||
# 用户基础信息请求头
|
||||
self._user_basic_headers = None
|
||||
|
||||
@@ -208,7 +210,8 @@ class SiteParserBase(metaclass=ABCMeta):
|
||||
self._get_page_content(
|
||||
url=urljoin(self._base_url, self._user_basic_page),
|
||||
params=self._user_basic_params,
|
||||
headers=self._user_basic_headers
|
||||
headers=self._user_basic_headers,
|
||||
**({"method": self._user_basic_method} if self._user_basic_method else {}),
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -325,12 +328,19 @@ class SiteParserBase(metaclass=ABCMeta):
|
||||
"""
|
||||
pass
|
||||
|
||||
def _get_page_content(self, url: str, params: dict = None, headers: dict = None):
|
||||
def _get_page_content(
|
||||
self,
|
||||
url: str,
|
||||
params: dict = None,
|
||||
headers: dict = None,
|
||||
method: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
获取页面内容
|
||||
:param url: 网页地址
|
||||
:param params: post参数
|
||||
:param headers: 额外的请求头
|
||||
:param method: 强制使用的 HTTP 请求方法
|
||||
:return:
|
||||
"""
|
||||
req_headers = None
|
||||
@@ -363,19 +373,19 @@ class SiteParserBase(metaclass=ABCMeta):
|
||||
cookie = self._site_cookie
|
||||
session = self._session
|
||||
|
||||
if params:
|
||||
if req_headers.get("Content-Type") == "application/json":
|
||||
if method == "post" or params:
|
||||
if (req_headers or {}).get("Content-Type") == "application/json":
|
||||
res = RequestUtils(cookies=cookie,
|
||||
session=session,
|
||||
timeout=60,
|
||||
proxies=proxies,
|
||||
headers=req_headers).post_res(url=url, json=params)
|
||||
headers=req_headers).post_res(url=url, json=params or {})
|
||||
else:
|
||||
res = RequestUtils(cookies=cookie,
|
||||
session=session,
|
||||
timeout=60,
|
||||
proxies=proxies,
|
||||
headers=req_headers).post_res(url=url, data=params)
|
||||
headers=req_headers).post_res(url=url, data=params or {})
|
||||
else:
|
||||
res = RequestUtils(cookies=cookie,
|
||||
session=session,
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
import json
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from app.log import logger
|
||||
from app.modules.indexer.parser import SiteParserBase, SiteSchema
|
||||
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
@@ -146,12 +148,44 @@ class HDDolbySiteUserInfo(SiteParserBase):
|
||||
|
||||
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
|
||||
"""
|
||||
解析未读消息链接,这里直接读出详情
|
||||
解析未读消息链接
|
||||
HDDolby 使用 API 模式,消息正文通过 _pase_unread_msgs 走网页接口读取。
|
||||
"""
|
||||
pass
|
||||
return None
|
||||
|
||||
def _parse_message_content(self, html_text) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
"""
|
||||
解析消息内容
|
||||
HDDolby 使用 API 模式,消息正文通过 _pase_unread_msgs 走网页接口读取。
|
||||
"""
|
||||
pass
|
||||
return None, None, None
|
||||
|
||||
def _pase_unread_msgs(self):
|
||||
"""
|
||||
HDDolby API 仅返回未读消息数量,正文需通过 NexusPHP 网页接口读取。
|
||||
"""
|
||||
if not self._site_cookie:
|
||||
if self.message_unread:
|
||||
logger.warn(
|
||||
f"{self._site_name} 未配置 Cookie,无法读取站点消息正文"
|
||||
f"(API 仅提供未读数量:{self.message_unread})"
|
||||
)
|
||||
return
|
||||
|
||||
nexus = NexusPhpSiteUserInfo(
|
||||
site_name=self._site_name,
|
||||
url=self._site_url,
|
||||
site_cookie=self._site_cookie,
|
||||
apikey=self.apikey,
|
||||
token=self.token,
|
||||
session=self._session,
|
||||
ua=self._ua,
|
||||
emulate=self._emulate,
|
||||
proxy=self._proxy,
|
||||
)
|
||||
nexus.message_unread = self.message_unread
|
||||
nexus.message_read_force = self.message_unread > 0
|
||||
nexus._pase_unread_msgs()
|
||||
self.message_unread_contents = nexus.message_unread_contents.copy()
|
||||
if self.message_unread_contents and not self.message_unread:
|
||||
self.message_unread = len(self.message_unread_contents)
|
||||
|
||||
@@ -2,109 +2,121 @@
|
||||
import json
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from app.log import logger
|
||||
from app.modules.indexer.parser import SiteParserBase, SiteSchema
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class TYemaSiteUserInfo(SiteParserBase):
|
||||
schema = SiteSchema.Yema
|
||||
class YemaSiteUserInfo(SiteParserBase):
|
||||
"""
|
||||
YemaPT 开放 API 用户数据解析器
|
||||
"""
|
||||
|
||||
def _parse_site_page(self, html_text: str):
|
||||
schema = SiteSchema.Yema
|
||||
request_mode = "apikey"
|
||||
|
||||
def _parse_site_page(self, html_text: str) -> None:
|
||||
"""
|
||||
获取站点页面地址
|
||||
配置 YemaPT 用户基本信息接口和认证请求头
|
||||
|
||||
:param html_text: API AuthKey 模式下的空首页数据
|
||||
"""
|
||||
self._user_traffic_page = None
|
||||
self._user_detail_page = None
|
||||
self._user_basic_page = "api/consumer/fetchSelfDetail"
|
||||
self._user_basic_page = "openApi/user/fetchBasicInfo.json"
|
||||
self._user_basic_params = {}
|
||||
self._user_basic_method = "post"
|
||||
self._user_detail_page = None
|
||||
self._user_traffic_page = None
|
||||
self._torrent_seeding_page = None
|
||||
self._sys_mail_unread_page = None
|
||||
self._user_mail_unread_page = None
|
||||
self._mail_unread_params = {}
|
||||
self._torrent_seeding_page = "/api/userTorrent/fetchSeedTorrentInfo"
|
||||
self._torrent_seeding_params = {
|
||||
# 虽然这个参数是无意义的,但这个 API 必须用 POST
|
||||
"status": "seeding"
|
||||
}
|
||||
self._torrent_seeding_headers = {}
|
||||
self._addition_headers = {
|
||||
"Authorization": self.apikey,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"User-Agent": self._ua,
|
||||
}
|
||||
|
||||
def _parse_logged_in(self, html_text):
|
||||
def _parse_user_base_info(self, html_text: str) -> None:
|
||||
"""
|
||||
判断是否登录成功, 通过判断是否存在用户信息
|
||||
暂时跳过检测,待后续优化
|
||||
:param html_text:
|
||||
:return:
|
||||
"""
|
||||
return True
|
||||
解析开放 API 返回的用户基本信息和促销流量
|
||||
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
"""
|
||||
解析用户基本信息,这里把_parse_user_traffic_info和_parse_user_detail_info合并到这里
|
||||
:param html_text: fetchBasicInfo 接口响应文本
|
||||
"""
|
||||
if not html_text:
|
||||
return None
|
||||
detail = json.loads(html_text)
|
||||
if not detail or not detail.get("success"):
|
||||
self.err_msg = "获取用户信息失败,未收到开放 API 响应"
|
||||
return
|
||||
user_info = detail.get("data", {})
|
||||
try:
|
||||
payload = json.loads(html_text)
|
||||
except (TypeError, json.JSONDecodeError) as err:
|
||||
self.err_msg = "获取用户信息失败,开放 API 响应不是有效 JSON"
|
||||
logger.warning(f"{self._site_name} {self.err_msg}:{str(err)}")
|
||||
return
|
||||
if not isinstance(payload, dict):
|
||||
self.err_msg = "获取用户信息失败,开放 API 响应结构无效"
|
||||
logger.warning(f"{self._site_name} {self.err_msg}")
|
||||
return
|
||||
if not payload.get("success") or not isinstance(payload.get("data"), dict):
|
||||
self.err_msg = payload.get("errorMessage") or "获取用户信息失败"
|
||||
logger.warning(f"{self._site_name} 获取用户信息失败:{self.err_msg}")
|
||||
return
|
||||
|
||||
user_info = payload["data"]
|
||||
self.userid = user_info.get("id")
|
||||
self.username = user_info.get("name")
|
||||
self.user_level = str(user_info.get("level")) if user_info.get("level") is not None else None
|
||||
self.user_level = str(user_info.get("level")) \
|
||||
if user_info.get("level") is not None else None
|
||||
self.join_at = StringUtils.unify_datetime_str(user_info.get("registerTime"))
|
||||
|
||||
self.upload = user_info.get('uploadSize')
|
||||
# 使用 promotionDownloadSize 获取真实下载量(考虑促销因素)
|
||||
if "promotionDownloadSize" in user_info:
|
||||
self.download = user_info.get('promotionDownloadSize')
|
||||
else:
|
||||
self.download = user_info.get('downloadSize')
|
||||
self.upload = int(user_info.get("promotionUploadSize") or 0)
|
||||
self.download = int(user_info.get("promotionDownloadSize") or 0)
|
||||
self.ratio = round(self.upload / (self.download or 1), 2)
|
||||
self.bonus = user_info.get("bonus")
|
||||
self.message_unread = 0
|
||||
self.bonus = float(user_info.get("bonus") or 0)
|
||||
|
||||
def _parse_user_traffic_info(self, html_text: str):
|
||||
def _parse_user_traffic_info(self, html_text: str) -> None:
|
||||
"""
|
||||
解析用户流量信息
|
||||
跳过独立流量页面,用户基本信息接口已经返回促销流量
|
||||
|
||||
:param html_text: 未使用的页面文本
|
||||
"""
|
||||
pass
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
def _parse_user_detail_info(self, html_text: str) -> None:
|
||||
"""
|
||||
解析用户详细信息
|
||||
跳过独立用户详情页面,开放 API 未提供该接口
|
||||
|
||||
:param html_text: 未使用的页面文本
|
||||
"""
|
||||
pass
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
|
||||
def _parse_user_torrent_seeding_info(
|
||||
self,
|
||||
html_text: str,
|
||||
multi_page: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
解析用户做种信息
|
||||
跳过做种统计,开放 API 未提供该接口
|
||||
|
||||
:param html_text: 未使用的页面文本
|
||||
:param multi_page: 是否为后续分页
|
||||
:return: 始终返回 None
|
||||
"""
|
||||
if not html_text:
|
||||
return None
|
||||
seeding_info = json.loads(html_text)
|
||||
if not seeding_info or not seeding_info.get("success") or not seeding_info.get("data"):
|
||||
return None
|
||||
|
||||
torrents = seeding_info.get("data")
|
||||
|
||||
self.seeding += torrents.get("num")
|
||||
self.seeding_size += torrents.get("fileSize")
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
|
||||
return next_page
|
||||
return None
|
||||
|
||||
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
|
||||
"""
|
||||
解析未读消息链接,这里直接读出详情
|
||||
"""
|
||||
pass
|
||||
跳过站内消息,开放 API 未提供该接口
|
||||
|
||||
def _parse_message_content(self, html_text) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
:param html_text: 未使用的页面文本
|
||||
:param msg_links: 未使用的消息链接容器
|
||||
:return: 始终返回 None
|
||||
"""
|
||||
解析消息内容
|
||||
return None
|
||||
|
||||
def _parse_message_content(
|
||||
self,
|
||||
html_text: str,
|
||||
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||||
"""
|
||||
pass
|
||||
跳过消息详情,开放 API 未提供该接口
|
||||
|
||||
:param html_text: 未使用的页面文本
|
||||
:return: 三个空值
|
||||
"""
|
||||
return None, None, None
|
||||
|
||||
+202
-138
@@ -1,34 +1,23 @@
|
||||
from typing import Tuple, List, Optional
|
||||
import base64
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas import MediaType
|
||||
from app.utils.http import RequestUtils, AsyncRequestUtils
|
||||
from app.utils.http import AsyncRequestUtils, RequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class YemaSpider:
|
||||
"""
|
||||
YemaPT API
|
||||
YemaPT 开放 API 索引器
|
||||
"""
|
||||
_indexerid = None
|
||||
_domain = None
|
||||
_name = ""
|
||||
_proxy = None
|
||||
_cookie = None
|
||||
_ua = None
|
||||
_size = 40
|
||||
_searchurl = "%sapi/torrent/fetchOpenTorrentList"
|
||||
_downloadurl = "%sapi/torrent/download?id=%s"
|
||||
_pageurl = "%s#/torrent/detail/%s/"
|
||||
_timeout = 15
|
||||
|
||||
# 分类
|
||||
_size = 100
|
||||
_movie_category = [4]
|
||||
_tv_category = [5, 13, 14, 17, 15, 6, 16]
|
||||
_tv_category = [5, 6, 13, 14, 15, 16, 17]
|
||||
|
||||
# 标签 https://wiki.yemapt.org/developer/constants
|
||||
_labels = {
|
||||
"1": "禁转",
|
||||
"2": "首发",
|
||||
@@ -44,173 +33,248 @@ class YemaSpider:
|
||||
"12": "完结",
|
||||
}
|
||||
|
||||
def __init__(self, indexer: dict):
|
||||
"""
|
||||
初始化 YemaPT 开放 API 索引器
|
||||
|
||||
:param indexer: 合并站点认证信息后的索引配置
|
||||
"""
|
||||
indexer = indexer or {}
|
||||
self._name = indexer.get("name") or "YemaPT"
|
||||
self._site_url = str(indexer.get("domain") or "https://www.yemapt.org/").rstrip("/")
|
||||
self._proxy = settings.PROXY if indexer.get("proxy") else None
|
||||
self._use_proxy = bool(indexer.get("proxy"))
|
||||
self._user_agent = indexer.get("ua") or settings.USER_AGENT
|
||||
self._api_key = indexer.get("apikey")
|
||||
self._timeout = indexer.get("timeout") or 15
|
||||
self._search_url = f"{self._site_url}/openApi/torrent/fetchOpenTorrentList.json"
|
||||
self._download_key_url = f"{self._site_url}/openApi/torrent/generateDownloadKey.json"
|
||||
|
||||
@classmethod
|
||||
def get_search_page_size(cls, keyword: Optional[str] = None) -> Optional[int]:
|
||||
"""
|
||||
获取搜索接口单页容量。
|
||||
获取搜索接口单页容量
|
||||
|
||||
:param keyword: 搜索关键字,YemaPT 不按关键字改变分页容量
|
||||
:return: 搜索接口单页容量
|
||||
"""
|
||||
return cls._size
|
||||
|
||||
def __init__(self, indexer: dict):
|
||||
self.systemconfig = SystemConfigOper()
|
||||
if indexer:
|
||||
self._indexerid = indexer.get('id')
|
||||
self._domain = indexer.get('domain')
|
||||
self._searchurl = self._searchurl % self._domain
|
||||
self._name = indexer.get('name')
|
||||
if indexer.get('proxy'):
|
||||
self._proxy = settings.PROXY
|
||||
self._cookie = indexer.get('cookie')
|
||||
self._ua = indexer.get('ua')
|
||||
self._timeout = indexer.get('timeout') or 15
|
||||
|
||||
def __get_params(self, keyword: str = None, page: Optional[int] = 0) -> dict:
|
||||
def _request_headers(self) -> dict:
|
||||
"""
|
||||
获取搜索参数
|
||||
构造开放 API 请求头
|
||||
|
||||
:return: 不包含 Cookie 的 AuthKey 请求头
|
||||
"""
|
||||
return {
|
||||
"Authorization": self._api_key,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"User-Agent": self._user_agent,
|
||||
}
|
||||
|
||||
def _build_params(
|
||||
self,
|
||||
keyword: Optional[str],
|
||||
page: Optional[int],
|
||||
) -> dict:
|
||||
"""
|
||||
构造公开种子列表查询参数
|
||||
|
||||
:param keyword: 搜索关键字
|
||||
:param page: MoviePilot 从 0 开始的页码
|
||||
:return: YemaPT 开放 API 请求体
|
||||
"""
|
||||
params = {
|
||||
"pageParam": {
|
||||
"current": page + 1,
|
||||
"current": int(page or 0) + 1,
|
||||
"pageSize": self._size,
|
||||
"total": self._size
|
||||
},
|
||||
"sorter": {}
|
||||
"sorter": {},
|
||||
}
|
||||
if keyword:
|
||||
params.update({
|
||||
"keyword": keyword,
|
||||
})
|
||||
params["keyword"] = keyword
|
||||
return params
|
||||
|
||||
def __parse_result(self, results: List[dict]) -> List[dict]:
|
||||
def _parse_result(self, results: List[dict]) -> List[dict]:
|
||||
"""
|
||||
解析搜索结果
|
||||
将开放 API 种子数据转换为 MoviePilot 标准字段
|
||||
|
||||
:param results: 公开种子列表接口 data 数组
|
||||
:return: MoviePilot 标准种子字典列表
|
||||
"""
|
||||
torrents = []
|
||||
if not results:
|
||||
return torrents
|
||||
|
||||
for result in results:
|
||||
category_value = result.get('categoryId')
|
||||
for result in results or []:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
category_value = result.get("categoryId")
|
||||
if category_value in self._tv_category:
|
||||
category = MediaType.TV.value
|
||||
elif category_value in self._movie_category:
|
||||
category = MediaType.MOVIE.value
|
||||
else:
|
||||
category = MediaType.UNKNOWN.value
|
||||
pass
|
||||
|
||||
torrentLabelIds = result.get('tagList', []) or []
|
||||
torrentLabels = []
|
||||
for labelId in torrentLabelIds:
|
||||
if self._labels.get(labelId) is not None:
|
||||
torrentLabels.append(self._labels.get(labelId))
|
||||
pass
|
||||
pass
|
||||
torrent = {
|
||||
'title': result.get('showName'),
|
||||
'description': result.get('shortDesc'),
|
||||
'enclosure': self.__get_download_url(result.get('id')),
|
||||
'pubdate': StringUtils.unify_datetime_str(result.get('listingTime')),
|
||||
'size': result.get('fileSize'),
|
||||
'seeders': result.get('seedNum'),
|
||||
'peers': result.get('leechNum'),
|
||||
'grabs': result.get('completedNum'),
|
||||
'downloadvolumefactor': self.__get_downloadvolumefactor(result.get('downloadPromotion')),
|
||||
'uploadvolumefactor': self.__get_uploadvolumefactor(result.get('uploadPromotion')),
|
||||
'freedate': StringUtils.unify_datetime_str(result.get('downloadPromotionEndTime')),
|
||||
'page_url': self._pageurl % (self._domain, result.get('id')),
|
||||
'labels': torrentLabels,
|
||||
'category': category
|
||||
}
|
||||
torrents.append(torrent)
|
||||
|
||||
labels = [
|
||||
self._labels[label_id]
|
||||
for label_id in result.get("tagList") or []
|
||||
if label_id in self._labels
|
||||
]
|
||||
torrent_id = result.get("id")
|
||||
torrents.append({
|
||||
"title": result.get("showName"),
|
||||
"description": result.get("shortDesc"),
|
||||
"enclosure": self._build_download_url(torrent_id),
|
||||
"pubdate": StringUtils.unify_datetime_str(result.get("listingTime")),
|
||||
"size": result.get("fileSize"),
|
||||
"seeders": result.get("seedNum"),
|
||||
"peers": result.get("leechNum"),
|
||||
"grabs": result.get("completedNum"),
|
||||
"downloadvolumefactor": self._download_factor(result.get("downloadPromotion")),
|
||||
"uploadvolumefactor": self._upload_factor(result.get("uploadPromotion")),
|
||||
"freedate": StringUtils.unify_datetime_str(result.get("downloadPromotionEndTime")),
|
||||
"page_url": f"{self._site_url}/#/torrent/detail/{torrent_id}/",
|
||||
"labels": labels,
|
||||
"hit_and_run": bool(result.get("hrPunishEnable")),
|
||||
"category": category,
|
||||
})
|
||||
return torrents
|
||||
|
||||
def search(self, keyword: str,
|
||||
mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
|
||||
"""
|
||||
搜索
|
||||
def _process_search_response(self, response) -> Tuple[bool, List[dict]]:
|
||||
"""
|
||||
校验开放 API 通用响应并解析搜索结果
|
||||
|
||||
res = RequestUtils(
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": f"{self._ua}",
|
||||
"Accept": "application/json, text/plain, */*"
|
||||
},
|
||||
cookies=self._cookie,
|
||||
:param response: RequestUtils 返回的响应对象
|
||||
:return: 是否失败及标准种子列表
|
||||
"""
|
||||
if response is None:
|
||||
logger.warning(f"{self._name} 搜索失败,无法连接开放 API")
|
||||
return True, []
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"{self._name} 搜索失败,HTTP 错误码:{response.status_code}")
|
||||
return True, []
|
||||
try:
|
||||
payload = response.json() or {}
|
||||
except (TypeError, ValueError) as err:
|
||||
logger.warning(f"{self._name} 搜索响应不是有效 JSON:{str(err)}")
|
||||
return True, []
|
||||
if not isinstance(payload, dict):
|
||||
logger.warning(f"{self._name} 搜索响应结构无效")
|
||||
return True, []
|
||||
if not payload.get("success"):
|
||||
logger.warning(f"{self._name} 搜索失败:{payload.get('errorMessage') or '未知错误'}")
|
||||
return True, []
|
||||
results = payload.get("data")
|
||||
if not isinstance(results, list):
|
||||
logger.warning(f"{self._name} 搜索响应 data 不是数组")
|
||||
return True, []
|
||||
return False, self._parse_result(results)
|
||||
|
||||
def search(
|
||||
self,
|
||||
keyword: Optional[str],
|
||||
mtype: MediaType = None,
|
||||
page: Optional[int] = 0,
|
||||
) -> Tuple[bool, List[dict]]:
|
||||
"""
|
||||
同步搜索 YemaPT 公开种子
|
||||
|
||||
:param keyword: 搜索关键字
|
||||
:param mtype: MoviePilot 媒体类型,开放 API 不支持直接按媒体类型查询
|
||||
:param page: MoviePilot 从 0 开始的页码
|
||||
:return: 是否失败及标准种子列表
|
||||
"""
|
||||
if not self._api_key:
|
||||
logger.warning(f"{self._name} 未配置 API AuthKey")
|
||||
return True, []
|
||||
response = RequestUtils(
|
||||
headers=self._request_headers(),
|
||||
proxies=self._proxy,
|
||||
referer=f"{self._domain}",
|
||||
timeout=self._timeout
|
||||
).post_res(url=self._searchurl, json=self.__get_params(keyword, page))
|
||||
if res and res.status_code == 200:
|
||||
results = res.json().get('data', []) or []
|
||||
return False, self.__parse_result(results)
|
||||
elif res is not None:
|
||||
logger.warn(f"{self._name} 搜索失败,错误码:{res.status_code}")
|
||||
return True, []
|
||||
else:
|
||||
logger.warn(f"{self._name} 搜索失败,无法连接 {self._domain}")
|
||||
return True, []
|
||||
timeout=self._timeout,
|
||||
).post_res(
|
||||
url=self._search_url,
|
||||
json=self._build_params(keyword, page),
|
||||
)
|
||||
return self._process_search_response(response)
|
||||
|
||||
async def async_search(self, keyword: str,
|
||||
mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
|
||||
async def async_search(
|
||||
self,
|
||||
keyword: Optional[str],
|
||||
mtype: MediaType = None,
|
||||
page: Optional[int] = 0,
|
||||
) -> Tuple[bool, List[dict]]:
|
||||
"""
|
||||
异步搜索
|
||||
异步搜索 YemaPT 公开种子
|
||||
|
||||
:param keyword: 搜索关键字
|
||||
:param mtype: MoviePilot 媒体类型,开放 API 不支持直接按媒体类型查询
|
||||
:param page: MoviePilot 从 0 开始的页码
|
||||
:return: 是否失败及标准种子列表
|
||||
"""
|
||||
res = await AsyncRequestUtils(
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": f"{self._ua}",
|
||||
"Accept": "application/json, text/plain, */*"
|
||||
},
|
||||
cookies=self._cookie,
|
||||
if not self._api_key:
|
||||
logger.warning(f"{self._name} 未配置 API AuthKey")
|
||||
return True, []
|
||||
response = await AsyncRequestUtils(
|
||||
headers=self._request_headers(),
|
||||
proxies=self._proxy,
|
||||
referer=f"{self._domain}",
|
||||
timeout=self._timeout
|
||||
).post_res(url=self._searchurl, json=self.__get_params(keyword, page))
|
||||
|
||||
if res and res.status_code == 200:
|
||||
results = res.json().get('data', []) or []
|
||||
return False, self.__parse_result(results)
|
||||
elif res is not None:
|
||||
logger.warn(f"{self._name} 搜索失败,错误码:{res.status_code}")
|
||||
return True, []
|
||||
else:
|
||||
logger.warn(f"{self._name} 搜索失败,无法连接 {self._domain}")
|
||||
return True, []
|
||||
timeout=self._timeout,
|
||||
).post_res(
|
||||
url=self._search_url,
|
||||
json=self._build_params(keyword, page),
|
||||
)
|
||||
return self._process_search_response(response)
|
||||
|
||||
@staticmethod
|
||||
def __get_downloadvolumefactor(discount: str) -> float:
|
||||
def _download_factor(promotion: str) -> float:
|
||||
"""
|
||||
获取下载系数
|
||||
转换下载促销类型
|
||||
|
||||
:param promotion: 开放 API 下载促销枚举
|
||||
:return: MoviePilot 下载系数
|
||||
"""
|
||||
discount_dict = {
|
||||
return {
|
||||
"free": 0,
|
||||
"half": 0.5,
|
||||
"none": 1
|
||||
}
|
||||
if discount:
|
||||
return discount_dict.get(discount, 1)
|
||||
return 1
|
||||
"none": 1,
|
||||
}.get(promotion, 1)
|
||||
|
||||
@staticmethod
|
||||
def __get_uploadvolumefactor(discount: str) -> float:
|
||||
def _upload_factor(promotion: str) -> float:
|
||||
"""
|
||||
获取上传系数
|
||||
转换上传促销类型
|
||||
|
||||
:param promotion: 开放 API 上传促销枚举
|
||||
:return: MoviePilot 上传系数
|
||||
"""
|
||||
discount_dict = {
|
||||
return {
|
||||
"none": 1,
|
||||
"one_half": 1.5,
|
||||
"double_upload": 2
|
||||
}
|
||||
if discount:
|
||||
return discount_dict.get(discount, 1)
|
||||
return 1
|
||||
"double_upload": 2,
|
||||
}.get(promotion, 1)
|
||||
|
||||
def __get_download_url(self, torrent_id: str) -> str:
|
||||
def _build_download_url(self, torrent_id: int) -> str:
|
||||
"""
|
||||
获取下载链接
|
||||
构造先生成下载凭证再获取种子文件的两段式链接
|
||||
|
||||
:param torrent_id: YemaPT 种子 ID
|
||||
:return: Base64 请求配置与下载凭证接口 URL
|
||||
"""
|
||||
return self._downloadurl % (self._domain, torrent_id)
|
||||
request_config = {
|
||||
"method": "post",
|
||||
"cookie": False,
|
||||
"header": {
|
||||
"Authorization": self._api_key,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
"params": {"id": torrent_id},
|
||||
"proxy": self._use_proxy,
|
||||
"success": "success",
|
||||
"result": "data",
|
||||
"result_base_url": self._site_url,
|
||||
"result_path": "api/torrent/download1",
|
||||
"result_query_param": "token",
|
||||
}
|
||||
encoded_config = base64.b64encode(
|
||||
json.dumps(request_config).encode("utf-8")
|
||||
).decode("ascii")
|
||||
return f"[{encoded_config}]{self._download_key_url}"
|
||||
|
||||
@@ -281,13 +281,14 @@ class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]):
|
||||
) for season, episodes in seasoninfo.items()]
|
||||
|
||||
def mediaserver_playing(self, server: str,
|
||||
count: Optional[int] = 20, username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: Jellyfin = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_resume(num=count, username=username)
|
||||
|
||||
def mediaserver_play_url(self, server: str, item_id: Union[str, int]) -> Optional[str]:
|
||||
@@ -315,13 +316,13 @@ class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]):
|
||||
return server_obj.get_season_episode_ids(str(item_id), season)
|
||||
|
||||
def mediaserver_latest(self, server: Optional[str] = None, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: Jellyfin = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_latest(num=count, username=username)
|
||||
|
||||
def mediaserver_latest_images(self,
|
||||
@@ -344,8 +345,7 @@ class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]):
|
||||
return []
|
||||
|
||||
links = []
|
||||
items: List[schemas.MediaServerPlayItem] = self.mediaserver_latest(server=server, count=count,
|
||||
username=username)
|
||||
items = self.mediaserver_latest(server=server, count=count, username=username) or []
|
||||
for item in items:
|
||||
if item.BackdropImageTags:
|
||||
image_url = server_obj.get_backdrop_url(item_id=item.id,
|
||||
|
||||
@@ -52,6 +52,17 @@ class Jellyfin:
|
||||
self.user = self.get_user()
|
||||
self.serverid = self.get_server_id()
|
||||
|
||||
def _request(self, headers: Optional[dict] = None, **kwargs: Any) -> RequestUtils:
|
||||
"""创建兼容不同 Jellyfin 版本鉴权方式的请求工具"""
|
||||
request_headers = dict(headers or {})
|
||||
if not any(str(name).lower() == "authorization" for name in request_headers):
|
||||
request_headers["Authorization"] = f'MediaBrowser Token="{self._apikey}"'
|
||||
if kwargs.get("accept_type") and "Accept" not in request_headers:
|
||||
request_headers["Accept"] = kwargs["accept_type"]
|
||||
if kwargs.get("content_type") and "Content-Type" not in request_headers:
|
||||
request_headers["Content-Type"] = kwargs["content_type"]
|
||||
return RequestUtils(headers=request_headers, **kwargs)
|
||||
|
||||
def get_jellyfin_folders(self) -> List[dict]:
|
||||
"""
|
||||
获取Jellyfin媒体库路径列表
|
||||
@@ -63,7 +74,7 @@ class Jellyfin:
|
||||
'api_key': self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
return res.json()
|
||||
else:
|
||||
@@ -85,7 +96,7 @@ class Jellyfin:
|
||||
'api_key': self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
library_items = res.json()
|
||||
librarys = []
|
||||
@@ -114,42 +125,50 @@ class Jellyfin:
|
||||
logger.error(f"连接Library/VirtualFolders 出错:" + str(e))
|
||||
return []
|
||||
|
||||
def __get_jellyfin_librarys(self, username: Optional[str] = None) -> List[dict]:
|
||||
def __get_jellyfin_librarys(self, username: Optional[str] = None) -> Optional[List[dict]]:
|
||||
"""
|
||||
获取Jellyfin媒体库的信息
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
if username:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return []
|
||||
return None
|
||||
# 使用标准库路径拼接结合统一 URL 规整,避免 host 尾部斜杠缺失导致的寻址偏移。
|
||||
url = UrlUtils.combine_url(self._host, posixpath.join("Users", str(user), "Views"))
|
||||
if not url:
|
||||
return []
|
||||
return None
|
||||
params = {"api_key": self._apikey}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
return res.json().get("Items")
|
||||
items = res.json().get("Items")
|
||||
return items if isinstance(items, list) else None
|
||||
else:
|
||||
logger.error(f"Users/Views 未获取到返回数据")
|
||||
return []
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Views 出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_librarys(self, username: Optional[str] = None, hidden: Optional[bool] = False) -> List[schemas.MediaServerLibrary]:
|
||||
def get_librarys(
|
||||
self,
|
||||
username: Optional[str] = None,
|
||||
hidden: Optional[bool] = False,
|
||||
) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库列表
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
source_libraries = self.__get_jellyfin_librarys(username)
|
||||
if source_libraries is None:
|
||||
return None
|
||||
libraries = []
|
||||
for library in self.__get_jellyfin_librarys(username) or []:
|
||||
for library in source_libraries:
|
||||
if hidden and self._sync_libraries and "all" not in self._sync_libraries \
|
||||
and library.get("Id") not in self._sync_libraries:
|
||||
continue
|
||||
@@ -191,7 +210,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
return len(res.json())
|
||||
else:
|
||||
@@ -212,7 +231,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
users = res.json()
|
||||
# 先查询是否有与当前用户名称匹配的
|
||||
@@ -268,7 +287,7 @@ class Jellyfin:
|
||||
return None
|
||||
url = f"{self._host}Users/authenticatebyname"
|
||||
try:
|
||||
res = RequestUtils(headers={
|
||||
res = self._request(headers={
|
||||
'X-Emby-Authorization': f'MediaBrowser Client="MoviePilot", '
|
||||
f'Device="requests", '
|
||||
f'DeviceId="1", '
|
||||
@@ -305,7 +324,7 @@ class Jellyfin:
|
||||
'api_key': self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
return res.json().get("Id")
|
||||
else:
|
||||
@@ -334,7 +353,7 @@ class Jellyfin:
|
||||
'api_key': self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
result = res.json()
|
||||
return schemas.Statistic(
|
||||
@@ -390,7 +409,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
res_items = res.json().get("Items")
|
||||
if res_items:
|
||||
@@ -427,7 +446,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
res_items = res.json().get("Items")
|
||||
if res_items:
|
||||
@@ -499,7 +518,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res_json = RequestUtils().get_res(url, params)
|
||||
res_json = self._request().get_res(url, params)
|
||||
if res_json:
|
||||
tv_info = res_json.json()
|
||||
res_items = tv_info.get("Items")
|
||||
@@ -540,7 +559,7 @@ class Jellyfin:
|
||||
"isMissing": "false",
|
||||
"api_key": self._apikey
|
||||
}
|
||||
res_json = RequestUtils().get_res(url, params)
|
||||
res_json = self._request().get_res(url, params)
|
||||
if not res_json:
|
||||
return {}
|
||||
episode_ids: Dict[int, str] = {}
|
||||
@@ -567,7 +586,7 @@ class Jellyfin:
|
||||
url = f"{self._host}Items/{item_id}/RemoteImages"
|
||||
params = {"api_key": self._apikey}
|
||||
try:
|
||||
res = RequestUtils(timeout=10).get_res(url, params)
|
||||
res = self._request(timeout=10).get_res(url, params)
|
||||
if res:
|
||||
images = res.json().get("Images") or []
|
||||
for image in images:
|
||||
@@ -592,7 +611,7 @@ class Jellyfin:
|
||||
url = f"{self._host}Items/{item_id}/PlaybackInfo"
|
||||
params = {"api_key": self._apikey}
|
||||
try:
|
||||
res = RequestUtils(timeout=10).get_res(url, params)
|
||||
res = self._request(timeout=10).get_res(url, params)
|
||||
if res:
|
||||
media_sources = res.json().get("MediaSources")
|
||||
if media_sources:
|
||||
@@ -626,7 +645,7 @@ class Jellyfin:
|
||||
_host = self._playhost
|
||||
url = f"{_host}Items/{item_id}/Images/{image_type}"
|
||||
try:
|
||||
res = RequestUtils().get_res(url)
|
||||
res = self._request().get_res(url)
|
||||
if res and res.status_code != 404:
|
||||
logger.info(f"影片图片链接:{res.url}")
|
||||
return res.url
|
||||
@@ -650,7 +669,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
return res.json()[index].get(key)
|
||||
else:
|
||||
@@ -671,7 +690,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().post_res(url, params=params)
|
||||
res = self._request().post_res(url, params=params)
|
||||
if res:
|
||||
return True
|
||||
else:
|
||||
@@ -868,7 +887,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res and res.status_code == 200:
|
||||
return self.__format_item_info(res.json())
|
||||
except Exception as e:
|
||||
@@ -895,7 +914,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey,
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if not res or res.status_code != 200:
|
||||
return None
|
||||
total_count = res.json().get("TotalRecordCount")
|
||||
@@ -929,7 +948,7 @@ class Jellyfin:
|
||||
"Limit": limit
|
||||
})
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if not res or res.status_code != 200:
|
||||
return None
|
||||
items = res.json().get("Items") or []
|
||||
@@ -955,7 +974,7 @@ class Jellyfin:
|
||||
.replace("[APIKEY]", self._apikey or '') \
|
||||
.replace("[USER]", self.user or '')
|
||||
try:
|
||||
return RequestUtils(accept_type="application/json").get_res(url=url)
|
||||
return self._request(accept_type="application/json").get_res(url=url)
|
||||
except Exception as e:
|
||||
logger.error(f"连接Jellyfin出错:" + str(e))
|
||||
return None
|
||||
@@ -973,7 +992,7 @@ class Jellyfin:
|
||||
.replace("[APIKEY]", self._apikey or '') \
|
||||
.replace("[USER]", self.user or '')
|
||||
try:
|
||||
return RequestUtils(
|
||||
return self._request(
|
||||
headers=headers
|
||||
).post_res(url=url, data=data)
|
||||
except Exception as e:
|
||||
@@ -1027,6 +1046,8 @@ class Jellyfin:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return None
|
||||
|
||||
url = f"{self._host}Users/{user}/Items/Resume"
|
||||
params = {
|
||||
@@ -1036,7 +1057,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey,
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
result = res.json().get("Items") or []
|
||||
ret_resume = []
|
||||
@@ -1059,7 +1080,7 @@ class Jellyfin:
|
||||
else:
|
||||
image = self.__get_local_image_by_id(item.get("Id"))
|
||||
# 小部分剧集无[xxx-S01E01-thumb.jpg]图片
|
||||
image_res = RequestUtils().get_res(image)
|
||||
image_res = self._request().get_res(image)
|
||||
if not image_res or image_res.status_code == 404:
|
||||
image = self.generate_image_link(item.get("Id"), "Backdrop", False)
|
||||
if item_type == MediaType.MOVIE.value:
|
||||
@@ -1083,7 +1104,7 @@ class Jellyfin:
|
||||
logger.error(f"Users/Items/Resume 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Items/Resume出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_latest(self, num=20, username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
@@ -1095,6 +1116,8 @@ class Jellyfin:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return None
|
||||
url = f"{self._host}Users/{user}/Items/Latest"
|
||||
params = {
|
||||
"Limit": 100,
|
||||
@@ -1103,7 +1126,7 @@ class Jellyfin:
|
||||
"api_key": self._apikey,
|
||||
}
|
||||
try:
|
||||
res = RequestUtils().get_res(url, params)
|
||||
res = self._request().get_res(url, params)
|
||||
if res:
|
||||
result = res.json() or []
|
||||
ret_latest = []
|
||||
@@ -1136,7 +1159,7 @@ class Jellyfin:
|
||||
logger.error(f"Users/Items/Latest 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Items/Latest出错:" + str(e))
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_user_library_folders(self):
|
||||
"""
|
||||
|
||||
@@ -293,23 +293,23 @@ class PlexModule(_ModuleBase, _MediaServerBase[Plex]):
|
||||
) for season, episodes in seasoninfo.items()]
|
||||
|
||||
def mediaserver_playing(self, server: str, count: Optional[int] = 20,
|
||||
**kwargs) -> List[schemas.MediaServerPlayItem]:
|
||||
**kwargs) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: Plex = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_resume(num=count)
|
||||
|
||||
def mediaserver_latest(self, server: Optional[str] = None, count: Optional[int] = 20,
|
||||
**kwargs) -> List[schemas.MediaServerPlayItem]:
|
||||
**kwargs) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: Plex = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_latest(num=count)
|
||||
|
||||
def mediaserver_latest_images(self,
|
||||
@@ -331,8 +331,7 @@ class PlexModule(_ModuleBase, _MediaServerBase[Plex]):
|
||||
return []
|
||||
|
||||
links = []
|
||||
items: List[schemas.MediaServerPlayItem] = self.mediaserver_latest(server=server, count=count,
|
||||
username=username)
|
||||
items = self.mediaserver_latest(server=server, count=count, username=username) or []
|
||||
for item in items:
|
||||
link = server_obj.get_remote_image_by_id(item_id=item.id,
|
||||
image_type="Backdrop",
|
||||
|
||||
@@ -122,17 +122,17 @@ class Plex:
|
||||
return [f"{self._host.rstrip('/') + url}?X-Plex-Token={self._token}" for url in
|
||||
list(poster_urls.keys())[:total_size]]
|
||||
|
||||
def get_librarys(self, hidden: Optional[bool] = False) -> List[schemas.MediaServerLibrary]:
|
||||
def get_librarys(self, hidden: Optional[bool] = False) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库列表
|
||||
"""
|
||||
if not self._plex:
|
||||
return []
|
||||
return None
|
||||
try:
|
||||
self._libraries = self._plex.library.sections()
|
||||
except Exception as err:
|
||||
logger.error(f"获取媒体服务器所有媒体库列表出错:{str(err)}")
|
||||
return []
|
||||
return None
|
||||
libraries = []
|
||||
for library in self._libraries:
|
||||
if hidden and self._sync_libraries and "all" not in self._sync_libraries \
|
||||
@@ -171,7 +171,7 @@ class Plex:
|
||||
sections = self._plex.library.sections()
|
||||
movie_count = tv_count = episode_count = 0
|
||||
# 媒体库白名单
|
||||
allow_library = [str(lib.id) for lib in self.get_librarys(hidden=True)]
|
||||
allow_library = [str(lib.id) for lib in self.get_librarys(hidden=True) or []]
|
||||
for sec in sections:
|
||||
if str(sec.key) not in allow_library:
|
||||
continue
|
||||
@@ -832,9 +832,12 @@ class Plex:
|
||||
获取继续观看的媒体
|
||||
"""
|
||||
if not self._plex:
|
||||
return []
|
||||
return None
|
||||
# 媒体库白名单
|
||||
allow_library = ",".join(map(str, (lib.id for lib in self.get_librarys(hidden=True))))
|
||||
libraries = self.get_librarys(hidden=True)
|
||||
if libraries is None:
|
||||
return None
|
||||
allow_library = ",".join(map(str, (lib.id for lib in libraries)))
|
||||
params = {"contentDirectoryID": allow_library}
|
||||
items = self._plex.fetchItems("/hubs/continueWatching/items",
|
||||
container_start=0,
|
||||
@@ -871,7 +874,10 @@ class Plex:
|
||||
if not self._plex:
|
||||
return None
|
||||
# 请求参数(除黑名单)
|
||||
allow_library = ",".join(map(str, (lib.id for lib in self.get_librarys(hidden=True))))
|
||||
libraries = self.get_librarys(hidden=True)
|
||||
if libraries is None:
|
||||
return None
|
||||
allow_library = ",".join(map(str, (lib.id for lib in libraries)))
|
||||
params = {
|
||||
"contentDirectoryID": allow_library,
|
||||
"count": num,
|
||||
|
||||
@@ -1265,7 +1265,8 @@ class TheMovieDbModule(_ModuleBase):
|
||||
vote_average: float,
|
||||
vote_count: int,
|
||||
release_date: str,
|
||||
page: Optional[int] = 1) -> Optional[List[MediaInfo]]:
|
||||
page: Optional[int] = 1,
|
||||
raise_exception: bool = False) -> Optional[List[MediaInfo]]:
|
||||
"""
|
||||
TMDB发现功能(异步版本)
|
||||
:param mtype: 媒体类型
|
||||
@@ -1291,7 +1292,7 @@ class TheMovieDbModule(_ModuleBase):
|
||||
"vote_count.gte": vote_count,
|
||||
"release_date.gte": release_date,
|
||||
"page": page
|
||||
})
|
||||
}, raise_exception=raise_exception)
|
||||
elif mtype == MediaType.TV:
|
||||
infos = await self.tmdb.async_discover_tvs({
|
||||
"sort_by": sort_by,
|
||||
@@ -1303,20 +1304,25 @@ class TheMovieDbModule(_ModuleBase):
|
||||
"vote_count.gte": vote_count,
|
||||
"first_air_date.gte": release_date,
|
||||
"page": page
|
||||
})
|
||||
}, raise_exception=raise_exception)
|
||||
else:
|
||||
return []
|
||||
if infos:
|
||||
return [MediaInfo(tmdb_info=info) for info in infos]
|
||||
return []
|
||||
|
||||
async def async_tmdb_trending(self, page: Optional[int] = 1) -> List[MediaInfo]:
|
||||
async def async_tmdb_trending(
|
||||
self, page: Optional[int] = 1, raise_exception: bool = False
|
||||
) -> List[MediaInfo]:
|
||||
"""
|
||||
TMDB流行趋势(异步版本)
|
||||
:param page: 第几页
|
||||
:return: TMDB信息列表
|
||||
"""
|
||||
trending = await self.tmdb.async_discover_trending(page=page)
|
||||
trending = await self.tmdb.async_discover_trending(
|
||||
page=page,
|
||||
raise_exception=raise_exception,
|
||||
)
|
||||
if trending:
|
||||
return [MediaInfo(tmdb_info=info) for info in trending]
|
||||
return []
|
||||
|
||||
@@ -1758,7 +1758,9 @@ class TmdbApi:
|
||||
ret_infos.append(tv)
|
||||
return ret_infos
|
||||
|
||||
async def async_discover_movies(self, params: dict) -> List[dict]:
|
||||
async def async_discover_movies(
|
||||
self, params: dict, raise_exception: bool = False
|
||||
) -> List[dict]:
|
||||
"""
|
||||
发现电影(异步版本)
|
||||
"""
|
||||
@@ -1771,9 +1773,13 @@ class TmdbApi:
|
||||
return items
|
||||
except Exception as e:
|
||||
logger.error(f"获取电影发现失败:{str(e)}")
|
||||
if raise_exception:
|
||||
raise
|
||||
return []
|
||||
|
||||
async def async_discover_tvs(self, params: dict) -> List[dict]:
|
||||
async def async_discover_tvs(
|
||||
self, params: dict, raise_exception: bool = False
|
||||
) -> List[dict]:
|
||||
"""
|
||||
发现电视剧(异步版本)
|
||||
"""
|
||||
@@ -1786,6 +1792,8 @@ class TmdbApi:
|
||||
return items
|
||||
except Exception as e:
|
||||
logger.error(f"获取电视剧发现失败:{str(e)}")
|
||||
if raise_exception:
|
||||
raise
|
||||
return []
|
||||
|
||||
async def async_search_persons(self, name: str) -> List[dict]:
|
||||
@@ -2006,7 +2014,9 @@ class TmdbApi:
|
||||
logger.error(str(e))
|
||||
return {}
|
||||
|
||||
async def async_discover_trending(self, page: Optional[int] = 1) -> List[dict]:
|
||||
async def async_discover_trending(
|
||||
self, page: Optional[int] = 1, raise_exception: bool = False
|
||||
) -> List[dict]:
|
||||
"""
|
||||
流行趋势(异步版本)
|
||||
"""
|
||||
@@ -2018,6 +2028,8 @@ class TmdbApi:
|
||||
return self._normalize_trending_infos(tmdbinfo)
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
if raise_exception:
|
||||
raise
|
||||
return []
|
||||
|
||||
async def async_get_movie_images(
|
||||
|
||||
@@ -332,14 +332,14 @@ class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]):
|
||||
|
||||
def mediaserver_playing(
|
||||
self, server: str, count: Optional[int] = 20, **kwargs
|
||||
) -> List[schemas.MediaServerPlayItem]:
|
||||
) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: Optional[TrimeMedia] = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return server_obj.get_resume(num=count) or []
|
||||
return None
|
||||
return server_obj.get_resume(num=count)
|
||||
|
||||
def mediaserver_play_url(
|
||||
self, server: str, item_id: Union[str, int]
|
||||
@@ -359,14 +359,14 @@ class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]):
|
||||
server: Optional[str] = None,
|
||||
count: Optional[int] = 20,
|
||||
**kwargs,
|
||||
) -> List[schemas.MediaServerPlayItem]:
|
||||
) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: Optional[TrimeMedia] = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return server_obj.get_latest(num=count) or []
|
||||
return None
|
||||
return server_obj.get_latest(num=count)
|
||||
|
||||
def mediaserver_latest_images(
|
||||
self,
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Union
|
||||
from urllib.parse import quote
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
@@ -108,6 +109,7 @@ class Api:
|
||||
"_host",
|
||||
"_token",
|
||||
"_apikey",
|
||||
"_access_code",
|
||||
"_api_path",
|
||||
"_request_utils",
|
||||
"_version",
|
||||
@@ -130,18 +132,51 @@ class Api:
|
||||
def version(self) -> Optional[Version]:
|
||||
return self._version
|
||||
|
||||
def __init__(self, host: str, apikey: str):
|
||||
@property
|
||||
def cookies(self) -> dict:
|
||||
"""
|
||||
当前会话的Cookies,开启访问码后包含访问码校验凭证
|
||||
"""
|
||||
return self._session.cookies.get_dict()
|
||||
|
||||
def __init__(self, host: str, apikey: str, access_code: Optional[str] = None):
|
||||
"""
|
||||
:param host: 飞牛服务端地址,如http://127.0.0.1:5666/v
|
||||
:param access_code: 访问码,未开启时为空
|
||||
"""
|
||||
self._api_path = "/api/v1"
|
||||
self._host = host.rstrip("/")
|
||||
self._apikey = apikey
|
||||
self._access_code = access_code
|
||||
self._token: Optional[str] = None
|
||||
self._version: Optional[Version] = None
|
||||
self._session = requests.Session()
|
||||
self._request_utils = RequestUtils(session=self._session, timeout=10)
|
||||
|
||||
def verify_access_code(self) -> bool:
|
||||
"""
|
||||
校验访问码,通过后会话获得访问凭证,否则无法访问登录页和各应用接口
|
||||
|
||||
:return: 未配置访问码或校验通过返回True
|
||||
"""
|
||||
if not self._access_code:
|
||||
return True
|
||||
# 访问码校验地址位于设备根路径,不在/v下
|
||||
root = self._host[: -len("/v")] if self._host.endswith("/v") else self._host
|
||||
url = f"{root}/c/{quote(self._access_code, safe='')}"
|
||||
res = self._request_utils.get_res(url, allow_redirects=True)
|
||||
if res is None:
|
||||
logger.error(f"校验飞牛访问码失败,无法访问 {url}")
|
||||
return False
|
||||
if res.status_code == 404:
|
||||
# 访问码错误或校验失败时返回404
|
||||
logger.error("飞牛访问码校验失败,请检查访问码是否正确")
|
||||
return False
|
||||
if not res.ok:
|
||||
logger.error(f"飞牛访问码校验失败,状态码:{res.status_code}")
|
||||
return False
|
||||
return True
|
||||
|
||||
def sys_version(self) -> Optional[Version]:
|
||||
"""
|
||||
飞牛影视版本号
|
||||
@@ -161,6 +196,9 @@ class Api:
|
||||
|
||||
:return: 成功返回token 否则返回None
|
||||
"""
|
||||
# 开启访问码后需先通过访问码校验,否则无法访问登录接口
|
||||
if not self.verify_access_code():
|
||||
return None
|
||||
if (
|
||||
res := self.request(
|
||||
"/login",
|
||||
|
||||
@@ -12,6 +12,7 @@ from app.utils.url import UrlUtils
|
||||
class TrimeMedia:
|
||||
_username: Optional[str] = None
|
||||
_password: Optional[str] = None
|
||||
_access_code: Optional[str] = None
|
||||
|
||||
_userinfo: Optional[fnapi.User] = None
|
||||
_host: Optional[str] = None
|
||||
@@ -28,6 +29,7 @@ class TrimeMedia:
|
||||
host: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
access_code: Optional[str] = None,
|
||||
play_host: Optional[str] = None,
|
||||
sync_libraries: Optional[list] = None,
|
||||
**kwargs,
|
||||
@@ -37,13 +39,14 @@ class TrimeMedia:
|
||||
return
|
||||
self._username = username
|
||||
self._password = password
|
||||
self._access_code = access_code
|
||||
self._host = host
|
||||
self._sync_libraries = sync_libraries or []
|
||||
|
||||
if not self.reconnect():
|
||||
logger.error(f"请检查服务端地址 {host}")
|
||||
return
|
||||
if result := self.__create_api(play_host):
|
||||
if result := self.__create_api(play_host, access_code):
|
||||
self._playhost = result.api.host
|
||||
result.api.close()
|
||||
elif play_host:
|
||||
@@ -69,11 +72,14 @@ class TrimeMedia:
|
||||
version: fnapi.Version
|
||||
|
||||
@staticmethod
|
||||
def __create_api(host: Optional[str]) -> Optional["TrimeMedia._ApiCreateResult"]:
|
||||
def __create_api(
|
||||
host: Optional[str], access_code: Optional[str] = None
|
||||
) -> Optional["TrimeMedia._ApiCreateResult"]:
|
||||
"""
|
||||
创建一个飞牛API
|
||||
|
||||
:param host: 服务端地址
|
||||
:param access_code: 访问码,未开启时为空
|
||||
:return: 如果地址无效、不可访问则返回None
|
||||
"""
|
||||
|
||||
@@ -85,16 +91,19 @@ class TrimeMedia:
|
||||
if not host.endswith("/v"):
|
||||
# 尝试补上结尾的/v 测试能否正常访问
|
||||
res = TrimeMedia._ApiCreateResult()
|
||||
res.api = fnapi.Api(host + "/v", api_key)
|
||||
if fnver := res.api.sys_version():
|
||||
res.api = fnapi.Api(host + "/v", api_key, access_code)
|
||||
# 开启访问码后,需先校验才能访问各应用接口
|
||||
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
|
||||
res.version = fnver
|
||||
return res
|
||||
res.api.close()
|
||||
# 测试用户配置的地址
|
||||
res = TrimeMedia._ApiCreateResult()
|
||||
res.api = fnapi.Api(host, api_key)
|
||||
if fnver := res.api.sys_version():
|
||||
res.api = fnapi.Api(host, api_key, access_code)
|
||||
if res.api.verify_access_code() and (fnver := res.api.sys_version()):
|
||||
res.version = fnver
|
||||
return res
|
||||
res.api.close()
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
@@ -130,7 +139,7 @@ class TrimeMedia:
|
||||
if not self.is_configured():
|
||||
return False
|
||||
self.disconnect()
|
||||
if result := self.__create_api(self._host):
|
||||
if result := self.__create_api(self._host, self._access_code):
|
||||
self._api = result.api
|
||||
self._version = result.version
|
||||
# 版本号:0.8.53, 服务版本:0.8.23
|
||||
@@ -163,16 +172,18 @@ class TrimeMedia:
|
||||
|
||||
def get_librarys(
|
||||
self, hidden: Optional[bool] = False
|
||||
) -> List[schemas.MediaServerLibrary]:
|
||||
) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库列表
|
||||
"""
|
||||
if not self.is_authenticated():
|
||||
return []
|
||||
return None
|
||||
if self._userinfo.is_admin == 1:
|
||||
mdb_list = self._api.mdb_list() or []
|
||||
mdb_list = self._api.mdb_list()
|
||||
else:
|
||||
mdb_list = self._api.mediadb_list() or []
|
||||
mdb_list = self._api.mediadb_list()
|
||||
if mdb_list is None:
|
||||
return None
|
||||
self._libraries = {lib.guid: lib for lib in mdb_list}
|
||||
libraries = []
|
||||
for library in self._libraries.values():
|
||||
@@ -584,8 +595,11 @@ class TrimeMedia:
|
||||
"""
|
||||
if not self.is_authenticated():
|
||||
return None
|
||||
items = self._api.play_list()
|
||||
if items is None:
|
||||
return None
|
||||
ret_resume = []
|
||||
for item in self._api.play_list() or []:
|
||||
for item in items:
|
||||
if len(ret_resume) == num:
|
||||
break
|
||||
if self.__is_library_blocked(item.ancestor_guid):
|
||||
@@ -599,14 +613,13 @@ class TrimeMedia:
|
||||
"""
|
||||
if not self.is_authenticated():
|
||||
return None
|
||||
items = (
|
||||
self._api.item_list(
|
||||
page=1,
|
||||
page_size=max(100, num * 5),
|
||||
types=[fnapi.Type.MOVIE, fnapi.Type.TV],
|
||||
)
|
||||
or []
|
||||
items = self._api.item_list(
|
||||
page=1,
|
||||
page_size=max(100, num * 5),
|
||||
types=[fnapi.Type.MOVIE, fnapi.Type.TV],
|
||||
)
|
||||
if items is None:
|
||||
return None
|
||||
latest = []
|
||||
for item in items:
|
||||
if len(latest) == num:
|
||||
@@ -679,4 +692,8 @@ class TrimeMedia:
|
||||
image_url, [self._api.host], strict=True
|
||||
):
|
||||
return None
|
||||
return {"Trim-MC-token": self._api.token}
|
||||
cookies = {"Trim-MC-token": self._api.token}
|
||||
if self._access_code:
|
||||
# 开启访问码后,图片请求也需要携带访问码校验凭证
|
||||
cookies.update(self._api.cookies)
|
||||
return cookies
|
||||
|
||||
@@ -304,14 +304,14 @@ class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]):
|
||||
|
||||
def mediaserver_playing(
|
||||
self, server: str, count: Optional[int] = 20, **kwargs
|
||||
) -> List[schemas.MediaServerPlayItem]:
|
||||
) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: Optional[Ugreen] = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return server_obj.get_resume(num=count) or []
|
||||
return None
|
||||
return server_obj.get_resume(num=count)
|
||||
|
||||
def mediaserver_play_url(
|
||||
self, server: str, item_id: Union[str, int]
|
||||
@@ -331,14 +331,14 @@ class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]):
|
||||
server: Optional[str] = None,
|
||||
count: Optional[int] = 20,
|
||||
**kwargs,
|
||||
) -> List[schemas.MediaServerPlayItem]:
|
||||
) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: Optional[Ugreen] = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return server_obj.get_latest(num=count) or []
|
||||
return None
|
||||
return server_obj.get_latest(num=count)
|
||||
|
||||
def mediaserver_latest_images(
|
||||
self,
|
||||
|
||||
@@ -489,15 +489,15 @@ class Api:
|
||||
return None
|
||||
return dict(result.data)
|
||||
|
||||
def media_list(self) -> list[dict]:
|
||||
def media_list(self) -> Optional[list[dict]]:
|
||||
"""
|
||||
获取首页媒体库列表(`media_lib_info_list`)。
|
||||
"""
|
||||
result = self.request("v1/video/homepage/media_list")
|
||||
if not result.success or not isinstance(result.data, Mapping):
|
||||
return []
|
||||
return None
|
||||
items = result.data.get("media_lib_info_list")
|
||||
return items if isinstance(items, list) else []
|
||||
return items if isinstance(items, list) else None
|
||||
|
||||
def media_lib_users(self) -> list[dict]:
|
||||
"""
|
||||
|
||||
@@ -520,7 +520,7 @@ class Ugreen:
|
||||
|
||||
return paths
|
||||
|
||||
def get_librarys(self, hidden: Optional[bool] = False) -> List[schemas.MediaServerLibrary]:
|
||||
def get_librarys(self, hidden: Optional[bool] = False) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取绿联影视媒体库列表
|
||||
|
||||
@@ -528,9 +528,11 @@ class Ugreen:
|
||||
:return: 媒体库列表
|
||||
"""
|
||||
if not self.is_authenticated() or not self._api:
|
||||
return []
|
||||
return None
|
||||
|
||||
media_libs = self._api.media_list()
|
||||
if media_libs is None:
|
||||
return None
|
||||
self._library_paths = self.__load_library_paths()
|
||||
libraries = []
|
||||
self._libraries = {}
|
||||
@@ -957,8 +959,8 @@ class Ugreen:
|
||||
|
||||
page_size = max(1, num or 12)
|
||||
data = self._api.recently_played(page=1, page_size=page_size)
|
||||
if not data:
|
||||
return []
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
ret_resume = []
|
||||
for item in data.get("video_arr") or []:
|
||||
@@ -982,8 +984,8 @@ class Ugreen:
|
||||
|
||||
page_size = max(1, num)
|
||||
data = self._api.recently_updated(page=1, page_size=page_size)
|
||||
if not data:
|
||||
return []
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
latest = []
|
||||
for item in data.get("video_arr") or []:
|
||||
|
||||
@@ -276,13 +276,13 @@ class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]):
|
||||
) for season, episodes in seasoninfo.items()]
|
||||
|
||||
def mediaserver_playing(self, server: str, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器正在播放信息
|
||||
"""
|
||||
server_obj: ZSpace = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_resume(num=count, username=username)
|
||||
|
||||
def mediaserver_play_url(self, server: str, item_id: Union[str, int]) -> Optional[str]:
|
||||
@@ -295,13 +295,13 @@ class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]):
|
||||
return server_obj.get_play_url(item_id)
|
||||
|
||||
def mediaserver_latest(self, server: Optional[str] = None, count: Optional[int] = 20,
|
||||
username: Optional[str] = None) -> List[schemas.MediaServerPlayItem]:
|
||||
username: Optional[str] = None) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
获取媒体服务器最新入库条目
|
||||
"""
|
||||
server_obj: ZSpace = self.get_instance(server)
|
||||
if not server_obj:
|
||||
return []
|
||||
return None
|
||||
return server_obj.get_latest(num=count, username=username)
|
||||
|
||||
def mediaserver_latest_images(self,
|
||||
@@ -324,8 +324,7 @@ class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]):
|
||||
return []
|
||||
|
||||
links = []
|
||||
items: List[schemas.MediaServerPlayItem] = self.mediaserver_latest(server=server, count=count,
|
||||
username=username)
|
||||
items = self.mediaserver_latest(server=server, count=count, username=username) or []
|
||||
for item in items:
|
||||
if item.BackdropImageTags:
|
||||
image_url = server_obj.get_backdrop_url(item_id=item.id,
|
||||
|
||||
@@ -198,39 +198,46 @@ class ZSpace:
|
||||
})
|
||||
return libraries
|
||||
|
||||
def __get_library_views(self, username: Optional[str] = None) -> List[dict]:
|
||||
def __get_library_views(self, username: Optional[str] = None) -> Optional[List[dict]]:
|
||||
"""
|
||||
获取极影视媒体库列表
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
if username:
|
||||
user = self.get_user(username)
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return []
|
||||
return None
|
||||
url = f"{self._host}emby/Users/{user}/Views"
|
||||
try:
|
||||
res = self.__request_utils().get_res(url)
|
||||
if res:
|
||||
return res.json().get("Items")
|
||||
items = res.json().get("Items")
|
||||
return items if isinstance(items, list) else None
|
||||
else:
|
||||
logger.error("Users/Views 未获取到返回数据")
|
||||
return []
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Views 出错:{e}")
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_librarys(self, username: Optional[str] = None, hidden: Optional[bool] = False) -> List[
|
||||
schemas.MediaServerLibrary]:
|
||||
def get_librarys(
|
||||
self,
|
||||
username: Optional[str] = None,
|
||||
hidden: Optional[bool] = False,
|
||||
) -> Optional[List[schemas.MediaServerLibrary]]:
|
||||
"""
|
||||
获取媒体服务器所有媒体库列表
|
||||
"""
|
||||
if not self._host or not self._apikey:
|
||||
return []
|
||||
return None
|
||||
source_libraries = self.__get_library_views(username)
|
||||
if source_libraries is None:
|
||||
return None
|
||||
libraries = []
|
||||
for library in self.__get_library_views(username) or []:
|
||||
for library in source_libraries:
|
||||
if hidden and self._sync_libraries and "all" not in self._sync_libraries \
|
||||
and library.get("Id") not in self._sync_libraries:
|
||||
continue
|
||||
@@ -1085,7 +1092,7 @@ class ZSpace:
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return []
|
||||
return None
|
||||
url = f"{self._host}emby/Users/{user}/Items/Resume"
|
||||
params = {
|
||||
"Limit": 100,
|
||||
@@ -1141,7 +1148,7 @@ class ZSpace:
|
||||
logger.error("Users/Items/Resume 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接Users/Items/Resume出错:{e}")
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_latest(self, num: Optional[int] = 20, username: Optional[str] = None) -> Optional[
|
||||
List[schemas.MediaServerPlayItem]]:
|
||||
@@ -1160,7 +1167,7 @@ class ZSpace:
|
||||
else:
|
||||
user = self.user
|
||||
if not user:
|
||||
return []
|
||||
return None
|
||||
url = f"{self._host}emby/Users/{user}/Items"
|
||||
params = {
|
||||
"Recursive": "true",
|
||||
@@ -1208,7 +1215,7 @@ class ZSpace:
|
||||
logger.debug("Users/Items?SortBy=DateCreated 未获取到返回数据")
|
||||
except Exception as e:
|
||||
logger.error(f"连接 Users/Items(DateCreated 排序)出错:{e}")
|
||||
return []
|
||||
return None
|
||||
|
||||
def get_user_library_folders(self):
|
||||
"""
|
||||
|
||||
-954
@@ -1,954 +0,0 @@
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Optional, Dict, List
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from watchfiles import Change, DefaultFilter, watch
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.chain.storage import StorageChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.cache import TTLCache, FileCache
|
||||
from app.core.config import settings
|
||||
from app.db.transferhistory_oper import TransferHistoryOper
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.log import logger
|
||||
from app.schemas import FileItem
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.mixins import ConfigReloadMixin
|
||||
from app.utils.singleton import SingletonClass
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
lock = Lock()
|
||||
snapshot_lock = Lock()
|
||||
|
||||
|
||||
class MonitorChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryChangeEvent:
|
||||
"""
|
||||
目录文件变化事件,隔离底层 watchfiles 事件结构。
|
||||
"""
|
||||
change_type: Change
|
||||
src_path: str
|
||||
is_directory: bool
|
||||
|
||||
|
||||
class LocalDirectoryWatcher:
|
||||
"""
|
||||
基于 watchfiles 的本地目录监控线程。
|
||||
"""
|
||||
_HANDLE_CHANGES = {Change.added, Change.modified}
|
||||
|
||||
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None):
|
||||
"""
|
||||
初始化本地目录监控。
|
||||
:param mon_path: 监控目录
|
||||
:param callback: 目录变化回调对象
|
||||
:param force_polling: 是否强制使用轮询模式,None 表示由 watchfiles 自动选择
|
||||
"""
|
||||
self._watch_path = mon_path
|
||||
self._callback = callback
|
||||
self._force_polling = force_polling
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._watch_filter = DefaultFilter()
|
||||
|
||||
@property
|
||||
def watch_path(self) -> Path:
|
||||
"""
|
||||
获取监控目录。
|
||||
:return: 监控目录
|
||||
"""
|
||||
return self._watch_path
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
启动本地目录监控线程。
|
||||
"""
|
||||
if not self._watch_path.exists():
|
||||
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
|
||||
if not self._watch_path.is_dir():
|
||||
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
|
||||
if self.is_alive():
|
||||
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
请求停止本地目录监控线程。
|
||||
"""
|
||||
self._stop_event.set()
|
||||
|
||||
def join(self, timeout: Optional[float] = None):
|
||||
"""
|
||||
等待本地目录监控线程退出。
|
||||
:param timeout: 最长等待秒数
|
||||
"""
|
||||
if self._thread:
|
||||
self._thread.join(timeout=timeout)
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否仍在运行。
|
||||
:return: 线程存活状态
|
||||
"""
|
||||
return bool(self._thread and self._thread.is_alive())
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
运行 watchfiles 主循环,并在快速模式不可用时回退到轮询。
|
||||
"""
|
||||
try:
|
||||
self._run_watch(force_polling=self._force_polling)
|
||||
except Exception as err:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if self._force_polling is True:
|
||||
logger.error(f"本地目录监控发生错误: {self._watch_path} - {err}")
|
||||
logger.debug(traceback.format_exc())
|
||||
return
|
||||
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
|
||||
try:
|
||||
self._run_watch(force_polling=True)
|
||||
except Exception as fallback_err:
|
||||
if not self._stop_event.is_set():
|
||||
logger.error(f"兼容模式监控 {self._watch_path} 仍然失败: {fallback_err}")
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def _run_watch(self, force_polling: Optional[bool]):
|
||||
"""
|
||||
执行一次 watchfiles 监控循环。
|
||||
:param force_polling: 是否强制轮询
|
||||
"""
|
||||
for changes in watch(
|
||||
str(self._watch_path),
|
||||
watch_filter=self._watch_filter,
|
||||
stop_event=self._stop_event,
|
||||
rust_timeout=1000,
|
||||
yield_on_timeout=True,
|
||||
force_polling=force_polling,
|
||||
recursive=True,
|
||||
ignore_permission_denied=True):
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
if not changes:
|
||||
continue
|
||||
self._handle_changes(changes)
|
||||
|
||||
def _handle_changes(self, changes: set[tuple[Change, str]]):
|
||||
"""
|
||||
将 watchfiles 原始变更转换为目录监控事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
"""
|
||||
changes = self._expand_added_directories(changes)
|
||||
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
||||
if change_type not in self._HANDLE_CHANGES:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
event = self._build_event(change_type=change_type, event_path=event_path)
|
||||
if not event or event.is_directory:
|
||||
continue
|
||||
file_size = self._get_file_size(event_path)
|
||||
if file_size is None:
|
||||
continue
|
||||
text = self._change_text(change_type)
|
||||
try:
|
||||
self._callback.event_handler(
|
||||
event=event,
|
||||
text=text,
|
||||
event_path=path_str,
|
||||
file_size=file_size
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
||||
|
||||
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
||||
"""
|
||||
将整体移入监控范围的新增目录展开为内部文件事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
:return: 包含目录内新增文件的变更集合
|
||||
"""
|
||||
expanded_changes = set(changes)
|
||||
for change_type, path_str in changes:
|
||||
if change_type != Change.added:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
try:
|
||||
if not event_path.is_dir():
|
||||
continue
|
||||
for nested_path in event_path.rglob("*"):
|
||||
if not nested_path.is_file():
|
||||
continue
|
||||
nested_path_str = nested_path.as_posix()
|
||||
if self._watch_filter(Change.added, nested_path_str):
|
||||
expanded_changes.add((Change.added, nested_path_str))
|
||||
except OSError as err:
|
||||
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
|
||||
return expanded_changes
|
||||
|
||||
@staticmethod
|
||||
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
||||
"""
|
||||
构建目录变化事件,路径已不存在时忽略。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:param event_path: 变化路径
|
||||
:return: 目录变化事件
|
||||
"""
|
||||
try:
|
||||
is_directory = event_path.is_dir()
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
|
||||
return None
|
||||
if not event_path.exists():
|
||||
return None
|
||||
return DirectoryChangeEvent(
|
||||
change_type=change_type,
|
||||
src_path=event_path.as_posix(),
|
||||
is_directory=is_directory
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_file_size(event_path: Path) -> Optional[int]:
|
||||
"""
|
||||
读取事件文件大小,文件已消失时返回 None。
|
||||
:param event_path: 事件文件路径
|
||||
:return: 文件大小
|
||||
"""
|
||||
try:
|
||||
return event_path.stat().st_size
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _change_text(change_type: Change) -> str:
|
||||
"""
|
||||
转换 watchfiles 事件类型为日志文案。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:return: 事件描述
|
||||
"""
|
||||
if change_type == Change.modified:
|
||||
return "修改"
|
||||
return "新增"
|
||||
|
||||
|
||||
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控处理链,单例模式
|
||||
"""
|
||||
CONFIG_WATCH = {SystemConfigKey.Directories.value}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 存储过照间隔(分钟)
|
||||
self._snapshot_interval = 5
|
||||
# TTL缓存,10秒钟有效
|
||||
self._cache = TTLCache(region="monitor", maxsize=1024, ttl=10)
|
||||
# 快照文件缓存
|
||||
self._snapshot_cache = FileCache(base=settings.CACHE_PATH / "snapshots")
|
||||
# 监控的文件扩展名
|
||||
self.all_exts = settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
|
||||
def on_config_changed(self):
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
return "目录监控"
|
||||
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None):
|
||||
"""
|
||||
保存快照到文件缓存
|
||||
:param storage: 存储名称
|
||||
:param snapshot: 快照数据
|
||||
:param last_snapshot_time: 上次快照时间戳
|
||||
:param file_count: 文件数量,用于调整监控间隔
|
||||
"""
|
||||
try:
|
||||
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
|
||||
if snapshot_time is None:
|
||||
snapshot_time = last_snapshot_time or time.time()
|
||||
snapshot_data = {
|
||||
'timestamp': snapshot_time,
|
||||
'file_count': file_count,
|
||||
'snapshot': snapshot
|
||||
}
|
||||
# 使用FileCache保存快照数据
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
|
||||
self._snapshot_cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
|
||||
logger.debug(f"快照已保存到缓存: {storage}")
|
||||
except Exception as e:
|
||||
logger.error(f"保存快照失败: {e}")
|
||||
|
||||
def reset_snapshot(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准
|
||||
:param storage: 存储名称
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
if self._snapshot_cache.exists(cache_key, region="snapshots"):
|
||||
self._snapshot_cache.delete(cache_key, region="snapshots")
|
||||
logger.info(f"快照已重置: {storage}")
|
||||
return True
|
||||
logger.debug(f"快照文件不存在,无需重置: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"重置快照失败: {storage} - {e}")
|
||||
return False
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)
|
||||
:param storage: 存储名称
|
||||
:param mon_path: 监控路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
|
||||
|
||||
# 生成快照
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=0 # 全量扫描,不使用增量
|
||||
)
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return False
|
||||
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
|
||||
|
||||
# 处理所有文件
|
||||
processed_count = 0
|
||||
for file_path, file_info in new_snapshot.items():
|
||||
try:
|
||||
if not self.__is_transfer_candidate_path(Path(file_path)):
|
||||
continue
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(file_path), file_size=file_size):
|
||||
processed_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||
|
||||
# 保存快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
|
||||
return False
|
||||
|
||||
def load_snapshot(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照
|
||||
:param storage: 存储名称
|
||||
:return: 快照数据或None
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_data = self._snapshot_cache.get(cache_key, region="snapshots")
|
||||
if snapshot_data:
|
||||
data = json.loads(snapshot_data.decode('utf-8'))
|
||||
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
|
||||
return data
|
||||
logger.debug(f"快照文件不存在: {storage}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"加载快照失败: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def adjust_monitor_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔
|
||||
:param file_count: 文件数量
|
||||
:return: 监控间隔(分钟)
|
||||
"""
|
||||
if file_count < 100:
|
||||
return 5 # 5分钟
|
||||
elif file_count < 500:
|
||||
return 10 # 10分钟
|
||||
elif file_count < 1000:
|
||||
return 15 # 15分钟
|
||||
else:
|
||||
return 30 # 30分钟
|
||||
|
||||
@staticmethod
|
||||
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件(只处理新增和修改,不处理删除)
|
||||
:param old_snapshot: 旧快照
|
||||
:param new_snapshot: 新快照
|
||||
:return: 变化信息
|
||||
"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_files = set(old_snapshot.keys())
|
||||
new_files = set(new_snapshot.keys())
|
||||
|
||||
# 新增文件
|
||||
changes['added'] = list(new_files - old_files)
|
||||
|
||||
# 修改文件(大小或时间变化)
|
||||
for file_path in old_files & new_files:
|
||||
old_info = old_snapshot[file_path]
|
||||
new_info = new_snapshot[file_path]
|
||||
|
||||
# 检查文件大小变化
|
||||
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
|
||||
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
|
||||
|
||||
# 检查修改时间变化(如果有的话)
|
||||
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
||||
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
||||
|
||||
if old_size != new_size or (old_time and new_time and old_time != new_time):
|
||||
changes['modified'].append(file_path)
|
||||
|
||||
return changes
|
||||
|
||||
@staticmethod
|
||||
def __is_bluray_sub(_path: Path) -> bool:
|
||||
"""
|
||||
判断是否蓝光原盘目录内的媒体流文件。
|
||||
"""
|
||||
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
|
||||
|
||||
@staticmethod
|
||||
def __get_bluray_dir(_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
获取蓝光原盘BDMV目录的上级目录。
|
||||
"""
|
||||
for p in _path.parents:
|
||||
if p.name == "BDMV":
|
||||
return p.parent
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
|
||||
"""
|
||||
判断路径后缀是否命中给定扩展名列表。
|
||||
"""
|
||||
if not file_path.suffix:
|
||||
return False
|
||||
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
|
||||
|
||||
def __is_transfer_candidate_path(self, file_path: Path) -> bool:
|
||||
"""
|
||||
判断监控事件路径是否需要进入整理链。
|
||||
"""
|
||||
if self.__has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
|
||||
return False
|
||||
return self.__has_suffix_in(file_path, self.all_exts)
|
||||
|
||||
@staticmethod
|
||||
def __build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
|
||||
"""
|
||||
生成整理记录使用的源路径。
|
||||
"""
|
||||
if is_bluray_folder:
|
||||
return f"{event_path.as_posix()}/"
|
||||
return event_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def __has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
|
||||
"""
|
||||
判断源文件是否已经存在整理记录。
|
||||
"""
|
||||
try:
|
||||
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
|
||||
except Exception as err:
|
||||
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
|
||||
"""
|
||||
统计目录下的文件数量(用于检测是否超过系统限制)
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查数量,避免长时间阻塞
|
||||
:return: 文件数量
|
||||
"""
|
||||
try:
|
||||
count = 0
|
||||
import os
|
||||
for root, dirs, files in os.walk(str(directory)):
|
||||
count += len(files)
|
||||
if count > max_check:
|
||||
return count
|
||||
return count
|
||||
except Exception as err:
|
||||
logger.debug(f"统计目录文件数量失败: {err}")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def check_system_limits() -> Dict[str, Any]:
|
||||
"""
|
||||
检查系统限制
|
||||
:return: 系统限制信息
|
||||
"""
|
||||
limits = {
|
||||
'max_user_watches': 0,
|
||||
'max_user_instances': 0,
|
||||
'current_watches': 0,
|
||||
'warnings': []
|
||||
}
|
||||
|
||||
try:
|
||||
system = platform.system()
|
||||
if system == 'Linux':
|
||||
# 检查 inotify 限制
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_watches'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 限制失败: {e}")
|
||||
limits['max_user_watches'] = 8192 # 默认值
|
||||
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_instances'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 实例限制失败: {e}")
|
||||
|
||||
# 检查当前使用的watches
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(['find', '/proc/*/fd', '-lname', 'anon_inode:inotify', '-printf', '%h\n'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
limits['current_watches'] = len(result.stdout.strip().split('\n'))
|
||||
except Exception as e:
|
||||
logger.debug(f"检查当前 inotify 使用失败: {e}")
|
||||
|
||||
except Exception as e:
|
||||
limits['warnings'].append(f"检查系统限制时出错: {e}")
|
||||
|
||||
return limits
|
||||
|
||||
@staticmethod
|
||||
def get_system_optimization_tips() -> List[str]:
|
||||
"""
|
||||
获取系统优化建议
|
||||
:return: 优化建议列表
|
||||
"""
|
||||
tips = []
|
||||
system = platform.system()
|
||||
|
||||
if system == 'Linux':
|
||||
tips.extend([
|
||||
"增加 inotify 监控数量限制:",
|
||||
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"sudo sysctl -p",
|
||||
"",
|
||||
"如果在Docker中运行,请在宿主机上执行以上命令"
|
||||
])
|
||||
elif system == 'Darwin':
|
||||
tips.extend([
|
||||
"macOS 系统优化建议:",
|
||||
"sudo sysctl kern.maxfiles=65536",
|
||||
"sudo sysctl kern.maxfilesperproc=32768",
|
||||
"ulimit -n 32768"
|
||||
])
|
||||
elif system == 'Windows':
|
||||
tips.extend([
|
||||
"Windows 系统优化建议:",
|
||||
"1. 关闭不必要的实时保护软件对监控目录的扫描",
|
||||
"2. 将监控目录添加到Windows Defender排除列表",
|
||||
"3. 确保有足够的可用内存"
|
||||
])
|
||||
|
||||
return tips
|
||||
|
||||
@staticmethod
|
||||
def should_use_polling(directory: Path, monitor_mode: str,
|
||||
file_count: int, limits: dict) -> tuple[bool, str]:
|
||||
"""
|
||||
判断是否应该使用轮询模式
|
||||
:param directory: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param file_count: 目录文件数量
|
||||
:param limits: 系统限制信息
|
||||
:return: (是否使用轮询, 原因)
|
||||
"""
|
||||
if monitor_mode == "compatibility":
|
||||
return True, "用户配置为兼容模式"
|
||||
|
||||
# 检查网络文件系统
|
||||
if SystemUtils.is_network_filesystem(directory):
|
||||
return True, "检测到网络文件系统,建议使用兼容模式"
|
||||
|
||||
max_watches = limits.get('max_user_watches')
|
||||
if max_watches and file_count > max_watches * 0.8:
|
||||
return True, f"目录文件数量({file_count})接近系统限制({max_watches})"
|
||||
return False, "使用快速模式"
|
||||
|
||||
def init(self):
|
||||
"""
|
||||
启动监控
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
|
||||
# 按下载目录去重
|
||||
monitor_dirs = list({f"{d.storage}_{d.download_path}": d for d in monitor_dirs}.values())
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
mon_storages = {}
|
||||
for mon_dir in monitor_dirs:
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
if mon_dir.monitor_type != "monitor":
|
||||
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
|
||||
continue
|
||||
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
if target_path.is_relative_to(mon_path):
|
||||
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
|
||||
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
|
||||
continue
|
||||
|
||||
# 启动监控
|
||||
if mon_dir.storage == "local":
|
||||
# 本地目录监控
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
try:
|
||||
# 统计文件数量并给出提示
|
||||
file_count = self.count_directory_files(mon_path)
|
||||
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
|
||||
|
||||
# 检查系统限制
|
||||
limits = self.check_system_limits()
|
||||
|
||||
# 检查是否需要使用轮询模式
|
||||
use_polling, reason = self.should_use_polling(mon_path,
|
||||
monitor_mode=mon_dir.monitor_mode,
|
||||
file_count=file_count,
|
||||
limits=limits)
|
||||
logger.info(f"监控模式决策: {reason}")
|
||||
|
||||
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
|
||||
logger.info(f"使用{mode_name}监控 {mon_path}")
|
||||
if not use_polling:
|
||||
if limits['warnings']:
|
||||
for warning in limits['warnings']:
|
||||
logger.warn(f"系统限制警告: {warning}")
|
||||
if limits['max_user_watches'] > 0:
|
||||
usage_percent = (file_count / limits['max_user_watches']) * 100
|
||||
logger.info(
|
||||
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
|
||||
|
||||
watcher = LocalDirectoryWatcher(
|
||||
mon_path=mon_path,
|
||||
callback=self,
|
||||
force_polling=True if use_polling else None
|
||||
)
|
||||
self._watchers.append(watcher)
|
||||
watcher.start()
|
||||
|
||||
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
|
||||
|
||||
except Exception as e:
|
||||
err_msg = str(e)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
|
||||
if "inotify" in err_msg.lower():
|
||||
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
|
||||
logger.error("解决方案:")
|
||||
tips = self.get_system_optimization_tips()
|
||||
for tip in tips:
|
||||
logger.error(f" {tip}")
|
||||
logger.error("执行上述命令后重启 MoviePilot")
|
||||
elif "permission" in err_msg.lower():
|
||||
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
|
||||
else:
|
||||
logger.error("建议尝试使用兼容模式进行监控")
|
||||
|
||||
messagehelper.put(f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}", title="目录监控")
|
||||
else:
|
||||
if not mon_storages.get(mon_dir.storage):
|
||||
mon_storages[mon_dir.storage] = []
|
||||
mon_storages[mon_dir.storage].append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self.load_snapshot(storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = self.adjust_monitor_interval(file_count)
|
||||
for path in paths:
|
||||
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': storage,
|
||||
'mon_paths': paths
|
||||
},
|
||||
id=f"monitor_{storage}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
|
||||
|
||||
# 启动定时服务
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
logger.info("定时监控服务已启动")
|
||||
|
||||
# 输出监控总结
|
||||
local_count = len([d for d in monitor_dirs if d.storage == "local" and d.monitor_type == "monitor"])
|
||||
remote_count = len([d for d in monitor_dirs if d.storage != "local" and d.monitor_type == "monitor"])
|
||||
logger.info(f"目录监控启动完成: 本地监控 {local_count} 个,远程监控 {remote_count} 个")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
"""
|
||||
轮询监控(改进版)
|
||||
"""
|
||||
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
|
||||
with snapshot_lock:
|
||||
try:
|
||||
# 加载上次快照数据
|
||||
old_snapshot_data = self.load_snapshot(storage)
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
|
||||
|
||||
# 判断是否为首次快照:检查快照文件是否存在且有效
|
||||
is_first_snapshot = old_snapshot_data is None
|
||||
new_snapshot = {}
|
||||
for mon_path in mon_paths:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
# 生成新快照(增量模式)
|
||||
snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
|
||||
if snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
continue
|
||||
new_snapshot.update(snapshot)
|
||||
file_count = len(snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {file_count} 个文件")
|
||||
file_count = len(new_snapshot)
|
||||
if not is_first_snapshot:
|
||||
# 比较快照找出变化
|
||||
changes = self.compare_snapshots(old_snapshot, new_snapshot)
|
||||
added_files = [
|
||||
file_path
|
||||
for file_path in changes['added']
|
||||
if self.__is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
modified_files = [
|
||||
file_path
|
||||
for file_path in changes['modified']
|
||||
if self.__is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
|
||||
# 处理新增文件
|
||||
handled_added_count = 0
|
||||
for new_file in added_files:
|
||||
file_info = new_snapshot.get(new_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
|
||||
handled_added_count += 1
|
||||
|
||||
# 处理修改文件
|
||||
handled_modified_count = 0
|
||||
for modified_file in modified_files:
|
||||
file_info = new_snapshot.get(modified_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self.__handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
|
||||
handled_modified_count += 1
|
||||
|
||||
if handled_added_count or handled_modified_count:
|
||||
logger.info(
|
||||
f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
|
||||
else:
|
||||
logger.debug(f"{storage} 无文件变化")
|
||||
else:
|
||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||
|
||||
# 保存新快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count, last_snapshot_time)
|
||||
|
||||
# 动态调整监控间隔
|
||||
new_interval = self.adjust_monitor_interval(file_count)
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}")
|
||||
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
|
||||
# 重新安排任务
|
||||
self._scheduler.modify_job(
|
||||
f"monitor_{storage}",
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
logger.info(f"{storage}:{monitor_scope} 监控间隔已调整为 {new_interval} 分钟")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}")
|
||||
logger.debug(traceback.format_exc())
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
"""
|
||||
处理文件变化
|
||||
:param event: 事件
|
||||
:param text: 事件描述
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if not event.is_directory:
|
||||
if not self.__is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
# 整理文件
|
||||
self.__handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
|
||||
|
||||
def __handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
|
||||
"""
|
||||
整理一个文件
|
||||
:param storage: 存储
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
:return: 是否进入整理链
|
||||
"""
|
||||
# 全程加锁
|
||||
with lock:
|
||||
is_bluray_folder = False
|
||||
# 蓝光原盘文件处理
|
||||
if self.__is_bluray_sub(event_path):
|
||||
event_path = self.__get_bluray_dir(event_path)
|
||||
if not event_path:
|
||||
return False
|
||||
is_bluray_folder = True
|
||||
elif not self.__is_transfer_candidate_path(event_path):
|
||||
return False
|
||||
|
||||
# TTL缓存控重
|
||||
if self._cache.get(str(event_path)):
|
||||
return False
|
||||
self._cache[str(event_path)] = True
|
||||
|
||||
src_path = self.__build_transfer_src_path(
|
||||
event_path=event_path,
|
||||
is_bluray_folder=is_bluray_folder,
|
||||
)
|
||||
has_transfer_history = self.__has_transfer_history(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if has_transfer_history is not False:
|
||||
return False
|
||||
|
||||
try:
|
||||
if is_bluray_folder:
|
||||
logger.info(f"开始整理蓝光原盘: {event_path}")
|
||||
else:
|
||||
logger.info(f"开始整理文件: {event_path}")
|
||||
# 开始整理
|
||||
TransferChain().do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=src_path,
|
||||
type="file" if not is_bluray_folder else "dir",
|
||||
name=event_path.name,
|
||||
basename=event_path.stem,
|
||||
extension=event_path.suffix[1:],
|
||||
size=file_size
|
||||
)
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
||||
return False
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
退出监控
|
||||
"""
|
||||
if self._watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in self._watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
self._watchers = []
|
||||
logger.info("本地目录监控服务已停止")
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
if self._cache:
|
||||
self._cache.close()
|
||||
if self._snapshot_cache:
|
||||
self._snapshot_cache.close()
|
||||
@@ -0,0 +1,14 @@
|
||||
"""
|
||||
目录监控包。
|
||||
|
||||
- watcher.py 本地目录监控线程(watchfiles)
|
||||
- syslimits.py 系统限制探测与监控模式决策
|
||||
- snapshot.py 远程快照存取与比对
|
||||
- dispatcher.py 监控事件到整理链的分发
|
||||
- poller.py 远程目录轮询监控
|
||||
- monitor.py Monitor 门面:装配、生命周期与健康检查
|
||||
"""
|
||||
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
|
||||
from app.monitor.monitor import Monitor
|
||||
|
||||
__all__ = ["DirectoryChangeEvent", "LocalDirectoryWatcher", "Monitor"]
|
||||
@@ -0,0 +1,210 @@
|
||||
import re
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import settings
|
||||
from app.db.transferhistory_oper import TransferHistoryOper
|
||||
from app.log import logger
|
||||
from app.schemas import FileItem
|
||||
|
||||
|
||||
class TransferDispatcher:
|
||||
"""
|
||||
将监控事件分发到整理链:候选判定、TTL 去重、整理历史查重与整理触发。
|
||||
"""
|
||||
# 历史查询失败待重试队列上限,防止长时间故障期间无限增长
|
||||
MAX_PENDING_RETRIES = 1000
|
||||
# 单个文件的最大重试次数(按健康检查周期计,60 次约 1 小时)
|
||||
MAX_RETRY_ATTEMPTS = 60
|
||||
|
||||
def __init__(self, all_exts: Optional[List[str]] = None, cache: Optional[Any] = None):
|
||||
"""
|
||||
初始化整理分发器。
|
||||
:param all_exts: 监控的文件扩展名,默认取系统配置
|
||||
:param cache: 去重缓存,默认使用 10 秒 TTL 缓存
|
||||
"""
|
||||
self.all_exts = all_exts if all_exts is not None else (
|
||||
settings.RMT_MEDIAEXT + settings.RMT_SUBEXT + settings.RMT_AUDIOEXT)
|
||||
self._cache = cache if cache is not None else TTLCache(region="monitor", maxsize=1024, ttl=10)
|
||||
self._lock = Lock()
|
||||
# 历史查询失败待重试的文件
|
||||
self._pending_retries: Dict[str, Dict[str, Any]] = {}
|
||||
self._pending_guard = Lock()
|
||||
|
||||
@staticmethod
|
||||
def _is_bluray_sub(_path: Path) -> bool:
|
||||
"""
|
||||
判断是否蓝光原盘目录内的媒体流文件。
|
||||
"""
|
||||
return True if re.search(r"BDMV[/\\]STREAM", _path.as_posix(), re.IGNORECASE) else False
|
||||
|
||||
@staticmethod
|
||||
def _get_bluray_dir(_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
获取蓝光原盘BDMV目录的上级目录。
|
||||
"""
|
||||
for p in _path.parents:
|
||||
if p.name == "BDMV":
|
||||
return p.parent
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _has_suffix_in(file_path: Path, extensions: List[str]) -> bool:
|
||||
"""
|
||||
判断路径后缀是否命中给定扩展名列表。
|
||||
"""
|
||||
if not file_path.suffix:
|
||||
return False
|
||||
return file_path.suffix.casefold() in {ext.casefold() for ext in extensions}
|
||||
|
||||
def is_transfer_candidate_path(self, file_path: Path) -> bool:
|
||||
"""
|
||||
判断监控事件路径是否需要进入整理链。
|
||||
"""
|
||||
if self._has_suffix_in(file_path, settings.DOWNLOAD_TMPEXT):
|
||||
return False
|
||||
return self._has_suffix_in(file_path, self.all_exts)
|
||||
|
||||
@staticmethod
|
||||
def _build_transfer_src_path(event_path: Path, is_bluray_folder: bool) -> str:
|
||||
"""
|
||||
生成整理记录使用的源路径。
|
||||
"""
|
||||
if is_bluray_folder:
|
||||
return f"{event_path.as_posix()}/"
|
||||
return event_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
|
||||
"""
|
||||
判断源文件是否已经存在整理记录。
|
||||
:return: True/False 查询成功,None 查询失败
|
||||
"""
|
||||
try:
|
||||
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
|
||||
except Exception as err:
|
||||
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pending_key(storage: str, event_path: Path) -> str:
|
||||
"""
|
||||
生成待重试文件的唯一键。
|
||||
"""
|
||||
return f"{storage}:{Path(event_path).as_posix()}"
|
||||
|
||||
def _register_pending(self, storage: str, event_path: Path, file_size: float = None):
|
||||
"""
|
||||
登记历史查询失败的文件待重试,重复失败累计次数,超限后放弃。
|
||||
:param storage: 存储
|
||||
:param event_path: 原始事件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
key = self._pending_key(storage, event_path)
|
||||
with self._pending_guard:
|
||||
entry = self._pending_retries.get(key)
|
||||
if entry:
|
||||
entry["attempts"] += 1
|
||||
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
|
||||
self._pending_retries.pop(key, None)
|
||||
logger.error(f"整理历史查询持续失败,已放弃重试: {key}")
|
||||
return
|
||||
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
|
||||
logger.error(f"整理重试队列已满,丢弃: {key}")
|
||||
return
|
||||
self._pending_retries[key] = {
|
||||
"storage": storage,
|
||||
"event_path": event_path,
|
||||
"file_size": file_size,
|
||||
"attempts": 1
|
||||
}
|
||||
logger.warn(f"整理历史查询失败,已登记待重试: {key}")
|
||||
|
||||
def _discard_pending(self, storage: str, event_path: Path):
|
||||
"""
|
||||
历史查询已得到确定结果,移除待重试登记。
|
||||
:param storage: 存储
|
||||
:param event_path: 原始事件路径
|
||||
"""
|
||||
with self._pending_guard:
|
||||
self._pending_retries.pop(self._pending_key(storage, event_path), None)
|
||||
|
||||
def retry_pending(self):
|
||||
"""
|
||||
重试历史查询失败的文件,由健康检查周期驱动。
|
||||
成功或得到确定结果的条目在 handle_file 内部自动移除。
|
||||
"""
|
||||
with self._pending_guard:
|
||||
items = list(self._pending_retries.values())
|
||||
for item in items:
|
||||
logger.info(f"重试整理: {item['storage']}:{item['event_path']}")
|
||||
self.handle_file(storage=item["storage"], event_path=item["event_path"],
|
||||
file_size=item["file_size"])
|
||||
|
||||
def handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
|
||||
"""
|
||||
整理一个文件。
|
||||
:param storage: 存储
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
:return: 是否进入整理链
|
||||
"""
|
||||
with self._lock:
|
||||
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
|
||||
origin_path = event_path
|
||||
is_bluray_folder = False
|
||||
# 蓝光原盘文件处理
|
||||
if self._is_bluray_sub(event_path):
|
||||
event_path = self._get_bluray_dir(event_path)
|
||||
if not event_path:
|
||||
return False
|
||||
is_bluray_folder = True
|
||||
elif not self.is_transfer_candidate_path(event_path):
|
||||
return False
|
||||
|
||||
# TTL缓存控重
|
||||
if self._cache.get(str(event_path)):
|
||||
return False
|
||||
self._cache[str(event_path)] = True
|
||||
|
||||
src_path = self._build_transfer_src_path(
|
||||
event_path=event_path,
|
||||
is_bluray_folder=is_bluray_folder,
|
||||
)
|
||||
has_transfer_history = self._has_transfer_history(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if has_transfer_history is None:
|
||||
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
|
||||
self._register_pending(storage=storage, event_path=origin_path, file_size=file_size)
|
||||
return False
|
||||
self._discard_pending(storage=storage, event_path=origin_path)
|
||||
if has_transfer_history:
|
||||
return False
|
||||
|
||||
try:
|
||||
if is_bluray_folder:
|
||||
logger.info(f"开始整理蓝光原盘: {event_path}")
|
||||
else:
|
||||
logger.info(f"开始整理文件: {event_path}")
|
||||
# 开始整理
|
||||
TransferChain().do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=src_path,
|
||||
type="file" if not is_bluray_folder else "dir",
|
||||
name=event_path.name,
|
||||
basename=event_path.stem,
|
||||
extension=event_path.suffix[1:],
|
||||
size=file_size
|
||||
)
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
||||
return False
|
||||
@@ -0,0 +1,508 @@
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from app.core.config import settings
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.log import logger
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.monitor.poller import RemotePoller
|
||||
from app.monitor.snapshot import SnapshotStore
|
||||
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.mixins import ConfigReloadMixin
|
||||
from app.utils.singleton import SingletonClass
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
||||
"""
|
||||
CONFIG_WATCH = {SystemConfigKey.Directories.value}
|
||||
# 目录监控健康检查间隔(秒)
|
||||
WATCHDOG_INTERVAL = 60
|
||||
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
|
||||
RECOVERY_STABLE_CYCLES = 5
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 本地目录监控列表读写锁
|
||||
self._watcher_lock = Lock()
|
||||
# 启动失败待重试的本地监控配置
|
||||
self._pending_locals: List[Dict[str, Any]] = []
|
||||
# 已告警的监控目录,避免重复推送
|
||||
self._alerted_paths: set = set()
|
||||
# 各监控目录已告警过的自动重启次数
|
||||
self._restart_marks: Dict[str, int] = {}
|
||||
# 各监控目录连续稳定的健康检查周期数
|
||||
self._stable_cycles: Dict[str, int] = {}
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 整理分发器
|
||||
self._dispatcher = TransferDispatcher()
|
||||
# 快照存储
|
||||
self._store = SnapshotStore()
|
||||
# 远程轮询监控
|
||||
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
|
||||
alert_cb=self.__poller_alert)
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
|
||||
def on_config_changed(self):
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
return "目录监控"
|
||||
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None):
|
||||
"""
|
||||
保存快照到文件缓存。
|
||||
"""
|
||||
self._store.save(storage, snapshot, file_count=file_count, last_snapshot_time=last_snapshot_time)
|
||||
|
||||
def load_snapshot(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照。
|
||||
"""
|
||||
return self._store.load(storage)
|
||||
|
||||
def reset_snapshot(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准。
|
||||
"""
|
||||
return self._store.reset(storage)
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
"""
|
||||
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
|
||||
|
||||
@staticmethod
|
||||
def adjust_monitor_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔。
|
||||
"""
|
||||
return SnapshotStore.adjust_interval(file_count)
|
||||
|
||||
@staticmethod
|
||||
def compare_snapshots(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件。
|
||||
"""
|
||||
return SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
|
||||
def init(self):
|
||||
"""
|
||||
启动监控
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
|
||||
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
|
||||
valid_dirs = []
|
||||
for mon_dir in monitor_dirs:
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
if mon_dir.monitor_type != "monitor":
|
||||
logger.debug(f"跳过监控配置 {mon_dir.download_path}:监控类型为 {mon_dir.monitor_type}")
|
||||
continue
|
||||
valid_dirs.append(mon_dir)
|
||||
|
||||
deduped: Dict[str, Any] = {}
|
||||
for mon_dir in valid_dirs:
|
||||
key = f"{mon_dir.storage}_{mon_dir.download_path}"
|
||||
if key in deduped:
|
||||
logger.warn(f"监控配置重复,忽略后一条: {mon_dir.download_path}"
|
||||
f"(媒体库 {mon_dir.library_path})")
|
||||
continue
|
||||
deduped[key] = mon_dir
|
||||
monitor_dirs = list(deduped.values())
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
|
||||
|
||||
mon_storages: Dict[str, List[Path]] = {}
|
||||
# 本地监控启动结果计数,用于输出真实的启动总结
|
||||
local_started = 0
|
||||
local_failed = 0
|
||||
for mon_dir in monitor_dirs:
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
if target_path.is_relative_to(mon_path):
|
||||
logger.warn(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控!")
|
||||
messagehelper.put(f"{target_path} 是监控目录 {mon_path} 的子目录,无法监控", title="目录监控")
|
||||
continue
|
||||
|
||||
# 启动监控
|
||||
if mon_dir.storage == "local":
|
||||
if self.__start_local_monitor(mon_path=mon_path, monitor_mode=mon_dir.monitor_mode):
|
||||
local_started += 1
|
||||
else:
|
||||
local_failed += 1
|
||||
else:
|
||||
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self._store.load(storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = SnapshotStore.adjust_interval(file_count)
|
||||
for path in paths:
|
||||
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': storage,
|
||||
'mon_paths': paths
|
||||
},
|
||||
id=f"monitor_{storage}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
|
||||
|
||||
# 监控健康检查:重建异常监控线程、重试启动失败目录、重试历史查询失败的文件
|
||||
if local_started or local_failed or mon_storages:
|
||||
self._scheduler.add_job(
|
||||
self.watchdog,
|
||||
'interval',
|
||||
seconds=self.WATCHDOG_INTERVAL,
|
||||
id="monitor_watchdog",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
|
||||
|
||||
# 启动定时服务
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
logger.info("定时监控服务已启动")
|
||||
|
||||
# 输出监控总结,报告实际启动成功数而不是配置数
|
||||
remote_count = sum(len(paths) for paths in mon_storages.values())
|
||||
summary = f"目录监控启动完成: 本地监控 {local_started} 个成功"
|
||||
if local_failed:
|
||||
summary += f"、{local_failed} 个失败(将自动退避重试)"
|
||||
summary += f",远程监控 {remote_count} 个"
|
||||
if local_failed:
|
||||
logger.warn(summary)
|
||||
else:
|
||||
logger.info(summary)
|
||||
|
||||
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
|
||||
"""
|
||||
启动单个本地目录监控,失败时登记待重试。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: 是否启动成功
|
||||
"""
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
try:
|
||||
# 检查是否需要使用轮询模式(兼容模式/网络存储不做启动期目录遍历)
|
||||
use_polling, reason, limits, file_count = decide_monitor_mode(mon_path, monitor_mode)
|
||||
logger.info(f"监控模式决策: {reason}")
|
||||
|
||||
mode_name = "兼容模式(轮询)" if use_polling else "快速模式"
|
||||
logger.info(f"使用{mode_name}监控 {mon_path}")
|
||||
if file_count is not None:
|
||||
logger.info(f"监控目录 {mon_path} 包含约 {file_count} 个文件")
|
||||
if not use_polling and limits:
|
||||
if limits['warnings']:
|
||||
for warning in limits['warnings']:
|
||||
logger.warn(f"系统限制警告: {warning}")
|
||||
if limits['max_user_watches'] > 0 and file_count is not None:
|
||||
usage_percent = (file_count / limits['max_user_watches']) * 100
|
||||
logger.info(
|
||||
f"系统监控资源使用率: {usage_percent:.1f}% ({file_count}/{limits['max_user_watches']})")
|
||||
|
||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||
poll_delay_ms = None
|
||||
if use_polling and SystemUtils.is_network_filesystem(mon_path):
|
||||
poll_delay_ms = LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
|
||||
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
|
||||
|
||||
watcher = LocalDirectoryWatcher(
|
||||
mon_path=mon_path,
|
||||
callback=self,
|
||||
force_polling=True if use_polling else None,
|
||||
poll_delay_ms=poll_delay_ms
|
||||
)
|
||||
# 启动成功后再登记,避免失败的监控残留在列表中
|
||||
watcher.start()
|
||||
with self._watcher_lock:
|
||||
self._watchers.append(watcher)
|
||||
self._pending_locals = [
|
||||
pending for pending in self._pending_locals
|
||||
if pending["mon_path"] != mon_path
|
||||
]
|
||||
self.__clear_alert(mon_path, f"本地目录监控已恢复: {mon_path} [{mode_name}]")
|
||||
|
||||
logger.info(f"✓ 本地目录监控已启动: {mon_path} [{mode_name}]")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
|
||||
return False
|
||||
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
|
||||
"""
|
||||
处理本地目录监控启动失败,登记待重试并按需告警。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param err: 启动异常
|
||||
"""
|
||||
err_msg = str(err)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
|
||||
if "inotify" in err_msg.lower():
|
||||
logger.error("inotify 相关错误,这通常是由于系统监控数量限制导致的")
|
||||
logger.error("解决方案:")
|
||||
for tip in get_system_optimization_tips():
|
||||
logger.error(f" {tip}")
|
||||
logger.error("执行上述命令后重启 MoviePilot")
|
||||
elif "permission" in err_msg.lower():
|
||||
logger.error("权限错误,请检查 MoviePilot 是否有足够的权限访问监控目录")
|
||||
elif isinstance(err, (FileNotFoundError, NotADirectoryError)):
|
||||
logger.error("监控目录当前不可用,网络存储/FUSE 挂载可能尚未就绪,将自动重试")
|
||||
elif monitor_mode != "compatibility":
|
||||
logger.error("建议尝试使用兼容模式进行监控")
|
||||
|
||||
with self._watcher_lock:
|
||||
if all(pending["mon_path"] != mon_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": mon_path,
|
||||
"monitor_mode": monitor_mode
|
||||
})
|
||||
self.__send_alert(mon_path,
|
||||
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
|
||||
f"将自动退避重试")
|
||||
|
||||
def watchdog(self):
|
||||
"""
|
||||
目录监控健康检查:重建崩溃或静默失效的监控线程,并重试启动失败的监控目录。
|
||||
"""
|
||||
try:
|
||||
self.__check_watchers()
|
||||
self.__retry_pending_locals()
|
||||
self._dispatcher.retry_pending()
|
||||
except Exception as e:
|
||||
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
||||
|
||||
def __check_watchers(self):
|
||||
"""
|
||||
检查本地目录监控线程状态,异常时重建。
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
watchers = list(self._watchers)
|
||||
for watcher in watchers:
|
||||
key = str(watcher.watch_path)
|
||||
if watcher.is_stalled():
|
||||
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
|
||||
elif not watcher.is_alive():
|
||||
reason = "监控线程已退出"
|
||||
else:
|
||||
# 线程已自愈,但崩溃过就要告警,避免自动重启把故障变成新的静默
|
||||
if watcher.restart_count > self._restart_marks.get(key, 0):
|
||||
self._restart_marks[key] = watcher.restart_count
|
||||
self._stable_cycles[key] = 0
|
||||
self.__send_alert(watcher.watch_path,
|
||||
f"目录监控发生错误并已自动重启"
|
||||
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
|
||||
else:
|
||||
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
|
||||
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
|
||||
if self._stable_cycles[key] >= self.RECOVERY_STABLE_CYCLES:
|
||||
self.__clear_alert(watcher.watch_path, f"目录监控已恢复正常: {watcher.watch_path}")
|
||||
continue
|
||||
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
|
||||
self.__send_alert(watcher.watch_path,
|
||||
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
|
||||
self.__rebuild_watcher(watcher)
|
||||
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
||||
"""
|
||||
重建一个本地目录监控线程。
|
||||
:param watcher: 需要重建的监控
|
||||
"""
|
||||
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
|
||||
watcher.stop()
|
||||
new_watcher = LocalDirectoryWatcher(
|
||||
mon_path=watcher.watch_path,
|
||||
callback=self,
|
||||
force_polling=watcher.force_polling,
|
||||
poll_delay_ms=watcher.poll_delay_ms
|
||||
)
|
||||
try:
|
||||
new_watcher.start()
|
||||
except Exception as e:
|
||||
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
|
||||
with self._watcher_lock:
|
||||
self._watchers = [item for item in self._watchers if item is not watcher]
|
||||
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": watcher.watch_path,
|
||||
# 重建沿用原监控模式,force_polling 为 True 即兼容模式
|
||||
"monitor_mode": "compatibility" if watcher.force_polling else "fast"
|
||||
})
|
||||
return
|
||||
with self._watcher_lock:
|
||||
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
||||
# 新监控的重启计数从零开始,同步重置告警基准
|
||||
self._restart_marks.pop(str(watcher.watch_path), None)
|
||||
self._stable_cycles.pop(str(watcher.watch_path), None)
|
||||
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
|
||||
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
|
||||
|
||||
def __retry_pending_locals(self):
|
||||
"""
|
||||
重试启动失败的本地目录监控,给网络存储/FUSE 挂载留出就绪时间。
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
pending = list(self._pending_locals)
|
||||
for item in pending:
|
||||
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
|
||||
if item.get("skip_cycles", 0) > 0:
|
||||
item["skip_cycles"] -= 1
|
||||
continue
|
||||
logger.info(f"重试启动本地目录监控: {item['mon_path']}")
|
||||
if not self.__start_local_monitor(mon_path=item["mon_path"], monitor_mode=item["monitor_mode"]):
|
||||
item["attempts"] = item.get("attempts", 0) + 1
|
||||
item["skip_cycles"] = min(item["attempts"], 10)
|
||||
|
||||
def __send_alert(self, mon_path: Path, message: str):
|
||||
"""
|
||||
推送目录监控异常告警,同一目录仅在状态变化时推送一次。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 告警内容
|
||||
"""
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key in self._alerted_paths:
|
||||
return
|
||||
self._alerted_paths.add(key)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
@staticmethod
|
||||
def __poller_alert(storage: str, message: str):
|
||||
"""
|
||||
远程轮询监控告警回调,复用消息渠道推送。
|
||||
:param storage: 存储名称
|
||||
:param message: 告警内容
|
||||
"""
|
||||
logger.warn(f"[{storage}] {message}")
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def __clear_alert(self, mon_path: Path, message: str):
|
||||
"""
|
||||
清除目录监控异常告警状态,并在此前告警过时推送恢复消息。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 恢复内容
|
||||
"""
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key not in self._alerted_paths:
|
||||
return
|
||||
self._alerted_paths.discard(key)
|
||||
logger.info(message)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
"""
|
||||
轮询监控:执行一轮快照并按结果动态调整监控间隔。
|
||||
"""
|
||||
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
|
||||
if file_count is None or not self._scheduler:
|
||||
return
|
||||
# 动态调整监控间隔
|
||||
new_interval = SnapshotStore.adjust_interval(file_count)
|
||||
try:
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}")
|
||||
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
|
||||
self._scheduler.modify_job(
|
||||
f"monitor_{storage}",
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
logger.info(f"{storage} 监控间隔已调整为 {new_interval} 分钟")
|
||||
except Exception as e:
|
||||
logger.error(f"调整监控间隔失败: {storage} - {e}")
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
"""
|
||||
处理文件变化。
|
||||
:param event: 事件
|
||||
:param text: 事件描述
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if event.is_directory:
|
||||
return
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
# 整理文件
|
||||
self._dispatcher.handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
退出监控
|
||||
"""
|
||||
# 先停定时服务,避免健康检查在停止过程中重建监控线程
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
with self._watcher_lock:
|
||||
watchers = self._watchers
|
||||
self._watchers = []
|
||||
self._pending_locals = []
|
||||
self._alerted_paths = set()
|
||||
self._restart_marks = {}
|
||||
self._stable_cycles = {}
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
logger.info("本地目录监控服务已停止")
|
||||
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
|
||||
@@ -0,0 +1,228 @@
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from app.chain.storage import StorageChain
|
||||
from app.log import logger
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.monitor.snapshot import SnapshotStore
|
||||
|
||||
|
||||
class RemotePoller:
|
||||
"""
|
||||
远程目录轮询监控:快照、比对并分发变化文件。
|
||||
"""
|
||||
# 同一存储连续异常达到该次数后推送告警
|
||||
FAILURE_ALERT_THRESHOLD = 3
|
||||
|
||||
def __init__(self, store: SnapshotStore, dispatcher: TransferDispatcher,
|
||||
alert_cb: Optional[Callable[[str, str], None]] = None):
|
||||
"""
|
||||
初始化远程轮询监控。
|
||||
:param store: 快照存储
|
||||
:param dispatcher: 整理分发器
|
||||
:param alert_cb: 告警回调 (storage, message)
|
||||
"""
|
||||
self._store = store
|
||||
self._dispatcher = dispatcher
|
||||
self._alert_cb = alert_cb
|
||||
# 快照锁按存储隔离,避免一个慢存储阻塞其他存储的轮询
|
||||
self._locks: Dict[str, Lock] = {}
|
||||
self._locks_guard = Lock()
|
||||
# 各存储连续异常次数
|
||||
self._failure_counts: Dict[str, int] = {}
|
||||
|
||||
def _get_lock(self, storage: str) -> Lock:
|
||||
"""
|
||||
获取指定存储的快照锁。
|
||||
:param storage: 存储名称
|
||||
:return: 快照锁
|
||||
"""
|
||||
with self._locks_guard:
|
||||
return self._locks.setdefault(storage, Lock())
|
||||
|
||||
def _note_failure(self, storage: str, reason: str):
|
||||
"""
|
||||
记录一次轮询异常,连续异常达到阈值时推送告警。
|
||||
:param storage: 存储名称
|
||||
:param reason: 异常原因
|
||||
"""
|
||||
count = self._failure_counts.get(storage, 0) + 1
|
||||
self._failure_counts[storage] = count
|
||||
logger.warn(f"远程目录监控异常(连续第 {count} 次): {storage} - {reason}")
|
||||
if count == self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
|
||||
self._alert_cb(storage,
|
||||
f"远程目录监控连续 {count} 次异常: {storage}\n原因: {reason}\n将继续按周期重试")
|
||||
|
||||
def _note_success(self, storage: str):
|
||||
"""
|
||||
记录一次轮询成功,此前告警过时推送恢复消息。
|
||||
:param storage: 存储名称
|
||||
"""
|
||||
if self._failure_counts.get(storage, 0) >= self.FAILURE_ALERT_THRESHOLD and self._alert_cb:
|
||||
self._alert_cb(storage, f"远程目录监控已恢复: {storage}")
|
||||
self._failure_counts[storage] = 0
|
||||
|
||||
def poll(self, storage: str, mon_paths: List[Path]) -> Optional[int]:
|
||||
"""
|
||||
执行一轮轮询监控。
|
||||
:param storage: 存储名称
|
||||
:param mon_paths: 监控路径列表
|
||||
:return: 基线文件数量,本轮无有效结果时返回 None
|
||||
"""
|
||||
monitor_scope = ",".join(str(mon_path) for mon_path in mon_paths) or "未配置路径"
|
||||
with self._get_lock(storage):
|
||||
try:
|
||||
# 加载上次快照数据,读取失败不能当作首次快照,否则会丢弃已有基线
|
||||
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
||||
if not load_ok:
|
||||
self._note_failure(storage, "读取快照基线失败,跳过本轮")
|
||||
return None
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
|
||||
is_first_snapshot = old_snapshot_data is None
|
||||
|
||||
new_snapshot = {}
|
||||
failed_paths = []
|
||||
for mon_path in mon_paths:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
# 生成新快照(增量模式)
|
||||
snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
|
||||
if snapshot is None:
|
||||
failed_paths.append(str(mon_path))
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
continue
|
||||
new_snapshot.update(snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {len(snapshot)} 个文件")
|
||||
|
||||
if failed_paths and (is_first_snapshot or len(failed_paths) == len(mon_paths)):
|
||||
# 首次基线必须完整建立;全部路径失败时本轮没有有效数据,均不落盘
|
||||
self._note_failure(storage, f"快照失败: {','.join(failed_paths)}")
|
||||
return None
|
||||
|
||||
# 增量快照只包含变化子树,必须与基线合并才是完整视图;
|
||||
# 直接把增量当基线会导致下一轮把未扫到的旧文件全部误判为新增
|
||||
merged_snapshot = {**old_snapshot, **new_snapshot}
|
||||
file_count = len(merged_snapshot)
|
||||
|
||||
if not is_first_snapshot:
|
||||
self._handle_changes(storage, old_snapshot, new_snapshot)
|
||||
else:
|
||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||
|
||||
# 保存合并后的基线
|
||||
if not self._store.save(storage, merged_snapshot, file_count, last_snapshot_time):
|
||||
self._note_failure(storage, "保存快照基线失败")
|
||||
return None
|
||||
|
||||
if failed_paths:
|
||||
# 部分路径失败:成功路径已合并,失败路径保留旧基线,下轮重试
|
||||
self._note_failure(storage, f"部分路径快照失败: {','.join(failed_paths)}")
|
||||
else:
|
||||
self._note_success(storage)
|
||||
return file_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询监控 {storage}:{monitor_scope} 出现错误:{e}\n{traceback.format_exc()}")
|
||||
self._note_failure(storage, str(e))
|
||||
return None
|
||||
|
||||
def _handle_changes(self, storage: str, old_snapshot: dict, new_snapshot: dict):
|
||||
"""
|
||||
比对快照并把变化文件送入整理链。
|
||||
:param storage: 存储名称
|
||||
:param old_snapshot: 旧基线
|
||||
:param new_snapshot: 本轮增量快照
|
||||
"""
|
||||
changes = SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
added_files = [
|
||||
file_path
|
||||
for file_path in changes['added']
|
||||
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
modified_files = [
|
||||
file_path
|
||||
for file_path in changes['modified']
|
||||
if self._dispatcher.is_transfer_candidate_path(Path(file_path))
|
||||
]
|
||||
|
||||
# 处理新增文件
|
||||
handled_added_count = 0
|
||||
for new_file in added_files:
|
||||
file_info = new_snapshot.get(new_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
|
||||
handled_added_count += 1
|
||||
|
||||
# 处理修改文件
|
||||
handled_modified_count = 0
|
||||
for modified_file in modified_files:
|
||||
file_info = new_snapshot.get(modified_file, {})
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
|
||||
handled_modified_count += 1
|
||||
|
||||
if handled_added_count or handled_modified_count:
|
||||
logger.info(f"{storage} 发现 {handled_added_count} 个新增文件,{handled_modified_count} 个修改文件")
|
||||
else:
|
||||
logger.debug(f"{storage} 无文件变化")
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
:param storage: 存储名称
|
||||
:param mon_path: 监控路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
|
||||
|
||||
# 生成快照
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=0 # 全量扫描,不使用增量
|
||||
)
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return False
|
||||
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
|
||||
|
||||
# 处理所有文件
|
||||
processed_count = 0
|
||||
for file_path, file_info in new_snapshot.items():
|
||||
try:
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
|
||||
continue
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(file_path),
|
||||
file_size=file_size):
|
||||
processed_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||
|
||||
# 全量扫描覆盖单个路径,与已有基线合并后落盘,避免覆盖其他监控路径的基线
|
||||
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if (load_ok and old_snapshot_data) else {}
|
||||
merged_snapshot = {**old_snapshot, **new_snapshot}
|
||||
self._store.save(storage, merged_snapshot, len(merged_snapshot))
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
|
||||
return False
|
||||
@@ -0,0 +1,148 @@
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from app.core.cache import FileCache
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class SnapshotStore:
|
||||
"""
|
||||
远程目录监控快照的存取与比对。
|
||||
"""
|
||||
|
||||
def __init__(self, cache: Optional[FileCache] = None):
|
||||
"""
|
||||
初始化快照存储。
|
||||
:param cache: 快照文件缓存,默认使用 CACHE_PATH/snapshots
|
||||
"""
|
||||
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
|
||||
|
||||
def save(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None) -> bool:
|
||||
"""
|
||||
保存快照到文件缓存。
|
||||
:param storage: 存储名称
|
||||
:param snapshot: 快照数据
|
||||
:param file_count: 文件数量,用于调整监控间隔
|
||||
:param last_snapshot_time: 上次快照时间戳
|
||||
:return: 是否保存成功
|
||||
"""
|
||||
try:
|
||||
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
|
||||
if snapshot_time is None:
|
||||
snapshot_time = last_snapshot_time or time.time()
|
||||
snapshot_data = {
|
||||
'timestamp': snapshot_time,
|
||||
'file_count': file_count,
|
||||
'snapshot': snapshot
|
||||
}
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_json = json.dumps(snapshot_data, ensure_ascii=False, indent=2)
|
||||
self._cache.set(cache_key, snapshot_json.encode('utf-8'), region="snapshots")
|
||||
logger.debug(f"快照已保存到缓存: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存快照失败: {e}")
|
||||
return False
|
||||
|
||||
def load_checked(self, storage: str) -> Tuple[Optional[Dict], bool]:
|
||||
"""
|
||||
从文件缓存加载快照,并区分「快照不存在」与「读取失败」。
|
||||
读取失败时不能当作首次快照处理,否则会静默丢弃已有基线。
|
||||
:param storage: 存储名称
|
||||
:return: (快照数据或None, 是否读取成功)
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
snapshot_data = self._cache.get(cache_key, region="snapshots")
|
||||
if snapshot_data:
|
||||
data = json.loads(snapshot_data.decode('utf-8'))
|
||||
logger.debug(f"成功加载快照: {storage}, 包含 {len(data.get('snapshot', {}))} 个文件")
|
||||
return data, True
|
||||
logger.debug(f"快照文件不存在: {storage}")
|
||||
return None, True
|
||||
except Exception as e:
|
||||
logger.error(f"加载快照失败: {e}")
|
||||
return None, False
|
||||
|
||||
def load(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件缓存加载快照。
|
||||
:param storage: 存储名称
|
||||
:return: 快照数据或None
|
||||
"""
|
||||
data, _ = self.load_checked(storage)
|
||||
return data
|
||||
|
||||
def reset(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准。
|
||||
:param storage: 存储名称
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
cache_key = f"{storage}_snapshot"
|
||||
if self._cache.exists(cache_key, region="snapshots"):
|
||||
self._cache.delete(cache_key, region="snapshots")
|
||||
logger.info(f"快照已重置: {storage}")
|
||||
return True
|
||||
logger.debug(f"快照文件不存在,无需重置: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"重置快照失败: {storage} - {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def compare(old_snapshot: Dict, new_snapshot: Dict) -> Dict[str, List]:
|
||||
"""
|
||||
比对快照,找出变化的文件(只处理新增和修改,不处理删除)。
|
||||
:param old_snapshot: 旧快照
|
||||
:param new_snapshot: 新快照
|
||||
:return: 变化信息
|
||||
"""
|
||||
changes = {
|
||||
'added': [],
|
||||
'modified': []
|
||||
}
|
||||
|
||||
old_files = set(old_snapshot.keys())
|
||||
new_files = set(new_snapshot.keys())
|
||||
|
||||
# 新增文件
|
||||
changes['added'] = list(new_files - old_files)
|
||||
|
||||
# 修改文件(大小或时间变化)
|
||||
for file_path in old_files & new_files:
|
||||
old_info = old_snapshot[file_path]
|
||||
new_info = new_snapshot[file_path]
|
||||
|
||||
# 检查文件大小变化
|
||||
old_size = old_info.get('size', 0) if isinstance(old_info, dict) else old_info
|
||||
new_size = new_info.get('size', 0) if isinstance(new_info, dict) else new_info
|
||||
|
||||
# 检查修改时间变化(如果有的话)
|
||||
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
||||
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
||||
|
||||
if old_size != new_size or (old_time and new_time and old_time != new_time):
|
||||
changes['modified'].append(file_path)
|
||||
|
||||
return changes
|
||||
|
||||
@staticmethod
|
||||
def adjust_interval(file_count: int) -> int:
|
||||
"""
|
||||
根据文件数量动态调整监控间隔。
|
||||
:param file_count: 文件数量
|
||||
:return: 监控间隔(分钟)
|
||||
"""
|
||||
if file_count < 100:
|
||||
return 5 # 5分钟
|
||||
elif file_count < 500:
|
||||
return 10 # 10分钟
|
||||
elif file_count < 1000:
|
||||
return 15 # 15分钟
|
||||
else:
|
||||
return 30 # 30分钟
|
||||
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from app.log import logger
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
def count_directory_entries(directory: Path, max_check: int = 10000) -> Tuple[int, int]:
|
||||
"""
|
||||
统计目录下的文件与子目录数量(用于检测是否超过系统限制)。
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查文件数量,避免长时间阻塞
|
||||
:return: (文件数量, 目录数量)
|
||||
"""
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
try:
|
||||
for _, dirs, files in os.walk(str(directory)):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > max_check:
|
||||
break
|
||||
except Exception as err:
|
||||
logger.debug(f"统计目录规模失败: {err}")
|
||||
return file_count, dir_count
|
||||
|
||||
|
||||
def count_directory_files(directory: Path, max_check: int = 10000) -> int:
|
||||
"""
|
||||
统计目录下的文件数量。
|
||||
:param directory: 目录路径
|
||||
:param max_check: 最大检查数量,避免长时间阻塞
|
||||
:return: 文件数量
|
||||
"""
|
||||
file_count, _ = count_directory_entries(directory, max_check=max_check)
|
||||
return file_count
|
||||
|
||||
|
||||
def check_system_limits() -> Dict[str, Any]:
|
||||
"""
|
||||
检查系统监控相关限制。
|
||||
:return: 系统限制信息
|
||||
"""
|
||||
limits = {
|
||||
'max_user_watches': 0,
|
||||
'max_user_instances': 0,
|
||||
'warnings': []
|
||||
}
|
||||
|
||||
try:
|
||||
if platform.system() == 'Linux':
|
||||
# 检查 inotify 限制
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_watches', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_watches'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 限制失败: {e}")
|
||||
limits['max_user_watches'] = 8192 # 默认值
|
||||
|
||||
try:
|
||||
with open('/proc/sys/fs/inotify/max_user_instances', 'r', encoding='utf-8', errors='replace') as f:
|
||||
limits['max_user_instances'] = int(f.read().strip())
|
||||
except Exception as e:
|
||||
logger.debug(f"读取 inotify 实例限制失败: {e}")
|
||||
except Exception as e:
|
||||
limits['warnings'].append(f"检查系统限制时出错: {e}")
|
||||
|
||||
return limits
|
||||
|
||||
|
||||
def get_system_optimization_tips() -> List[str]:
|
||||
"""
|
||||
获取系统优化建议。
|
||||
:return: 优化建议列表
|
||||
"""
|
||||
tips = []
|
||||
system = platform.system()
|
||||
|
||||
if system == 'Linux':
|
||||
tips.extend([
|
||||
"增加 inotify 监控数量限制:",
|
||||
"echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf",
|
||||
"sudo sysctl -p",
|
||||
"",
|
||||
"如果在Docker中运行,请在宿主机上执行以上命令"
|
||||
])
|
||||
elif system == 'Darwin':
|
||||
tips.extend([
|
||||
"macOS 系统优化建议:",
|
||||
"sudo sysctl kern.maxfiles=65536",
|
||||
"sudo sysctl kern.maxfilesperproc=32768",
|
||||
"ulimit -n 32768"
|
||||
])
|
||||
elif system == 'Windows':
|
||||
tips.extend([
|
||||
"Windows 系统优化建议:",
|
||||
"1. 关闭不必要的实时保护软件对监控目录的扫描",
|
||||
"2. 将监控目录添加到Windows Defender排除列表",
|
||||
"3. 确保有足够的可用内存"
|
||||
])
|
||||
|
||||
return tips
|
||||
|
||||
|
||||
def decide_monitor_mode(directory: Path,
|
||||
monitor_mode: str) -> Tuple[bool, str, Optional[Dict[str, Any]], Optional[int]]:
|
||||
"""
|
||||
决策监控模式。兼容模式与网络文件系统直接短路,只有快速模式候选才统计
|
||||
目录规模与系统限制,避免启动期对网络挂载做无谓的全量遍历。
|
||||
|
||||
inotify 的 max_user_watches 按监视点(目录)计数,因此用目录数量而不是
|
||||
文件数量与上限比较。
|
||||
|
||||
:param directory: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: (是否使用轮询, 原因, 系统限制信息或None, 文件数量或None)
|
||||
"""
|
||||
if monitor_mode == "compatibility":
|
||||
return True, "用户配置为兼容模式", None, None
|
||||
|
||||
# 检查网络文件系统
|
||||
if SystemUtils.is_network_filesystem(directory):
|
||||
return True, "检测到网络文件系统,建议使用兼容模式", None, None
|
||||
|
||||
limits = check_system_limits()
|
||||
file_count, dir_count = count_directory_entries(directory)
|
||||
max_watches = limits.get('max_user_watches')
|
||||
if max_watches and dir_count > max_watches * 0.8:
|
||||
return (True, f"目录数量({dir_count})接近 inotify 监控上限({max_watches})",
|
||||
limits, file_count)
|
||||
return False, "使用快速模式", limits, file_count
|
||||
@@ -0,0 +1,303 @@
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from watchfiles import Change, DefaultFilter, watch
|
||||
|
||||
from app.log import logger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryChangeEvent:
|
||||
"""
|
||||
目录文件变化事件,隔离底层 watchfiles 事件结构。
|
||||
"""
|
||||
change_type: Change
|
||||
src_path: str
|
||||
is_directory: bool
|
||||
|
||||
|
||||
class LocalDirectoryWatcher:
|
||||
"""
|
||||
基于 watchfiles 的本地目录监控线程。
|
||||
"""
|
||||
_HANDLE_CHANGES = {Change.added, Change.modified}
|
||||
# 监控循环异常退出后的重启退避秒数,网络存储/FUSE 挂载抖动通常是暂时的
|
||||
RESTART_BACKOFF = (5, 15, 30, 60, 120, 300)
|
||||
# 单次监控循环存活超过该秒数视为已恢复,重置退避
|
||||
HEALTHY_UPTIME = 60
|
||||
# 超过该秒数监控循环没有任何活动,判定为静默失效
|
||||
STALL_TIMEOUT = 600
|
||||
# 轮询模式目录扫描间隔(毫秒):本地磁盘用 watchfiles 默认值
|
||||
POLL_DELAY_LOCAL_MS = 300
|
||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||
POLL_DELAY_NETWORK_MS = 5000
|
||||
|
||||
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
|
||||
poll_delay_ms: Optional[int] = None):
|
||||
"""
|
||||
初始化本地目录监控。
|
||||
:param mon_path: 监控目录
|
||||
:param callback: 目录变化回调对象
|
||||
:param force_polling: 是否强制使用轮询模式,None 表示由 watchfiles 自动选择
|
||||
:param poll_delay_ms: 轮询模式目录扫描间隔(毫秒),仅轮询时生效
|
||||
"""
|
||||
self._watch_path = mon_path
|
||||
self._callback = callback
|
||||
self._force_polling = force_polling
|
||||
self._poll_delay_ms = poll_delay_ms or self.POLL_DELAY_LOCAL_MS
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._watch_filter = DefaultFilter()
|
||||
# 最近一次监控循环活动时间(monotonic),用于检测静默失效
|
||||
self._last_activity: float = 0.0
|
||||
# 累计自动重启次数
|
||||
self._restart_count: int = 0
|
||||
|
||||
@property
|
||||
def watch_path(self) -> Path:
|
||||
"""
|
||||
获取监控目录。
|
||||
:return: 监控目录
|
||||
"""
|
||||
return self._watch_path
|
||||
|
||||
@property
|
||||
def force_polling(self) -> Optional[bool]:
|
||||
"""
|
||||
获取监控模式配置,重建监控线程时沿用。
|
||||
:return: 是否强制轮询
|
||||
"""
|
||||
return self._force_polling
|
||||
|
||||
@property
|
||||
def restart_count(self) -> int:
|
||||
"""
|
||||
获取累计自动重启次数。
|
||||
:return: 自动重启次数
|
||||
"""
|
||||
return self._restart_count
|
||||
|
||||
@property
|
||||
def poll_delay_ms(self) -> int:
|
||||
"""
|
||||
获取轮询模式目录扫描间隔(毫秒),重建监控线程时沿用。
|
||||
:return: 扫描间隔
|
||||
"""
|
||||
return self._poll_delay_ms
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
启动本地目录监控线程。
|
||||
"""
|
||||
if not self._watch_path.exists():
|
||||
raise FileNotFoundError(f"监控目录不存在: {self._watch_path}")
|
||||
if not self._watch_path.is_dir():
|
||||
raise NotADirectoryError(f"监控路径不是目录: {self._watch_path}")
|
||||
if self.is_alive():
|
||||
logger.info(f"本地目录监控已在运行中: {self._watch_path}")
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._mark_activity()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"MoviePilot-DirectoryWatcher-{self._watch_path.name}",
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
请求停止本地目录监控线程。
|
||||
"""
|
||||
self._stop_event.set()
|
||||
|
||||
def join(self, timeout: Optional[float] = None):
|
||||
"""
|
||||
等待本地目录监控线程退出。
|
||||
:param timeout: 最长等待秒数
|
||||
"""
|
||||
if self._thread:
|
||||
self._thread.join(timeout=timeout)
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否仍在运行。
|
||||
:return: 线程存活状态
|
||||
"""
|
||||
return bool(self._thread and self._thread.is_alive())
|
||||
|
||||
def is_stalled(self) -> bool:
|
||||
"""
|
||||
判断监控线程是否已静默失效(线程存活但监控循环长时间无任何活动)。
|
||||
:return: 是否静默失效
|
||||
"""
|
||||
if self._stop_event.is_set() or not self.is_alive():
|
||||
return False
|
||||
if not self._last_activity:
|
||||
return False
|
||||
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
|
||||
|
||||
def _mark_activity(self):
|
||||
"""
|
||||
记录一次监控循环活动时间,作为静默失效检测的心跳。
|
||||
"""
|
||||
self._last_activity = time.monotonic()
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
运行 watchfiles 主循环,异常时退避重启,避免一次故障导致监控永久停摆。
|
||||
"""
|
||||
# 快速模式失败后降级为轮询,降级后的失败一律走退避重启
|
||||
force_polling = self._force_polling
|
||||
attempt = 0
|
||||
while not self._stop_event.is_set():
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
self._mark_activity()
|
||||
self._run_watch(force_polling=force_polling)
|
||||
# 正常返回表示收到停止信号
|
||||
return
|
||||
except Exception as err:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
# 崩溃堆栈按 ERROR 级输出,生产环境 LOG_LEVEL=ERROR 时也能落盘
|
||||
logger.error(f"本地目录监控异常堆栈: {self._watch_path}\n{traceback.format_exc()}")
|
||||
if force_polling is not True:
|
||||
logger.warn(f"快速模式监控 {self._watch_path} 失败,将自动切换到兼容模式: {err}")
|
||||
force_polling = True
|
||||
continue
|
||||
if time.monotonic() - started_at >= self.HEALTHY_UPTIME:
|
||||
# 上一轮监控已稳定运行过,重新从最短间隔开始退避
|
||||
attempt = 0
|
||||
delay = self.RESTART_BACKOFF[min(attempt, len(self.RESTART_BACKOFF) - 1)]
|
||||
attempt += 1
|
||||
self._restart_count += 1
|
||||
logger.error(f"本地目录监控发生错误,{delay} 秒后自动重启"
|
||||
f"(累计第 {self._restart_count} 次): {self._watch_path} - {err}")
|
||||
if self._stop_event.wait(timeout=delay):
|
||||
return
|
||||
|
||||
def _run_watch(self, force_polling: Optional[bool]):
|
||||
"""
|
||||
执行一次 watchfiles 监控循环。
|
||||
:param force_polling: 是否强制轮询
|
||||
"""
|
||||
for changes in watch(
|
||||
str(self._watch_path),
|
||||
watch_filter=self._watch_filter,
|
||||
stop_event=self._stop_event,
|
||||
rust_timeout=1000,
|
||||
yield_on_timeout=True,
|
||||
force_polling=force_polling,
|
||||
poll_delay_ms=self._poll_delay_ms,
|
||||
recursive=True,
|
||||
ignore_permission_denied=True):
|
||||
self._mark_activity()
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
if not changes:
|
||||
continue
|
||||
self._handle_changes(changes)
|
||||
self._mark_activity()
|
||||
|
||||
def _handle_changes(self, changes: set[tuple[Change, str]]):
|
||||
"""
|
||||
将 watchfiles 原始变更转换为目录监控事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
"""
|
||||
changes = self._expand_added_directories(changes)
|
||||
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
||||
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
|
||||
self._mark_activity()
|
||||
if change_type not in self._HANDLE_CHANGES:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
event = self._build_event(change_type=change_type, event_path=event_path)
|
||||
if not event or event.is_directory:
|
||||
continue
|
||||
file_size = self._get_file_size(event_path)
|
||||
if file_size is None:
|
||||
continue
|
||||
text = self._change_text(change_type)
|
||||
try:
|
||||
self._callback.event_handler(
|
||||
event=event,
|
||||
text=text,
|
||||
event_path=path_str,
|
||||
file_size=file_size
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
||||
|
||||
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
||||
"""
|
||||
将整体移入监控范围的新增目录展开为内部文件事件。
|
||||
:param changes: watchfiles 返回的变更集合
|
||||
:return: 包含目录内新增文件的变更集合
|
||||
"""
|
||||
expanded_changes = set(changes)
|
||||
for change_type, path_str in changes:
|
||||
if change_type != Change.added:
|
||||
continue
|
||||
event_path = Path(path_str)
|
||||
try:
|
||||
if not event_path.is_dir():
|
||||
continue
|
||||
for nested_path in event_path.rglob("*"):
|
||||
if not nested_path.is_file():
|
||||
continue
|
||||
nested_path_str = nested_path.as_posix()
|
||||
if self._watch_filter(Change.added, nested_path_str):
|
||||
expanded_changes.add((Change.added, nested_path_str))
|
||||
except OSError as err:
|
||||
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
|
||||
return expanded_changes
|
||||
|
||||
@staticmethod
|
||||
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
||||
"""
|
||||
构建目录变化事件,路径已不存在时忽略。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:param event_path: 变化路径
|
||||
:return: 目录变化事件
|
||||
"""
|
||||
try:
|
||||
is_directory = event_path.is_dir()
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控事件路径失败: {event_path} - {err}")
|
||||
return None
|
||||
if not event_path.exists():
|
||||
return None
|
||||
return DirectoryChangeEvent(
|
||||
change_type=change_type,
|
||||
src_path=event_path.as_posix(),
|
||||
is_directory=is_directory
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_file_size(event_path: Path) -> Optional[int]:
|
||||
"""
|
||||
读取事件文件大小,文件已消失时返回 None。
|
||||
:param event_path: 事件文件路径
|
||||
:return: 文件大小
|
||||
"""
|
||||
try:
|
||||
return event_path.stat().st_size
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件大小失败: {event_path} - {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _change_text(change_type: Change) -> str:
|
||||
"""
|
||||
转换 watchfiles 事件类型为日志文案。
|
||||
:param change_type: watchfiles 变化类型
|
||||
:return: 事件描述
|
||||
"""
|
||||
if change_type == Change.modified:
|
||||
return "修改"
|
||||
return "新增"
|
||||
@@ -277,6 +277,20 @@ class _PluginBase(metaclass=ABCMeta):
|
||||
plugin_id = self.__class__.__name__
|
||||
self.plugindata.save(plugin_id, key, value)
|
||||
|
||||
async def async_save_data(
|
||||
self, key: str, value: Any, plugin_id: Optional[str] = None
|
||||
) -> None:
|
||||
"""
|
||||
异步保存插件数据
|
||||
|
||||
:param key: 数据键
|
||||
:param value: 数据值
|
||||
:param plugin_id: 插件ID
|
||||
"""
|
||||
if not plugin_id:
|
||||
plugin_id = self.__class__.__name__
|
||||
await self.plugindata.async_save(plugin_id, key, value)
|
||||
|
||||
def get_data(self, key: Optional[str] = None, plugin_id: Optional[str] = None) -> Any:
|
||||
"""
|
||||
获取插件数据
|
||||
@@ -287,6 +301,20 @@ class _PluginBase(metaclass=ABCMeta):
|
||||
plugin_id = self.__class__.__name__
|
||||
return self.plugindata.get_data(plugin_id, key)
|
||||
|
||||
async def async_get_data(
|
||||
self, key: Optional[str] = None, plugin_id: Optional[str] = None
|
||||
) -> Any:
|
||||
"""
|
||||
异步获取插件数据
|
||||
|
||||
:param key: 数据键
|
||||
:param plugin_id: 插件ID
|
||||
:return: 指定键的数据值或插件的全部数据
|
||||
"""
|
||||
if not plugin_id:
|
||||
plugin_id = self.__class__.__name__
|
||||
return await self.plugindata.async_get_data(plugin_id, key)
|
||||
|
||||
def del_data(self, key: str, plugin_id: Optional[str] = None) -> Any:
|
||||
"""
|
||||
删除插件数据
|
||||
|
||||
@@ -94,6 +94,7 @@ class AgentLLMProviderEventData(ChainEventData):
|
||||
user_agent: Optional[str] = Field(default=None, description="OpenAI兼容接口User-Agent")
|
||||
use_proxy: Optional[bool] = Field(default=None, description="是否使用系统代理")
|
||||
thinking_level: Optional[str] = Field(default=None, description="思考模式级别")
|
||||
api_protocol: Optional[str] = Field(default=None, description="OpenAI兼容接口API协议:auto/chat_completions/responses")
|
||||
selected_provider_id: Optional[str] = Field(default=None, description="插件侧供应商ID")
|
||||
selected_provider_name: Optional[str] = Field(default=None, description="插件侧供应商名称")
|
||||
source: Optional[str] = Field(default=None, description="选择来源")
|
||||
|
||||
@@ -36,3 +36,12 @@ class OperationInterrupted(KeyboardInterrupt):
|
||||
用于表示操作被中断
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class StorageQueryError(Exception):
|
||||
"""
|
||||
用于表示存储查询无法确认结果的异常类。
|
||||
当文件信息查询因网络、限流或接口错误失败(区别于「确认不存在」)时抛出,
|
||||
调用方不应把该状态当作文件不存在处理。
|
||||
"""
|
||||
pass
|
||||
|
||||
+16
-2
@@ -1,9 +1,13 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field
|
||||
from app.schemas.types import StorageSchema
|
||||
|
||||
# Windows 盘符绝对路径,如 Z:/Downloads 或 Z:\Downloads
|
||||
WINDOWS_DRIVE_PATTERN = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
class FileURI(BaseModel):
|
||||
# 文件路径
|
||||
@@ -13,10 +17,19 @@ class FileURI(BaseModel):
|
||||
|
||||
@property
|
||||
def uri(self) -> str:
|
||||
"""
|
||||
文件 URI,本地存储直接返回路径,其他存储带上存储前缀
|
||||
"""
|
||||
return self.path if self.storage == "local" else f"{self.storage}:{self.path}"
|
||||
|
||||
@classmethod
|
||||
def from_uri(cls, uri: str) -> "FileURI":
|
||||
"""
|
||||
解析文件 URI 为存储类型和路径
|
||||
|
||||
:param uri: 文件 URI,如 /media/movie、u115:/media/movie 或 Windows 盘符路径 Z:/media
|
||||
:return: FileURI 对象
|
||||
"""
|
||||
storage, path = 'local', uri
|
||||
for s in StorageSchema:
|
||||
protocol = f"{s.value}:"
|
||||
@@ -24,11 +37,13 @@ class FileURI(BaseModel):
|
||||
path = uri[len(protocol):]
|
||||
storage = s.value
|
||||
break
|
||||
if not path.startswith("/"):
|
||||
# Windows 盘符路径本身就是绝对路径,补上根斜杠会得到 /Z:/xxx 这样的非法路径
|
||||
if not path.startswith("/") and not WINDOWS_DRIVE_PATTERN.match(path):
|
||||
path = "/" + path
|
||||
path = Path(path).as_posix()
|
||||
return cls(storage=storage, path=path)
|
||||
|
||||
|
||||
class FileItem(FileURI):
|
||||
# 类型 dir/file
|
||||
type: Optional[str] = None
|
||||
@@ -68,4 +83,3 @@ class StorageUsage(BaseModel):
|
||||
class StorageTransType(BaseModel):
|
||||
# 传输类型
|
||||
transtype: Optional[dict] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@@ -95,6 +95,26 @@ class PluginSidebarNavItem(BaseModel):
|
||||
order: int = Field(default=0, description="同组内排序,越小越靠前")
|
||||
|
||||
|
||||
class PluginRatingRequest(BaseModel):
|
||||
"""插件评分请求"""
|
||||
|
||||
rating: float = Field(
|
||||
ge=0.1,
|
||||
le=5.0,
|
||||
multiple_of=0.1,
|
||||
description="评分,范围 0.1 至 5.0,精确到 0.1",
|
||||
)
|
||||
|
||||
|
||||
class PluginRating(BaseModel):
|
||||
"""插件评分结果"""
|
||||
|
||||
plugin_id: str = Field(description="插件 ID")
|
||||
average_rating: float = Field(default=0.0, description="平均评分")
|
||||
rating_count: int = Field(default=0, description="评分人数")
|
||||
user_rating: Optional[float] = Field(default=None, description="当前安装实例评分")
|
||||
|
||||
|
||||
class PluginMemoryInfo(BaseModel):
|
||||
"""插件内存信息"""
|
||||
plugin_id: str = Field(description="插件ID")
|
||||
|
||||
+21
-1
@@ -1,7 +1,7 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +40,26 @@ class MediaServerConf(BaseModel):
|
||||
# 自动同步间隔(小时),未设置时使用旧全局配置
|
||||
sync_interval: Optional[int] = None
|
||||
|
||||
@field_validator("sync_interval", mode="before")
|
||||
@classmethod
|
||||
def validate_sync_interval(cls, value: Any) -> Optional[int]:
|
||||
"""
|
||||
兼容前端清空输入框后残留的空字符串等非法值,避免历史配置导致模块初始化失败
|
||||
|
||||
:param value: 原始配置值
|
||||
:return: 合法的间隔小时数,无法解析时返回 None
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class DownloaderConf(BaseModel):
|
||||
"""
|
||||
|
||||
+16
-7
@@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, List, Literal, Optional
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -60,9 +60,7 @@ class TransferTask(BaseModel):
|
||||
fileitem: FileItem
|
||||
meta: Optional[Any] = None
|
||||
mediainfo: Optional[Any] = None
|
||||
media_source: Optional[
|
||||
Literal["themoviedb", "douban", "bangumi", "anilist"]
|
||||
] = None
|
||||
media_source: Optional[str] = None
|
||||
target_directory: Optional[TransferDirectoryConf] = None
|
||||
target_storage: Optional[str] = None
|
||||
target_path: Optional[Path] = None
|
||||
@@ -220,9 +218,7 @@ class ManualTransferItem(BaseModel):
|
||||
# AniList ID
|
||||
anilistid: Optional[int] = None
|
||||
# 媒体数据源
|
||||
media_source: Optional[
|
||||
Literal["themoviedb", "douban", "bangumi", "anilist"]
|
||||
] = None
|
||||
media_source: Optional[str] = None
|
||||
# 数据源原生ID
|
||||
media_id: Optional[str] = None
|
||||
# 类型
|
||||
@@ -253,6 +249,19 @@ class ManualTransferItem(BaseModel):
|
||||
episode_group: Optional[str] = None
|
||||
# 仅预览,不执行整理
|
||||
preview: Optional[bool] = False
|
||||
# 重新整理,清理命中的成功历史及其旧目标
|
||||
reorganize: Optional[bool] = False
|
||||
|
||||
|
||||
class ManualTransferHistoryInfo(BaseModel):
|
||||
"""
|
||||
手动整理命中的成功历史摘要
|
||||
"""
|
||||
|
||||
# 是否应显示重新整理操作
|
||||
reorganize: bool = False
|
||||
# 命中的成功历史数量
|
||||
history_count: int = 0
|
||||
|
||||
|
||||
class ManualTransferTargetPath(BaseModel):
|
||||
|
||||
@@ -215,6 +215,8 @@ class SystemConfigKey(Enum):
|
||||
NotificationSwitchs = "NotificationSwitchs"
|
||||
# 目录配置
|
||||
Directories = "Directories"
|
||||
# 挂载型本地盘是否删除空目录
|
||||
MountedLocalDiskDeleteEmptyDirs = "MountedLocalDiskDeleteEmptyDirs"
|
||||
# 存储配置
|
||||
Storages = "Storages"
|
||||
# 搜索站点范围
|
||||
@@ -279,6 +281,8 @@ class SystemConfigKey(Enum):
|
||||
SetupWizardState = "SetupWizardState"
|
||||
# 绿联影视登录会话缓存
|
||||
UgreenSessionCache = "UgreenSessionCache"
|
||||
# 共享媒体识别成功次数
|
||||
MediaRecognizeShareCount = "MediaRecognizeShareCount"
|
||||
|
||||
|
||||
# 处理进度Key字典
|
||||
|
||||
+86
-3
@@ -3,10 +3,11 @@ import collections
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from contextlib import AsyncExitStack, contextmanager, asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
|
||||
import chardet
|
||||
import httpx
|
||||
@@ -72,6 +73,53 @@ _DEFAULT_MAX_CONNECTIONS = 40
|
||||
_DEFAULT_KEEPALIVE_EXPIRY = 30
|
||||
# 同步 requests.Session 复用连接时,遇到对端或代理关闭 keep-alive 后允许重试的方法
|
||||
_REQUESTS_RETRY_IDEMPOTENT_METHODS = ("GET", "HEAD", "OPTIONS")
|
||||
|
||||
# 代理走 CONNECT 隧道时,httpx 默认开启的 HTTP/2 多路复用会把并发请求叠加到极少数隧道上;
|
||||
# 隧道被代理节点切换或空闲回收打断后,复用其上的所有请求会同时失败。按 (proxy, host) 熔断:
|
||||
# 命中一次连接层失败就记录下次允许再尝试 h2 的时间戳(time.monotonic 基准),冷却期内该
|
||||
# (proxy, host) 的请求直接退化为 http1.1;冷却期结束后自动恢复尝试 h2。
|
||||
_H2_PROXY_BREAKER_COOLDOWN = 1800 # 30 分钟
|
||||
_h2_proxy_breaker_lock = threading.Lock()
|
||||
_h2_proxy_retry_at: Dict[Tuple[str, str], float] = {}
|
||||
# 只有这些错误是"h2 隧道被打断"的特征(对应实测日志里的 SEND_HEADERS in CLOSED、
|
||||
# EndOfStream 等);超时、连接失败、代理不可达等错误换 h1 一样会发生,
|
||||
# 不应触发熔断,也不值得付出一次注定同样失败的 h1 重试
|
||||
_H2_TUNNEL_BREAK_ERRORS = (
|
||||
httpx.RemoteProtocolError,
|
||||
httpx.LocalProtocolError,
|
||||
httpx.ReadError,
|
||||
httpx.WriteError,
|
||||
httpx.CloseError,
|
||||
)
|
||||
|
||||
|
||||
def _h2_proxy_breaker_key(proxy: str, url: str) -> Tuple[str, str]:
|
||||
try:
|
||||
host = httpx.URL(url).host or ""
|
||||
except Exception:
|
||||
host = url
|
||||
return proxy, host
|
||||
|
||||
|
||||
def _h2_proxy_allowed(proxy: Optional[str], url: str) -> bool:
|
||||
"""判断给定代理 + 目标 host 当前是否允许尝试 h2(未处于熔断冷却期)"""
|
||||
if not proxy:
|
||||
return True
|
||||
with _h2_proxy_breaker_lock:
|
||||
retry_at = _h2_proxy_retry_at.get(_h2_proxy_breaker_key(proxy, url), 0.0)
|
||||
return time.monotonic() >= retry_at
|
||||
|
||||
|
||||
def _trip_h2_proxy_breaker(proxy: str, url: str) -> None:
|
||||
"""记录一次 h2 连接层失败,熔断该 (proxy, host) 冷却期内的 h2 尝试"""
|
||||
now = time.monotonic()
|
||||
with _h2_proxy_breaker_lock:
|
||||
# 顺手清掉已过冷却期的条目,防止长期运行下字典无限增长
|
||||
for key in [k for k, retry_at in _h2_proxy_retry_at.items() if now >= retry_at]:
|
||||
del _h2_proxy_retry_at[key]
|
||||
_h2_proxy_retry_at[_h2_proxy_breaker_key(proxy, url)] = now + _H2_PROXY_BREAKER_COOLDOWN
|
||||
|
||||
|
||||
# 持有 LRU 淘汰后正在异步关闭的 transport task,避免 fire-and-forget 被 GC 警告
|
||||
_pending_eviction_tasks: set[asyncio.Task] = set()
|
||||
|
||||
@@ -1087,13 +1135,48 @@ class AsyncRequestUtils:
|
||||
self._client, method, url, raise_exception, **kwargs
|
||||
)
|
||||
|
||||
# 代理走 CONNECT 隧道时 h2 多路复用容易被隧道打断放大成批量失败(见
|
||||
# _h2_proxy_allowed 处注释);仅对幂等方法做"h2 失败就地降级 h1 重试",
|
||||
# 避免非幂等请求在服务端可能已收到数据的情况下重复产生副作用
|
||||
http2 = self._http2 and _h2_proxy_allowed(self._proxies, url)
|
||||
if not (http2 and self._proxies and method.upper() in _REQUESTS_RETRY_IDEMPOTENT_METHODS):
|
||||
return await self._dispatch_request(
|
||||
http2, cookies_dict, method, url, raise_exception, **kwargs
|
||||
)
|
||||
|
||||
try:
|
||||
return await self._dispatch_request(
|
||||
True, cookies_dict, method, url, True, **kwargs
|
||||
)
|
||||
except _H2_TUNNEL_BREAK_ERRORS as e:
|
||||
logger.debug(f"h2 代理连接层失败,熔断 {url} 所在 host 并降级 h1 重试: {e!r}")
|
||||
_trip_h2_proxy_breaker(self._proxies, url)
|
||||
return await self._dispatch_request(
|
||||
False, cookies_dict, method, url, raise_exception, **kwargs
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
# 与 h2 隧道无关的失败(超时、连接失败等):不熔断也不重试,
|
||||
# 恢复调用方原本的 raise_exception 语义
|
||||
if raise_exception:
|
||||
raise
|
||||
error_msg = str(e) or f"未知网络错误 (URL: {url}, Method: {method.upper()})"
|
||||
logger.debug(f"异步请求失败: {error_msg}")
|
||||
return None
|
||||
|
||||
async def _dispatch_request(
|
||||
self, http2: bool, cookies_dict: Optional[dict], method: str, url: str,
|
||||
raise_exception: bool, **kwargs
|
||||
) -> Optional[httpx.Response]:
|
||||
"""
|
||||
按给定 http2 开关构建/复用底层连接并发起请求,供 request() 的 h2/h1 熔断切换复用
|
||||
"""
|
||||
# 共享底层 transport(连接池+TLS 复用),每次请求创建轻量 AsyncClient。
|
||||
# AsyncClient 持有的 cookie jar 仅存活于本次请求 lifecycle,
|
||||
# 既复用握手又彻底避免 jar 跨调用累积。
|
||||
transport = _get_shared_async_transport(
|
||||
proxy=self._proxies,
|
||||
verify=self._verify,
|
||||
http2=self._http2,
|
||||
http2=http2,
|
||||
max_keepalive_connections=self._max_keepalive_connections,
|
||||
max_connections=self._max_connections,
|
||||
keepalive_expiry=self._keepalive_expiry,
|
||||
@@ -1113,7 +1196,7 @@ class AsyncRequestUtils:
|
||||
|
||||
# 兜底:没有运行中的事件循环时,临时客户端走完即关
|
||||
async with httpx.AsyncClient(
|
||||
http2=self._http2,
|
||||
http2=http2,
|
||||
proxy=self._proxies,
|
||||
timeout=self._timeout,
|
||||
verify=self._verify,
|
||||
|
||||
+13
-3
@@ -828,10 +828,13 @@ class SystemUtils:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_network_filesystem(directory: Path) -> bool:
|
||||
def is_network_filesystem(
|
||||
directory: Path, include_local_fuse: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
检测是否为网络文件系统
|
||||
:param directory: 目录路径
|
||||
:param include_local_fuse: 是否将本地 FUSE 挂载视为挂载文件系统
|
||||
:return: 是否为网络文件系统
|
||||
"""
|
||||
try:
|
||||
@@ -849,7 +852,10 @@ class SystemUtils:
|
||||
"fuseblk",
|
||||
# TBD
|
||||
]
|
||||
if any(fs in output for fs in local_fs):
|
||||
if (
|
||||
not include_local_fuse
|
||||
and any(fs in output for fs in local_fs)
|
||||
):
|
||||
return False
|
||||
network_fs = ['nfs', 'cifs', 'smbfs', 'fuse', 'sshfs', 'ftpfs']
|
||||
return any(fs in output for fs in network_fs)
|
||||
@@ -859,7 +865,11 @@ class SystemUtils:
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.lower()
|
||||
return 'nfs' in output or 'smbfs' in output
|
||||
return (
|
||||
'nfs' in output
|
||||
or 'smbfs' in output
|
||||
or (include_local_fuse and 'fuse' in output)
|
||||
)
|
||||
elif system == 'Windows':
|
||||
# Windows 检查网络驱动器
|
||||
return str(directory).startswith('\\\\')
|
||||
|
||||
@@ -481,6 +481,9 @@ moviepilot tool run search_torrents media_type=movie tmdb_id=12345
|
||||
- `tool list` 用于动态发现当前服务可调用的工具
|
||||
- `tool show` 会输出参数名、类型和描述
|
||||
- `tool run` 参数格式固定为 `key=value`
|
||||
- `read_file`、`write_file`、`edit_file` 和 `execute_command`
|
||||
属于内置 Agent 的本地敏感能力,不通过 MCP/`moviepilot tool` 暴露;插件开发时
|
||||
由 Agent 按当前用户权限直接调用这些工具。
|
||||
|
||||
## Scheduler 命令
|
||||
|
||||
|
||||
@@ -55,6 +55,43 @@ pip install -r requirements.txt
|
||||
pip install -r requirements-dev.in
|
||||
```
|
||||
|
||||
### 2.1 本地启动脚本
|
||||
|
||||
不需要打开 IDE 时,可以直接使用仓库内的启动脚本。脚本会自动定位项目根目录和虚拟环境,并以模块方式启动后端,避免 `ModuleNotFoundError: No module named 'app'`。
|
||||
|
||||
```bash
|
||||
# 默认启动后端开发服务,前台运行,按 Ctrl+C 停止
|
||||
./scripts/start-local.sh
|
||||
./scripts/start-local.sh backend
|
||||
|
||||
# 如果已经安装前端发布包,可启动完整的前后端服务
|
||||
./scripts/start-local.sh service start
|
||||
|
||||
# 管理完整服务
|
||||
./scripts/start-local.sh stop
|
||||
./scripts/start-local.sh restart
|
||||
./scripts/start-local.sh status
|
||||
./scripts/start-local.sh logs --follow
|
||||
```
|
||||
|
||||
默认会使用 `DEBUG=true` 和 `DEV=true`,与 IDE 开发启动保持一致;如果不需要热重载,可以这样启动以降低资源占用:
|
||||
|
||||
```bash
|
||||
DEV=false ./scripts/start-local.sh
|
||||
```
|
||||
|
||||
脚本会优先使用 `CONFIG_DIR`,其次使用 `MOVIEPILOT_CONFIG_DIR`,再检测 `~/Documents/moviepilot`,最后回退到仓库内的 `config` 目录。需要使用其他配置目录时,可以这样运行:
|
||||
|
||||
```bash
|
||||
MOVIEPILOT_CONFIG_DIR=/path/to/moviepilot-config ./scripts/start-local.sh
|
||||
```
|
||||
|
||||
首次使用前如果脚本没有执行权限,运行:
|
||||
|
||||
```bash
|
||||
chmod +x scripts/start-local.sh
|
||||
```
|
||||
|
||||
### 3. 修改主程序依赖
|
||||
|
||||
新增或升级依赖时,先确认依赖属于哪个层级:
|
||||
|
||||
+34
-3
@@ -31,6 +31,12 @@ MCP 使用系统配置中的 `API_TOKEN` 作为认证密钥,文档中的 API K
|
||||
- `tools/call`: 调用特定工具。
|
||||
- `ping`: 连接存活检测。
|
||||
|
||||
### 动态插件工具
|
||||
|
||||
`tools/list` 会同时返回 MoviePilot 内置工具和已启用插件通过 `get_agent_tools()` 声明的工具。插件启动、停止、重载或配置生效后,MCP 工具管理器会在下一次列出或调用工具时按注册表版本惰性刷新,避免继续暴露已移除的工具或遗漏新工具。
|
||||
|
||||
MCP 当前不会主动发送工具列表变更通知(`listChanged=false`)。如果客户端缓存了工具列表,插件状态变化后需要让客户端重新请求 `tools/list`;无法手动刷新的客户端应重新连接 MCP 服务或新建会话。
|
||||
|
||||
---
|
||||
|
||||
## 4. 客户端配置示例
|
||||
@@ -124,7 +130,8 @@ FastAPI 异常响应保留 `detail` 字段,并在错误详情为文本时返
|
||||
| GET | `/api/v1/media/{mediaid}` | 查询媒体详情,`mediaid` 支持 `tmdb:`、`douban:`、`bangumi:`、`anilist:` 及插件自定义来源前缀 |
|
||||
| POST | `/api/v1/media/scrape/{storage}` | 刮削媒体元数据;请求体为 `FileItem`,可选查询参数 `media_source`、`media_id`、`type_name`(电影/电视剧)可指定本次刮削媒体 |
|
||||
| POST | `/api/v1/transfer/manual/target-path` | 匹配手动整理目标路径;请求体可用 `media_source` + `media_id` 指定数据源原生ID |
|
||||
| POST | `/api/v1/transfer/manual` | 手动整理;请求体可用 `media_source` + `media_id` 指定本次识别与刮削数据源,同时兼容 `tmdbid`、`doubanid`、`bangumiid`、`anilistid` |
|
||||
| POST | `/api/v1/transfer/manual/history` | 查询文件、批量文件或目录命中的成功整理历史摘要,用于进入手动整理界面时显示重新整理状态 |
|
||||
| POST | `/api/v1/transfer/manual` | 手动整理;请求体可用 `media_source` + `media_id` 指定本次识别与刮削数据源,同时兼容 `tmdbid`、`doubanid`、`bangumiid`、`anilistid`;命中失败历史时自动清理旧目标和记录后重试,`reorganize=true` 时清理命中的成功历史和非移动模式旧目标后重新整理 |
|
||||
|
||||
#### 搜索 / 种子 / 字幕
|
||||
|
||||
@@ -142,6 +149,8 @@ FastAPI 异常响应保留 `detail` 字段,并在错误详情为文本时返
|
||||
| GET | `/api/v1/search/last/context` | 获取上一次搜索结果及可复用搜索参数,`params.result_type` 为 `torrent` 或 `subtitle` |
|
||||
| POST | `/api/v1/search/recommend` | 获取 AI 推荐资源,请求体:`filtered_indices`、`check_only`、`force` |
|
||||
|
||||
渐进式搜索在无业务事件时每 15 秒发送 `{"type":"heartbeat"}`,客户端应将其仅用于连接保活。超过 48 条的最终 `replace` 会分批发送:首批 `type=replace`,后续批次 `type=append`,所有批次均带 `replace_batch=true`、从 0 开始的 `batch_index`、`batch_count` 和最终 `total_items`;客户端必须按顺序收齐后再原子替换结果。最终 `done` 在已发送 `replace` 后不重复携带 `items`。
|
||||
|
||||
#### AniList 榜单 / 探索
|
||||
|
||||
AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-chinese` 代理查询。代理不可用时自动回退 AniList 官方 GraphQL,并合并 `anilist-chinese` 每日数据集;媒体标题优先使用项目提供的中文标题,未提供中文标题时回退 AniList 原语言标题。
|
||||
@@ -194,9 +203,15 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| :--- | :--- | :--- |
|
||||
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存及识别成功、失败条目统计 |
|
||||
| GET | `/api/v1/tmdb/cache` | 查询 TheMovieDb 识别缓存统计、共享识别累计成功命中次数及开关状态 |
|
||||
| DELETE | `/api/v1/tmdb/cache/{cache_key}` | 按缓存键删除单条 TheMovieDb 识别缓存,缓存键需要进行 URL 编码 |
|
||||
| DELETE | `/api/v1/tmdb/cache` | 清空全部 TheMovieDb 识别缓存 |
|
||||
| GET | `/api/v1/douban/cache` | 查询豆瓣识别缓存统计、共享识别累计成功命中次数及开关状态 |
|
||||
| DELETE | `/api/v1/douban/cache/{cache_key}` | 按缓存键删除单条豆瓣识别缓存,缓存键需要进行 URL 编码 |
|
||||
| DELETE | `/api/v1/douban/cache` | 清空全部豆瓣识别缓存 |
|
||||
|
||||
缓存查询响应的 `data` 包含 `count`、`recognized`、`unrecognized`、`data`,以及共享识别统计字段
|
||||
`shared_recognized` 和开关字段 `shared_recognize_enabled`。共享命中次数仅在共享结果驱动的二次媒体识别成功后累计。
|
||||
|
||||
### 插件补充接口
|
||||
|
||||
@@ -204,13 +219,29 @@ AniList 榜单、探索、详情、人物和推荐接口优先通过 `anilist-ch
|
||||
|
||||
按需读取指定已安装插件的最新远端更新说明。该接口用于前端在用户点击“查看更新说明”时再实时访问插件仓库,避免加载已安装插件列表时批量请求网络。
|
||||
|
||||
**GET** `/api/v1/plugin/rating?plugin_ids={plugin_id,...}`
|
||||
|
||||
批量查询插件平均分、评分人数和当前安装实例评分。`plugin_ids` 省略时查询中心端已有的全部插件评分。
|
||||
|
||||
**GET** `/api/v1/plugin/rating/{plugin_id}`
|
||||
|
||||
查询单个插件平均分、评分人数和当前安装实例评分。中心端暂不可用时返回该插件的零评分结果。
|
||||
|
||||
**POST** `/api/v1/plugin/rating/{plugin_id}`
|
||||
|
||||
为已安装插件提交当前安装实例评分,请求体为 `{"rating": 4.5}`。评分范围为 `0.1` 至 `5.0`,精确到 `0.1`;同一安装实例再次提交会更新原评分。
|
||||
|
||||
### 1. 列出所有工具
|
||||
|
||||
**GET** `/api/v1/mcp/tools`
|
||||
|
||||
获取所有可用的MCP工具列表。
|
||||
|
||||
工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。
|
||||
内置工具的 `inputSchema` 只包含实际执行业务所需的参数,不包含用于解释调用原因的通用 `explanation` 参数,以减少 Agent 上下文消耗。插件工具的参数结构由插件自身声明。
|
||||
|
||||
内置 Agent 的本地文件与命令工具 `read_file`、`write_file`、`edit_file`、
|
||||
`execute_command` 不通过 MCP 暴露。这些工具在 Agent 运行时执行独立的
|
||||
用户权限与路径边界检查;MCP 隐藏列表只负责收敛接口暴露面,不替代权限控制。
|
||||
|
||||
媒体相关 MCP 工具(如 `query_media_detail`、`search_torrents`、`query_library_exists`、`add_subscribe`、`transfer_file`)接受 `tmdb_id`/`tmdbid`、`douban_id`/`doubanid`、`bangumi_id`/`bangumiid`、`anilist_id`/`anilistid`,也接受 `media_source` + `media_id`。工具返回的媒体、订阅、下载和整理记录会同步带回可用的四种专用 ID 及通用主身份。
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ testpaths =
|
||||
tests
|
||||
timeout = 120
|
||||
timeout_method = thread
|
||||
asyncio_mode = strict
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
# 仅对「无法在本仓修复根因」的已知上游/三方弃用告警做精确忽略,保持测试输出干净、
|
||||
# 让本仓自身的新告警更醒目。本仓代码引发的告警一律不在此忽略,应在源码/用例处修复。
|
||||
filterwarnings =
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Cython~=3.2.5
|
||||
pylint~=4.0.6
|
||||
pytest~=9.0.3
|
||||
pytest-asyncio~=1.4.0
|
||||
pytest-cov~=7.1.0
|
||||
pytest-timeout~=2.4.0
|
||||
uv~=0.11.23
|
||||
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
MOVIEPILOT_BIN="$PROJECT_ROOT/moviepilot"
|
||||
VENV_PYTHON="$PROJECT_ROOT/venv/bin/python"
|
||||
|
||||
show_usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/start-local.sh 启动后端开发服务(前台运行)
|
||||
./scripts/start-local.sh backend 启动后端开发服务(前台运行)
|
||||
./scripts/start-local.sh service start 启动后端和已安装的前端服务
|
||||
./scripts/start-local.sh service start --safe 以安全模式启动完整服务
|
||||
./scripts/start-local.sh stop|restart|status 管理后端和前端服务
|
||||
./scripts/start-local.sh logs [OPTIONS] 查看后端日志
|
||||
./scripts/start-local.sh help 显示本帮助
|
||||
EOF
|
||||
}
|
||||
|
||||
if [[ ! -x "$MOVIEPILOT_BIN" ]]; then
|
||||
printf '未找到本地 CLI:%s\n' "$MOVIEPILOT_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -x "$VENV_PYTHON" ]]; then
|
||||
printf '未找到项目虚拟环境:%s\n请先执行:%s install deps\n' "$VENV_PYTHON" "$MOVIEPILOT_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 显式传入配置目录,避免被仓库中的临时 .moviepilot.env 覆盖。
|
||||
if [[ -z "${CONFIG_DIR:-}" ]]; then
|
||||
if [[ -n "${MOVIEPILOT_CONFIG_DIR:-}" ]]; then
|
||||
CONFIG_DIR="$MOVIEPILOT_CONFIG_DIR"
|
||||
elif [[ -d "${HOME:-}/Documents/moviepilot" ]]; then
|
||||
CONFIG_DIR="${HOME}/Documents/moviepilot"
|
||||
else
|
||||
CONFIG_DIR="$PROJECT_ROOT/config"
|
||||
fi
|
||||
fi
|
||||
export CONFIG_DIR
|
||||
export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
|
||||
export DEBUG="${DEBUG:-true}"
|
||||
export DEV="${DEV:-true}"
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
set -- backend
|
||||
fi
|
||||
|
||||
command_name="$1"
|
||||
shift
|
||||
|
||||
case "$command_name" in
|
||||
backend|start)
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
printf '后端模块启动不接受额外参数;完整服务请使用:%s service start [OPTIONS]\n' "$0" >&2
|
||||
exit 2
|
||||
fi
|
||||
exec "$VENV_PYTHON" -m app.main
|
||||
;;
|
||||
service)
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
set -- start
|
||||
fi
|
||||
exec "$MOVIEPILOT_BIN" "$@"
|
||||
;;
|
||||
stop|restart|status|logs|doctor|config|version)
|
||||
exec "$MOVIEPILOT_BIN" "$command_name" "$@"
|
||||
;;
|
||||
help|--help|-h)
|
||||
show_usage
|
||||
;;
|
||||
*)
|
||||
printf '未知命令:%s\n\n' "$command_name" >&2
|
||||
show_usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: create-moviepilot-plugin
|
||||
version: 2
|
||||
version: 3
|
||||
description: >-
|
||||
Use this skill when the user asks to create, modify, debug, validate, or
|
||||
scaffold a MoviePilot local plugin. Covers MoviePilot V2 plugin development,
|
||||
@@ -11,7 +11,7 @@ description: >-
|
||||
sidebar pages, commands, services, workflow actions, agent tools, and local
|
||||
install/reload flows. Also use for Chinese requests mentioning 编写插件、本地插件源,
|
||||
插件开发, V2插件, 插件市场, 本地安装插件, 插件热加载, 前端联邦, 侧栏入口, Vue插件页面.
|
||||
allowed-tools: list_directory read_file write_file edit_file execute_command query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
|
||||
allowed-tools: list_directory read_file write_file edit_file execute_command search_web browse_webpage query_system_settings update_system_settings query_market_plugins install_plugin reload_plugin query_installed_plugins
|
||||
---
|
||||
|
||||
# Create MoviePilot Plugin
|
||||
@@ -33,6 +33,33 @@ a local plugin source and installed into the running MoviePilot instance.
|
||||
- When working in or from `MoviePilot-Plugins`, read its `README.md`,
|
||||
`docs/Repository_Guide.md`, and `docs/V2_Plugin_Development.md`. For
|
||||
scenario-specific extensions, read the matching `docs/faq/*.md`.
|
||||
|
||||
## Code Tool Workflow
|
||||
|
||||
- Use `execute_command(action="run")` with `rg` and narrow globs or paths to
|
||||
locate plugin classes, extension points, tests, and package entries. Use
|
||||
`list_directory` only when inspecting one known folder or a configured remote
|
||||
storage backend.
|
||||
- Read the relevant implementation and adjacent example before editing.
|
||||
- Before using a Python or Node.js dependency API, determine the exact installed
|
||||
or locked version from requirements, package manifests, lockfiles, local
|
||||
package source, and `.pyi`/`.d.ts` declarations. If those are insufficient,
|
||||
use `search_web` with the official documentation domain and `browse_webpage`
|
||||
to read the matching version. Do not guess API signatures from memory or mix
|
||||
examples from different major versions. Search the relevant package directory,
|
||||
`.venv`, or `node_modules` directly with `rg` instead of scanning the entire
|
||||
project without bounds.
|
||||
- Use `edit_file` for localized changes. Its `old_text` must identify one exact
|
||||
location by default; add surrounding context instead of enabling
|
||||
`replace_all` unless every match intentionally changes.
|
||||
- Use `write_file` for new files. Existing files require `overwrite=true` for a
|
||||
full rewrite; first call `read_file(include_metadata=true)` and pass its
|
||||
`sha256` as `expected_sha256` when replacing previously read content.
|
||||
- Use `execute_command(action="run")` for short validation, Git, and diagnostic
|
||||
commands. Use `action="start"` only for interactive or long-running commands,
|
||||
then continue through the returned session ID.
|
||||
- Do not use shell redirection or inline scripts to perform source edits or to
|
||||
bypass a file-tool permission error.
|
||||
- When the plugin uses Vue federation, also read
|
||||
`MoviePilot-Frontend/docs/module-federation-guide.md`,
|
||||
`MoviePilot-Frontend/docs/federation-troubleshooting.md`,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user