Compare commits

...

9 Commits
v2.13.13 ... v2

Author SHA1 Message Date
jxxghp
dc773337d3 fix: bind browser sessions to dedicated worker threads 2026-06-22 22:18:12 +08:00
jxxghp
5c649ff1d1 fix(search): suppress linter warning for SitesHelper import 2026-06-22 21:53:41 +08:00
jxxghp
3407cc8edd Preserve empty tool selections 2026-06-22 21:29:15 +08:00
jxxghp
f9ea0118d9 Handle Telegramify 1.2 compatibility 2026-06-22 20:16:37 +08:00
jxxghp
ad73434e2c fix(agent): stabilize tool selector logging 2026-06-22 19:06:14 +08:00
jxxghp
a6afa0fbc0 fix(agent): log tool selector results 2026-06-22 18:48:04 +08:00
jxxghp
3306d196b7 Refine existing implementation 2026-06-22 18:21:20 +08:00
InfinityPacer
e44a6f41b5 chore: split runtime and development dependencies (#5985) 2026-06-22 17:44:06 +08:00
jxxghp
7a7b27858e Remove redundant Cython Docker install 2026-06-22 14:06:19 +08:00
26 changed files with 1458 additions and 625 deletions

View File

@@ -23,24 +23,15 @@ jobs:
uses: actions/cache@v4 uses: actions/cache@v4
with: with:
path: ~/.cache/pip path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/requirements.in') }} key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.in', '**/requirements-dev.in', '**/requirements.txt') }}
restore-keys: | restore-keys: |
${{ runner.os }}-pip- ${{ runner.os }}-pip-
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade pip setuptools wheel
pip install pylint # Pylint 属于开发/静态检查依赖,统一通过 dev 入口安装。
# 安装项目依赖 pip install -r requirements-dev.in
if [ -f requirements.txt ]; then
echo "📦 安装 requirements.txt 中的依赖..."
pip install -r requirements.txt
elif [ -f requirements.in ]; then
echo "📦 安装 requirements.in 中的依赖..."
pip install -r requirements.in
else
echo "⚠️ 未找到依赖文件,仅安装 pylint"
fi
- name: Verify pylint config - name: Verify pylint config
run: | run: |
@@ -88,4 +79,4 @@ jobs:
run: | run: |
echo "🎉 Pylint 检查完成!" echo "🎉 Pylint 检查完成!"
echo "✅ 没有发现语法错误或严重问题" echo "✅ 没有发现语法错误或严重问题"
echo "📊 详细报告已保存为构建工件" echo "📊 详细报告已保存为构建工件"

View File

@@ -38,16 +38,15 @@ jobs:
uses: actions/cache@v5 uses: actions/cache@v5
with: with:
path: ~/.cache/pip path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.in', '**/requirements.txt') }} key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.in', '**/requirements-dev.in', '**/requirements.txt') }}
restore-keys: | restore-keys: |
${{ runner.os }}-pip- ${{ runner.os }}-pip-
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade pip setuptools wheel
# 用 requirements.in 还原 CI / 全新环境(含 pytest 与 moviepilot-rust 等可选扩展), # 单测需要开发/测试依赖;运行时入口 requirements.in 不携带测试与构建辅助工具。
# 与本地"干净 venv 复现"一致;测试运行器 pytest 已在 requirements.in 中声明。 pip install -r requirements-dev.in
pip install -r requirements.in
- name: Run tests - name: Run tests
timeout-minutes: 10 timeout-minutes: 10

View File

@@ -838,6 +838,8 @@ class MoviePilotAgent:
detail = cls._exception_detail_text(error).lower() detail = cls._exception_detail_text(error).lower()
if "no endpoints found that support image input" in detail: if "no endpoints found that support image input" in detail:
return True return True
if "not a vlm" in detail or "text-only prompts" in detail:
return True
if "unknown variant" in detail and "image_url" in detail: if "unknown variant" in detail and "image_url" in detail:
return True return True
if "image input" not in detail and "images" not in detail: if "image input" not in detail and "images" not in detail:

View File

@@ -691,7 +691,9 @@ class AgentCapabilityManager:
@staticmethod @staticmethod
def supports_image_input() -> bool: def supports_image_input() -> bool:
"""当前 Agent 是否启用图片输入能力。""" """当前 Agent 是否启用图片输入能力。"""
return bool(settings.LLM_SUPPORT_IMAGE_INPUT) from app.agent.llm.helper import LLMHelper
return LLMHelper.supports_image_input()
@staticmethod @staticmethod
def supports_audio_input() -> bool: def supports_audio_input() -> bool:

View File

@@ -5,7 +5,7 @@ import inspect
import json import json
import time import time
from functools import wraps from functools import wraps
from typing import Any, List from typing import Any, List, Optional
from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import AIMessage, AIMessageChunk
@@ -700,11 +700,85 @@ class LLMHelper:
return {} return {}
@staticmethod @staticmethod
def supports_image_input() -> bool: def _metadata_supports_image_input(metadata: Any) -> Optional[bool]:
"""从模型元数据中读取图片输入能力,未知时返回 None。"""
if not isinstance(metadata, dict):
return None
modalities = metadata.get("modalities") or {}
input_modalities = modalities.get("input")
if isinstance(input_modalities, str):
input_modalities = [input_modalities]
if isinstance(input_modalities, list):
normalized_modalities = {
str(item or "").strip().lower() for item in input_modalities
}
return "image" in normalized_modalities
return None
@classmethod
def _resolve_catalog_image_input_support(
cls,
provider: Optional[str] = None,
model: Optional[str] = None,
base_url: Optional[str] = None,
base_url_preset: Optional[str] = None,
) -> Optional[bool]:
"""复用 provider 目录缓存解析当前模型是否支持图片输入。"""
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).strip()
model_name = str(model if model is not None else settings.LLM_MODEL).strip()
if not provider_name or not model_name:
return None
try:
from app.agent.llm.provider import LLMProviderManager
metadata = LLMProviderManager().resolve_cached_model_metadata(
provider_id=provider_name,
model_id=model_name,
base_url=base_url if base_url is not None else settings.LLM_BASE_URL,
base_url_preset_id=(
base_url_preset
if base_url_preset is not None
else settings.LLM_BASE_URL_PRESET
),
)
except Exception as err:
logger.debug(f"解析模型图片能力失败: {err}")
return None
return cls._metadata_supports_image_input(metadata)
@classmethod
def supports_image_input(
cls,
provider: Optional[str] = None,
model: Optional[str] = None,
base_url: Optional[str] = None,
base_url_preset: Optional[str] = None,
) -> bool:
""" """
判断当前模型是否启用了图片输入能力。 判断当前模型是否启用了图片输入能力。
用户开关为总开关;当内置模型目录明确标注当前模型不支持 image 输入时,
即使总开关开启也降级为纯文本,避免文本模型收到 `image_url` 内容块后
被兼容端点以 400 拒绝。无参调用保持旧版“只读总开关”语义,
未知自定义模型也保持原有开关语义。
""" """
return bool(settings.LLM_SUPPORT_IMAGE_INPUT) if not settings.LLM_SUPPORT_IMAGE_INPUT:
return False
if provider is None and model is None:
return True
image_support = cls._resolve_catalog_image_input_support(
provider=provider,
model=model,
base_url=base_url,
base_url_preset=base_url_preset,
)
if image_support is not None:
return image_support
return True
@staticmethod @staticmethod
def _build_legacy_runtime( def _build_legacy_runtime(
@@ -798,6 +872,41 @@ class LLMHelper:
return True return True
return None return None
@staticmethod
def _attach_runtime_metadata(model: Any, runtime: dict[str, Any]) -> None:
"""
将 MoviePilot 已解析出的 provider 运行时信息挂到模型实例上。
这些字段只供内部中间件识别协议能力,不参与 LangChain 请求序列化。
"""
runtime_metadata = {
"runtime": runtime.get("runtime"),
"provider_id": runtime.get("provider_id"),
"base_url": runtime.get("base_url"),
}
def _set_metadata_attr(name: str, value: Any) -> None:
try:
setattr(model, name, value)
except Exception:
object.__setattr__(model, name, value)
try:
_set_metadata_attr("_moviepilot_llm_runtime", runtime_metadata["runtime"])
_set_metadata_attr(
"_moviepilot_llm_provider_id",
runtime_metadata["provider_id"],
)
_set_metadata_attr("_moviepilot_llm_base_url", runtime_metadata["base_url"])
except Exception as err:
logger.debug(f"LLM运行时元数据附加失败: {str(err)}")
profile = getattr(model, "profile", None)
if isinstance(profile, dict):
profile["moviepilot_runtime"] = runtime_metadata["runtime"]
profile["moviepilot_provider_id"] = runtime_metadata["provider_id"]
profile["moviepilot_base_url"] = runtime_metadata["base_url"]
@classmethod @classmethod
def _resolve_thinking_level( def _resolve_thinking_level(
cls, cls,
@@ -1011,6 +1120,7 @@ class LLMHelper:
"max_input_tokens": int(max_input_tokens), "max_input_tokens": int(max_input_tokens),
} }
cls._attach_runtime_metadata(model, runtime)
return model return model
@staticmethod @staticmethod

View File

@@ -1564,6 +1564,70 @@ class LLMProviderManager(metaclass=Singleton):
return models[candidate] return models[candidate]
return None return None
def _cached_models_dev_model(
self,
provider_id: str,
model_id: str,
base_url: Optional[str] = None,
base_url_preset_id: Optional[str] = None,
) -> dict[str, Any] | None:
"""从已缓存或内置的 models.dev 数据中同步读取模型元数据。"""
try:
spec = self.get_provider(provider_id)
except LLMProviderError:
return None
models_dev_provider_id = self._resolve_provider_models_dev_provider_id(
spec,
base_url,
base_url_preset_id=base_url_preset_id,
)
if not models_dev_provider_id:
return None
payload = self._cached_models_dev_payload().get(models_dev_provider_id, {}) or {}
models = payload.get("models") if isinstance(payload, dict) else None
if not isinstance(models, dict):
return None
candidates = [model_id]
if model_id.startswith("models/"):
candidates.append(model_id.removeprefix("models/"))
for candidate in candidates:
if candidate in models:
return models[candidate]
return None
def resolve_cached_model_metadata(
self,
provider_id: str,
model_id: Optional[str],
base_url: Optional[str] = None,
base_url_preset_id: Optional[str] = None,
) -> dict[str, Any] | None:
"""同步解析缓存中的模型元数据,不触发远端 models.dev 刷新。"""
if not model_id:
return None
metadata = self._cached_models_dev_model(
provider_id,
model_id,
base_url=base_url,
base_url_preset_id=base_url_preset_id,
)
if metadata:
return metadata
if provider_id == "chatgpt":
return self._cached_models_dev_model("openai", model_id)
if provider_id == "openai":
return (
self._cached_models_dev_payload()
.get("openai", {})
.get("models", {})
.get(model_id)
)
return None
@staticmethod @staticmethod
def _normalize_model_record( def _normalize_model_record(
model_id: str, model_id: str,

View File

@@ -1,6 +1,6 @@
"""MoviePilot 自定义工具筛选中间件。""" """MoviePilot 自定义工具筛选中间件。"""
from dataclasses import replace from dataclasses import dataclass, replace
import json import json
from collections.abc import Awaitable, Callable from collections.abc import Awaitable, Callable
from typing import Annotated, Any, NotRequired from typing import Annotated, Any, NotRequired
@@ -20,7 +20,7 @@ from langchain.agents.middleware.tool_selection import (
LLMToolSelectorMiddleware, LLMToolSelectorMiddleware,
) )
from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig from langchain_core.runnables import RunnableConfig
from langchain_core.tools import BaseTool from langchain_core.tools import BaseTool
from langgraph.runtime import Runtime from langgraph.runtime import Runtime
@@ -68,19 +68,25 @@ class ToolSelectionStateUpdate(TypedDict):
selected_tool_names: list[str] | None selected_tool_names: list[str] | None
@dataclass(frozen=True)
class _ToolSelectionAttempt:
"""工具筛选尝试结果,用于统一记录最终日志。"""
request: ModelRequest
selected_tool_names: list[str]
status: str
detail: str = ""
class ToolSelectorMiddleware(LLMToolSelectorMiddleware): class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
""" """
为 DeepSeek 兼容端点提供更稳妥的工具筛选实现 使用 provider-neutral JSON 提示执行工具筛选。
LangChain 默认会通过 `with_structured_output()` 走 OpenAI LangChain 默认会通过 `with_structured_output()` 走 provider-specific
`response_format=json_schema` 路径,但 DeepSeek 官方 OpenAI 兼容端点公开文档 结构化输出能力,不同 OpenAI/Anthropic 兼容端点对 `response_format`、
仅保证 `json_object` 模式可用。对于 `deepseek-reasoner`,这会在工具筛选阶段 JSON schema 和工具绑定的支持并不一致。工具筛选只是 Agent 执行前的
提前触发 400导致 Agent 还没真正开始执行工具就失败。 辅助优化,失败时也会恢复使用全部工具,因此这里统一使用文本提示约束
模型返回 `{"tools": [...]}` 并手动解析,避免在筛选阶段引入额外兼容分支。
因此这里仅在识别到 DeepSeek 模型/端点时,退回到显式 JSON 输出模式:
1. 使用 `response_format={"type": "json_object"}`
2. 在提示词中明确约束返回 JSON 结构;
3. 手动解析 `{"tools": [...]}`,其余模型继续沿用 LangChain 默认实现。
另外LangChain 原生工具筛选挂在 `wrap_model_call` 上,会在同一条用户请求 另外LangChain 原生工具筛选挂在 `wrap_model_call` 上,会在同一条用户请求
的每次“模型回合”前都重新筛选一次工具。对于会多轮调用工具的复杂任务, 的每次“模型回合”前都重新筛选一次工具。对于会多轮调用工具的复杂任务,
@@ -322,21 +328,16 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
request: ModelRequest[ContextT], request: ModelRequest[ContextT],
) -> ModelRequest[ContextT]: ) -> ModelRequest[ContextT]:
""" """
处理工具筛选响应,并保留空结果回退所有工具的 MoviePilot 策略 处理工具筛选响应,并在正常空结果时禁用可筛选工具
""" """
if response.get("tools") == []: if response.get("tools") == []:
logger.warning("工具筛选结果为空,将恢复使用所有工具。")
always_included_tools: list[BaseTool] = [ always_included_tools: list[BaseTool] = [
tool tool
for tool in request.tools for tool in request.tools
if not isinstance(tool, dict) and tool.name in self.always_include if not isinstance(tool, dict) and tool.name in self.always_include
] ]
provider_tools = [tool for tool in request.tools if isinstance(tool, dict)] provider_tools = [tool for tool in request.tools if isinstance(tool, dict)]
return request.override(tools=[*always_included_tools, *provider_tools])
return request.override(
tools=[*available_tools, *always_included_tools, *provider_tools]
)
response["tools"] = self._complete_low_count_selection( response["tools"] = self._complete_low_count_selection(
selected_tool_names=[ selected_tool_names=[
@@ -347,47 +348,21 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
valid_tool_names=valid_tool_names, valid_tool_names=valid_tool_names,
available_tools=available_tools, available_tools=available_tools,
) )
return super()._process_selection_response( modified_request = super()._process_selection_response(
response, response,
available_tools, available_tools,
valid_tool_names, valid_tool_names,
request, request,
) )
return modified_request
@staticmethod
def _is_deepseek_compatible_model(model: BaseChatModel) -> bool:
"""
判断当前模型是否应当走 DeepSeek JSON 兼容分支。
除了官方 `langchain_deepseek`,用户也可能通过 OpenAI-compatible
配置把 DeepSeek 端点接到 `ChatOpenAI`。因此这里同时检查模块名、模型名
和 Base URL避免只靠单一条件漏判。
"""
module_name = type(model).__module__.lower()
model_name = (
str(getattr(model, "model_name", "") or getattr(model, "model", ""))
.strip()
.lower()
)
base_url = (
str(getattr(model, "openai_api_base", "") or getattr(model, "api_base", ""))
.strip()
.lower()
)
return (
"deepseek" in module_name
or model_name.startswith("deepseek-")
or "api.deepseek.com" in base_url
)
@staticmethod @staticmethod
def _parse_json_object(text: str) -> dict[str, Any]: def _parse_json_object(text: str) -> dict[str, Any]:
""" """
解析模型返回的 JSON。 解析模型返回的 JSON。
DeepSeek 在 JSON 模式下通常会返回纯 JSON但这里仍做一层兜底 不同模型可能偶发输出 Markdown 围栏或前后说明文本,因此这里从
兼容模型偶发输出围栏或前后说明文本的情况 响应中提取第一个 JSON 对象作为兜底
""" """
stripped_text = text.strip() stripped_text = text.strip()
if not stripped_text: if not stripped_text:
@@ -440,16 +415,16 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
) )
return f"Capability groups from tool tags:\n{rendered_groups}\n\n" return f"Capability groups from tool tags:\n{rendered_groups}\n\n"
def _build_deepseek_selection_prompt(self, selection_request: Any) -> str: def _build_json_selection_prompt(self, selection_request: Any) -> str:
""" """
为 DeepSeek 生成显式 JSON 输出提示。 生成显式 JSON 输出提示。
DeepSeek 官方文档要求在 JSON 输出模式下,提示词中必须明确包含 JSON 使用纯提示约束可覆盖更多兼容端点,避免在工具筛选阶段依赖某个
约束,否则兼容端点可能返回空内容或无意义输出 provider 专属的 `response_format` 或 schema 能力
""" """
limit_instruction = "" limit_instruction = ""
if self.max_tools: if self.max_tools:
limit_instruction = f"- Select up to {self.max_tools} tools. IF NO TOOLS ARE RELEVANT, DO NOT RETURN AN EMPTY ARRAY. SELECT THE MOST APPLICABLE ONES TO ENSURE THE REQUEST IS HANDLED." limit_instruction = f"- Select up to {self.max_tools} tools. Return an empty array if no tools are relevant."
return ( return (
f"{selection_request.system_message}\n\n" f"{selection_request.system_message}\n\n"
@@ -469,7 +444,7 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
def _normalize_selection_response(self, response: Any) -> dict[str, list[str]]: def _normalize_selection_response(self, response: Any) -> dict[str, list[str]]:
""" """
解析并标准化 DeepSeek JSON 模式的工具筛选结果。 解析并标准化显式 JSON 模式的工具筛选结果。
""" """
content = getattr(response, "content", response) content = getattr(response, "content", response)
text = LLMHelper.extract_text_content(content) text = LLMHelper.extract_text_content(content)
@@ -486,22 +461,21 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
logger.debug(f"工具筛选标准化结果: {normalized_tools}") logger.debug(f"工具筛选标准化结果: {normalized_tools}")
return {"tools": normalized_tools} return {"tools": normalized_tools}
async def _aselect_tools_with_deepseek( async def _aselect_tools_with_json_prompt(
self, selection_request: Any self, selection_request: Any
) -> dict[str, list[str]]: ) -> dict[str, list[str]]:
""" """
使用 DeepSeek 兼容的 JSON 输出模式执行异步工具筛选。 使用 JSON 提示执行异步工具筛选。
:param selection_request: LangChain 工具筛选请求
:return: 标准化后的工具名列表
""" """
logger.debug("工具筛选走 DeepSeek JSON 兼容分支") logger.debug("工具筛选走 JSON 提示分支")
structured_model = selection_request.model.bind( response = await selection_request.model.ainvoke(
response_format={"type": "json_object"}
)
response = await structured_model.ainvoke(
[ [
{ SystemMessage(
"role": "system", content=self._build_json_selection_prompt(selection_request)
"content": self._build_deepseek_selection_prompt(selection_request), ),
},
selection_request.last_user_message, selection_request.last_user_message,
] ]
) )
@@ -512,6 +486,31 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
"""从已筛选后的请求中提取最终工具名,保留原有顺序。""" """从已筛选后的请求中提取最终工具名,保留原有顺序。"""
return [tool.name for tool in request.tools if not isinstance(tool, dict)] return [tool.name for tool in request.tools if not isinstance(tool, dict)]
@staticmethod
def _count_request_tools(request: ModelRequest) -> int:
"""统计当前请求中的 LangChain 工具数量,不包含 provider 原生工具字典。"""
return len([tool for tool in request.tools if not isinstance(tool, dict)])
@classmethod
def _log_selection_attempt(cls, attempt: _ToolSelectionAttempt) -> None:
"""按工具筛选最终状态记录稳定日志。"""
tool_count = cls._count_request_tools(attempt.request)
if attempt.status == "selected":
selected_text = ", ".join(attempt.selected_tool_names) or "无有效工具"
logger.info(f"工具筛选结果: {selected_text}")
return
if attempt.status == "failed_fallback":
logger.warning(
f"工具筛选失败,将恢复使用所有工具(共 {tool_count} 个): {attempt.detail}"
)
return
if attempt.status == "skipped":
logger.info(f"工具筛选跳过: {attempt.detail}")
return
if attempt.status == "reused":
selected_text = ", ".join(attempt.selected_tool_names) or "无有效工具"
logger.info(f"工具筛选复用已有结果: {selected_text}")
@staticmethod @staticmethod
def _apply_selected_tools( def _apply_selected_tools(
request: ModelRequest[ContextT], request: ModelRequest[ContextT],
@@ -523,9 +522,6 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
这里只复用首次筛选出的客户端工具名provider-specific 的 dict 工具仍然 这里只复用首次筛选出的客户端工具名provider-specific 的 dict 工具仍然
原样保留,避免破坏 LangChain/provider 自身的工具绑定约定。 原样保留,避免破坏 LangChain/provider 自身的工具绑定约定。
""" """
if not selected_tool_names:
return request
current_tools_by_name = { current_tools_by_name = {
tool.name: tool for tool in request.tools if not isinstance(tool, dict) tool.name: tool for tool in request.tools if not isinstance(tool, dict)
} }
@@ -546,30 +542,43 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
这里单独抽成 helper便于首次筛选后缓存结果也便于测试覆盖 这里单独抽成 helper便于首次筛选后缓存结果也便于测试覆盖
“首轮筛选,后续复用”的行为。 “首轮筛选,后续复用”的行为。
""" """
return (await self._aselect_request_once_with_status(request)).request
async def _aselect_request_once_with_status(
self, request: ModelRequest[ContextT]
) -> _ToolSelectionAttempt:
"""
执行一次真实工具筛选,并携带最终状态供调用方统一记录日志。
"""
selection_request = self._prepare_selection_request(request) selection_request = self._prepare_selection_request(request)
if selection_request is None: if selection_request is None:
return request return _ToolSelectionAttempt(
request=request,
selected_tool_names=self._extract_selected_tool_names(request),
status="skipped",
detail="没有需要筛选的工具",
)
if not self._is_deepseek_compatible_model(selection_request.model): try:
captured_request: ModelRequest[ContextT] = request response = await self._aselect_tools_with_json_prompt(selection_request)
modified_request = self._process_selection_response(
async def _capture_handler( response,
updated_request: ModelRequest[ContextT], selection_request.available_tools,
) -> ModelRequest[ContextT]: selection_request.valid_tool_names,
nonlocal captured_request request,
captured_request = updated_request )
return updated_request return _ToolSelectionAttempt(
request=modified_request,
await super().awrap_model_call(request, _capture_handler) selected_tool_names=self._extract_selected_tool_names(modified_request),
return captured_request status="selected",
)
response = await self._aselect_tools_with_deepseek(selection_request) except Exception as err:
return self._process_selection_response( return _ToolSelectionAttempt(
response, request=request,
selection_request.available_tools, selected_tool_names=self._extract_selected_tool_names(request),
selection_request.valid_tool_names, status="failed_fallback",
request, detail=str(err),
) )
async def abefore_agent( # noqa async def abefore_agent( # noqa
self, self,
@@ -584,9 +593,37 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
不会为每次模型回合重复追加一笔 selector LLM 开销。 不会为每次模型回合重复追加一笔 selector LLM 开销。
""" """
if "selected_tool_names" in state: if "selected_tool_names" in state:
self._log_selection_attempt(
_ToolSelectionAttempt(
request=ModelRequest(
model=self.model,
tools=list(self.selection_tools),
messages=state["messages"],
state=state,
runtime=runtime,
),
selected_tool_names=state.get("selected_tool_names") or [],
status="reused",
)
)
return None return None
if not self.selection_tools or self.model is None: if not self.selection_tools or self.model is None:
detail = "没有可筛选工具" if not self.selection_tools else "未配置筛选模型"
self._log_selection_attempt(
_ToolSelectionAttempt(
request=ModelRequest(
model=self.model,
tools=list(self.selection_tools),
messages=state["messages"],
state=state,
runtime=runtime,
),
selected_tool_names=[],
status="skipped",
detail=detail,
)
)
return ToolSelectionStateUpdate(selected_tool_names=None) return ToolSelectionStateUpdate(selected_tool_names=None)
selection_request = ModelRequest( selection_request = ModelRequest(
@@ -596,9 +633,10 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
state=state, state=state,
runtime=runtime, runtime=runtime,
) )
modified_request = await self._aselect_request_once(selection_request) attempt = await self._aselect_request_once_with_status(selection_request)
selected_tool_names = self._extract_selected_tool_names(modified_request) self._log_selection_attempt(attempt)
return ToolSelectionStateUpdate(selected_tool_names=selected_tool_names or None) selected_tool_names = attempt.selected_tool_names
return ToolSelectionStateUpdate(selected_tool_names=selected_tool_names)
async def awrap_model_call( async def awrap_model_call(
self, self,
@@ -619,11 +657,13 @@ class ToolSelectorMiddleware(LLMToolSelectorMiddleware):
and self.selection_tools and self.selection_tools
and self.model is not None and self.model is not None
): ):
request = await self._aselect_request_once(request) attempt = await self._aselect_request_once_with_status(request)
selected_tool_names = self._extract_selected_tool_names(request) or None self._log_selection_attempt(attempt)
request = attempt.request
selected_tool_names = attempt.selected_tool_names
request.state["selected_tool_names"] = selected_tool_names # noqa request.state["selected_tool_names"] = selected_tool_names # noqa
if selected_tool_names: if selected_tool_names is not None:
request = self._apply_selected_tools(request, selected_tool_names) request = self._apply_selected_tools(request, selected_tool_names)
return await handler(request) return await handler(request)

View File

@@ -9,7 +9,7 @@ from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.search import SearchChain from app.chain.search import SearchChain
from app.db.systemconfig_oper import SystemConfigOper from app.db.systemconfig_oper import SystemConfigOper
from app.helper.sites import SitesHelper from app.helper.sites import SitesHelper # noqa
from app.log import logger from app.log import logger
from app.schemas.types import MediaType, SystemConfigKey from app.schemas.types import MediaType, SystemConfigKey
from ._torrent_search_utils import ( from ._torrent_search_utils import (

View File

@@ -173,7 +173,7 @@ class MessageChain(ChainBase):
images = CommingMessage.MessageImage.normalize_list(images) images = CommingMessage.MessageImage.normalize_list(images)
processing_status = None processing_status = None
continues_async = False processing_finish_deferred = False
try: try:
# 语音输入只用于转写为文本,不默认改变回复形式。 # 语音输入只用于转写为文本,不默认改变回复形式。
has_audio_input = bool(audio_refs) has_audio_input = bool(audio_refs)
@@ -228,7 +228,7 @@ class MessageChain(ChainBase):
text=text, text=text,
) )
continues_async = self._handle_message_core( processing_finish_deferred = self._handle_message_core(
channel=channel, channel=channel,
source=source, source=source,
userid=userid, userid=userid,
@@ -241,9 +241,9 @@ class MessageChain(ChainBase):
files=files, files=files,
has_audio_input=has_audio_input, has_audio_input=has_audio_input,
processing_status=processing_status, processing_status=processing_status,
) ) is True
finally: finally:
if continues_async is not True: if not processing_finish_deferred:
self._mark_message_processing_finished( self._mark_message_processing_finished(
channel=channel, channel=channel,
source=source, source=source,
@@ -1278,7 +1278,10 @@ class MessageChain(ChainBase):
# 将可直接输入给 LLM 的附件统一转换为 data URL # 将可直接输入给 LLM 的附件统一转换为 data URL
original_images = images original_images = images
all_files = list(files or []) all_files = list(files or [])
if images and LLMHelper.supports_image_input(): if images and LLMHelper.supports_image_input(
provider=settings.LLM_PROVIDER,
model=settings.LLM_MODEL,
):
images = self._download_attachments_to_data_urls( images = self._download_attachments_to_data_urls(
images, channel, source images, channel, source
) )

View File

@@ -2,6 +2,7 @@ import ipaddress
import threading import threading
import time import time
import uuid import uuid
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Any, Callable, Optional, Protocol from typing import Any, Callable, Optional, Protocol
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -144,8 +145,11 @@ class BrowserSessionHelper:
PRIVATE_HOST_SUFFIXES = (".localhost", ".local", ".lan", ".home", ".internal") PRIVATE_HOST_SUFFIXES = (".localhost", ".local", ".lan", ".home", ".internal")
PRIVATE_HOSTNAMES = {"localhost", "ip6-localhost", "ip6-loopback"} PRIVATE_HOSTNAMES = {"localhost", "ip6-localhost", "ip6-loopback"}
REF_ATTRIBUTE = "data-moviepilot-agent-ref" REF_ATTRIBUTE = "data-moviepilot-agent-ref"
SESSION_WORKER_NAME_PREFIX = "browser-session"
_sessions: dict[str, _BrowserSessionState] = {} _sessions: dict[str, _BrowserSessionState] = {}
_session_executors: dict[str, ThreadPoolExecutor] = {}
_session_thread_ids: dict[str, int] = {}
_sessions_lock = threading.RLock() _sessions_lock = threading.RLock()
def __init__(self, headless: bool = True, viewport: Optional[dict[str, int]] = None): def __init__(self, headless: bool = True, viewport: Optional[dict[str, int]] = None):
@@ -211,7 +215,9 @@ class BrowserSessionHelper:
关闭所有 Agent 浏览器会话。 关闭所有 Agent 浏览器会话。
""" """
with cls._sessions_lock: with cls._sessions_lock:
session_keys = list(cls._sessions.keys()) session_keys = list(
set(cls._sessions.keys()) | set(cls._session_executors.keys())
)
for session_key in session_keys: for session_key in session_keys:
cls.close_session(session_key) cls.close_session(session_key)
@@ -223,12 +229,27 @@ class BrowserSessionHelper:
:param session_key: 会话标识 :param session_key: 会话标识
:return: 找到并关闭会话时返回 True :return: 找到并关闭会话时返回 True
""" """
with cls._sessions_lock: if cls._is_current_session_thread(session_key):
session = cls._sessions.pop(session_key, None) closed = cls._close_session_in_thread(session_key)
if not session: cls._shutdown_session_executor(session_key, wait=False)
return False return closed
cls._close_session_state(session)
return True executor = cls._get_existing_session_executor(session_key)
if executor:
future = executor.submit(
cls._run_session_task,
session_key,
cls._close_session_in_thread,
session_key,
)
try:
return future.result()
finally:
cls._shutdown_session_executor(session_key)
closed = cls._close_session_in_thread(session_key)
cls._shutdown_session_executor(session_key)
return closed
def with_session( def with_session(
self, self,
@@ -249,6 +270,34 @@ class BrowserSessionHelper:
:return: 回调函数返回值 :return: 回调函数返回值
""" """
self._prune_sessions() self._prune_sessions()
return self._run_in_session_thread(
session_key,
self._with_session_in_thread,
session_key,
callback,
user_agent=user_agent,
cookies=cookies,
timeout=timeout,
)
def _with_session_in_thread(
self,
session_key: str,
callback: Callable[[_BrowserSessionState], Any],
user_agent: Optional[str] = None,
cookies: Optional[str] = None,
timeout: Optional[int] = 30,
) -> Any:
"""
在会话专属线程内获取浏览器会话并执行回调。
:param session_key: 会话标识
:param callback: 使用浏览器会话执行操作的回调函数
:param user_agent: 新建会话时使用的 User-Agent
:param cookies: 本次操作要注入的 Cookie 请求头
:param timeout: 默认操作超时时间,单位秒
:return: 回调函数返回值
"""
session = self._get_or_create_session( session = self._get_or_create_session(
session_key=session_key, session_key=session_key,
user_agent=user_agent, user_agent=user_agent,
@@ -263,6 +312,164 @@ class BrowserSessionHelper:
session.active_page.set_extra_http_headers({"cookie": cookies}) session.active_page.set_extra_http_headers({"cookie": cookies})
return callback(session) return callback(session)
@classmethod
def _run_in_session_thread(
cls,
session_key: str,
callback: Callable[..., Any],
*args: Any,
**kwargs: Any,
) -> Any:
"""
将浏览器同步 API 调用投递到当前会话固定的单线程执行器。
:param session_key: 会话标识
:param callback: 需要在会话线程中运行的回调
:param args: 回调位置参数
:param kwargs: 回调关键字参数
:return: 回调返回值
"""
if cls._is_current_session_thread(session_key):
return callback(*args, **kwargs)
for _ in range(2):
executor = cls._get_session_executor(session_key)
try:
future = executor.submit(
cls._run_session_task,
session_key,
callback,
*args,
**kwargs,
)
except RuntimeError:
cls._discard_session_executor(session_key, executor)
continue
try:
return future.result()
except Exception:
with cls._sessions_lock:
has_session = session_key in cls._sessions
if not has_session:
cls._shutdown_session_executor(session_key)
raise
raise RuntimeError("浏览器会话线程已关闭")
@classmethod
def _run_session_task(
cls,
session_key: str,
callback: Callable[..., Any],
*args: Any,
**kwargs: Any,
) -> Any:
"""
记录当前会话工作线程后执行实际任务。
:param session_key: 会话标识
:param callback: 需要执行的回调
:param args: 回调位置参数
:param kwargs: 回调关键字参数
:return: 回调返回值
"""
with cls._sessions_lock:
cls._session_thread_ids[session_key] = threading.get_ident()
return callback(*args, **kwargs)
@classmethod
def _is_current_session_thread(cls, session_key: str) -> bool:
"""
判断当前代码是否已运行在指定会话的固定线程内。
:param session_key: 会话标识
:return: 当前线程是会话工作线程时返回 True
"""
with cls._sessions_lock:
thread_id = cls._session_thread_ids.get(session_key)
return thread_id == threading.get_ident()
@classmethod
def _get_session_executor(cls, session_key: str) -> ThreadPoolExecutor:
"""
获取或创建指定会话的单线程执行器。
:param session_key: 会话标识
:return: 会话专属执行器
"""
with cls._sessions_lock:
executor = cls._session_executors.get(session_key)
if executor:
return executor
executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix=cls.SESSION_WORKER_NAME_PREFIX,
)
cls._session_executors[session_key] = executor
return executor
@classmethod
def _get_existing_session_executor(
cls, session_key: str
) -> Optional[ThreadPoolExecutor]:
"""
获取指定会话已存在的执行器。
:param session_key: 会话标识
:return: 已存在的执行器,不存在时返回 None
"""
with cls._sessions_lock:
return cls._session_executors.get(session_key)
@classmethod
def _discard_session_executor(
cls,
session_key: str,
executor: ThreadPoolExecutor,
) -> None:
"""
丢弃已经关闭的会话执行器。
:param session_key: 会话标识
:param executor: 需要从缓存中移除的执行器
"""
with cls._sessions_lock:
if cls._session_executors.get(session_key) is executor:
cls._session_executors.pop(session_key, None)
@classmethod
def _shutdown_session_executor(
cls,
session_key: str,
wait: bool = True,
) -> None:
"""
关闭并移除指定会话的执行器。
:param session_key: 会话标识
:param wait: 是否等待工作线程退出
"""
with cls._sessions_lock:
executor = cls._session_executors.pop(session_key, None)
cls._session_thread_ids.pop(session_key, None)
if executor:
executor.shutdown(wait=wait, cancel_futures=True)
@classmethod
def _close_session_in_thread(cls, session_key: str) -> bool:
"""
在会话固定线程内关闭并移除浏览器会话。
:param session_key: 会话标识
:return: 找到并关闭会话时返回 True
"""
with cls._sessions_lock:
session = cls._sessions.pop(session_key, None)
if not session:
return False
cls._close_session_state(session)
return True
def open_tab( def open_tab(
self, self,
session: _BrowserSessionState, session: _BrowserSessionState,
@@ -479,24 +686,29 @@ class BrowserSessionHelper:
if session: if session:
return session return session
context = self._launch_context( context = self._launch_context(
headless=self.headless, headless=self.headless,
user_agent=user_agent, user_agent=user_agent,
viewport=self.viewport, viewport=self.viewport,
) )
page = context.new_page() page = context.new_page()
if cookies: if cookies:
page.set_extra_http_headers({"cookie": cookies}) page.set_extra_http_headers({"cookie": cookies})
session = _BrowserSessionState( session = _BrowserSessionState(
session_key=session_key, session_key=session_key,
context=context, context=context,
pages=[page], pages=[page],
user_agent=user_agent, user_agent=user_agent,
cookies=cookies, cookies=cookies,
) )
with self._sessions_lock:
existing_session = self._sessions.get(session_key)
if existing_session:
self._close_session_state(session)
return existing_session
self._sessions[session_key] = session self._sessions[session_key] = session
self._enforce_session_limit() self._enforce_session_limit(protect_session_key=session_key)
return session return session
@classmethod @classmethod
def _prune_sessions(cls) -> None: def _prune_sessions(cls) -> None:
@@ -511,14 +723,29 @@ class BrowserSessionHelper:
cls.close_session(session_key) cls.close_session(session_key)
@classmethod @classmethod
def _enforce_session_limit(cls) -> None: def _enforce_session_limit(cls, protect_session_key: Optional[str] = None) -> None:
while len(cls._sessions) > cls.MAX_SESSIONS: """
oldest_key = min( 清理超过数量上限的旧会话。
cls._sessions,
key=lambda key: cls._sessions[key].last_used_at, :param protect_session_key: 本次刚创建、需要优先保留的会话标识
) """
session = cls._sessions.pop(oldest_key) while True:
cls._close_session_state(session) with cls._sessions_lock:
if len(cls._sessions) <= cls.MAX_SESSIONS:
return
candidate_keys = [
session_key
for session_key in cls._sessions
if session_key != protect_session_key
]
if not candidate_keys:
return
oldest_key = min(
candidate_keys,
key=lambda key: cls._sessions[key].last_used_at,
)
if not cls.close_session(oldest_key):
return
@staticmethod @staticmethod
def _close_session_state(session: _BrowserSessionState) -> None: def _close_session_state(session: _BrowserSessionState) -> None:

View File

@@ -15,8 +15,15 @@ from telebot.types import (
InlineKeyboardButton, InlineKeyboardButton,
InputMediaPhoto, InputMediaPhoto,
) )
from telegramify_markdown import entities_to_markdownv2, standardize, telegramify # noqa from telegramify_markdown import standardize, telegramify # noqa
from telegramify_markdown.content import ContentTypes, File, Photo, Text try:
from telegramify_markdown import entities_to_markdownv2 # noqa
except ImportError:
entities_to_markdownv2 = None
try:
from telegramify_markdown.content import ContentTypes, File, Photo, Text
except ImportError:
from telegramify_markdown.type import ContentTypes, File, Photo, Text
from app.core.config import settings from app.core.config import settings
from app.core.context import MediaInfo, Context from app.core.context import MediaInfo, Context
@@ -255,14 +262,22 @@ class Telegram:
@staticmethod @staticmethod
def _telegramify_item_text(item: Text) -> str: def _telegramify_item_text(item: Text) -> str:
"""将 telegramify 文本片段转换为 Telegram MarkdownV2 字符串。""" """将 telegramify 文本片段转换为 Telegram MarkdownV2 字符串。"""
return entities_to_markdownv2(item.text, item.entities) if hasattr(item, "content"):
return item.content
if entities_to_markdownv2:
return entities_to_markdownv2(item.text, item.entities)
return standardize(item.text)
@staticmethod @staticmethod
def _telegramify_item_caption(item: Text | File | Photo) -> str: def _telegramify_item_caption(item: Text | File | Photo) -> str:
"""将 telegramify 文本或媒体片段转换为 Telegram MarkdownV2 caption。""" """将 telegramify 文本或媒体片段转换为 Telegram MarkdownV2 caption。"""
if isinstance(item, Text): if isinstance(item, Text):
return Telegram._telegramify_item_text(item) return Telegram._telegramify_item_text(item)
return entities_to_markdownv2(item.caption_text, item.caption_entities) if hasattr(item, "caption"):
return item.caption
if entities_to_markdownv2:
return entities_to_markdownv2(item.caption_text, item.caption_entities)
return standardize(item.caption_text)
@staticmethod @staticmethod
def _normalize_parse_mode(parse_mode: Optional[str] = None) -> str: def _normalize_parse_mode(parse_mode: Optional[str] = None) -> str:

View File

@@ -86,7 +86,6 @@ RUN python3 -m venv ${VENV_PATH} \
&& ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip3.12 \ && ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip3.12 \
&& ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip-compile \ && ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip-compile \
&& ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip-sync \ && ln -sf /usr/local/bin/uv-pip-compat ${VENV_PATH}/bin/pip-sync \
&& pip install "Cython~=3.1.2" \
&& pip-compile requirements.in -o requirements.txt \ && pip-compile requirements.in -o requirements.txt \
&& pip install -r requirements.txt && pip install -r requirements.txt

View File

@@ -1,6 +1,6 @@
## 开发环境设置指南 ## 开发环境设置指南
本文档旨在帮助开发者快速设置开发环境,并介绍如何使用 `pip-tools` 管理依赖项和使用 `safety` 进行安全检查 本文档旨在帮助开发者快速设置开发环境,并说明主程序、开发测试、构建工具和插件依赖的管理边界
### 环境准备 ### 环境准备
@@ -33,57 +33,38 @@ Rust 加速扩展通过 `moviepilot-rust` PyPI 包安装,主项目本地开发
虚拟环境确保项目的依赖项与系统全局环境隔离,防止冲突。 虚拟环境确保项目的依赖项与系统全局环境隔离,防止冲突。
### 2. 使用 pip-tools 管理依赖项 ### 2. 依赖分层与安装
我们使用 `pip-tools` 来管理项目的 Python 依赖项,这有助于保持 `requirements.txt` 文件的一致性和更新性。 主程序依赖按使用场景分层,避免运行时镜像携带只在开发、测试或构建时需要的工具:
#### 安装 pip-tools | 文件 | 用途 | 典型安装场景 |
| --- | --- | --- |
| `requirements.in` | 主程序运行时依赖。只放启动、后台任务、插件运行框架和内置功能在生产环境需要导入的包。 | Docker 镜像、CLI 本地运行、运行时依赖自愈。 |
| `requirements-dev.in` | 开发、测试、静态检查和源码构建辅助依赖。 | CI 单测、本地跑测、Pylint、显式源码构建。 |
| `requirements.txt` | 兼容入口,默认只委托到 `requirements.in`。它不是跨平台完整锁文件,不应在本地开发机上直接维护一份平台相关锁定结果。 | 旧脚本、Docker 运行时恢复、CLI 安装入口。 |
首先,您需要安装 `pip-tools` 以便管理依赖 运行主程序只需要安装运行时依赖:
```bash ```bash
pip install pip-tools pip install -r requirements.txt
``` ```
#### 管理依赖项 开发、测试、静态检查或执行源码编译时安装开发依赖入口:
1. **修改 `requirements.in` 文件** ```bash
pip install -r requirements-dev.in
```
`requirements.in` 文件是项目依赖项的源文件。要添加或更新依赖项,请直接编辑该文件。 ### 3. 修改主程序依赖
2. **更新特定的依赖项** 新增或升级依赖时,先确认依赖属于哪个层级
如果你只想更新 `requirements.in` 中的某个特定依赖包,而不影响其他依赖项,可以使用 `--upgrade-package` 选项,指定要升级的包: 1. **运行时依赖**:被 `app/` 生产代码直接导入,或是生产功能、后台任务、插件框架启动必需,写入 `requirements.in`。
2. **开发 / 测试 / 静态检查 / 构建依赖**只用于单测、覆盖率、lint 辅助、源码构建等,不应进入生产运行时,写入 `requirements-dev.in`。
3. **工具依赖**`pip-tools`、`uv`、`safety` 这类安装或审计工具不属于主程序运行依赖,按脚本或 CI 场景显式安装。
4. **插件依赖**:由插件声明并在插件安装阶段处理,不直接并入主程序 `requirements.in`。
```bash ### 4. 准备资源与插件目录
pip-compile --upgrade-package <package-name> requirements.in
```
例如,要只升级 `requests` 这个包,你可以运行以下命令:
```bash
pip-compile --upgrade-package requests requirements.in
```
3. **全量更新依赖项**
如果你想更新 `requirements.in` 中的所有依赖包,运行以下命令生成或更新 `requirements.txt` 文件:
```bash
pip-compile requirements.in
```
这将根据 `requirements.in` 中指定的依赖项生成一个锁定的 `requirements.txt` 文件。
4. **安装依赖项**
使用以下命令安装 `requirements.txt` 文件中列出的依赖项:
```bash
pip install -r requirements.txt
```
### 3. 准备资源与插件目录
本地源码开发时,主程序需要读取资源文件和插件源码。相关文件需要放到主程序实际加载的目录下: 本地源码开发时,主程序需要读取资源文件和插件源码。相关文件需要放到主程序实际加载的目录下:
@@ -92,9 +73,9 @@ pip install pip-tools
如果资源文件没有放到 `app/helper/`,站点索引、规则和内置资源相关能力可能无法按本地开发预期工作;如果插件没有放到 `app/plugins/`,主程序也不会在本地运行时发现该插件。 如果资源文件没有放到 `app/helper/`,站点索引、规则和内置资源相关能力可能无法按本地开发预期工作;如果插件没有放到 `app/plugins/`,主程序也不会在本地运行时发现该插件。
### 4. 运行安全检查 ### 5. 运行安全检查
我们使用 `safety` 工具检查依赖项中是否存在已知安全漏洞。请确保在每次更新依赖项后都运行安全检查,以确保项目的安全性 我们使用 `safety` 工具检查依赖项中是否存在已知安全漏洞。更新运行时依赖后,应至少检查运行时入口;更新开发测试依赖时,也应覆盖开发入口
#### 安装 safety #### 安装 safety
@@ -106,7 +87,7 @@ pip install safety
#### 执行安全检查 #### 执行安全检查
运行以下命令检查 `requirements.txt` 文件中列出的依赖项是否存在安全漏洞 运行以下命令检查运行时入口
```bash ```bash
safety check -r requirements.txt --policy-file=safety.policy.yml > safety_report.txt safety check -r requirements.txt --policy-file=safety.policy.yml > safety_report.txt
@@ -114,11 +95,11 @@ safety check -r requirements.txt --policy-file=safety.policy.yml > safety_report
这将生成一个名为 `safety_report.txt` 的报告文件,您可以查看其中的漏洞报告并进行相应处理。 这将生成一个名为 `safety_report.txt` 的报告文件,您可以查看其中的漏洞报告并进行相应处理。
### 5. 提交代码前的检查 ### 6. 提交代码前的检查
在提交代码之前,请确保完成以下步骤: 在提交代码之前,请确保完成以下步骤:
1. **确依赖项已更新**:如果您对 `requirements.in` 进行了更改,请重新生成 `requirements.txt` 并安装依赖 1. **确依赖分层正确**:运行时包进入 `requirements.in`;测试、覆盖率、静态检查和构建辅助进入 `requirements-dev.in`;插件依赖不并入主程序运行时依赖。
2. **运行安全检查**:确保 `safety` 检查通过,没有新的安全漏洞。 2. **运行安全检查**:确保 `safety` 检查通过,没有新的安全漏洞。
@@ -128,9 +109,10 @@ safety check -r requirements.txt --policy-file=safety.policy.yml > safety_report
pytest pytest
``` ```
### 6. 参考资源 ### 7. 参考资源
- [pip-tools 官方文档](https://github.com/jazzband/pip-tools) - [pip-tools 官方文档](https://github.com/jazzband/pip-tools)
- [uv 官方文档](https://docs.astral.sh/uv/)
- [safety 官方文档](https://pyup.io/safety/) - [safety 官方文档](https://pyup.io/safety/)
- [MoviePilot-Resources](https://github.com/jxxghp/MoviePilot-Resources) - [MoviePilot-Resources](https://github.com/jxxghp/MoviePilot-Resources)
- [MoviePilot-Plugins](https://github.com/jxxghp/MoviePilot-Plugins) - [MoviePilot-Plugins](https://github.com/jxxghp/MoviePilot-Plugins)

View File

@@ -104,10 +104,11 @@
| Item | Detail | | Item | Detail |
|---|---| |---|---|
| Source file | `requirements.in`edit this to add or upgrade dependencies | | Runtime source | `requirements.in`production/runtime dependencies only |
| Lock file | `requirements.txt` — generated by `pip-compile`; never edit manually | | Dev/test/lint/build source | `requirements-dev.in` — includes runtime plus pytest, coverage tooling, pylint, and build support |
| Tool | `pip-tools` (`pip-compile`, `pip-sync`) | | Compatibility entry | `requirements.txt` — delegates to `requirements.in`; not a committed cross-platform lock |
| Install | `pip install -r requirements.txt` | | Runtime install | `pip install -r requirements.txt` |
| Dev/test/lint/build install | `pip install -r requirements-dev.in` |
--- ---

View File

@@ -12,11 +12,11 @@ python3 -m venv venv
source venv/bin/activate # macOS / Linux source venv/bin/activate # macOS / Linux
.\venv\Scripts\activate # Windows .\venv\Scripts\activate # Windows
# Install pip-tools # Install runtime dependencies
pip install pip-tools
# Install project dependencies
pip install -r requirements.txt pip install -r requirements.txt
# Install development/test/lint/build dependencies
pip install -r requirements-dev.in
``` ```
--- ---
@@ -24,20 +24,17 @@ pip install -r requirements.txt
## Dependency Management ## Dependency Management
```bash ```bash
# Compile requirements.txt from requirements.in (full recompile) # Install runtime dependencies
pip-compile requirements.in
# Upgrade a single package without touching others
pip-compile --upgrade-package <package-name> requirements.in
# Install from the generated lock file
pip install -r requirements.txt pip install -r requirements.txt
# Install test/lint/build dependencies
pip install -r requirements-dev.in
``` ```
**Rules:** **Rules:**
- Always edit `requirements.in` to add or change dependencies. - Runtime dependencies belong in `requirements.in`.
- Never edit `requirements.txt` manually — it is a generated lock file. - Test, coverage, lint, and explicit build tooling belong in `requirements-dev.in`.
- After any change to `requirements.in`, re-run `pip-compile requirements.in` and commit both files together. - `requirements.txt` is a compatibility entry that delegates to `requirements.in`; do not replace it with a local cross-platform lock file.
--- ---
@@ -83,7 +80,7 @@ pylint app/chain/download.py
## Security Scan ## Security Scan
```bash ```bash
# Run safety check against the lock file # Run safety check against the runtime compatibility entry
safety check -r requirements.txt --policy-file=safety.policy.yml safety check -r requirements.txt --policy-file=safety.policy.yml
# Save report to file # Save report to file
@@ -91,7 +88,7 @@ safety check -r requirements.txt --policy-file=safety.policy.yml > safety_report
``` ```
**Rules:** **Rules:**
- Run after every change to `requirements.txt`. - Run after runtime dependency changes; include `requirements-dev.in` when development/test/lint/build dependencies change.
- No new high-severity vulnerabilities may be introduced. - No new high-severity vulnerabilities may be introduced.
--- ---

View File

@@ -111,7 +111,7 @@ except:
## What Not To Do ## What Not To Do
- Do not introduce new third-party libraries without updating `requirements.in` and running `pip-compile`. - Do not introduce new third-party libraries without placing them in the correct dependency entry: runtime packages in `requirements.in`, test/lint/build tooling in `requirements-dev.in`.
- Do not use `requests` or `httpx` directly for external HTTP calls — use `RequestUtils` from `app/utils/http.py`. - Do not use `requests` or `httpx` directly for external HTTP calls — use `RequestUtils` from `app/utils/http.py`.
- Do not issue raw SQLAlchemy queries from chains, modules, or endpoints — use the `*_oper.py` classes. - Do not issue raw SQLAlchemy queries from chains, modules, or endpoints — use the `*_oper.py` classes.
- Do not add TODO or FIXME without context. Only keep one if it is genuinely deferred and cannot be addressed in the current task. - Do not add TODO or FIXME without context. Only keep one if it is genuinely deferred and cannot be addressed in the current task.

View File

@@ -57,7 +57,7 @@ pylint app/
safety check -r requirements.txt --policy-file=safety.policy.yml safety check -r requirements.txt --policy-file=safety.policy.yml
``` ```
- Run after every change to `requirements.txt`. - Run after runtime dependency changes; scan the development dependency entry as well when `requirements-dev.in` changes.
- No new high-severity vulnerabilities may be introduced. - No new high-severity vulnerabilities may be introduced.
- If a vulnerability cannot be patched immediately, document it explicitly in the PR description. - If a vulnerability cannot be patched immediately, document it explicitly in the PR description.
@@ -129,7 +129,7 @@ Before marking any task as complete:
- [ ] Related pytest tests pass - [ ] Related pytest tests pass
- [ ] No new pylint error-level issues in `pylint app/` - [ ] No new pylint error-level issues in `pylint app/`
- [ ] If dependencies changed: `pip-compile requirements.in` was run and `safety check` passes - [ ] If dependencies changed: the package is in the correct runtime or dev dependency entry, and `safety check` passes for the affected entry
- [ ] If CLI behavior changed: `docs/cli.md` and related tests are updated - [ ] If CLI behavior changed: `docs/cli.md` and related tests are updated
- [ ] If MCP/API behavior changed: `docs/mcp-api.md` and related skill files are updated - [ ] If MCP/API behavior changed: `docs/mcp-api.md` and related skill files are updated
- [ ] If database schema changed: a new Alembic migration exists under `database/versions/` - [ ] If database schema changed: a new Alembic migration exists under `database/versions/`

View File

@@ -100,11 +100,10 @@ ci: improve docker build cache
When updating a dependency: When updating a dependency:
1. Update `requirements.in` with the new version constraint. 1. Decide the dependency layer: runtime packages go to `requirements.in`; test, coverage, lint, and explicit build tooling go to `requirements-dev.in`.
2. Run `pip-compile requirements.in` to regenerate `requirements.txt`. 2. Keep `requirements.txt` as the compatibility entry that delegates to `requirements.in`; do not commit a locally generated cross-platform lock file.
3. Run `safety check -r requirements.txt --policy-file=safety.policy.yml`. 3. Run `safety check -r requirements.txt --policy-file=safety.policy.yml`; include the dev dependency entry when `requirements-dev.in` changed.
4. Run the full test suite: `pytest`. 4. Run the full test suite: `pytest`.
5. Commit both `requirements.in` and `requirements.txt` together.
--- ---

View File

@@ -15,7 +15,7 @@ python tests/run.py # 等价于 pytest 全量(参数透
- 不再使用 `python -m unittest discover`:它不导入 `tests` 包、收不到纯函数用例,且绕过 `conftest.py` 的隔离。 - 不再使用 `python -m unittest discover`:它不导入 `tests` 包、收不到纯函数用例,且绕过 `conftest.py` 的隔离。
- 不再依赖 `python tests/test_xxx.py` 直跑:所有 `if __name__ == "__main__": unittest.main()` 尾巴已移除。 - 不再依赖 `python tests/test_xxx.py` 直跑:所有 `if __name__ == "__main__": unittest.main()` 尾巴已移除。
- **复现 CI 用干净环境**:建议用一个仅 `pip install -r requirements.in pytest` 的虚拟环境运行,避免本地额外包或编译产物掩盖问题。 - **复现 CI 用干净环境**:建议用一个仅 `pip install -r requirements-dev.in` 的虚拟环境运行,避免本地额外包或编译产物掩盖问题。
## 隔离模型(`tests/conftest.py` ## 隔离模型(`tests/conftest.py`
@@ -133,4 +133,4 @@ def test_recognize_prefers_explicit_id(sample_meta, monkeypatch):
- **门禁**`.github/workflows/test.yml` 在指向 `v2` 的 `pull_request` / `push` 及手动触发时,用 `python tests/run.py` 跑全量单测。 - **门禁**`.github/workflows/test.yml` 在指向 `v2` 的 `pull_request` / `push` 及手动触发时,用 `python tests/run.py` 跑全量单测。
- **PR**`python tests/run.py` 确认全绿、且 socket 探针零真实出站,避免把红的改动推上去空耗门禁。 - **PR**`python tests/run.py` 确认全绿、且 socket 探针零真实出站,避免把红的改动推上去空耗门禁。
- 复现 CI 用仅安装 `requirements.in` 的干净环境(含 pytest 与可选扩展),保证可选扩展、动态模块的存在性与 CI 一致 - 复现 CI 用仅安装 `requirements-dev.in` 的干净环境`requirements.in` 只承载运行时依赖pytest 与覆盖率插件由开发依赖入口提供

7
requirements-dev.in Normal file
View File

@@ -0,0 +1,7 @@
-r requirements.in
Cython~=3.2.5
pylint~=4.0.6
pytest~=9.0.3
pytest-cov~=7.1.0
pytest-timeout~=2.4.0

View File

@@ -1,4 +1,3 @@
Cython~=3.2.5
moviepilot-rust~=0.1.10 moviepilot-rust~=0.1.10
pydantic>=2.13.4,<3.0.0 pydantic>=2.13.4,<3.0.0
pydantic-settings>=2.14.1,<3.0.0 pydantic-settings>=2.14.1,<3.0.0
@@ -38,8 +37,8 @@ beautifulsoup4~=4.15.0
pillow~=12.2.0 pillow~=12.2.0
pillow-avif-plugin~=1.5.5 pillow-avif-plugin~=1.5.5
pyTelegramBotAPI~=4.34.0 pyTelegramBotAPI~=4.34.0
telegramify-markdown~=1.1.5 telegramify-markdown~=1.2.0
cloakbrowser~=0.3.31 cloakbrowser~=0.4.0
torrentool~=1.2.0 torrentool~=1.2.0
fast-bencode~=1.1.8 fast-bencode~=1.1.8
slack-bolt~=1.28.0 slack-bolt~=1.28.0
@@ -91,6 +90,3 @@ google-genai~=2.8.0
ddgs~=9.14.4 ddgs~=9.14.4
websocket-client~=1.9.0 websocket-client~=1.9.0
lark-oapi~=1.6.8 lark-oapi~=1.6.8
pytest~=9.0.3
pytest-cov~=7.1.0
pytest-timeout~=2.4.0

View File

@@ -0,0 +1,101 @@
from unittest.mock import AsyncMock, patch
from app.agent import MoviePilotAgent
from app.agent.llm import AgentCapabilityManager, LLMHelper
from app.chain.message import MessageChain
from app.core.config import settings
from app.schemas.types import MessageChannel
def test_llm_supports_image_input_uses_model_catalog_text_only(monkeypatch):
"""内置目录明确为纯文本模型时,应自动关闭图片输入。"""
monkeypatch.setattr(settings, "LLM_SUPPORT_IMAGE_INPUT", True)
assert not LLMHelper.supports_image_input(
provider="minimax",
model="MiniMax-M2.7",
)
def test_llm_supports_image_input_keeps_known_vision_model(monkeypatch):
"""内置目录明确为视觉模型时,应允许图片输入。"""
monkeypatch.setattr(settings, "LLM_SUPPORT_IMAGE_INPUT", True)
assert LLMHelper.supports_image_input(
provider="zhipuai",
model="glm-5v-turbo",
)
def test_llm_supports_image_input_keeps_unknown_model_override(monkeypatch):
"""未知自定义模型保持用户开关语义,避免误伤私有视觉模型。"""
monkeypatch.setattr(settings, "LLM_SUPPORT_IMAGE_INPUT", True)
assert LLMHelper.supports_image_input(
provider="custom-provider",
model="custom-vlm-model",
)
def test_agent_capability_manager_delegates_image_support():
"""Agent 能力管理器应复用统一的模型图片能力判断。"""
with patch.object(LLMHelper, "supports_image_input", return_value=False) as supports:
assert not AgentCapabilityManager.supports_image_input()
supports.assert_called_once_with()
def test_handle_ai_message_routes_text_only_model_images_to_files(monkeypatch):
"""纯文本模型收到图片消息时,应降级为文件附件而非 image_url 内容块。"""
chain = MessageChain()
monkeypatch.setattr(settings, "AI_AGENT_ENABLE", True)
monkeypatch.setattr(settings, "LLM_SUPPORT_IMAGE_INPUT", True)
monkeypatch.setattr(settings, "LLM_PROVIDER", "minimax")
monkeypatch.setattr(settings, "LLM_MODEL", "MiniMax-M2.7")
with patch.object(
chain, "_get_or_create_session_id", return_value="session-1"
), patch.object(
chain, "_download_attachments_to_data_urls"
) as download_images, patch.object(
chain,
"_prepare_agent_files",
return_value=[
{
"name": "image_1.jpg",
"mime_type": "image/jpeg",
"local_path": "/tmp/image_1.jpg",
"status": "ready",
}
],
) as prepare_files, patch(
"app.chain.message.agent_manager.process_message", new_callable=AsyncMock
) as process_message, patch(
"app.chain.message.asyncio.run_coroutine_threadsafe",
side_effect=lambda coro, _loop: coro.close(),
):
chain._handle_ai_message(
text="/ai 帮我看看这张图",
channel=MessageChannel.Telegram,
source="telegram-test",
userid="10001",
username="tester",
images=["tg://file_id/image-1"],
)
download_images.assert_not_called()
prepare_files.assert_called_once()
assert prepare_files.call_args.kwargs["files"][0].ref == "tg://file_id/image-1"
assert process_message.call_args.kwargs["images"] is None
assert process_message.call_args.kwargs["files"][0]["local_path"] == "/tmp/image_1.jpg"
def test_unsupported_image_error_recognizes_vlm_text_only_message():
"""兼容端点返回 not a VLM 时,应识别为图片输入能力错误。"""
error = Exception(
"Error code: 400 - {'code': 20041, 'message': "
"'The model is not a VLM (Vision Language Model). "
"Please use text-only prompts.'}"
)
assert MoviePilotAgent._is_unsupported_image_input_error(error)

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,8 @@
from __future__ import annotations from __future__ import annotations
import json import json
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Optional from typing import Optional
from unittest.mock import patch from unittest.mock import patch
@@ -45,6 +47,7 @@ class _FakePage:
self.clicks = [] self.clicks = []
self.fills = [] self.fills = []
self.selects = [] self.selects = []
self.close_thread_id = None
def set_extra_http_headers(self, headers: dict[str, str]) -> None: def set_extra_http_headers(self, headers: dict[str, str]) -> None:
"""记录额外请求头。""" """记录额外请求头。"""
@@ -124,6 +127,7 @@ class _FakePage:
def close(self) -> None: def close(self) -> None:
"""记录页面关闭状态。""" """记录页面关闭状态。"""
self.close_thread_id = threading.get_ident()
self.closed = True self.closed = True
@@ -133,6 +137,7 @@ class _FakeContext:
def __init__(self, pages: Optional[list[_FakePage]] = None) -> None: def __init__(self, pages: Optional[list[_FakePage]] = None) -> None:
self.pages = pages or [_FakePage()] self.pages = pages or [_FakePage()]
self.closed = False self.closed = False
self.close_thread_id = None
def new_page(self) -> _FakePage: def new_page(self) -> _FakePage:
"""返回或创建模拟页面。""" """返回或创建模拟页面。"""
@@ -146,6 +151,7 @@ class _FakeContext:
def close(self) -> None: def close(self) -> None:
"""记录上下文关闭状态。""" """记录上下文关闭状态。"""
self.close_thread_id = threading.get_ident()
self.closed = True self.closed = True
@@ -250,6 +256,55 @@ def test_browser_session_helper_reuses_page_within_session():
assert not context.closed assert not context.closed
def test_browser_session_helper_runs_same_session_on_one_worker_thread():
"""同一 session_key 的浏览器操作应固定在同一个工作线程。"""
page = _FakePage()
context = _FakeContext([page])
helper = BrowserSessionHelper()
caller_thread_ids = set()
session_thread_ids = []
barrier = threading.Barrier(2)
def _run_from_caller_thread() -> int:
"""从外部调用线程进入同一个浏览器会话。"""
caller_thread_ids.add(threading.get_ident())
barrier.wait(timeout=1)
return helper.with_session(
"session-1",
lambda _session: threading.get_ident(),
)
with patch.object(BrowserSessionHelper, "_launch_context", return_value=context):
with ThreadPoolExecutor(max_workers=2) as executor:
futures = [
executor.submit(_run_from_caller_thread),
executor.submit(_run_from_caller_thread),
]
session_thread_ids = [future.result(timeout=1) for future in futures]
assert len(caller_thread_ids) == 2
assert len(set(session_thread_ids)) == 1
assert session_thread_ids[0] not in caller_thread_ids
def test_browser_session_helper_closes_session_on_worker_thread():
"""关闭会话时应在创建浏览器对象的工作线程内释放资源。"""
page = _FakePage()
context = _FakeContext([page])
helper = BrowserSessionHelper()
with patch.object(BrowserSessionHelper, "_launch_context", return_value=context):
session_thread_id = helper.with_session(
"session-1",
lambda _session: threading.get_ident(),
)
closed = BrowserSessionHelper.close_session("session-1")
assert closed is True
assert page.close_thread_id == session_thread_id
assert context.close_thread_id == session_thread_id
def test_browse_webpage_returns_snapshot_with_refs_after_goto(): def test_browse_webpage_returns_snapshot_with_refs_after_goto():
"""goto 后应返回包含可交互元素 ref 的页面快照。""" """goto 后应返回包含可交互元素 ref 的页面快照。"""
page = _FakePage() page = _FakePage()

View File

@@ -589,6 +589,59 @@ class LlmHelperTestCallTest(unittest.TestCase):
) )
self.assertEqual(llm_calls[0].get("default_headers"), {"X-Test": "1"}) self.assertEqual(llm_calls[0].get("default_headers"), {"X-Test": "1"})
def test_get_llm_attaches_runtime_metadata(self):
"""LLM 实例应带上内部 runtime 元数据,供 Agent 中间件判断兼容分支。"""
class _FakeProviderManager:
async def resolve_runtime(self, **kwargs):
return {
"provider_id": kwargs["provider_id"],
"runtime": "anthropic_compatible",
"model_id": kwargs["model"],
"api_key": kwargs["api_key"],
"base_url": kwargs["base_url"],
"default_headers": None,
"use_responses_api": None,
"model_record": None,
"model_metadata": None,
}
class _FakeChatAnthropic:
def __init__(self, **kwargs):
self.model = kwargs["model"]
self.profile = None
provider_module = ModuleType("app.agent.llm.provider")
provider_module.LLMProviderManager = _FakeProviderManager
anthropic_module = ModuleType("langchain_anthropic")
anthropic_module.ChatAnthropic = _FakeChatAnthropic
with patch.dict(
sys.modules,
{
"app.agent.llm.provider": provider_module,
"langchain_anthropic": anthropic_module,
},
):
model = asyncio.run(
llm_module.LLMHelper.get_llm(
provider="minimax",
model="MiniMax-M2.7",
api_key="sk-test",
base_url="https://api.minimaxi.com/anthropic/v1",
)
)
self.assertEqual(
getattr(model, "_moviepilot_llm_runtime"),
"anthropic_compatible",
)
self.assertEqual(getattr(model, "_moviepilot_llm_provider_id"), "minimax")
self.assertEqual(
getattr(model, "_moviepilot_llm_base_url"),
"https://api.minimaxi.com/anthropic/v1",
)
def test_get_llm_applies_proxy_only_when_enabled(self): def test_get_llm_applies_proxy_only_when_enabled(self):
"""LLM 构造时应按独立开关决定是否传入系统代理。""" """LLM 构造时应按独立开关决定是否传入系统代理。"""
calls = [] calls = []

View File

@@ -253,6 +253,15 @@ def test_send_msg_markdown_escaping(telegram):
assert send_kwargs["text"].startswith("*测试标题*\n") assert send_kwargs["text"].startswith("*测试标题*\n")
def test_telegramify_new_content_fields_are_used_directly():
"""新版telegramify对象应直接使用已渲染的MarkdownV2字段"""
text_item = SimpleNamespace(content="已转义\\_文本")
file_item = SimpleNamespace(caption="已转义\\_说明")
assert Telegram._telegramify_item_text(text_item) == "已转义\\_文本"
assert Telegram._telegramify_item_caption(file_item) == "已转义\\_说明"
def test_send_msg_with_html_parse_mode_keeps_html(telegram): def test_send_msg_with_html_parse_mode_keeps_html(telegram):
"""HTML模式发送时应保留调用方传入的HTML内容""" """HTML模式发送时应保留调用方传入的HTML内容"""
result = telegram.send_msg( result = telegram.send_msg(