mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
refactor(agent): 按需加载 Agent 运行时 (#6336)
This commit is contained in:
@@ -81,6 +81,62 @@ Xvfb,因此不能用同一个 `0 → 0` / `0 → 1` 不变量衡量。三轮 B
|
||||
- 非默认场景结果保存在 `samples/<scenario>/<variant>-<index>/`,可与同 campaign 的 idle 样本并存,
|
||||
Markdown 中位数会按场景分组,不会混算。
|
||||
|
||||
## Agent 惰性物化场景
|
||||
|
||||
PERF-003 在既有 `AI_AGENT_ENABLE=false` 固定配置下增加两个 After-only 场景。探针只向主 MoviePilot
|
||||
Python 进程发送信号;OpenAPI 生成和工具目录构造均发生在该解释器内,不通过 `docker exec` 启动
|
||||
第二个 Python,也不调用真实 Agent、LLM provider 或外部 MCP。
|
||||
|
||||
先以 `f2e548e1` 冻结 Before,候选提交完成后把 `AFTER_COMMIT` 替换为其精确 commit:
|
||||
|
||||
```bash
|
||||
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
|
||||
--campaign v3-perf-003 \
|
||||
build --before-ref f2e548e1 --after-ref AFTER_COMMIT
|
||||
|
||||
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
|
||||
--campaign v3-perf-003 \
|
||||
seed --browser-source-volume mp-perf-v3-browser-seed --replace
|
||||
```
|
||||
|
||||
正式 idle-default 三组 A/B 仍使用原 `run` 合同;下面两个动作场景在同一 build/seed 后单独采 After,
|
||||
不会覆盖 idle 结果:
|
||||
|
||||
```bash
|
||||
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
|
||||
--campaign v3-perf-003 \
|
||||
run --before-ref f2e548e1 --after-ref AFTER_COMMIT \
|
||||
--browser-source-volume mp-perf-v3-browser-seed \
|
||||
--points 1,5,10,30 --replace --keep-resources
|
||||
|
||||
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
|
||||
--campaign v3-perf-003 \
|
||||
sample --variant after --index 1 --scenario agent-disabled-router --points 1,5,10,30
|
||||
|
||||
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
|
||||
--campaign v3-perf-003 \
|
||||
sample --variant after --index 2 --scenario agent-tool-catalog --points 1,5,10,30
|
||||
```
|
||||
|
||||
- `agent-disabled-router`:直接从主进程 FastAPI app 生成完整 OpenAPI,确认 Agent、LLM、MCP、OpenAI、
|
||||
Anthropic 路由在禁用态仍存在,同时 callback、LLM helper、工具域、orchestrator、LangGraph 和 provider SDK
|
||||
前后保持 0,工具工厂不物化;
|
||||
- `agent-tool-catalog`:通过主进程已有的 `moviepilot_tool_manager.list_tools()` 首次构建现有工具目录和 JSON Schema,
|
||||
要求动作前工具域未物化,动作后仅工具 base/catalog/factory/impl 物化;目录还必须无身份碰撞、Schema
|
||||
digest 完整,重复读取复用同一 snapshot/revision。结果记录工具数、Schema 摘要、plugin revision 与
|
||||
factory revision;
|
||||
- 固定哨兵覆盖 `app.agent.orchestrator`、`app.agent.callback`、`app.agent.llm.helper`、工具
|
||||
`base/catalog/factory/impl`、`langgraph`、`langchain`、`langchain_core`、`openai`、`anthropic`、
|
||||
`google.genai`、`boto3`、`botocore`。其中 `langchain/langchain_core` 可能由完整 Schema 聚合形成既有
|
||||
基线,只记录数量与变化,不作为禁用态归零门禁;
|
||||
- JSON 保留动作前后 Engine、PSS/USS、线程、完整 `sys.modules`、materialization observation、revision、
|
||||
网络累计值和浏览器卷指纹;动作前后容器网络收发必须为 0,Markdown 另汇总 Agent 场景与各定时点的
|
||||
模块哨兵峰值;
|
||||
- 启用态 Agent 生命周期不会在该无凭据场景中伪造。现有 `get_running_agent_manager()` 是严格只读、
|
||||
non-materializing 的运行态 getter,`begin_agent_shutdown()` 也只是关闭轴;二者都不是安全启用入口。
|
||||
启用态必须由正式 startup/service lifecycle 驱动,只有宿主形成明确不创建 provider/client、不会外联的
|
||||
公共初始化合同后,才适合加入同一测量门禁。
|
||||
|
||||
## 完整三组 A/B
|
||||
|
||||
```bash
|
||||
|
||||
@@ -12,6 +12,20 @@ import sys
|
||||
_OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR")
|
||||
_SCENARIO = os.environ.get("MP_PERF_SCENARIO", "idle-default")
|
||||
_ACTIVATION_TIMEOUT = float(os.environ.get("MP_PERF_ACTIVATION_TIMEOUT", "120"))
|
||||
_AGENT_SCENARIOS = {"agent-disabled-router", "agent-tool-catalog"}
|
||||
_AGENT_HEAVY_MODULE_PREFIXES = tuple(
|
||||
prefix
|
||||
for prefix in os.environ.get(
|
||||
"MP_PERF_AGENT_MODULE_PREFIXES",
|
||||
(
|
||||
"app.agent.orchestrator,app.agent.callback,app.agent.llm.helper,"
|
||||
"app.agent.tools.base,app.agent.tools.catalog,"
|
||||
"app.agent.tools.factory,app.agent.tools.impl,langgraph,langchain,"
|
||||
"langchain_core,openai,anthropic,google.genai,boto3,botocore"
|
||||
),
|
||||
).split(",")
|
||||
if prefix
|
||||
)
|
||||
_snapshot_index = 0
|
||||
_activation_started = False
|
||||
_browser_resources: list[object] = []
|
||||
@@ -112,6 +126,244 @@ def _enum_value(value):
|
||||
return getattr(value, "value", value)
|
||||
|
||||
|
||||
def _stable_digest(value: object) -> str:
|
||||
"""计算不依赖对象地址的 JSON 摘要。"""
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
content = json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _agent_module_observation() -> dict[str, object]:
|
||||
"""记录 Agent 重模块在目标解释器中的精确加载状态。"""
|
||||
prefix_counts = {
|
||||
prefix: sum(
|
||||
1
|
||||
for module_name in sys.modules
|
||||
if module_name == prefix or module_name.startswith(f"{prefix}.")
|
||||
)
|
||||
for prefix in _AGENT_HEAVY_MODULE_PREFIXES
|
||||
}
|
||||
matching_modules = sorted(
|
||||
module_name
|
||||
for module_name in sys.modules
|
||||
if any(
|
||||
module_name == prefix or module_name.startswith(f"{prefix}.")
|
||||
for prefix in _AGENT_HEAVY_MODULE_PREFIXES
|
||||
)
|
||||
)
|
||||
return {
|
||||
"total_modules": len(sys.modules),
|
||||
"prefix_counts": prefix_counts,
|
||||
"matching_modules": matching_modules,
|
||||
"matching_sha256": _stable_digest(matching_modules),
|
||||
}
|
||||
|
||||
|
||||
def _read_agent_runtime() -> dict[str, object]:
|
||||
"""读取轻量 Agent loader 的公开只读状态,不触发 capability 首用。"""
|
||||
try:
|
||||
from app.agent.runtime_loader import is_tool_factory_materialized
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"tool_factory_materialized": is_tool_factory_materialized(),
|
||||
}
|
||||
except Exception as error: # pragma: no cover - 候选未就绪或真实 runtime 错误
|
||||
return {
|
||||
"available": False,
|
||||
"error_type": type(error).__name__,
|
||||
"error": str(error),
|
||||
}
|
||||
|
||||
|
||||
def _probe_router_openapi(app_instance=None, settings_object=None) -> dict[str, object]:
|
||||
"""在主进程中生成 OpenAPI,并验证禁用态 Agent 路由仍完整存在。"""
|
||||
if app_instance is None:
|
||||
from app.factory import app as app_instance
|
||||
if settings_object is None:
|
||||
from app.runtime.config import settings as settings_object
|
||||
|
||||
required_paths = (
|
||||
"/api/v1/message/agent/stream",
|
||||
"/api/v1/message/agent/sessions",
|
||||
"/api/v1/openai/v1/chat/completions",
|
||||
"/api/v1/openai/v1/responses",
|
||||
"/api/v1/anthropic/v1/messages",
|
||||
"/api/v1/llm/manage",
|
||||
"/api/v1/mcp",
|
||||
"/api/v1/mcp/tools",
|
||||
)
|
||||
schema = app_instance.openapi()
|
||||
route_paths = sorted(
|
||||
{
|
||||
str(route.path)
|
||||
for route in app_instance.routes
|
||||
if getattr(route, "path", None)
|
||||
}
|
||||
)
|
||||
openapi_paths = sorted((schema.get("paths") or {}).keys())
|
||||
missing_routes = [path for path in required_paths if path not in route_paths]
|
||||
missing_openapi_paths = [
|
||||
path for path in required_paths if path not in openapi_paths
|
||||
]
|
||||
agent_enabled = bool(settings_object.AI_AGENT_ENABLE)
|
||||
return {
|
||||
"success": not agent_enabled
|
||||
and not missing_routes
|
||||
and not missing_openapi_paths,
|
||||
"ai_agent_enable": agent_enabled,
|
||||
"required_paths": list(required_paths),
|
||||
"missing_routes": missing_routes,
|
||||
"missing_openapi_paths": missing_openapi_paths,
|
||||
"route_count": len(route_paths),
|
||||
"openapi_path_count": len(openapi_paths),
|
||||
"openapi_sha256": _stable_digest(schema),
|
||||
"openapi_title": (schema.get("info") or {}).get("title"),
|
||||
"openapi_version": (schema.get("info") or {}).get("version"),
|
||||
}
|
||||
|
||||
|
||||
def _probe_tool_catalog(manager=None) -> dict[str, object]:
|
||||
"""通过稳定工具管理入口首次生成目录与 JSON Schema。"""
|
||||
if manager is None:
|
||||
from app.agent.tools.manager import moviepilot_tool_manager
|
||||
|
||||
manager = moviepilot_tool_manager
|
||||
definitions = manager.list_tools()
|
||||
catalog = manager.catalog
|
||||
serialized_definitions = [
|
||||
{
|
||||
"name": definition.name,
|
||||
"input_schema": definition.input_schema,
|
||||
}
|
||||
for definition in definitions
|
||||
]
|
||||
schema_count = sum(
|
||||
isinstance(definition.input_schema, dict) for definition in definitions
|
||||
)
|
||||
entries = catalog.entries if catalog is not None else ()
|
||||
collisions = catalog.collisions if catalog is not None else {}
|
||||
source_counts: dict[str, int] = {}
|
||||
serialized_entries = []
|
||||
for entry in entries:
|
||||
source_counts[entry.source] = source_counts.get(entry.source, 0) + 1
|
||||
serialized_entries.append(
|
||||
{
|
||||
"name": entry.name,
|
||||
"source": entry.source,
|
||||
"schema_digest": entry.schema_digest,
|
||||
}
|
||||
)
|
||||
first_catalog_sha256 = _stable_digest(serialized_entries)
|
||||
first_schemas_sha256 = _stable_digest(serialized_definitions)
|
||||
repeated_definitions = manager.list_tools()
|
||||
repeated_catalog = manager.catalog
|
||||
repeated_serialized_definitions = [
|
||||
{
|
||||
"name": definition.name,
|
||||
"input_schema": definition.input_schema,
|
||||
}
|
||||
for definition in repeated_definitions
|
||||
]
|
||||
repeated_entries = repeated_catalog.entries if repeated_catalog is not None else ()
|
||||
repeated_serialized_entries = [
|
||||
{
|
||||
"name": entry.name,
|
||||
"source": entry.source,
|
||||
"schema_digest": entry.schema_digest,
|
||||
}
|
||||
for entry in repeated_entries
|
||||
]
|
||||
repeated_catalog_sha256 = _stable_digest(repeated_serialized_entries)
|
||||
repeated_schemas_sha256 = _stable_digest(repeated_serialized_definitions)
|
||||
schema_digests_complete = all(
|
||||
isinstance(entry.schema_digest, str) and len(entry.schema_digest) == 64
|
||||
for entry in entries
|
||||
)
|
||||
repeat_revision_unchanged = bool(
|
||||
catalog is not None
|
||||
and repeated_catalog is not None
|
||||
and repeated_catalog.plugin_revision == catalog.plugin_revision
|
||||
and repeated_catalog.factory_revision == catalog.factory_revision
|
||||
)
|
||||
repeat_stable = bool(
|
||||
repeated_catalog is catalog
|
||||
and len(repeated_definitions) == len(definitions)
|
||||
and repeated_catalog_sha256 == first_catalog_sha256
|
||||
and repeated_schemas_sha256 == first_schemas_sha256
|
||||
and repeat_revision_unchanged
|
||||
)
|
||||
return {
|
||||
"success": bool(definitions)
|
||||
and catalog is not None
|
||||
and len(entries) == len(definitions)
|
||||
and schema_count == len(definitions)
|
||||
and not collisions
|
||||
and schema_digests_complete
|
||||
and repeat_stable,
|
||||
"tool_count": len(definitions),
|
||||
"schema_count": schema_count,
|
||||
"catalog_entry_count": len(entries),
|
||||
"collision_names": sorted(collisions),
|
||||
"plugin_revision": catalog.plugin_revision if catalog is not None else None,
|
||||
"factory_revision": catalog.factory_revision if catalog is not None else None,
|
||||
"schemas_sha256": first_schemas_sha256,
|
||||
"catalog_sha256": first_catalog_sha256,
|
||||
"source_counts": source_counts,
|
||||
"schema_digests_complete": schema_digests_complete,
|
||||
"repeat_tool_count": len(repeated_definitions),
|
||||
"repeat_catalog_same_object": repeated_catalog is catalog,
|
||||
"repeat_catalog_sha256": repeated_catalog_sha256,
|
||||
"repeat_schemas_sha256": repeated_schemas_sha256,
|
||||
"repeat_revision_unchanged": repeat_revision_unchanged,
|
||||
"repeat_stable": repeat_stable,
|
||||
}
|
||||
|
||||
|
||||
def _activate_agent_scenario(
|
||||
scenario: str,
|
||||
*,
|
||||
app_instance=None,
|
||||
settings_object=None,
|
||||
tool_manager=None,
|
||||
runtime_reader=None,
|
||||
) -> dict[str, object]:
|
||||
"""执行 Agent 禁用态路由或首次工具目录的进程内场景。"""
|
||||
if scenario not in _AGENT_SCENARIOS:
|
||||
raise ValueError(f"场景不支持 Agent 激活:{scenario}")
|
||||
runtime_reader = runtime_reader or _read_agent_runtime
|
||||
modules_before = _agent_module_observation()
|
||||
runtime_before = runtime_reader()
|
||||
|
||||
if scenario == "agent-disabled-router":
|
||||
action = _probe_router_openapi(
|
||||
app_instance=app_instance,
|
||||
settings_object=settings_object,
|
||||
)
|
||||
else:
|
||||
action = _probe_tool_catalog(manager=tool_manager)
|
||||
|
||||
modules_after = _agent_module_observation()
|
||||
runtime_after = runtime_reader()
|
||||
return {
|
||||
"requested": True,
|
||||
"action": scenario.removeprefix("agent-"),
|
||||
"success": bool(action.get("success")),
|
||||
"modules": {"before": modules_before, "after": modules_after},
|
||||
"observations": {"before": runtime_before, "after": runtime_after},
|
||||
"router_openapi": action if scenario == "agent-disabled-router" else None,
|
||||
"tool_catalog": action if scenario == "agent-tool-catalog" else None,
|
||||
}
|
||||
|
||||
|
||||
def _read_display_runtime() -> dict[str, object]:
|
||||
"""读取 host.display 的只读状态和观测,不触发资源激活。"""
|
||||
try:
|
||||
@@ -338,8 +590,12 @@ def _run_activation() -> None:
|
||||
"started_at": _utc_now(),
|
||||
}
|
||||
try:
|
||||
result["browser"] = _activate_browser_scenario(_SCENARIO)
|
||||
result["success"] = bool(result["browser"]["success"])
|
||||
if _SCENARIO in _AGENT_SCENARIOS:
|
||||
result["agent"] = _activate_agent_scenario(_SCENARIO)
|
||||
result["success"] = bool(result["agent"]["success"])
|
||||
else:
|
||||
result["browser"] = _activate_browser_scenario(_SCENARIO)
|
||||
result["success"] = bool(result["browser"]["success"])
|
||||
except Exception as error: # pragma: no cover - 真实集成错误由 marker 保存
|
||||
result.update(
|
||||
{
|
||||
|
||||
@@ -32,7 +32,9 @@ DEFAULT_SUBSTRATE = (
|
||||
)
|
||||
DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed"
|
||||
DEFAULT_SCENARIO = "idle-default"
|
||||
SCENARIOS = (DEFAULT_SCENARIO, "browser-headless", "browser-headed")
|
||||
BROWSER_SCENARIOS = ("browser-headless", "browser-headed")
|
||||
AGENT_SCENARIOS = ("agent-disabled-router", "agent-tool-catalog")
|
||||
SCENARIOS = (DEFAULT_SCENARIO, *BROWSER_SCENARIOS, *AGENT_SCENARIOS)
|
||||
CAMPAIGN_LABEL = "org.moviepilot.perf.campaign"
|
||||
ROLE_LABEL = "org.moviepilot.perf.role"
|
||||
SOURCE_LABEL = "org.moviepilot.perf.source-commit"
|
||||
@@ -43,6 +45,35 @@ CRITICAL_SUBSTRATE_PATHS = (
|
||||
"scripts/uv-pip-compat.sh",
|
||||
)
|
||||
SEED_COMPATIBILITY_PATHS = ("database/versions",)
|
||||
AGENT_HEAVY_MODULE_PREFIXES = (
|
||||
"app.agent.orchestrator",
|
||||
"app.agent.callback",
|
||||
"app.agent.llm.helper",
|
||||
"app.agent.tools.base",
|
||||
"app.agent.tools.catalog",
|
||||
"app.agent.tools.factory",
|
||||
"app.agent.tools.impl",
|
||||
"langgraph",
|
||||
"langchain",
|
||||
"langchain_core",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"google.genai",
|
||||
"boto3",
|
||||
"botocore",
|
||||
)
|
||||
AGENT_SCHEMA_BASELINE_PREFIXES = ("langchain", "langchain_core")
|
||||
AGENT_NONMATERIALIZATION_PREFIXES = tuple(
|
||||
prefix
|
||||
for prefix in AGENT_HEAVY_MODULE_PREFIXES
|
||||
if prefix not in AGENT_SCHEMA_BASELINE_PREFIXES
|
||||
)
|
||||
AGENT_TOOL_CATALOG_PREFIXES = (
|
||||
"app.agent.tools.base",
|
||||
"app.agent.tools.catalog",
|
||||
"app.agent.tools.factory",
|
||||
"app.agent.tools.impl",
|
||||
)
|
||||
MODULE_PREFIXES = (
|
||||
"lark_oapi",
|
||||
"slack_bolt",
|
||||
@@ -50,12 +81,10 @@ MODULE_PREFIXES = (
|
||||
"discord",
|
||||
"plexapi",
|
||||
"telebot",
|
||||
"langgraph",
|
||||
"langchain",
|
||||
"app.agent",
|
||||
"app.agent.orchestrator",
|
||||
"app.agent.tools",
|
||||
"app.modules",
|
||||
*AGENT_HEAVY_MODULE_PREFIXES,
|
||||
)
|
||||
BALANCED_RUN_ORDER = (
|
||||
("before", 1),
|
||||
@@ -593,6 +622,7 @@ def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, s
|
||||
"MP_PERF_ACTIVATION_TIMEOUT": str(
|
||||
getattr(args, "activation_timeout", 180)
|
||||
),
|
||||
"MP_PERF_AGENT_MODULE_PREFIXES": ",".join(AGENT_HEAVY_MODULE_PREFIXES),
|
||||
}
|
||||
)
|
||||
return environment
|
||||
@@ -1133,7 +1163,7 @@ def capture_activation_snapshot(
|
||||
output_dir: Path,
|
||||
phase: str,
|
||||
) -> dict[str, Any]:
|
||||
"""采集浏览器激活边界的 Engine、进程和进程内 import 状态。"""
|
||||
"""采集场景动作边界的 Engine、进程和进程内 import 状态。"""
|
||||
engine = capture_engine_stats(container)
|
||||
processes = capture_processes(container)
|
||||
modules = capture_modules(container, output_dir, processes["main_python"])
|
||||
@@ -1248,17 +1278,195 @@ def evaluate_browser_activation(
|
||||
}
|
||||
|
||||
|
||||
def activate_browser_scenario(
|
||||
def _agent_prefix_counts(snapshot: dict[str, Any]) -> dict[str, int]:
|
||||
"""从模块快照提取 PERF-003 Agent 重模块哨兵。"""
|
||||
counts = snapshot["modules"].get("prefix_counts") or {}
|
||||
return {
|
||||
prefix: int(counts.get(prefix) or 0) for prefix in AGENT_HEAVY_MODULE_PREFIXES
|
||||
}
|
||||
|
||||
|
||||
def evaluate_agent_activation(
|
||||
scenario: str,
|
||||
pre: dict[str, Any],
|
||||
post: dict[str, Any],
|
||||
marker: dict[str, Any],
|
||||
expected_pid: Optional[int] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""验证禁用态路由与首次工具目录的惰性物化不变量。"""
|
||||
agent = marker.get("agent") or {}
|
||||
observations = agent.get("observations") or {}
|
||||
runtime_before = observations.get("before") or {}
|
||||
runtime_after = observations.get("after") or {}
|
||||
prefix_before = _agent_prefix_counts(pre)
|
||||
prefix_after = _agent_prefix_counts(post)
|
||||
forbidden_before = {
|
||||
prefix: prefix_before[prefix]
|
||||
for prefix in AGENT_NONMATERIALIZATION_PREFIXES
|
||||
if prefix_before[prefix]
|
||||
}
|
||||
pre_xvfb = pre["processes"]["xvfb"]
|
||||
post_xvfb = post["processes"]["xvfb"]
|
||||
network_delta = {
|
||||
"rx_bytes": int(post["engine"]["network_rx_bytes"])
|
||||
- int(pre["engine"]["network_rx_bytes"]),
|
||||
"tx_bytes": int(post["engine"]["network_tx_bytes"])
|
||||
- int(pre["engine"]["network_tx_bytes"]),
|
||||
}
|
||||
errors: list[str] = []
|
||||
|
||||
if marker.get("scenario") != scenario:
|
||||
errors.append("进程内 marker 的场景与采集请求不一致")
|
||||
if expected_pid is not None and marker.get("pid") != expected_pid:
|
||||
errors.append("进程内 marker 不是目标 MoviePilot Python 进程写出")
|
||||
if not marker.get("success") or not agent.get("success"):
|
||||
errors.append("主 MoviePilot Python 进程未完成 Agent 场景动作")
|
||||
if forbidden_before:
|
||||
errors.append("Agent 场景动作前已经加载必须延迟物化的模块")
|
||||
if pre_xvfb["count"] != 0 or post_xvfb["count"] != 0:
|
||||
errors.append("Agent 场景不得物化 Xvfb")
|
||||
if not runtime_before.get("available") or not runtime_after.get("available"):
|
||||
errors.append("主进程未提供轻量 Agent runtime 只读观测")
|
||||
if runtime_before.get("tool_factory_materialized") is not False:
|
||||
errors.append("Agent 场景动作前工具工厂必须未物化")
|
||||
if any(network_delta.values()):
|
||||
errors.append("Agent 场景动作产生了容器网络收发")
|
||||
|
||||
revision = {"plugin": None, "factory": None}
|
||||
action_summary: dict[str, Any]
|
||||
if scenario == "agent-disabled-router":
|
||||
router = agent.get("router_openapi") or {}
|
||||
if router.get("ai_agent_enable") is not False:
|
||||
errors.append("router/OpenAPI 场景必须运行在 AI_AGENT_ENABLE=false")
|
||||
if router.get("missing_routes") or router.get("missing_openapi_paths"):
|
||||
errors.append("禁用态缺少 Agent 相关 router 或 OpenAPI path")
|
||||
forbidden_after = {
|
||||
prefix: prefix_after[prefix]
|
||||
for prefix in AGENT_NONMATERIALIZATION_PREFIXES
|
||||
if prefix_after[prefix]
|
||||
}
|
||||
if forbidden_after:
|
||||
errors.append("生成完整 OpenAPI 后加载了必须延迟物化的模块")
|
||||
if runtime_after.get("tool_factory_materialized") is not False:
|
||||
errors.append("生成完整 OpenAPI 不得物化工具工厂")
|
||||
action_summary = {
|
||||
"route_count": router.get("route_count"),
|
||||
"openapi_path_count": router.get("openapi_path_count"),
|
||||
"openapi_sha256": router.get("openapi_sha256"),
|
||||
}
|
||||
elif scenario == "agent-tool-catalog":
|
||||
catalog = agent.get("tool_catalog") or {}
|
||||
if prefix_after["app.agent.tools.factory"] < 1:
|
||||
errors.append("首次工具目录动作后未加载工具工厂")
|
||||
if prefix_after["app.agent.tools.impl"] < 1:
|
||||
errors.append("首次工具目录动作后未加载工具实现")
|
||||
allowed_prefixes = {
|
||||
*AGENT_TOOL_CATALOG_PREFIXES,
|
||||
*AGENT_SCHEMA_BASELINE_PREFIXES,
|
||||
}
|
||||
unexpected_prefixes = {
|
||||
prefix: count
|
||||
for prefix, count in prefix_after.items()
|
||||
if prefix not in allowed_prefixes and count
|
||||
}
|
||||
if unexpected_prefixes:
|
||||
errors.append("首次工具目录动作加载了非目录所需的 Agent/provider 重模块")
|
||||
if runtime_after.get("tool_factory_materialized") is not True:
|
||||
errors.append("首次工具目录动作后工具工厂未标记为已物化")
|
||||
if (
|
||||
not catalog.get("success")
|
||||
or not catalog.get("tool_count")
|
||||
or catalog.get("schema_count") != catalog.get("tool_count")
|
||||
or catalog.get("catalog_entry_count") != catalog.get("tool_count")
|
||||
or catalog.get("collision_names")
|
||||
or not catalog.get("schema_digests_complete")
|
||||
or not catalog.get("repeat_stable")
|
||||
or catalog.get("repeat_tool_count") != catalog.get("tool_count")
|
||||
):
|
||||
errors.append("工具目录、JSON Schema 或重复读取稳定性不满足合同")
|
||||
if catalog.get("plugin_revision") is None or not catalog.get(
|
||||
"factory_revision"
|
||||
):
|
||||
errors.append("工具目录缺少 plugin/factory revision")
|
||||
revision = {
|
||||
"plugin": catalog.get("plugin_revision"),
|
||||
"factory": catalog.get("factory_revision"),
|
||||
}
|
||||
action_summary = {
|
||||
"tool_count": catalog.get("tool_count"),
|
||||
"schema_count": catalog.get("schema_count"),
|
||||
"schemas_sha256": catalog.get("schemas_sha256"),
|
||||
"collision_names": catalog.get("collision_names") or [],
|
||||
"repeat_stable": catalog.get("repeat_stable"),
|
||||
}
|
||||
else:
|
||||
errors.append(f"未知 Agent 场景:{scenario}")
|
||||
action_summary = {}
|
||||
|
||||
return {
|
||||
"passed": not errors,
|
||||
"errors": errors,
|
||||
"expected": (
|
||||
"router/OpenAPI 完整且必须延迟物化的模块保持 0"
|
||||
if scenario == "agent-disabled-router"
|
||||
else "首次工具目录后仅物化工具域及 Schema 基线"
|
||||
),
|
||||
"observed": {
|
||||
"pre_xvfb_count": pre_xvfb["count"],
|
||||
"post_xvfb_count": post_xvfb["count"],
|
||||
"prefix_before": prefix_before,
|
||||
"prefix_after": prefix_after,
|
||||
"network_delta": network_delta,
|
||||
"tool_factory_materialized_before": runtime_before.get(
|
||||
"tool_factory_materialized"
|
||||
),
|
||||
"tool_factory_materialized_after": runtime_after.get(
|
||||
"tool_factory_materialized"
|
||||
),
|
||||
},
|
||||
"action": action_summary,
|
||||
"revision": revision,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_scenario_activation(
|
||||
scenario: str,
|
||||
pre: dict[str, Any],
|
||||
post: dict[str, Any],
|
||||
marker: dict[str, Any],
|
||||
expected_pid: Optional[int] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""按场景族分派外部采样验收。"""
|
||||
if scenario in BROWSER_SCENARIOS:
|
||||
return evaluate_browser_activation(
|
||||
scenario,
|
||||
pre,
|
||||
post,
|
||||
marker,
|
||||
expected_pid=expected_pid,
|
||||
)
|
||||
if scenario in AGENT_SCENARIOS:
|
||||
return evaluate_agent_activation(
|
||||
scenario,
|
||||
pre,
|
||||
post,
|
||||
marker,
|
||||
expected_pid=expected_pid,
|
||||
)
|
||||
raise HarnessError(f"未知激活场景:{scenario}")
|
||||
|
||||
|
||||
def activate_sample_scenario(
|
||||
container,
|
||||
output_dir: Path,
|
||||
scenario: str,
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
"""通过 SIGUSR2 让目标 MoviePilot 解释器执行场景激活并回收 marker。"""
|
||||
"""通过 SIGUSR2 让目标 MoviePilot 解释器执行动作并回收 marker。"""
|
||||
pre = capture_activation_snapshot(container, output_dir, "pre-activation")
|
||||
main_python = pre["processes"]["main_python"]
|
||||
if not main_python:
|
||||
raise HarnessError("未找到主 Python 进程,无法触发浏览器场景")
|
||||
raise HarnessError("未找到主 Python 进程,无法触发测量场景")
|
||||
marker_path = output_dir / "modules" / f"activation-{main_python['pid']}.json"
|
||||
marker_path.unlink(missing_ok=True)
|
||||
|
||||
@@ -1274,12 +1482,12 @@ def activate_browser_scenario(
|
||||
raise HarnessError("等待场景激活 marker 时容器提前退出")
|
||||
time.sleep(0.05)
|
||||
if not marker_path.exists():
|
||||
raise HarnessError(f"浏览器场景激活在 {timeout:.0f}s 内未完成")
|
||||
raise HarnessError(f"测量场景动作在 {timeout:.0f}s 内未完成")
|
||||
|
||||
marker_received_at = time.monotonic()
|
||||
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
post = capture_activation_snapshot(container, output_dir, "post-activation")
|
||||
validation = evaluate_browser_activation(
|
||||
validation = evaluate_scenario_activation(
|
||||
scenario,
|
||||
pre,
|
||||
post,
|
||||
@@ -1304,7 +1512,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"""执行一个隔离样本并在约定时间点采集完整指标。"""
|
||||
scenario = getattr(args, "scenario", DEFAULT_SCENARIO)
|
||||
if scenario != DEFAULT_SCENARIO and args.variant != "after":
|
||||
raise HarnessError("浏览器激活场景只用于验证包含 app.sdk.browser 的 After 候选")
|
||||
raise HarnessError("非默认场景只用于验证包含候选公共 API 的 After 版本")
|
||||
client = require_docker_client()
|
||||
build = load_build_manifest(args)
|
||||
config_seed, browser_seed = require_seed_volumes(client, args)
|
||||
@@ -1391,7 +1599,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
|
||||
measurement_origin_at = settled_at
|
||||
|
||||
if scenario != DEFAULT_SCENARIO:
|
||||
activation = activate_browser_scenario(
|
||||
activation = activate_sample_scenario(
|
||||
container,
|
||||
output_dir,
|
||||
scenario,
|
||||
@@ -1614,8 +1822,13 @@ def build_markdown_report(
|
||||
)
|
||||
lines.append("| " + " | ".join(row) + " |")
|
||||
|
||||
activated_samples = [sample for sample in samples if sample.get("activation")]
|
||||
if activated_samples:
|
||||
browser_activated_samples = [
|
||||
sample
|
||||
for sample in samples
|
||||
if sample.get("activation")
|
||||
and sample.get("scenario", DEFAULT_SCENARIO) in BROWSER_SCENARIOS
|
||||
]
|
||||
if browser_activated_samples:
|
||||
activation_headers = [
|
||||
"场景",
|
||||
"版本",
|
||||
@@ -1644,7 +1857,7 @@ def build_markdown_report(
|
||||
]
|
||||
)
|
||||
for sample in sorted(
|
||||
activated_samples,
|
||||
browser_activated_samples,
|
||||
key=lambda item: (
|
||||
item.get("scenario", DEFAULT_SCENARIO),
|
||||
variant_order.get(item["variant"], 99),
|
||||
@@ -1692,6 +1905,144 @@ def build_markdown_report(
|
||||
]
|
||||
lines.append("| " + " | ".join(activation_row) + " |")
|
||||
|
||||
agent_activated_samples = [
|
||||
sample
|
||||
for sample in samples
|
||||
if sample.get("activation")
|
||||
and sample.get("scenario", DEFAULT_SCENARIO) in AGENT_SCENARIOS
|
||||
]
|
||||
if agent_activated_samples:
|
||||
agent_headers = [
|
||||
"场景",
|
||||
"样本",
|
||||
"动作(s)",
|
||||
"Pre/Post WS(MiB)",
|
||||
"Pre/Post Python PSS(MiB)",
|
||||
"Pre/Post sys.modules",
|
||||
"Factory observation",
|
||||
"模块哨兵 Pre",
|
||||
"模块哨兵 Post",
|
||||
"Router/OpenAPI 或 Tools/Schemas",
|
||||
"Plugin/Factory revision",
|
||||
"Action RX/TX Δ(KiB)",
|
||||
"验收",
|
||||
]
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Agent 场景动作",
|
||||
"",
|
||||
"| " + " | ".join(agent_headers) + " |",
|
||||
"| " + " | ".join(["---"] * len(agent_headers)) + " |",
|
||||
]
|
||||
)
|
||||
|
||||
def format_prefix_counts(counts: dict[str, int]) -> str:
|
||||
"""仅展开已加载前缀,全部未加载时输出明确零状态。"""
|
||||
loaded = [f"{prefix}={count}" for prefix, count in counts.items() if count]
|
||||
return ", ".join(loaded) if loaded else "全部 0"
|
||||
|
||||
for sample in sorted(
|
||||
agent_activated_samples,
|
||||
key=lambda item: (
|
||||
item.get("scenario", DEFAULT_SCENARIO),
|
||||
item["sample_index"],
|
||||
),
|
||||
):
|
||||
activation = sample["activation"]
|
||||
pre = activation["pre"]
|
||||
post = activation["post"]
|
||||
validation = activation["validation"]
|
||||
observed = validation["observed"]
|
||||
action = validation["action"]
|
||||
revision = validation["revision"]
|
||||
pre_python = pre["processes"].get("main_python") or {}
|
||||
post_python = post["processes"].get("main_python") or {}
|
||||
if sample.get("scenario") == "agent-disabled-router":
|
||||
action_result = (
|
||||
f"{action.get('route_count')}/{action.get('openapi_path_count')}"
|
||||
)
|
||||
else:
|
||||
action_result = (
|
||||
f"{action.get('tool_count')}/{action.get('schema_count')}; "
|
||||
f"repeat={'Y' if action.get('repeat_stable') else 'N'}; "
|
||||
f"collision={len(action.get('collision_names') or [])}"
|
||||
)
|
||||
factory_revision = str(revision.get("factory") or "")
|
||||
revision_result = (
|
||||
f"{revision.get('plugin')}/{factory_revision[:12]}"
|
||||
if factory_revision
|
||||
else "不适用"
|
||||
)
|
||||
agent_row = [
|
||||
sample.get("scenario", DEFAULT_SCENARIO),
|
||||
str(sample["sample_index"]),
|
||||
f"{float(activation.get('worker_elapsed_seconds') or 0):.2f}",
|
||||
f"{format_mib(pre['engine']['working_set_bytes'])}/"
|
||||
f"{format_mib(post['engine']['working_set_bytes'])}",
|
||||
f"{format_kib_as_mib(pre_python.get('pss_kib'))}/"
|
||||
f"{format_kib_as_mib(post_python.get('pss_kib'))}",
|
||||
f"{pre['modules'].get('count')}/{post['modules'].get('count')}",
|
||||
(
|
||||
f"{observed.get('tool_factory_materialized_before')}→"
|
||||
f"{observed.get('tool_factory_materialized_after')}"
|
||||
),
|
||||
format_prefix_counts(observed.get("prefix_before") or {}),
|
||||
format_prefix_counts(observed.get("prefix_after") or {}),
|
||||
action_result,
|
||||
revision_result,
|
||||
(
|
||||
f"{format_bytes_as_kib(post['engine']['network_rx_bytes'] - pre['engine']['network_rx_bytes'])}/"
|
||||
f"{format_bytes_as_kib(post['engine']['network_tx_bytes'] - pre['engine']['network_tx_bytes'])}"
|
||||
),
|
||||
"通过" if validation["passed"] else "失败",
|
||||
]
|
||||
lines.append("| " + " | ".join(agent_row) + " |")
|
||||
|
||||
sentinel_samples = [sample for sample in samples if sample.get("measurements")]
|
||||
if sentinel_samples:
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Agent 模块哨兵",
|
||||
"",
|
||||
"每行记录该样本所有定时采样点的最大模块数;精确时间点数据保留在 JSON。",
|
||||
"`langchain` 与 `langchain_core` 只记录 Schema 基线,不参与归零门禁。",
|
||||
"",
|
||||
"| 场景 | 版本 | 样本 | 重模块峰值 |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for sample in sorted(
|
||||
sentinel_samples,
|
||||
key=lambda item: (
|
||||
item.get("scenario", DEFAULT_SCENARIO),
|
||||
variant_order.get(item["variant"], 99),
|
||||
item["sample_index"],
|
||||
),
|
||||
):
|
||||
peaks = {
|
||||
prefix: max(
|
||||
int(
|
||||
measurement.get("modules", {})
|
||||
.get("prefix_counts", {})
|
||||
.get(prefix, 0)
|
||||
)
|
||||
for measurement in sample["measurements"]
|
||||
)
|
||||
for prefix in AGENT_HEAVY_MODULE_PREFIXES
|
||||
}
|
||||
peak_text = (
|
||||
", ".join(
|
||||
f"{prefix}={count}" for prefix, count in peaks.items() if count
|
||||
)
|
||||
or "全部 0"
|
||||
)
|
||||
lines.append(
|
||||
f"| {sample.get('scenario', DEFAULT_SCENARIO)} | "
|
||||
f"{sample['variant']} | {sample['sample_index']} | {peak_text} |"
|
||||
)
|
||||
|
||||
lines.extend(["", "## 中位数对照", ""])
|
||||
for scenario in scenarios:
|
||||
scenario_samples = [
|
||||
|
||||
@@ -36,6 +36,29 @@ def snapshot(xvfb_count: int, xvfb_pss_kib: int = 0) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def agent_snapshot(prefix_counts: dict[str, int], xvfb_count: int = 0) -> dict:
|
||||
"""构造包含 Agent 模块哨兵的场景边界快照。"""
|
||||
return {
|
||||
"engine": {
|
||||
"working_set_bytes": 500 * 1024 * 1024,
|
||||
"network_rx_bytes": 1024,
|
||||
"network_tx_bytes": 512,
|
||||
},
|
||||
"processes": {
|
||||
"main_python": {
|
||||
"pss_kib": 400 * 1024,
|
||||
"uss_kib": 390 * 1024,
|
||||
"threads": 8,
|
||||
},
|
||||
"xvfb": {"count": xvfb_count, "pss_kib": 0},
|
||||
},
|
||||
"modules": {
|
||||
"count": 3000,
|
||||
"prefix_counts": prefix_counts,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def managed_resource(before_generation: int, after_generation: int) -> dict:
|
||||
"""构造 host.display single-flight 观测。"""
|
||||
observations = []
|
||||
@@ -141,12 +164,54 @@ def test_browser_scenario_uses_isolated_resource_and_result_names(
|
||||
)
|
||||
|
||||
|
||||
def test_browser_scenario_rejects_before_without_touching_docker() -> None:
|
||||
"""旧基线不具备 SDK/display 冷启动不变量,非默认场景只接受 After。"""
|
||||
def test_agent_scenarios_are_explicit_and_keep_idle_prefix_contract() -> None:
|
||||
"""PERF-003 暴露完整哨兵,并把 Schema 基线排除在归零门禁外。"""
|
||||
harness = load_module(
|
||||
"moviepilot_perf_agent_cli",
|
||||
PERF_DIR / "moviepilot_docker_ab.py",
|
||||
)
|
||||
expected_prefixes = {
|
||||
"app.agent.orchestrator",
|
||||
"app.agent.callback",
|
||||
"app.agent.llm.helper",
|
||||
"app.agent.tools.base",
|
||||
"app.agent.tools.catalog",
|
||||
"app.agent.tools.factory",
|
||||
"app.agent.tools.impl",
|
||||
"langgraph",
|
||||
"langchain",
|
||||
"langchain_core",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"google.genai",
|
||||
"boto3",
|
||||
"botocore",
|
||||
}
|
||||
|
||||
assert set(harness.AGENT_SCENARIOS) == {
|
||||
"agent-disabled-router",
|
||||
"agent-tool-catalog",
|
||||
}
|
||||
assert set(harness.AGENT_HEAVY_MODULE_PREFIXES) == expected_prefixes
|
||||
assert expected_prefixes.issubset(harness.MODULE_PREFIXES)
|
||||
assert set(harness.AGENT_SCHEMA_BASELINE_PREFIXES) == {
|
||||
"langchain",
|
||||
"langchain_core",
|
||||
}
|
||||
assert not set(harness.AGENT_SCHEMA_BASELINE_PREFIXES).intersection(
|
||||
harness.AGENT_NONMATERIALIZATION_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["browser-headless", "agent-disabled-router"])
|
||||
def test_non_default_scenario_rejects_before_without_touching_docker(
|
||||
scenario: str,
|
||||
) -> None:
|
||||
"""旧基线不具备候选公共合同,所有非默认场景只接受 After。"""
|
||||
harness = load_module(
|
||||
"moviepilot_perf_after_only", PERF_DIR / "moviepilot_docker_ab.py"
|
||||
)
|
||||
args = argparse.Namespace(scenario="browser-headless", variant="before")
|
||||
args = argparse.Namespace(scenario=scenario, variant="before")
|
||||
|
||||
with pytest.raises(harness.HarnessError, match="After"):
|
||||
harness.command_sample(args)
|
||||
@@ -222,6 +287,132 @@ def test_activation_validation_enforces_headless_and_headed_invariants() -> None
|
||||
assert invalid["single_flight"]["passed"] is False
|
||||
|
||||
|
||||
def test_agent_activation_validation_enforces_lazy_boundaries() -> None:
|
||||
"""禁用态延迟重 Agent 域,首次目录只允许工具域物化。"""
|
||||
harness = load_module(
|
||||
"moviepilot_perf_agent_validation",
|
||||
PERF_DIR / "moviepilot_docker_ab.py",
|
||||
)
|
||||
zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES}
|
||||
catalog_loaded = dict(zero)
|
||||
catalog_loaded["app.agent.tools.base"] = 1
|
||||
catalog_loaded["app.agent.tools.catalog"] = 1
|
||||
catalog_loaded["app.agent.tools.factory"] = 1
|
||||
catalog_loaded["app.agent.tools.impl"] = 82
|
||||
schema_baseline = dict(zero)
|
||||
schema_baseline["langchain_core"] = 5
|
||||
router_marker = {
|
||||
"scenario": "agent-disabled-router",
|
||||
"pid": 42,
|
||||
"success": True,
|
||||
"agent": {
|
||||
"success": True,
|
||||
"observations": {
|
||||
"before": {
|
||||
"available": True,
|
||||
"tool_factory_materialized": False,
|
||||
},
|
||||
"after": {
|
||||
"available": True,
|
||||
"tool_factory_materialized": False,
|
||||
},
|
||||
},
|
||||
"router_openapi": {
|
||||
"success": True,
|
||||
"ai_agent_enable": False,
|
||||
"missing_routes": [],
|
||||
"missing_openapi_paths": [],
|
||||
"route_count": 200,
|
||||
"openapi_path_count": 180,
|
||||
"openapi_sha256": "schema",
|
||||
},
|
||||
},
|
||||
}
|
||||
catalog_marker = {
|
||||
"scenario": "agent-tool-catalog",
|
||||
"pid": 42,
|
||||
"success": True,
|
||||
"agent": {
|
||||
"success": True,
|
||||
"observations": {
|
||||
"before": {
|
||||
"available": True,
|
||||
"tool_factory_materialized": False,
|
||||
},
|
||||
"after": {
|
||||
"available": True,
|
||||
"tool_factory_materialized": True,
|
||||
},
|
||||
},
|
||||
"tool_catalog": {
|
||||
"success": True,
|
||||
"tool_count": 82,
|
||||
"schema_count": 82,
|
||||
"catalog_entry_count": 82,
|
||||
"collision_names": [],
|
||||
"plugin_revision": 0,
|
||||
"factory_revision": "factory-revision",
|
||||
"schemas_sha256": "schemas",
|
||||
"schema_digests_complete": True,
|
||||
"repeat_tool_count": 82,
|
||||
"repeat_stable": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
router = harness.evaluate_agent_activation(
|
||||
"agent-disabled-router",
|
||||
agent_snapshot(schema_baseline),
|
||||
agent_snapshot(schema_baseline),
|
||||
router_marker,
|
||||
expected_pid=42,
|
||||
)
|
||||
catalog = harness.evaluate_agent_activation(
|
||||
"agent-tool-catalog",
|
||||
agent_snapshot(zero),
|
||||
agent_snapshot(catalog_loaded),
|
||||
catalog_marker,
|
||||
expected_pid=42,
|
||||
)
|
||||
invalid_loaded = dict(catalog_loaded)
|
||||
invalid_loaded["app.agent.orchestrator"] = 1
|
||||
invalid = harness.evaluate_agent_activation(
|
||||
"agent-tool-catalog",
|
||||
agent_snapshot(zero),
|
||||
agent_snapshot(invalid_loaded),
|
||||
catalog_marker,
|
||||
expected_pid=42,
|
||||
)
|
||||
callback_loaded = dict(catalog_loaded)
|
||||
callback_loaded["app.agent.callback"] = 1
|
||||
invalid_callback = harness.evaluate_agent_activation(
|
||||
"agent-tool-catalog",
|
||||
agent_snapshot(zero),
|
||||
agent_snapshot(callback_loaded),
|
||||
catalog_marker,
|
||||
expected_pid=42,
|
||||
)
|
||||
network_post = agent_snapshot(catalog_loaded)
|
||||
network_post["engine"]["network_tx_bytes"] += 1
|
||||
invalid_network = harness.evaluate_agent_activation(
|
||||
"agent-tool-catalog",
|
||||
agent_snapshot(zero),
|
||||
network_post,
|
||||
catalog_marker,
|
||||
expected_pid=42,
|
||||
)
|
||||
|
||||
assert router["passed"] is True
|
||||
assert router["action"]["openapi_path_count"] == 180
|
||||
assert catalog["passed"] is True
|
||||
assert catalog["revision"]["factory"] == "factory-revision"
|
||||
assert invalid["passed"] is False
|
||||
assert any("非目录" in error for error in invalid["errors"])
|
||||
assert invalid_callback["passed"] is False
|
||||
assert invalid_network["passed"] is False
|
||||
assert any("网络" in error for error in invalid_network["errors"])
|
||||
|
||||
|
||||
def test_sitecustomize_acquires_headed_display_concurrently_in_same_process() -> None:
|
||||
"""headed probe 并发走公开 SDK 冷启动,并只保留一个上下文。"""
|
||||
probe = load_module(
|
||||
@@ -283,6 +474,161 @@ def test_sitecustomize_headless_uses_one_headless_context() -> None:
|
||||
assert result["single_flight_probe"]["requested"] is False
|
||||
|
||||
|
||||
def test_sitecustomize_router_probe_generates_complete_openapi_without_http() -> None:
|
||||
"""禁用态探针直接读取主进程 app,不发起 HTTP 或外部请求。"""
|
||||
probe = load_module(
|
||||
"moviepilot_perf_sitecustomize_router",
|
||||
PERF_DIR / "instrument" / "sitecustomize.py",
|
||||
)
|
||||
required_paths = [
|
||||
"/api/v1/message/agent/stream",
|
||||
"/api/v1/message/agent/sessions",
|
||||
"/api/v1/openai/v1/chat/completions",
|
||||
"/api/v1/openai/v1/responses",
|
||||
"/api/v1/anthropic/v1/messages",
|
||||
"/api/v1/llm/manage",
|
||||
"/api/v1/mcp",
|
||||
"/api/v1/mcp/tools",
|
||||
]
|
||||
|
||||
class FakeApp:
|
||||
"""只实现 Router/OpenAPI 探针使用的 FastAPI 合同。"""
|
||||
|
||||
routes = [SimpleNamespace(path=path) for path in required_paths]
|
||||
|
||||
@staticmethod
|
||||
def openapi() -> dict:
|
||||
return {
|
||||
"info": {"title": "MoviePilot", "version": "v3"},
|
||||
"paths": {path: {"get": {}} for path in required_paths},
|
||||
}
|
||||
|
||||
result = probe._probe_router_openapi(
|
||||
app_instance=FakeApp(),
|
||||
settings_object=SimpleNamespace(AI_AGENT_ENABLE=False),
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["route_count"] == len(required_paths)
|
||||
assert result["openapi_path_count"] == len(required_paths)
|
||||
assert result["missing_routes"] == []
|
||||
assert result["missing_openapi_paths"] == []
|
||||
|
||||
|
||||
def test_sitecustomize_tool_catalog_probe_records_schema_and_revisions() -> None:
|
||||
"""首次目录探针保留工具数、Schema 摘要和双 revision。"""
|
||||
probe = load_module(
|
||||
"moviepilot_perf_sitecustomize_catalog",
|
||||
PERF_DIR / "instrument" / "sitecustomize.py",
|
||||
)
|
||||
definitions = [
|
||||
SimpleNamespace(
|
||||
name="query_media",
|
||||
input_schema={"type": "object", "properties": {}},
|
||||
),
|
||||
SimpleNamespace(
|
||||
name="add_subscribe",
|
||||
input_schema={"type": "object", "properties": {"title": {}}},
|
||||
),
|
||||
]
|
||||
catalog = SimpleNamespace(
|
||||
entries=(
|
||||
SimpleNamespace(
|
||||
name="query_media",
|
||||
source="builtin",
|
||||
schema_digest="a" * 64,
|
||||
),
|
||||
SimpleNamespace(
|
||||
name="add_subscribe",
|
||||
source="builtin",
|
||||
schema_digest="b" * 64,
|
||||
),
|
||||
),
|
||||
collisions={},
|
||||
plugin_revision=7,
|
||||
factory_revision="factory-revision",
|
||||
)
|
||||
|
||||
class FakeManager:
|
||||
"""按真实管理器合同在 list_tools 后发布 catalog。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.catalog = None
|
||||
|
||||
def list_tools(self):
|
||||
self.catalog = catalog
|
||||
return definitions
|
||||
|
||||
result = probe._probe_tool_catalog(manager=FakeManager())
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["tool_count"] == 2
|
||||
assert result["schema_count"] == 2
|
||||
assert result["plugin_revision"] == 7
|
||||
assert result["factory_revision"] == "factory-revision"
|
||||
assert len(result["schemas_sha256"]) == 64
|
||||
assert len(result["catalog_sha256"]) == 64
|
||||
assert result["source_counts"] == {"builtin": 2}
|
||||
assert result["schema_digests_complete"] is True
|
||||
assert result["repeat_catalog_same_object"] is True
|
||||
assert result["repeat_revision_unchanged"] is True
|
||||
assert result["repeat_stable"] is True
|
||||
|
||||
|
||||
def test_sitecustomize_agent_scenario_records_before_and_after_observations(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Agent 场景在同一目标解释器内记录模块与 materialization 边界。"""
|
||||
probe = load_module(
|
||||
"moviepilot_perf_sitecustomize_agent",
|
||||
PERF_DIR / "instrument" / "sitecustomize.py",
|
||||
)
|
||||
module_observations = iter(
|
||||
[
|
||||
{"total_modules": 100, "prefix_counts": {}, "matching_modules": []},
|
||||
{
|
||||
"total_modules": 190,
|
||||
"prefix_counts": {
|
||||
"app.agent.tools.factory": 1,
|
||||
"app.agent.tools.impl": 82,
|
||||
},
|
||||
"matching_modules": ["app.agent.tools.factory"],
|
||||
},
|
||||
]
|
||||
)
|
||||
runtime_observations = iter(
|
||||
[
|
||||
{"available": True, "tool_factory_materialized": False},
|
||||
{"available": True, "tool_factory_materialized": True},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
probe,
|
||||
"_agent_module_observation",
|
||||
lambda: next(module_observations),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
probe,
|
||||
"_probe_tool_catalog",
|
||||
lambda manager=None: {
|
||||
"success": True,
|
||||
"tool_count": 82,
|
||||
"schema_count": 82,
|
||||
},
|
||||
)
|
||||
|
||||
result = probe._activate_agent_scenario(
|
||||
"agent-tool-catalog",
|
||||
runtime_reader=lambda: next(runtime_observations),
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["observations"]["before"]["tool_factory_materialized"] is False
|
||||
assert result["observations"]["after"]["tool_factory_materialized"] is True
|
||||
assert result["modules"]["before"]["total_modules"] == 100
|
||||
assert result["modules"]["after"]["total_modules"] == 190
|
||||
|
||||
|
||||
def test_sitecustomize_serializes_managed_resource_facade(monkeypatch) -> None:
|
||||
"""进程探针按公开只读 facade 记录 generation 与 activate observation。"""
|
||||
observation = SimpleNamespace(
|
||||
@@ -357,6 +703,28 @@ def test_sitecustomize_signal_worker_publishes_atomic_marker(tmp_path: Path) ->
|
||||
assert not list(tmp_path.glob("*.tmp"))
|
||||
|
||||
|
||||
def test_sitecustomize_signal_worker_dispatches_agent_scenario(tmp_path: Path) -> None:
|
||||
"""SIGUSR2 worker 对 Agent 场景也在当前 PID 发布完整 marker。"""
|
||||
probe = load_module(
|
||||
"moviepilot_perf_sitecustomize_agent_marker",
|
||||
PERF_DIR / "instrument" / "sitecustomize.py",
|
||||
)
|
||||
probe._OUTPUT_DIR = str(tmp_path)
|
||||
probe._SCENARIO = "agent-disabled-router"
|
||||
probe._activate_agent_scenario = lambda scenario: {
|
||||
"success": scenario == "agent-disabled-router"
|
||||
}
|
||||
|
||||
probe._run_activation()
|
||||
|
||||
marker_path = tmp_path / f"activation-{os.getpid()}.json"
|
||||
payload = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
assert payload["pid"] == os.getpid()
|
||||
assert payload["scenario"] == "agent-disabled-router"
|
||||
assert payload["agent"]["success"] is True
|
||||
assert "browser" not in payload
|
||||
|
||||
|
||||
def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> None:
|
||||
"""非默认场景报告包含激活证据,并按场景隔离中位数。"""
|
||||
harness = load_module(
|
||||
@@ -428,3 +796,73 @@ def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> No
|
||||
assert "Single-flight" in report
|
||||
assert "### `browser-headed`" in report
|
||||
assert "1.25" in report
|
||||
|
||||
|
||||
def test_markdown_reports_agent_observation_revision_and_sentinel() -> None:
|
||||
"""Agent 场景报告展示物化边界、revision 与定时哨兵峰值。"""
|
||||
harness = load_module(
|
||||
"moviepilot_perf_agent_report",
|
||||
PERF_DIR / "moviepilot_docker_ab.py",
|
||||
)
|
||||
zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES}
|
||||
loaded = dict(zero)
|
||||
loaded["app.agent.tools.factory"] = 1
|
||||
loaded["app.agent.tools.impl"] = 82
|
||||
pre = agent_snapshot(zero)
|
||||
post = agent_snapshot(loaded)
|
||||
activation = {
|
||||
"worker_elapsed_seconds": 2.5,
|
||||
"pre": pre,
|
||||
"post": post,
|
||||
"marker": {"success": True},
|
||||
"validation": {
|
||||
"passed": True,
|
||||
"observed": {
|
||||
"prefix_before": zero,
|
||||
"prefix_after": loaded,
|
||||
"tool_factory_materialized_before": False,
|
||||
"tool_factory_materialized_after": True,
|
||||
},
|
||||
"action": {
|
||||
"tool_count": 82,
|
||||
"schema_count": 82,
|
||||
"repeat_stable": True,
|
||||
"collision_names": [],
|
||||
},
|
||||
"revision": {"plugin": 7, "factory": "1234567890abcdef"},
|
||||
},
|
||||
}
|
||||
sample = {
|
||||
"scenario": "agent-tool-catalog",
|
||||
"variant": "after",
|
||||
"sample_index": 1,
|
||||
"http_ready_seconds": 7.0,
|
||||
"activation": activation,
|
||||
"measurements": [
|
||||
{
|
||||
"target_minute": 1.0,
|
||||
"engine": post["engine"],
|
||||
"processes": post["processes"],
|
||||
"modules": {
|
||||
"count": 3082,
|
||||
"prefix_counts": loaded,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
build = {
|
||||
"campaign": "fake",
|
||||
"platform": "linux/arm64",
|
||||
"before_commit": "before",
|
||||
"after_commit": "after",
|
||||
"substrate": {"reference": "frozen"},
|
||||
}
|
||||
|
||||
report = harness.build_markdown_report(build, None, [sample])
|
||||
|
||||
assert "## Agent 场景动作" in report
|
||||
assert "False→True" in report
|
||||
assert "82/82; repeat=Y; collision=0" in report
|
||||
assert "7/1234567890ab" in report
|
||||
assert "## Agent 模块哨兵" in report
|
||||
assert "app.agent.tools.impl=82" in report
|
||||
|
||||
Reference in New Issue
Block a user