refactor(agent): 按需加载 Agent 运行时 (#6336)

This commit is contained in:
InfinityPacer
2026-08-16 19:25:24 +08:00
committed by GitHub
parent e5f0c53069
commit 5b367011c5
56 changed files with 5059 additions and 628 deletions
+56
View File
@@ -81,6 +81,62 @@ Xvfb,因此不能用同一个 `0 → 0` / `0 → 1` 不变量衡量。三轮 B
- 非默认场景结果保存在 `samples/<scenario>/<variant>-<index>/`,可与同 campaign 的 idle 样本并存,
Markdown 中位数会按场景分组,不会混算。
## Agent 惰性物化场景
PERF-003 在既有 `AI_AGENT_ENABLE=false` 固定配置下增加两个 After-only 场景。探针只向主 MoviePilot
Python 进程发送信号;OpenAPI 生成和工具目录构造均发生在该解释器内,不通过 `docker exec` 启动
第二个 Python,也不调用真实 Agent、LLM provider 或外部 MCP。
先以 `f2e548e1` 冻结 Before,候选提交完成后把 `AFTER_COMMIT` 替换为其精确 commit
```bash
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-003 \
build --before-ref f2e548e1 --after-ref AFTER_COMMIT
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-003 \
seed --browser-source-volume mp-perf-v3-browser-seed --replace
```
正式 idle-default 三组 A/B 仍使用原 `run` 合同;下面两个动作场景在同一 build/seed 后单独采 After
不会覆盖 idle 结果:
```bash
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-003 \
run --before-ref f2e548e1 --after-ref AFTER_COMMIT \
--browser-source-volume mp-perf-v3-browser-seed \
--points 1,5,10,30 --replace --keep-resources
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-003 \
sample --variant after --index 1 --scenario agent-disabled-router --points 1,5,10,30
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-003 \
sample --variant after --index 2 --scenario agent-tool-catalog --points 1,5,10,30
```
- `agent-disabled-router`:直接从主进程 FastAPI app 生成完整 OpenAPI,确认 Agent、LLM、MCP、OpenAI、
Anthropic 路由在禁用态仍存在,同时 callback、LLM helper、工具域、orchestrator、LangGraph 和 provider SDK
前后保持 0,工具工厂不物化;
- `agent-tool-catalog`:通过主进程已有的 `moviepilot_tool_manager.list_tools()` 首次构建现有工具目录和 JSON Schema
要求动作前工具域未物化,动作后仅工具 base/catalog/factory/impl 物化;目录还必须无身份碰撞、Schema
digest 完整,重复读取复用同一 snapshot/revision。结果记录工具数、Schema 摘要、plugin revision 与
factory revision
- 固定哨兵覆盖 `app.agent.orchestrator``app.agent.callback``app.agent.llm.helper`、工具
`base/catalog/factory/impl``langgraph``langchain``langchain_core``openai``anthropic`
`google.genai``boto3``botocore`。其中 `langchain/langchain_core` 可能由完整 Schema 聚合形成既有
基线,只记录数量与变化,不作为禁用态归零门禁;
- JSON 保留动作前后 Engine、PSS/USS、线程、完整 `sys.modules`、materialization observation、revision、
网络累计值和浏览器卷指纹;动作前后容器网络收发必须为 0,Markdown 另汇总 Agent 场景与各定时点的
模块哨兵峰值;
- 启用态 Agent 生命周期不会在该无凭据场景中伪造。现有 `get_running_agent_manager()` 是严格只读、
non-materializing 的运行态 getter`begin_agent_shutdown()` 也只是关闭轴;二者都不是安全启用入口。
启用态必须由正式 startup/service lifecycle 驱动,只有宿主形成明确不创建 provider/client、不会外联的
公共初始化合同后,才适合加入同一测量门禁。
## 完整三组 A/B
```bash
+258 -2
View File
@@ -12,6 +12,20 @@ import sys
_OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR")
_SCENARIO = os.environ.get("MP_PERF_SCENARIO", "idle-default")
_ACTIVATION_TIMEOUT = float(os.environ.get("MP_PERF_ACTIVATION_TIMEOUT", "120"))
_AGENT_SCENARIOS = {"agent-disabled-router", "agent-tool-catalog"}
_AGENT_HEAVY_MODULE_PREFIXES = tuple(
prefix
for prefix in os.environ.get(
"MP_PERF_AGENT_MODULE_PREFIXES",
(
"app.agent.orchestrator,app.agent.callback,app.agent.llm.helper,"
"app.agent.tools.base,app.agent.tools.catalog,"
"app.agent.tools.factory,app.agent.tools.impl,langgraph,langchain,"
"langchain_core,openai,anthropic,google.genai,boto3,botocore"
),
).split(",")
if prefix
)
_snapshot_index = 0
_activation_started = False
_browser_resources: list[object] = []
@@ -112,6 +126,244 @@ def _enum_value(value):
return getattr(value, "value", value)
def _stable_digest(value: object) -> str:
"""计算不依赖对象地址的 JSON 摘要。"""
import hashlib
import json
content = json.dumps(
value,
ensure_ascii=False,
allow_nan=False,
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(content.encode("utf-8")).hexdigest()
def _agent_module_observation() -> dict[str, object]:
"""记录 Agent 重模块在目标解释器中的精确加载状态。"""
prefix_counts = {
prefix: sum(
1
for module_name in sys.modules
if module_name == prefix or module_name.startswith(f"{prefix}.")
)
for prefix in _AGENT_HEAVY_MODULE_PREFIXES
}
matching_modules = sorted(
module_name
for module_name in sys.modules
if any(
module_name == prefix or module_name.startswith(f"{prefix}.")
for prefix in _AGENT_HEAVY_MODULE_PREFIXES
)
)
return {
"total_modules": len(sys.modules),
"prefix_counts": prefix_counts,
"matching_modules": matching_modules,
"matching_sha256": _stable_digest(matching_modules),
}
def _read_agent_runtime() -> dict[str, object]:
"""读取轻量 Agent loader 的公开只读状态,不触发 capability 首用。"""
try:
from app.agent.runtime_loader import is_tool_factory_materialized
return {
"available": True,
"tool_factory_materialized": is_tool_factory_materialized(),
}
except Exception as error: # pragma: no cover - 候选未就绪或真实 runtime 错误
return {
"available": False,
"error_type": type(error).__name__,
"error": str(error),
}
def _probe_router_openapi(app_instance=None, settings_object=None) -> dict[str, object]:
"""在主进程中生成 OpenAPI,并验证禁用态 Agent 路由仍完整存在。"""
if app_instance is None:
from app.factory import app as app_instance
if settings_object is None:
from app.runtime.config import settings as settings_object
required_paths = (
"/api/v1/message/agent/stream",
"/api/v1/message/agent/sessions",
"/api/v1/openai/v1/chat/completions",
"/api/v1/openai/v1/responses",
"/api/v1/anthropic/v1/messages",
"/api/v1/llm/manage",
"/api/v1/mcp",
"/api/v1/mcp/tools",
)
schema = app_instance.openapi()
route_paths = sorted(
{
str(route.path)
for route in app_instance.routes
if getattr(route, "path", None)
}
)
openapi_paths = sorted((schema.get("paths") or {}).keys())
missing_routes = [path for path in required_paths if path not in route_paths]
missing_openapi_paths = [
path for path in required_paths if path not in openapi_paths
]
agent_enabled = bool(settings_object.AI_AGENT_ENABLE)
return {
"success": not agent_enabled
and not missing_routes
and not missing_openapi_paths,
"ai_agent_enable": agent_enabled,
"required_paths": list(required_paths),
"missing_routes": missing_routes,
"missing_openapi_paths": missing_openapi_paths,
"route_count": len(route_paths),
"openapi_path_count": len(openapi_paths),
"openapi_sha256": _stable_digest(schema),
"openapi_title": (schema.get("info") or {}).get("title"),
"openapi_version": (schema.get("info") or {}).get("version"),
}
def _probe_tool_catalog(manager=None) -> dict[str, object]:
"""通过稳定工具管理入口首次生成目录与 JSON Schema。"""
if manager is None:
from app.agent.tools.manager import moviepilot_tool_manager
manager = moviepilot_tool_manager
definitions = manager.list_tools()
catalog = manager.catalog
serialized_definitions = [
{
"name": definition.name,
"input_schema": definition.input_schema,
}
for definition in definitions
]
schema_count = sum(
isinstance(definition.input_schema, dict) for definition in definitions
)
entries = catalog.entries if catalog is not None else ()
collisions = catalog.collisions if catalog is not None else {}
source_counts: dict[str, int] = {}
serialized_entries = []
for entry in entries:
source_counts[entry.source] = source_counts.get(entry.source, 0) + 1
serialized_entries.append(
{
"name": entry.name,
"source": entry.source,
"schema_digest": entry.schema_digest,
}
)
first_catalog_sha256 = _stable_digest(serialized_entries)
first_schemas_sha256 = _stable_digest(serialized_definitions)
repeated_definitions = manager.list_tools()
repeated_catalog = manager.catalog
repeated_serialized_definitions = [
{
"name": definition.name,
"input_schema": definition.input_schema,
}
for definition in repeated_definitions
]
repeated_entries = repeated_catalog.entries if repeated_catalog is not None else ()
repeated_serialized_entries = [
{
"name": entry.name,
"source": entry.source,
"schema_digest": entry.schema_digest,
}
for entry in repeated_entries
]
repeated_catalog_sha256 = _stable_digest(repeated_serialized_entries)
repeated_schemas_sha256 = _stable_digest(repeated_serialized_definitions)
schema_digests_complete = all(
isinstance(entry.schema_digest, str) and len(entry.schema_digest) == 64
for entry in entries
)
repeat_revision_unchanged = bool(
catalog is not None
and repeated_catalog is not None
and repeated_catalog.plugin_revision == catalog.plugin_revision
and repeated_catalog.factory_revision == catalog.factory_revision
)
repeat_stable = bool(
repeated_catalog is catalog
and len(repeated_definitions) == len(definitions)
and repeated_catalog_sha256 == first_catalog_sha256
and repeated_schemas_sha256 == first_schemas_sha256
and repeat_revision_unchanged
)
return {
"success": bool(definitions)
and catalog is not None
and len(entries) == len(definitions)
and schema_count == len(definitions)
and not collisions
and schema_digests_complete
and repeat_stable,
"tool_count": len(definitions),
"schema_count": schema_count,
"catalog_entry_count": len(entries),
"collision_names": sorted(collisions),
"plugin_revision": catalog.plugin_revision if catalog is not None else None,
"factory_revision": catalog.factory_revision if catalog is not None else None,
"schemas_sha256": first_schemas_sha256,
"catalog_sha256": first_catalog_sha256,
"source_counts": source_counts,
"schema_digests_complete": schema_digests_complete,
"repeat_tool_count": len(repeated_definitions),
"repeat_catalog_same_object": repeated_catalog is catalog,
"repeat_catalog_sha256": repeated_catalog_sha256,
"repeat_schemas_sha256": repeated_schemas_sha256,
"repeat_revision_unchanged": repeat_revision_unchanged,
"repeat_stable": repeat_stable,
}
def _activate_agent_scenario(
scenario: str,
*,
app_instance=None,
settings_object=None,
tool_manager=None,
runtime_reader=None,
) -> dict[str, object]:
"""执行 Agent 禁用态路由或首次工具目录的进程内场景。"""
if scenario not in _AGENT_SCENARIOS:
raise ValueError(f"场景不支持 Agent 激活:{scenario}")
runtime_reader = runtime_reader or _read_agent_runtime
modules_before = _agent_module_observation()
runtime_before = runtime_reader()
if scenario == "agent-disabled-router":
action = _probe_router_openapi(
app_instance=app_instance,
settings_object=settings_object,
)
else:
action = _probe_tool_catalog(manager=tool_manager)
modules_after = _agent_module_observation()
runtime_after = runtime_reader()
return {
"requested": True,
"action": scenario.removeprefix("agent-"),
"success": bool(action.get("success")),
"modules": {"before": modules_before, "after": modules_after},
"observations": {"before": runtime_before, "after": runtime_after},
"router_openapi": action if scenario == "agent-disabled-router" else None,
"tool_catalog": action if scenario == "agent-tool-catalog" else None,
}
def _read_display_runtime() -> dict[str, object]:
"""读取 host.display 的只读状态和观测,不触发资源激活。"""
try:
@@ -338,8 +590,12 @@ def _run_activation() -> None:
"started_at": _utc_now(),
}
try:
result["browser"] = _activate_browser_scenario(_SCENARIO)
result["success"] = bool(result["browser"]["success"])
if _SCENARIO in _AGENT_SCENARIOS:
result["agent"] = _activate_agent_scenario(_SCENARIO)
result["success"] = bool(result["agent"]["success"])
else:
result["browser"] = _activate_browser_scenario(_SCENARIO)
result["success"] = bool(result["browser"]["success"])
except Exception as error: # pragma: no cover - 真实集成错误由 marker 保存
result.update(
{
+366 -15
View File
@@ -32,7 +32,9 @@ DEFAULT_SUBSTRATE = (
)
DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed"
DEFAULT_SCENARIO = "idle-default"
SCENARIOS = (DEFAULT_SCENARIO, "browser-headless", "browser-headed")
BROWSER_SCENARIOS = ("browser-headless", "browser-headed")
AGENT_SCENARIOS = ("agent-disabled-router", "agent-tool-catalog")
SCENARIOS = (DEFAULT_SCENARIO, *BROWSER_SCENARIOS, *AGENT_SCENARIOS)
CAMPAIGN_LABEL = "org.moviepilot.perf.campaign"
ROLE_LABEL = "org.moviepilot.perf.role"
SOURCE_LABEL = "org.moviepilot.perf.source-commit"
@@ -43,6 +45,35 @@ CRITICAL_SUBSTRATE_PATHS = (
"scripts/uv-pip-compat.sh",
)
SEED_COMPATIBILITY_PATHS = ("database/versions",)
AGENT_HEAVY_MODULE_PREFIXES = (
"app.agent.orchestrator",
"app.agent.callback",
"app.agent.llm.helper",
"app.agent.tools.base",
"app.agent.tools.catalog",
"app.agent.tools.factory",
"app.agent.tools.impl",
"langgraph",
"langchain",
"langchain_core",
"openai",
"anthropic",
"google.genai",
"boto3",
"botocore",
)
AGENT_SCHEMA_BASELINE_PREFIXES = ("langchain", "langchain_core")
AGENT_NONMATERIALIZATION_PREFIXES = tuple(
prefix
for prefix in AGENT_HEAVY_MODULE_PREFIXES
if prefix not in AGENT_SCHEMA_BASELINE_PREFIXES
)
AGENT_TOOL_CATALOG_PREFIXES = (
"app.agent.tools.base",
"app.agent.tools.catalog",
"app.agent.tools.factory",
"app.agent.tools.impl",
)
MODULE_PREFIXES = (
"lark_oapi",
"slack_bolt",
@@ -50,12 +81,10 @@ MODULE_PREFIXES = (
"discord",
"plexapi",
"telebot",
"langgraph",
"langchain",
"app.agent",
"app.agent.orchestrator",
"app.agent.tools",
"app.modules",
*AGENT_HEAVY_MODULE_PREFIXES,
)
BALANCED_RUN_ORDER = (
("before", 1),
@@ -593,6 +622,7 @@ def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, s
"MP_PERF_ACTIVATION_TIMEOUT": str(
getattr(args, "activation_timeout", 180)
),
"MP_PERF_AGENT_MODULE_PREFIXES": ",".join(AGENT_HEAVY_MODULE_PREFIXES),
}
)
return environment
@@ -1133,7 +1163,7 @@ def capture_activation_snapshot(
output_dir: Path,
phase: str,
) -> dict[str, Any]:
"""采集浏览器激活边界的 Engine、进程和进程内 import 状态。"""
"""采集场景动作边界的 Engine、进程和进程内 import 状态。"""
engine = capture_engine_stats(container)
processes = capture_processes(container)
modules = capture_modules(container, output_dir, processes["main_python"])
@@ -1248,17 +1278,195 @@ def evaluate_browser_activation(
}
def activate_browser_scenario(
def _agent_prefix_counts(snapshot: dict[str, Any]) -> dict[str, int]:
"""从模块快照提取 PERF-003 Agent 重模块哨兵。"""
counts = snapshot["modules"].get("prefix_counts") or {}
return {
prefix: int(counts.get(prefix) or 0) for prefix in AGENT_HEAVY_MODULE_PREFIXES
}
def evaluate_agent_activation(
scenario: str,
pre: dict[str, Any],
post: dict[str, Any],
marker: dict[str, Any],
expected_pid: Optional[int] = None,
) -> dict[str, Any]:
"""验证禁用态路由与首次工具目录的惰性物化不变量。"""
agent = marker.get("agent") or {}
observations = agent.get("observations") or {}
runtime_before = observations.get("before") or {}
runtime_after = observations.get("after") or {}
prefix_before = _agent_prefix_counts(pre)
prefix_after = _agent_prefix_counts(post)
forbidden_before = {
prefix: prefix_before[prefix]
for prefix in AGENT_NONMATERIALIZATION_PREFIXES
if prefix_before[prefix]
}
pre_xvfb = pre["processes"]["xvfb"]
post_xvfb = post["processes"]["xvfb"]
network_delta = {
"rx_bytes": int(post["engine"]["network_rx_bytes"])
- int(pre["engine"]["network_rx_bytes"]),
"tx_bytes": int(post["engine"]["network_tx_bytes"])
- int(pre["engine"]["network_tx_bytes"]),
}
errors: list[str] = []
if marker.get("scenario") != scenario:
errors.append("进程内 marker 的场景与采集请求不一致")
if expected_pid is not None and marker.get("pid") != expected_pid:
errors.append("进程内 marker 不是目标 MoviePilot Python 进程写出")
if not marker.get("success") or not agent.get("success"):
errors.append("主 MoviePilot Python 进程未完成 Agent 场景动作")
if forbidden_before:
errors.append("Agent 场景动作前已经加载必须延迟物化的模块")
if pre_xvfb["count"] != 0 or post_xvfb["count"] != 0:
errors.append("Agent 场景不得物化 Xvfb")
if not runtime_before.get("available") or not runtime_after.get("available"):
errors.append("主进程未提供轻量 Agent runtime 只读观测")
if runtime_before.get("tool_factory_materialized") is not False:
errors.append("Agent 场景动作前工具工厂必须未物化")
if any(network_delta.values()):
errors.append("Agent 场景动作产生了容器网络收发")
revision = {"plugin": None, "factory": None}
action_summary: dict[str, Any]
if scenario == "agent-disabled-router":
router = agent.get("router_openapi") or {}
if router.get("ai_agent_enable") is not False:
errors.append("router/OpenAPI 场景必须运行在 AI_AGENT_ENABLE=false")
if router.get("missing_routes") or router.get("missing_openapi_paths"):
errors.append("禁用态缺少 Agent 相关 router 或 OpenAPI path")
forbidden_after = {
prefix: prefix_after[prefix]
for prefix in AGENT_NONMATERIALIZATION_PREFIXES
if prefix_after[prefix]
}
if forbidden_after:
errors.append("生成完整 OpenAPI 后加载了必须延迟物化的模块")
if runtime_after.get("tool_factory_materialized") is not False:
errors.append("生成完整 OpenAPI 不得物化工具工厂")
action_summary = {
"route_count": router.get("route_count"),
"openapi_path_count": router.get("openapi_path_count"),
"openapi_sha256": router.get("openapi_sha256"),
}
elif scenario == "agent-tool-catalog":
catalog = agent.get("tool_catalog") or {}
if prefix_after["app.agent.tools.factory"] < 1:
errors.append("首次工具目录动作后未加载工具工厂")
if prefix_after["app.agent.tools.impl"] < 1:
errors.append("首次工具目录动作后未加载工具实现")
allowed_prefixes = {
*AGENT_TOOL_CATALOG_PREFIXES,
*AGENT_SCHEMA_BASELINE_PREFIXES,
}
unexpected_prefixes = {
prefix: count
for prefix, count in prefix_after.items()
if prefix not in allowed_prefixes and count
}
if unexpected_prefixes:
errors.append("首次工具目录动作加载了非目录所需的 Agent/provider 重模块")
if runtime_after.get("tool_factory_materialized") is not True:
errors.append("首次工具目录动作后工具工厂未标记为已物化")
if (
not catalog.get("success")
or not catalog.get("tool_count")
or catalog.get("schema_count") != catalog.get("tool_count")
or catalog.get("catalog_entry_count") != catalog.get("tool_count")
or catalog.get("collision_names")
or not catalog.get("schema_digests_complete")
or not catalog.get("repeat_stable")
or catalog.get("repeat_tool_count") != catalog.get("tool_count")
):
errors.append("工具目录、JSON Schema 或重复读取稳定性不满足合同")
if catalog.get("plugin_revision") is None or not catalog.get(
"factory_revision"
):
errors.append("工具目录缺少 plugin/factory revision")
revision = {
"plugin": catalog.get("plugin_revision"),
"factory": catalog.get("factory_revision"),
}
action_summary = {
"tool_count": catalog.get("tool_count"),
"schema_count": catalog.get("schema_count"),
"schemas_sha256": catalog.get("schemas_sha256"),
"collision_names": catalog.get("collision_names") or [],
"repeat_stable": catalog.get("repeat_stable"),
}
else:
errors.append(f"未知 Agent 场景:{scenario}")
action_summary = {}
return {
"passed": not errors,
"errors": errors,
"expected": (
"router/OpenAPI 完整且必须延迟物化的模块保持 0"
if scenario == "agent-disabled-router"
else "首次工具目录后仅物化工具域及 Schema 基线"
),
"observed": {
"pre_xvfb_count": pre_xvfb["count"],
"post_xvfb_count": post_xvfb["count"],
"prefix_before": prefix_before,
"prefix_after": prefix_after,
"network_delta": network_delta,
"tool_factory_materialized_before": runtime_before.get(
"tool_factory_materialized"
),
"tool_factory_materialized_after": runtime_after.get(
"tool_factory_materialized"
),
},
"action": action_summary,
"revision": revision,
}
def evaluate_scenario_activation(
scenario: str,
pre: dict[str, Any],
post: dict[str, Any],
marker: dict[str, Any],
expected_pid: Optional[int] = None,
) -> dict[str, Any]:
"""按场景族分派外部采样验收。"""
if scenario in BROWSER_SCENARIOS:
return evaluate_browser_activation(
scenario,
pre,
post,
marker,
expected_pid=expected_pid,
)
if scenario in AGENT_SCENARIOS:
return evaluate_agent_activation(
scenario,
pre,
post,
marker,
expected_pid=expected_pid,
)
raise HarnessError(f"未知激活场景:{scenario}")
def activate_sample_scenario(
container,
output_dir: Path,
scenario: str,
timeout: float,
) -> dict[str, Any]:
"""通过 SIGUSR2 让目标 MoviePilot 解释器执行场景激活并回收 marker。"""
"""通过 SIGUSR2 让目标 MoviePilot 解释器执行动作并回收 marker。"""
pre = capture_activation_snapshot(container, output_dir, "pre-activation")
main_python = pre["processes"]["main_python"]
if not main_python:
raise HarnessError("未找到主 Python 进程,无法触发浏览器场景")
raise HarnessError("未找到主 Python 进程,无法触发测量场景")
marker_path = output_dir / "modules" / f"activation-{main_python['pid']}.json"
marker_path.unlink(missing_ok=True)
@@ -1274,12 +1482,12 @@ def activate_browser_scenario(
raise HarnessError("等待场景激活 marker 时容器提前退出")
time.sleep(0.05)
if not marker_path.exists():
raise HarnessError(f"浏览器场景激活{timeout:.0f}s 内未完成")
raise HarnessError(f"测量场景动作{timeout:.0f}s 内未完成")
marker_received_at = time.monotonic()
marker = json.loads(marker_path.read_text(encoding="utf-8"))
post = capture_activation_snapshot(container, output_dir, "post-activation")
validation = evaluate_browser_activation(
validation = evaluate_scenario_activation(
scenario,
pre,
post,
@@ -1304,7 +1512,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
"""执行一个隔离样本并在约定时间点采集完整指标。"""
scenario = getattr(args, "scenario", DEFAULT_SCENARIO)
if scenario != DEFAULT_SCENARIO and args.variant != "after":
raise HarnessError("浏览器激活场景只用于验证包含 app.sdk.browser 的 After 候选")
raise HarnessError("非默认场景只用于验证包含候选公共 API 的 After 版本")
client = require_docker_client()
build = load_build_manifest(args)
config_seed, browser_seed = require_seed_volumes(client, args)
@@ -1391,7 +1599,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
measurement_origin_at = settled_at
if scenario != DEFAULT_SCENARIO:
activation = activate_browser_scenario(
activation = activate_sample_scenario(
container,
output_dir,
scenario,
@@ -1614,8 +1822,13 @@ def build_markdown_report(
)
lines.append("| " + " | ".join(row) + " |")
activated_samples = [sample for sample in samples if sample.get("activation")]
if activated_samples:
browser_activated_samples = [
sample
for sample in samples
if sample.get("activation")
and sample.get("scenario", DEFAULT_SCENARIO) in BROWSER_SCENARIOS
]
if browser_activated_samples:
activation_headers = [
"场景",
"版本",
@@ -1644,7 +1857,7 @@ def build_markdown_report(
]
)
for sample in sorted(
activated_samples,
browser_activated_samples,
key=lambda item: (
item.get("scenario", DEFAULT_SCENARIO),
variant_order.get(item["variant"], 99),
@@ -1692,6 +1905,144 @@ def build_markdown_report(
]
lines.append("| " + " | ".join(activation_row) + " |")
agent_activated_samples = [
sample
for sample in samples
if sample.get("activation")
and sample.get("scenario", DEFAULT_SCENARIO) in AGENT_SCENARIOS
]
if agent_activated_samples:
agent_headers = [
"场景",
"样本",
"动作(s)",
"Pre/Post WS(MiB)",
"Pre/Post Python PSS(MiB)",
"Pre/Post sys.modules",
"Factory observation",
"模块哨兵 Pre",
"模块哨兵 Post",
"Router/OpenAPI 或 Tools/Schemas",
"Plugin/Factory revision",
"Action RX/TX Δ(KiB)",
"验收",
]
lines.extend(
[
"",
"## Agent 场景动作",
"",
"| " + " | ".join(agent_headers) + " |",
"| " + " | ".join(["---"] * len(agent_headers)) + " |",
]
)
def format_prefix_counts(counts: dict[str, int]) -> str:
"""仅展开已加载前缀,全部未加载时输出明确零状态。"""
loaded = [f"{prefix}={count}" for prefix, count in counts.items() if count]
return ", ".join(loaded) if loaded else "全部 0"
for sample in sorted(
agent_activated_samples,
key=lambda item: (
item.get("scenario", DEFAULT_SCENARIO),
item["sample_index"],
),
):
activation = sample["activation"]
pre = activation["pre"]
post = activation["post"]
validation = activation["validation"]
observed = validation["observed"]
action = validation["action"]
revision = validation["revision"]
pre_python = pre["processes"].get("main_python") or {}
post_python = post["processes"].get("main_python") or {}
if sample.get("scenario") == "agent-disabled-router":
action_result = (
f"{action.get('route_count')}/{action.get('openapi_path_count')}"
)
else:
action_result = (
f"{action.get('tool_count')}/{action.get('schema_count')}; "
f"repeat={'Y' if action.get('repeat_stable') else 'N'}; "
f"collision={len(action.get('collision_names') or [])}"
)
factory_revision = str(revision.get("factory") or "")
revision_result = (
f"{revision.get('plugin')}/{factory_revision[:12]}"
if factory_revision
else "不适用"
)
agent_row = [
sample.get("scenario", DEFAULT_SCENARIO),
str(sample["sample_index"]),
f"{float(activation.get('worker_elapsed_seconds') or 0):.2f}",
f"{format_mib(pre['engine']['working_set_bytes'])}/"
f"{format_mib(post['engine']['working_set_bytes'])}",
f"{format_kib_as_mib(pre_python.get('pss_kib'))}/"
f"{format_kib_as_mib(post_python.get('pss_kib'))}",
f"{pre['modules'].get('count')}/{post['modules'].get('count')}",
(
f"{observed.get('tool_factory_materialized_before')}"
f"{observed.get('tool_factory_materialized_after')}"
),
format_prefix_counts(observed.get("prefix_before") or {}),
format_prefix_counts(observed.get("prefix_after") or {}),
action_result,
revision_result,
(
f"{format_bytes_as_kib(post['engine']['network_rx_bytes'] - pre['engine']['network_rx_bytes'])}/"
f"{format_bytes_as_kib(post['engine']['network_tx_bytes'] - pre['engine']['network_tx_bytes'])}"
),
"通过" if validation["passed"] else "失败",
]
lines.append("| " + " | ".join(agent_row) + " |")
sentinel_samples = [sample for sample in samples if sample.get("measurements")]
if sentinel_samples:
lines.extend(
[
"",
"## Agent 模块哨兵",
"",
"每行记录该样本所有定时采样点的最大模块数;精确时间点数据保留在 JSON。",
"`langchain` 与 `langchain_core` 只记录 Schema 基线,不参与归零门禁。",
"",
"| 场景 | 版本 | 样本 | 重模块峰值 |",
"| --- | --- | --- | --- |",
]
)
for sample in sorted(
sentinel_samples,
key=lambda item: (
item.get("scenario", DEFAULT_SCENARIO),
variant_order.get(item["variant"], 99),
item["sample_index"],
),
):
peaks = {
prefix: max(
int(
measurement.get("modules", {})
.get("prefix_counts", {})
.get(prefix, 0)
)
for measurement in sample["measurements"]
)
for prefix in AGENT_HEAVY_MODULE_PREFIXES
}
peak_text = (
", ".join(
f"{prefix}={count}" for prefix, count in peaks.items() if count
)
or "全部 0"
)
lines.append(
f"| {sample.get('scenario', DEFAULT_SCENARIO)} | "
f"{sample['variant']} | {sample['sample_index']} | {peak_text} |"
)
lines.extend(["", "## 中位数对照", ""])
for scenario in scenarios:
scenario_samples = [
+441 -3
View File
@@ -36,6 +36,29 @@ def snapshot(xvfb_count: int, xvfb_pss_kib: int = 0) -> dict:
}
def agent_snapshot(prefix_counts: dict[str, int], xvfb_count: int = 0) -> dict:
"""构造包含 Agent 模块哨兵的场景边界快照。"""
return {
"engine": {
"working_set_bytes": 500 * 1024 * 1024,
"network_rx_bytes": 1024,
"network_tx_bytes": 512,
},
"processes": {
"main_python": {
"pss_kib": 400 * 1024,
"uss_kib": 390 * 1024,
"threads": 8,
},
"xvfb": {"count": xvfb_count, "pss_kib": 0},
},
"modules": {
"count": 3000,
"prefix_counts": prefix_counts,
},
}
def managed_resource(before_generation: int, after_generation: int) -> dict:
"""构造 host.display single-flight 观测。"""
observations = []
@@ -141,12 +164,54 @@ def test_browser_scenario_uses_isolated_resource_and_result_names(
)
def test_browser_scenario_rejects_before_without_touching_docker() -> None:
"""旧基线不具备 SDK/display 冷启动不变量,非默认场景只接受 After"""
def test_agent_scenarios_are_explicit_and_keep_idle_prefix_contract() -> None:
"""PERF-003 暴露完整哨兵,并把 Schema 基线排除在归零门禁外"""
harness = load_module(
"moviepilot_perf_agent_cli",
PERF_DIR / "moviepilot_docker_ab.py",
)
expected_prefixes = {
"app.agent.orchestrator",
"app.agent.callback",
"app.agent.llm.helper",
"app.agent.tools.base",
"app.agent.tools.catalog",
"app.agent.tools.factory",
"app.agent.tools.impl",
"langgraph",
"langchain",
"langchain_core",
"openai",
"anthropic",
"google.genai",
"boto3",
"botocore",
}
assert set(harness.AGENT_SCENARIOS) == {
"agent-disabled-router",
"agent-tool-catalog",
}
assert set(harness.AGENT_HEAVY_MODULE_PREFIXES) == expected_prefixes
assert expected_prefixes.issubset(harness.MODULE_PREFIXES)
assert set(harness.AGENT_SCHEMA_BASELINE_PREFIXES) == {
"langchain",
"langchain_core",
}
assert not set(harness.AGENT_SCHEMA_BASELINE_PREFIXES).intersection(
harness.AGENT_NONMATERIALIZATION_PREFIXES
)
@pytest.mark.parametrize("scenario", ["browser-headless", "agent-disabled-router"])
def test_non_default_scenario_rejects_before_without_touching_docker(
scenario: str,
) -> None:
"""旧基线不具备候选公共合同,所有非默认场景只接受 After。"""
harness = load_module(
"moviepilot_perf_after_only", PERF_DIR / "moviepilot_docker_ab.py"
)
args = argparse.Namespace(scenario="browser-headless", variant="before")
args = argparse.Namespace(scenario=scenario, variant="before")
with pytest.raises(harness.HarnessError, match="After"):
harness.command_sample(args)
@@ -222,6 +287,132 @@ def test_activation_validation_enforces_headless_and_headed_invariants() -> None
assert invalid["single_flight"]["passed"] is False
def test_agent_activation_validation_enforces_lazy_boundaries() -> None:
"""禁用态延迟重 Agent 域,首次目录只允许工具域物化。"""
harness = load_module(
"moviepilot_perf_agent_validation",
PERF_DIR / "moviepilot_docker_ab.py",
)
zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES}
catalog_loaded = dict(zero)
catalog_loaded["app.agent.tools.base"] = 1
catalog_loaded["app.agent.tools.catalog"] = 1
catalog_loaded["app.agent.tools.factory"] = 1
catalog_loaded["app.agent.tools.impl"] = 82
schema_baseline = dict(zero)
schema_baseline["langchain_core"] = 5
router_marker = {
"scenario": "agent-disabled-router",
"pid": 42,
"success": True,
"agent": {
"success": True,
"observations": {
"before": {
"available": True,
"tool_factory_materialized": False,
},
"after": {
"available": True,
"tool_factory_materialized": False,
},
},
"router_openapi": {
"success": True,
"ai_agent_enable": False,
"missing_routes": [],
"missing_openapi_paths": [],
"route_count": 200,
"openapi_path_count": 180,
"openapi_sha256": "schema",
},
},
}
catalog_marker = {
"scenario": "agent-tool-catalog",
"pid": 42,
"success": True,
"agent": {
"success": True,
"observations": {
"before": {
"available": True,
"tool_factory_materialized": False,
},
"after": {
"available": True,
"tool_factory_materialized": True,
},
},
"tool_catalog": {
"success": True,
"tool_count": 82,
"schema_count": 82,
"catalog_entry_count": 82,
"collision_names": [],
"plugin_revision": 0,
"factory_revision": "factory-revision",
"schemas_sha256": "schemas",
"schema_digests_complete": True,
"repeat_tool_count": 82,
"repeat_stable": True,
},
},
}
router = harness.evaluate_agent_activation(
"agent-disabled-router",
agent_snapshot(schema_baseline),
agent_snapshot(schema_baseline),
router_marker,
expected_pid=42,
)
catalog = harness.evaluate_agent_activation(
"agent-tool-catalog",
agent_snapshot(zero),
agent_snapshot(catalog_loaded),
catalog_marker,
expected_pid=42,
)
invalid_loaded = dict(catalog_loaded)
invalid_loaded["app.agent.orchestrator"] = 1
invalid = harness.evaluate_agent_activation(
"agent-tool-catalog",
agent_snapshot(zero),
agent_snapshot(invalid_loaded),
catalog_marker,
expected_pid=42,
)
callback_loaded = dict(catalog_loaded)
callback_loaded["app.agent.callback"] = 1
invalid_callback = harness.evaluate_agent_activation(
"agent-tool-catalog",
agent_snapshot(zero),
agent_snapshot(callback_loaded),
catalog_marker,
expected_pid=42,
)
network_post = agent_snapshot(catalog_loaded)
network_post["engine"]["network_tx_bytes"] += 1
invalid_network = harness.evaluate_agent_activation(
"agent-tool-catalog",
agent_snapshot(zero),
network_post,
catalog_marker,
expected_pid=42,
)
assert router["passed"] is True
assert router["action"]["openapi_path_count"] == 180
assert catalog["passed"] is True
assert catalog["revision"]["factory"] == "factory-revision"
assert invalid["passed"] is False
assert any("非目录" in error for error in invalid["errors"])
assert invalid_callback["passed"] is False
assert invalid_network["passed"] is False
assert any("网络" in error for error in invalid_network["errors"])
def test_sitecustomize_acquires_headed_display_concurrently_in_same_process() -> None:
"""headed probe 并发走公开 SDK 冷启动,并只保留一个上下文。"""
probe = load_module(
@@ -283,6 +474,161 @@ def test_sitecustomize_headless_uses_one_headless_context() -> None:
assert result["single_flight_probe"]["requested"] is False
def test_sitecustomize_router_probe_generates_complete_openapi_without_http() -> None:
"""禁用态探针直接读取主进程 app,不发起 HTTP 或外部请求。"""
probe = load_module(
"moviepilot_perf_sitecustomize_router",
PERF_DIR / "instrument" / "sitecustomize.py",
)
required_paths = [
"/api/v1/message/agent/stream",
"/api/v1/message/agent/sessions",
"/api/v1/openai/v1/chat/completions",
"/api/v1/openai/v1/responses",
"/api/v1/anthropic/v1/messages",
"/api/v1/llm/manage",
"/api/v1/mcp",
"/api/v1/mcp/tools",
]
class FakeApp:
"""只实现 Router/OpenAPI 探针使用的 FastAPI 合同。"""
routes = [SimpleNamespace(path=path) for path in required_paths]
@staticmethod
def openapi() -> dict:
return {
"info": {"title": "MoviePilot", "version": "v3"},
"paths": {path: {"get": {}} for path in required_paths},
}
result = probe._probe_router_openapi(
app_instance=FakeApp(),
settings_object=SimpleNamespace(AI_AGENT_ENABLE=False),
)
assert result["success"] is True
assert result["route_count"] == len(required_paths)
assert result["openapi_path_count"] == len(required_paths)
assert result["missing_routes"] == []
assert result["missing_openapi_paths"] == []
def test_sitecustomize_tool_catalog_probe_records_schema_and_revisions() -> None:
"""首次目录探针保留工具数、Schema 摘要和双 revision。"""
probe = load_module(
"moviepilot_perf_sitecustomize_catalog",
PERF_DIR / "instrument" / "sitecustomize.py",
)
definitions = [
SimpleNamespace(
name="query_media",
input_schema={"type": "object", "properties": {}},
),
SimpleNamespace(
name="add_subscribe",
input_schema={"type": "object", "properties": {"title": {}}},
),
]
catalog = SimpleNamespace(
entries=(
SimpleNamespace(
name="query_media",
source="builtin",
schema_digest="a" * 64,
),
SimpleNamespace(
name="add_subscribe",
source="builtin",
schema_digest="b" * 64,
),
),
collisions={},
plugin_revision=7,
factory_revision="factory-revision",
)
class FakeManager:
"""按真实管理器合同在 list_tools 后发布 catalog。"""
def __init__(self) -> None:
self.catalog = None
def list_tools(self):
self.catalog = catalog
return definitions
result = probe._probe_tool_catalog(manager=FakeManager())
assert result["success"] is True
assert result["tool_count"] == 2
assert result["schema_count"] == 2
assert result["plugin_revision"] == 7
assert result["factory_revision"] == "factory-revision"
assert len(result["schemas_sha256"]) == 64
assert len(result["catalog_sha256"]) == 64
assert result["source_counts"] == {"builtin": 2}
assert result["schema_digests_complete"] is True
assert result["repeat_catalog_same_object"] is True
assert result["repeat_revision_unchanged"] is True
assert result["repeat_stable"] is True
def test_sitecustomize_agent_scenario_records_before_and_after_observations(
monkeypatch,
) -> None:
"""Agent 场景在同一目标解释器内记录模块与 materialization 边界。"""
probe = load_module(
"moviepilot_perf_sitecustomize_agent",
PERF_DIR / "instrument" / "sitecustomize.py",
)
module_observations = iter(
[
{"total_modules": 100, "prefix_counts": {}, "matching_modules": []},
{
"total_modules": 190,
"prefix_counts": {
"app.agent.tools.factory": 1,
"app.agent.tools.impl": 82,
},
"matching_modules": ["app.agent.tools.factory"],
},
]
)
runtime_observations = iter(
[
{"available": True, "tool_factory_materialized": False},
{"available": True, "tool_factory_materialized": True},
]
)
monkeypatch.setattr(
probe,
"_agent_module_observation",
lambda: next(module_observations),
)
monkeypatch.setattr(
probe,
"_probe_tool_catalog",
lambda manager=None: {
"success": True,
"tool_count": 82,
"schema_count": 82,
},
)
result = probe._activate_agent_scenario(
"agent-tool-catalog",
runtime_reader=lambda: next(runtime_observations),
)
assert result["success"] is True
assert result["observations"]["before"]["tool_factory_materialized"] is False
assert result["observations"]["after"]["tool_factory_materialized"] is True
assert result["modules"]["before"]["total_modules"] == 100
assert result["modules"]["after"]["total_modules"] == 190
def test_sitecustomize_serializes_managed_resource_facade(monkeypatch) -> None:
"""进程探针按公开只读 facade 记录 generation 与 activate observation。"""
observation = SimpleNamespace(
@@ -357,6 +703,28 @@ def test_sitecustomize_signal_worker_publishes_atomic_marker(tmp_path: Path) ->
assert not list(tmp_path.glob("*.tmp"))
def test_sitecustomize_signal_worker_dispatches_agent_scenario(tmp_path: Path) -> None:
"""SIGUSR2 worker 对 Agent 场景也在当前 PID 发布完整 marker。"""
probe = load_module(
"moviepilot_perf_sitecustomize_agent_marker",
PERF_DIR / "instrument" / "sitecustomize.py",
)
probe._OUTPUT_DIR = str(tmp_path)
probe._SCENARIO = "agent-disabled-router"
probe._activate_agent_scenario = lambda scenario: {
"success": scenario == "agent-disabled-router"
}
probe._run_activation()
marker_path = tmp_path / f"activation-{os.getpid()}.json"
payload = json.loads(marker_path.read_text(encoding="utf-8"))
assert payload["pid"] == os.getpid()
assert payload["scenario"] == "agent-disabled-router"
assert payload["agent"]["success"] is True
assert "browser" not in payload
def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> None:
"""非默认场景报告包含激活证据,并按场景隔离中位数。"""
harness = load_module(
@@ -428,3 +796,73 @@ def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> No
assert "Single-flight" in report
assert "### `browser-headed`" in report
assert "1.25" in report
def test_markdown_reports_agent_observation_revision_and_sentinel() -> None:
"""Agent 场景报告展示物化边界、revision 与定时哨兵峰值。"""
harness = load_module(
"moviepilot_perf_agent_report",
PERF_DIR / "moviepilot_docker_ab.py",
)
zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES}
loaded = dict(zero)
loaded["app.agent.tools.factory"] = 1
loaded["app.agent.tools.impl"] = 82
pre = agent_snapshot(zero)
post = agent_snapshot(loaded)
activation = {
"worker_elapsed_seconds": 2.5,
"pre": pre,
"post": post,
"marker": {"success": True},
"validation": {
"passed": True,
"observed": {
"prefix_before": zero,
"prefix_after": loaded,
"tool_factory_materialized_before": False,
"tool_factory_materialized_after": True,
},
"action": {
"tool_count": 82,
"schema_count": 82,
"repeat_stable": True,
"collision_names": [],
},
"revision": {"plugin": 7, "factory": "1234567890abcdef"},
},
}
sample = {
"scenario": "agent-tool-catalog",
"variant": "after",
"sample_index": 1,
"http_ready_seconds": 7.0,
"activation": activation,
"measurements": [
{
"target_minute": 1.0,
"engine": post["engine"],
"processes": post["processes"],
"modules": {
"count": 3082,
"prefix_counts": loaded,
},
}
],
}
build = {
"campaign": "fake",
"platform": "linux/arm64",
"before_commit": "before",
"after_commit": "after",
"substrate": {"reference": "frozen"},
}
report = harness.build_markdown_report(build, None, [sample])
assert "## Agent 场景动作" in report
assert "False→True" in report
assert "82/82; repeat=Y; collision=0" in report
assert "7/1234567890ab" in report
assert "## Agent 模块哨兵" in report
assert "app.agent.tools.impl=82" in report