refactor(runtime): activate managed resources on demand (#6334)

This commit is contained in:
InfinityPacer
2026-08-16 16:44:45 +08:00
committed by GitHub
parent 7e851dbfa7
commit b8b59ae20a
27 changed files with 3068 additions and 116 deletions
+31
View File
@@ -50,6 +50,37 @@ ${PYTHON} scripts/perf/moviepilot_docker_ab.py \
开发 harness 时可以用小数分钟做短冒烟,例如 `--points 0,0.02`。正式数据必须保持
`1,5,10,30`
未指定 `--scenario` 时仍使用 `idle-default`,样本目录和 Docker 资源名称与既有命令保持一致。
## 浏览器激活场景
浏览器场景使用 campaign browser seed 的独立克隆卷,不直接挂载或写入固定来源卷,也不会在样本
阶段下载浏览器。容器保持 internal network;探针只发送信号,`app.sdk.browser` 的导入、浏览器上下文
创建和本地 `data:` 页面校验都发生在主 MoviePilot Python 进程中。
非默认浏览器场景用于候选实现的 After 激活门禁;Before 不具备新 SDK,且旧实现启动时已经常驻
Xvfb,因此不能用同一个 `0 → 0` / `0 → 1` 不变量衡量。三轮 Before/After 空载收益仍由默认
`run``idle-default` 场景完成,浏览器场景用三个隔离的 After sample 记录冷激活成本。
```bash
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-002-headless \
sample --variant after --index 1 --scenario browser-headless --points 1,5,10,30
../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \
--campaign v3-perf-002-headed \
sample --variant after --index 1 --scenario browser-headed --points 1,5,10,30
```
- `browser-headless`:一次真实 headless context 激活,要求 Xvfb `0 → 0`
- `browser-headed`:主进程内两个线程通过屏障并发调用
`launch_browser_context(headless=False)`;要求两个真实 SDK 冷启动调用成功、额外上下文关闭后只保留一个、
Capability observation 只有一个 `headed_browser_launch` generation/start,且 Xvfb `0 → 1`
- 激活完成后再开始 `1/5/10/30m` 计时,JSON 保留激活前后 Engine 网络、working set、进程
PSS/USS/RSS/线程、Xvfb 数量/PSS、`sys.modules` 和进程内 marker
- 非默认场景结果保存在 `samples/<scenario>/<variant>-<index>/`,可与同 campaign 的 idle 样本并存,
Markdown 中位数会按场景分组,不会混算。
## 完整三组 A/B
```bash
+343 -1
View File
@@ -1,12 +1,41 @@
"""MoviePilot Docker A/B 测量时使用的最小 ``sys.modules`` 快照探针。"""
"""MoviePilot Docker 测量进程内使用的最小诊断探针。"""
from __future__ import annotations
import os
import signal
import sys
# 场景激活依赖只在收到信号后加载,避免改变 idle-default 的 import 基线。
# pylint: disable=import-outside-toplevel
_OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR")
_SCENARIO = os.environ.get("MP_PERF_SCENARIO", "idle-default")
_ACTIVATION_TIMEOUT = float(os.environ.get("MP_PERF_ACTIVATION_TIMEOUT", "120"))
_snapshot_index = 0
_activation_started = False
_browser_resources: list[object] = []
def _utc_now() -> str:
"""返回稳定、可机器解析的 UTC 时间。"""
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat()
def _atomic_write_json(path, payload: dict[str, object]) -> None:
"""原子发布结果,避免采集端读取到半写 marker。"""
import json
path.parent.mkdir(parents=True, exist_ok=True)
temporary_path = path.with_suffix(f"{path.suffix}.tmp")
with temporary_path.open("w", encoding="utf-8") as output:
json.dump(payload, output, ensure_ascii=False, indent=2, sort_keys=True)
output.write("\n")
output.flush()
os.fsync(output.fileno())
os.replace(temporary_path, path)
def _dump_modules(_signum, _frame) -> None:
@@ -31,5 +60,318 @@ def _dump_modules(_signum, _frame) -> None:
os.replace(temporary_path, final_path)
def _launch_one_browser(
*,
index: int,
headless: bool,
launcher,
start_gate,
) -> tuple[dict[str, object], list[object]]:
"""启动一个本地 data URL 浏览器上下文并返回可序列化结果。"""
import time
if start_gate is not None:
start_gate.wait(timeout=min(_ACTIVATION_TIMEOUT, 10))
started_at = time.perf_counter()
retained: list[object] = []
result: dict[str, object] = {
"index": index,
"headless": headless,
"started_at_monotonic": started_at,
}
try:
context = launcher(headless=headless)
retained.append(context)
page = context.new_page()
retained.append(page)
page.goto("data:text/html,<title>MoviePilot Browser Probe</title>")
title = page.title()
result.update(
{
"success": title == "MoviePilot Browser Probe",
"page_title": title,
"context_type": type(context).__name__,
}
)
if not result["success"]:
result["error"] = "本地 data URL 标题校验失败"
except Exception as error: # pragma: no cover - 真实浏览器错误由 marker 保存
result.update(
{
"success": False,
"error_type": type(error).__name__,
"error": str(error),
}
)
result["elapsed_seconds"] = time.perf_counter() - started_at
return result, retained
def _enum_value(value):
"""把 runtime 枚举降为 JSON 标量。"""
return getattr(value, "value", value)
def _read_display_runtime() -> dict[str, object]:
"""读取 host.display 的只读状态和观测,不触发资源激活。"""
try:
from app.runtime.managed_resources import (
managed_resource_observations,
managed_resource_snapshot,
)
snapshot = managed_resource_snapshot("host.display")
observations = managed_resource_observations("host.display")
return {
"available": True,
"snapshot": {
"capability_id": snapshot.capability_id,
"materialization": _enum_value(snapshot.materialization),
"lifecycle": _enum_value(snapshot.lifecycle),
"generation": snapshot.generation,
"visible": snapshot.visible,
"error": snapshot.error,
},
"observations": [
{
"capability_id": item.capability_id,
"generation": item.generation,
"operation": item.operation,
"outcome": item.outcome,
"reason": item.reason,
"materialization": _enum_value(item.materialization),
"lifecycle": _enum_value(item.lifecycle),
"duration_ms": item.duration_ms,
"error": item.error,
}
for item in observations
],
}
except Exception as error: # pragma: no cover - 核心未就绪或真实 runtime 错误
return {
"available": False,
"error_type": type(error).__name__,
"error": str(error),
}
def _close_browser_resources(resources: list[object]) -> list[dict[str, str]]:
"""逆序关闭一次探针创建的页面与上下文,并返回可序列化错误。"""
errors: list[dict[str, str]] = []
for resource in reversed(resources):
close = getattr(resource, "close", None)
if not callable(close):
continue
try:
close()
except Exception as error: # pragma: no cover - 真实浏览器错误由 marker 保存
errors.append(
{
"resource_type": type(resource).__name__,
"error_type": type(error).__name__,
"error": str(error),
}
)
return errors
def _activate_browser_scenario(
scenario: str,
launcher=None,
) -> dict[str, object]:
"""通过公开 SDK 执行真实浏览器激活,headed 使用并发冷启动探针。"""
import threading
import time
if scenario not in {"browser-headless", "browser-headed"}:
raise ValueError(f"场景不支持浏览器激活:{scenario}")
if launcher is None:
from app.sdk.browser import launch_browser_context
launcher = launch_browser_context
display_before = _read_display_runtime()
headless = scenario == "browser-headless"
concurrency = 1 if headless else 2
launch_results: list[dict[str, object] | None] = [None] * concurrency
cleanup_error_slots: list[list[dict[str, str]]] = [
[] for _index in range(concurrency)
]
retained_slots = [False] * concurrency
start_gate = None if headless else threading.Barrier(concurrency)
completion_gate = None if headless else threading.Barrier(concurrency)
def launch(index: int) -> None:
result, resources = _launch_one_browser(
index=index,
headless=headless,
launcher=launcher,
start_gate=start_gate,
)
launch_results[index] = result
if headless:
if result.get("success"):
_browser_resources.extend(resources)
result["retained"] = True
retained_slots[index] = True
else:
cleanup_error_slots[index] = _close_browser_resources(resources)
return
try:
completion_gate.wait(timeout=min(_ACTIVATION_TIMEOUT, 30))
except threading.BrokenBarrierError:
cleanup_error_slots[index] = _close_browser_resources(resources)
result.update(
{
"success": False,
"error_type": "BrokenBarrierError",
"error": "并发浏览器启动未能完成同线程清理协调",
}
)
return
successful_indices = [
candidate_index
for candidate_index, item in enumerate(launch_results)
if item is not None and bool(item.get("success"))
]
retained_index = min(successful_indices) if successful_indices else None
if index == retained_index:
# 保留对象不再跨线程使用;容器退出会回收浏览器及其 worker 进程。
_browser_resources.extend(resources)
result["retained"] = True
retained_slots[index] = True
else:
# Playwright sync/greenlet 对象必须在创建它的线程内关闭。
cleanup_error_slots[index] = _close_browser_resources(resources)
if headless:
launch(0)
else:
threads = [
threading.Thread(
target=launch,
args=(index,),
name=f"mp-perf-browser-launch-{index}",
daemon=True,
)
for index in range(concurrency)
]
for thread in threads:
thread.start()
deadline = time.monotonic() + _ACTIVATION_TIMEOUT
for thread in threads:
thread.join(timeout=max(deadline - time.monotonic(), 0))
serialized_launches = [
item
if item is not None
else {
"index": index,
"success": False,
"error_type": "TimeoutError",
"error": "浏览器启动未在进程内超时前完成",
}
for index, item in enumerate(launch_results)
]
launch_starts = [
float(item["started_at_monotonic"])
for item in serialized_launches
if "started_at_monotonic" in item
]
successful_indices = [
index
for index, item in enumerate(serialized_launches)
if bool(item.get("success"))
]
cleanup_errors = [
error for slot_errors in cleanup_error_slots for error in slot_errors
]
retained_count = sum(retained_slots)
expected_successes = concurrency
browser_success = (
len(successful_indices) == expected_successes and not cleanup_errors
)
return {
"requested": True,
"headless": headless,
"concurrency": concurrency,
"successes": len(successful_indices),
"retained_contexts": retained_count,
"launches": serialized_launches,
"cleanup_errors": cleanup_errors,
"success": browser_success,
"managed_resource": {
"before": display_before,
"after": _read_display_runtime(),
},
"single_flight_probe": {
"requested": not headless,
"concurrent_callers": concurrency if not headless else 0,
"successful_callers": len(successful_indices) if not headless else 0,
"barrier_used": not headless,
"launch_start_spread_ms": (
(max(launch_starts) - min(launch_starts)) * 1000
if launch_starts
else None
),
"all_callers_succeeded": len(successful_indices) == expected_successes,
"calls": serialized_launches if not headless else [],
},
}
def _run_activation() -> None:
"""在目标解释器的工作线程中运行激活并发布完成 marker。"""
if not _OUTPUT_DIR:
return
import time
from pathlib import Path
started_at = time.perf_counter()
result: dict[str, object] = {
"schema_version": 1,
"scenario": _SCENARIO,
"pid": os.getpid(),
"started_at": _utc_now(),
}
try:
result["browser"] = _activate_browser_scenario(_SCENARIO)
result["success"] = bool(result["browser"]["success"])
except Exception as error: # pragma: no cover - 真实集成错误由 marker 保存
result.update(
{
"success": False,
"error_type": type(error).__name__,
"error": str(error),
}
)
result["elapsed_seconds"] = time.perf_counter() - started_at
result["completed_at"] = _utc_now()
_atomic_write_json(
Path(_OUTPUT_DIR) / f"activation-{os.getpid()}.json",
result,
)
def _request_activation(_signum, _frame) -> None:
"""SIGUSR2 只调度一次工作线程,真实 import 与启动仍在目标进程内完成。"""
global _activation_started
if not _OUTPUT_DIR or _activation_started:
return
import threading
_activation_started = True
threading.Thread(
target=_run_activation,
name="mp-perf-scenario-activation",
daemon=True,
).start()
if _OUTPUT_DIR and hasattr(signal, "SIGUSR1"):
signal.signal(signal.SIGUSR1, _dump_modules)
if _OUTPUT_DIR and hasattr(signal, "SIGUSR2"):
signal.signal(signal.SIGUSR2, _request_activation)
+394 -55
View File
@@ -31,6 +31,8 @@ DEFAULT_SUBSTRATE = (
"sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720"
)
DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed"
DEFAULT_SCENARIO = "idle-default"
SCENARIOS = (DEFAULT_SCENARIO, "browser-headless", "browser-headed")
CAMPAIGN_LABEL = "org.moviepilot.perf.campaign"
ROLE_LABEL = "org.moviepilot.perf.role"
SOURCE_LABEL = "org.moviepilot.perf.source-commit"
@@ -587,6 +589,10 @@ def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, s
{
"PYTHONPATH": "/opt/moviepilot-perf/instrument",
"MP_PERF_OUTPUT_DIR": "/opt/moviepilot-perf/out/modules",
"MP_PERF_SCENARIO": getattr(args, "scenario", DEFAULT_SCENARIO),
"MP_PERF_ACTIVATION_TIMEOUT": str(
getattr(args, "activation_timeout", 180)
),
}
)
return environment
@@ -1103,7 +1109,9 @@ def sample_volume_names(
index: int,
) -> tuple[str, str]:
"""返回单个样本的隔离配置和浏览器卷名称。"""
prefix = f"{resource_prefix(args)}-{variant}-{index}"
scenario = getattr(args, "scenario", DEFAULT_SCENARIO)
scenario_segment = "" if scenario == DEFAULT_SCENARIO else f"-{scenario}"
prefix = f"{resource_prefix(args)}{scenario_segment}-{variant}-{index}"
return f"{prefix}-config", f"{prefix}-browser"
@@ -1113,11 +1121,190 @@ def sample_result_directory(
index: int,
) -> Path:
"""返回单个样本的原始结果目录。"""
return campaign_directory(args) / "samples" / f"{variant}-{index}"
scenario = getattr(args, "scenario", DEFAULT_SCENARIO)
sample_root = campaign_directory(args) / "samples"
if scenario == DEFAULT_SCENARIO:
return sample_root / f"{variant}-{index}"
return sample_root / scenario / f"{variant}-{index}"
def capture_activation_snapshot(
container,
output_dir: Path,
phase: str,
) -> dict[str, Any]:
"""采集浏览器激活边界的 Engine、进程和进程内 import 状态。"""
engine = capture_engine_stats(container)
processes = capture_processes(container)
modules = capture_modules(container, output_dir, processes["main_python"])
return {
"phase": phase,
"captured_at": utc_now(),
"engine": engine,
"processes": processes,
"modules": modules,
}
def evaluate_browser_activation(
scenario: str,
pre: dict[str, Any],
post: dict[str, Any],
marker: dict[str, Any],
expected_pid: Optional[int] = None,
) -> dict[str, Any]:
"""按场景不变量判断浏览器与 display 的真实激活是否有效。"""
pre_xvfb = pre["processes"]["xvfb"]
post_xvfb = post["processes"]["xvfb"]
browser = marker.get("browser") or {}
managed_resource = browser.get("managed_resource") or {}
managed_before = managed_resource.get("before") or {}
managed_after = managed_resource.get("after") or {}
before_observations = managed_before.get("observations") or []
after_observations = managed_after.get("observations") or []
observation_prefix_matches = (
after_observations[: len(before_observations)] == before_observations
)
new_observations = (
after_observations[len(before_observations) :]
if observation_prefix_matches
else after_observations
)
display_starts = [
item
for item in new_observations
if item.get("operation") == "activate" and item.get("outcome") == "started"
]
display_successes = [
item
for item in new_observations
if item.get("operation") == "activate" and item.get("outcome") == "succeeded"
]
display_start_reasons = [item.get("reason") for item in display_starts]
before_generation = (managed_before.get("snapshot") or {}).get("generation")
after_generation = (managed_after.get("snapshot") or {}).get("generation")
errors: list[str] = []
if marker.get("scenario") != scenario:
errors.append("进程内 marker 的场景与采集请求不一致")
if expected_pid is not None and marker.get("pid") != expected_pid:
errors.append("进程内 marker 不是目标 MoviePilot Python 进程写出")
if not marker.get("success") or not browser.get("success"):
errors.append("主 MoviePilot Python 进程未完成浏览器激活")
if browser.get("retained_contexts") != 1:
errors.append("激活后必须保留一个浏览器上下文供 post activation 采样")
if not managed_before.get("available") or not managed_after.get("available"):
errors.append("主进程未提供 host.display managed resource 观测")
process_single_flight = browser.get("single_flight_probe") or {}
single_flight = {
"requested": scenario == "browser-headed",
"concurrent_callers": int(process_single_flight.get("concurrent_callers") or 0),
"successful_callers": int(process_single_flight.get("successful_callers") or 0),
"xvfb_process_delta": int(post_xvfb["count"]) - int(pre_xvfb["count"]),
"generation_before": before_generation,
"generation_after": after_generation,
"activation_start_count": len(display_starts),
"activation_success_count": len(display_successes),
"activation_start_reasons": display_start_reasons,
"observation_prefix_matches": observation_prefix_matches,
"passed": None,
}
if scenario == "browser-headless":
if pre_xvfb["count"] != 0 or post_xvfb["count"] != 0:
errors.append("headless 激活前后都不得存在 Xvfb")
if display_starts or before_generation != after_generation:
errors.append("headless 激活不得申请 host.display")
elif scenario == "browser-headed":
if pre_xvfb["count"] != 0:
errors.append("headed 冷激活前必须没有 Xvfb")
if post_xvfb["count"] != 1:
errors.append("headed 并发激活后必须恰好存在一个 Xvfb")
single_flight["passed"] = (
single_flight["concurrent_callers"] == 2
and single_flight["successful_callers"] == 2
and single_flight["xvfb_process_delta"] == 1
and single_flight["activation_start_count"] == 1
and single_flight["activation_success_count"] == 1
and single_flight["activation_start_reasons"] == ["headed_browser_launch"]
and before_generation is not None
and after_generation == before_generation + 1
)
if not single_flight["passed"]:
errors.append("headed 并发请求未证明 display single-flight")
else:
errors.append(f"未知浏览器场景:{scenario}")
return {
"passed": not errors,
"errors": errors,
"expected": ("Xvfb 0→0" if scenario == "browser-headless" else "Xvfb 0→1"),
"observed": {
"pre_xvfb_count": pre_xvfb["count"],
"pre_xvfb_pss_kib": pre_xvfb["pss_kib"],
"post_xvfb_count": post_xvfb["count"],
"post_xvfb_pss_kib": post_xvfb["pss_kib"],
},
"single_flight": single_flight,
}
def activate_browser_scenario(
container,
output_dir: Path,
scenario: str,
timeout: float,
) -> dict[str, Any]:
"""通过 SIGUSR2 让目标 MoviePilot 解释器执行场景激活并回收 marker。"""
pre = capture_activation_snapshot(container, output_dir, "pre-activation")
main_python = pre["processes"]["main_python"]
if not main_python:
raise HarnessError("未找到主 Python 进程,无法触发浏览器场景")
marker_path = output_dir / "modules" / f"activation-{main_python['pid']}.json"
marker_path.unlink(missing_ok=True)
requested_at = time.monotonic()
result = container.exec_run(["kill", "-USR2", str(main_python["pid"])])
if result.exit_code != 0:
raise HarnessError("向主 Python 进程发送场景激活信号失败")
deadline = requested_at + timeout
while time.monotonic() < deadline:
if marker_path.exists():
break
if not container_running(container):
raise HarnessError("等待场景激活 marker 时容器提前退出")
time.sleep(0.05)
if not marker_path.exists():
raise HarnessError(f"浏览器场景激活在 {timeout:.0f}s 内未完成")
marker_received_at = time.monotonic()
marker = json.loads(marker_path.read_text(encoding="utf-8"))
post = capture_activation_snapshot(container, output_dir, "post-activation")
validation = evaluate_browser_activation(
scenario,
pre,
post,
marker,
expected_pid=main_python["pid"],
)
return {
"scenario": scenario,
"trigger": "SIGUSR2-to-main-python",
"main_python_pid": main_python["pid"],
"orchestrator_elapsed_seconds": marker_received_at - requested_at,
"post_capture_elapsed_seconds": time.monotonic() - marker_received_at,
"worker_elapsed_seconds": marker.get("elapsed_seconds"),
"pre": pre,
"post": post,
"marker": marker,
"validation": validation,
}
def command_sample(args: argparse.Namespace) -> dict[str, Any]:
"""执行一个隔离样本并在约定时间点采集完整指标。"""
scenario = getattr(args, "scenario", DEFAULT_SCENARIO)
if scenario != DEFAULT_SCENARIO and args.variant != "after":
raise HarnessError("浏览器激活场景只用于验证包含 app.sdk.browser 的 After 候选")
client = require_docker_client()
build = load_build_manifest(args)
config_seed, browser_seed = require_seed_volumes(client, args)
@@ -1144,13 +1331,21 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
clone_volume(client, image, browser_seed, browser_volume.name)
browser_before = volume_fingerprint(client, image, browser_volume.name)
network = ensure_internal_network(client, args)
container_name = f"{resource_prefix(args)}-{args.variant}-{args.index}"
scenario_segment = "" if scenario == DEFAULT_SCENARIO else f"-{scenario}"
container_name = (
f"{resource_prefix(args)}{scenario_segment}-{args.variant}-{args.index}"
)
role = (
f"sample-{args.variant}-{args.index}"
if scenario == DEFAULT_SCENARIO
else f"sample-{scenario}-{args.variant}-{args.index}"
)
container = create_app_container(
client,
args,
image=image,
name=container_name,
role=f"sample-{args.variant}-{args.index}",
role=role,
config_volume=config_volume.name,
browser_volume=browser_volume.name,
network_name=network.name,
@@ -1161,6 +1356,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
"campaign": args.campaign,
"variant": args.variant,
"sample_index": args.index,
"scenario": scenario,
"source_commit": build[f"{args.variant}_commit"],
"image": image,
"started_at": utc_now(),
@@ -1171,6 +1367,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
"network": "internal",
"database": "sqlite-seed-clone",
"browser": "prewarmed-seed-clone",
"scenario": scenario,
},
"browser_before": browser_before,
"measurements": [],
@@ -1191,9 +1388,27 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
result["http_ready_seconds"] = ready_seconds
result["settled_seconds"] = settled_at - started_at
result["settled_wait_seconds_after_ready"] = settled_at - ready_at
measurement_origin_at = settled_at
if scenario != DEFAULT_SCENARIO:
activation = activate_browser_scenario(
container,
output_dir,
scenario,
args.activation_timeout,
)
result["activation"] = activation
atomic_write_json(output_dir / "result.partial.json", result)
if not activation["validation"]["passed"]:
details = "; ".join(activation["validation"]["errors"])
raise HarnessError(f"{scenario} 场景激活不满足验收条件:{details}")
measurement_origin_at = time.monotonic()
result["measurement_origin"] = "post-activation"
else:
result["measurement_origin"] = "settled"
for point in args.points:
deadline = settled_at + point * 60
deadline = measurement_origin_at + point * 60
remaining = deadline - time.monotonic()
if remaining > 0:
time.sleep(remaining)
@@ -1201,7 +1416,12 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]:
raise HarnessError(f"容器在 {point:g}m 采样前退出")
print(f"[{args.variant}-{args.index}] sampling {point:g}m")
result["measurements"].append(
capture_measurement(container, output_dir, point, settled_at)
capture_measurement(
container,
output_dir,
point,
measurement_origin_at,
)
)
atomic_write_json(output_dir / "result.partial.json", result)
assert_no_app_env(container)
@@ -1236,7 +1456,7 @@ def load_sample_results(args: argparse.Namespace) -> list[dict[str, Any]]:
results = []
if not sample_root.exists():
return results
for path in sorted(sample_root.glob("*/result.json")):
for path in sorted(sample_root.rglob("result.json")):
results.append(json.loads(path.read_text(encoding="utf-8")))
return results
@@ -1282,6 +1502,10 @@ def build_markdown_report(
samples: list[dict[str, Any]],
) -> str:
"""生成不含本机路径和凭据的 Markdown 汇总。"""
scenarios = sorted(
{sample.get("scenario", DEFAULT_SCENARIO) for sample in samples}
) or [DEFAULT_SCENARIO]
show_scenario = any(scenario != DEFAULT_SCENARIO for scenario in scenarios)
points = sorted(
{
float(measurement["target_minute"])
@@ -1314,7 +1538,8 @@ def build_markdown_report(
)
headers = (
["版本", "样本", "HTTP ready(s)"]
(["场景"] if show_scenario else [])
+ ["版本", "样本", "HTTP ready(s)"]
+ [f"{point:g}m WS(MiB)" for point in points]
+ [
"末次 Python PSS(MiB)",
@@ -1333,11 +1558,12 @@ def build_markdown_report(
for sample in sorted(
samples,
key=lambda item: (
item.get("scenario", DEFAULT_SCENARIO),
variant_order.get(item["variant"], 99),
item["sample_index"],
),
):
row = [
row = ([sample.get("scenario", DEFAULT_SCENARIO)] if show_scenario else []) + [
sample["variant"],
str(sample["sample_index"]),
f"{sample.get('http_ready_seconds', 0):.2f}"
@@ -1388,53 +1614,149 @@ def build_markdown_report(
)
lines.append("| " + " | ".join(row) + " |")
activated_samples = [sample for sample in samples if sample.get("activation")]
if activated_samples:
activation_headers = [
"场景",
"版本",
"样本",
"激活(s)",
"Pre WS(MiB)",
"Post WS(MiB)",
"Pre Python PSS(MiB)",
"Post Python PSS(MiB)",
"Pre Xvfb",
"Post Xvfb",
"Post Xvfb PSS(MiB)",
"Activation RX Δ(KiB)",
"Activation TX Δ(KiB)",
"Browser",
"Single-flight generation/start",
"验收",
]
lines.extend(
[
"",
"## 场景激活",
"",
"| " + " | ".join(activation_headers) + " |",
"| " + " | ".join(["---"] * len(activation_headers)) + " |",
]
)
for sample in sorted(
activated_samples,
key=lambda item: (
item.get("scenario", DEFAULT_SCENARIO),
variant_order.get(item["variant"], 99),
item["sample_index"],
),
):
activation = sample["activation"]
pre = activation["pre"]
post = activation["post"]
marker = activation["marker"]
validation = activation["validation"]
pre_python = pre["processes"].get("main_python") or {}
post_python = post["processes"].get("main_python") or {}
single_flight = validation["single_flight"]
activation_row = [
sample.get("scenario", DEFAULT_SCENARIO),
sample["variant"],
str(sample["sample_index"]),
f"{float(activation.get('worker_elapsed_seconds') or 0):.2f}",
format_mib(pre["engine"]["working_set_bytes"]),
format_mib(post["engine"]["working_set_bytes"]),
format_kib_as_mib(pre_python.get("pss_kib")),
format_kib_as_mib(post_python.get("pss_kib")),
str(pre["processes"]["xvfb"]["count"]),
str(post["processes"]["xvfb"]["count"]),
format_kib_as_mib(post["processes"]["xvfb"]["pss_kib"]),
format_bytes_as_kib(
post["engine"]["network_rx_bytes"]
- pre["engine"]["network_rx_bytes"]
),
format_bytes_as_kib(
post["engine"]["network_tx_bytes"]
- pre["engine"]["network_tx_bytes"]
),
"成功" if marker.get("success") else "失败",
(
f"{single_flight.get('generation_after')}/"
f"{single_flight.get('activation_start_count')}"
if single_flight.get("passed") is True
else "不适用"
if single_flight.get("passed") is None
else "失败"
),
"通过" if validation["passed"] else "失败",
]
lines.append("| " + " | ".join(activation_row) + " |")
lines.extend(["", "## 中位数对照", ""])
if points:
lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |")
lines.append("| --- | ---: | ---: | ---: | ---: |")
for point in points:
before_values = [
measurement_at(sample, point)["engine"]["working_set_bytes"]
for sample in samples
if sample["variant"] == "before" and measurement_at(sample, point)
]
after_values = [
measurement_at(sample, point)["engine"]["working_set_bytes"]
for sample in samples
if sample["variant"] == "after" and measurement_at(sample, point)
]
before_median = median(before_values)
after_median = median(after_values)
if before_median is None or after_median is None:
lines.append(f"| {point:g}m | — | — | — | — |")
continue
delta = after_median - before_median
percent = delta / before_median * 100 if before_median else 0
lines.append(
f"| {point:g}m | {format_mib(before_median)} | "
f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | {percent:.1f}% |"
for scenario in scenarios:
scenario_samples = [
sample
for sample in samples
if sample.get("scenario", DEFAULT_SCENARIO) == scenario
]
if show_scenario:
lines.extend([f"### `{scenario}`", ""])
if points:
lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |")
lines.append("| --- | ---: | ---: | ---: | ---: |")
for point in points:
before_values = [
measurement_at(sample, point)["engine"]["working_set_bytes"]
for sample in scenario_samples
if sample["variant"] == "before" and measurement_at(sample, point)
]
after_values = [
measurement_at(sample, point)["engine"]["working_set_bytes"]
for sample in scenario_samples
if sample["variant"] == "after" and measurement_at(sample, point)
]
before_median = median(before_values)
after_median = median(after_values)
if before_median is None or after_median is None:
lines.append(f"| {point:g}m | — | — | — | — |")
continue
delta = after_median - before_median
percent = delta / before_median * 100 if before_median else 0
lines.append(
f"| {point:g}m | {format_mib(before_median)} | "
f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | "
f"{percent:.1f}% |"
)
lines.append("")
lines.extend(["## 启动时间", ""])
for scenario in scenarios:
scenario_samples = [
sample
for sample in samples
if sample.get("scenario", DEFAULT_SCENARIO) == scenario
]
ready_before = median(
sample["http_ready_seconds"]
for sample in scenario_samples
if sample["variant"] == "before" and "http_ready_seconds" in sample
)
ready_after = median(
sample["http_ready_seconds"]
for sample in scenario_samples
if sample["variant"] == "after" and "http_ready_seconds" in sample
)
scenario_prefix = f"`{scenario}`" if show_scenario else ""
if ready_before is not None and ready_after is not None:
startup_change = (
(ready_after - ready_before) / ready_before * 100 if ready_before else 0
)
ready_before = median(
sample["http_ready_seconds"]
for sample in samples
if sample["variant"] == "before" and "http_ready_seconds" in sample
)
ready_after = median(
sample["http_ready_seconds"]
for sample in samples
if sample["variant"] == "after" and "http_ready_seconds" in sample
)
lines.extend(["", "## 启动时间", ""])
if ready_before is not None and ready_after is not None:
startup_change = (
(ready_after - ready_before) / ready_before * 100 if ready_before else 0
)
lines.append(
f"Before 中位数 {ready_before:.2f}sAfter 中位数 {ready_after:.2f}s"
f"变化 {startup_change:.1f}%。"
)
else:
lines.append("样本尚不完整。")
lines.append(
f"{scenario_prefix}Before 中位数 {ready_before:.2f}s"
f"After 中位数 {ready_after:.2f}s,变化 {startup_change:.1f}%。"
)
else:
lines.append(f"{scenario_prefix}样本尚不完整。")
lines.extend(
[
"",
@@ -1605,6 +1927,12 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--memory", default="2g")
parser.add_argument("--ready-timeout", type=int, default=300)
parser.add_argument("--settle-timeout", type=int, default=300)
parser.add_argument(
"--activation-timeout",
type=int,
default=180,
help="进程内场景激活完成 marker 的等待秒数",
)
parser.add_argument("--stop-timeout", type=int, default=120)
subparsers = parser.add_subparsers(dest="command", required=True)
@@ -1624,6 +1952,12 @@ def build_parser() -> argparse.ArgumentParser:
sample.add_argument(
"--points", type=parse_points, default=parse_points("1,5,10,30")
)
sample.add_argument(
"--scenario",
choices=SCENARIOS,
default=DEFAULT_SCENARIO,
help="样本场景;默认保持 PERF-001 idle-default 行为",
)
sample.add_argument("--replace", action="store_true")
run = subparsers.add_parser("run", help="完整执行 build、seed 和三组平衡 A/B")
@@ -1652,7 +1986,12 @@ def main(argv: Optional[list[str]] = None) -> int:
args.browser_source_volume = DEFAULT_BROWSER_SOURCE_VOLUME
if args.cpus <= 0:
parser.error("--cpus 必须大于 0")
if args.ready_timeout <= 0 or args.settle_timeout <= 0 or args.stop_timeout <= 0:
if (
args.ready_timeout <= 0
or args.settle_timeout <= 0
or args.activation_timeout <= 0
or args.stop_timeout <= 0
):
parser.error("timeout 必须大于 0")
try:
if args.command == "build":
+430
View File
@@ -0,0 +1,430 @@
"""PERF Docker harness 场景协议的无 Docker fake 测试。"""
from __future__ import annotations
import argparse
import importlib.util
import json
import os
import sys
import threading
import time
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
PERF_DIR = Path(__file__).resolve().parent
def load_module(name: str, path: Path):
"""从脚本路径加载模块,避免要求 scripts 变成运行时 Python package。"""
spec = importlib.util.spec_from_file_location(name, path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def snapshot(xvfb_count: int, xvfb_pss_kib: int = 0) -> dict:
"""构造只包含验收字段的进程快照。"""
return {
"processes": {
"xvfb": {"count": xvfb_count, "pss_kib": xvfb_pss_kib},
}
}
def managed_resource(before_generation: int, after_generation: int) -> dict:
"""构造 host.display single-flight 观测。"""
observations = []
if after_generation > before_generation:
observations = [
{
"operation": "activate",
"outcome": "started",
"generation": after_generation,
"reason": "headed_browser_launch",
},
{
"operation": "activate",
"outcome": "succeeded",
"generation": after_generation,
},
]
return {
"before": {
"available": True,
"snapshot": {"generation": before_generation},
"observations": [],
},
"after": {
"available": True,
"snapshot": {"generation": after_generation},
"observations": observations,
},
}
class FakePage:
"""验证本地 data URL 的同步页面替身。"""
def __init__(self) -> None:
self.url = ""
def goto(self, url: str) -> None:
self.url = url
def title(self) -> str:
assert self.url.startswith("data:text/html,")
return "MoviePilot Browser Probe"
def close(self) -> None:
"""模拟 Playwright page 对称关闭。"""
class FakeContext:
"""浏览器上下文替身。"""
def new_page(self) -> FakePage:
return FakePage()
def close(self) -> None:
"""模拟 CloakBrowser context 对称关闭。"""
def test_default_cli_and_paths_keep_idle_contract(tmp_path: Path) -> None:
"""未指定 scenario 时保持既有 idle 命令和资源路径。"""
harness = load_module("moviepilot_perf_cli", PERF_DIR / "moviepilot_docker_ab.py")
args = harness.build_parser().parse_args(
[
"--campaign",
"fake",
"--output-dir",
str(tmp_path),
"sample",
"--variant",
"after",
"--index",
"1",
]
)
assert args.scenario == "idle-default"
assert harness.sample_volume_names(args, "after", 1) == (
"mpperf-fake-after-1-config",
"mpperf-fake-after-1-browser",
)
assert harness.sample_result_directory(args, "after", 1) == (
tmp_path / "fake" / "samples" / "after-1"
)
def test_browser_scenario_uses_isolated_resource_and_result_names(
tmp_path: Path,
) -> None:
"""不同激活场景不会覆盖 idle 样本或彼此复用可写卷。"""
harness = load_module("moviepilot_perf_paths", PERF_DIR / "moviepilot_docker_ab.py")
args = argparse.Namespace(
campaign="fake",
output_dir=tmp_path,
scenario="browser-headed",
)
assert harness.sample_volume_names(args, "after", 2) == (
"mpperf-fake-browser-headed-after-2-config",
"mpperf-fake-browser-headed-after-2-browser",
)
assert harness.sample_result_directory(args, "after", 2) == (
tmp_path / "fake" / "samples" / "browser-headed" / "after-2"
)
def test_browser_scenario_rejects_before_without_touching_docker() -> None:
"""旧基线不具备 SDK/display 冷启动不变量,非默认场景只接受 After。"""
harness = load_module(
"moviepilot_perf_after_only", PERF_DIR / "moviepilot_docker_ab.py"
)
args = argparse.Namespace(scenario="browser-headless", variant="before")
with pytest.raises(harness.HarnessError, match="After"):
harness.command_sample(args)
def test_activation_validation_enforces_headless_and_headed_invariants() -> None:
"""headless 保持无 Xvfbheaded 两调用只能产生一个 Xvfb。"""
harness = load_module(
"moviepilot_perf_validation",
PERF_DIR / "moviepilot_docker_ab.py",
)
headless_marker = {
"scenario": "browser-headless",
"pid": 42,
"success": True,
"browser": {
"success": True,
"concurrency": 1,
"successes": 1,
"retained_contexts": 1,
"managed_resource": managed_resource(0, 0),
"single_flight_probe": {
"concurrent_callers": 0,
"successful_callers": 0,
},
},
}
headed_marker = {
"scenario": "browser-headed",
"pid": 42,
"success": True,
"browser": {
"success": True,
"concurrency": 2,
"successes": 2,
"retained_contexts": 1,
"managed_resource": managed_resource(0, 1),
"single_flight_probe": {
"concurrent_callers": 2,
"successful_callers": 2,
},
},
}
headless = harness.evaluate_browser_activation(
"browser-headless",
snapshot(0),
snapshot(0),
headless_marker,
expected_pid=42,
)
headed = harness.evaluate_browser_activation(
"browser-headed",
snapshot(0),
snapshot(1, 72 * 1024),
headed_marker,
expected_pid=42,
)
invalid = harness.evaluate_browser_activation(
"browser-headed",
snapshot(0),
snapshot(2, 144 * 1024),
headed_marker,
expected_pid=42,
)
assert headless["passed"] is True
assert headed["passed"] is True
assert headed["single_flight"]["passed"] is True
assert headed["single_flight"]["generation_after"] == 1
assert headed["single_flight"]["activation_start_count"] == 1
assert invalid["passed"] is False
assert invalid["single_flight"]["passed"] is False
def test_sitecustomize_acquires_headed_display_concurrently_in_same_process() -> None:
"""headed probe 并发走公开 SDK 冷启动,并只保留一个上下文。"""
probe = load_module(
"moviepilot_perf_sitecustomize",
PERF_DIR / "instrument" / "sitecustomize.py",
)
browser_calls: list[tuple[int, bool]] = []
closed_contexts: list[tuple[int, int]] = []
lock = threading.Lock()
class TrackedContext(FakeContext):
"""记录并发探针关闭的额外浏览器上下文。"""
def __init__(self, index: int) -> None:
self.index = index
def close(self) -> None:
closed_contexts.append((self.index, threading.get_ident()))
def launcher(*, headless: bool) -> FakeContext:
with lock:
browser_calls.append((threading.get_ident(), headless))
index = len(browser_calls) - 1
return TrackedContext(index)
result = probe._activate_browser_scenario(
"browser-headed",
launcher=launcher,
)
assert result["success"] is True
assert result["successes"] == 2
assert result["retained_contexts"] == 1
assert len(browser_calls) == 2
assert len({thread_id for thread_id, _headless in browser_calls}) == 2
assert all(headless is False for _thread_id, headless in browser_calls)
assert len(closed_contexts) == 1
closed_index, closed_thread_id = closed_contexts[0]
assert closed_thread_id == browser_calls[closed_index][0]
assert result["single_flight_probe"]["barrier_used"] is True
def test_sitecustomize_headless_uses_one_headless_context() -> None:
"""headless probe 只启动一个无显示上下文。"""
probe = load_module(
"moviepilot_perf_sitecustomize_headless",
PERF_DIR / "instrument" / "sitecustomize.py",
)
calls: list[bool] = []
def launcher(*, headless: bool) -> FakeContext:
calls.append(headless)
return FakeContext()
result = probe._activate_browser_scenario("browser-headless", launcher=launcher)
assert result["success"] is True
assert calls == [True]
assert result["single_flight_probe"]["requested"] is False
def test_sitecustomize_serializes_managed_resource_facade(monkeypatch) -> None:
"""进程探针按公开只读 facade 记录 generation 与 activate observation。"""
observation = SimpleNamespace(
capability_id="host.display",
generation=1,
operation="activate",
outcome="started",
reason="fake",
materialization="materialized",
lifecycle="starting",
duration_ms=0.5,
error=None,
)
runtime_snapshot = SimpleNamespace(
capability_id="host.display",
materialization="materialized",
lifecycle="running",
generation=1,
visible=True,
error=None,
)
facade = ModuleType("app.runtime.managed_resources")
def managed_resource_snapshot(capability_id: str):
assert capability_id == "host.display"
return runtime_snapshot
def managed_resource_observations(capability_id=None):
assert capability_id == "host.display"
return (observation,)
facade.managed_resource_snapshot = managed_resource_snapshot
facade.managed_resource_observations = managed_resource_observations
monkeypatch.setitem(sys.modules, "app.runtime.managed_resources", facade)
probe = load_module(
"moviepilot_perf_sitecustomize_observation",
PERF_DIR / "instrument" / "sitecustomize.py",
)
result = probe._read_display_runtime()
assert result["available"] is True
assert result["snapshot"]["generation"] == 1
assert result["observations"][0]["operation"] == "activate"
assert result["observations"][0]["outcome"] == "started"
def test_sitecustomize_signal_worker_publishes_atomic_marker(tmp_path: Path) -> None:
"""信号回调只调度目标进程工作线程,并发布带 PID 的完成 marker。"""
probe = load_module(
"moviepilot_perf_sitecustomize_marker",
PERF_DIR / "instrument" / "sitecustomize.py",
)
probe._OUTPUT_DIR = str(tmp_path)
probe._SCENARIO = "browser-headless"
probe._activation_started = False
probe._activate_browser_scenario = lambda scenario: {
"success": scenario == "browser-headless"
}
probe._request_activation(None, None)
marker_path = tmp_path / f"activation-{os.getpid()}.json"
deadline = time.monotonic() + 2
while time.monotonic() < deadline and not marker_path.exists():
time.sleep(0.01)
payload = json.loads(marker_path.read_text(encoding="utf-8"))
assert payload["pid"] == os.getpid()
assert payload["scenario"] == "browser-headless"
assert payload["success"] is True
assert not list(tmp_path.glob("*.tmp"))
def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> None:
"""非默认场景报告包含激活证据,并按场景隔离中位数。"""
harness = load_module(
"moviepilot_perf_report", PERF_DIR / "moviepilot_docker_ab.py"
)
process_data = {
"main_python": {"pss_kib": 400 * 1024, "uss_kib": 390 * 1024, "threads": 8},
"xvfb": {"count": 0, "pss_kib": 0},
}
post_process_data = {
"main_python": {"pss_kib": 410 * 1024, "uss_kib": 400 * 1024, "threads": 10},
"xvfb": {"count": 1, "pss_kib": 72 * 1024},
}
activation = {
"worker_elapsed_seconds": 1.25,
"pre": {
"engine": {
"working_set_bytes": 500 * 1024 * 1024,
"network_rx_bytes": 1024,
"network_tx_bytes": 512,
},
"processes": process_data,
},
"post": {
"engine": {
"working_set_bytes": 600 * 1024 * 1024,
"network_rx_bytes": 3072,
"network_tx_bytes": 1536,
},
"processes": post_process_data,
},
"marker": {"success": True},
"validation": {
"passed": True,
"single_flight": {"passed": True},
},
}
sample = {
"scenario": "browser-headed",
"variant": "after",
"sample_index": 1,
"http_ready_seconds": 7.0,
"activation": activation,
"measurements": [
{
"target_minute": 1.0,
"engine": {
"working_set_bytes": 610 * 1024 * 1024,
"network_rx_bytes": 1024,
"network_tx_bytes": 512,
},
"processes": post_process_data,
"modules": {"count": 3000},
}
],
}
build = {
"campaign": "fake",
"platform": "linux/arm64",
"before_commit": "before",
"after_commit": "after",
"substrate": {"reference": "frozen"},
}
report = harness.build_markdown_report(build, None, [sample])
assert "## 场景激活" in report
assert "browser-headed" in report
assert "Single-flight" in report
assert "### `browser-headed`" in report
assert "1.25" in report