docs: add diagnostic tools and final audit report for v5.1 hardening

check_otp_timing.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+离线验证 TempmailService 的 OTP 时间锚点过滤行为。
+
+场景 1:
+- 30 秒内先后收到两封邮件
+- 在两封邮件之间设置新的 otp_sent_at
+- 期望过滤第一封，命中第二封
+
+场景 2:
+- 第二封邮件已经入箱后才刷新 otp_sent_at
+- 期望复现严格时间过滤导致第二封也被排除的窗口
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+from typing import Any, Dict, List, Optional
+
+import src.services.tempmail as tempmail_module
+from src.services.tempmail import TempmailService
+
+
+@dataclass(frozen=True)
+class Scenario:
+    name: str
+    anchor_offset_seconds: int
+    expected_code: Optional[str]
+    expected_message: str
+
+
+class FakeResponse:
+    def __init__(self, payload: Dict[str, Any], status_code: int = 200):
+        self._payload = payload
+        self.status_code = status_code
+
+    def json(self) -> Dict[str, Any]:
+        return self._payload
+
+
+class FakeHTTPClient:
+    def __init__(self, payload: Dict[str, Any]):
+        self.payload = payload
+        self.calls: List[Dict[str, Any]] = []
+
+    def get(self, url: str, **kwargs: Any) -> FakeResponse:
+        self.calls.append({"url": url, "kwargs": kwargs})
+        return FakeResponse(self.payload)
+
+
+class FakeClock:
+    def __init__(self, start: float):
+        self.current = float(start)
+
+    def time(self) -> float:
+        return self.current
+
+    def sleep(self, seconds: float) -> None:
+        self.current += float(seconds)
+
+
+def build_inbox_payload(base_timestamp: int) -> Dict[str, Any]:
+    return {
+        "emails": [
+            {
+                "id": "mail-1",
+                "received_at": base_timestamp + 10,
+                "from": "noreply@openai.com",
+                "subject": "First OTP",
+                "body": "111111",
+            },
+            {
+                "id": "mail-2",
+                "received_at": base_timestamp + 20,
+                "from": "noreply@openai.com",
+                "subject": "Second OTP",
+                "body": "222222",
+            },
+        ]
+    }
+
+
+def run_scenario(scenario: Scenario) -> Dict[str, Any]:
+    base_timestamp = 1_700_000_000
+    service = TempmailService({"base_url": "https://api.tempmail.test"})
+    service._email_cache["tester@example.com"] = {"token": "token-1"}
+    service.http_client = FakeHTTPClient(build_inbox_payload(base_timestamp))
+
+    fake_clock = FakeClock(start=base_timestamp + scenario.anchor_offset_seconds)
+    anchor_timestamp = fake_clock.time()
+    original_time = tempmail_module.time.time
+    original_sleep = tempmail_module.time.sleep
+
+    try:
+        tempmail_module.time.time = fake_clock.time
+        tempmail_module.time.sleep = fake_clock.sleep
+        code = service.get_verification_code(
+            email="tester@example.com",
+            timeout=1,
+            otp_sent_at=anchor_timestamp,
+        )
+    finally:
+        tempmail_module.time.time = original_time
+        tempmail_module.time.sleep = original_sleep
+
+    passed = code == scenario.expected_code
+    return {
+        "name": scenario.name,
+        "anchor_timestamp": anchor_timestamp,
+        "code": code,
+        "passed": passed,
+        "http_calls": len(service.http_client.calls),
+        "message": scenario.expected_message,
+    }
+
+
+def main() -> int:
+    logging.getLogger("src.services.tempmail").setLevel(logging.ERROR)
+
+    scenarios = [
+        Scenario(
+            name="anchor_between_two_emails",
+            anchor_offset_seconds=15,
+            expected_code="222222",
+            expected_message="新锚点位于两封邮件之间，第一封应被过滤，第二封应被命中。",
+        ),
+        Scenario(
+            name="anchor_set_after_second_email",
+            anchor_offset_seconds=21,
+            expected_code=None,
+            expected_message="锚点晚于第二封邮件时，严格大于过滤会把第二封也排除，复现登录阶段的竞态窗口。",
+        ),
+    ]
+
+    print("Tempmail OTP timing check")
+    print("=========================")
+
+    failed = False
+    for scenario in scenarios:
+        result = run_scenario(scenario)
+        status = "PASS" if result["passed"] else "FAIL"
+        print(f"{status} {result['name']}")
+        print(f"  anchor_timestamp={result['anchor_timestamp']}")
+        print(f"  returned_code={result['code']}")
+        print(f"  inbox_polls={result['http_calls']}")
+        print(f"  note={result['message']}")
+        if not result["passed"]:
+            failed = True
+
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/reviews/FUNCTIONAL-AVAILABILITY-REPORT-2026-03-24.md

@@ -0,0 +1,165 @@
+# 功能可用性实测报告
+
+日期: 2026-03-24
+范围:
+- 启动命令: `uv run python -m src.web.app`
+- 监听地址: `http://127.0.0.1:15555`
+- 隔离数据库: `tests_runtime/e2e_runtime_1774308869.db`
+- 实测脚本: `tests/e2e/runtime_functionality_check.py`
+
+## 1. 执行摘要
+
+本次按真实服务链路完成了以下验证:
+
+- 服务存活检查通过。
+- `POST /api/registration/create` 可创建受控模拟任务。
+- `GET /api/ws/task/{task_uuid}` 可实时推送日志与状态，任务完成时收到 `completed`。
+- 任务完成后数据库状态符合 Task 1、Task 5 预期。
+- 批量计数探针通过 `/api/registration/batch/{batch_id}` 验证，符合 Task 2 预期。
+- 重启后僵尸任务被自动标记失败，符合 Task 4 预期。
+
+结论:
+
+- 本次新增的真实服务验证 harness 可用。
+- Task 1-5 中本次可通过真实服务直接观测的加固点均已生效。
+
+## 2. 实测过程
+
+### 2.1 端口处理
+
+`15555` 端口初始被已有容器 `codex-manager-webui-1` 占用。为执行指定启动命令，先停止该容器，实测结束后已恢复。
+
+### 2.2 执行命令
+
+1. 静态验证
+   - `uv run python -m pytest tests/test_account_token_sync_status.py tests/test_batch_task_manager.py tests/test_task_manager_status_broadcast.py tests/test_task_recovery.py tests/test_registration_email_service_failover.py`
+   - 结果: `16 passed`
+   - `uv run python -m py_compile src/web/app.py src/web/routes/registration.py tests/e2e/runtime_functionality_check.py`
+   - 结果: 退出码 `0`
+
+2. 真实服务启动
+   - `APP_DATABASE_URL='sqlite:////Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db' APP_HOST='127.0.0.1' APP_PORT='15555' uv run python -m src.web.app`
+
+3. live 实测
+   - `uv run python tests/e2e/runtime_functionality_check.py --mode live --base-url http://127.0.0.1:15555 --ws-url ws://127.0.0.1:15555 --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --report-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_functionality_report_1774308869.json`
+
+4. recovery 准备
+   - `uv run python tests/e2e/runtime_functionality_check.py --mode prepare-recovery --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --state-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_state_1774308869.json`
+
+5. 服务重启后 recovery 实测
+   - `uv run python tests/e2e/runtime_functionality_check.py --mode verify-recovery --base-url http://127.0.0.1:15555 --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --state-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_state_1774308869.json --report-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_report_1774308869.json`
+
+## 3. 验证结果
+
+### 3.1 服务存活
+
+- `GET /api/registration/tasks?page=1&page_size=1` 返回 `200`。
+
+### 3.2 模拟任务创建与 WebSocket
+
+- 创建任务 UUID: `a8f4da41-354c-4d89-9634-c582a032c70b`
+- 批量探针 ID: `2e8cfce4-bf20-4f0b-8839-a94e8e141472`
+- WebSocket 收到 3 条状态消息:
+  - `pending`
+  - `running`
+  - `completed`
+- WebSocket 收到 6 条实时日志，包含:
+  - Token 同步探针写库
+  - OTP 超时退避 3 次
+  - 批量计数探针完成
+
+判定:
+
+- 日志不是任务结束后一次性补发，而是在运行过程中实时推送。
+
+### 3.3 Task 1 验证: Token 同步
+
+数据库中以下账号状态正确:
+
+- `mock-seeded-a8f4da41@example.test`
+  - `access_token` 已保存
+  - `refresh_token` 已保存
+  - `token_sync_status = pending`
+
+- `mock-tokenless-a8f4da41@example.test`
+  - 先创建无 token，再更新 `access_token`
+  - `token_sync_status = pending`
+
+- `mock-partial-a8f4da41@example.test`
+  - 清空 `refresh_token` 后仍保留 `access_token`
+  - `token_sync_status = pending`
+
+Outlook 配置探针:
+
+- `mock-outlook-a8f4da41@example.test`
+  - `refresh_token` 已从 `old-second` 更新为 `new-second`
+
+### 3.4 Task 2 验证: 批量计数
+
+`GET /api/registration/batch/2e8cfce4-bf20-4f0b-8839-a94e8e141472` 返回:
+
+- `total = 3`
+- `completed = 3`
+- `success = 2`
+- `failed = 1`
+- `finished = true`
+- `progress = 3/3`
+
+判定:
+
+- 批量计数与任务结果一致，收口正确。
+
+### 3.5 Task 3 验证: 单任务状态广播
+
+任务完成时，WebSocket 最后一条状态消息为:
+
+- `status = completed`
+- `email = mock-seeded-a8f4da41@example.test`
+- `email_service = tempmail`
+
+判定:
+
+- 单任务状态广播已生效。
+
+### 3.6 Task 4 验证: 僵尸任务恢复
+
+重启前手工插入任务:
+
+- `stale-e738842e-74d8-400d-859e-1b283eab1a95`
+- 初始状态: `running`
+
+重启后观测结果:
+
+- 状态变为 `failed`
+- `error_message = 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。`
+- `logs` 已追加系统收敛日志
+- `completed_at` 已写入
+
+服务启动日志同时出现:
+
+- `已收敛 1 个僵尸任务: stale-e7`
+
+### 3.7 Task 5 验证: OTP 超时退避
+
+模拟任务内部连续触发 3 次二阶段 OTP 超时，记录到任务结果:
+
+- 第 1 次: `failures = 1`, `delay_seconds = 30`
+- 第 2 次: `failures = 2`, `delay_seconds = 60`
+- 第 3 次: `failures = 3`, `delay_seconds = 3600`
+
+判定:
+
+- 深度冷却逻辑已生效。
+
+## 4. 产物
+
+- `tests/e2e/runtime_functionality_check.py`
+- `tests_runtime/runtime_functionality_report_1774308869.json`
+- `tests_runtime/runtime_recovery_report_1774308869.json`
+- `tests_runtime/runtime_recovery_state_1774308869.json`
+- `tests_runtime/e2e_runtime_1774308869.db`
+
+## 5. 观察到的问题
+
+- `python -m src.web.app` 启动时会出现 `runpy` 的重复导入告警，但不影响服务启动与本次验证结果。
+- 启动日志中打印的 `host` 和 `database` 仍显示为数据库配置值 `0.0.0.0 / sqlite:///data/database.db`，与本次通过环境变量注入的真实运行参数不一致。实测链路实际使用了隔离数据库，但日志口径存在偏差。

probe_tempmail.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+Tempmail.lol API 探针。
+
+用途：
+1. 创建测试收件箱或复用现有 token。
+2. 拉取 /inbox 原始 JSON 并原样打印。
+3. 检查邮件对象里是否存在 received_at/date 等时间字段。
+"""
+
+import argparse
+import json
+import sys
+import time
+from typing import Any, Dict, Iterable
+
+import httpx
+
+
+DEFAULT_BASE_URL = "https://api.tempmail.lol/v2"
+TIME_FIELDS = ("received_at", "date", "created_at", "createdAt", "timestamp")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="抓取 Tempmail.lol 收件箱原始 JSON")
+    parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Tempmail API 基础地址")
+    parser.add_argument("--token", help="已有 inbox token；未提供时自动创建新邮箱")
+    parser.add_argument("--poll-count", type=int, default=1, help="轮询次数")
+    parser.add_argument("--poll-interval", type=float, default=3.0, help="轮询间隔秒数")
+    parser.add_argument("--timeout", type=float, default=20.0, help="HTTP 超时时间")
+    return parser.parse_args()
+
+
+def dump_json(title: str, payload: Dict[str, Any]) -> None:
+    print(f"\n===== {title} =====")
+    print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
+
+
+def summarize_time_fields(emails: Iterable[Dict[str, Any]]) -> None:
+    for index, message in enumerate(emails, start=1):
+        present_fields = {name: message.get(name) for name in TIME_FIELDS if name in message}
+        print(f"email[{index}] 时间字段: {json.dumps(present_fields, ensure_ascii=False, default=str)}")
+
+
+def create_inbox(client: httpx.Client, base_url: str) -> Dict[str, Any]:
+    response = client.post(
+        f"{base_url}/inbox/create",
+        headers={
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        },
+        json={},
+    )
+    print(f"CREATE_STATUS {response.status_code}")
+    response.raise_for_status()
+    payload = response.json()
+    dump_json("CREATE_RESPONSE", payload)
+    return payload
+
+
+def fetch_inbox(client: httpx.Client, base_url: str, token: str) -> Dict[str, Any]:
+    response = client.get(
+        f"{base_url}/inbox",
+        params={"token": token},
+        headers={"Accept": "application/json"},
+    )
+    print(f"INBOX_STATUS {response.status_code}")
+    response.raise_for_status()
+    payload = response.json()
+    dump_json("INBOX_RESPONSE", payload)
+    emails = payload.get("emails", []) if isinstance(payload, dict) else []
+    if isinstance(emails, list):
+        summarize_time_fields([mail for mail in emails if isinstance(mail, dict)])
+    else:
+        print(f"emails 字段不是列表: {type(emails).__name__}")
+    return payload
+
+
+def main() -> int:
+    args = parse_args()
+    with httpx.Client(timeout=args.timeout) as client:
+        token = args.token
+        if not token:
+            inbox = create_inbox(client, args.base_url)
+            token = str(inbox.get("token", "")).strip()
+            address = str(inbox.get("address", "")).strip()
+            print(f"ADDRESS {address}")
+            print(f"TOKEN {token}")
+
+        if not token:
+            print("未拿到 token，无法继续拉取 inbox", file=sys.stderr)
+            return 1
+
+        for attempt in range(1, args.poll_count + 1):
+            print(f"\n----- poll {attempt}/{args.poll_count} -----")
+            fetch_inbox(client, args.base_url, token)
+            if attempt < args.poll_count:
+                time.sleep(args.poll_interval)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/e2e/runtime_functionality_check.py

@@ -0,0 +1,278 @@
+import argparse
+import asyncio
+import json
+import sqlite3
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List
+
+import httpx
+import websockets
+
+
+STALE_ERROR = "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。"
+
+
+def _write_json(path: Path, payload: Dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _load_json(path: Path) -> Dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _connect_db(db_path: Path) -> sqlite3.Connection:
+    return sqlite3.connect(db_path, timeout=5)
+
+
+def _fetchone_dict(conn: sqlite3.Connection, sql: str, params: tuple[Any, ...]) -> Dict[str, Any]:
+    conn.row_factory = sqlite3.Row
+    row = conn.execute(sql, params).fetchone()
+    return dict(row) if row else {}
+
+
+def _assert(condition: bool, message: str) -> None:
+    if not condition:
+        raise AssertionError(message)
+
+
+def _health_check(client: httpx.Client, report: Dict[str, Any]) -> None:
+    response = client.get("/api/registration/tasks", params={"page": 1, "page_size": 1})
+    report["health"] = {"status_code": response.status_code, "body": response.json()}
+    _assert(response.status_code == 200, "健康检查失败")
+
+
+async def _collect_task_websocket(ws_url: str, task_uuid: str) -> Dict[str, Any]:
+    endpoint = f"{ws_url}/api/ws/task/{task_uuid}"
+    messages: List[Dict[str, Any]] = []
+    started_at = time.time()
+
+    async with websockets.connect(endpoint, open_timeout=10, close_timeout=5) as websocket:
+        while time.time() - started_at < 30:
+            raw_message = await asyncio.wait_for(websocket.recv(), timeout=10)
+            payload = json.loads(raw_message)
+            messages.append(payload)
+            if payload.get("type") == "status" and payload.get("status") in {"completed", "failed"}:
+                break
+
+    logs = [message for message in messages if message.get("type") == "log"]
+    statuses = [message for message in messages if message.get("type") == "status"]
+    return {
+        "messages": messages,
+        "log_count": len(logs),
+        "status_count": len(statuses),
+        "live_log_count": sum(1 for message in logs if "timestamp" in message),
+        "final_status": statuses[-1]["status"] if statuses else None,
+    }
+
+
+def _poll_task_completion(client: httpx.Client, task_uuid: str) -> Dict[str, Any]:
+    deadline = time.time() + 20
+    while time.time() < deadline:
+        response = client.get(f"/api/registration/tasks/{task_uuid}")
+        response.raise_for_status()
+        payload = response.json()
+        if payload["status"] in {"completed", "failed"}:
+            return payload
+        time.sleep(0.2)
+    raise TimeoutError(f"任务未在预期时间内结束: {task_uuid}")
+
+
+def _validate_live_database(
+    db_path: Path,
+    task_uuid: str,
+    batch_id: str,
+    checks: Dict[str, Any],
+    report: Dict[str, Any],
+) -> None:
+    with _connect_db(db_path) as conn:
+        seeded = _fetchone_dict(
+            conn,
+            "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?",
+            (checks["seeded_account_email"],),
+        )
+        tokenless = _fetchone_dict(
+            conn,
+            "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?",
+            (checks["tokenless_account_email"],),
+        )
+        partial = _fetchone_dict(
+            conn,
+            "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?",
+            (checks["partial_account_email"],),
+        )
+        task_row = _fetchone_dict(
+            conn,
+            "SELECT task_uuid, status, logs, result FROM registration_tasks WHERE task_uuid = ?",
+            (task_uuid,),
+        )
+        outlook_row = _fetchone_dict(
+            conn,
+            "SELECT config FROM email_services WHERE id = ?",
+            (checks["outlook_service_id"],),
+        )
+
+    _assert(seeded.get("token_sync_status") == "pending", "seeded 账号 token_sync_status 异常")
+    _assert(tokenless.get("access_token") == "mock-access-token-updated", "tokenless 账号 access_token 未写入")
+    _assert(tokenless.get("token_sync_status") == "pending", "tokenless 账号 token_sync_status 异常")
+    _assert(partial.get("access_token") == "mock-access-token-partial", "partial 账号 access_token 丢失")
+    _assert(partial.get("refresh_token") == "", "partial 账号 refresh_token 未清空")
+    _assert(partial.get("token_sync_status") == "pending", "partial 账号 token_sync_status 异常")
+    _assert(task_row.get("status") == "completed", "模拟任务数据库状态不是 completed")
+    _assert(task_row.get("logs"), "模拟任务日志未落库")
+
+    task_result = json.loads(task_row["result"]) if task_row.get("result") else {}
+    outlook_config = json.loads(outlook_row["config"]) if outlook_row.get("config") else {}
+    second_account = next(
+        account for account in outlook_config.get("accounts", [])
+        if account.get("email") == checks["outlook_account_email"]
+    )
+
+    batch_snapshot = task_result["hardening_checks"]["batch_counter"]["snapshot"]
+    backoff_states = task_result["hardening_checks"]["otp_timeout_backoff"]["states"]
+
+    _assert(second_account["refresh_token"] == "new-second", "Outlook refresh_token 未更新")
+    _assert(batch_snapshot["completed"] == 3, "批量 completed 计数异常")
+    _assert(batch_snapshot["success"] == 2, "批量 success 计数异常")
+    _assert(batch_snapshot["failed"] == 1, "批量 failed 计数异常")
+    _assert(batch_snapshot["status"] == "completed", "批量状态异常")
+    _assert(batch_snapshot["finished"] is True, "批量 finished 标记异常")
+    _assert(backoff_states[-1]["delay_seconds"] == 3600, "OTP 深度冷却未生效")
+    _assert(backoff_states[-1]["failures"] == 3, "OTP 连续失败次数异常")
+
+    report["database"] = {
+        "task_uuid": task_uuid,
+        "batch_id": batch_id,
+        "seeded_account": seeded,
+        "tokenless_account": tokenless,
+        "partial_account": partial,
+        "task_result": task_result,
+        "outlook_second_account": second_account,
+    }
+
+
+def run_live_mode(base_url: str, ws_url: str, db_path: Path, report_path: Path) -> None:
+    report: Dict[str, Any] = {"mode": "live", "base_url": base_url, "db_path": str(db_path)}
+    with httpx.Client(base_url=base_url, timeout=httpx.Timeout(10, read=30)) as client:
+        _health_check(client, report)
+
+        create_response = client.post(
+            "/api/registration/create",
+            json={
+                "email_service_type": "tempmail",
+                "start_delay_ms": 600,
+                "log_delay_ms": 150,
+            },
+        )
+        create_response.raise_for_status()
+        created = create_response.json()
+        task_uuid = created["task"]["task_uuid"]
+        batch_id = created["batch_id"]
+        checks = created["checks"]
+        report["create"] = created
+
+        ws_report = asyncio.run(_collect_task_websocket(ws_url, task_uuid))
+        report["websocket"] = ws_report
+        _assert(ws_report["final_status"] == "completed", "WebSocket 未收到 completed 状态")
+        _assert(ws_report["log_count"] >= 4, "WebSocket 日志数量不足")
+        _assert(ws_report["live_log_count"] >= 1, "未捕获到实时日志广播")
+
+        task_payload = _poll_task_completion(client, task_uuid)
+        report["task"] = task_payload
+        runtime_checks = {
+            **checks,
+            "outlook_service_id": task_payload["result"]["hardening_checks"]["outlook_refresh"]["service_id"],
+            "backoff_service_id": task_payload["result"]["hardening_checks"]["otp_timeout_backoff"]["service_id"],
+        }
+
+        batch_response = client.get(f"/api/registration/batch/{batch_id}")
+        batch_response.raise_for_status()
+        report["batch_api"] = batch_response.json()
+        _assert(report["batch_api"]["completed"] == 3, "批量状态 API completed 异常")
+        _assert(report["batch_api"]["success"] == 2, "批量状态 API success 异常")
+        _assert(report["batch_api"]["failed"] == 1, "批量状态 API failed 异常")
+        _assert(report["batch_api"]["finished"] is True, "批量状态 API finished 异常")
+
+    _validate_live_database(db_path, task_uuid, batch_id, runtime_checks, report)
+    _write_json(report_path, report)
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+def run_prepare_recovery_mode(db_path: Path, state_path: Path) -> None:
+    stale_task_uuid = f"stale-{uuid.uuid4()}"
+    now = time.strftime("%Y-%m-%d %H:%M:%S")
+    with _connect_db(db_path) as conn:
+        conn.execute(
+            """
+            INSERT INTO registration_tasks (task_uuid, status, logs, created_at, started_at)
+            VALUES (?, 'running', '[00:00:00] stale task', ?, ?)
+            """,
+            (stale_task_uuid, now, now),
+        )
+        conn.commit()
+
+    payload = {
+        "stale_task_uuid": stale_task_uuid,
+        "db_path": str(db_path),
+        "prepared_at": now,
+    }
+    _write_json(state_path, payload)
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+
+def run_verify_recovery_mode(base_url: str, db_path: Path, state_path: Path, report_path: Path) -> None:
+    state = _load_json(state_path)
+    report: Dict[str, Any] = {
+        "mode": "verify-recovery",
+        "base_url": base_url,
+        "db_path": str(db_path),
+        "state": state,
+    }
+
+    with httpx.Client(base_url=base_url, timeout=httpx.Timeout(10, read=30)) as client:
+        _health_check(client, report)
+
+    with _connect_db(db_path) as conn:
+        stale_task = _fetchone_dict(
+            conn,
+            "SELECT task_uuid, status, error_message, logs, completed_at FROM registration_tasks WHERE task_uuid = ?",
+            (state["stale_task_uuid"],),
+        )
+
+    _assert(stale_task.get("status") == "failed", "僵尸任务未在重启后标记为 failed")
+    _assert(stale_task.get("error_message") == STALE_ERROR, "僵尸任务 error_message 不匹配")
+    _assert(STALE_ERROR in (stale_task.get("logs") or ""), "僵尸任务日志未追加系统收敛说明")
+    _assert(bool(stale_task.get("completed_at")), "僵尸任务 completed_at 缺失")
+
+    report["recovery"] = stale_task
+    _write_json(report_path, report)
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="真实服务功能可用性验证脚本")
+    parser.add_argument("--mode", choices=["live", "prepare-recovery", "verify-recovery"], required=True)
+    parser.add_argument("--base-url", default="http://127.0.0.1:15555")
+    parser.add_argument("--ws-url", default="ws://127.0.0.1:15555")
+    parser.add_argument("--db-path", required=True)
+    parser.add_argument("--report-path", default="tests_runtime/runtime_functionality_report.json")
+    parser.add_argument("--state-path", default="tests_runtime/runtime_recovery_state.json")
+    args = parser.parse_args()
+
+    db_path = Path(args.db_path).resolve()
+    report_path = Path(args.report_path).resolve()
+    state_path = Path(args.state_path).resolve()
+
+    if args.mode == "live":
+        run_live_mode(args.base_url, args.ws_url, db_path, report_path)
+        return
+    if args.mode == "prepare-recovery":
+        run_prepare_recovery_mode(db_path, state_path)
+        return
+    run_verify_recovery_mode(args.base_url, db_path, state_path, report_path)
+
+
+if __name__ == "__main__":
+    main()

tests_runtime/runtime_functionality_report_1774308869.json

@@ -0,0 +1,292 @@
+{
+  "mode": "live",
+  "base_url": "http://127.0.0.1:15555",
+  "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db",
+  "health": {
+    "status_code": 200,
+    "body": {
+      "total": 4,
+      "tasks": [
+        {
+          "id": 4,
+          "task_uuid": "9079068e-e3f5-4fa7-8e1c-810ce1c352da",
+          "status": "completed",
+          "email_service_id": null,
+          "proxy": null,
+          "logs": null,
+          "result": null,
+          "error_message": null,
+          "created_at": "2026-03-23T23:34:58.715238",
+          "started_at": "2026-03-23T23:34:58.718370",
+          "completed_at": "2026-03-23T23:34:58.718376"
+        }
+      ]
+    }
+  },
+  "create": {
+    "task": {
+      "id": 5,
+      "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+      "status": "pending",
+      "email_service_id": null,
+      "proxy": null,
+      "logs": null,
+      "result": null,
+      "error_message": null,
+      "created_at": "2026-03-23T23:35:28.629402",
+      "started_at": null,
+      "completed_at": null
+    },
+    "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472",
+    "checks": {
+      "seeded_account_email": "mock-seeded-a8f4da41@example.test",
+      "tokenless_account_email": "mock-tokenless-a8f4da41@example.test",
+      "partial_account_email": "mock-partial-a8f4da41@example.test",
+      "outlook_account_email": "mock-outlook-a8f4da41@example.test",
+      "backoff_service_name": "mock-backoff-a8f4da41"
+    }
+  },
+  "websocket": {
+    "messages": [
+      {
+        "type": "status",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "status": "pending"
+      },
+      {
+        "type": "status",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "status": "running",
+        "timestamp": "2026-03-23T23:35:29.258537",
+        "email_service": "tempmail"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] 任务已启动，开始执行真实链路探针",
+        "timestamp": "2026-03-23T23:35:29.258717"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] Token 同步与 Outlook refresh_token 探针已写入数据库",
+        "timestamp": "2026-03-23T23:35:29.462037"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] OTP 超时退避 #1: failures=1, delay=30",
+        "timestamp": "2026-03-23T23:35:29.618496"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] OTP 超时退避 #2: failures=2, delay=60",
+        "timestamp": "2026-03-23T23:35:29.772745"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] OTP 超时退避 #3: failures=3, delay=3600",
+        "timestamp": "2026-03-23T23:35:29.926635"
+      },
+      {
+        "type": "log",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "message": "[模拟] 批量计数探针已完成",
+        "timestamp": "2026-03-23T23:35:30.102423"
+      },
+      {
+        "type": "status",
+        "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+        "status": "completed",
+        "timestamp": "2026-03-23T23:35:30.287066",
+        "email": "mock-seeded-a8f4da41@example.test",
+        "email_service": "tempmail"
+      }
+    ],
+    "log_count": 6,
+    "status_count": 3,
+    "live_log_count": 6,
+    "final_status": "completed"
+  },
+  "task": {
+    "id": 5,
+    "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+    "status": "completed",
+    "email_service_id": null,
+    "proxy": null,
+    "logs": "[模拟] 任务已启动，开始执行真实链路探针\n[模拟] Token 同步与 Outlook refresh_token 探针已写入数据库\n[模拟] OTP 超时退避 #1: failures=1, delay=30\n[模拟] OTP 超时退避 #2: failures=2, delay=60\n[模拟] OTP 超时退避 #3: failures=3, delay=3600\n[模拟] 批量计数探针已完成\n[模拟] 任务完成，所有探针已收口",
+    "result": {
+      "email": "mock-seeded-a8f4da41@example.test",
+      "email_service": "tempmail",
+      "hardening_checks": {
+        "token_sync": {
+          "seeded_account_id": 4,
+          "tokenless_account_id": 5,
+          "partial_account_id": 6
+        },
+        "outlook_refresh": {
+          "service_id": 3,
+          "email": "mock-outlook-a8f4da41@example.test"
+        },
+        "batch_counter": {
+          "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472",
+          "task_uuids": [
+            "03c182b4-d5d3-4939-b2a0-eda844c402d9",
+            "224f2a9f-c0f3-4d97-8e92-4c2e772a675b",
+            "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8"
+          ],
+          "snapshot": {
+            "status": "completed",
+            "total": 3,
+            "completed": 3,
+            "success": 2,
+            "failed": 1,
+            "skipped": 0,
+            "cancelled": false,
+            "current_index": 0,
+            "finished": true,
+            "task_uuids": [
+              "03c182b4-d5d3-4939-b2a0-eda844c402d9",
+              "224f2a9f-c0f3-4d97-8e92-4c2e772a675b",
+              "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8"
+            ]
+          }
+        },
+        "otp_timeout_backoff": {
+          "service_id": 4,
+          "states": [
+            {
+              "failures": 1,
+              "delay_seconds": 30,
+              "opened_until": 1774308959.612146,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #1"
+            },
+            {
+              "failures": 2,
+              "delay_seconds": 60,
+              "opened_until": 1774308989.7684338,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #2"
+            },
+            {
+              "failures": 3,
+              "delay_seconds": 3600,
+              "opened_until": 1774312529.923651,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #3"
+            }
+          ]
+        }
+      }
+    },
+    "error_message": null,
+    "created_at": "2026-03-23T23:35:28.629402",
+    "started_at": "2026-03-23T23:35:29.251251",
+    "completed_at": "2026-03-23T23:35:30.252298"
+  },
+  "batch_api": {
+    "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472",
+    "total": 3,
+    "completed": 3,
+    "success": 2,
+    "failed": 1,
+    "current_index": 0,
+    "cancelled": false,
+    "finished": true,
+    "progress": "3/3"
+  },
+  "database": {
+    "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b",
+    "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472",
+    "seeded_account": {
+      "email": "mock-seeded-a8f4da41@example.test",
+      "access_token": "mock-access-token-seeded",
+      "refresh_token": "mock-refresh-token-seeded",
+      "token_sync_status": "pending"
+    },
+    "tokenless_account": {
+      "email": "mock-tokenless-a8f4da41@example.test",
+      "access_token": "mock-access-token-updated",
+      "refresh_token": null,
+      "token_sync_status": "pending"
+    },
+    "partial_account": {
+      "email": "mock-partial-a8f4da41@example.test",
+      "access_token": "mock-access-token-partial",
+      "refresh_token": "",
+      "token_sync_status": "pending"
+    },
+    "task_result": {
+      "email": "mock-seeded-a8f4da41@example.test",
+      "email_service": "tempmail",
+      "hardening_checks": {
+        "token_sync": {
+          "seeded_account_id": 4,
+          "tokenless_account_id": 5,
+          "partial_account_id": 6
+        },
+        "outlook_refresh": {
+          "service_id": 3,
+          "email": "mock-outlook-a8f4da41@example.test"
+        },
+        "batch_counter": {
+          "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472",
+          "task_uuids": [
+            "03c182b4-d5d3-4939-b2a0-eda844c402d9",
+            "224f2a9f-c0f3-4d97-8e92-4c2e772a675b",
+            "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8"
+          ],
+          "snapshot": {
+            "status": "completed",
+            "total": 3,
+            "completed": 3,
+            "success": 2,
+            "failed": 1,
+            "skipped": 0,
+            "cancelled": false,
+            "current_index": 0,
+            "finished": true,
+            "task_uuids": [
+              "03c182b4-d5d3-4939-b2a0-eda844c402d9",
+              "224f2a9f-c0f3-4d97-8e92-4c2e772a675b",
+              "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8"
+            ]
+          }
+        },
+        "otp_timeout_backoff": {
+          "service_id": 4,
+          "states": [
+            {
+              "failures": 1,
+              "delay_seconds": 30,
+              "opened_until": 1774308959.612146,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #1"
+            },
+            {
+              "failures": 2,
+              "delay_seconds": 60,
+              "opened_until": 1774308989.7684338,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #2"
+            },
+            {
+              "failures": 3,
+              "delay_seconds": 3600,
+              "opened_until": 1774312529.923651,
+              "retry_after": null,
+              "last_error": "模拟 OTP 超时 #3"
+            }
+          ]
+        }
+      }
+    },
+    "outlook_second_account": {
+      "email": "mock-outlook-a8f4da41@example.test",
+      "refresh_token": "new-second"
+    }
+  }
+}

tests_runtime/runtime_recovery_report_1774308869.json

@@ -0,0 +1,38 @@
+{
+  "mode": "verify-recovery",
+  "base_url": "http://127.0.0.1:15555",
+  "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db",
+  "state": {
+    "stale_task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95",
+    "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db",
+    "prepared_at": "2026-03-24 07:35:40"
+  },
+  "health": {
+    "status_code": 200,
+    "body": {
+      "total": 9,
+      "tasks": [
+        {
+          "id": 9,
+          "task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95",
+          "status": "failed",
+          "email_service_id": null,
+          "proxy": null,
+          "logs": "[00:00:00] stale task\n[系统] 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。",
+          "result": null,
+          "error_message": "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。",
+          "created_at": "2026-03-24T07:35:40",
+          "started_at": "2026-03-24T07:35:40",
+          "completed_at": "2026-03-23T23:35:57.292019"
+        }
+      ]
+    }
+  },
+  "recovery": {
+    "task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95",
+    "status": "failed",
+    "error_message": "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。",
+    "logs": "[00:00:00] stale task\n[系统] 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。",
+    "completed_at": "2026-03-23 23:35:57.292019"
+  }
+}

tests_runtime/runtime_recovery_state_1774308869.json

@@ -0,0 +1,5 @@
+{
+  "stale_task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95",
+  "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db",
+  "prepared_at": "2026-03-24 07:35:40"
+}