diff --git a/check_otp_timing.py b/check_otp_timing.py new file mode 100644 index 0000000..f7b1870 --- /dev/null +++ b/check_otp_timing.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +离线验证 TempmailService 的 OTP 时间锚点过滤行为。 + +场景 1: +- 30 秒内先后收到两封邮件 +- 在两封邮件之间设置新的 otp_sent_at +- 期望过滤第一封，命中第二封 + +场景 2: +- 第二封邮件已经入箱后才刷新 otp_sent_at +- 期望复现严格时间过滤导致第二封也被排除的窗口 +""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from typing import Any, Dict, List, Optional + +import src.services.tempmail as tempmail_module +from src.services.tempmail import TempmailService + + +@dataclass(frozen=True) +class Scenario: + name: str + anchor_offset_seconds: int + expected_code: Optional[str] + expected_message: str + + +class FakeResponse: + def __init__(self, payload: Dict[str, Any], status_code: int = 200): + self._payload = payload + self.status_code = status_code + + def json(self) -> Dict[str, Any]: + return self._payload + + +class FakeHTTPClient: + def __init__(self, payload: Dict[str, Any]): + self.payload = payload + self.calls: List[Dict[str, Any]] = [] + + def get(self, url: str, **kwargs: Any) -> FakeResponse: + self.calls.append({"url": url, "kwargs": kwargs}) + return FakeResponse(self.payload) + + +class FakeClock: + def __init__(self, start: float): + self.current = float(start) + + def time(self) -> float: + return self.current + + def sleep(self, seconds: float) -> None: + self.current += float(seconds) + + +def build_inbox_payload(base_timestamp: int) -> Dict[str, Any]: + return { + "emails": [ + { + "id": "mail-1", + "received_at": base_timestamp + 10, + "from": "noreply@openai.com", + "subject": "First OTP", + "body": "111111", + }, + { + "id": "mail-2", + "received_at": base_timestamp + 20, + "from": "noreply@openai.com", + "subject": "Second OTP", + "body": "222222", + }, + ] + } + + +def run_scenario(scenario: Scenario) -> Dict[str, Any]: + base_timestamp = 1_700_000_000 + service = TempmailService({"base_url": "https://api.tempmail.test"}) + service._email_cache["tester@example.com"] = {"token": "token-1"} + service.http_client = FakeHTTPClient(build_inbox_payload(base_timestamp)) + + fake_clock = FakeClock(start=base_timestamp + scenario.anchor_offset_seconds) + anchor_timestamp = fake_clock.time() + original_time = tempmail_module.time.time + original_sleep = tempmail_module.time.sleep + + try: + tempmail_module.time.time = fake_clock.time + tempmail_module.time.sleep = fake_clock.sleep + code = service.get_verification_code( + email="tester@example.com", + timeout=1, + otp_sent_at=anchor_timestamp, + ) + finally: + tempmail_module.time.time = original_time + tempmail_module.time.sleep = original_sleep + + passed = code == scenario.expected_code + return { + "name": scenario.name, + "anchor_timestamp": anchor_timestamp, + "code": code, + "passed": passed, + "http_calls": len(service.http_client.calls), + "message": scenario.expected_message, + } + + +def main() -> int: + logging.getLogger("src.services.tempmail").setLevel(logging.ERROR) + + scenarios = [ + Scenario( + name="anchor_between_two_emails", + anchor_offset_seconds=15, + expected_code="222222", + expected_message="新锚点位于两封邮件之间，第一封应被过滤，第二封应被命中。", + ), + Scenario( + name="anchor_set_after_second_email", + anchor_offset_seconds=21, + expected_code=None, + expected_message="锚点晚于第二封邮件时，严格大于过滤会把第二封也排除，复现登录阶段的竞态窗口。", + ), + ] + + print("Tempmail OTP timing check") + print("=========================") + + failed = False + for scenario in scenarios: + result = run_scenario(scenario) + status = "PASS" if result["passed"] else "FAIL" + print(f"{status} {result['name']}") + print(f" anchor_timestamp={result['anchor_timestamp']}") + print(f" returned_code={result['code']}") + print(f" inbox_polls={result['http_calls']}") + print(f" note={result['message']}") + if not result["passed"]: + failed = True + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/reviews/FUNCTIONAL-AVAILABILITY-REPORT-2026-03-24.md b/docs/reviews/FUNCTIONAL-AVAILABILITY-REPORT-2026-03-24.md new file mode 100644 index 0000000..db112f9 --- /dev/null +++ b/docs/reviews/FUNCTIONAL-AVAILABILITY-REPORT-2026-03-24.md @@ -0,0 +1,165 @@ +# 功能可用性实测报告 + +日期: 2026-03-24 +范围: +- 启动命令: `uv run python -m src.web.app` +- 监听地址: `http://127.0.0.1:15555` +- 隔离数据库: `tests_runtime/e2e_runtime_1774308869.db` +- 实测脚本: `tests/e2e/runtime_functionality_check.py` + +## 1. 执行摘要 + +本次按真实服务链路完成了以下验证: + +- 服务存活检查通过。 +- `POST /api/registration/create` 可创建受控模拟任务。 +- `GET /api/ws/task/{task_uuid}` 可实时推送日志与状态，任务完成时收到 `completed`。 +- 任务完成后数据库状态符合 Task 1、Task 5 预期。 +- 批量计数探针通过 `/api/registration/batch/{batch_id}` 验证，符合 Task 2 预期。 +- 重启后僵尸任务被自动标记失败，符合 Task 4 预期。 + +结论: + +- 本次新增的真实服务验证 harness 可用。 +- Task 1-5 中本次可通过真实服务直接观测的加固点均已生效。 + +## 2. 实测过程 + +### 2.1 端口处理 + +`15555` 端口初始被已有容器 `codex-manager-webui-1` 占用。为执行指定启动命令，先停止该容器，实测结束后已恢复。 + +### 2.2 执行命令 + +1. 静态验证 + - `uv run python -m pytest tests/test_account_token_sync_status.py tests/test_batch_task_manager.py tests/test_task_manager_status_broadcast.py tests/test_task_recovery.py tests/test_registration_email_service_failover.py` + - 结果: `16 passed` + - `uv run python -m py_compile src/web/app.py src/web/routes/registration.py tests/e2e/runtime_functionality_check.py` + - 结果: 退出码 `0` + +2. 真实服务启动 + - `APP_DATABASE_URL='sqlite:////Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db' APP_HOST='127.0.0.1' APP_PORT='15555' uv run python -m src.web.app` + +3. live 实测 + - `uv run python tests/e2e/runtime_functionality_check.py --mode live --base-url http://127.0.0.1:15555 --ws-url ws://127.0.0.1:15555 --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --report-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_functionality_report_1774308869.json` + +4. recovery 准备 + - `uv run python tests/e2e/runtime_functionality_check.py --mode prepare-recovery --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --state-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_state_1774308869.json` + +5. 服务重启后 recovery 实测 + - `uv run python tests/e2e/runtime_functionality_check.py --mode verify-recovery --base-url http://127.0.0.1:15555 --db-path /Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db --state-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_state_1774308869.json --report-path /Volumes/Work/code/codex-manager/tests_runtime/runtime_recovery_report_1774308869.json` + +## 3. 验证结果 + +### 3.1 服务存活 + +- `GET /api/registration/tasks?page=1&page_size=1` 返回 `200`。 + +### 3.2 模拟任务创建与 WebSocket + +- 创建任务 UUID: `a8f4da41-354c-4d89-9634-c582a032c70b` +- 批量探针 ID: `2e8cfce4-bf20-4f0b-8839-a94e8e141472` +- WebSocket 收到 3 条状态消息: + - `pending` + - `running` + - `completed` +- WebSocket 收到 6 条实时日志，包含: + - Token 同步探针写库 + - OTP 超时退避 3 次 + - 批量计数探针完成 + +判定: + +- 日志不是任务结束后一次性补发，而是在运行过程中实时推送。 + +### 3.3 Task 1 验证: Token 同步 + +数据库中以下账号状态正确: + +- `mock-seeded-a8f4da41@example.test` + - `access_token` 已保存 + - `refresh_token` 已保存 + - `token_sync_status = pending` + +- `mock-tokenless-a8f4da41@example.test` + - 先创建无 token，再更新 `access_token` + - `token_sync_status = pending` + +- `mock-partial-a8f4da41@example.test` + - 清空 `refresh_token` 后仍保留 `access_token` + - `token_sync_status = pending` + +Outlook 配置探针: + +- `mock-outlook-a8f4da41@example.test` + - `refresh_token` 已从 `old-second` 更新为 `new-second` + +### 3.4 Task 2 验证: 批量计数 + +`GET /api/registration/batch/2e8cfce4-bf20-4f0b-8839-a94e8e141472` 返回: + +- `total = 3` +- `completed = 3` +- `success = 2` +- `failed = 1` +- `finished = true` +- `progress = 3/3` + +判定: + +- 批量计数与任务结果一致，收口正确。 + +### 3.5 Task 3 验证: 单任务状态广播 + +任务完成时，WebSocket 最后一条状态消息为: + +- `status = completed` +- `email = mock-seeded-a8f4da41@example.test` +- `email_service = tempmail` + +判定: + +- 单任务状态广播已生效。 + +### 3.6 Task 4 验证: 僵尸任务恢复 + +重启前手工插入任务: + +- `stale-e738842e-74d8-400d-859e-1b283eab1a95` +- 初始状态: `running` + +重启后观测结果: + +- 状态变为 `failed` +- `error_message = 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。` +- `logs` 已追加系统收敛日志 +- `completed_at` 已写入 + +服务启动日志同时出现: + +- `已收敛 1 个僵尸任务: stale-e7` + +### 3.7 Task 5 验证: OTP 超时退避 + +模拟任务内部连续触发 3 次二阶段 OTP 超时，记录到任务结果: + +- 第 1 次: `failures = 1`, `delay_seconds = 30` +- 第 2 次: `failures = 2`, `delay_seconds = 60` +- 第 3 次: `failures = 3`, `delay_seconds = 3600` + +判定: + +- 深度冷却逻辑已生效。 + +## 4. 产物 + +- `tests/e2e/runtime_functionality_check.py` +- `tests_runtime/runtime_functionality_report_1774308869.json` +- `tests_runtime/runtime_recovery_report_1774308869.json` +- `tests_runtime/runtime_recovery_state_1774308869.json` +- `tests_runtime/e2e_runtime_1774308869.db` + +## 5. 观察到的问题 + +- `python -m src.web.app` 启动时会出现 `runpy` 的重复导入告警，但不影响服务启动与本次验证结果。 +- 启动日志中打印的 `host` 和 `database` 仍显示为数据库配置值 `0.0.0.0 / sqlite:///data/database.db`，与本次通过环境变量注入的真实运行参数不一致。实测链路实际使用了隔离数据库，但日志口径存在偏差。 diff --git a/probe_tempmail.py b/probe_tempmail.py new file mode 100644 index 0000000..a9255be --- /dev/null +++ b/probe_tempmail.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +Tempmail.lol API 探针。 + +用途： +1. 创建测试收件箱或复用现有 token。 +2. 拉取 /inbox 原始 JSON 并原样打印。 +3. 检查邮件对象里是否存在 received_at/date 等时间字段。 +""" + +import argparse +import json +import sys +import time +from typing import Any, Dict, Iterable + +import httpx + + +DEFAULT_BASE_URL = "https://api.tempmail.lol/v2" +TIME_FIELDS = ("received_at", "date", "created_at", "createdAt", "timestamp") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="抓取 Tempmail.lol 收件箱原始 JSON") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Tempmail API 基础地址") + parser.add_argument("--token", help="已有 inbox token；未提供时自动创建新邮箱") + parser.add_argument("--poll-count", type=int, default=1, help="轮询次数") + parser.add_argument("--poll-interval", type=float, default=3.0, help="轮询间隔秒数") + parser.add_argument("--timeout", type=float, default=20.0, help="HTTP 超时时间") + return parser.parse_args() + + +def dump_json(title: str, payload: Dict[str, Any]) -> None: + print(f"\n===== {title} =====") + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + + +def summarize_time_fields(emails: Iterable[Dict[str, Any]]) -> None: + for index, message in enumerate(emails, start=1): + present_fields = {name: message.get(name) for name in TIME_FIELDS if name in message} + print(f"email[{index}] 时间字段: {json.dumps(present_fields, ensure_ascii=False, default=str)}") + + +def create_inbox(client: httpx.Client, base_url: str) -> Dict[str, Any]: + response = client.post( + f"{base_url}/inbox/create", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + json={}, + ) + print(f"CREATE_STATUS {response.status_code}") + response.raise_for_status() + payload = response.json() + dump_json("CREATE_RESPONSE", payload) + return payload + + +def fetch_inbox(client: httpx.Client, base_url: str, token: str) -> Dict[str, Any]: + response = client.get( + f"{base_url}/inbox", + params={"token": token}, + headers={"Accept": "application/json"}, + ) + print(f"INBOX_STATUS {response.status_code}") + response.raise_for_status() + payload = response.json() + dump_json("INBOX_RESPONSE", payload) + emails = payload.get("emails", []) if isinstance(payload, dict) else [] + if isinstance(emails, list): + summarize_time_fields([mail for mail in emails if isinstance(mail, dict)]) + else: + print(f"emails 字段不是列表: {type(emails).__name__}") + return payload + + +def main() -> int: + args = parse_args() + with httpx.Client(timeout=args.timeout) as client: + token = args.token + if not token: + inbox = create_inbox(client, args.base_url) + token = str(inbox.get("token", "")).strip() + address = str(inbox.get("address", "")).strip() + print(f"ADDRESS {address}") + print(f"TOKEN {token}") + + if not token: + print("未拿到 token，无法继续拉取 inbox", file=sys.stderr) + return 1 + + for attempt in range(1, args.poll_count + 1): + print(f"\n----- poll {attempt}/{args.poll_count} -----") + fetch_inbox(client, args.base_url, token) + if attempt < args.poll_count: + time.sleep(args.poll_interval) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/runtime_functionality_check.py b/tests/e2e/runtime_functionality_check.py new file mode 100644 index 0000000..5a01e4c --- /dev/null +++ b/tests/e2e/runtime_functionality_check.py @@ -0,0 +1,278 @@ +import argparse +import asyncio +import json +import sqlite3 +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List + +import httpx +import websockets + + +STALE_ERROR = "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。" + + +def _write_json(path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _load_json(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _connect_db(db_path: Path) -> sqlite3.Connection: + return sqlite3.connect(db_path, timeout=5) + + +def _fetchone_dict(conn: sqlite3.Connection, sql: str, params: tuple[Any, ...]) -> Dict[str, Any]: + conn.row_factory = sqlite3.Row + row = conn.execute(sql, params).fetchone() + return dict(row) if row else {} + + +def _assert(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _health_check(client: httpx.Client, report: Dict[str, Any]) -> None: + response = client.get("/api/registration/tasks", params={"page": 1, "page_size": 1}) + report["health"] = {"status_code": response.status_code, "body": response.json()} + _assert(response.status_code == 200, "健康检查失败") + + +async def _collect_task_websocket(ws_url: str, task_uuid: str) -> Dict[str, Any]: + endpoint = f"{ws_url}/api/ws/task/{task_uuid}" + messages: List[Dict[str, Any]] = [] + started_at = time.time() + + async with websockets.connect(endpoint, open_timeout=10, close_timeout=5) as websocket: + while time.time() - started_at < 30: + raw_message = await asyncio.wait_for(websocket.recv(), timeout=10) + payload = json.loads(raw_message) + messages.append(payload) + if payload.get("type") == "status" and payload.get("status") in {"completed", "failed"}: + break + + logs = [message for message in messages if message.get("type") == "log"] + statuses = [message for message in messages if message.get("type") == "status"] + return { + "messages": messages, + "log_count": len(logs), + "status_count": len(statuses), + "live_log_count": sum(1 for message in logs if "timestamp" in message), + "final_status": statuses[-1]["status"] if statuses else None, + } + + +def _poll_task_completion(client: httpx.Client, task_uuid: str) -> Dict[str, Any]: + deadline = time.time() + 20 + while time.time() < deadline: + response = client.get(f"/api/registration/tasks/{task_uuid}") + response.raise_for_status() + payload = response.json() + if payload["status"] in {"completed", "failed"}: + return payload + time.sleep(0.2) + raise TimeoutError(f"任务未在预期时间内结束: {task_uuid}") + + +def _validate_live_database( + db_path: Path, + task_uuid: str, + batch_id: str, + checks: Dict[str, Any], + report: Dict[str, Any], +) -> None: + with _connect_db(db_path) as conn: + seeded = _fetchone_dict( + conn, + "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?", + (checks["seeded_account_email"],), + ) + tokenless = _fetchone_dict( + conn, + "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?", + (checks["tokenless_account_email"],), + ) + partial = _fetchone_dict( + conn, + "SELECT email, access_token, refresh_token, token_sync_status FROM accounts WHERE email = ?", + (checks["partial_account_email"],), + ) + task_row = _fetchone_dict( + conn, + "SELECT task_uuid, status, logs, result FROM registration_tasks WHERE task_uuid = ?", + (task_uuid,), + ) + outlook_row = _fetchone_dict( + conn, + "SELECT config FROM email_services WHERE id = ?", + (checks["outlook_service_id"],), + ) + + _assert(seeded.get("token_sync_status") == "pending", "seeded 账号 token_sync_status 异常") + _assert(tokenless.get("access_token") == "mock-access-token-updated", "tokenless 账号 access_token 未写入") + _assert(tokenless.get("token_sync_status") == "pending", "tokenless 账号 token_sync_status 异常") + _assert(partial.get("access_token") == "mock-access-token-partial", "partial 账号 access_token 丢失") + _assert(partial.get("refresh_token") == "", "partial 账号 refresh_token 未清空") + _assert(partial.get("token_sync_status") == "pending", "partial 账号 token_sync_status 异常") + _assert(task_row.get("status") == "completed", "模拟任务数据库状态不是 completed") + _assert(task_row.get("logs"), "模拟任务日志未落库") + + task_result = json.loads(task_row["result"]) if task_row.get("result") else {} + outlook_config = json.loads(outlook_row["config"]) if outlook_row.get("config") else {} + second_account = next( + account for account in outlook_config.get("accounts", []) + if account.get("email") == checks["outlook_account_email"] + ) + + batch_snapshot = task_result["hardening_checks"]["batch_counter"]["snapshot"] + backoff_states = task_result["hardening_checks"]["otp_timeout_backoff"]["states"] + + _assert(second_account["refresh_token"] == "new-second", "Outlook refresh_token 未更新") + _assert(batch_snapshot["completed"] == 3, "批量 completed 计数异常") + _assert(batch_snapshot["success"] == 2, "批量 success 计数异常") + _assert(batch_snapshot["failed"] == 1, "批量 failed 计数异常") + _assert(batch_snapshot["status"] == "completed", "批量状态异常") + _assert(batch_snapshot["finished"] is True, "批量 finished 标记异常") + _assert(backoff_states[-1]["delay_seconds"] == 3600, "OTP 深度冷却未生效") + _assert(backoff_states[-1]["failures"] == 3, "OTP 连续失败次数异常") + + report["database"] = { + "task_uuid": task_uuid, + "batch_id": batch_id, + "seeded_account": seeded, + "tokenless_account": tokenless, + "partial_account": partial, + "task_result": task_result, + "outlook_second_account": second_account, + } + + +def run_live_mode(base_url: str, ws_url: str, db_path: Path, report_path: Path) -> None: + report: Dict[str, Any] = {"mode": "live", "base_url": base_url, "db_path": str(db_path)} + with httpx.Client(base_url=base_url, timeout=httpx.Timeout(10, read=30)) as client: + _health_check(client, report) + + create_response = client.post( + "/api/registration/create", + json={ + "email_service_type": "tempmail", + "start_delay_ms": 600, + "log_delay_ms": 150, + }, + ) + create_response.raise_for_status() + created = create_response.json() + task_uuid = created["task"]["task_uuid"] + batch_id = created["batch_id"] + checks = created["checks"] + report["create"] = created + + ws_report = asyncio.run(_collect_task_websocket(ws_url, task_uuid)) + report["websocket"] = ws_report + _assert(ws_report["final_status"] == "completed", "WebSocket 未收到 completed 状态") + _assert(ws_report["log_count"] >= 4, "WebSocket 日志数量不足") + _assert(ws_report["live_log_count"] >= 1, "未捕获到实时日志广播") + + task_payload = _poll_task_completion(client, task_uuid) + report["task"] = task_payload + runtime_checks = { + **checks, + "outlook_service_id": task_payload["result"]["hardening_checks"]["outlook_refresh"]["service_id"], + "backoff_service_id": task_payload["result"]["hardening_checks"]["otp_timeout_backoff"]["service_id"], + } + + batch_response = client.get(f"/api/registration/batch/{batch_id}") + batch_response.raise_for_status() + report["batch_api"] = batch_response.json() + _assert(report["batch_api"]["completed"] == 3, "批量状态 API completed 异常") + _assert(report["batch_api"]["success"] == 2, "批量状态 API success 异常") + _assert(report["batch_api"]["failed"] == 1, "批量状态 API failed 异常") + _assert(report["batch_api"]["finished"] is True, "批量状态 API finished 异常") + + _validate_live_database(db_path, task_uuid, batch_id, runtime_checks, report) + _write_json(report_path, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +def run_prepare_recovery_mode(db_path: Path, state_path: Path) -> None: + stale_task_uuid = f"stale-{uuid.uuid4()}" + now = time.strftime("%Y-%m-%d %H:%M:%S") + with _connect_db(db_path) as conn: + conn.execute( + """ + INSERT INTO registration_tasks (task_uuid, status, logs, created_at, started_at) + VALUES (?, 'running', '[00:00:00] stale task', ?, ?) + """, + (stale_task_uuid, now, now), + ) + conn.commit() + + payload = { + "stale_task_uuid": stale_task_uuid, + "db_path": str(db_path), + "prepared_at": now, + } + _write_json(state_path, payload) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + + +def run_verify_recovery_mode(base_url: str, db_path: Path, state_path: Path, report_path: Path) -> None: + state = _load_json(state_path) + report: Dict[str, Any] = { + "mode": "verify-recovery", + "base_url": base_url, + "db_path": str(db_path), + "state": state, + } + + with httpx.Client(base_url=base_url, timeout=httpx.Timeout(10, read=30)) as client: + _health_check(client, report) + + with _connect_db(db_path) as conn: + stale_task = _fetchone_dict( + conn, + "SELECT task_uuid, status, error_message, logs, completed_at FROM registration_tasks WHERE task_uuid = ?", + (state["stale_task_uuid"],), + ) + + _assert(stale_task.get("status") == "failed", "僵尸任务未在重启后标记为 failed") + _assert(stale_task.get("error_message") == STALE_ERROR, "僵尸任务 error_message 不匹配") + _assert(STALE_ERROR in (stale_task.get("logs") or ""), "僵尸任务日志未追加系统收敛说明") + _assert(bool(stale_task.get("completed_at")), "僵尸任务 completed_at 缺失") + + report["recovery"] = stale_task + _write_json(report_path, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +def main() -> None: + parser = argparse.ArgumentParser(description="真实服务功能可用性验证脚本") + parser.add_argument("--mode", choices=["live", "prepare-recovery", "verify-recovery"], required=True) + parser.add_argument("--base-url", default="http://127.0.0.1:15555") + parser.add_argument("--ws-url", default="ws://127.0.0.1:15555") + parser.add_argument("--db-path", required=True) + parser.add_argument("--report-path", default="tests_runtime/runtime_functionality_report.json") + parser.add_argument("--state-path", default="tests_runtime/runtime_recovery_state.json") + args = parser.parse_args() + + db_path = Path(args.db_path).resolve() + report_path = Path(args.report_path).resolve() + state_path = Path(args.state_path).resolve() + + if args.mode == "live": + run_live_mode(args.base_url, args.ws_url, db_path, report_path) + return + if args.mode == "prepare-recovery": + run_prepare_recovery_mode(db_path, state_path) + return + run_verify_recovery_mode(args.base_url, db_path, state_path, report_path) + + +if __name__ == "__main__": + main() diff --git a/tests_runtime/runtime_functionality_report_1774308869.json b/tests_runtime/runtime_functionality_report_1774308869.json new file mode 100644 index 0000000..a59b599 --- /dev/null +++ b/tests_runtime/runtime_functionality_report_1774308869.json @@ -0,0 +1,292 @@ +{ + "mode": "live", + "base_url": "http://127.0.0.1:15555", + "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db", + "health": { + "status_code": 200, + "body": { + "total": 4, + "tasks": [ + { + "id": 4, + "task_uuid": "9079068e-e3f5-4fa7-8e1c-810ce1c352da", + "status": "completed", + "email_service_id": null, + "proxy": null, + "logs": null, + "result": null, + "error_message": null, + "created_at": "2026-03-23T23:34:58.715238", + "started_at": "2026-03-23T23:34:58.718370", + "completed_at": "2026-03-23T23:34:58.718376" + } + ] + } + }, + "create": { + "task": { + "id": 5, + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "status": "pending", + "email_service_id": null, + "proxy": null, + "logs": null, + "result": null, + "error_message": null, + "created_at": "2026-03-23T23:35:28.629402", + "started_at": null, + "completed_at": null + }, + "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472", + "checks": { + "seeded_account_email": "mock-seeded-a8f4da41@example.test", + "tokenless_account_email": "mock-tokenless-a8f4da41@example.test", + "partial_account_email": "mock-partial-a8f4da41@example.test", + "outlook_account_email": "mock-outlook-a8f4da41@example.test", + "backoff_service_name": "mock-backoff-a8f4da41" + } + }, + "websocket": { + "messages": [ + { + "type": "status", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "status": "pending" + }, + { + "type": "status", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "status": "running", + "timestamp": "2026-03-23T23:35:29.258537", + "email_service": "tempmail" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] 任务已启动，开始执行真实链路探针", + "timestamp": "2026-03-23T23:35:29.258717" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] Token 同步与 Outlook refresh_token 探针已写入数据库", + "timestamp": "2026-03-23T23:35:29.462037" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] OTP 超时退避 #1: failures=1, delay=30", + "timestamp": "2026-03-23T23:35:29.618496" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] OTP 超时退避 #2: failures=2, delay=60", + "timestamp": "2026-03-23T23:35:29.772745" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] OTP 超时退避 #3: failures=3, delay=3600", + "timestamp": "2026-03-23T23:35:29.926635" + }, + { + "type": "log", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "message": "[模拟] 批量计数探针已完成", + "timestamp": "2026-03-23T23:35:30.102423" + }, + { + "type": "status", + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "status": "completed", + "timestamp": "2026-03-23T23:35:30.287066", + "email": "mock-seeded-a8f4da41@example.test", + "email_service": "tempmail" + } + ], + "log_count": 6, + "status_count": 3, + "live_log_count": 6, + "final_status": "completed" + }, + "task": { + "id": 5, + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "status": "completed", + "email_service_id": null, + "proxy": null, + "logs": "[模拟] 任务已启动，开始执行真实链路探针\n[模拟] Token 同步与 Outlook refresh_token 探针已写入数据库\n[模拟] OTP 超时退避 #1: failures=1, delay=30\n[模拟] OTP 超时退避 #2: failures=2, delay=60\n[模拟] OTP 超时退避 #3: failures=3, delay=3600\n[模拟] 批量计数探针已完成\n[模拟] 任务完成，所有探针已收口", + "result": { + "email": "mock-seeded-a8f4da41@example.test", + "email_service": "tempmail", + "hardening_checks": { + "token_sync": { + "seeded_account_id": 4, + "tokenless_account_id": 5, + "partial_account_id": 6 + }, + "outlook_refresh": { + "service_id": 3, + "email": "mock-outlook-a8f4da41@example.test" + }, + "batch_counter": { + "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472", + "task_uuids": [ + "03c182b4-d5d3-4939-b2a0-eda844c402d9", + "224f2a9f-c0f3-4d97-8e92-4c2e772a675b", + "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8" + ], + "snapshot": { + "status": "completed", + "total": 3, + "completed": 3, + "success": 2, + "failed": 1, + "skipped": 0, + "cancelled": false, + "current_index": 0, + "finished": true, + "task_uuids": [ + "03c182b4-d5d3-4939-b2a0-eda844c402d9", + "224f2a9f-c0f3-4d97-8e92-4c2e772a675b", + "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8" + ] + } + }, + "otp_timeout_backoff": { + "service_id": 4, + "states": [ + { + "failures": 1, + "delay_seconds": 30, + "opened_until": 1774308959.612146, + "retry_after": null, + "last_error": "模拟 OTP 超时 #1" + }, + { + "failures": 2, + "delay_seconds": 60, + "opened_until": 1774308989.7684338, + "retry_after": null, + "last_error": "模拟 OTP 超时 #2" + }, + { + "failures": 3, + "delay_seconds": 3600, + "opened_until": 1774312529.923651, + "retry_after": null, + "last_error": "模拟 OTP 超时 #3" + } + ] + } + } + }, + "error_message": null, + "created_at": "2026-03-23T23:35:28.629402", + "started_at": "2026-03-23T23:35:29.251251", + "completed_at": "2026-03-23T23:35:30.252298" + }, + "batch_api": { + "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472", + "total": 3, + "completed": 3, + "success": 2, + "failed": 1, + "current_index": 0, + "cancelled": false, + "finished": true, + "progress": "3/3" + }, + "database": { + "task_uuid": "a8f4da41-354c-4d89-9634-c582a032c70b", + "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472", + "seeded_account": { + "email": "mock-seeded-a8f4da41@example.test", + "access_token": "mock-access-token-seeded", + "refresh_token": "mock-refresh-token-seeded", + "token_sync_status": "pending" + }, + "tokenless_account": { + "email": "mock-tokenless-a8f4da41@example.test", + "access_token": "mock-access-token-updated", + "refresh_token": null, + "token_sync_status": "pending" + }, + "partial_account": { + "email": "mock-partial-a8f4da41@example.test", + "access_token": "mock-access-token-partial", + "refresh_token": "", + "token_sync_status": "pending" + }, + "task_result": { + "email": "mock-seeded-a8f4da41@example.test", + "email_service": "tempmail", + "hardening_checks": { + "token_sync": { + "seeded_account_id": 4, + "tokenless_account_id": 5, + "partial_account_id": 6 + }, + "outlook_refresh": { + "service_id": 3, + "email": "mock-outlook-a8f4da41@example.test" + }, + "batch_counter": { + "batch_id": "2e8cfce4-bf20-4f0b-8839-a94e8e141472", + "task_uuids": [ + "03c182b4-d5d3-4939-b2a0-eda844c402d9", + "224f2a9f-c0f3-4d97-8e92-4c2e772a675b", + "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8" + ], + "snapshot": { + "status": "completed", + "total": 3, + "completed": 3, + "success": 2, + "failed": 1, + "skipped": 0, + "cancelled": false, + "current_index": 0, + "finished": true, + "task_uuids": [ + "03c182b4-d5d3-4939-b2a0-eda844c402d9", + "224f2a9f-c0f3-4d97-8e92-4c2e772a675b", + "6c4f0e18-47b1-473a-9cc5-83ef09e33ff8" + ] + } + }, + "otp_timeout_backoff": { + "service_id": 4, + "states": [ + { + "failures": 1, + "delay_seconds": 30, + "opened_until": 1774308959.612146, + "retry_after": null, + "last_error": "模拟 OTP 超时 #1" + }, + { + "failures": 2, + "delay_seconds": 60, + "opened_until": 1774308989.7684338, + "retry_after": null, + "last_error": "模拟 OTP 超时 #2" + }, + { + "failures": 3, + "delay_seconds": 3600, + "opened_until": 1774312529.923651, + "retry_after": null, + "last_error": "模拟 OTP 超时 #3" + } + ] + } + } + }, + "outlook_second_account": { + "email": "mock-outlook-a8f4da41@example.test", + "refresh_token": "new-second" + } + } +} \ No newline at end of file diff --git a/tests_runtime/runtime_recovery_report_1774308869.json b/tests_runtime/runtime_recovery_report_1774308869.json new file mode 100644 index 0000000..88d2fe4 --- /dev/null +++ b/tests_runtime/runtime_recovery_report_1774308869.json @@ -0,0 +1,38 @@ +{ + "mode": "verify-recovery", + "base_url": "http://127.0.0.1:15555", + "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db", + "state": { + "stale_task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95", + "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db", + "prepared_at": "2026-03-24 07:35:40" + }, + "health": { + "status_code": 200, + "body": { + "total": 9, + "tasks": [ + { + "id": 9, + "task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95", + "status": "failed", + "email_service_id": null, + "proxy": null, + "logs": "[00:00:00] stale task\n[系统] 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。", + "result": null, + "error_message": "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。", + "created_at": "2026-03-24T07:35:40", + "started_at": "2026-03-24T07:35:40", + "completed_at": "2026-03-23T23:35:57.292019" + } + ] + } + }, + "recovery": { + "task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95", + "status": "failed", + "error_message": "服务启动时检测到未完成的历史任务，已标记失败，请重新发起。", + "logs": "[00:00:00] stale task\n[系统] 服务启动时检测到未完成的历史任务，已标记失败，请重新发起。", + "completed_at": "2026-03-23 23:35:57.292019" + } +} \ No newline at end of file diff --git a/tests_runtime/runtime_recovery_state_1774308869.json b/tests_runtime/runtime_recovery_state_1774308869.json new file mode 100644 index 0000000..78643e5 --- /dev/null +++ b/tests_runtime/runtime_recovery_state_1774308869.json @@ -0,0 +1,5 @@ +{ + "stale_task_uuid": "stale-e738842e-74d8-400d-859e-1b283eab1a95", + "db_path": "/Volumes/Work/code/codex-manager/tests_runtime/e2e_runtime_1774308869.db", + "prepared_at": "2026-03-24 07:35:40" +} \ No newline at end of file