mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-04 23:17:20 +08:00
fix(runtime): 收敛模块关闭线程所有权 (#6443)
* fix(runtime): bound module shutdown owners * fix(runtime): declare blocking shutdown owners --------- Co-authored-by: jxxghp <jxxghp@gmail.com>
This commit is contained in:
@@ -84,7 +84,7 @@ class Telegram:
|
|||||||
_typing_command_max_duration_seconds = 30
|
_typing_command_max_duration_seconds = 30
|
||||||
_typing_callback_max_duration_seconds = 60
|
_typing_callback_max_duration_seconds = 60
|
||||||
_typing_join_timeout_seconds = 1
|
_typing_join_timeout_seconds = 1
|
||||||
_polling_join_timeout_seconds = 10
|
_shutdown_timeout_seconds = 10
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -1743,9 +1743,25 @@ class Telegram:
|
|||||||
# 清理菜单命令
|
# 清理菜单命令
|
||||||
self._bot.delete_my_commands()
|
self._bot.delete_my_commands()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _stop_bot_with_deadline(bot: TeleBot, deadline: float) -> bool:
|
||||||
|
"""停止 SDK polling,并在共享 deadline 内等待 worker 收敛。"""
|
||||||
|
bot.stop_polling()
|
||||||
|
if not bot.threaded or not bot.worker_pool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
workers = tuple(bot.worker_pool.workers)
|
||||||
|
for worker in workers:
|
||||||
|
worker.stop()
|
||||||
|
for worker in workers:
|
||||||
|
if worker is threading.current_thread():
|
||||||
|
continue
|
||||||
|
worker.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||||
|
return all(not worker.is_alive() for worker in workers)
|
||||||
|
|
||||||
def stop(self) -> bool:
|
def stop(self) -> bool:
|
||||||
"""
|
"""
|
||||||
停止 Telegram 消息接收服务,并返回 polling/typing owner 是否收敛。
|
停止 Telegram 消息接收服务,并返回 SDK/polling/typing owner 是否收敛。
|
||||||
"""
|
"""
|
||||||
converged = True
|
converged = True
|
||||||
with self._typing_lifecycle_lock:
|
with self._typing_lifecycle_lock:
|
||||||
@@ -1757,16 +1773,24 @@ class Telegram:
|
|||||||
|
|
||||||
bot = self._bot
|
bot = self._bot
|
||||||
polling_thread = self._polling_thread
|
polling_thread = self._polling_thread
|
||||||
|
deadline = time.monotonic() + self._shutdown_timeout_seconds
|
||||||
|
transport_converged = True
|
||||||
if bot:
|
if bot:
|
||||||
bot.stop_bot()
|
if not self._stop_bot_with_deadline(bot, deadline):
|
||||||
|
converged = False
|
||||||
|
transport_converged = False
|
||||||
|
logger.error("Telegram SDK worker 未在关闭预算内退出")
|
||||||
if (
|
if (
|
||||||
polling_thread
|
polling_thread
|
||||||
and polling_thread.is_alive()
|
and polling_thread.is_alive()
|
||||||
and polling_thread is not threading.current_thread()
|
and polling_thread is not threading.current_thread()
|
||||||
):
|
):
|
||||||
polling_thread.join(timeout=self._polling_join_timeout_seconds)
|
polling_thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||||
if polling_thread and polling_thread.is_alive():
|
if polling_thread and polling_thread.is_alive():
|
||||||
logger.error("Telegram polling 线程未在关闭预算内退出")
|
logger.error("Telegram polling 线程未在关闭预算内退出")
|
||||||
|
converged = False
|
||||||
|
transport_converged = False
|
||||||
|
if not transport_converged:
|
||||||
return False
|
return False
|
||||||
self._polling_thread = None
|
self._polling_thread = None
|
||||||
self._bot = None
|
self._bot = None
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from app.runtime.extensions.module_manager import ModuleManager
|
|||||||
from app.runtime.extensions.module.dispatcher import ModuleInvocationDispatcher
|
from app.runtime.extensions.module.dispatcher import ModuleInvocationDispatcher
|
||||||
from app.runtime.extensions.plugin_manager import PluginManager
|
from app.runtime.extensions.plugin_manager import PluginManager
|
||||||
from app.runtime.events import EventHandlerBinding, EventManager
|
from app.runtime.events import EventHandlerBinding, EventManager
|
||||||
|
from app.runtime.execution import run_in_threadpool_to_completion
|
||||||
from app.runtime.observability import record_metric
|
from app.runtime.observability import record_metric
|
||||||
from app.runtime.state import SystemHelper
|
from app.runtime.state import SystemHelper
|
||||||
from app.runtime.settings import configure_runtime_setting_provider
|
from app.runtime.settings import configure_runtime_setting_provider
|
||||||
@@ -611,12 +612,16 @@ async def stop_modules() -> bool:
|
|||||||
name: str,
|
name: str,
|
||||||
callback: Callable[[], object],
|
callback: Callable[[], object],
|
||||||
*,
|
*,
|
||||||
|
offload: bool = False,
|
||||||
record_failure: bool = True,
|
record_failure: bool = True,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""执行单个关闭步骤,失败时继续收口并保留诚实结果。"""
|
"""执行单个关闭步骤,失败时继续收口并保留诚实结果。"""
|
||||||
nonlocal all_converged
|
nonlocal all_converged
|
||||||
try:
|
try:
|
||||||
result = callback()
|
if offload:
|
||||||
|
result = await run_in_threadpool_to_completion(callback)
|
||||||
|
else:
|
||||||
|
result = callback()
|
||||||
if inspect.isawaitable(result):
|
if inspect.isawaitable(result):
|
||||||
result = await result
|
result = await result
|
||||||
converged = result is not False
|
converged = result is not False
|
||||||
@@ -633,14 +638,14 @@ async def stop_modules() -> bool:
|
|||||||
return converged
|
return converged
|
||||||
|
|
||||||
await run_step("图片代理安全日志合并器", close_image_proxy_block_log_coalescer)
|
await run_step("图片代理安全日志合并器", close_image_proxy_block_log_coalescer)
|
||||||
await run_step("模块", lambda: ModuleManager().shutdown())
|
await run_step("模块", lambda: ModuleManager().shutdown(), offload=True)
|
||||||
await run_step("事件消费", lambda: EventManager().stop_async())
|
await run_step("事件消费", lambda: EventManager().stop_async())
|
||||||
await run_step("浏览器会话", close_browser_sessions)
|
await run_step("浏览器会话", close_browser_sessions, offload=True)
|
||||||
await run_step("托管资源", stop_managed_resources)
|
await run_step("托管资源", stop_managed_resources)
|
||||||
await run_step("DoH服务", lambda: DohHelper().shutdown())
|
await run_step("DoH服务", lambda: DohHelper().shutdown(), offload=True)
|
||||||
await run_step("线程池", lambda: ThreadHelper().shutdown())
|
await run_step("线程池", lambda: ThreadHelper().shutdown(), offload=True)
|
||||||
await run_step("消息服务", stop_message)
|
await run_step("消息服务", stop_message, offload=True)
|
||||||
await run_step("Redis缓存连接", lambda: RedisHelper().close())
|
await run_step("Redis缓存连接", lambda: RedisHelper().close(), offload=True)
|
||||||
await run_step("异步Redis缓存连接", lambda: AsyncRedisHelper().close())
|
await run_step("异步Redis缓存连接", lambda: AsyncRedisHelper().close())
|
||||||
# Web Agent 的取消 finally 可能还要写入最终展示快照,必须先完成任务收尾,再关闭写入准入。
|
# Web Agent 的取消 finally 可能还要写入最终展示快照,必须先完成任务收尾,再关闭写入准入。
|
||||||
web_agent_drained = await run_step(
|
web_agent_drained = await run_step(
|
||||||
@@ -672,8 +677,8 @@ async def stop_modules() -> bool:
|
|||||||
else:
|
else:
|
||||||
all_converged = False
|
all_converged = False
|
||||||
logger.error("数据库任务未收敛,跳过数据库连接关闭以避免运行中事务使用已释放连接")
|
logger.error("数据库任务未收敛,跳过数据库连接关闭以避免运行中事务使用已释放连接")
|
||||||
await run_step("前端服务", stop_frontend)
|
await run_step("前端服务", stop_frontend, offload=True)
|
||||||
await run_step("临时文件", clear_temp)
|
await run_step("临时文件", clear_temp, offload=True)
|
||||||
return all_converged
|
return all_converged
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import asyncio
|
|||||||
import inspect
|
import inspect
|
||||||
import time
|
import time
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from typing import Callable
|
from typing import Awaitable, Callable
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
@@ -33,6 +33,7 @@ from app.foundation.environment import is_free_threaded_runtime, is_gil_enabled
|
|||||||
|
|
||||||
settings = RuntimeSettingsCompat()
|
settings = RuntimeSettingsCompat()
|
||||||
from app.runtime.health import get_application_health
|
from app.runtime.health import get_application_health
|
||||||
|
from app.runtime.execution import run_in_threadpool_to_completion
|
||||||
from app.runtime.topology import validate_process_topology
|
from app.runtime.topology import validate_process_topology
|
||||||
from app.runtime.tasks import TaskRegistry, configure_task_registry
|
from app.runtime.tasks import TaskRegistry, configure_task_registry
|
||||||
from app.adapters.external.server import MoviePilotServerHelper
|
from app.adapters.external.server import MoviePilotServerHelper
|
||||||
@@ -135,32 +136,38 @@ async def run_shutdown_step(
|
|||||||
timeout_seconds: float | None = None,
|
timeout_seconds: float | None = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛。"""
|
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛。"""
|
||||||
try:
|
|
||||||
|
async def invoke() -> object:
|
||||||
|
"""在主循环调用 owner,并等待其可能返回的异步结果。"""
|
||||||
result = callback()
|
result = callback()
|
||||||
if inspect.isawaitable(result):
|
if inspect.isawaitable(result):
|
||||||
task = asyncio.ensure_future(result)
|
return await result
|
||||||
|
return result
|
||||||
|
|
||||||
def _consume_shutdown_result(done: asyncio.Future) -> None:
|
try:
|
||||||
"""消费延迟收敛任务的最终异常,避免事件循环产生未取回异常。"""
|
task = asyncio.create_task(invoke(), name=f"shutdown.{name}")
|
||||||
try:
|
|
||||||
done.result()
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
pass
|
|
||||||
except Exception as err:
|
|
||||||
logger.error(f"关闭{name}最终收尾失败:{err}")
|
|
||||||
|
|
||||||
task.add_done_callback(_consume_shutdown_result)
|
def _consume_shutdown_result(done: asyncio.Future) -> None:
|
||||||
if timeout_seconds:
|
"""消费延迟收敛任务的最终异常,避免事件循环产生未取回异常。"""
|
||||||
try:
|
try:
|
||||||
result = await asyncio.wait_for(
|
done.result()
|
||||||
asyncio.shield(task), timeout=timeout_seconds
|
except asyncio.CancelledError:
|
||||||
)
|
pass
|
||||||
except asyncio.TimeoutError:
|
except Exception as err:
|
||||||
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
|
logger.error(f"关闭{name}最终收尾失败:{err}")
|
||||||
task.cancel()
|
|
||||||
return False
|
task.add_done_callback(_consume_shutdown_result)
|
||||||
else:
|
if timeout_seconds:
|
||||||
result = await task
|
try:
|
||||||
|
result = await asyncio.wait_for(
|
||||||
|
asyncio.shield(task), timeout=timeout_seconds
|
||||||
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
|
||||||
|
task.cancel()
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
result = await task
|
||||||
if result is False:
|
if result is False:
|
||||||
logger.error("关闭%s未收敛,资源所有权保持不变", name)
|
logger.error("关闭%s未收敛,资源所有权保持不变", name)
|
||||||
return False
|
return False
|
||||||
@@ -170,6 +177,17 @@ async def run_shutdown_step(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def offload_shutdown_callback(
|
||||||
|
callback: Callable[[], object],
|
||||||
|
) -> Callable[[], Awaitable[object]]:
|
||||||
|
"""把明确会阻塞的同步关闭 owner 包装为异步生命周期回调。"""
|
||||||
|
|
||||||
|
async def invoke() -> object:
|
||||||
|
return await run_in_threadpool_to_completion(callback)
|
||||||
|
|
||||||
|
return invoke
|
||||||
|
|
||||||
|
|
||||||
async def run_startup_step(
|
async def run_startup_step(
|
||||||
name: str,
|
name: str,
|
||||||
callback: Callable[[], object],
|
callback: Callable[[], object],
|
||||||
@@ -384,7 +402,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
|||||||
dependencies=("插件备份恢复",),
|
dependencies=("插件备份恢复",),
|
||||||
mode=LifecycleMode.NORMAL_ONLY,
|
mode=LifecycleMode.NORMAL_ONLY,
|
||||||
start=init_plugins,
|
start=init_plugins,
|
||||||
stop=finalize_plugins,
|
stop=offload_shutdown_callback(finalize_plugins),
|
||||||
start_order=90,
|
start_order=90,
|
||||||
stop_order=60,
|
stop_order=60,
|
||||||
start_timeout_seconds=300,
|
start_timeout_seconds=300,
|
||||||
@@ -395,7 +413,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
|||||||
name="插件变更监控",
|
name="插件变更监控",
|
||||||
dependencies=("插件",),
|
dependencies=("插件",),
|
||||||
mode=LifecycleMode.NORMAL_ONLY,
|
mode=LifecycleMode.NORMAL_ONLY,
|
||||||
stop=stop_plugin_monitor,
|
stop=offload_shutdown_callback(stop_plugin_monitor),
|
||||||
stop_order=8,
|
stop_order=8,
|
||||||
stop_timeout_seconds=10,
|
stop_timeout_seconds=10,
|
||||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||||
@@ -405,7 +423,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
|||||||
dependencies=("插件",),
|
dependencies=("插件",),
|
||||||
mode=LifecycleMode.NORMAL_ONLY,
|
mode=LifecycleMode.NORMAL_ONLY,
|
||||||
start=init_scheduler,
|
start=init_scheduler,
|
||||||
stop=stop_scheduler,
|
stop=offload_shutdown_callback(stop_scheduler),
|
||||||
start_order=100,
|
start_order=100,
|
||||||
stop_order=50,
|
stop_order=50,
|
||||||
start_timeout_seconds=120,
|
start_timeout_seconds=120,
|
||||||
@@ -496,7 +514,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
|||||||
dependencies=("命令服务",),
|
dependencies=("命令服务",),
|
||||||
mode=LifecycleMode.NORMAL_ONLY,
|
mode=LifecycleMode.NORMAL_ONLY,
|
||||||
start=init_workflow,
|
start=init_workflow,
|
||||||
stop=stop_workflow,
|
stop=offload_shutdown_callback(stop_workflow),
|
||||||
start_order=140,
|
start_order=140,
|
||||||
stop_order=20,
|
stop_order=20,
|
||||||
start_timeout_seconds=120,
|
start_timeout_seconds=120,
|
||||||
@@ -507,7 +525,9 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
|||||||
name="插件备份",
|
name="插件备份",
|
||||||
dependencies=("插件",),
|
dependencies=("插件",),
|
||||||
mode=LifecycleMode.NORMAL_ONLY,
|
mode=LifecycleMode.NORMAL_ONLY,
|
||||||
stop=lambda: SystemChain().backup_plugins(),
|
stop=offload_shutdown_callback(
|
||||||
|
lambda: SystemChain().backup_plugins()
|
||||||
|
),
|
||||||
stop_order=10,
|
stop_order=10,
|
||||||
stop_timeout_seconds=300,
|
stop_timeout_seconds=300,
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -224,6 +224,22 @@ preflight 会验证 Python 3.14、GIL 状态、`thread_inherit_context`、MovieP
|
|||||||
依赖、语义、驱动、启动或样本完整性不成立。该工具只用于隔离的本地长 A/B,不接真实凭据、用户数据库、
|
依赖、语义、驱动、启动或样本完整性不成立。该工具只用于隔离的本地长 A/B,不接真实凭据、用户数据库、
|
||||||
媒体目录或外网,也不加入常规 CI。
|
媒体目录或外网,也不加入常规 CI。
|
||||||
|
|
||||||
|
## 模块关闭事件循环 A/B
|
||||||
|
|
||||||
|
`module_shutdown_ab.py` 使用隔离资源 owner,分别测量 `stop_modules()` 内部同步关闭和生命周期总入口的
|
||||||
|
同步关闭。默认制造 50ms 同步等待,并观察 10ms 心跳是否在关闭完成前执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
../.venv-test/bin/python scripts/perf/module_shutdown_ab.py \
|
||||||
|
--block-ms 50 \
|
||||||
|
--heartbeat-ms 10 \
|
||||||
|
--samples 7
|
||||||
|
```
|
||||||
|
|
||||||
|
分别在 Before/After revision 运行相同参数并保存 JSON。总关闭耗时应保持接近固定等待时间;心跳中位延迟
|
||||||
|
用于判断事件循环是否被同步 owner 占用,所有样本在关闭完成前执行心跳属于正确性门禁。探针不启动真实
|
||||||
|
模块、线程池、数据库、配置或网络。
|
||||||
|
|
||||||
## TaskRegistry 跨线程提交 A/B
|
## TaskRegistry 跨线程提交 A/B
|
||||||
|
|
||||||
`task_registry_ab.py` 验证目标事件循环尚未分发 callback 时执行 shutdown,pending completion 与原始
|
`task_registry_ab.py` 验证目标事件循环尚未分发 callback 时执行 shutdown,pending completion 与原始
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
"""测量同步资源关闭期间的事件循环响应与关闭总耗时。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Awaitable, Callable
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
if str(PROJECT_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
from app.testing.bootstrap import install_sites_stub, isolate_config_dir
|
||||||
|
|
||||||
|
isolate_config_dir()
|
||||||
|
install_sites_stub()
|
||||||
|
|
||||||
|
from app.startup import lifecycle
|
||||||
|
from app.startup.initializers import modules as modules_initializer
|
||||||
|
|
||||||
|
|
||||||
|
async def _noop_async() -> None:
|
||||||
|
"""提供不产生外部副作用的异步关闭替身。"""
|
||||||
|
|
||||||
|
|
||||||
|
def _noop_sync() -> None:
|
||||||
|
"""提供不产生外部副作用的同步关闭替身。"""
|
||||||
|
|
||||||
|
|
||||||
|
def configure_module_probe(block_seconds: float) -> None:
|
||||||
|
"""把模块关闭依赖替换为隔离 owner,仅保留一个固定同步等待。"""
|
||||||
|
|
||||||
|
def blocking_module_shutdown() -> bool:
|
||||||
|
time.sleep(block_seconds)
|
||||||
|
return True
|
||||||
|
|
||||||
|
modules_initializer.ModuleManager = lambda: SimpleNamespace(
|
||||||
|
shutdown=blocking_module_shutdown
|
||||||
|
)
|
||||||
|
modules_initializer.EventManager = lambda: SimpleNamespace(
|
||||||
|
stop_async=_noop_async
|
||||||
|
)
|
||||||
|
modules_initializer.DohHelper = lambda: SimpleNamespace(shutdown=_noop_sync)
|
||||||
|
modules_initializer.ThreadHelper = lambda: SimpleNamespace(shutdown=_noop_sync)
|
||||||
|
modules_initializer.RedisHelper = lambda: SimpleNamespace(close=_noop_sync)
|
||||||
|
modules_initializer.AsyncRedisHelper = lambda: SimpleNamespace(close=_noop_async)
|
||||||
|
modules_initializer.close_image_proxy_block_log_coalescer = _noop_async
|
||||||
|
modules_initializer.close_browser_sessions = _noop_sync
|
||||||
|
modules_initializer.stop_managed_resources = _noop_async
|
||||||
|
modules_initializer.stop_message = _noop_sync
|
||||||
|
modules_initializer.shutdown_web_agent_background_tasks = _noop_async
|
||||||
|
modules_initializer.wait_web_agent_background_tasks = _noop_async
|
||||||
|
modules_initializer.get_configured_agent_chat_persistence = lambda: SimpleNamespace(
|
||||||
|
begin_shutdown=_noop_sync,
|
||||||
|
shutdown=_noop_async,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def stop_database_worker() -> None:
|
||||||
|
modules_initializer._database_worker = None
|
||||||
|
|
||||||
|
modules_initializer.stop_database_worker = stop_database_worker
|
||||||
|
modules_initializer.close_database = _noop_async
|
||||||
|
modules_initializer.stop_frontend = _noop_sync
|
||||||
|
modules_initializer.clear_temp = _noop_sync
|
||||||
|
modules_initializer._database_worker = None
|
||||||
|
|
||||||
|
|
||||||
|
async def sample(
|
||||||
|
shutdown: Callable[[], Awaitable[object]],
|
||||||
|
*,
|
||||||
|
heartbeat_seconds: float,
|
||||||
|
) -> dict[str, float | bool]:
|
||||||
|
"""执行一次关闭并测量同期心跳的实际唤醒延迟。"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
started_at = loop.time()
|
||||||
|
heartbeat_at: float | None = None
|
||||||
|
|
||||||
|
async def heartbeat() -> None:
|
||||||
|
nonlocal heartbeat_at
|
||||||
|
await asyncio.sleep(heartbeat_seconds)
|
||||||
|
heartbeat_at = loop.time()
|
||||||
|
|
||||||
|
heartbeat_task = asyncio.create_task(heartbeat())
|
||||||
|
await shutdown()
|
||||||
|
shutdown_finished_at = loop.time()
|
||||||
|
completed_before_shutdown = heartbeat_task.done()
|
||||||
|
await heartbeat_task
|
||||||
|
assert heartbeat_at is not None
|
||||||
|
return {
|
||||||
|
"heartbeat_delay_ms": (heartbeat_at - started_at) * 1000,
|
||||||
|
"heartbeat_completed_before_shutdown": completed_before_shutdown,
|
||||||
|
"shutdown_ms": (shutdown_finished_at - started_at) * 1000,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def run_samples(
|
||||||
|
*,
|
||||||
|
block_seconds: float,
|
||||||
|
heartbeat_seconds: float,
|
||||||
|
samples: int,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""顺序采集模块内部和生命周期总入口两类同步关闭样本。"""
|
||||||
|
configure_module_probe(block_seconds)
|
||||||
|
module_samples = [
|
||||||
|
await sample(
|
||||||
|
modules_initializer.stop_modules,
|
||||||
|
heartbeat_seconds=heartbeat_seconds,
|
||||||
|
)
|
||||||
|
for _ in range(samples)
|
||||||
|
]
|
||||||
|
lifecycle_samples = [
|
||||||
|
await sample(
|
||||||
|
lambda: lifecycle.run_shutdown_step(
|
||||||
|
"probe.sync_owner",
|
||||||
|
lifecycle.offload_shutdown_callback(
|
||||||
|
lambda: time.sleep(block_seconds)
|
||||||
|
),
|
||||||
|
timeout_seconds=max(1.0, block_seconds * 4),
|
||||||
|
),
|
||||||
|
heartbeat_seconds=heartbeat_seconds,
|
||||||
|
)
|
||||||
|
for _ in range(samples)
|
||||||
|
]
|
||||||
|
|
||||||
|
def summarize(values: list[dict[str, float | bool]]) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"samples": values,
|
||||||
|
"heartbeat_delay_median_ms": statistics.median(
|
||||||
|
float(value["heartbeat_delay_ms"]) for value in values
|
||||||
|
),
|
||||||
|
"shutdown_median_ms": statistics.median(
|
||||||
|
float(value["shutdown_ms"]) for value in values
|
||||||
|
),
|
||||||
|
"heartbeat_completed_before_shutdown": all(
|
||||||
|
bool(value["heartbeat_completed_before_shutdown"])
|
||||||
|
for value in values
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"block_ms": block_seconds * 1000,
|
||||||
|
"heartbeat_target_ms": heartbeat_seconds * 1000,
|
||||||
|
"module_step": summarize(module_samples),
|
||||||
|
"lifecycle_step": summarize(lifecycle_samples),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""解析固定阻塞、心跳间隔和样本数。"""
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--block-ms", type=float, default=50.0)
|
||||||
|
parser.add_argument("--heartbeat-ms", type=float, default=10.0)
|
||||||
|
parser.add_argument("--samples", type=int, default=7)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""运行隔离样本并输出 JSON。"""
|
||||||
|
args = parse_args()
|
||||||
|
if args.block_ms <= 0 or args.heartbeat_ms <= 0 or args.samples < 1:
|
||||||
|
raise SystemExit("block-ms、heartbeat-ms 和 samples 必须大于 0")
|
||||||
|
result = asyncio.run(
|
||||||
|
run_samples(
|
||||||
|
block_seconds=args.block_ms / 1000,
|
||||||
|
heartbeat_seconds=args.heartbeat_ms / 1000,
|
||||||
|
samples=args.samples,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+4
-2
@@ -13,8 +13,8 @@
|
|||||||
"runtime_to_db": [],
|
"runtime_to_db": [],
|
||||||
"workflow_to_db": []
|
"workflow_to_db": []
|
||||||
},
|
},
|
||||||
"edge_count": 6601,
|
"edge_count": 6603,
|
||||||
"edge_sha256": "4ebc1db325d75ac04418e89c199dc2b09eb95905e6e03a4101b7780d6b184c7c",
|
"edge_sha256": "70ed941e5ae3893b29167aa191357df42847de4547ce4a0b8d29acf38bbf924d",
|
||||||
"edges": [
|
"edges": [
|
||||||
"app -> app.runtime",
|
"app -> app.runtime",
|
||||||
"app -> app.runtime.compat",
|
"app -> app.runtime.compat",
|
||||||
@@ -6302,6 +6302,7 @@
|
|||||||
"app.startup.initializers.modules -> app.runtime.cache",
|
"app.startup.initializers.modules -> app.runtime.cache",
|
||||||
"app.startup.initializers.modules -> app.runtime.config",
|
"app.startup.initializers.modules -> app.runtime.config",
|
||||||
"app.startup.initializers.modules -> app.runtime.events",
|
"app.startup.initializers.modules -> app.runtime.events",
|
||||||
|
"app.startup.initializers.modules -> app.runtime.execution",
|
||||||
"app.startup.initializers.modules -> app.runtime.extensions",
|
"app.startup.initializers.modules -> app.runtime.extensions",
|
||||||
"app.startup.initializers.modules -> app.runtime.extensions.module",
|
"app.startup.initializers.modules -> app.runtime.extensions.module",
|
||||||
"app.startup.initializers.modules -> app.runtime.extensions.module.dispatcher",
|
"app.startup.initializers.modules -> app.runtime.extensions.module.dispatcher",
|
||||||
@@ -6404,6 +6405,7 @@
|
|||||||
"app.startup.lifecycle -> app.foundation.environment",
|
"app.startup.lifecycle -> app.foundation.environment",
|
||||||
"app.startup.lifecycle -> app.runtime",
|
"app.startup.lifecycle -> app.runtime",
|
||||||
"app.startup.lifecycle -> app.runtime.config",
|
"app.startup.lifecycle -> app.runtime.config",
|
||||||
|
"app.startup.lifecycle -> app.runtime.execution",
|
||||||
"app.startup.lifecycle -> app.runtime.health",
|
"app.startup.lifecycle -> app.runtime.health",
|
||||||
"app.startup.lifecycle -> app.runtime.log",
|
"app.startup.lifecycle -> app.runtime.log",
|
||||||
"app.startup.lifecycle -> app.runtime.settings",
|
"app.startup.lifecycle -> app.runtime.settings",
|
||||||
|
|||||||
@@ -1054,6 +1054,26 @@ def test_stop_modules_propagates_doh_nonconvergence(monkeypatch):
|
|||||||
_assert_completed_once(dependency)
|
_assert_completed_once(dependency)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_stop_modules_keeps_event_loop_responsive_during_sync_owner_wait(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""同步 owner 的有界等待不得占用主事件循环。"""
|
||||||
|
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
|
||||||
|
release = threading.Event()
|
||||||
|
timer = threading.Timer(0.05, release.set)
|
||||||
|
dependencies["module"].side_effect = lambda: release.wait(timeout=1.0)
|
||||||
|
|
||||||
|
heartbeat = asyncio.create_task(asyncio.sleep(0.01))
|
||||||
|
timer.start()
|
||||||
|
try:
|
||||||
|
await modules_initializer.stop_modules()
|
||||||
|
finally:
|
||||||
|
timer.join(timeout=1.0)
|
||||||
|
|
||||||
|
assert heartbeat.done()
|
||||||
|
|
||||||
|
|
||||||
def test_stop_modules_drains_web_agent_tasks_before_persistence(monkeypatch):
|
def test_stop_modules_drains_web_agent_tasks_before_persistence(monkeypatch):
|
||||||
"""关闭时先收口 Web Agent,再关闭持久化准入和数据库任务。"""
|
"""关闭时先收口 Web Agent,再关闭持久化准入和数据库任务。"""
|
||||||
order = []
|
order = []
|
||||||
@@ -1120,7 +1140,8 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
|
|||||||
"get_configured_agent_chat_persistence",
|
"get_configured_agent_chat_persistence",
|
||||||
MagicMock(return_value=persistence),
|
MagicMock(return_value=persistence),
|
||||||
)
|
)
|
||||||
stop_database_worker = AsyncMock()
|
database_worker_stopped = asyncio.Event()
|
||||||
|
stop_database_worker = AsyncMock(side_effect=database_worker_stopped.set)
|
||||||
monkeypatch.setattr(modules_initializer, "stop_database_worker", stop_database_worker)
|
monkeypatch.setattr(modules_initializer, "stop_database_worker", stop_database_worker)
|
||||||
monkeypatch.setattr(modules_initializer, "_database_worker", object())
|
monkeypatch.setattr(modules_initializer, "_database_worker", object())
|
||||||
|
|
||||||
@@ -1135,6 +1156,7 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
|
|||||||
completed = await shutdown
|
completed = await shutdown
|
||||||
|
|
||||||
assert completed is False
|
assert completed is False
|
||||||
|
await asyncio.wait_for(database_worker_stopped.wait(), timeout=1.0)
|
||||||
stop_database_worker.assert_awaited_once_with()
|
stop_database_worker.assert_awaited_once_with()
|
||||||
|
|
||||||
|
|
||||||
@@ -1177,6 +1199,54 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No
|
|||||||
await asyncio.wait_for(settled.wait(), timeout=0.2)
|
await asyncio.wait_for(settled.wait(), timeout=0.2)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_shutdown_step_bounds_sync_owner_without_blocking_event_loop() -> None:
|
||||||
|
"""同步 owner 超时后应及时返回,并继续持有 worker 直至真实终态。"""
|
||||||
|
started = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
settled = threading.Event()
|
||||||
|
|
||||||
|
def blocking_shutdown() -> None:
|
||||||
|
started.set()
|
||||||
|
release.wait(timeout=1.0)
|
||||||
|
settled.set()
|
||||||
|
|
||||||
|
heartbeat = asyncio.create_task(asyncio.sleep(0.01))
|
||||||
|
shutdown = asyncio.create_task(
|
||||||
|
lifecycle.run_shutdown_step(
|
||||||
|
"同步阻塞 owner",
|
||||||
|
lifecycle.offload_shutdown_callback(blocking_shutdown),
|
||||||
|
timeout_seconds=0.02,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert await asyncio.to_thread(started.wait, 0.2)
|
||||||
|
started_at = asyncio.get_running_loop().time()
|
||||||
|
completed = await shutdown
|
||||||
|
|
||||||
|
assert completed is False
|
||||||
|
assert asyncio.get_running_loop().time() - started_at < 0.2
|
||||||
|
assert heartbeat.done()
|
||||||
|
assert not settled.is_set()
|
||||||
|
|
||||||
|
release.set()
|
||||||
|
assert await asyncio.to_thread(settled.wait, 0.2)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_shutdown_step_calls_awaitable_wrapper_on_event_loop() -> None:
|
||||||
|
"""普通 callable 可在主循环构造并返回需要等待的异步结果。"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
def shutdown_wrapper() -> asyncio.Task[None]:
|
||||||
|
assert asyncio.get_running_loop() is loop
|
||||||
|
return loop.create_task(asyncio.sleep(0))
|
||||||
|
|
||||||
|
assert await lifecycle.run_shutdown_step(
|
||||||
|
"异步包装 owner",
|
||||||
|
shutdown_wrapper,
|
||||||
|
) is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_shutdown_step_reports_explicit_nonconvergence() -> None:
|
async def test_shutdown_step_reports_explicit_nonconvergence() -> None:
|
||||||
"""同步和异步 owner 显式返回 False 时都必须向生命周期传播失败。"""
|
"""同步和异步 owner 显式返回 False 时都必须向生命周期传播失败。"""
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
|
import asyncio
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from telebot import TeleBot
|
||||||
|
|
||||||
from app.modules import _MessageBase
|
from app.modules import _MessageBase
|
||||||
from app.modules.discord import DiscordModule
|
from app.modules.discord import DiscordModule
|
||||||
@@ -23,6 +26,7 @@ from app.modules.wechat import WechatModule
|
|||||||
from app.modules.wechat.wechatbot import WeChatBot
|
from app.modules.wechat.wechatbot import WeChatBot
|
||||||
from app.modules.wechatclawbot import WechatClawBotModule
|
from app.modules.wechatclawbot import WechatClawBotModule
|
||||||
from app.modules.wechatclawbot.wechatclawbot import WechatClawBot
|
from app.modules.wechatclawbot.wechatclawbot import WechatClawBot
|
||||||
|
from app.runtime.execution import run_in_threadpool_to_completion
|
||||||
|
|
||||||
|
|
||||||
def test_config_reload_stops_before_initializing_latest_generation():
|
def test_config_reload_stops_before_initializing_latest_generation():
|
||||||
@@ -180,6 +184,8 @@ def test_telegram_stop_closes_sdk_and_waits_for_polling_thread():
|
|||||||
"""客户端停止完成后不得保留 SDK worker 或 polling 线程句柄。"""
|
"""客户端停止完成后不得保留 SDK worker 或 polling 线程句柄。"""
|
||||||
client = Telegram.__new__(Telegram)
|
client = Telegram.__new__(Telegram)
|
||||||
bot = Mock()
|
bot = Mock()
|
||||||
|
bot.threaded = False
|
||||||
|
bot.worker_pool = None
|
||||||
client._bot = bot
|
client._bot = bot
|
||||||
polling_thread = Mock()
|
polling_thread = Mock()
|
||||||
polling_thread.is_alive.side_effect = [True, False]
|
polling_thread.is_alive.side_effect = [True, False]
|
||||||
@@ -193,9 +199,9 @@ def test_telegram_stop_closes_sdk_and_waits_for_polling_thread():
|
|||||||
assert client.stop() is True
|
assert client.stop() is True
|
||||||
assert client.stop() is True
|
assert client.stop() is True
|
||||||
|
|
||||||
bot.stop_bot.assert_called_once_with()
|
bot.stop_polling.assert_called_once_with()
|
||||||
polling_thread.join.assert_called_once_with(
|
polling_thread.join.assert_called_once_with(
|
||||||
timeout=client._polling_join_timeout_seconds
|
timeout=pytest.approx(client._shutdown_timeout_seconds, abs=0.1)
|
||||||
)
|
)
|
||||||
assert client._bot is None
|
assert client._bot is None
|
||||||
assert client._polling_thread is None
|
assert client._polling_thread is None
|
||||||
@@ -205,11 +211,13 @@ def test_telegram_stop_keeps_polling_owner_when_thread_misses_deadline():
|
|||||||
"""polling 超过关闭预算时必须返回未收敛并保留原 owner。"""
|
"""polling 超过关闭预算时必须返回未收敛并保留原 owner。"""
|
||||||
client = Telegram.__new__(Telegram)
|
client = Telegram.__new__(Telegram)
|
||||||
bot = Mock()
|
bot = Mock()
|
||||||
|
bot.threaded = False
|
||||||
|
bot.worker_pool = None
|
||||||
polling_thread = Mock()
|
polling_thread = Mock()
|
||||||
polling_thread.is_alive.return_value = True
|
polling_thread.is_alive.return_value = True
|
||||||
client._bot = bot
|
client._bot = bot
|
||||||
client._polling_thread = polling_thread
|
client._polling_thread = polling_thread
|
||||||
client._polling_join_timeout_seconds = 0.01
|
client._shutdown_timeout_seconds = 0.01
|
||||||
client._typing_tasks = {}
|
client._typing_tasks = {}
|
||||||
client._typing_stop_flags = {}
|
client._typing_stop_flags = {}
|
||||||
client._typing_lock = threading.RLock()
|
client._typing_lock = threading.RLock()
|
||||||
@@ -218,11 +226,60 @@ def test_telegram_stop_keeps_polling_owner_when_thread_misses_deadline():
|
|||||||
|
|
||||||
assert client.stop() is False
|
assert client.stop() is False
|
||||||
|
|
||||||
polling_thread.join.assert_called_once_with(timeout=0.01)
|
polling_thread.join.assert_called_once()
|
||||||
|
remaining_timeout = polling_thread.join.call_args.kwargs["timeout"]
|
||||||
|
assert 0 <= remaining_timeout <= client._shutdown_timeout_seconds
|
||||||
assert client._bot is bot
|
assert client._bot is bot
|
||||||
assert client._polling_thread is polling_thread
|
assert client._polling_thread is polling_thread
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_telegram_stop_bounds_real_sdk_worker_and_retries_after_release():
|
||||||
|
"""真实 SDK worker 阻塞时应保留 owner,释放后重试可以完整收敛。"""
|
||||||
|
bot = TeleBot("123:test", threaded=True, num_threads=1)
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
def blocking_callback() -> None:
|
||||||
|
entered.set()
|
||||||
|
release.wait(timeout=1.0)
|
||||||
|
|
||||||
|
bot.worker_pool.put(blocking_callback)
|
||||||
|
assert await asyncio.to_thread(entered.wait, 0.2)
|
||||||
|
|
||||||
|
client = Telegram.__new__(Telegram)
|
||||||
|
client._bot = bot
|
||||||
|
client._polling_thread = None
|
||||||
|
client._shutdown_timeout_seconds = 0.02
|
||||||
|
client._typing_tasks = {}
|
||||||
|
client._typing_stop_flags = {}
|
||||||
|
client._typing_lock = threading.RLock()
|
||||||
|
client._typing_lifecycle_lock = threading.RLock()
|
||||||
|
client._typing_accepting = True
|
||||||
|
|
||||||
|
heartbeat = asyncio.create_task(asyncio.sleep(0.005))
|
||||||
|
started_at = time.monotonic()
|
||||||
|
try:
|
||||||
|
assert await run_in_threadpool_to_completion(client.stop) is False
|
||||||
|
assert time.monotonic() - started_at < 0.2
|
||||||
|
assert heartbeat.done()
|
||||||
|
assert client._bot is bot
|
||||||
|
assert any(worker.is_alive() for worker in bot.worker_pool.workers)
|
||||||
|
|
||||||
|
release.set()
|
||||||
|
for worker in bot.worker_pool.workers:
|
||||||
|
await asyncio.to_thread(worker.join, 0.2)
|
||||||
|
|
||||||
|
assert await run_in_threadpool_to_completion(client.stop) is True
|
||||||
|
assert client._bot is None
|
||||||
|
assert client._polling_thread is None
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
for worker in bot.worker_pool.workers:
|
||||||
|
worker.stop()
|
||||||
|
await asyncio.to_thread(worker.join, 0.2)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"module_type",
|
"module_type",
|
||||||
[
|
[
|
||||||
|
|||||||
Reference in New Issue
Block a user