GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-04 06:56:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(runtime): 收口调度与事件异步任务生命周期 (#6415)

Browse Source
This commit is contained in:
InfinityPacer
2026-08-23 17:17:59 +08:00
committed by GitHub
parent e7bfc9dd43
commit e1d7918297
18 changed files with 1937 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/endpoints/site.py
+7 -3
View File
@@ -185,6 +185,7 @@ async def cookie_cloud_sync(
@router.get("/reset", summary="重置站点", response_model=_SchemaResponse[None])
async def reset(
task_registry: Annotated[TaskRegistry, Depends(get_background_task_registry)],
command: SiteMutationCommand = Depends(get_site_mutation_command),
_: ApiPrincipal = Depends(get_current_active_superuser_async),
) -> Any:
@@ -194,9 +195,12 @@ async def reset(
result = await command.reset()
await get_configured_system_config().async_set(SystemConfigKey.IndexerSites, [])
await get_configured_system_config().async_set(SystemConfigKey.RssSites, [])
# 启动定时服务
Scheduler().start("cookiecloud", manual=True)
# 插件站点删除
resolve_background_task_registry(task_registry).create_sync(
Scheduler().start,
job_id="cookiecloud",
owner="api.site.reset",
manual=True,
)
return _SchemaResponse(success=result.success, message="站点已重置！")
app/runtime/event/dispatch.py
+14 -4
View File
@@ -29,6 +29,7 @@ class EventDispatcher:
event_loop: Callable[[], Any],
event_factory: Callable[..., Any],
error_handler: Callable[..., None],
async_handle_sink: Callable[[Any], bool] | None = None,
) -> None:
"""注入注册表、绑定器、执行器和错误策略回调。"""
self._registry = registry
@@ -37,6 +38,7 @@ class EventDispatcher:
self._event_loop = event_loop
self._event_factory = event_factory
self._error_handler = error_handler
self._async_handle_sink = async_handle_sink
def dispatch_chain(self, event: Any) -> bool:
"""同步按优先级顺序执行链式事件快照。"""
@@ -119,10 +121,18 @@ class EventDispatcher:
correlation_id=event.correlation_id,
)
if inspect.iscoroutinefunction(handler):
asyncio.run_coroutine_threadsafe(
self.safe_invoke_async(handler, isolated),
self._event_loop(),
)
coroutine = self.safe_invoke_async(handler, isolated)
if self._async_handle_sink:
self._async_handle_sink(coroutine)
continue
try:
asyncio.run_coroutine_threadsafe(coroutine, self._event_loop())
except RuntimeError:
coroutine.close()
logger.warning(
"事件 %s 的异步处理器无法投递，事件循环已停止",
event.event_type,
)
else:
self._executor().submit(
self.safe_invoke_sync,
app/runtime/events.py
+158 -5
View File
@@ -1,7 +1,10 @@
import asyncio
import concurrent.futures
import random
import threading
import traceback
import uuid
from dataclasses import dataclass
from queue import Empty, PriorityQueue
from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type
@@ -28,6 +31,15 @@ DEFAULT_EVENT_PRIORITY = 10 # 事件的默认优先级
MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间（秒）
MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间（秒）
_EVENT_STOP_SENTINEL = object()
@dataclass(slots=True)
class _EventAsyncHandle:
"""记录异步广播的取消代理和真实完成信号。"""
handle: concurrent.futures.Future[Any]
completion: concurrent.futures.Future[Any]
class Event:
@@ -109,6 +121,10 @@ class EventManager(metaclass=Singleton):
self.__lock = threading.Lock()
# 退出事件
self.__event = threading.Event()
# 广播异步处理器的生命周期由事件总线自己持有，避免关闭后仍向主循环运行。
self.__lifecycle_lock = threading.RLock()
self.__lifecycle_state = "new"
self.__async_handles: Dict[int, _EventAsyncHandle] = {}
# 由上层管理器注册的处理器实例解析器
self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {}
# 由启动组合层注入的错误通知回调
@@ -138,6 +154,7 @@ class EventManager(metaclass=Singleton):
event_loop=lambda: global_vars.loop,
event_factory=Event,
error_handler=lambda **kwargs: self.__handle_event_error(**kwargs),
async_handle_sink=self.__register_async_handle,
)
def register_handler_instance_resolver(
@@ -165,8 +182,16 @@ class EventManager(metaclass=Singleton):
"""
开始广播事件处理线程
"""
with self.__lifecycle_lock:
if self.__lifecycle_state == "running":
return
if self.__lifecycle_state == "stopping":
logger.warning("事件处理仍在停止，忽略重复启动")
return
self.__lifecycle_state = "running"
self.__event.set()
self.__consumer_threads = []
# 启动消费者线程用于处理广播事件
self.__event.set()
for _ in range(MIN_EVENT_CONSUMER_THREADS):
thread = threading.Thread(target=self.__broadcast_consumer_loop, daemon=True)
thread.start()
@@ -177,14 +202,140 @@ class EventManager(metaclass=Singleton):
停止广播事件处理线程
"""
logger.info("正在停止事件处理...")
self.__event.clear() # 停止广播事件处理
consumer_threads = self.__begin_stop()
try:
# 通过遍历保存的线程来等待它们完成
for consumer_thread in self.__consumer_threads:
consumer_thread.join()
self.__join_consumer_threads(consumer_threads)
self.__discard_stop_sentinels()
self.__cancel_async_handles()
logger.info("事件处理停止完成")
except Exception as e:
logger.error(f"停止事件处理线程出错：{str(e)} - {traceback.format_exc()}")
finally:
with self.__lifecycle_lock:
self.__lifecycle_state = "stopped"
self.__consumer_threads = []
async def stop_async(self) -> None:
"""停止广播消费者并等待已投递的异步处理器收口。"""
logger.info("正在停止事件处理...")
consumer_threads = self.__begin_stop()
try:
if consumer_threads:
await asyncio.to_thread(
self.__join_consumer_threads,
consumer_threads,
)
self.__discard_stop_sentinels()
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
for handle in handles:
handle.handle.cancel()
if handles:
await asyncio.gather(
*(
asyncio.shield(asyncio.wrap_future(handle.completion))
for handle in handles
),
return_exceptions=True,
)
logger.info("事件处理停止完成")
except Exception as e:
logger.error(f"停止事件处理线程出错：{str(e)} - {traceback.format_exc()}")
raise
with self.__lifecycle_lock:
self.__lifecycle_state = "stopped"
self.__consumer_threads = []
def __begin_stop(self) -> tuple[threading.Thread, ...]:
"""关闭提交入口并唤醒消费者线程。"""
with self.__lifecycle_lock:
self.__lifecycle_state = "stopping"
self.__event.clear()
consumer_threads = tuple(self.__consumer_threads)
if consumer_threads:
self.__event_queue.put((float("-inf"), _EVENT_STOP_SENTINEL))
return consumer_threads
@staticmethod
def __join_consumer_threads(consumer_threads: tuple[threading.Thread, ...]) -> None:
"""在线程池或同步兼容入口中等待事件消费者退出。"""
for consumer_thread in consumer_threads:
consumer_thread.join()
def __discard_stop_sentinels(self) -> None:
"""清理仅用于唤醒消费者的标记，保留尚未消费的业务事件。"""
pending = []
while True:
try:
item = self.__event_queue.get_nowait()
except Empty:
break
if item[1] is not _EVENT_STOP_SENTINEL:
pending.append(item)
for item in pending:
self.__event_queue.put(item)
def __cancel_async_handles(self) -> None:
"""请求取消所有仍由事件总线持有的异步处理器。"""
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
for handle in handles:
handle.handle.cancel()
def __register_async_handle(
self,
coroutine: Any,
) -> bool:
"""在同一生命周期临界区提交并登记异步广播处理器。"""
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
coroutine.close()
return False
completion: concurrent.futures.Future[Any] = concurrent.futures.Future()
started = threading.Event()
async def _tracked() -> None:
started.set()
try:
result = await coroutine
except asyncio.CancelledError:
if not completion.done():
completion.cancel()
except Exception as err:
if not completion.done():
completion.set_exception(err)
else:
if not completion.done():
completion.set_result(result)
tracked = _tracked()
try:
handle = asyncio.run_coroutine_threadsafe(tracked, global_vars.loop)
except RuntimeError:
tracked.close()
coroutine.close()
logger.warning("异步事件处理器无法投递，事件循环已停止")
return False
self.__async_handles[id(completion)] = _EventAsyncHandle(
handle=handle,
completion=completion,
)
def _complete_unstarted_submission(
submitted: concurrent.futures.Future[Any],
) -> None:
if submitted.cancelled() and not started.is_set():
coroutine.close()
completion.cancel()
handle.add_done_callback(_complete_unstarted_submission)
completion.add_done_callback(self.__remove_async_handle)
return True
def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None:
"""异步处理器完成后移除其 owner 句柄。"""
with self.__lifecycle_lock:
self.__async_handles.pop(id(handle), None)
def check(self, etype: Union[EventType, ChainEventType]) -> bool:
"""
@@ -429,6 +580,8 @@ class EventManager(metaclass=Singleton):
while self.__event.is_set():
try:
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
if event is _EVENT_STOP_SENTINEL:
break
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
app/scheduler.py
+653 -179
View File
File diff suppressed because it is too large Load Diff
app/startup/modules_initializer.py
+1 -1
View File
@@ -580,7 +580,7 @@ async def stop_modules():
await run_step("AI智能体", stop_agent)
await run_step("模块", lambda: ModuleManager().shutdown())
await run_step("事件消费", lambda: EventManager().stop())
await run_step("事件消费", lambda: EventManager().stop_async())
await run_step("浏览器会话", close_browser_sessions)
await run_step("托管资源", stop_managed_resources)
await run_step("DoH服务", lambda: DohHelper().shutdown())
app/startup/scheduler_initializer.py
+2 -2
View File
@@ -16,7 +16,7 @@ def init_scheduler():
def stop_scheduler():
"""
停止定时器；生命周期事件循环中返回有限等待的兼容协程。
停止定时器；生命周期事件循环中返回可等待的收口协程。
"""
scheduler = Scheduler()
try:
@@ -24,7 +24,7 @@ def stop_scheduler():
except RuntimeError:
scheduler.stop()
return None
return scheduler.async_stop()
return scheduler.stop_async()
def restart_scheduler():
docs/adr/0007-background-action-reliability.md
+2 -2
View File
@@ -86,8 +86,8 @@ Event Contract Registry 是 53 个事件的逐项机器清单。下表按相同
- IMDb 同步 `clear_cache()` ABI 在事件循环内触发的异步缓存清理登记为
`module.imdb.cache_clear`；同步调用方式和无运行事件循环时的立即清理行为保持不变，宿主关停后不再
接受新的清理任务。
- Scheduler 的协程作业与异步进度收尾由 Scheduler 自有任务集合持有；同步 `start()` / `stop()` ABI 保持，
生命周期关闭入口额外等待有限预算，跨线程提交的 Future 也会在停止时收到取消请求。
- Scheduler 的协程作业与异步进度收尾由 Scheduler 自有句柄表持有；同步 `start()` / `stop()` ABI 保持，
生命周期关闭入口等待目标事件循环确认真实收尾，跨线程取消代理不作为任务完成凭据。
### Transfer pending / 文件整理
docs/refactor/backend-architecture-next-stage.md
+2 -2
View File
@@ -839,8 +839,8 @@ ADR 必须逐个映射当前 Event、BackgroundTasks、Scheduler job、Agent tas
- IMDb 同步清缓存兼容入口在运行事件循环时改由 TaskRegistry 登记异步缓存清理任务，owner 为
`module.imdb.cache_clear`；同步签名、模块调用方式和无事件循环时的立即清理语义保持不变。
- Scheduler 的协程作业和异步进度收尾不再使用无主 `create_task` 或丢弃跨线程 Future；由 Scheduler 自有
任务集合登记、停止时取消，生命周期入口通过异步兼容包装器在有限预算内等待收口，保留旧同步
`Scheduler.start()` / `Scheduler.stop()` 与插件调度 ABI。
句柄表登记并在停止时取消，completion 只在目标事件循环确认真实收尾后完成。关闭总预算由宿主生命周期
统一控制，保留旧同步 `Scheduler.start()` / `Scheduler.stop()` 与插件调度 ABI。
#### ARCH-251：用现有数据库做首个 durable side-effect pilot
tests/fixtures/architecture/dependency-baseline.json
Vendored
+33 -4
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6451,
"edge_sha256": "9075d8717e384580cd6a41bc36685438770db4a8d8f18c57e6c494f32937113a",
"edge_count": 6479,
"edge_sha256": "a65f8d4024e2299b37510359c7ffea91219f0aa0ed9673b74a3eb22d51d6c67f",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -1510,6 +1510,8 @@
"app.api.context -> app.application.subscription.delete",
"app.api.context -> app.application.subscription.identity",
"app.api.context -> app.application.subscription.mutation",
"app.api.context -> app.runtime",
"app.api.context -> app.runtime.tasks",
"app.api.context -> app.startup",
"app.api.context -> app.startup.context",
"app.api.dependencies.agent -> app.api",
@@ -1599,6 +1601,7 @@
"app.api.dependencies.subscription -> app.runtime",
"app.api.dependencies.subscription -> app.runtime.events",
"app.api.dependencies.subscription -> app.runtime.log",
"app.api.dependencies.subscription -> app.runtime.tasks",
"app.api.dependencies.subscription -> app.schemas",
"app.api.dependencies.subscription -> app.schemas.types",
"app.api.dependencies.subscription -> app.startup",
@@ -1682,6 +1685,7 @@
"app.api.endpoints.anthropic -> app.agent",
"app.api.endpoints.anthropic -> app.agent.runtime_loader",
"app.api.endpoints.anthropic -> app.api",
"app.api.endpoints.anthropic -> app.api.context",
"app.api.endpoints.anthropic -> app.api.endpoints",
"app.api.endpoints.anthropic -> app.api.endpoints.openai",
"app.api.endpoints.anthropic -> app.api.openai_utils",
@@ -1689,6 +1693,8 @@
"app.api.endpoints.anthropic -> app.api.presentation.sse",
"app.api.endpoints.anthropic -> app.application",
"app.api.endpoints.anthropic -> app.application.configuration",
"app.api.endpoints.anthropic -> app.runtime",
"app.api.endpoints.anthropic -> app.runtime.tasks",
"app.api.endpoints.anthropic -> app.schemas",
"app.api.endpoints.anthropic -> app.schemas.openai",
"app.api.endpoints.auth -> app.api",
@@ -1833,6 +1839,7 @@
"app.api.endpoints.history -> app.runtime.config",
"app.api.endpoints.history -> app.runtime.log",
"app.api.endpoints.history -> app.runtime.progress",
"app.api.endpoints.history -> app.runtime.tasks",
"app.api.endpoints.history -> app.schemas",
"app.api.endpoints.history -> app.schemas.common",
"app.api.endpoints.history -> app.schemas.history",
@@ -1945,6 +1952,7 @@
"app.api.endpoints.message -> app.adapters.web.security",
"app.api.endpoints.message -> app.adapters.web.security.access",
"app.api.endpoints.message -> app.api",
"app.api.endpoints.message -> app.api.context",
"app.api.endpoints.message -> app.api.dependencies",
"app.api.endpoints.message -> app.api.dependencies.agent",
"app.api.endpoints.message -> app.api.dependencies.auth",
@@ -1961,6 +1969,7 @@
"app.api.endpoints.message -> app.runtime.extensions",
"app.api.endpoints.message -> app.runtime.extensions.service_config",
"app.api.endpoints.message -> app.runtime.log",
"app.api.endpoints.message -> app.runtime.tasks",
"app.api.endpoints.message -> app.schemas",
"app.api.endpoints.message -> app.schemas.message",
"app.api.endpoints.message -> app.schemas.response",
@@ -2029,11 +2038,14 @@
"app.api.endpoints.openai -> app.agent.contracts",
"app.api.endpoints.openai -> app.agent.runtime_loader",
"app.api.endpoints.openai -> app.api",
"app.api.endpoints.openai -> app.api.context",
"app.api.endpoints.openai -> app.api.openai_utils",
"app.api.endpoints.openai -> app.api.presentation",
"app.api.endpoints.openai -> app.api.presentation.sse",
"app.api.endpoints.openai -> app.application",
"app.api.endpoints.openai -> app.application.configuration",
"app.api.endpoints.openai -> app.runtime",
"app.api.endpoints.openai -> app.runtime.tasks",
"app.api.endpoints.openai -> app.schemas",
"app.api.endpoints.openai -> app.schemas.openai",
"app.api.endpoints.openai -> app.schemas.types",
@@ -2048,6 +2060,7 @@
"app.api.endpoints.plugin -> app.adapters.web.security",
"app.api.endpoints.plugin -> app.adapters.web.security.access",
"app.api.endpoints.plugin -> app.api",
"app.api.endpoints.plugin -> app.api.context",
"app.api.endpoints.plugin -> app.api.dependencies",
"app.api.endpoints.plugin -> app.api.dependencies.auth",
"app.api.endpoints.plugin -> app.api.dependencies.plugin",
@@ -2069,6 +2082,7 @@
"app.api.endpoints.plugin -> app.runtime.extensions.plugin",
"app.api.endpoints.plugin -> app.runtime.extensions.plugin.contracts",
"app.api.endpoints.plugin -> app.runtime.log",
"app.api.endpoints.plugin -> app.runtime.tasks",
"app.api.endpoints.plugin -> app.schemas",
"app.api.endpoints.plugin -> app.schemas.common",
"app.api.endpoints.plugin -> app.schemas.exception",
@@ -2122,6 +2136,7 @@
"app.api.endpoints.site -> app.adapters.web.security",
"app.api.endpoints.site -> app.adapters.web.security.access",
"app.api.endpoints.site -> app.api",
"app.api.endpoints.site -> app.api.context",
"app.api.endpoints.site -> app.api.dependencies",
"app.api.endpoints.site -> app.api.dependencies.auth",
"app.api.endpoints.site -> app.api.dependencies.site",
@@ -2145,6 +2160,7 @@
"app.api.endpoints.site -> app.domain.site",
"app.api.endpoints.site -> app.runtime",
"app.api.endpoints.site -> app.runtime.log",
"app.api.endpoints.site -> app.runtime.tasks",
"app.api.endpoints.site -> app.schemas",
"app.api.endpoints.site -> app.schemas.common",
"app.api.endpoints.site -> app.schemas.response",
@@ -2180,6 +2196,7 @@
"app.api.endpoints.subscribe -> app.adapters.web.security",
"app.api.endpoints.subscribe -> app.adapters.web.security.access",
"app.api.endpoints.subscribe -> app.api",
"app.api.endpoints.subscribe -> app.api.context",
"app.api.endpoints.subscribe -> app.api.dependencies",
"app.api.endpoints.subscribe -> app.api.dependencies.auth",
"app.api.endpoints.subscribe -> app.api.dependencies.subscription",
@@ -2201,6 +2218,7 @@
"app.api.endpoints.subscribe -> app.domain.metainfo",
"app.api.endpoints.subscribe -> app.runtime",
"app.api.endpoints.subscribe -> app.runtime.events",
"app.api.endpoints.subscribe -> app.runtime.tasks",
"app.api.endpoints.subscribe -> app.schemas",
"app.api.endpoints.subscribe -> app.schemas.common",
"app.api.endpoints.subscribe -> app.schemas.event",
@@ -2347,9 +2365,12 @@
"app.api.endpoints.webhook -> app.adapters.web.security",
"app.api.endpoints.webhook -> app.adapters.web.security.access",
"app.api.endpoints.webhook -> app.api",
"app.api.endpoints.webhook -> app.api.context",
"app.api.endpoints.webhook -> app.api.response",
"app.api.endpoints.webhook -> app.chain",
"app.api.endpoints.webhook -> app.chain.webhook",
"app.api.endpoints.webhook -> app.runtime",
"app.api.endpoints.webhook -> app.runtime.tasks",
"app.api.endpoints.webhook -> app.schemas",
"app.api.endpoints.webhook -> app.schemas.response",
"app.api.endpoints.workflow -> app.adapters",
@@ -2555,6 +2576,8 @@
"app.application.mediaserver -> app.schemas.mediaserver",
"app.application.mediaserver -> app.schemas.system",
"app.application.mediaserver -> app.schemas.types",
"app.application.messaging.agent -> app.runtime",
"app.application.messaging.agent -> app.runtime.tasks",
"app.application.messaging.agent -> app.schemas",
"app.application.messaging.agent -> app.schemas.types",
"app.application.messaging.chat -> app.application",
@@ -3619,7 +3642,6 @@
"app.db.oper.passkey -> app.db.base",
"app.db.oper.passkey -> app.db.models",
"app.db.oper.passkey -> app.db.models.passkey",
"app.db.oper.passkey -> app.db.uow",
"app.db.oper.plugindata -> app.db",
"app.db.oper.plugindata -> app.db.base",
"app.db.oper.plugindata -> app.db.models",
@@ -4300,6 +4322,7 @@
"app.modules.imdb.api -> app.runtime.cache",
"app.modules.imdb.api -> app.runtime.log",
"app.modules.imdb.api -> app.runtime.settings",
"app.modules.imdb.api -> app.runtime.tasks",
"app.modules.indexer -> app.application",
"app.modules.indexer -> app.application.site",
"app.modules.indexer -> app.application.site.health",
@@ -5626,6 +5649,7 @@
"app.runtime.extensions.plugin_manager -> app.foundation.version",
"app.runtime.extensions.plugin_manager -> app.runtime",
"app.runtime.extensions.plugin_manager -> app.runtime.events",
"app.runtime.extensions.plugin_manager -> app.runtime.execution",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions.plugin",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions.plugin.access",
@@ -5994,6 +6018,8 @@
"app.startup.context -> app.application.subscription.identity",
"app.startup.context -> app.application.subscription.mutation",
"app.startup.context -> app.application.workflow",
"app.startup.context -> app.runtime",
"app.startup.context -> app.runtime.tasks",
"app.startup.database -> app.adapters",
"app.startup.database -> app.adapters.system",
"app.startup.database -> app.adapters.system.backup",
@@ -6057,6 +6083,7 @@
"app.startup.lifecycle -> app.runtime.log",
"app.startup.lifecycle -> app.runtime.settings",
"app.startup.lifecycle -> app.runtime.state",
"app.startup.lifecycle -> app.runtime.tasks",
"app.startup.lifecycle -> app.runtime.topology",
"app.startup.lifecycle -> app.startup",
"app.startup.lifecycle -> app.startup.cache_initializer",
@@ -6171,6 +6198,7 @@
"app.startup.modules_initializer -> app.runtime.observability",
"app.startup.modules_initializer -> app.runtime.settings",
"app.startup.modules_initializer -> app.runtime.state",
"app.startup.modules_initializer -> app.runtime.tasks",
"app.startup.modules_initializer -> app.runtime.thread",
"app.startup.modules_initializer -> app.scheduler",
"app.startup.modules_initializer -> app.schemas",
@@ -6468,7 +6496,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 799,
"module_count": 800,
"modules": [
"app",
"app.adapters",
@@ -7161,6 +7189,7 @@
"app.runtime.scheduling",
"app.runtime.settings",
"app.runtime.state",
"app.runtime.tasks",
"app.runtime.thread",
"app.runtime.topology",
"app.scheduler",
tests/test_agent_scheduled_tasks.py
+73
View File
@@ -127,9 +127,15 @@ def _add_agent_task(trigger_type: str, trigger_value: str, prefix: str):
def _build_agent_task_scheduler(reconcile: bool = False) -> Scheduler:
"""构造不启动后台线程的 Agent 任务调度器。"""
scheduler = object.__new__(Scheduler)
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._scheduler = BackgroundScheduler(timezone=settings.TZ)
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler._agent_task_interruptions_reconciled = False
if reconcile:
scheduler._reconcile_agent_task_interruptions()
@@ -316,6 +322,11 @@ def test_scheduler_registers_and_removes_agent_task_job() -> None:
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._scheduler = BackgroundScheduler(timezone=settings.TZ)
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
next_run_at = scheduler.update_agent_task_job(task.id)
job_id = scheduler._get_agent_task_job_id(task.id)
@@ -642,6 +653,63 @@ async def test_scheduler_config_reload_does_not_interrupt_running_agent_task() -
manager.process_message.assert_not_awaited()
@pytest.mark.anyio
async def test_scheduler_config_reload_preserves_active_agent_task(
monkeypatch,
) -> None:
"""配置热重载只替换后续计划，已开始的 AgentTask 仍按真实结果收口。"""
task = _add_agent_task("cron", "0 * * * *", "reload-active")
scheduler = _build_agent_task_scheduler()
scheduler.init_agent_task_jobs()
started = asyncio.Event()
release = asyncio.Event()
async def process_message(**_kwargs) -> str:
started.set()
await release.wait()
return "执行完成"
manager = SimpleNamespace(
execute_scheduled_task=AgentManager.execute_scheduled_task,
process_message=process_message,
)
manager.execute_scheduled_task = AgentManager.execute_scheduled_task.__get__(manager)
monkeypatch.setattr(
"app.agent.runtime_loader.get_running_agent_manager",
lambda: manager,
)
monkeypatch.setattr(
scheduler,
"init",
Mock(side_effect=lambda **_kwargs: setattr(
scheduler,
"_lifecycle_state",
"running",
)),
)
job_id = scheduler._get_agent_task_job_id(task.id)
assert scheduler.start(job_id) is True
await asyncio.wait_for(started.wait(), timeout=1)
assert AgentTaskOper().get(task.id).last_status == "running"
await scheduler.on_config_changed()
assert AgentTaskOper().get(task.id).last_status == "running"
assert scheduler._handles
release.set()
async def wait_until_released() -> None:
while scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
completed = AgentTaskOper().get(task.id)
assert completed.last_status == "success"
assert completed.last_result == "执行完成"
def test_scheduler_restart_keeps_interrupted_cron_future_schedule() -> None:
"""周期任务中断后只保留下次正常调度，不抹掉本轮中断事实。"""
task = _add_agent_task("cron", "0 * * * *", "restart-cron")
@@ -699,6 +767,11 @@ def test_scheduler_starts_registered_agent_task_without_waiting() -> None:
"running": False,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler.start = Mock()
assert scheduler.start_agent_task(7) is True
tests/test_database_backup_scheduler.py
+7
View File
@@ -1,6 +1,7 @@
"""数据库备份与宿主调度器的接入合同。"""
import ast
import threading
from dataclasses import replace
from pathlib import Path
from unittest.mock import Mock
@@ -27,6 +28,12 @@ def _scheduler() -> Scheduler:
scheduler = object.__new__(Scheduler)
scheduler._scheduler = _SchedulerStub()
scheduler._jobs = {}
scheduler._lock = threading.RLock()
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
tests/test_event_dispatch_snapshot.py
+172 -1
View File
@@ -1,7 +1,12 @@
"""事件调度订阅快照的并发回归测试。"""
"""事件调度订阅快照和生命周期回归测试。"""
import asyncio
import threading
import pytest
from app.runtime.config import global_vars
from app.runtime import events as events_module
from app.runtime.events import Event, eventmanager
from app.schemas.types import ChainEventType, EventType
@@ -37,6 +42,26 @@ def isolated_eventmanager(monkeypatch):
"_EventManager__executor",
_ImmediateExecutor(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__event",
threading.Event(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__consumer_threads",
[],
)
monkeypatch.setattr(
eventmanager,
"_EventManager__lifecycle_state",
"new",
)
monkeypatch.setattr(
eventmanager,
"_EventManager__async_handles",
{},
)
return eventmanager
@@ -158,3 +183,149 @@ async def test_async_chain_dispatch_uses_subscription_snapshot(
calls.clear()
assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True
assert calls == ["mutating", "late"]
@pytest.mark.asyncio
async def test_async_broadcast_handles_are_cancelled_on_shutdown(isolated_eventmanager):
"""事件总线关闭时必须取消并收口已投递的异步广播处理器。"""
global_vars.set_loop(asyncio.get_running_loop())
handler_count = 5
active = 0
cancelled = 0
all_active = asyncio.Event()
async def handler(_event):
nonlocal active, cancelled
active += 1
if active == handler_count:
all_active.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled += 1
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
consumer_threads = tuple(isolated_eventmanager._EventManager__consumer_threads)
for _ in range(handler_count):
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"shutdown"}},
)
await asyncio.wait_for(all_active.wait(), timeout=2)
assert len(isolated_eventmanager._EventManager__async_handles) == handler_count
await isolated_eventmanager.stop_async()
await asyncio.sleep(0)
assert cancelled == handler_count
assert isolated_eventmanager._EventManager__async_handles == {}
assert not any(
thread.is_alive()
for thread in consumer_threads
)
@pytest.mark.asyncio
async def test_async_broadcast_shutdown_waits_for_handler_cleanup(
isolated_eventmanager,
) -> None:
"""提交代理变为 cancelled 后仍须等待处理器 finally 真正完成。"""
global_vars.set_loop(asyncio.get_running_loop())
started = asyncio.Event()
cancelling = asyncio.Event()
cleanup_release = asyncio.Event()
async def handler(_event):
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelling.set()
await cleanup_release.wait()
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
isolated_eventmanager.send_event(EventType.ConfigChanged, {"key": {"shutdown"}})
await asyncio.wait_for(started.wait(), timeout=2)
stop_task = asyncio.create_task(isolated_eventmanager.stop_async())
await asyncio.wait_for(cancelling.wait(), timeout=1)
await asyncio.sleep(0)
assert not stop_task.done()
assert isolated_eventmanager._EventManager__lifecycle_state == "stopping"
stop_task.cancel()
with pytest.raises(asyncio.CancelledError):
await stop_task
assert isolated_eventmanager._EventManager__async_handles
assert isolated_eventmanager._EventManager__lifecycle_state == "stopping"
cleanup_release.set()
async def wait_until_released() -> None:
while isolated_eventmanager._EventManager__async_handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
await isolated_eventmanager.stop_async()
assert isolated_eventmanager._EventManager__async_handles == {}
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
@pytest.mark.asyncio
async def test_async_broadcast_submission_is_registered_before_stop_snapshot(
isolated_eventmanager,
monkeypatch,
) -> None:
"""事件处理器提交和 owner 登记不得被关闭快照从中切开。"""
global_vars.set_loop(asyncio.get_running_loop())
submission_entered = threading.Event()
submission_release = threading.Event()
real_submit = events_module.asyncio.run_coroutine_threadsafe
def delayed_submit(coroutine, loop):
handle = real_submit(coroutine, loop)
submission_entered.set()
submission_release.wait(timeout=1)
return handle
monkeypatch.setattr(
events_module.asyncio,
"run_coroutine_threadsafe",
delayed_submit,
)
isolated_eventmanager.start()
async def handler() -> None:
await asyncio.Event().wait()
submit_thread = threading.Thread(
target=isolated_eventmanager._EventManager__register_async_handle,
args=(handler(),),
)
submit_thread.start()
assert await asyncio.to_thread(submission_entered.wait, 1)
stop_result = []
stop_thread = threading.Thread(
target=lambda: stop_result.append(
isolated_eventmanager._EventManager__begin_stop()
)
)
stop_thread.start()
await asyncio.sleep(0.02)
assert stop_thread.is_alive()
submission_release.set()
await asyncio.to_thread(submit_thread.join, 1)
await asyncio.to_thread(stop_thread.join, 1)
assert not submit_thread.is_alive()
assert not stop_thread.is_alive()
assert len(isolated_eventmanager._EventManager__async_handles) == 1
await isolated_eventmanager.stop_async()
assert isolated_eventmanager._EventManager__async_handles == {}
tests/test_lifecycle_shutdown.py
+1 -1
View File
@@ -694,7 +694,7 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
dependencies = {}
for name, method_name in (
("ModuleManager", "shutdown"),
("EventManager", "stop"),
("EventManager", "stop_async"),
("DohHelper", "shutdown"),
("ThreadHelper", "shutdown"),
("RedisHelper", "close"),
tests/test_plugin_local_sync.py
+5
View File
@@ -122,6 +122,11 @@ def _build_scheduler_for_plugin_reload(jobs: dict, backend) -> Scheduler:
scheduler._lock = threading.RLock()
scheduler._jobs = jobs
scheduler._scheduler = backend
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
tests/test_scheduler_cache_expiry.py
+8 -3
View File
@@ -54,13 +54,13 @@ def test_scheduler_initializer_stop_preserves_sync_abi(monkeypatch):
assert scheduler_initializer.stop_scheduler() is None
scheduler.stop.assert_called_once_with()
scheduler.async_stop.assert_not_called()
scheduler.stop_async.assert_not_called()
def test_scheduler_initializer_stop_awaits_in_running_loop(monkeypatch):
"""生命周期事件循环中的停止入口应返回可等待的异步收口。"""
scheduler = Mock()
scheduler.async_stop = AsyncMock()
scheduler.stop_async = AsyncMock()
monkeypatch.setattr(scheduler_initializer, "Scheduler", Mock(return_value=scheduler))
async def scenario():
@@ -69,7 +69,7 @@ def test_scheduler_initializer_stop_awaits_in_running_loop(monkeypatch):
await result
asyncio.run(scenario())
scheduler.async_stop.assert_awaited_once_with()
scheduler.stop_async.assert_awaited_once_with()
scheduler.stop.assert_not_called()
@@ -117,6 +117,11 @@ def test_clear_cache_is_manual_only(monkeypatch):
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._lifecycle_state = "new"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler._agent_task_interruptions_reconciled = True
scheduler._auth_count = 0
scheduler._auth_message = False
tests/test_scheduler_lifecycle.py
+724
View File
@@ -0,0 +1,724 @@
"""Scheduler 任务句柄、generation 与 AgentTask reservation 回归。"""
import asyncio
import gc
import threading
import warnings
import pytest
from app import scheduler as scheduler_module
from app.runtime.config import global_vars
from app.scheduler import Scheduler
class _ProgressStub:
"""隔离 scheduler 生命周期测试的同步进度后端。"""
def __init__(self, _key: str) -> None:
"""接收进度键但不连接外部后端。"""
def start(self) -> None:
"""记录进度开始。"""
def update(self, **_kwargs) -> None:
"""忽略中间进度。"""
def get(self):
"""返回空的历史进度。"""
return None
class _AsyncProgressStub:
"""隔离 scheduler 生命周期测试的异步进度后端。"""
def __init__(self, _key: str) -> None:
"""接收进度键但不连接外部后端。"""
async def get(self):
"""返回空的历史进度。"""
return None
async def update(self, **_kwargs) -> None:
"""忽略中间进度。"""
async def end(self, **_kwargs) -> None:
"""记录终态但不访问外部缓存。"""
def _scheduler(job_id: str, func) -> Scheduler:
"""构造已启动但不拥有 APScheduler 线程的实例。"""
scheduler = object.__new__(Scheduler)
scheduler._scheduler = None
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {
job_id: {
"name": "生命周期测试",
"provider_name": "测试",
"func": func,
"running": False,
"_generation": 1,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {job_id: 1}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
@pytest.mark.anyio
async def test_stop_async_cancels_and_awaits_scheduler_owned_job(monkeypatch) -> None:
"""关闭后已投递协程必须取消并完成收尾，不得遗留 owner 句柄。"""
started = asyncio.Event()
cleaned = asyncio.Event()
async def job():
started.set()
try:
await asyncio.Event().wait()
finally:
cleaned.set()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("lifecycle-job", job)
assert scheduler.start("lifecycle-job") is True
await asyncio.wait_for(started.wait(), timeout=1)
assert scheduler._handles
await scheduler.stop_async()
assert cleaned.is_set()
assert scheduler._jobs["lifecycle-job"]["running"] is False
assert scheduler._jobs["lifecycle-job"]["last_error"] == "任务已取消"
assert scheduler._handles == {}
assert scheduler._lifecycle_state == "stopped"
@pytest.mark.anyio
async def test_foreign_loop_submission_runs_on_main_loop_and_finishes_before_stop(
monkeypatch,
) -> None:
"""自建事件循环提交的任务仍由应用主循环拥有并完成取消收尾。"""
main_loop = asyncio.get_running_loop()
started = asyncio.Event()
cancelling = asyncio.Event()
cleanup_release = asyncio.Event()
execution_loop = None
async def job() -> None:
nonlocal execution_loop
execution_loop = asyncio.get_running_loop()
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelling.set()
await cleanup_release.wait()
raise
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
scheduler = _scheduler("foreign-loop-job", job)
def submit_from_foreign_loop() -> bool:
async def submit() -> bool:
return scheduler.start("foreign-loop-job")
return asyncio.run(submit())
assert await asyncio.to_thread(submit_from_foreign_loop) is True
await asyncio.wait_for(started.wait(), timeout=1)
assert execution_loop is main_loop
stop_task = asyncio.create_task(scheduler.stop_async())
await asyncio.wait_for(cancelling.wait(), timeout=1)
await asyncio.sleep(0)
assert not stop_task.done()
assert scheduler._lifecycle_state == "stopping"
stop_task.cancel()
with pytest.raises(asyncio.CancelledError):
await stop_task
assert scheduler._handles
assert scheduler._lifecycle_state == "stopping"
cleanup_release.set()
async def wait_until_released() -> None:
while scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
await scheduler.stop_async()
assert scheduler._handles == {}
assert scheduler._lifecycle_state == "stopped"
@pytest.mark.anyio
async def test_cross_thread_submission_is_registered_before_stop_snapshot(
monkeypatch,
) -> None:
"""跨线程提交与 owner 登记必须对关闭快照表现为同一原子操作。"""
main_loop = asyncio.get_running_loop()
registration_entered = threading.Event()
registration_release = threading.Event()
async def job() -> None:
await asyncio.Event().wait()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
scheduler = _scheduler("atomic-submit", job)
register_handle = scheduler._register_handle
def delayed_register(**kwargs) -> bool:
registration_entered.set()
registration_release.wait(timeout=1)
return register_handle(**kwargs)
monkeypatch.setattr(scheduler, "_register_handle", delayed_register)
submit_thread = threading.Thread(target=scheduler.start, args=("atomic-submit",))
submit_thread.start()
assert await asyncio.to_thread(registration_entered.wait, 1)
stop_result = []
stop_thread = threading.Thread(target=lambda: stop_result.append(scheduler._begin_stop()))
stop_thread.start()
await asyncio.sleep(0.02)
assert stop_thread.is_alive()
registration_release.set()
await asyncio.to_thread(submit_thread.join, 1)
await asyncio.to_thread(stop_thread.join, 1)
assert not submit_thread.is_alive()
assert not stop_thread.is_alive()
assert len(stop_result[0][1]) == 1
for handle in stop_result[0][1]:
scheduler._cancel_handle(handle)
await scheduler._await_cancelled_handles(stop_result[0][1])
@pytest.mark.anyio
async def test_submit_to_loop_tracks_internal_progress_or_finish_tasks() -> None:
"""进度和收尾协程也必须归 Scheduler 所有并可在关闭时收口。"""
started = asyncio.Event()
cancelled = asyncio.Event()
async def pending() -> None:
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise
scheduler = _scheduler("internal-task", lambda: None)
scheduler._submit_to_loop(
pending(),
job_id="internal-task",
generation=1,
)
await asyncio.wait_for(started.wait(), timeout=1)
assert len(scheduler._handles) == 1
await scheduler.stop_async()
assert cancelled.is_set()
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_sync_job_callback_and_finish_handles_are_owned(monkeypatch) -> None:
"""同步任务回投的进度与收尾句柄都必须纳入关闭收口。"""
update_started = asyncio.Event()
finish_started = asyncio.Event()
gate = asyncio.Event()
cancelled = 0
class BlockingProgress:
"""让进度和收尾停在异步后端，便于验证 owner registry。"""
def __init__(self, _key: str) -> None:
pass
async def update(self, **_kwargs) -> None:
nonlocal cancelled
update_started.set()
try:
await gate.wait()
except asyncio.CancelledError:
cancelled += 1
raise
async def get(self):
nonlocal cancelled
finish_started.set()
try:
await gate.wait()
except asyncio.CancelledError:
cancelled += 1
raise
return None
async def end(self, **_kwargs) -> None:
pass
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", BlockingProgress)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", asyncio.get_running_loop())
def job(progress_callback) -> None:
progress_callback(value=50)
scheduler = _scheduler("callback-handles", job)
await asyncio.to_thread(scheduler.start, "callback-handles")
await asyncio.wait_for(
asyncio.gather(update_started.wait(), finish_started.wait()),
timeout=1,
)
assert len(scheduler._handles) == 2
await scheduler.stop_async()
assert cancelled == 2
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_stale_progress_cannot_update_replaced_job(monkeypatch) -> None:
"""旧 generation 的延迟进度不得写入新注册的同 ID 任务。"""
updates = []
class RecordingProgress:
def __init__(self, _key: str) -> None:
pass
async def update(self, **kwargs) -> None:
updates.append(kwargs)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", RecordingProgress)
scheduler = _scheduler("generation-progress", lambda: None)
old_job = scheduler._jobs["generation-progress"]
callback = scheduler._Scheduler__build_progress_callback(
"generation-progress",
old_job,
)
scheduler._jobs["generation-progress"] = {
"name": "新一代",
"provider_name": "测试",
"running": True,
"_generation": 2,
}
callback(value=42, text="旧进度")
await asyncio.sleep(0)
await asyncio.sleep(0)
assert updates == []
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_replaced_job_keeps_active_state_without_stale_progress(monkeypatch) -> None:
"""同 ID 新 generation 显示真实运行态，但不继承旧任务进度详情。"""
detail = {}
class RecordingProgress:
def __init__(self, _key: str) -> None:
pass
def start(self) -> None:
pass
def update(self, **kwargs) -> None:
detail.update(kwargs)
def get(self):
return detail
class RecordingAsyncProgress:
def __init__(self, _key: str) -> None:
pass
async def get(self):
return detail
monkeypatch.setattr(scheduler_module, "ProgressHelper", RecordingProgress)
monkeypatch.setattr(
scheduler_module,
"AsyncProgressHelper",
RecordingAsyncProgress,
)
scheduler = _scheduler("generation-cache", lambda: None)
old_job = scheduler._Scheduler__prepare_job("generation-cache")
assert old_job is not None
assert detail["data"]["_generation"] == 1
scheduler._jobs["generation-cache"] = {
"name": "新一代",
"provider_name": "测试",
"running": False,
"_generation": 2,
}
progress = scheduler.get_progress("generation-cache")
assert progress is not None
assert progress.status == "running"
assert progress.enable is True
assert progress.value == 0
assert "_generation" not in progress.data
async_progress = await scheduler.aget_progress("generation-cache")
assert async_progress is not None
assert async_progress.status == "running"
assert async_progress.enable is True
assert async_progress.value == 0
assert "_generation" not in async_progress.data
@pytest.mark.anyio
async def test_stale_generation_cannot_finish_replaced_job(monkeypatch) -> None:
"""旧 generation 收尾不得改写同 ID 的新任务状态或进度。"""
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("generation-job", lambda: None)
old_job = scheduler._jobs["generation-job"]
old_job["running"] = True
new_job = {
"name": "新一代",
"provider_name": "测试",
"running": True,
"_generation": 2,
}
scheduler._jobs["generation-job"] = new_job
await scheduler._Scheduler__finish_job(
job_id="generation-job",
job=old_job,
generation=1,
success=True,
)
assert new_job["running"] is True
assert "last_finished_at" not in new_job
assert old_job["running"] is True
def test_agent_task_manual_start_has_single_reservation() -> None:
"""并发手动触发同一 AgentTask 时只能有一个调用获得 reservation。"""
scheduler = _scheduler("agent-task-1", lambda: None)
scheduler._jobs["agent-task-1"].update(
name="AgentTask",
owner="agent",
)
entered = threading.Event()
release = threading.Event()
results = []
def start(*_args, **_kwargs):
entered.set()
release.wait(timeout=1)
return True
scheduler.start = start
first = threading.Thread(
target=lambda: results.append(scheduler.start_agent_task(1)),
)
first.start()
assert entered.wait(timeout=1)
second = scheduler.start_agent_task(1)
release.set()
first.join(timeout=1)
assert second is False
assert results == [True]
assert scheduler._agent_task_reservations == {}
def test_scheduler_rejects_new_submission_after_stop() -> None:
"""进入 stopping/stopped 后不得再从旧 scheduler 提交任务。"""
scheduler = _scheduler("stopped-job", lambda: None)
scheduler._lifecycle_state = "stopping"
assert scheduler.start("stopped-job") is False
assert scheduler._jobs["stopped-job"]["running"] is False
@pytest.mark.anyio
async def test_config_reload_does_not_restart_scheduler_during_shutdown() -> None:
"""系统关闭开始后到达的配置事件不得重新打开调度入口。"""
scheduler = _scheduler("shutdown-reload", lambda: None)
scheduler._lifecycle_state = "stopping"
scheduler.init = lambda **_kwargs: pytest.fail("关闭阶段不得重新初始化调度器")
await scheduler.on_config_changed()
assert scheduler._lifecycle_state == "stopping"
@pytest.mark.anyio
async def test_concurrent_config_reload_waits_for_old_scheduler_shutdown(
monkeypatch,
) -> None:
"""并发配置事件合并为一次重建，旧调度线程池结束前不得启动新实例。"""
shutdown_started = threading.Event()
shutdown_release = threading.Event()
class BlockingScheduler:
running = True
@staticmethod
def remove_all_jobs() -> None:
pass
@staticmethod
def shutdown() -> None:
shutdown_started.set()
shutdown_release.wait(timeout=1)
scheduler = _scheduler("reload-once", lambda: None)
scheduler._scheduler = BlockingScheduler()
init_calls = 0
def init(**_kwargs) -> None:
nonlocal init_calls
init_calls += 1
scheduler._lifecycle_state = "running"
monkeypatch.setattr(scheduler, "init", init)
first = asyncio.create_task(scheduler.on_config_changed())
assert await asyncio.to_thread(shutdown_started.wait, 1)
await scheduler.on_config_changed()
assert init_calls == 0
assert scheduler._lifecycle_state == "reloading"
shutdown_release.set()
await asyncio.wait_for(first, timeout=1)
assert init_calls == 1
assert scheduler._lifecycle_state == "running"
@pytest.mark.anyio
async def test_config_reload_preserves_overlap_guard_across_job_generations(
monkeypatch,
) -> None:
"""热重载替换任务定义后，同 ID 旧任务结束前不得启动新 generation。"""
started = asyncio.Event()
release = asyncio.Event()
finished = asyncio.Event()
run_count = 0
async def job() -> None:
nonlocal run_count
run_count += 1
if run_count == 1:
started.set()
await release.wait()
finished.set()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("reload-overlap", job)
class ActiveScheduler:
"""提供列表接口所需的最小 APScheduler 状态。"""
running = True
@staticmethod
def get_jobs() -> list:
"""当前用例只关注正在运行任务，不提供后续计划。"""
return []
def init(**_kwargs) -> None:
replacement = {
"name": "生命周期测试",
"provider_name": "测试",
"func": job,
"running": False,
}
scheduler._assign_job_generation("reload-overlap", replacement)
scheduler._jobs = {"reload-overlap": replacement}
scheduler._scheduler = ActiveScheduler()
scheduler._lifecycle_state = "running"
monkeypatch.setattr(scheduler, "init", init)
assert scheduler.start("reload-overlap") is True
await asyncio.wait_for(started.wait(), timeout=1)
await scheduler.on_config_changed()
progress = scheduler.get_progress("reload-overlap")
assert progress is not None
assert progress.status == "running"
assert progress.enable is True
listed = scheduler.list()
assert len(listed) == 1
assert listed[0].id == "reload-overlap"
assert listed[0].status == "正在运行"
assert scheduler.start("reload-overlap") is False
assert run_count == 1
assert len(scheduler._handles) == 1
release.set()
await asyncio.wait_for(finished.wait(), timeout=1)
async def wait_until_released() -> None:
while scheduler._active_job_generations or scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
assert scheduler.start("reload-overlap") is True
async def wait_until_second_run_finishes() -> None:
while scheduler._active_job_generations or scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_second_run_finishes(), timeout=1)
assert run_count == 2
def test_stop_between_prepare_and_submission_releases_active_generation(
monkeypatch,
) -> None:
"""关闭插入准备与提交之间时，不得遗留未实际运行的 generation。"""
calls = 0
scheduler = _scheduler("stop-race", None)
async def job() -> None:
nonlocal calls
calls += 1
scheduler._jobs["stop-race"]["func"] = job
original_prepare = scheduler._Scheduler__prepare_job
def prepare_then_stop(job_id: str):
prepared = original_prepare(job_id)
scheduler._begin_stop()
return prepared
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", prepare_then_stop)
assert scheduler.start("stop-race") is False
assert calls == 0
assert scheduler._handles == {}
assert scheduler._active_job_generations == {}
assert scheduler._jobs["stop-race"]["running"] is False
assert scheduler._jobs["stop-race"]["last_error"] == "任务未提交"
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", original_prepare)
scheduler._lifecycle_state = "running"
assert scheduler.start("stop-race") is True
assert calls == 1
assert scheduler._active_job_generations == {}
@pytest.mark.anyio
async def test_cross_thread_rejection_closes_unstarted_business_coroutine(
monkeypatch,
) -> None:
"""跨线程提交被关闭门禁拒绝时，包装与业务协程都必须释放。"""
main_loop = asyncio.get_running_loop()
prepared = threading.Event()
release = threading.Event()
calls = 0
scheduler = _scheduler("cross-thread-stop-race", None)
async def job() -> None:
nonlocal calls
calls += 1
scheduler._jobs["cross-thread-stop-race"]["func"] = job
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
original_prepare = scheduler._Scheduler__prepare_job
def prepare_then_wait(job_id: str):
result = original_prepare(job_id)
prepared.set()
release.wait(timeout=1)
return result
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", prepare_then_wait)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always", RuntimeWarning)
start_task = asyncio.create_task(
asyncio.to_thread(scheduler.start, "cross-thread-stop-race")
)
assert await asyncio.to_thread(prepared.wait, 1)
scheduler._begin_stop()
release.set()
assert await asyncio.wait_for(start_task, timeout=1) is False
gc.collect()
assert calls == 0
assert scheduler._active_job_generations == {}
assert scheduler._handles == {}
assert not any("was never awaited" in str(item.message) for item in captured)
def test_cancelled_cross_thread_proxy_waits_for_target_loop_cleanup(
monkeypatch,
) -> None:
"""跨线程代理提前取消后，真实完成信号必须等待目标循环清理。"""
target_loop = asyncio.new_event_loop()
loop_blocked = threading.Event()
loop_release = threading.Event()
loop_drained = threading.Event()
loop_errors = []
loop_thread = threading.Thread(target=target_loop.run_forever)
loop_thread.start()
def block_target_loop() -> None:
loop_blocked.set()
loop_release.wait(timeout=1)
target_loop.set_exception_handler(
lambda _loop, context: loop_errors.append(context)
)
target_loop.call_soon_threadsafe(block_target_loop)
assert loop_blocked.wait(timeout=1)
scheduler = _scheduler("cancel-before-start", None)
calls = 0
async def business() -> None:
nonlocal calls
calls += 1
scheduler._jobs["cancel-before-start"]["func"] = business
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", target_loop)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always", RuntimeWarning)
try:
assert scheduler.start("cancel-before-start") is True
scheduler_handle = next(iter(scheduler._handles.values()))
scheduler._cancel_handle(scheduler_handle)
assert not scheduler_handle.completion.done()
loop_release.set()
target_loop.call_soon_threadsafe(loop_drained.set)
assert loop_drained.wait(timeout=1)
assert scheduler_handle.completion.done()
gc.collect()
finally:
target_loop.call_soon_threadsafe(target_loop.stop)
loop_thread.join(timeout=1)
target_loop.close()
assert calls == 0
assert loop_errors == []
assert scheduler._active_job_generations == {}
assert scheduler._handles == {}
assert not any("was never awaited" in str(item.message) for item in captured)
tests/test_scheduler_progress.py
+15 -7
View File
@@ -14,8 +14,6 @@ def _build_scheduler(job_id, func):
scheduler._scheduler = None
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._async_tasks = set()
scheduler._accepting_async_tasks = True
scheduler._jobs = {
job_id: {
"name": "测试定时服务",
@@ -24,6 +22,11 @@ def _build_scheduler(job_id, func):
"running": False,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
@@ -161,7 +164,7 @@ def test_scheduler_records_cancelled_async_job_as_failed():
async def run_task():
job = scheduler._Scheduler__prepare_job(job_id)
with pytest.raises(asyncio.CancelledError):
await scheduler._Scheduler__run_coro_job(task(), job_id, job)
await scheduler._Scheduler__run_coro_job(task, job_id, job)
scheduler = _build_scheduler(job_id, task)
asyncio.run(run_task())
@@ -173,7 +176,7 @@ def test_scheduler_records_cancelled_async_job_as_failed():
assert progress.error == "任务已取消"
def test_scheduler_async_stop_cancels_owned_async_jobs():
def test_scheduler_stop_async_cancels_owned_async_jobs():
"""Scheduler 关停应取消并等待自身登记的异步作业。"""
job_id = f"test-owned-task-{uuid4()}"
started = asyncio.Event()
@@ -194,13 +197,13 @@ def test_scheduler_async_stop_cancels_owned_async_jobs():
"""在当前事件循环启动并收口异步作业。"""
scheduler.start(job_id)
await started.wait()
assert len(scheduler._async_tasks) == 1
await scheduler.async_stop(timeout_seconds=1)
assert len(scheduler._handles) == 1
await scheduler.stop_async()
asyncio.run(run_task())
assert cancelled.is_set()
assert scheduler._async_tasks == set()
assert scheduler._handles == {}
def test_scheduler_returns_none_for_unknown_job():
@@ -209,5 +212,10 @@ def test_scheduler_returns_none_for_unknown_job():
scheduler = object.__new__(Scheduler)
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
assert scheduler.get_progress(job_id) is None
tests/test_site_reset_scheduler.py
+60
View File
@@ -0,0 +1,60 @@
from unittest.mock import AsyncMock, Mock
import pytest
from app.api.endpoints import site as site_endpoint
from app.application.site.mutation import SiteMutationResult
from app.runtime.tasks import TaskRegistry
class _TaskRegistry(TaskRegistry):
"""记录站点重置提交的同步后台任务。"""
def __init__(self) -> None:
"""初始化任务记录。"""
super().__init__()
self.calls: list[tuple] = []
def create_sync(self, function, *args, owner: str, **kwargs) -> None:
"""保存函数、参数和 owner，避免端点测试启动真实任务。"""
self.calls.append((function, args, kwargs, owner))
@pytest.mark.asyncio
async def test_reset_submits_cookiecloud_after_site_transaction(monkeypatch):
"""站点重置提交 CookieCloud 后台任务，不在请求事件循环内直接执行。"""
command = Mock()
command.reset = AsyncMock(return_value=SiteMutationResult(success=True))
task_registry = _TaskRegistry()
scheduler = Mock()
system_config = Mock()
system_config.async_set = AsyncMock()
monkeypatch.setattr(site_endpoint, "Scheduler", Mock(return_value=scheduler))
monkeypatch.setattr(
site_endpoint,
"get_configured_system_config",
Mock(return_value=system_config),
)
response = await site_endpoint.reset(
task_registry=task_registry,
command=command,
_=Mock(),
)
assert response.success is True
command.reset.assert_awaited_once_with()
scheduler.start.assert_not_called()
system_config.async_set.assert_any_await(
site_endpoint.SystemConfigKey.IndexerSites, []
)
system_config.async_set.assert_any_await(site_endpoint.SystemConfigKey.RssSites, [])
assert task_registry.calls == [
(
scheduler.start,
(),
{"job_id": "cookiecloud", "manual": True},
"api.site.reset",
)
]