From 7f09927c47ffbc853bf921ab9660fe3a21a08302 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 23 Aug 2026 17:20:24 +0800 Subject: [PATCH] refactor: own shutdown lifecycle boundaries --- app/agent/capabilities/adapter.py | 6 +- app/agent/middleware/subagents.py | 40 +- app/agent/orchestrator.py | 167 ++++-- app/agent/runtime_loader.py | 19 +- app/agent/tools/base.py | 143 +++++- app/agent/tools/impl/_plugin_tool_utils.py | 150 +++--- app/agent/tools/impl/update_plugin_config.py | 78 +-- app/api/dependencies/plugin.py | 1 + app/api/endpoints/plugin.py | 260 ++++++---- app/application/plugin/config.py | 38 +- app/application/plugin/install.py | 44 +- app/application/plugin/runtime.py | 7 +- app/application/transfer.py | 199 +++++-- app/chain/transfer.py | 407 ++++++++++++--- app/foundation/singleton.py | 28 +- app/monitor/monitor.py | 456 ++++++++++++++-- app/monitor/recovery.py | 85 ++- app/runtime/event/dispatch.py | 10 + app/runtime/events.py | 206 +++++++- app/runtime/extensions/plugin/admission.py | 112 ++++ app/runtime/extensions/plugin/lifecycle.py | 190 +++++-- app/runtime/extensions/plugin/monitor.py | 85 ++- app/runtime/extensions/plugin_manager.py | 360 ++++++++++--- app/runtime/tasks.py | 5 +- app/schemas/exception.py | 10 + app/startup/agent_initializer.py | 92 +++- app/startup/lifecycle/__init__.py | 206 ++++++-- app/startup/modules_initializer.py | 19 +- app/startup/monitor_initializer.py | 31 +- app/startup/plugins_initializer.py | 158 ++++-- app/startup/transfer_initializer.py | 12 + .../backend-architecture-next-stage.md | 36 +- .../architecture/dependency-baseline.json | 8 +- .../startup-performance-baseline.json | 272 +++++----- tests/test_agent_lazy_initializer.py | 116 ++++- tests/test_agent_lifecycle.py | 48 +- tests/test_agent_plugin_tools.py | 40 +- tests/test_agent_runtime_loader.py | 37 +- tests/test_agent_side_effect_boundaries.py | 57 +- tests/test_agent_tool_timeouts.py | 60 +++ tests/test_event_dispatch_snapshot.py | 410 ++++++++++++++- tests/test_lifecycle_shutdown.py | 458 ++++++++++++++++- tests/test_monitor_gap_recovery.py | 15 +- tests/test_monitor_lifecycle.py | 345 +++++++++++++ tests/test_monitor_mount_isolation.py | 23 +- tests/test_monitor_resilience.py | 14 +- tests/test_monitor_watchfiles.py | 15 +- tests/test_plugin_config_command.py | 30 +- tests/test_plugin_endpoint.py | 79 +++ tests/test_plugin_install_command.py | 31 ++ tests/test_plugin_lifecycle_status.py | 223 ++++++++ tests/test_plugin_monitor_lifecycle.py | 424 ++++++++++++++- tests/test_plugin_mutation_admission.py | 155 ++++++ tests/test_singleton.py | 48 ++ tests/test_task_registry.py | 8 +- tests/test_transfer_failed_retry_scheduler.py | 126 +++++ ...ansfer_failure_notification_aggregation.py | 132 +++++ tests/test_transfer_pending_replay.py | 32 +- tests/test_transfer_worker_lifecycle.py | 485 ++++++++++++++++++ 59 files changed, 6393 insertions(+), 958 deletions(-) create mode 100644 app/runtime/extensions/plugin/admission.py create mode 100644 tests/test_monitor_lifecycle.py create mode 100644 tests/test_plugin_mutation_admission.py create mode 100644 tests/test_transfer_failed_retry_scheduler.py create mode 100644 tests/test_transfer_worker_lifecycle.py diff --git a/app/agent/capabilities/adapter.py b/app/agent/capabilities/adapter.py index 04b3510d0..9edaa8499 100644 --- a/app/agent/capabilities/adapter.py +++ b/app/agent/capabilities/adapter.py @@ -155,7 +155,11 @@ class AgentServiceAdapter: raise CapabilityAdapterContractError( f"{spec.entrypoint}.close() 必须返回 awaitable" ) - await result + converged = await result + if converged is False: + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.close() 返回未收敛，保留 service owner" + ) @staticmethod async def cleanup( diff --git a/app/agent/middleware/subagents.py b/app/agent/middleware/subagents.py index 990c05aa0..e3fc2a6c5 100644 --- a/app/agent/middleware/subagents.py +++ b/app/agent/middleware/subagents.py @@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): ) self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS) self._tasks: dict[str, _SubAgentRuntimeTask] = {} + self._accepting_tasks = True + self._close_cancel_requested: set[asyncio.Task] = set() self.tools = [ StructuredTool.from_function( coroutine=self._control_task, @@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None: """记录任务完成时间并取出异常避免未读取告警。""" + cancel_requested = getattr(self, "_close_cancel_requested", None) + if cancel_requested is not None: + cancel_requested.discard(task) record = self._tasks.get(task_id) if record: record.finished_at = datetime.now() @@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): @staticmethod async def _cancel_records( records: list[_SubAgentRuntimeTask], + *, + cancel_requested: Optional[set[asyncio.Task]] = None, ) -> list[_SubAgentRuntimeTask]: """取消一组任务，并返回等待上限内仍未收敛的记录。""" cancellable_tasks = [ @@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): if cancellable_tasks: logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}") for task in cancellable_tasks: + if cancel_requested is not None and task in cancel_requested: + continue task.cancel() + if cancel_requested is not None: + cancel_requested.add(task) if not cancellable_tasks: return [] @@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}") return [record for record in records if record.task in pending] - async def close(self) -> None: - """取消脱离当前 Agent 回合的子代理任务。""" + def seal(self) -> None: + """封住新的 detached 子代理提交，既有任务继续由记录表持有。""" + self._accepting_tasks = False + + async def close(self) -> bool: + """有限等待 detached 子代理；超时保留记录并返回 False。""" + self.seal() + if not hasattr(self, "_close_cancel_requested"): + self._close_cancel_requested = set() unfinished_records = [ record for record in self._tasks.values() if not record.task.done() ] @@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): logger.info( f"关闭子代理任务控制器，取消未完成任务: tasks={len(unfinished_records)}" ) - await self._cancel_records(unfinished_records) + pending_records = await self._cancel_records( + unfinished_records, + cancel_requested=self._close_cancel_requested, + ) + if pending_records: + self._tasks = { + record.task_id: record + for record in pending_records + } + return False self._tasks.clear() + self._close_cancel_requested.clear() + return True @staticmethod def _pipeline_description( @@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): previous_results: list[tuple[_SubAgentRuntimeTask, str]] = [] timeout = normalized_timeout_ms / 1000 for step_index, spec in enumerate(specs, start=1): + if not self._accepting_tasks: + return records, "子代理任务控制器正在关闭，不能再启动新任务。" record = self._create_pipeline_record(spec) records.append(record) pipeline_description = self._pipeline_description( @@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): """管理异步子代理任务。""" logger.info(f"收到子代理管控操作: action={action}") if action in {"start", "run", "pipeline"}: + if not self._accepting_tasks: + error = "子代理任务控制器正在关闭，不能再启动新任务。" + return self._json_response({"success": False, "error": error}) specs, error = self._normalize_specs( description=description, subagent_type=subagent_type, diff --git a/app/agent/orchestrator.py b/app/agent/orchestrator.py index ffc5f295a..030c90abe 100644 --- a/app/agent/orchestrator.py +++ b/app/agent/orchestrator.py @@ -436,6 +436,7 @@ class MoviePilotAgent: self._agent_started_at: Optional[datetime] = None self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None self._subagent_middlewares: tuple[Any, ...] = () + self._shutdown_started = False self._last_agent_cache_hit = False # 流式token管理 @@ -1751,10 +1752,25 @@ class MoviePilotAgent: return None @staticmethod - async def _close_subagent_middleware_instances( + def _seal_subagent_middleware_instances( middlewares: tuple[Any, ...], ) -> None: - """释放不再由 Agent 图持有的子代理控制器。""" + """同步封住子代理控制器的新任务入口，不等待既有任务退出。""" + for middleware in middlewares: + seal = getattr(middleware, "seal", None) + if not callable(seal): + continue + try: + seal() + except Exception as error: + logger.debug(f"封住子代理中间件失败: {error}") + + @staticmethod + async def _close_subagent_middleware_instances( + middlewares: tuple[Any, ...], + ) -> tuple[Any, ...]: + """关闭子代理控制器，并返回仍持有未收敛任务的实例。""" + pending_middlewares = [] for middleware in middlewares: close = getattr(middleware, "close", None) if not callable(close): @@ -1762,9 +1778,30 @@ class MoviePilotAgent: try: result = close() if inspect.isawaitable(result): - await result + result = await result + if result is False: + pending_middlewares.append(middleware) except Exception as error: logger.debug(f"关闭子代理中间件失败: {error}") + pending_middlewares.append(middleware) + return tuple(pending_middlewares) + + @staticmethod + def _merge_subagent_middleware_owners( + *groups: tuple[Any, ...], + ) -> tuple[Any, ...]: + """按对象身份合并当前图与延迟收敛控制器的 owner 集合。""" + merged = [] + for group in groups: + for middleware in group: + if not any(middleware is existing for existing in merged): + merged.append(middleware) + return tuple(merged) + + def begin_shutdown(self) -> None: + """在任何异步等待前封住当前 Agent 的 detached 子代理提交。""" + self._shutdown_started = True + self._seal_subagent_middleware_instances(self._subagent_middlewares) async def _cache_agent( self, @@ -1786,7 +1823,11 @@ class MoviePilotAgent: for replacement in subagent_middlewares ) ) - await self._close_subagent_middleware_instances(previous_middlewares) + if self._shutdown_started: + self._seal_subagent_middleware_instances(subagent_middlewares) + pending_middlewares = await self._close_subagent_middleware_instances( + previous_middlewares + ) self._compiled_agent_bundle = _CompiledAgentBundle( signature=signature, agent=agent, @@ -1798,15 +1839,21 @@ class MoviePilotAgent: mcp_config_signature=mcp_config_signature, catalog_checked_at=datetime.now(), ) - self._subagent_middlewares = subagent_middlewares + self._subagent_middlewares = self._merge_subagent_middleware_owners( + pending_middlewares, + subagent_middlewares, + ) return agent - async def _invalidate_cached_agent(self) -> None: - """使当前图失效，并释放只属于该图的子代理控制器。""" + async def _invalidate_cached_agent(self) -> bool: + """使当前图失效，未收敛的子代理控制器继续由 Agent 持有。""" subagent_middlewares = self._subagent_middlewares - self._subagent_middlewares = () self._compiled_agent_bundle = None - await self._close_subagent_middleware_instances(subagent_middlewares) + pending_middlewares = await self._close_subagent_middleware_instances( + subagent_middlewares + ) + self._subagent_middlewares = pending_middlewares + return not pending_middlewares @staticmethod def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]: @@ -2003,9 +2050,13 @@ class MoviePilotAgent: if cached_agent: # 签名相同表示已编译图中的精确工具实例仍有效；新建快照仅用于复核。 cached_bundle.catalog_checked_at = datetime.now() - await self._close_subagent_middleware_instances( + pending_middlewares = await self._close_subagent_middleware_instances( temporary_subagent_middlewares ) + self._subagent_middlewares = self._merge_subagent_middleware_owners( + self._subagent_middlewares, + pending_middlewares, + ) temporary_subagent_middlewares = () logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}") return cached_agent @@ -2123,14 +2174,22 @@ class MoviePilotAgent: temporary_subagent_middlewares = () return cached_agent except asyncio.CancelledError: - await self._close_subagent_middleware_instances( + pending_middlewares = await self._close_subagent_middleware_instances( temporary_subagent_middlewares ) + self._subagent_middlewares = self._merge_subagent_middleware_owners( + self._subagent_middlewares, + pending_middlewares, + ) raise except Exception as e: - await self._close_subagent_middleware_instances( + pending_middlewares = await self._close_subagent_middleware_instances( temporary_subagent_middlewares ) + self._subagent_middlewares = self._merge_subagent_middleware_owners( + self._subagent_middlewares, + pending_middlewares, + ) logger.error(f"创建 Agent 失败: {e}") raise @@ -2539,14 +2598,20 @@ class MoviePilotAgent: ) ) - async def cleanup(self): + async def cleanup(self) -> bool: """ - 清理智能体资源 + 清理智能体资源；detached 子代理未收敛时保留 owner 并返回 False。 """ - await self._invalidate_cached_agent() + self.begin_shutdown() + if not await self._invalidate_cached_agent(): + logger.error( + f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}" + ) + return False self._pending_secret_confirmation = None self.protected_output_callback = None logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}") + return True @dataclass @@ -2626,6 +2691,7 @@ class AgentManager: self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {} self._session_cancel_requested: set[str] = set() self._close_finalizer_task: Optional[asyncio.Task] = None + self._closed = False self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT # 接收门禁与队列写入共用一把锁，确保关闭开始后不会再创建 worker。 self._lifecycle_lock = asyncio.Lock() @@ -2693,22 +2759,33 @@ class AgentManager: async with self._lifecycle_lock: if self._accepting_tasks: return + if self._close_finalizer_task and not self._close_finalizer_task.done(): + raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭") memory_manager.initialize() if not self._idle_cleanup_task or self._idle_cleanup_task.done(): self._idle_cleanup_task = asyncio.create_task( self._cleanup_idle_sessions() ) self._accepting_tasks = True + self._closed = False - async def close(self): + async def close(self) -> bool: """ - 关闭管理器 + 关闭管理器，并诚实返回全部会话 owner 是否已经收敛。 """ async with self._lifecycle_lock: + if self._closed: + return True if self._close_finalizer_task and not self._close_finalizer_task.done(): - return + return False # 门禁必须先关闭；锁内完成清理可阻止等待中的请求在收口期间重新入队。 self._accepting_tasks = False + # 子代理提交门禁同样必须在第一次 await 前关闭，否则 detached task + # 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。 + for agent in self.active_agents.values(): + begin_shutdown = getattr(agent, "begin_shutdown", None) + if callable(begin_shutdown): + begin_shutdown() if self._idle_cleanup_task: self._idle_cleanup_task.cancel() try: @@ -2751,8 +2828,8 @@ class AgentManager: self._session_workers.pop(session_id, None) for session_id, agent in list(self.active_agents.items()): if session_id not in timed_out_session_ids: - await agent.cleanup() - self.active_agents.pop(session_id, None) + if await agent.cleanup() is not False: + self.active_agents.pop(session_id, None) logger.error( "AgentManager 关闭时仍有 worker 未收敛，" f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束" @@ -2760,13 +2837,21 @@ class AgentManager: self._close_finalizer_task = asyncio.create_task( self._finish_deferred_close(timed_out_workers) ) - return + return False self._session_workers.clear() - for agent in list(self.active_agents.values()): - await agent.cleanup() - self.active_agents.clear() + for session_id, agent in list(self.active_agents.items()): + if await agent.cleanup() is not False: + self.active_agents.pop(session_id, None) + if self.active_agents: + logger.error( + "AgentManager 仍有 %d 个 detached 子代理 owner 未收敛", + len(self.active_agents), + ) + return False await memory_manager.close() + self._closed = True + return True def _record_session_activity(self, session_id: str, user_id: str) -> None: """ @@ -3072,7 +3157,10 @@ class AgentManager: and isinstance(task.agent_factory, type) and not isinstance(existing_agent, task.agent_factory) ): - await existing_agent.cleanup() + if await existing_agent.cleanup() is False: + raise AgentManagerUnavailableError( + f"Agent 会话 {session_id} 仍有子代理任务在停止" + ) self.active_agents.pop(session_id, None) if session_id not in self.active_agents: @@ -3249,7 +3337,11 @@ class AgentManager: # 清理agent if session_id in self.active_agents: agent = self.active_agents[session_id] - await agent.cleanup() + if await agent.cleanup() is False: + logger.error( + f"会话 {session_id} 仍有子代理 owner 未收敛，保留会话与记忆" + ) + return del self.active_agents[session_id] memory_manager.clear_memory(session_id, user_id) logger.info(f"会话 {session_id} 的记忆已清空") @@ -3290,9 +3382,14 @@ class AgentManager: self._session_deferred_cleanup_tasks.pop(session_id, None) self._session_queue_rejections.pop(session_id, None) self._session_last_queue_wait_ms.pop(session_id, None) - agent = self.active_agents.pop(session_id, None) + agent = self.active_agents.get(session_id) if agent: - await agent.cleanup() + if await agent.cleanup() is False: + logger.error( + f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛" + ) + return + self.active_agents.pop(session_id, None) memory_manager.clear_memory(session_id, user_id) logger.info(f"会话 {session_id} 的记忆已清空") @@ -3310,15 +3407,19 @@ class AgentManager: for session_id, worker in workers: if self._session_workers.get(session_id) is worker: self._session_workers.pop(session_id, None) - agent = self.active_agents.pop(session_id, None) - if agent: - await agent.cleanup() for session_id, agent in list(self.active_agents.items()): - await agent.cleanup() - self.active_agents.pop(session_id, None) + if await agent.cleanup() is not False: + self.active_agents.pop(session_id, None) self._session_shutdown_pending.clear() self._session_cancel_requested.clear() + if self.active_agents: + logger.error( + "AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛", + len(self.active_agents), + ) + return await memory_manager.close() + self._closed = True finally: self._close_finalizer_task = None diff --git a/app/agent/runtime_loader.py b/app/agent/runtime_loader.py index ed72e86bd..ec762efe2 100644 --- a/app/agent/runtime_loader.py +++ b/app/agent/runtime_loader.py @@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import ( build_agent_capability_registry, should_run_agent_service, ) -from app.runtime.capabilities.model import CapabilityMaterializationState +from app.runtime.capabilities.model import ( + CapabilityLifecycleState, + CapabilityMaterializationState, +) from app.runtime.capabilities.runtime import CapabilityRuntime @@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None: await close() -async def begin_agent_shutdown() -> None: - """不可逆关闭首用闸门，并等待全部同步及异步能力释放。""" - try: - await _ensure_runtime().shutdown_async(reason="application_shutdown") - finally: - await close_materialized_terminal_sessions() +async def begin_agent_shutdown() -> bool: + """不可逆关闭首用闸门，并返回 Agent service 是否真实收敛。""" + runtime = _ensure_runtime() + await runtime.shutdown_async(reason="application_shutdown") + return ( + runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle + is CapabilityLifecycleState.STOPPED + ) diff --git a/app/agent/tools/base.py b/app/agent/tools/base.py index 845c4198a..dd6c272c9 100644 --- a/app/agent/tools/base.py +++ b/app/agent/tools/base.py @@ -3,7 +3,7 @@ import inspect import json import threading from abc import ABCMeta, abstractmethod -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor from contextvars import Context, copy_context from functools import partial from pathlib import Path @@ -168,33 +168,130 @@ _blocking_semaphores = { for bucket, limit in _BLOCKING_BUCKET_LIMITS.items() } _blocking_executors: dict[str, ThreadPoolExecutor] = {} -_blocking_executor_lock = threading.Lock() +_blocking_retiring_executors: set[ThreadPoolExecutor] = set() +_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {} +_blocking_executor_lock = threading.RLock() +_blocking_executor_accepting = True -def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor: - """按桶懒加载线程池，避免在导入阶段创建过多 worker。""" +def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None: + """在同步调用到达终态后撤销 Future 与 retiring executor owner。""" with _blocking_executor_lock: + executor = _blocking_futures.pop(future, None) + if executor is None or executor not in _blocking_retiring_executors: + return + if executor not in _blocking_futures.values(): + _blocking_retiring_executors.discard(executor) + + +def _submit_blocking_call( + bucket: str, + bound_call: Callable[[], Any], +) -> ConcurrentFuture[Any]: + """在提交门禁内原子取得 executor、提交调用并登记 Future owner。""" + context = copy_context() + with _blocking_executor_lock: + if not _blocking_executor_accepting: + raise RuntimeError("Agent 工具阻塞执行器正在关闭，不能再提交新任务") executor = _blocking_executors.get(bucket) - if executor: - return executor - - limit = _BLOCKING_BUCKET_LIMITS[bucket] - executor = ThreadPoolExecutor( - max_workers=limit, - thread_name_prefix=f"agent-tool-{bucket}", - ) - _blocking_executors[bucket] = executor - return executor + if executor is None: + limit = _BLOCKING_BUCKET_LIMITS[bucket] + executor = ThreadPoolExecutor( + max_workers=limit, + thread_name_prefix=f"agent-tool-{bucket}", + ) + _blocking_executors[bucket] = executor + # 长期 worker 保持空底层上下文，每个任务只在自己的调用快照内运行。 + future = Context().run(executor.submit, context.run, bound_call) + _blocking_futures[future] = executor + future.add_done_callback(_discard_blocking_future) + return future -def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None: - """关闭 Agent 工具阻塞线程池，释放长期运行进程或测试环境中的 worker。""" +def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]: + """撤销活动 executor 的提交资格，并保留其运行 Future 对应的 owner。""" with _blocking_executor_lock: - executors = list(_blocking_executors.values()) + executors = tuple(_blocking_executors.values()) _blocking_executors.clear() - + _blocking_retiring_executors.update(executors) for executor in executors: - executor.shutdown(wait=wait, cancel_futures=cancel_futures) + executor.shutdown(wait=False, cancel_futures=cancel_futures) + with _blocking_executor_lock: + owned_executors = set(_blocking_futures.values()) + _blocking_retiring_executors.intersection_update(owned_executors) + return executors + + +def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None: + """原子封住新阻塞工具提交，并请求取消尚未开始的同步调用。""" + global _blocking_executor_accepting + with _blocking_executor_lock: + _blocking_executor_accepting = False + _retire_blocking_executors(cancel_futures=cancel_futures) + + +def reopen_blocking_executors() -> bool: + """仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。""" + global _blocking_executor_accepting + with _blocking_executor_lock: + if _blocking_futures or _blocking_retiring_executors: + return False + _blocking_executor_accepting = True + return True + + +async def close_blocking_executors( + *, + timeout_seconds: float, + cancel_futures: bool = True, +) -> bool: + """有限等待全部阻塞工具 Future，超时保留 Future 与 executor owner。""" + begin_blocking_executor_shutdown(cancel_futures=cancel_futures) + with _blocking_executor_lock: + futures = tuple(_blocking_futures) + wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures) + if wrapped_futures: + done, pending = await asyncio.wait( + wrapped_futures, + timeout=max(0.0, timeout_seconds), + ) + if done: + await asyncio.gather(*done, return_exceptions=True) + for pending_future in pending: + pending_future.add_done_callback( + lambda completed: completed.exception() + if not completed.cancelled() + else None + ) + + with _blocking_executor_lock: + unfinished = tuple( + future for future in _blocking_futures if not future.done() + ) + retiring_count = len(_blocking_retiring_executors) + if unfinished: + logger.error( + "Agent 阻塞工具未在 %.1f 秒内收敛：futures=%d，executors=%d", + max(0.0, timeout_seconds), + len(unfinished), + retiring_count, + ) + return False + return True + + +def shutdown_blocking_executors( + *, + wait: bool = True, + cancel_futures: bool = False, +) -> bool: + """同步清理测试 owner；非等待模式下保留尚未收敛的 executor 句柄。""" + executors = _retire_blocking_executors(cancel_futures=cancel_futures) + for executor in executors: + if wait: + executor.shutdown(wait=True, cancel_futures=cancel_futures) + with _blocking_executor_lock: + return not _blocking_futures and not _blocking_retiring_executors class ToolExecutionTimeoutError(TimeoutError): @@ -226,13 +323,7 @@ async def run_agent_blocking( await semaphore.acquire() try: - context = copy_context() - # 长期 worker 保持空底层上下文，每个任务只在自己的调用快照内运行。 - future = Context().run( - _get_blocking_executor(bucket_name).submit, - context.run, - bound_call, - ) + future = _submit_blocking_call(bucket_name, bound_call) except Exception: semaphore.release() raise diff --git a/app/agent/tools/impl/_plugin_tool_utils.py b/app/agent/tools/impl/_plugin_tool_utils.py index a7ce7c67e..1647064ec 100644 --- a/app/agent/tools/impl/_plugin_tool_utils.py +++ b/app/agent/tools/impl/_plugin_tool_utils.py @@ -2,6 +2,7 @@ import json import shutil +from contextvars import copy_context from pathlib import Path from typing import Any, Optional @@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None: def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus: """重载插件实例并重新注册其命令、定时任务和 API。""" - runtime_status = get_plugin_manager().reload_plugin(plugin_id) - refresh_plugin_registrations(plugin_id) - return runtime_status + plugin_manager = get_plugin_manager() + with plugin_manager.mutation(f"重载插件 {plugin_id}"): + runtime_status = plugin_manager.reload_plugin(plugin_id) + refresh_plugin_registrations(plugin_id) + return runtime_status def summarize_plugin(plugin: Any) -> dict[str, Any]: @@ -351,8 +354,10 @@ async def install_plugin_runtime( async def reload_runtime(target_id: str) -> object: """通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。""" + mutation_context = copy_context() return await run_agent_blocking( "plugin", + mutation_context.run, plugin_manager.reload_plugin_tree, target_id, ) @@ -371,31 +376,32 @@ async def install_plugin_runtime( ) return result - with plugin_manager.suppress_plugin_monitor(plugin_id): - result = await PluginInstallCommand( - installed_plugins_reader=lambda: SystemConfigOper().get( - SystemConfigKey.UserInstalledPlugins - ) or [], - installed_plugins_writer=save_installed_plugins, - plugin_ids_provider=plugin_manager.get_plugin_ids, - compatibility_checker=skip_compatibility_check, - package_installer=install_package, - package_checkpointer=package_manager.async_checkpoint, - package_committer=package_manager.async_commit, - package_rollback=package_manager.async_rollback, - install_reporter=lambda target_id, target_repo: ( - MoviePilotServerHelper.async_install_plugin_reg( - plugin_id=target_id, - repo_url=target_repo, - ) - ), - plugin_reloader=reload_runtime, - registration_refresher=refresh_registrations, - ).execute( - plugin_id=plugin_id, - repo_url=repo_url, - force=force, - ) + result = await PluginInstallCommand( + installed_plugins_reader=lambda: SystemConfigOper().get( + SystemConfigKey.UserInstalledPlugins + ) or [], + installed_plugins_writer=save_installed_plugins, + plugin_ids_provider=plugin_manager.get_plugin_ids, + compatibility_checker=skip_compatibility_check, + package_installer=install_package, + package_checkpointer=package_manager.async_checkpoint, + package_committer=package_manager.async_commit, + package_rollback=package_manager.async_rollback, + install_reporter=lambda target_id, target_repo: ( + MoviePilotServerHelper.async_install_plugin_reg( + plugin_id=target_id, + repo_url=target_repo, + ) + ), + plugin_reloader=reload_runtime, + registration_refresher=refresh_registrations, + mutation=plugin_manager.mutation, + package_write_guard=plugin_manager.suppress_plugin_monitor, + ).execute( + plugin_id=plugin_id, + repo_url=repo_url, + force=force, + ) return result.success, result.message, result.refreshed_only @@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]: from app.agent.tools.base import run_agent_blocking plugin_manager = get_plugin_manager() - virtual_instance = plugin_manager.get_plugin_instance(plugin_id) - source_instances = plugin_manager.get_plugin_source_instances(plugin_id) - if not virtual_instance and source_instances: - instance_ids = "、".join(item.instance_id for item in source_instances) - raise ValueError(f"请先卸载该插件的分身：{instance_ids}") + with plugin_manager.mutation(f"卸载插件 {plugin_id}"): + virtual_instance = plugin_manager.get_plugin_instance(plugin_id) + source_instances = plugin_manager.get_plugin_source_instances(plugin_id) + if not virtual_instance and source_instances: + instance_ids = "、".join(item.instance_id for item in source_instances) + raise ValueError(f"请先卸载该插件的分身：{instance_ids}") - config_oper = SystemConfigOper() - install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or [] - if plugin_id in install_plugins: - install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id] - await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins) - - remove_plugin_api(plugin_id) - remove_plugin_job(plugin_id) - - plugin_class = plugin_manager.plugins.get(plugin_id) - was_clone = bool(getattr(plugin_class, "is_clone", False)) - clone_files_removed = False - - if virtual_instance: - plugin_manager.delete_plugin_config(plugin_id, force=True) - plugin_manager.delete_plugin_data(plugin_id, force=True) - plugin_manager.delete_plugin_instance(plugin_id) - elif was_clone: - plugin_manager.delete_plugin_config(plugin_id) - plugin_manager.delete_plugin_data(plugin_id) - plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower() - try: - clone_files_removed = await run_agent_blocking( - "plugin", - _remove_plugin_directory, - plugin_base_dir, + config_oper = SystemConfigOper() + install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or [] + if plugin_id in install_plugins: + install_plugins = [ + plugin for plugin in install_plugins if plugin != plugin_id + ] + await config_oper.async_set( + SystemConfigKey.UserInstalledPlugins, + install_plugins, ) - if clone_files_removed: - plugin_manager.plugins.pop(plugin_id, None) - except Exception: - clone_files_removed = False - remove_plugin_from_folders(plugin_id) - plugin_manager.remove_plugin(plugin_id) + remove_plugin_api(plugin_id) + remove_plugin_job(plugin_id) - return { - "was_clone": was_clone, - "clone_files_removed": clone_files_removed, - } + plugin_class = plugin_manager.plugins.get(plugin_id) + was_clone = bool(getattr(plugin_class, "is_clone", False)) + clone_files_removed = False + + if virtual_instance: + plugin_manager.delete_plugin_config(plugin_id, force=True) + plugin_manager.delete_plugin_data(plugin_id, force=True) + plugin_manager.delete_plugin_instance(plugin_id) + elif was_clone: + plugin_manager.delete_plugin_config(plugin_id) + plugin_manager.delete_plugin_data(plugin_id) + plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower() + try: + clone_files_removed = await run_agent_blocking( + "plugin", + _remove_plugin_directory, + plugin_base_dir, + ) + if clone_files_removed: + plugin_manager.plugins.pop(plugin_id, None) + except Exception: + clone_files_removed = False + + remove_plugin_from_folders(plugin_id) + plugin_manager.remove_plugin(plugin_id) + + return { + "was_clone": was_clone, + "clone_files_removed": clone_files_removed, + } diff --git a/app/agent/tools/impl/update_plugin_config.py b/app/agent/tools/impl/update_plugin_config.py index 9585150b4..bea02a67b 100644 --- a/app/agent/tools/impl/update_plugin_config.py +++ b/app/agent/tools/impl/update_plugin_config.py @@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool): ) plugin_manager = get_plugin_manager() - current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {}) + with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"): + current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {}) - # merge 模式以当前保存值为基准，replace 模式则从空配置开始重建。 - next_config = {} if replace else dict(current_config) - if updates: - next_config.update(updates) - for key in remove_keys: - next_config.pop(key, None) + # merge 模式以当前保存值为基准，replace 模式则从空配置开始重建。 + next_config = {} if replace else dict(current_config) + if updates: + next_config.update(updates) + for key in remove_keys: + next_config.pop(key, None) - changed_keys = sorted( - key - for key in set(current_config.keys()) | set(next_config.keys()) - if current_config.get(key) != next_config.get(key) - or (key in current_config) != (key in next_config) - ) - - if not await plugin_manager.async_save_plugin_config(plugin_id, next_config): - return json.dumps( - { - "success": False, - "message": f"保存插件 {plugin_id} 配置失败", - }, - ensure_ascii=False, + changed_keys = sorted( + key + for key in set(current_config.keys()) | set(next_config.keys()) + if current_config.get(key) != next_config.get(key) + or (key in current_config) != (key in next_config) ) - return json.dumps( - { - "success": True, - **plugin_info, - "message": "插件配置已保存，请调用 reload_plugin 使最新配置生效", - "replace": replace, - "changed_keys": changed_keys, - "removed_keys": remove_keys, - "config_requires_reload": True, - "previous_config": current_config, - "saved_config": next_config, - }, - ensure_ascii=False, - indent=2, - default=str, - ) + if not await plugin_manager.async_save_plugin_config( + plugin_id, + next_config, + ): + return json.dumps( + { + "success": False, + "message": f"保存插件 {plugin_id} 配置失败", + }, + ensure_ascii=False, + ) + + return json.dumps( + { + "success": True, + **plugin_info, + "message": "插件配置已保存，请调用 reload_plugin 使最新配置生效", + "replace": replace, + "changed_keys": changed_keys, + "removed_keys": remove_keys, + "config_requires_reload": True, + "previous_config": current_config, + "saved_config": next_config, + }, + ensure_ascii=False, + indent=2, + default=str, + ) async def run( self, diff --git a/app/api/dependencies/plugin.py b/app/api/dependencies/plugin.py index abc6c4566..d3c12a492 100644 --- a/app/api/dependencies/plugin.py +++ b/app/api/dependencies/plugin.py @@ -40,4 +40,5 @@ def get_plugin_config_command() -> PluginConfigCommand: reload_runtime=manager.reload_plugin, publish_reset=publish_reset, refresh_registrations=refresh_registrations, + mutation=manager.mutation, ) diff --git a/app/api/endpoints/plugin.py b/app/api/endpoints/plugin.py index 9cf7e5bcc..b18a65c22 100644 --- a/app/api/endpoints/plugin.py +++ b/app/api/endpoints/plugin.py @@ -57,7 +57,10 @@ from app.api.dependencies.plugin import ( from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.market import PluginHelper from app.adapters.system.plugin.package import PluginPackageManager -from app.schemas.exception import PersistenceUnavailableError +from app.schemas.exception import ( + PersistenceUnavailableError, + PluginMutationRejectedError, +) from app.runtime.log import logger from app.schemas.types import SystemConfigKey from app.api.context import get_background_task_registry, resolve_background_task_registry @@ -520,10 +523,15 @@ def reload_plugin( """ 重新加载插件 """ - # 重新加载插件 - runtime_status = PluginManager().reload_plugin(plugin_id) - # 注册插件服务 - register_plugin(plugin_id) + plugin_manager = PluginManager() + try: + with plugin_manager.mutation(f"重载插件 {plugin_id}"): + # 重新加载插件 + runtime_status = plugin_manager.reload_plugin(plugin_id) + # 注册插件服务 + register_plugin(plugin_id) + except PluginMutationRejectedError as error: + return _SchemaResponse(success=False, message=str(error)) if runtime_status is _SchemaPluginRuntimeStatus.ACTIVE: return _SchemaResponse(success=True) return _SchemaResponse( @@ -600,14 +608,15 @@ async def install( ), plugin_reloader=reload_runtime, registration_refresher=refresh_registrations, + mutation=plugin_manager.mutation, + package_write_guard=plugin_manager.suppress_plugin_monitor, + ) + result = await command.execute( + plugin_id=plugin_id, + repo_url=repo_url, + release_version=release_version, + force=bool(force), ) - with plugin_manager.suppress_plugin_monitor(plugin_id): - result = await command.execute( - plugin_id=plugin_id, - repo_url=repo_url, - release_version=release_version, - force=bool(force), - ) if not result.success: return _SchemaResponse(success=False, message=result.message) return _SchemaResponse(success=True) @@ -888,11 +897,12 @@ async def save_plugin_folders( 保存插件文件夹分组配置 """ try: - await get_configured_system_config().async_set( - SystemConfigKey.PluginFolders, - folders, - ) - return _SchemaResponse(success=True) + with PluginManager().mutation("保存插件文件夹配置"): + await get_configured_system_config().async_set( + SystemConfigKey.PluginFolders, + folders, + ) + return _SchemaResponse(success=True) except PersistenceUnavailableError: raise except Exception as e: @@ -909,18 +919,26 @@ async def create_plugin_folder( """ 创建新的插件文件夹 """ - folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} - if folder_name not in folders: - folders[folder_name] = [] - await get_configured_system_config().async_set( - SystemConfigKey.PluginFolders, - folders, - ) - return _SchemaResponse( - success=True, message=f"文件夹 '{folder_name}' 创建成功" - ) - else: - return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 已存在") + try: + with PluginManager().mutation(f"创建插件文件夹 {folder_name}"): + folders = ( + get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} + ) + if folder_name not in folders: + folders[folder_name] = [] + await get_configured_system_config().async_set( + SystemConfigKey.PluginFolders, + folders, + ) + return _SchemaResponse( + success=True, message=f"文件夹 '{folder_name}' 创建成功" + ) + return _SchemaResponse( + success=False, + message=f"文件夹 '{folder_name}' 已存在", + ) + except PluginMutationRejectedError as error: + return _SchemaResponse(success=False, message=str(error)) @router.delete( @@ -932,15 +950,26 @@ async def delete_plugin_folder( """ 删除插件文件夹 """ - folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} - if folder_name in folders: - del folders[folder_name] - await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders) - return _SchemaResponse( - success=True, message=f"文件夹 '{folder_name}' 删除成功" - ) - else: - return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 不存在") + try: + with PluginManager().mutation(f"删除插件文件夹 {folder_name}"): + folders = ( + get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} + ) + if folder_name in folders: + del folders[folder_name] + await get_configured_system_config().async_set( + SystemConfigKey.PluginFolders, + folders, + ) + return _SchemaResponse( + success=True, message=f"文件夹 '{folder_name}' 删除成功" + ) + return _SchemaResponse( + success=False, + message=f"文件夹 '{folder_name}' 不存在", + ) + except PluginMutationRejectedError as error: + return _SchemaResponse(success=False, message=str(error)) @router.put( @@ -956,12 +985,22 @@ async def update_folder_plugins( """ 更新指定文件夹中的插件列表 """ - folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} - folders[folder_name] = plugin_ids - await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders) - return _SchemaResponse( - success=True, message=f"文件夹 '{folder_name}' 中的插件已更新" - ) + try: + with PluginManager().mutation(f"更新插件文件夹 {folder_name}"): + folders = ( + get_configured_system_config().get(SystemConfigKey.PluginFolders) or {} + ) + folders[folder_name] = plugin_ids + await get_configured_system_config().async_set( + SystemConfigKey.PluginFolders, + folders, + ) + return _SchemaResponse( + success=True, + message=f"文件夹 '{folder_name}' 中的插件已更新", + ) + except PluginMutationRejectedError as error: + return _SchemaResponse(success=False, message=str(error)) @router.post( @@ -975,23 +1014,24 @@ def clone_plugin( """ 创建插件分身 """ + plugin_manager = PluginManager() try: - success, message = PluginManager().clone_plugin( - plugin_id=plugin_id, - suffix=clone_data.suffix, - name=clone_data.name, - description=clone_data.description, - version=clone_data.version, - icon=clone_data.icon, - ) + with plugin_manager.mutation(f"创建插件 {plugin_id} 分身"): + success, message = plugin_manager.clone_plugin( + plugin_id=plugin_id, + suffix=clone_data.suffix, + name=clone_data.name, + description=clone_data.description, + version=clone_data.version, + icon=clone_data.icon, + ) - if success: - # 分身服务已完成运行态加载，此处只补齐宿主注册。 - register_plugin(message) - # 将分身插件添加到原插件所在的文件夹中 - _add_clone_to_plugin_folder(plugin_id, message) - return _SchemaResponse(success=True, message="插件分身创建成功") - else: + if success: + # 分身服务已完成运行态加载，此处只补齐宿主注册。 + register_plugin(message) + # 将分身插件添加到原插件所在的文件夹中 + _add_clone_to_plugin_folder(plugin_id, message) + return _SchemaResponse(success=True, message="插件分身创建成功") return _SchemaResponse(success=False, message=message) except Exception as e: logger.error(f"创建插件分身失败：{str(e)}") @@ -1034,54 +1074,60 @@ def uninstall_plugin( 卸载插件 """ plugin_manager = PluginManager() - virtual_instance = plugin_manager.get_plugin_instance(plugin_id) - source_instances = plugin_manager.get_plugin_source_instances(plugin_id) - if not virtual_instance and source_instances: - instance_ids = "、".join(item.instance_id for item in source_instances) - return _SchemaResponse( - success=False, - message=f"请先卸载该插件的分身：{instance_ids}", - ) - config_oper = get_configured_system_config() - # 删除已安装信息 - install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or [] - for plugin in install_plugins: - if plugin == plugin_id: - install_plugins.remove(plugin) - break - config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins) - # 移除插件API - remove_plugin_api(plugin_id) - # 移除插件服务 - remove_plugin_job(plugin_id) - # 判断是否为分身 - plugin_class = plugin_manager.plugins.get(plugin_id) - if virtual_instance: - plugin_manager.delete_plugin_config(plugin_id, force=True) - plugin_manager.delete_plugin_data(plugin_id, force=True) - plugin_manager.delete_plugin_instance(plugin_id) - elif getattr(plugin_class, "is_clone", False): - # 如果是分身插件，则删除分身数据和配置 - plugin_manager.delete_plugin_config(plugin_id) - plugin_manager.delete_plugin_data(plugin_id) - # 删除分身文件 - plugin_base_dir = ( - get_api_runtime_config_snapshot().root_path - / "app" - / "plugins" - / plugin_id.lower() - ) - if plugin_base_dir.exists(): - try: - shutil.rmtree(plugin_base_dir) - plugin_manager.plugins.pop(plugin_id, None) - except Exception as e: - logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}") - # 从插件文件夹中移除该插件 - remove_plugin_from_folders(plugin_id) - # 移除插件 - plugin_manager.remove_plugin(plugin_id) - return _SchemaResponse(success=True) + try: + with plugin_manager.mutation(f"卸载插件 {plugin_id}"): + virtual_instance = plugin_manager.get_plugin_instance(plugin_id) + source_instances = plugin_manager.get_plugin_source_instances(plugin_id) + if not virtual_instance and source_instances: + instance_ids = "、".join(item.instance_id for item in source_instances) + return _SchemaResponse( + success=False, + message=f"请先卸载该插件的分身：{instance_ids}", + ) + config_oper = get_configured_system_config() + # 删除已安装信息 + install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or [] + for plugin in install_plugins: + if plugin == plugin_id: + install_plugins.remove(plugin) + break + config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins) + # 移除插件API + remove_plugin_api(plugin_id) + # 移除插件服务 + remove_plugin_job(plugin_id) + # 判断是否为分身 + plugin_class = plugin_manager.plugins.get(plugin_id) + if virtual_instance: + plugin_manager.delete_plugin_config(plugin_id, force=True) + plugin_manager.delete_plugin_data(plugin_id, force=True) + plugin_manager.delete_plugin_instance(plugin_id) + elif getattr(plugin_class, "is_clone", False): + # 如果是分身插件，则删除分身数据和配置 + plugin_manager.delete_plugin_config(plugin_id) + plugin_manager.delete_plugin_data(plugin_id) + # 删除分身文件 + plugin_base_dir = ( + get_api_runtime_config_snapshot().root_path + / "app" + / "plugins" + / plugin_id.lower() + ) + if plugin_base_dir.exists(): + try: + shutil.rmtree(plugin_base_dir) + plugin_manager.plugins.pop(plugin_id, None) + except Exception as e: + logger.error( + f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}" + ) + # 从插件文件夹中移除该插件 + remove_plugin_from_folders(plugin_id) + # 移除插件 + plugin_manager.remove_plugin(plugin_id) + return _SchemaResponse(success=True) + except PluginMutationRejectedError as error: + return _SchemaResponse(success=False, message=str(error)) def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str): diff --git a/app/application/plugin/config.py b/app/application/plugin/config.py index 8c4e94d81..2e000fe0e 100644 --- a/app/application/plugin/config.py +++ b/app/application/plugin/config.py @@ -4,7 +4,9 @@ from __future__ import annotations from collections.abc import Callable from dataclasses import dataclass -from typing import Any +from typing import Any, ContextManager + +from app.schemas.exception import PluginMutationRejectedError @dataclass(frozen=True, slots=True) @@ -29,6 +31,7 @@ class PluginConfigCommand: reload_runtime: Callable[[str], Any], publish_reset: Callable[[str], Any], refresh_registrations: Callable[[str], Any], + mutation: Callable[[str], ContextManager[None]], ) -> None: """保存插件管理 Facade 和运行时注册刷新端口。""" self._save_config = save_config @@ -39,21 +42,30 @@ class PluginConfigCommand: self._reload_runtime = reload_runtime self._publish_reset = publish_reset self._refresh_registrations = refresh_registrations + self._mutation = mutation def update(self, plugin_id: str, config: dict) -> PluginConfigResult: """保存配置并按既有顺序重新初始化实例及运行时注册。""" - if not self._save_config(plugin_id, config, False): - return PluginConfigResult(False, "插件配置保存失败") - self._initialize(plugin_id, config) - self._refresh_registrations(plugin_id) - return PluginConfigResult(True) + try: + with self._mutation(f"更新插件 {plugin_id} 配置"): + if not self._save_config(plugin_id, config, False): + return PluginConfigResult(False, "插件配置保存失败") + self._initialize(plugin_id, config) + self._refresh_registrations(plugin_id) + return PluginConfigResult(True) + except PluginMutationRejectedError as error: + return PluginConfigResult(False, str(error)) def reset(self, plugin_id: str) -> PluginConfigResult: """通知插件补偿后停止实例、删除配置数据并重建运行态。""" - self._publish_reset(plugin_id) - self._stop(plugin_id) - self._delete_config(plugin_id, True) - self._delete_data(plugin_id, True) - self._reload_runtime(plugin_id) - self._refresh_registrations(plugin_id) - return PluginConfigResult(True) + try: + with self._mutation(f"重置插件 {plugin_id} 配置和数据"): + self._publish_reset(plugin_id) + self._stop(plugin_id) + self._delete_config(plugin_id, True) + self._delete_data(plugin_id, True) + self._reload_runtime(plugin_id) + self._refresh_registrations(plugin_id) + return PluginConfigResult(True) + except PluginMutationRejectedError as error: + return PluginConfigResult(False, str(error)) diff --git a/app/application/plugin/install.py b/app/application/plugin/install.py index 6f46e09cc..56f7fd18e 100644 --- a/app/application/plugin/install.py +++ b/app/application/plugin/install.py @@ -5,11 +5,12 @@ from __future__ import annotations import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass, field -from typing import Any, Optional +from typing import Any, ContextManager, Optional from app.schemas.exception import PersistenceUnavailableError from app.application.plugin.lifecycle import plugin_lifecycle from app.runtime.log import logger +from app.schemas.exception import PluginMutationRejectedError InstalledPluginsReader = Callable[[], list[str]] @@ -25,6 +26,8 @@ PackageCheckpointAction = Callable[[Any], Awaitable[object]] InstallReporter = Callable[[str, Optional[str]], Awaitable[object]] PluginReloader = Callable[[str], Awaitable[object]] PluginRegistrationRefresher = Callable[[str], Awaitable[object]] +PluginMutationAdmission = Callable[[str], ContextManager[None]] +PluginPackageWriteGuard = Callable[[str], ContextManager[None]] @dataclass(frozen=True, slots=True) @@ -94,6 +97,8 @@ class PluginInstallCommand: install_reporter: InstallReporter, plugin_reloader: PluginReloader, registration_refresher: PluginRegistrationRefresher, + mutation: PluginMutationAdmission, + package_write_guard: PluginPackageWriteGuard, ) -> None: """保存安装用例所需端口，不绑定数据库、网络或运行时实现。""" self._installed_plugins_reader = installed_plugins_reader @@ -107,6 +112,8 @@ class PluginInstallCommand: self._install_reporter = install_reporter self._plugin_reloader = plugin_reloader self._registration_refresher = registration_refresher + self._mutation = mutation + self._package_write_guard = package_write_guard async def execute( self, @@ -120,20 +127,29 @@ class PluginInstallCommand: state = _InstallState() async with plugin_lifecycle.hold(plugin_id): try: - return await self._execute_locked( - plugin_id=plugin_id, - repo_url=repo_url, - release_version=release_version, - force=force, - state=state, + with self._mutation(f"安装插件 {plugin_id}"): + with self._package_write_guard(plugin_id): + try: + return await self._execute_locked( + plugin_id=plugin_id, + repo_url=repo_url, + release_version=release_version, + force=force, + state=state, + ) + except asyncio.CancelledError: + await self._rollback_cancelled( + plugin_id=plugin_id, + original_plugins=state.original_plugins, + state=state, + ) + raise + except PluginMutationRejectedError as error: + return PluginInstallResult( + success=False, + message=str(error), + failure_stage="admission", ) - except asyncio.CancelledError: - await self._rollback_cancelled( - plugin_id=plugin_id, - original_plugins=state.original_plugins, - state=state, - ) - raise async def _execute_locked( self, diff --git a/app/application/plugin/runtime.py b/app/application/plugin/runtime.py index 347573614..eecf9f854 100644 --- a/app/application/plugin/runtime.py +++ b/app/application/plugin/runtime.py @@ -3,12 +3,15 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any, Protocol - +from typing import Any, ContextManager, Protocol class PluginRuntime(Protocol): """声明入口层消费的插件宿主能力。""" + def mutation(self, operation: str) -> ContextManager[None]: + """为完整插件可变事务取得停机准入 lease。""" + ... + def __getattr__(self, name: str) -> Any: """允许兼容门面按既有 V3 方法名访问插件宿主能力。""" diff --git a/app/application/transfer.py b/app/application/transfer.py index 834837f9d..c57308e49 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -191,9 +191,16 @@ class TransferFailureNotificationAggregator: NOTIFICATION_DEBOUNCE_SECONDS = 30 def __init__(self) -> None: - """初始化分组缓冲和定时器。""" + """初始化分组缓冲、回调、定时器与关闭状态。""" self._buffers: dict[str, list[TransferFailureNotification]] = {} + self._callbacks: dict[ + str, + Callable[[list[TransferFailureNotification]], None], + ] = {} self._timers: dict[str, asyncio.TimerHandle] = {} + self._generations: dict[str, int] = {} + self._lock = threading.Lock() + self._closed = False def schedule( self, @@ -204,48 +211,128 @@ class TransferFailureNotificationAggregator: loop: asyncio.AbstractEventLoop, ) -> None: """从整理线程安全地把失败快照加入事件循环中的聚合缓冲。""" - loop.call_soon_threadsafe( - self._schedule_on_loop, - group_key, - notification, - callback, - loop, - ) + # 先在调用线程登记快照，关闭流程才能覆盖已接收但尚未进入事件循环的通知。 + with self._lock: + if self._closed: + raise RuntimeError("整理失败通知聚合器正在关闭，不能再接收通知") + self._buffers.setdefault(group_key, []).append(notification) + self._callbacks[group_key] = callback + generation = self._generations.get(group_key, 0) + 1 + self._generations[group_key] = generation + try: + loop.call_soon_threadsafe( + self._schedule_on_loop, + group_key, + generation, + callback, + loop, + ) + except Exception as err: + logger.error( + f"创建整理失败通知聚合定时器失败，将立即发送 " + f"(group={group_key}): {err}" + ) + self.flush(group_key, generation, callback) def _schedule_on_loop( self, group_key: str, - notification: TransferFailureNotification, + generation: int, callback: Callable[[list[TransferFailureNotification]], None], loop: asyncio.AbstractEventLoop, ) -> None: - """在所属事件循环中更新缓冲并重置静默窗口。""" - self._buffers.setdefault(group_key, []).append(notification) - timer = self._timers.pop(group_key, None) - if timer: - timer.cancel() - self._timers[group_key] = loop.call_later( - self.NOTIFICATION_DEBOUNCE_SECONDS, - self.flush, - group_key, - callback, - ) + """在所属事件循环中为已登记缓冲重置静默窗口。""" + schedule_error: Optional[Exception] = None + with self._lock: + if ( + self._closed + or group_key not in self._buffers + or self._generations.get(group_key) != generation + ): + return + timer = self._timers.pop(group_key, None) + if timer: + timer.cancel() + try: + self._timers[group_key] = loop.call_later( + self.NOTIFICATION_DEBOUNCE_SECONDS, + self.flush, + group_key, + generation, + callback, + ) + except Exception as err: + schedule_error = err + if schedule_error is not None: + logger.error( + f"创建整理失败通知聚合定时器失败，将立即发送 " + f"(group={group_key}): {schedule_error}" + ) + self.flush(group_key, generation, callback) def flush( self, group_key: str, + generation: int, callback: Callable[[list[TransferFailureNotification]], None], ) -> None: """发送一个分组内的聚合结果并释放缓冲。""" - notifications = self._buffers.pop(group_key, []) - self._timers.pop(group_key, None) + with self._lock: + # 新通知已登记但 timer 重置回调尚未执行时，旧代不得提前发送新批次。 + if self._generations.get(group_key) != generation: + return + notifications = self._buffers.pop(group_key, []) + self._callbacks.pop(group_key, None) + timer = self._timers.pop(group_key, None) + self._generations.pop(group_key, None) + if timer: + timer.cancel() if not notifications: return + self._deliver(group_key, notifications, callback) + + @staticmethod + def _deliver( + group_key: str, + notifications: list[TransferFailureNotification], + callback: Callable[[list[TransferFailureNotification]], None], + ) -> None: + """调用聚合通知回调，并统一观察发送异常。""" try: callback(notifications) except Exception as err: logger.error(f"发送整理失败聚合通知失败 (group={group_key}): {err}") + def close(self) -> None: + """停止接收新通知，取消定时器并同步发送全部已缓冲通知。""" + with self._lock: + if self._closed and not self._buffers and not self._timers: + return + self._closed = True + timers = list(self._timers.values()) + pending = [] + orphaned = [] + for group_key, notifications in self._buffers.items(): + callback = self._callbacks.get(group_key) + if callback is None: + orphaned.append((group_key, len(notifications))) + continue + pending.append((group_key, notifications, callback)) + self._timers.clear() + self._generations.clear() + self._buffers.clear() + self._callbacks.clear() + + for timer in timers: + timer.cancel() + for group_key, notification_count in orphaned: + logger.error( + f"整理失败通知聚合缓冲缺少发送回调，无法刷新 " + f"(group={group_key}, count={notification_count})" + ) + for group_key, notifications, callback in pending: + self._deliver(group_key, notifications, callback) + # 作业锁：JobManager 与 TransferChain 共享，保护整理作业视图。 job_lock = threading.Lock() @@ -267,7 +354,8 @@ class JobManager: # 记录仍由主程序整理线程直接执行的任务，避免把阻塞中的本地任务误判为失活 _active_executions: set[Tuple[str, str]] = set() - def __init__(self): + def __init__(self) -> None: + """初始化当前进程内的整理作业状态。""" self._job_view = {} self._season_episodes = {} self._meta_to_media_ids = {} @@ -1011,25 +1099,68 @@ class JobManager: class FailedRetryScheduler: """ - 负责失败整理记录的 debounce 聚合与 AI 重试调度。 + 负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。 + + 缓冲不提供持久化保证；关闭时会取消尚未触发的记录，由上层 durable + 工作流在后续阶段承接需要跨进程保证的重试意图。 """ RETRY_TRANSFER_DEBOUNCE_SECONDS = 300 - def __init__(self): + def __init__(self) -> None: + """初始化重试缓冲、定时器、活跃任务集合与关闭状态。""" super().__init__() self._retry_transfer_buffer: dict[str, list[int]] = {} self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {} + self._retry_transfer_generations: dict[str, int] = {} + self._retry_transfer_tasks: set[asyncio.Task[None]] = set() self._retry_transfer_lock = asyncio.Lock() + self._closed = False - async def close(self): + async def close(self) -> None: + """停止接收重试，取消定时器，并等待活跃 flush 任务完成取消。""" async with self._retry_transfer_lock: + self._closed = True timers = list(self._retry_transfer_timers.values()) + buffered_count = sum( + len(history_ids) + for history_ids in self._retry_transfer_buffer.values() + ) self._retry_transfer_timers.clear() + self._retry_transfer_generations.clear() self._retry_transfer_buffer.clear() + tasks = tuple(self._retry_transfer_tasks) for timer in timers: timer.cancel() + if buffered_count: + logger.warning( + f"智能体重试整理调度器关闭，取消 {buffered_count} 条未持久化缓冲记录" + ) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + def _start_retry_transfer_task(self, group_key: str, generation: int) -> None: + """把定时器到期后的 flush 建为具名且受本调度器管理的任务。""" + if self._closed: + return + task = asyncio.create_task( + self._flush_retry_transfer(group_key, generation), + name="transfer.failed_retry.flush", + ) + self._retry_transfer_tasks.add(task) + task.add_done_callback(self._observe_retry_transfer_task) + + def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None: + """移除已结束任务，并观察未被 flush 逻辑处理的异常。""" + self._retry_transfer_tasks.discard(task) + if task.cancelled(): + return + exception = task.exception() + if exception is not None: + logger.error(f"智能体重试整理后台任务异常: {exception}") @staticmethod def _build_retry_transfer_template_context( @@ -1054,7 +1185,7 @@ class FailedRetryScheduler: template_context=template_context, ) - async def schedule_retry(self, history_id: int, group_key: str = ""): + async def schedule_retry(self, history_id: int, group_key: str = "") -> None: """ 同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。 """ @@ -1062,6 +1193,8 @@ class FailedRetryScheduler: group_key = f"_default_{history_id}" async with self._retry_transfer_lock: + if self._closed: + raise RuntimeError("智能体重试整理调度器正在关闭，不能再接收任务") if group_key not in self._retry_transfer_buffer: self._retry_transfer_buffer[group_key] = [] if history_id not in self._retry_transfer_buffer[group_key]: @@ -1075,18 +1208,26 @@ class FailedRetryScheduler: self._retry_transfer_timers[group_key].cancel() loop = asyncio.get_running_loop() + generation = self._retry_transfer_generations.get(group_key, 0) + 1 + self._retry_transfer_generations[group_key] = generation self._retry_transfer_timers[group_key] = loop.call_later( self.RETRY_TRANSFER_DEBOUNCE_SECONDS, - lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)), + self._start_retry_transfer_task, + group_key, + generation, ) - async def _flush_retry_transfer(self, group_key: str): + async def _flush_retry_transfer(self, group_key: str, generation: int) -> None: """ 延迟定时器到期后，取出该分组的所有 history_id 并合并为一次 agent 调用。 """ async with self._retry_transfer_lock: + # callback 到期与真正取得锁之间可能有新记录续期；旧代不能提前取走新批次。 + if self._retry_transfer_generations.get(group_key) != generation: + return history_ids = self._retry_transfer_buffer.pop(group_key, []) self._retry_transfer_timers.pop(group_key, None) + self._retry_transfer_generations.pop(group_key, None) if not history_ids: return diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 2b238ea46..9aa6f837c 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -2,9 +2,12 @@ import asyncio import queue import re import threading +import time import traceback import uuid from collections import Counter +from concurrent.futures import CancelledError as FutureCancelledError +from concurrent.futures import Future from copy import deepcopy from pathlib import Path from typing import List, Optional, Tuple, Union, Dict, Callable, Any @@ -87,10 +90,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo 文件整理处理链 """ + # worker 在构造期启动；若中途失败，单例仍需先发布给 lifespan 清理入口。 + _retain_failed_singleton = True + CONFIG_WATCH = { "TRANSFER_THREADS", } + _WORKER_RESTART_TIMEOUT_SECONDS = 30.0 + _WORKER_CLOSE_TIMEOUT_SECONDS = 30.0 + _QUEUE_STOP_SENTINEL = object() + @staticmethod def _transfer_result_payload( task: TransferTask, @@ -108,7 +118,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo "transfer_history_id": history_id, } - def __init__(self): + def __init__(self) -> None: """初始化文件整理处理链。""" super().__init__() # 主要媒体文件后缀 @@ -141,7 +151,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._progress = ProgressHelper(ProgressKey.FileTransfer) # 队列相关状态 self._threads = [] + self._retiring_threads: List[threading.Thread] = [] self._queue_active = False + # 每一代 worker 使用独立停止信号，避免热更新启动新 worker 后旧线程重新取任务 + self._worker_stop_event = threading.Event() + # 生命周期操作串行化；状态锁只保护短临界区，不覆盖同步文件 I/O 等待 + self._worker_lifecycle_lock = threading.RLock() + self._worker_state_lock = threading.RLock() + self._closing = False + # pending 回放同样由整理链持有，关闭时可阻止继续处理下一条登记 + self._replay_thread: Optional[threading.Thread] = None + self._replay_stop_event = threading.Event() self._active_tasks = 0 self._processed_num = 0 self._fail_num = 0 @@ -149,33 +169,210 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # 启动整理任务 self.__init() - def __init(self): - """ - 启动文件整理线程 - """ - self._queue_active = True - for i in range(self.runtime_config.transfer_threads): - logger.info(f"启动文件整理线程 {i + 1} ...") - thread = threading.Thread( - target=self.__start_transfer, name=f"transfer-{i}", daemon=True + def __init(self) -> bool: + """启动一代文件整理线程，并返回是否成功取得 worker 所有权。""" + with self._worker_lifecycle_lock: + with self._worker_state_lock: + if self._closing: + logger.warning("文件整理链已进入关闭状态，拒绝重新启动 worker") + return False + self._retiring_threads = [ + thread for thread in self._retiring_threads if thread.is_alive() + ] + alive_threads = [thread for thread in self._threads if thread.is_alive()] + if alive_threads: + logger.error( + "上一代文件整理线程尚未收敛，拒绝并行启动新 worker：%s", + ", ".join(thread.name for thread in alive_threads), + ) + self._threads = alive_threads + return False + stop_event = threading.Event() + threads = [ + threading.Thread( + target=self.__start_transfer, + args=(stop_event,), + name=f"transfer-{index}", + daemon=True, + ) + for index in range(self.runtime_config.transfer_threads) + ] + self._worker_stop_event = stop_event + self._threads = threads + self._queue_active = True + for index, thread in enumerate(threads): + logger.info(f"启动文件整理线程 {index + 1} ...") + thread.start() + return True + + @staticmethod + def __join_threads( + threads: List[threading.Thread], deadline: float + ) -> List[threading.Thread]: + """在统一截止时间内等待线程，返回仍未收敛且继续由调用方持有的线程。""" + current_thread = threading.current_thread() + for thread in threads: + if thread is current_thread or not thread.is_alive(): + continue + thread.join(timeout=max(0.0, deadline - time.monotonic())) + return [thread for thread in threads if thread.is_alive()] + + def __acquire_worker_lifecycle_lock(self, deadline: float) -> bool: + """在统一截止时间内取得 worker 生命周期锁，并兼容同线程 RLock 重入。""" + return self._worker_lifecycle_lock.acquire( + timeout=max(0.0, deadline - time.monotonic()) + ) + + def __request_worker_stop(self) -> List[threading.Thread]: + """发布当前 worker 代的停止信号，并用哨兵唤醒空闲线程。""" + with self._worker_state_lock: + self._queue_active = False + self._worker_stop_event.set() + current_threads = list(self._threads) + threads = [*self._retiring_threads, *current_threads] + for _ in current_threads: + self._queue.put(self._QUEUE_STOP_SENTINEL) + return threads + + def __stop(self, timeout_seconds: float = _WORKER_RESTART_TIMEOUT_SECONDS) -> bool: + """在锁等待与线程 join 的共享预算内停止当前 worker 代。""" + deadline = time.monotonic() + max(0.0, timeout_seconds) + if not self.__acquire_worker_lifecycle_lock(deadline): + logger.error( + "未在 %.1f 秒内取得文件整理 worker 生命周期锁", + max(0.0, timeout_seconds), ) - self._threads.append(thread) - thread.start() + return False + try: + threads = self.__request_worker_stop() + alive_threads = self.__join_threads(threads, deadline) + with self._worker_state_lock: + self._threads = [] + self._retiring_threads = alive_threads + if alive_threads: + logger.error( + "文件整理线程未在 %.1f 秒内收敛，仍由 TransferChain 持有：%s", + max(0.0, timeout_seconds), + ", ".join(thread.name for thread in alive_threads), + ) + return False + logger.info("文件整理线程已停止") + return True + finally: + self._worker_lifecycle_lock.release() - def __stop(self): + def close_workers(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool: """ - 停止文件整理进程 - """ - self._queue_active = False - for thread in self._threads: - thread.join() - self._threads = [] - logger.info("文件整理线程已停止") + 关闭整理 worker 与 pending 回放，并拒绝后续队列写入。 - def on_config_changed(self): + 这是宿主生命周期使用的同步边界。停止信号只能阻止线程领取下一项工作，不能 + 取消已经进入同步文件或数据库 I/O 的调用；超过预算时保留活线程句柄并返回 + False，调用方据此避免过早释放仍被使用的数据库等下游资源。 + :param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数 + :return: 全部后台线程均已收敛时返回 True，否则返回 False + """ + deadline = time.monotonic() + max(0.0, timeout_seconds) + if not self.__acquire_worker_lifecycle_lock(deadline): + logger.error( + "未在 %.1f 秒内取得整理后台生命周期锁，关闭未开始", + max(0.0, timeout_seconds), + ) + return False + try: + with self._worker_state_lock: + self._closing = True + worker_threads = self.__request_worker_stop() + replay_thread = self._replay_thread + self._replay_stop_event.set() + + alive_workers = self.__join_threads(worker_threads, deadline) + alive_replays = self.__join_threads( + [replay_thread] if replay_thread else [], deadline + ) + with self._worker_state_lock: + self._threads = [] + self._retiring_threads = alive_workers + if replay_thread and not alive_replays and self._replay_thread is replay_thread: + self._replay_thread = None + + alive_threads = [*alive_workers, *alive_replays] + if alive_threads: + logger.error( + "整理后台线程未在 %.1f 秒内收敛，仍由 TransferChain 持有：%s", + max(0.0, timeout_seconds), + ", ".join(thread.name for thread in alive_threads), + ) + return False + logger.info("文件整理 worker 与待处理回放线程已关闭") + return True + finally: + self._worker_lifecycle_lock.release() + + async def close(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool: + """ + 收口整理线程、失败通知和 AI 重试，并返回依赖是否可以安全释放。 + + 同步文件 I/O 在线程内无法被 asyncio 取消，因此先在线程池中执行有界 + ``close_workers``。只有 worker 与 replay 全部退出后才关闭通知和重试；若 + 超时则保留这些依赖，供仍在运行的整理回调继续使用。 + :param timeout_seconds: worker 与 replay 共享的最大等待秒数 + :return: 所有整理后台 owner 均已收敛时返回 True + """ + workers_closed = await asyncio.to_thread( + self.close_workers, + timeout_seconds, + ) + if not workers_closed: + return False + self.failure_notification_aggregator.close() + await self.retry_scheduler.close() + return True + + @staticmethod + def _observe_failed_retry_schedule(future: Future[None]) -> None: + """观察跨线程 AI 重试调度的完成结果，避免关闭竞态变成静默异常。""" + try: + future.result() + except FutureCancelledError: + return + except Exception as err: + logger.error(f"触发AI智能体重试整理失败: {err}") + + def _schedule_failed_transfer_retry( + self, + history_id: int, + group_key: str, + ) -> None: + """把失败历史提交到主事件循环，并持续持有和观察调度结果。""" + retry_coroutine = self.retry_scheduler.schedule_retry( + history_id, + group_key=group_key, + ) + try: + future = asyncio.run_coroutine_threadsafe( + retry_coroutine, + global_vars.loop, + ) + except Exception as err: + retry_coroutine.close() + logger.error(f"触发AI智能体重试整理失败: {err}") + return + future.add_done_callback(self._observe_failed_retry_schedule) + logger.info(f"已触发AI智能体重试整理历史记录 #{history_id}") + + def on_config_changed(self) -> None: """配置变更时重启文件整理线程。""" - self.__stop() - self.__init() + with self._worker_lifecycle_lock: + if self._closing: + logger.info("文件整理链正在关闭，忽略 worker 配置热更新") + return + if not self.__stop( + timeout_seconds=self._WORKER_RESTART_TIMEOUT_SECONDS + ): + logger.warning( + "旧文件整理 worker 仍在收尾；其停止信号保持有效，新一代接管后续队列" + ) + self.__init() def __default_callback( self, task: TransferTask, transferinfo: TransferInfo, / @@ -344,19 +541,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo and self.runtime_config.ai_agent_enable and self.runtime_config.ai_agent_retry_transfer ): - try: - # 使用 download_hash 或源文件父目录作为分组键， - # 同一批次（如同一个种子）的失败记录会被合并为一次agent调用 - group_key = build_transfer_failure_group_key(task) - asyncio.run_coroutine_threadsafe( - self.retry_scheduler.schedule_retry( - history.id, group_key=group_key - ), - global_vars.loop, - ) - logger.info(f"已触发AI智能体重试整理历史记录 #{history.id}") - except Exception as e: - logger.error(f"触发AI智能体重试整理失败: {e}") + # 使用 download_hash 或源文件父目录作为分组键， + # 同一批次（如同一个种子）的失败记录会被合并为一次agent调用 + self._schedule_failed_transfer_retry( + history.id, + build_transfer_failure_group_key(task), + ) # 返回失败 ret_status = False @@ -653,7 +843,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo :param task: 任务信息 :return: True表示任务已添加到队列，False表示任务无效或已存在（重复） """ - return self._transfer_queue_service().put(task, self.__default_callback) + with self._worker_state_lock: + if self._closing: + logger.warning("文件整理链已关闭，拒绝新的队列任务") + return False + return self._transfer_queue_service().put(task, self.__default_callback) def _transfer_queue_service(self) -> TransferQueueService: """构建保持旧队列对象和私有兼容接缝的应用服务。""" @@ -667,26 +861,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo expire_tasks=self.__expire_stale_transfer_tasks, ) - def replay_pending(self): + def replay_pending(self) -> None: """ 回放上次进程退出时仍未整理完的文件。 在后台线程执行：回放要 stat 源文件，而启动期挂载可能尚未就绪甚至处于 挂死状态，同步执行会把整个启动流程堵住。 """ - threading.Thread( - target=self.__replay_pending, - name="MoviePilot-TransferReplay", - daemon=True - ).start() + with self._worker_state_lock: + if self._closing: + logger.info("文件整理链正在关闭，跳过待处理文件回放") + return + if self._replay_thread and self._replay_thread.is_alive(): + logger.info("待处理文件回放已在运行，跳过重复启动") + return + stop_event = threading.Event() + thread = threading.Thread( + target=self.__run_replay_pending, + args=(stop_event,), + name="MoviePilot-TransferReplay", + daemon=True, + ) + self._replay_stop_event = stop_event + self._replay_thread = thread + # 在状态锁内启动，避免 close_workers 看到尚未 start 的线程后错误 join。 + thread.start() - def __replay_pending(self): + def __run_replay_pending(self, stop_event: threading.Event) -> None: + """执行一次受控回放，并在自然结束后释放当前线程句柄。""" + try: + self.__replay_pending(stop_event) + finally: + with self._worker_state_lock: + if self._replay_thread is threading.current_thread(): + self._replay_thread = None + + def __replay_pending( + self, stop_event: Optional[threading.Event] = None + ) -> None: """ 把落盘登记的待整理文件重新送回整理入口。 只回放「存储 + 源路径」这一最小事实，重新走完整的识别与整理流程， 已经整理完成的由整理历史查重挡掉，因此不存在重复整理的问题。 + :param stop_event: 宿主关闭信号；只阻止处理下一条登记，不取消运行中的同步 I/O """ + stop_event = stop_event or threading.Event() + if stop_event.is_set(): + return try: pendings = self._pendingoper.list_all() except Exception as err: @@ -697,8 +919,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo logger.info(f"发现 {len(pendings)} 个上次未整理完的文件，正在重新送入整理链 ...") replayed = 0 for storage, src_path in pendings: + if stop_event.is_set(): + break try: fileitem, should_discard = self.__build_replay_fileitem(storage, src_path) + # stat 等同步 I/O 返回后重新检查，关闭期间不得注销尚未完成的登记。 + if stop_event.is_set(): + break if not fileitem: if should_discard: # 源文件确认已消失，注销登记避免每次启动重复回放 @@ -708,7 +935,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo replayed += 1 except Exception as err: logger.error(f"回放待整理文件失败：{storage}:{src_path} - {err}") - logger.info(f"✓ 待整理文件回放完成，{replayed} 个文件已重新送入整理链") + if stop_event.is_set(): + logger.info( + "待整理文件回放收到关闭请求，已送入 %s 个文件，其余登记保持待处理", + replayed, + ) + else: + logger.info(f"✓ 待整理文件回放完成，{replayed} 个文件已重新送入整理链") @staticmethod def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]: @@ -876,21 +1109,69 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.jobview.try_remove_job(task) self._finish_scrape_batch_task(task) - def __start_transfer(self): + def __settle_transfer_progress_if_idle(self) -> None: + """在没有 active 或未结算真实任务时结束进度并重置本批计数。""" + with task_lock: + # unfinished_tasks 同时覆盖队列内任务和已被其他 worker 取走、尚未来得及 + # 登记 active 的任务。持有 Queue 自身互斥锁完成判断和计数重置，使并发 + # enqueue 只能发生在旧批次归零之后；仍在 deque 中的停止哨兵不算真实任务。 + with self._queue.all_tasks_done: + queued_stop_sentinels = sum( + item is self._QUEUE_STOP_SENTINEL + for item in self._queue.queue + ) + has_unsettled_tasks = ( + self._queue.unfinished_tasks > queued_stop_sentinels + ) + if ( + self._active_tasks != 0 + or self._processed_num <= 0 + or has_unsettled_tasks + ): + return + processed_num = self._processed_num + fail_num = self._fail_num + self._total_num = 0 + self._processed_num = 0 + self._fail_num = 0 + __end_msg = ( + f"整理队列处理完成，共整理 {processed_num} 个文件，" + f"失败 {fail_num} 个" + ) + logger.info(__end_msg) + self._progress.update(value=100, text=__end_msg) + self._progress.end() + + def __start_transfer(self, stop_event: threading.Event) -> None: """ - 处理队列 + 处理当前 worker 代的队列，停止后不领取下一项任务。 + + :param stop_event: 当前 worker 代专属停止信号，热更新后不会被重新清除 """ - while not global_vars.is_system_stopped and self._queue_active: + while not global_vars.is_system_stopped and not stop_event.is_set(): try: item: TransferQueue = self._queue.get( block=True, timeout=self._transfer_interval ) + if item is self._QUEUE_STOP_SENTINEL: + self._queue.task_done() + self.__settle_transfer_progress_if_idle() + if stop_event.is_set() or global_vars.is_system_stopped: + break + continue + if stop_event.is_set() or global_vars.is_system_stopped: + # 关闭信号与 queue.get 竞态时，把尚未处理的任务放回队列；其 + # TransferPending 登记保持不变，供同进程重启 worker 或下次启动回放。 + self._queue.put(item) + self._queue.task_done() + break if not item: continue task = item.task if not task: self._queue.task_done() + self.__settle_transfer_progress_if_idle() continue # 文件信息 @@ -966,18 +1247,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo with task_lock: # 减少运行中的任务数 self._active_tasks -= 1 - # 检查是否所有任务都已完成且队列为空 - if self._active_tasks == 0 and self._queue.empty(): - # 结束进度 - __end_msg = f"整理队列处理完成，共整理 {self._processed_num} 个文件，失败 {self._fail_num} 个" - logger.info(__end_msg) - self._progress.update(value=100, text=__end_msg) - self._progress.end() - # 重置计数，_total_num 一并归零，否则会作为历史最大值一直 - # 累积，令后续批次的「当前共 N 个文件」与进度百分比失真 - self._total_num = 0 - self._processed_num = 0 - self._fail_num = 0 + self.__settle_transfer_progress_if_idle() except queue.Empty: # 即使队列空了，如果还有任务在运行，也不应该结束进度 @@ -1113,18 +1383,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo and self.runtime_config.ai_agent_enable and self.runtime_config.ai_agent_retry_transfer ): - try: - # 使用 download_hash 或源文件父目录作为分组键 - group_key = build_transfer_failure_group_key(task) - asyncio.run_coroutine_threadsafe( - self.retry_scheduler.schedule_retry( - his.id, group_key=group_key - ), - global_vars.loop, - ) - logger.info(f"已触发AI智能体重试整理历史记录 #{his.id}") - except Exception as e: - logger.error(f"触发AI智能体重试整理失败: {e}") + # 使用 download_hash 或源文件父目录作为分组键 + self._schedule_failed_transfer_retry( + his.id, + build_transfer_failure_group_key(task), + ) return False, "未识别到媒体信息" diff --git a/app/foundation/singleton.py b/app/foundation/singleton.py index a4d711fa2..2a66c4802 100644 --- a/app/foundation/singleton.py +++ b/app/foundation/singleton.py @@ -14,14 +14,24 @@ class Singleton(abc.ABCMeta, type): def get_existing_instance(cls, *args, **kwargs): """按相同参数返回已创建实例，不触发初始化""" key = (cls, args, frozenset(kwargs.items())) - return cls._instances.get(key) + with cls._lock: + return cls._instances.get(key) def __call__(cls, *args, **kwargs): """按类和构造参数创建或复用实例。""" key = (cls, args, frozenset(kwargs.items())) with cls._lock: if key not in cls._instances: - cls._instances[key] = super().__call__(*args, **kwargs) + if getattr(cls, "_retain_failed_singleton", False): + # 启动线程的 lifecycle owner 必须先发布身份再执行 __init__； + # 构造中途抛错时保留实例，启动失败清理才能找到已创建的 owner。 + instance = cls.__new__(cls, *args, **kwargs) + if not isinstance(instance, cls): + return instance + cls._instances[key] = instance + cls.__init__(instance, *args, **kwargs) + else: + cls._instances[key] = super().__call__(*args, **kwargs) return cls._instances[key] @@ -42,13 +52,23 @@ class SingletonClass(abc.ABCMeta, type): def get_existing_instance(cls): """返回已创建实例，不触发初始化""" - return cls._instances.get(cls) + with cls._lock: + return cls._instances.get(cls) def __call__(cls, *args, **kwargs): """按类创建或复用唯一实例。""" with cls._lock: if cls not in cls._instances: - cls._instances[cls] = super().__call__(*args, **kwargs) + if getattr(cls, "_retain_failed_singleton", False): + # 与参数化单例保持相同的 owner 发布顺序；锁会阻止其他线程 + # 在 __init__ 返回或抛错前读取半构造实例。 + instance = cls.__new__(cls, *args, **kwargs) + if not isinstance(instance, cls): + return instance + cls._instances[cls] = instance + cls.__init__(instance, *args, **kwargs) + else: + cls._instances[cls] = super().__call__(*args, **kwargs) return cls._instances[cls] diff --git a/app/monitor/monitor.py b/app/monitor/monitor.py index af96b5956..526ba32b2 100644 --- a/app/monitor/monitor.py +++ b/app/monitor/monitor.py @@ -1,4 +1,5 @@ import time +import threading import traceback from functools import partial from pathlib import Path @@ -27,6 +28,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): """ 目录监控门面，单例模式：装配本地/远程监控、维护生命周期与健康检查。 """ + # watcher/scheduler 在构造期启动；异常时保留实例供启动失败屏障收口。 + _retain_failed_singleton = True # 除目录配置外，同时监听仅在监控线程创建时读取的环境变量：这两项经 # /system/env 保存后运行时值虽已更新，但已运行的监控不会重新决策模式， # 必须触发 init() 全量重建才能生效（MONITOR_RESCAN_DELAYS 为实时解析，无需在列） @@ -54,11 +57,25 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): REBUILD_KEY_PREFIX = "rebuild:" PROBE_KEY = "probe" PENDING_KEY = "pending" + RELOAD_STOP_TIMEOUT = 30.0 + LIFECYCLE_CLOSE_TIMEOUT = 90.0 - def __init__(self): + def __init__(self) -> None: + """初始化目录监控依赖、owner 注册表与当前 lifespan 状态。""" super().__init__() + # 生命周期操作串行化；永久关闭请求用 Event 提前封口，不能被阻塞 I/O 挡住。 + self._lifecycle_lock = threading.RLock() + self._owner_lock = threading.Lock() + self._work_stop_event = threading.Event() + self._shutdown_event = threading.Event() + self._closed = False + self._compensation_threads: Dict[int, threading.Thread] = {} + self._scheduler_shutdown_thread: Optional[threading.Thread] = None + self._scheduler_shutdown_succeeded = False # 本地目录监控服务 self._watchers = [] + # 已请求停止但尚未退出的 watcher，只保留 owner 身份，不再参与健康检查 + self._retired_watchers = [] # 本地目录监控列表读写锁 self._watcher_lock = Lock() # 启动失败待重试的本地监控配置 @@ -87,9 +104,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher, alert_cb=self.__poller_alert) # 启动目录监控和文件整理 - self.init() + if not self.init(): + raise RuntimeError("目录监控 owner 初始化未收敛") - def on_config_changed(self): + def on_config_changed(self) -> None: + """配置变化时重建监控；lifespan 封口后拒绝重新启动。""" + if self._shutdown_event.is_set(): + logger.info("目录监控已进入生命周期封口，忽略配置热更新") + return self.init() def get_reload_name(self): @@ -118,6 +140,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): """ 强制全量扫描并处理所有文件（包括已存在的文件）。 """ + if not self.__accepting_work(): + return False return self._poller.force_full_scan(storage=storage, mon_path=mon_path) @staticmethod @@ -134,24 +158,61 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): """ return SnapshotStore.compare(old_snapshot, new_snapshot) - def init(self): + def init(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool: + """在旧 owner 收敛后启动一代监控，关闭中的生命周期拒绝重开。""" + if self._shutdown_event.is_set(): + logger.info("目录监控生命周期已关闭，跳过启动") + return False + deadline = time.monotonic() + max(0.0, timeout) + if not self._lifecycle_lock.acquire( + timeout=max(0.0, deadline - time.monotonic()) + ): + logger.error("目录监控未在 %.1f 秒内取得重载所有权", max(0.0, timeout)) + return False + try: + if self._shutdown_event.is_set(): + return False + if not self.__stop_owned(deadline=deadline, close=False): + logger.error("旧目录监控 owner 未收敛，取消本次配置热更新") + return False + if self._shutdown_event.is_set(): + return False + if not self._recovery.reopen(): + logger.error("目录监控恢复线程仍未退出，取消本次配置热更新") + return False + if self._shutdown_event.is_set(): + self._recovery.request_stop() + return False + self._work_stop_event.clear() + return self.__initialize_monitors() + finally: + self._lifecycle_lock.release() + + def __initialize_monitors(self) -> bool: """ - 启动监控 + 在已取得生命周期所有权后启动监控。 + + 永久关闭请求可以在本方法阻塞于 FUSE 时从其他线程提前设置；每个可能产生 + 新 owner 的边界都重新检查，确保解冻后只能退出，不能穿透旧生命周期。 """ - # 停止现有任务 - self.stop() + if not self.__accepting_work(): + return False # 读取目录配置 monitor_dirs = DirectoryHelper().get_download_dirs() + if not self.__accepting_work(): + return False if not monitor_dirs: logger.info("未找到任何目录监控配置") - return + return True messagehelper = MessageHelper() # 先筛出有效的监控配置，再按下载目录去重，避免非监控配置顶掉监控配置 valid_dirs = [] for mon_dir in monitor_dirs: + if not self.__accepting_work(): + return False if not mon_dir.library_path: logger.warn(f"跳过监控配置 {mon_dir.download_path}：未设置媒体库目录") continue @@ -172,6 +233,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置") # 启动定时服务进程 + if not self.__accepting_work(): + return False self._scheduler = BackgroundScheduler(timezone=get_runtime_setting("TZ")) mon_storages: Dict[str, List[Path]] = {} @@ -179,6 +242,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): local_started = 0 local_failed = 0 for mon_dir in monitor_dirs: + if not self.__accepting_work(): + self.__abort_interrupted_startup() + return False # 检查媒体库目录是不是下载目录的子目录 mon_path = Path(mon_dir.download_path) target_path = Path(mon_dir.library_path) @@ -197,6 +263,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): mon_storages.setdefault(mon_dir.storage, []).append(mon_path) for storage, paths in mon_storages.items(): + if not self.__accepting_work(): + self.__abort_interrupted_startup() + return False # 远程目录监控 - 使用智能间隔 # 先尝试加载已有快照获取文件数量 snapshot_data = self._store.load(storage) @@ -238,6 +307,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]") # 启动定时服务 + if not self.__accepting_work(): + self.__abort_interrupted_startup() + return False if self._scheduler.get_jobs(): self._scheduler.print_jobs() self._scheduler.start() @@ -253,6 +325,47 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.warn(summary) else: logger.info(summary) + return True + + def __accepting_work(self) -> bool: + """判断当前 monitor 代是否仍允许创建 owner 和派发整理工作。""" + return not self._work_stop_event.is_set() and not self._shutdown_event.is_set() + + @property + def lifecycle_closed(self) -> bool: + """返回当前 Monitor 是否已经被应用生命周期永久封口。""" + return self._closed or self._shutdown_event.is_set() + + def reopen(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool: + """确认旧 owner 已收敛后，为新的应用 lifespan 显式解除永久封口。""" + if not self.lifecycle_closed: + return True + deadline = time.monotonic() + max(0.0, timeout) + if not self._lifecycle_lock.acquire( + timeout=max(0.0, deadline - time.monotonic()) + ): + logger.error("旧目录监控生命周期仍在收尾，无法重新开启") + return False + try: + if not self.__stop_owned(deadline=deadline, close=True): + logger.error("旧目录监控 owner 仍未收敛，无法重新开启") + return False + if not self._recovery.reopen(): + logger.error("旧目录监控恢复线程仍存活，无法重新开启") + return False + self._closed = False + self._shutdown_event.clear() + return True + finally: + self._lifecycle_lock.release() + + def __abort_interrupted_startup(self) -> None: + """关闭请求穿透阻塞启动后，在当前生命周期锁内收拢已经创建的 owner。""" + logger.info("目录监控启动期间收到关闭请求，正在回收本次已创建 owner") + self.__stop_owned( + deadline=time.monotonic() + self.RELOAD_STOP_TIMEOUT, + close=self._shutdown_event.is_set(), + ) def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool: """ @@ -261,6 +374,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): :param monitor_mode: 配置的监控模式 :return: 是否启动成功 """ + if not self.__accepting_work(): + return False logger.info(f"正在启动本地目录监控: {mon_path}") logger.info("*** 重要提示：目录监控只处理新增和修改的文件，不会处理监控启动前已存在的文件 ***") @@ -298,6 +413,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): # 启动成功后再登记，避免失败的监控残留在列表中 watcher.start() with self._watcher_lock: + if not self.__accepting_work(): + watcher.stop() + if watcher.is_alive(): + self._retired_watchers.append(watcher) + return False self._watchers.append(watcher) self._pending_locals = [ pending for pending in self._pending_locals @@ -312,13 +432,15 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e) return False - def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception): + def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception) -> None: """ 处理本地目录监控启动失败，登记待重试并按需告警。 :param mon_path: 监控目录 :param monitor_mode: 配置的监控模式 :param err: 启动异常 """ + if not self.__accepting_work(): + return err_msg = str(err) logger.error(f"启动本地目录监控失败: {mon_path}") logger.error(f"错误详情: {err_msg}") @@ -346,7 +468,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n" f"将自动退避重试") - def watchdog(self): + def watchdog(self) -> None: """ 目录监控健康检查：检测监控线程状态并驱动恢复。 @@ -356,8 +478,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): 动作会把看门狗冻死在它自己要修复的挂载上，随后停滞检测、告警、重试驱动 全部静默失效——这正是全进程雪崩的起点。 """ + if not self.__accepting_work(): + return try: broken = self.__check_watchers() + if not self.__accepting_work(): + return self.__drive_recovery(broken) except Exception as e: logger.error(f"目录监控健康检查出现错误：{e}\n{traceback.format_exc()}") @@ -372,6 +498,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): """ with self._watcher_lock: watchers = list(self._watchers) + self._retired_watchers = [ + watcher for watcher in self._retired_watchers if watcher.is_alive() + ] isolated = set(self._isolated) # 探测已确认挂载恢复的目录，本轮直接送去重建 resumed = list(self._pending_rebuild.values()) @@ -412,7 +541,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): broken.append(watcher) return broken - def __drive_recovery(self, broken: List[LocalDirectoryWatcher]): + def __drive_recovery(self, broken: List[LocalDirectoryWatcher]) -> None: """ 把所有会触碰挂载的恢复动作派发到一次性工作线程，并等待有限时间。 @@ -421,6 +550,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): 可放弃的子进程探测来确认挂载何时恢复。 :param broken: 需要重建的监控列表 """ + if not self.__accepting_work(): + return actions: Dict[str, Callable[[], None]] = {} rebuilds: Dict[str, LocalDirectoryWatcher] = {} for watcher in broken: @@ -432,6 +563,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): actions[self.PENDING_KEY] = self.__drive_pending results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT) + if not self.__accepting_work(): + return for key, state in results.items(): if state is RecoveryState.COMPLETED: @@ -445,11 +578,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成" f"（{state.value}），将在后续健康检查周期重试: {key}") - def __enter_isolation(self, watcher: LocalDirectoryWatcher): + def __enter_isolation(self, watcher: LocalDirectoryWatcher) -> None: """ 将一个监控目录转入挂载级故障隔离：停止对它的一切新访问，等待探测恢复。 :param watcher: 重建未能返回的监控 """ + if not self.__accepting_work(): + return key = str(watcher.watch_path) with self._watcher_lock: if key in self._isolated: @@ -466,7 +601,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): f"已暂停对该目录的所有访问，正在周期探测挂载，恢复后将自动重建监控并补扫", stage="isolated") - def __probe_isolated(self): + def __probe_isolated(self) -> None: """ 对隔离中的监控目录做可放弃探测，挂载恢复应答后解除隔离并重建监控。 @@ -474,11 +609,17 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): 超时可被 kill，因此本线程不会像内联 stat 那样永久冻死。探测通过后的重建 仍有极小概率再次卡住，届时本线程会被下一轮的 BUSY 判定跳过，不再泄漏。 """ + if not self.__accepting_work(): + return with self._watcher_lock: keys = list(self._isolated) for key in keys: + if not self.__accepting_work(): + return mon_path = Path(key) if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT): + if not self.__accepting_work(): + return with self._watcher_lock: entry = self._isolated.get(key) if not entry: @@ -489,6 +630,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.warn(f"挂载探测未通过（累计 {failures} 次，已隔离 " f"{int(time.time() - since)} 秒），继续隔离: {mon_path}") continue + if not self.__accepting_work(): + return with self._watcher_lock: entry = self._isolated.pop(key, None) if not entry: @@ -502,19 +645,24 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): with self._watcher_lock: self._pending_rebuild[key] = entry["watcher"] - def __drive_pending(self): + def __drive_pending(self) -> None: """ 驱动两条待重试队列。两者都会访问挂载（启动重试走目录遍历与 exists， 整理重试走 stat），必须在恢复工作线程里执行而不是看门狗线程里。 """ + if not self.__accepting_work(): + return self.__retry_pending_locals() - self._dispatcher.retry_pending() + if self.__accepting_work(): + self._dispatcher.retry_pending() - def __rebuild_watcher(self, watcher: LocalDirectoryWatcher): + def __rebuild_watcher(self, watcher: LocalDirectoryWatcher) -> None: """ 重建一个本地目录监控线程。 :param watcher: 需要重建的监控 """ + if not self.__accepting_work(): + return # 卡死的线程阻塞在底层调用中无法强制回收，只能请求停止后由守护线程自然退出 watcher.stop() new_watcher = LocalDirectoryWatcher( @@ -527,8 +675,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): new_watcher.start() except Exception as e: logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}") + if not self.__accepting_work(): + return with self._watcher_lock: + # 旧 watcher 可能仍卡在 FUSE 调用里；只有真实退出后才能移除句柄。 self._watchers = [item for item in self._watchers if item is not watcher] + if watcher.is_alive(): + self._retired_watchers.append(watcher) if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals): self._pending_locals.append({ "mon_path": watcher.watch_path, @@ -537,9 +690,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): }) return with self._watcher_lock: + if not self.__accepting_work(): + new_watcher.stop() + if new_watcher.is_alive(): + self._retired_watchers.append(new_watcher) + return registered = any(item is watcher for item in self._watchers) if registered: self._watchers = [new_watcher if item is watcher else item for item in self._watchers] + if watcher.is_alive(): + # stop() 只能发信号，FUSE 上的旧线程可能永久不返回；继续持有它， + # 让生命周期关闭如实失败，而不是把 daemon 泄漏伪装成已收敛。 + self._retired_watchers.append(watcher) if not registered: # 卡死的重建线程可能在挂载恢复后才解冻并走到这里，而该目录此时已由 # 隔离恢复路径重建过。此处若直接放行，新建的监控既不在 _watchers 里 @@ -547,6 +709,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): # 派发事件的孤儿线程，必须就地停掉 logger.warn(f"目录监控已由其他路径重建，停止本次重建的冗余监控: {watcher.watch_path}") new_watcher.stop() + if new_watcher.is_alive(): + with self._watcher_lock: + self._retired_watchers.append(new_watcher) return # 新监控的重启计数从零开始，同步重置告警基准 self._restart_marks.pop(str(watcher.watch_path), None) @@ -557,7 +722,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): self.__start_compensation(mon_path=watcher.watch_path, since=watcher.last_activity_time) - def __start_compensation(self, mon_path: Path, since: float): + def __start_compensation(self, mon_path: Path, since: float) -> None: """ 在后台线程发起补偿扫描，避免遍历目录阻塞健康检查周期。 :param mon_path: 监控目录 @@ -567,14 +732,34 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): # 从未活动过说明没有可靠的停摆起点，全量补扫代价不可控，跳过 logger.debug(f"监控无活动记录，跳过补偿扫描: {mon_path}") return - Thread( - target=self.__compensate_scan, + thread = Thread( + target=self.__run_compensation, kwargs={"mon_path": mon_path, "since": since}, name=f"MoviePilot-MonitorCompensation-{mon_path.name}", daemon=True - ).start() + ) + with self._owner_lock: + # 与 close() 的 owner 快照共用同一把锁；封口先发生时绝不再启动， + # 登记先发生时 close() 必然能看到并等待这条线程。 + if not self.__accepting_work(): + return + self._compensation_threads[id(thread)] = thread + try: + thread.start() + except Exception: + self._compensation_threads.pop(id(thread), None) + raise - def __compensate_scan(self, mon_path: Path, since: float): + def __run_compensation(self, mon_path: Path, since: float) -> None: + """执行补偿扫描，并在真实终态后释放 owner 句柄。""" + current_thread = threading.current_thread() + try: + self.__compensate_scan(mon_path=mon_path, since=since) + finally: + with self._owner_lock: + self._compensation_threads.pop(id(current_thread), None) + + def __compensate_scan(self, mon_path: Path, since: float) -> None: """ 补扫监控停摆期间落地的文件。 @@ -586,7 +771,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): :param mon_path: 监控目录 :param since: 停摆起点（墙钟时间戳），仅用于统计与日志 """ + if not self.__accepting_work(): + return candidates = self.__collect_compensation_files(mon_path) + if not self.__accepting_work(): + return if candidates is None: return # mtime 不再作为过滤条件，但仍是「最可能是新文件」的排序依据 @@ -599,6 +788,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold) handled = 0 for file_path, file_modify_time, file_size in candidates: + if not self.__accepting_work(): + return if self._dispatcher.handle_file( storage="local", event_path=file_path, @@ -634,14 +825,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): return None return candidates - def __retry_pending_locals(self): + def __retry_pending_locals(self) -> None: """ 重试启动失败的本地目录监控，给网络存储/FUSE 挂载留出就绪时间。 """ + if not self.__accepting_work(): + return with self._watcher_lock: pending = list(self._pending_locals) isolated = set(self._isolated) for item in pending: + if not self.__accepting_work(): + return if str(item["mon_path"]) in isolated: # 隔离中的挂载不接受任何新访问：启动重试要走目录遍历与 exists， # 在「请求永不返回」的挂载上会再冻死一个线程 @@ -655,7 +850,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): item["attempts"] = item.get("attempts", 0) + 1 item["skip_cycles"] = min(item["attempts"], 10) - def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"): + def __send_alert(self, mon_path: Path, message: str, stage: str = "fault") -> None: """ 推送目录监控异常告警，同一目录在同一阶段仅推送一次。 :param mon_path: 监控目录 @@ -664,6 +859,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): （监控已暂停访问、等待挂载恢复），只按目录去重会把这条 关键消息吞掉，因此阶段变化时重新推送 """ + if not self.__accepting_work(): + return key = str(mon_path) with self._watcher_lock: if self._alerted_paths.get(key) == stage: @@ -672,7 +869,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): MessageHelper().put(message, title="目录监控") @staticmethod - def __poller_alert(storage: str, message: str): + def __poller_alert(storage: str, message: str) -> None: """ 远程轮询监控告警回调，复用消息渠道推送。 :param storage: 存储名称 @@ -681,12 +878,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.warn(f"[{storage}] {message}") MessageHelper().put(message, title="目录监控") - def __clear_alert(self, mon_path: Path, message: str): + def __clear_alert(self, mon_path: Path, message: str) -> None: """ 清除目录监控异常告警状态，并在此前告警过时推送恢复消息。 :param mon_path: 监控目录 :param message: 恢复内容 """ + if not self.__accepting_work(): + return key = str(mon_path) with self._watcher_lock: if key not in self._alerted_paths: @@ -695,12 +894,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): logger.info(message) MessageHelper().put(message, title="目录监控") - def polling_observer(self, storage: str, mon_paths: List[Path]): + def polling_observer(self, storage: str, mon_paths: List[Path]) -> None: """ 轮询监控：执行一轮快照并按结果动态调整监控间隔。 """ + if not self.__accepting_work(): + return file_count = self._poller.poll(storage=storage, mon_paths=mon_paths) - if file_count is None or not self._scheduler: + if not self.__accepting_work() or file_count is None or not self._scheduler: return # 动态调整监控间隔 new_interval = SnapshotStore.adjust_interval(file_count) @@ -716,7 +917,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): except Exception as e: logger.error(f"调整监控间隔失败: {storage} - {e}") - def event_handler(self, event, text: str, event_path: str, file_size: float = None): + def event_handler( + self, event: Any, text: str, event_path: str, + file_size: Optional[float] = None + ) -> None: """ 处理文件变化。 :param event: 事件 @@ -724,7 +928,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): :param event_path: 事件文件路径 :param file_size: 文件大小 """ - if event.is_directory: + if not self.__accepting_work() or event.is_directory: return if not self._dispatcher.is_transfer_candidate_path(Path(event_path)): return @@ -733,6 +937,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): file_modify_time = Path(event_path).stat().st_mtime except OSError as err: logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}") + if not self.__accepting_work(): + return # 整理文件 handle_kwargs = { "storage": "local", @@ -743,55 +949,191 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass): handle_kwargs["file_modify_time"] = file_modify_time self._dispatcher.handle_file(**handle_kwargs) - def event_unreadable(self, event_path: Path): + def event_unreadable(self, event_path: Path) -> None: """ 处理读取失败的监控事件，登记待重试。 :param event_path: 事件文件路径 """ + if not self.__accepting_work(): + return event_path = Path(event_path) if not self._dispatcher.is_transfer_candidate_path(event_path): return self._dispatcher.register_unreadable(storage="local", event_path=event_path) - def stop(self): + def stop(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool: + """在共享预算内临时停止全部监控 owner，供配置热重载使用。""" + return self.__stop_with_budget(timeout=timeout, close=False) + + def close(self, timeout: float = LIFECYCLE_CLOSE_TIMEOUT) -> bool: + """永久封口当前 lifespan，并在共享预算内等待全部监控 owner。""" + return self.__stop_with_budget(timeout=timeout, close=True) + + def __stop_with_budget(self, timeout: float, close: bool) -> bool: + """从等待生命周期锁开始计算一次停止操作的完整预算。""" + timeout = max(0.0, timeout) + deadline = time.monotonic() + timeout + # 先封住工作入口再等待锁。配置重载若正阻塞在 FUSE 上，解冻后也只能清理， + # 不能在外层生命周期已经超时返回后继续创建 watcher 或派发整理任务。 + self._work_stop_event.set() + if close: + self._shutdown_event.set() + if not self._lifecycle_lock.acquire( + timeout=max(0.0, deadline - time.monotonic()) + ): + logger.error("目录监控未在 %.1f 秒内取得停机所有权", timeout) + return False + try: + return self.__stop_owned(deadline=deadline, close=close) + finally: + self._lifecycle_lock.release() + + def __stop_owned(self, deadline: float, close: bool) -> bool: """ - 退出监控 + 在已取得生命周期锁时等待全部 owner，共享调用方给出的绝对截止时间。 + + 超时只报告未收敛并保留所有活句柄；只有 watcher、补偿扫描、恢复动作及 + scheduler shutdown 都到达终态后，才清空这一代的内存状态。 """ - # 先停定时服务，避免健康检查在停止过程中重建监控线程 - if self._scheduler: - self._scheduler.remove_all_jobs() - if self._scheduler.running: - try: - self._scheduler.shutdown() - logger.info("定时监控服务已停止") - except Exception as e: - logger.error(f"停止定时服务出现了错误：{e}") - self._scheduler = None - # 待重试条目按停止前的监控范围登记，重载后范围可能变化，一并清理 + self._work_stop_event.set() + if close: + self._closed = True + self._shutdown_event.set() + self._recovery.request_stop() + self.__request_scheduler_stop() + + with self._watcher_lock: + watchers = tuple(self._watchers) + tuple(self._retired_watchers) + if watchers: + logger.info("正在停止本地目录监控服务...") + for watcher in watchers: + try: + watcher.stop() + except Exception as err: + logger.error(f"请求停止目录监控服务出现了错误：{err}") + + recovery_converged = self._recovery.close(deadline=deadline) + current_thread = threading.current_thread() + # 恢复线程可能在封口后才从 FUSE 调用返回，并登记一个已经请求停止的新 + # watcher；RecoveryExecutor 收敛后重新取快照，确保它也使用剩余预算 join。 + with self._watcher_lock: + registered_watchers = tuple(self._watchers) + tuple(self._retired_watchers) + for watcher in registered_watchers: + try: + watcher.stop() + watcher.join(timeout=max(0.0, deadline - time.monotonic())) + except Exception as err: + logger.error(f"等待目录监控服务停止出现了错误：{err}") + + with self._owner_lock: + compensation_threads = tuple(self._compensation_threads.values()) + scheduler_thread = self._scheduler_shutdown_thread + for thread in compensation_threads: + if thread is not current_thread and thread.is_alive(): + thread.join(timeout=max(0.0, deadline - time.monotonic())) + if ( + scheduler_thread is not None + and scheduler_thread is not current_thread + and scheduler_thread.is_alive() + ): + scheduler_thread.join(timeout=max(0.0, deadline - time.monotonic())) + + with self._watcher_lock: + alive_watchers = tuple( + watcher + for watcher in (*self._watchers, *self._retired_watchers) + if watcher.is_alive() + ) + with self._owner_lock: + alive_compensations = tuple( + thread for thread in self._compensation_threads.values() + if thread.is_alive() + ) + scheduler_thread = self._scheduler_shutdown_thread + scheduler_converged = ( + self._scheduler is None + or ( + scheduler_thread is not None + and not scheduler_thread.is_alive() + and self._scheduler_shutdown_succeeded + ) + ) + alive_recoveries = self._recovery.running_threads() + if ( + alive_watchers + or alive_compensations + or alive_recoveries + or not recovery_converged + or not scheduler_converged + ): + logger.error( + "目录监控 owner 未在截止时间内收敛：watcher=%d，补偿=%d，恢复=%d，scheduler=%s", + len(alive_watchers), + len(alive_compensations), + len(alive_recoveries), + "未收敛" if not scheduler_converged else "已收敛", + ) + return False + + # 仅在所有 owner 真实终止后清理业务状态，确保失败重试仍能找到原句柄。 self._dispatcher.clear_pending() with self._watcher_lock: - watchers = self._watchers self._watchers = [] + self._retired_watchers = [] self._pending_locals = [] self._alerted_paths = {} self._restart_marks = {} self._stable_cycles = {} self._isolated = {} self._pending_rebuild = {} - # 已冻死的恢复线程无法回收，这里只是不再跟踪它们，避免重载后同名目录 - # 被残留记录误判为 BUSY 而永远拿不到重建机会 self._recovery.clear() + with self._owner_lock: + self._compensation_threads.clear() + self._scheduler = None + self._scheduler_shutdown_thread = None + self._scheduler_shutdown_succeeded = False if watchers: - logger.info("正在停止本地目录监控服务...") - for watcher in watchers: - try: - watcher.stop() - watcher.join(timeout=5) - if watcher.is_alive(): - logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}") - else: - logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}") - except Exception as e: - logger.error(f"停止目录监控服务出现了错误：{e}") logger.info("本地目录监控服务已停止") + return True + + def __request_scheduler_stop(self) -> None: + """创建并登记唯一 scheduler shutdown 线程，避免阻塞生命周期调用线程。""" + with self._owner_lock: + running = self._scheduler_shutdown_thread + if running is not None and running.is_alive(): + return + if running is not None and self._scheduler_shutdown_succeeded: + return + scheduler = self._scheduler + if scheduler is None: + return + self._scheduler_shutdown_succeeded = False + thread = threading.Thread( + target=self.__shutdown_scheduler, + args=(scheduler,), + name="MoviePilot-MonitorSchedulerShutdown", + daemon=True, + ) + self._scheduler_shutdown_thread = thread + try: + thread.start() + except Exception as err: + self._scheduler_shutdown_thread = None + logger.error(f"启动定时监控停止线程失败：{err}") + + def __shutdown_scheduler(self, scheduler: BackgroundScheduler) -> None: + """停止 scheduler 及其在途 job，并把真实终态写回 owner 注册表。""" + succeeded = False + try: + scheduler.remove_all_jobs() + if scheduler.running: + scheduler.shutdown(wait=True) + succeeded = True + logger.info("定时监控服务已停止") + except Exception as err: + logger.error(f"停止定时服务出现了错误：{err}") + finally: + with self._owner_lock: + if self._scheduler is scheduler: + self._scheduler_shutdown_succeeded = succeeded # 缓存与快照存储是共享后端的代理，生命周期由应用全局管理，这里不再关闭 diff --git a/app/monitor/recovery.py b/app/monitor/recovery.py index 423018f42..841f86f65 100644 --- a/app/monitor/recovery.py +++ b/app/monitor/recovery.py @@ -1,5 +1,5 @@ """ -监控恢复动作的可放弃执行单元。 +监控恢复动作的有限等待执行单元。 FUSE/网络挂载有两种故障形态，下游程序的免疫力完全不同： @@ -11,8 +11,8 @@ FUSE/网络挂载有两种故障形态，下游程序的免疫力完全不同： 本模块提供 block 型故障下唯一可行的两种自保手段： 1. RecoveryExecutor —— 把会触碰挂载的动作放进一次性守护线程执行，调用方只 - 等待有限时间。超时即放弃该线程（它会作为守护线程悬挂到进程退出），换取 - 调用方（健康检查这个全局自愈单点）永远活着。 + 等待有限时间。超时后当前健康检查不再等待，但线程句柄仍由生命周期 owner + 持有到真实终态，换取看门狗可继续检测且停机屏障不会伪装收敛。 2. probe_path —— 用子进程而非线程做挂载探测。子进程可以被 kill，因此探测 本身是可放弃的，隔离期间可以无限次周期重试而不累积不可回收的资源。 """ @@ -41,7 +41,7 @@ class RecoveryState(str, Enum): """ # 动作已在限定时间内执行完毕（内部抛异常也算完成，异常已记录） COMPLETED = "completed" - # 超时仍未返回，判定为 block 型挂载故障，线程已被放弃 + # 超时仍未返回，判定为 block 型挂载故障，本轮不再等待但继续持有线程 TIMEOUT = "timeout" # 同 key 的上一个动作仍未结束，本次未提交，避免持续泄漏冻死的线程 BUSY = "busy" @@ -55,10 +55,12 @@ class RecoveryExecutor: 健康检查周期都会在同一个死挂载上多泄漏一个线程。 """ - def __init__(self): + def __init__(self) -> None: + """初始化在途线程注册表和当前生命周期的提交状态。""" # key -> 该 key 最近一次提交的执行线程 self._running: Dict[str, threading.Thread] = {} self._lock = threading.Lock() + self._accepting = True def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]: """ @@ -85,26 +87,70 @@ class RecoveryExecutor: thread.join(timeout=max(0.0, deadline - time.monotonic())) if thread.is_alive(): results[key] = RecoveryState.TIMEOUT - logger.error(f"恢复动作超过 {timeout} 秒未返回，判定挂载无响应并放弃该线程: {key}") + logger.error(f"恢复动作超过 {timeout} 秒未返回，判定挂载无响应，本轮不再等待: {key}") else: results[key] = RecoveryState.COMPLETED return results - def discard(self, key: str): + def discard(self, key: str) -> None: """ - 丢弃一个 key 的在途记录。监控停止或配置重载时调用，避免残留条目 - 让重建后的同名目录被误判为 BUSY。 + 清理一个已到终态的 key；活线程继续保留，供生命周期停机屏障追踪。 :param key: 动作标识 """ with self._lock: - self._running.pop(key, None) + thread = self._running.get(key) + if thread is None or not thread.is_alive(): + self._running.pop(key, None) - def clear(self): + def clear(self) -> None: """ - 清空全部在途记录。已经冻死的线程无法回收，这里只是不再跟踪它们。 + 清理已经完成的记录，仍存活的线程继续由执行器持有。 + + 兼容旧调用名，但不再丢弃挂死线程：遗失句柄会让宿主错误释放它仍可能使用 + 的数据库和整理链资源。 """ with self._lock: + self._running = { + key: thread + for key, thread in self._running.items() + if thread.is_alive() + } + + def request_stop(self) -> None: + """封住新恢复动作提交，既有线程只能自然完成。""" + with self._lock: + self._accepting = False + + def close(self, deadline: float) -> bool: + """在绝对截止时间内等待全部恢复线程，超时继续保留活线程句柄。""" + self.request_stop() + with self._lock: + threads = tuple(self._running.values()) + current_thread = threading.current_thread() + for thread in threads: + if thread is current_thread or not thread.is_alive(): + continue + thread.join(timeout=max(0.0, deadline - time.monotonic())) + self.clear() + with self._lock: + return not any(thread.is_alive() for thread in self._running.values()) + + def reopen(self) -> bool: + """在旧恢复线程全部结束后，为新的 Monitor 生命周期恢复提交。""" + self.clear() + with self._lock: + if any(thread.is_alive() for thread in self._running.values()): + return False self._running.clear() + self._accepting = True + return True + + def running_threads(self) -> tuple[threading.Thread, ...]: + """返回仍存活且由执行器持有的恢复线程快照。""" + with self._lock: + return tuple( + thread for thread in self._running.values() if thread.is_alive() + ) def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]: """ @@ -114,6 +160,8 @@ class RecoveryExecutor: :return: 执行线程，未启动时为 None """ with self._lock: + if not self._accepting: + return None running = self._running.get(key) if running is not None and running.is_alive(): return None @@ -124,11 +172,18 @@ class RecoveryExecutor: daemon=True ) self._running[key] = thread - thread.start() - return thread + # 登记与 start 必须处于同一所有权临界区。否则 close() 可能把尚未 + # is_alive() 的句柄当成已结束并移除，随后该线程才真正启动。 + try: + thread.start() + except BaseException: + if self._running.get(key) is thread: + self._running.pop(key, None) + raise + return thread @staticmethod - def _execute(key: str, action: Callable[[], None]): + def _execute(key: str, action: Callable[[], None]) -> None: """ 执行一个恢复动作，异常只记录不外抛，避免一个目录的失败连累整批恢复。 :param key: 动作标识 diff --git a/app/runtime/event/dispatch.py b/app/runtime/event/dispatch.py index a7adcc4c3..2cd87e81f 100644 --- a/app/runtime/event/dispatch.py +++ b/app/runtime/event/dispatch.py @@ -30,6 +30,9 @@ class EventDispatcher: event_factory: Callable[..., Any], error_handler: Callable[..., None], async_handle_sink: Callable[[Any], bool] | None = None, + sync_handle_sink: ( + Callable[[Callable[..., Any], tuple[Any, ...]], bool] | None + ) = None, ) -> None: """注入注册表、绑定器、执行器和错误策略回调。""" self._registry = registry @@ -39,6 +42,7 @@ class EventDispatcher: self._event_factory = event_factory self._error_handler = error_handler self._async_handle_sink = async_handle_sink + self._sync_handle_sink = sync_handle_sink def dispatch_chain(self, event: Any) -> bool: """同步按优先级顺序执行链式事件快照。""" @@ -134,6 +138,12 @@ class EventDispatcher: event.event_type, ) else: + if self._sync_handle_sink: + self._sync_handle_sink( + self.safe_invoke_sync, + (handler, isolated), + ) + continue self._executor().submit( self.safe_invoke_sync, handler, diff --git a/app/runtime/events.py b/app/runtime/events.py index f4b387061..bee558470 100644 --- a/app/runtime/events.py +++ b/app/runtime/events.py @@ -4,6 +4,7 @@ import random import threading import traceback import uuid +from contextvars import ContextVar from dataclasses import dataclass from queue import Empty, PriorityQueue from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type @@ -32,6 +33,10 @@ MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数 INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间（秒） MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间（秒） _EVENT_STOP_SENTINEL = object() +_CURRENT_EVENT_HANDLER_OWNER: ContextVar[object | None] = ContextVar( + "current_event_handler_owner", + default=None, +) @dataclass(slots=True) @@ -121,10 +126,11 @@ class EventManager(metaclass=Singleton): self.__lock = threading.Lock() # 退出事件 self.__event = threading.Event() - # 广播异步处理器的生命周期由事件总线自己持有，避免关闭后仍向主循环运行。 + # 广播处理器由事件总线统一持有，确保插件卸载前可以建立结算屏障。 self.__lifecycle_lock = threading.RLock() self.__lifecycle_state = "new" - self.__async_handles: Dict[int, _EventAsyncHandle] = {} + self.__sync_handles: Dict[object, concurrent.futures.Future[Any]] = {} + self.__async_handles: Dict[object, _EventAsyncHandle] = {} # 由上层管理器注册的处理器实例解析器 self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {} # 由启动组合层注入的错误通知回调 @@ -155,6 +161,7 @@ class EventManager(metaclass=Singleton): event_factory=Event, error_handler=lambda **kwargs: self.__handle_event_error(**kwargs), async_handle_sink=self.__register_async_handle, + sync_handle_sink=self.__register_sync_handle, ) def register_handler_instance_resolver( @@ -188,6 +195,9 @@ class EventManager(metaclass=Singleton): if self.__lifecycle_state == "stopping": logger.warning("事件处理仍在停止，忽略重复启动") return + if self.__lifecycle_state == "sealed": + logger.warning("事件处理已封口，忽略重复启动") + return self.__lifecycle_state = "running" self.__event.set() self.__consumer_threads = [] @@ -199,14 +209,19 @@ class EventManager(metaclass=Singleton): def stop(self): """ - 停止广播事件处理线程 + 兼容同步关闭入口，等待同步处理器并请求取消异步处理器。 + + 调用线程无法安全等待主事件循环完成异步清理；需要完整异步收口时应使用 + stop_async()。 """ logger.info("正在停止事件处理...") consumer_threads = self.__begin_stop() try: self.__join_consumer_threads(consumer_threads) self.__discard_stop_sentinels() - self.__cancel_async_handles() + current_owner = _CURRENT_EVENT_HANDLER_OWNER.get() + self.__cancel_async_handles(exclude_owner=current_owner) + self.__wait_sync_handles(exclude_owner=current_owner) logger.info("事件处理停止完成") except Exception as e: logger.error(f"停止事件处理线程出错：{str(e)} - {traceback.format_exc()}") @@ -216,7 +231,7 @@ class EventManager(metaclass=Singleton): self.__consumer_threads = [] async def stop_async(self) -> None: - """停止广播消费者并等待已投递的异步处理器收口。""" + """停止广播消费者，等待同步处理器并取消收口异步处理器。""" logger.info("正在停止事件处理...") consumer_threads = self.__begin_stop() try: @@ -226,15 +241,29 @@ class EventManager(metaclass=Singleton): consumer_threads, ) self.__discard_stop_sentinels() + current_owner = _CURRENT_EVENT_HANDLER_OWNER.get() with self.__lifecycle_lock: - handles = tuple(self.__async_handles.values()) - for handle in handles: + async_handles = tuple( + handle + for owner, handle in self.__async_handles.items() + if owner is not current_owner + ) + sync_handles = tuple( + handle + for owner, handle in self.__sync_handles.items() + if owner is not current_owner + ) + for handle in async_handles: handle.handle.cancel() - if handles: + if async_handles or sync_handles: await asyncio.gather( *( asyncio.shield(asyncio.wrap_future(handle.completion)) - for handle in handles + for handle in async_handles + ), + *( + asyncio.shield(asyncio.wrap_future(handle)) + for handle in sync_handles ), return_exceptions=True, ) @@ -272,16 +301,81 @@ class EventManager(metaclass=Singleton): break if item[1] is not _EVENT_STOP_SENTINEL: pending.append(item) + self.__event_queue.task_done() for item in pending: self.__event_queue.put(item) - def __cancel_async_handles(self) -> None: - """请求取消所有仍由事件总线持有的异步处理器。""" + def __cancel_async_handles( + self, + *, + exclude_owner: object | None = None, + ) -> None: + """请求取消异步处理器，并避免 handler 关闭事件总线时取消自身。""" with self.__lifecycle_lock: - handles = tuple(self.__async_handles.values()) + handles = tuple( + handle + for owner, handle in self.__async_handles.items() + if owner is not exclude_owner + ) for handle in handles: handle.handle.cancel() + def __wait_sync_handles( + self, + *, + exclude_owner: object | None = None, + ) -> None: + """等待同步处理器完成，并避免 handler 关闭事件总线时等待自身。""" + with self.__lifecycle_lock: + handles = tuple( + handle + for owner, handle in self.__sync_handles.items() + if owner is not exclude_owner + ) + if handles: + concurrent.futures.wait(handles) + + def __register_sync_handle( + self, + callback: Callable[..., Any], + args: tuple[Any, ...], + ) -> bool: + """在同一生命周期临界区提交并登记同步广播处理器。""" + with self.__lifecycle_lock: + if self.__lifecycle_state != "running": + logger.warning( + "事件处理处于 %s 状态，拒绝同步广播处理器", + self.__lifecycle_state, + ) + return False + try: + owner = object() + + def _tracked_sync() -> Any: + """在同步 handler 调用栈中发布当前事件 owner。""" + context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner) + try: + return callback(*args) + finally: + _CURRENT_EVENT_HANDLER_OWNER.reset(context_token) + + handle = self.__executor.submit(_tracked_sync) + except RuntimeError: + logger.warning("同步事件处理器无法投递，线程池已停止") + return False + self.__sync_handles[owner] = handle + handle.add_done_callback( + lambda _completed, current_owner=owner: ( + self.__remove_sync_handle(current_owner) + ) + ) + return True + + def __remove_sync_handle(self, owner: object) -> None: + """同步处理器完成后移除其 owner 句柄。""" + with self.__lifecycle_lock: + self.__sync_handles.pop(owner, None) + def __register_async_handle( self, coroutine: Any, @@ -290,12 +384,18 @@ class EventManager(metaclass=Singleton): with self.__lifecycle_lock: if self.__lifecycle_state != "running": coroutine.close() + logger.warning( + "事件处理处于 %s 状态，拒绝异步广播处理器", + self.__lifecycle_state, + ) return False completion: concurrent.futures.Future[Any] = concurrent.futures.Future() started = threading.Event() + owner = object() async def _tracked() -> None: started.set() + context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner) try: result = await coroutine except asyncio.CancelledError: @@ -307,6 +407,8 @@ class EventManager(metaclass=Singleton): else: if not completion.done(): completion.set_result(result) + finally: + _CURRENT_EVENT_HANDLER_OWNER.reset(context_token) tracked = _tracked() try: @@ -316,7 +418,7 @@ class EventManager(metaclass=Singleton): coroutine.close() logger.warning("异步事件处理器无法投递，事件循环已停止") return False - self.__async_handles[id(completion)] = _EventAsyncHandle( + self.__async_handles[owner] = _EventAsyncHandle( handle=handle, completion=completion, ) @@ -329,13 +431,54 @@ class EventManager(metaclass=Singleton): completion.cancel() handle.add_done_callback(_complete_unstarted_submission) - completion.add_done_callback(self.__remove_async_handle) + completion.add_done_callback( + lambda _completed, current_owner=owner: ( + self.__remove_async_handle(current_owner) + ) + ) return True - def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None: + def __remove_async_handle(self, owner: object) -> None: """异步处理器完成后移除其 owner 句柄。""" with self.__lifecycle_lock: - self.__async_handles.pop(id(handle), None) + self.__async_handles.pop(owner, None) + + async def drain_async( + self, + timeout: Optional[float] = None, + *, + seal: bool = False, + ) -> bool: + """ + 等待已接纳的广播事件及其派生事件自然结算。 + + seal=True 会在确认稳定空闲的同一临界区关闭后续广播提交，供插件卸载前 + 建立不可穿透的投递屏障；超时或停止交错时保持原提交状态并返回 False。 + 广播处理器调用栈内无法等待自身完成，因此会立即返回 False，且不执行封口。 + """ + if _CURRENT_EVENT_HANDLER_OWNER.get() is not None: + logger.warning("事件处理器内部不能建立事件投递屏障") + return False + loop = asyncio.get_running_loop() + deadline = None if timeout is None else loop.time() + max(timeout, 0) + while True: + with self.__lifecycle_lock: + if self.__lifecycle_state not in {"running", "sealed"}: + return False + with self.__event_queue.all_tasks_done: + queue_idle = self.__event_queue.unfinished_tasks == 0 + handlers_idle = not self.__sync_handles and not self.__async_handles + if queue_idle and handlers_idle: + if seal: + self.__lifecycle_state = "sealed" + return True + if deadline is not None: + remaining = deadline - loop.time() + if remaining <= 0: + return False + await asyncio.sleep(min(0.01, remaining)) + else: + await asyncio.sleep(0.01) def check(self, etype: Union[EventType, ChainEventType]) -> bool: """ @@ -470,7 +613,15 @@ class EventManager(metaclass=Singleton): :param event: 要处理的事件对象 """ logger.debug(f"Triggering broadcast event: {event}") - self.__event_queue.put((event.priority, event)) + with self.__lifecycle_lock: + if self.__lifecycle_state in {"sealed", "stopping", "stopped"}: + logger.warning( + "事件处理处于 %s 状态，拒绝广播事件 %s", + self.__lifecycle_state, + event.event_type, + ) + return None + self.__event_queue.put((event.priority, event)) record_metric( "event.queue.depth", self.__event_queue.qsize(), @@ -580,15 +731,18 @@ class EventManager(metaclass=Singleton): while self.__event.is_set(): try: priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait) - if event is _EVENT_STOP_SENTINEL: - break - record_metric( - "event.queue.depth", - self.__event_queue.qsize(), - delivery="broadcast", - ) - rate_limiter.reset() - self.__dispatch_broadcast_event(event) + try: + if event is _EVENT_STOP_SENTINEL: + break + record_metric( + "event.queue.depth", + self.__event_queue.qsize(), + delivery="broadcast", + ) + rate_limiter.reset() + self.__dispatch_broadcast_event(event) + finally: + self.__event_queue.task_done() except Empty: rate_limiter.current_wait = rate_limiter.current_wait * random.uniform(1, 1 + jitter_factor) rate_limiter.trigger_limit() diff --git a/app/runtime/extensions/plugin/admission.py b/app/runtime/extensions/plugin/admission.py new file mode 100644 index 000000000..6656624f4 --- /dev/null +++ b/app/runtime/extensions/plugin/admission.py @@ -0,0 +1,112 @@ +"""插件宿主可变事务的停机准入。""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass + +from app.schemas.exception import PluginMutationRejectedError + + +@dataclass(slots=True) +class _MutationContext: + """记录一个可跨协程和受控线程传播的事务上下文。""" + + admission: "PluginMutationAdmission" + holders: int = 0 + open: bool = True + + +class PluginMutationAdmission: + """在停机封口与插件可变事务之间维护真实 owner 计数。""" + + def __init__(self) -> None: + """初始化开放准入、活动 owner 计数和事务上下文。""" + self._condition = threading.Condition() + self._accepting = True + self._active_count = 0 + self._current_context: ContextVar[_MutationContext | None] = ContextVar( + "plugin_mutation_context", + default=None, + ) + + @property + def active_count(self) -> int: + """返回尚未退出的 lease 数量，用于停机诊断和严格卸载判断。""" + with self._condition: + return self._active_count + + @property + def accepting(self) -> bool: + """返回当前生命周期是否仍接纳新的根事务。""" + with self._condition: + return self._accepting + + def is_held(self) -> bool: + """判断当前执行上下文是否持有仍有效的本 admission lease。""" + context = self._current_context.get() + return bool( + context + and context.admission is self + and context.open + and context.holders > 0 + ) + + @contextmanager + def hold(self, operation: str) -> Iterator[None]: + """取得可变事务 lease；封口后仅允许已获准事务的嵌套调用。""" + context = self._current_context.get() + nested = bool( + context + and context.admission is self + and context.open + and context.holders > 0 + ) + context_token = None + if not nested: + context = _MutationContext(admission=self) + context_token = self._current_context.set(context) + assert context is not None + + acquired = False + try: + with self._condition: + if not self._accepting and not nested: + raise PluginMutationRejectedError(operation) + self._active_count += 1 + context.holders += 1 + acquired = True + yield + finally: + if acquired: + with self._condition: + self._active_count -= 1 + context.holders -= 1 + if context.holders == 0: + context.open = False + self._condition.notify_all() + if context_token is not None: + self._current_context.reset(context_token) + + def seal(self) -> int: + """原子停止接纳根事务，并返回封口瞬间的活动 lease 数量。""" + with self._condition: + self._accepting = False + return self._active_count + + def wait_until_idle(self) -> None: + """自然等待全部已获准 lease 退出，不取消或遗失其 owner。""" + with self._condition: + while self._active_count: + self._condition.wait() + + def reopen(self) -> bool: + """仅在没有遗留 lease 时为新的应用生命周期重新开放准入。""" + with self._condition: + if self._active_count: + return False + self._accepting = True + return True diff --git a/app/runtime/extensions/plugin/lifecycle.py b/app/runtime/extensions/plugin/lifecycle.py index d8d3025f4..cfcef5b9b 100644 --- a/app/runtime/extensions/plugin/lifecycle.py +++ b/app/runtime/extensions/plugin/lifecycle.py @@ -5,6 +5,7 @@ from __future__ import annotations import traceback from collections.abc import Callable from functools import wraps +import threading import time from typing import Any, Optional, ParamSpec, TypeVar, cast @@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla try: result = func(*args, **kwargs) statuses = result.values() if isinstance(result, dict) else (result,) - if PluginRuntimeStatus.LOAD_FAILED in statuses: + if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses: outcome = "error" return result except BaseException: @@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla class PluginLifecycle: """管理插件发现、初始化、启停和热重载，不持有市场或 HTTP 路由职责。""" + _EVENT_HANDLERS_QUIESCED = "__event_handlers__" + def __init__( self, *, @@ -83,6 +86,8 @@ class PluginLifecycle: self._runtime_status_writer = runtime_status_writer self._logger = log self._event_sender = event_sender + self._lifecycle_lock = threading.RLock() + self._quiesced_hooks: dict[str, set[str]] = {} @observe_plugin_lifecycle("start") def start( @@ -116,6 +121,7 @@ class PluginLifecycle: self._classes[current_id] = plugin instance = plugin() instance.init_plugin(self._plugin_config(current_id)) + self._quiesced_hooks.pop(current_id, None) self._running[current_id] = instance self._logger.info( f"加载插件：{current_id} 版本：{instance.plugin_version}" @@ -156,31 +162,168 @@ class PluginLifecycle: @observe_plugin_lifecycle("stop") def stop(self, plugin_id: Optional[str] = None) -> None: - """停止指定插件或全部插件，并清理模块缓存。""" + """按旧单阶段 ABI 先解绑 handler，再停止并强制卸载插件。""" + with self._lifecycle_lock: + plugins = self._select_running_plugins(plugin_id) + self._quiesce_selected(plugins) + self._finalize( + plugin_id, + require_quiesced=False, + disable_events=not self._handlers_quiesced(plugins), + ) + + @observe_plugin_lifecycle("quiesce") + def quiesce(self, plugin_id: Optional[str] = None) -> bool: + """先解绑事件 handler，再按旧 hook 顺序停止生产者并保留实例。""" + with self._lifecycle_lock: + plugins = self._select_running_plugins(plugin_id) + return self._quiesce_selected(plugins) + + @observe_plugin_lifecycle("quiesce_handlers") + def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool: + """禁止目标插件接收新事件，保留实例供在途 handler 和后续 hook 使用。""" + with self._lifecycle_lock: + plugins = self._select_running_plugins(plugin_id) + return self._disable_selected_handlers(plugins) + + @observe_plugin_lifecycle("quiesce_services") + def quiesce_services(self, plugin_id: Optional[str] = None) -> bool: + """在事件结算屏障后执行旧 close、stop_service hook。""" + with self._lifecycle_lock: + plugins = self._select_running_plugins(plugin_id) + if not self._handlers_quiesced(plugins): + self._logger.warning("插件事件 handler 尚未全部停用，拒绝关闭插件资源") + return False + return self._quiesce_hooks(plugins) + + def _quiesce_selected(self, plugins: dict[str, Any]) -> bool: + """兼容单阶段调用：先停用全部 handler，再执行稳定快照的旧 hooks。""" + if not self._disable_selected_handlers(plugins): + return False + return self._quiesce_hooks(plugins) + + def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool: + """先停用稳定快照的全部事件入口，任一失败时不执行破坏性 hook。""" + all_converged = True + for current_id, plugin in plugins.items(): + completed = self._quiesced_hooks.setdefault(current_id, set()) + if self._EVENT_HANDLERS_QUIESCED in completed: + continue + try: + self._disable_events(type(plugin)) + except Exception as error: # noqa: BLE001 插件边界必须隔离 + all_converged = False + self._logger.warning( + f"停用插件 {current_id} 的事件 handler 时发生错误: {error}" + ) + continue + completed.add(self._EVENT_HANDLERS_QUIESCED) + return all_converged + + def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool: + """执行旧 ABI hooks，并只重试尚未成功的步骤。""" + all_converged = True + for current_id, plugin in plugins.items(): + completed = self._quiesced_hooks.setdefault(current_id, set()) + for hook_name in ("close", "stop_service"): + if hook_name in completed: + continue + hook = getattr(plugin, hook_name, None) + if not callable(hook): + completed.add(hook_name) + continue + try: + result = hook() + except Exception as error: # noqa: BLE001 插件边界必须隔离 + all_converged = False + self._logger.warning( + f"停止插件 {current_id} 的 {hook_name} 时发生错误: {error}" + ) + continue + if result is False: + all_converged = False + self._logger.warning( + f"停止插件 {current_id} 的 {hook_name} 未收敛" + ) + continue + completed.add(hook_name) + return all_converged + + @observe_plugin_lifecycle("finalize") + def finalize(self, plugin_id: Optional[str] = None) -> bool: + """在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。""" + return self._finalize( + plugin_id, + require_quiesced=True, + disable_events=False, + ) + + def _finalize( + self, + plugin_id: Optional[str], + *, + require_quiesced: bool, + disable_events: bool = True, + ) -> bool: + """按严格或兼容策略卸载插件，并在清理失败时保留实例所有权。""" + with self._lifecycle_lock: + plugins = self._select_running_plugins(plugin_id) + if require_quiesced and any( + not self._is_quiesced(current_id, plugin) + for current_id, plugin in plugins.items() + ): + self._logger.warning("插件后台服务尚未全部收敛，拒绝卸载运行实例") + return False + + try: + if disable_events: + for plugin in plugins.values(): + self._disable_events(type(plugin)) + self._clear_modules(plugin_id) + self._clear_tools() + except Exception as error: # noqa: BLE001 保留实例所有权供后续重试 + self._logger.warning(f"卸载插件运行实例时发生错误: {error}") + return False + + if plugin_id: + self._classes.pop(plugin_id, None) + self._running.pop(plugin_id, None) + self._quiesced_hooks.pop(plugin_id, None) + else: + self._classes.clear() + self._running.clear() + self._quiesced_hooks.clear() + self._logger.info("插件停止完成") + return True + + def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]: + """返回本阶段处理的稳定实例快照，并保持旧停机日志语义。""" if plugin_id: self._logger.info(f"正在停止插件 {plugin_id}...") plugin = self._running.get(plugin_id) plugins = {plugin_id: plugin} if plugin else {} if not plugin: self._logger.debug(f"插件 {plugin_id} 不存在或未加载") - else: - self._logger.info("正在停止所有插件...") - plugins = dict(self._running) + return plugins + self._logger.info("正在停止所有插件...") + return dict(self._running) - for current_id, plugin in plugins.items(): - self._disable_events(type(plugin)) - self._stop_plugin(plugin) + def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool: + """判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。""" + required = {self._EVENT_HANDLERS_QUIESCED} | { + hook_name + for hook_name in ("close", "stop_service") + if callable(getattr(plugin, hook_name, None)) + } + return required.issubset(self._quiesced_hooks.get(plugin_id, set())) - if plugin_id: - self._classes.pop(plugin_id, None) - self._running.pop(plugin_id, None) - self._clear_modules(plugin_id) - else: - self._classes.clear() - self._running.clear() - self._clear_modules(None) - self._clear_tools() - self._logger.info("插件停止完成") + def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool: + """判断稳定快照中的全部插件是否已经停用事件入口。""" + return all( + self._EVENT_HANDLERS_QUIESCED + in self._quiesced_hooks.get(plugin_id, set()) + for plugin_id in plugins + ) @observe_plugin_lifecycle("reload") def reload( @@ -194,14 +337,3 @@ class PluginLifecycle: status = self.start(plugin_id)[plugin_id] self._event_sender(reload_event, data={"plugin_id": plugin_id}) return status - - def _stop_plugin(self, plugin: Any) -> None: - """按插件旧 ABI 顺序关闭资源和服务。""" - try: - if hasattr(plugin, "close"): - plugin.close() - if hasattr(plugin, "stop_service"): - plugin.stop_service() - except Exception as error: # noqa: BLE001 - name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__ - self._logger.warning(f"停止插件 {name} 时发生错误: {error}") diff --git a/app/runtime/extensions/plugin/monitor.py b/app/runtime/extensions/plugin/monitor.py index 342972c2b..48f181d29 100644 --- a/app/runtime/extensions/plugin/monitor.py +++ b/app/runtime/extensions/plugin/monitor.py @@ -2,8 +2,8 @@ from __future__ import annotations -import time import threading +import time from collections.abc import Callable from pathlib import Path from typing import Any, Optional @@ -27,6 +27,8 @@ class PluginMonitorController: self._logger = log self._thread: Optional[threading.Thread] = None self._stop_event = threading.Event() + self._lifecycle_lock = threading.RLock() + self._closed = False @property def stop_event(self) -> threading.Event: @@ -35,32 +37,73 @@ class PluginMonitorController: def reload(self, enabled: bool) -> None: """按当前配置停止旧线程，并在启用时创建新线程。""" - self.stop() - if enabled: + stopped = self.stop() + if enabled and stopped: self.start() def start(self) -> None: """启动唯一的守护监控线程。""" - if self._thread and self._thread.is_alive(): - self._logger.info("插件文件修改监测已经在运行中...") - return - self._logger.info("开始监测插件文件修改...") - self._stop_event.clear() - self._thread = threading.Thread(target=self._runner, daemon=True) - self._thread.start() + with self._lifecycle_lock: + if self._closed: + self._logger.info("插件文件修改监测已进入停机封口，跳过启动") + return + if self._thread and self._thread.is_alive(): + self._logger.info("插件文件修改监测已经在运行中...") + return + self._logger.info("开始监测插件文件修改...") + self._stop_event.clear() + self._thread = threading.Thread(target=self._runner, daemon=True) + self._thread.start() - def stop(self) -> None: - """请求监控线程退出，并在限定时间内等待其清理。""" - if not self._thread or not self._thread.is_alive(): - self._logger.info("未启用插件文件修改监测，无需停止") - return - self._logger.info("正在停止插件文件修改监测...") + def reopen(self) -> bool: + """为新的应用生命周期解除封口，仍有旧线程时拒绝重开。""" + with self._lifecycle_lock: + if self._thread and self._thread.is_alive(): + self._logger.warning("旧插件文件监测线程仍在运行，无法开启新生命周期") + return False + self._thread = None + self._closed = False + return True + + def stop(self, timeout: float = 5.0) -> bool: + """临时停止监控线程，并返回其是否在预算内真正退出。""" + return self._stop_with_budget(timeout=timeout, close=False) + + def close(self, timeout: float = 5.0) -> bool: + """永久封口当前生命周期，并返回监控线程是否真正退出。""" + return self._stop_with_budget(timeout=timeout, close=True) + + def _stop_with_budget(self, *, timeout: float, close: bool) -> bool: + """在同一预算内取得生命周期锁、设置封口并等待线程退出。""" + timeout = max(0.0, timeout) + deadline = time.monotonic() + timeout self._stop_event.set() - self._thread.join(timeout=5) - if self._thread.is_alive(): - self._logger.warning("插件文件修改监测线程在5秒内未能正常停止。") - self._thread = None - self._logger.info("插件文件修改监测停止完成") + if not self._lifecycle_lock.acquire(timeout=timeout): + self._logger.warning( + f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权。" + ) + return False + try: + if close: + self._closed = True + thread = self._thread + self._stop_event.set() + if not thread or not thread.is_alive(): + self._thread = None + self._logger.info("未启用插件文件修改监测，无需停止") + return True + self._logger.info("正在停止插件文件修改监测...") + thread.join(timeout=max(0.0, deadline - time.monotonic())) + if thread.is_alive(): + self._logger.warning( + f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。" + ) + return False + self._thread = None + self._logger.info("插件文件修改监测停止完成") + return True + finally: + self._lifecycle_lock.release() class PluginChangeMonitor: diff --git a/app/runtime/extensions/plugin_manager.py b/app/runtime/extensions/plugin_manager.py index c505f18bb..2aa44c2d9 100644 --- a/app/runtime/extensions/plugin_manager.py +++ b/app/runtime/extensions/plugin_manager.py @@ -1,9 +1,21 @@ +import asyncio +import concurrent.futures import inspect import posixpath import threading from contextlib import contextmanager from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple +from typing import ( + Any, + Callable, + ContextManager, + Dict, + List, + Optional, + Tuple, + Type, + Union, +) from watchfiles import watch @@ -17,6 +29,7 @@ from app.runtime.execution import run_in_threadpool_to_completion from app.runtime.log import logger from app.runtime.observability import observe_compat_facade from app.runtime.settings import RuntimeSettingsCompat +from app.runtime.thread import ThreadHelper settings = RuntimeSettingsCompat() from app.runtime.events import EventHandlerBinding, eventmanager @@ -39,6 +52,7 @@ from app.runtime.extensions.plugin.sync import ( ) from app.runtime.extensions.plugin.clone import PluginCloneService from app.runtime.extensions.plugin.access import PluginAccessPolicy +from app.runtime.extensions.plugin.admission import PluginMutationAdmission from app.runtime.extensions.plugin.catalog import PluginCatalogFacade from app.runtime.extensions.plugin.paths import PluginPathResolver from app.runtime.extensions.plugin.dependency import ( @@ -47,6 +61,7 @@ from app.runtime.extensions.plugin.dependency import ( PluginDependencyService, ) from app.runtime.extensions.plugin.storage import PluginConfigStore, PluginInstanceStore +from app.schemas.exception import PluginMutationRejectedError from app.schemas.types import EventType, SystemConfigKey LegacyDiagnosticsConfigurator = Callable[..., None] @@ -195,6 +210,15 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): runner=self._run_file_watcher, log=logger, ) + self._plugin_quiesce_lock = threading.RLock() + self._plugin_quiesce_future: Optional[ + concurrent.futures.Future[bool] + ] = None + self._plugin_service_quiesce_future: Optional[ + concurrent.futures.Future[bool] + ] = None + self._plugin_mutation_admission = PluginMutationAdmission() + self._plugin_runtime_closed = False self._plugin_dependencies = PluginDependencyService( system=get_plugin_system, log=logger, @@ -315,12 +339,16 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): def init_config(self): """按最新系统配置完整重启插件。""" - # 停止已有插件 - self.stop() - classification = self.classify_plugins() - self.apply_plugin_dependency_classification(classification) - for plugin_id in classification.ready: - self.start(plugin_id) + try: + with self.mutation("配置热重载"): + # 停止已有插件 + self.stop() + classification = self.classify_plugins() + self.apply_plugin_dependency_classification(classification) + for plugin_id in classification.ready: + self.start(plugin_id) + except PluginMutationRejectedError as error: + logger.warning(str(error)) def start(self, pid: Optional[str] = None) -> Dict[str, PluginRuntimeStatus]: """ @@ -328,8 +356,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param pid: 插件ID，为空加载所有插件 """ - _legacy_diagnostics_configurator(enabled=settings.DEBUG, emitter=logger.warning) - return self._plugin_lifecycle.start(pid) + try: + with self.mutation("启动插件"): + with self._plugin_quiesce_lock: + _legacy_diagnostics_configurator( + enabled=settings.DEBUG, + emitter=logger.warning, + ) + return self._plugin_lifecycle.start(pid) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + if pid: + return {pid: PluginRuntimeStatus.LOAD_FAILED} + return {} def init_plugin(self, plugin_id: str, conf: dict): """ @@ -337,7 +376,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param plugin_id: 插件ID :param conf: 插件配置 """ - self._plugin_lifecycle.initialize(plugin_id, conf) + try: + with self.mutation("初始化插件配置"): + with self._plugin_quiesce_lock: + self._plugin_lifecycle.initialize(plugin_id, conf) + except PluginMutationRejectedError as error: + logger.warning(str(error)) def clear_plugin_agent_tools_cache(self) -> None: """ @@ -356,12 +400,127 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): """兼容读取旧私有字段，实际版本由独立工具目录持有。""" return self._plugin_tool_catalog.revision - def stop(self, pid: Optional[str] = None): + def stop(self, pid: Optional[str] = None) -> None: """ 停止插件服务 :param pid: 插件ID，为空停止所有插件 """ - self._plugin_lifecycle.stop(pid) + try: + with self.mutation("停止插件"): + with self._plugin_quiesce_lock: + self._plugin_lifecycle.stop(pid) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + + def mutation(self, operation: str) -> ContextManager[None]: + """为一个完整插件可变事务取得可跨异步边界传播的准入 lease。""" + return self._plugin_mutation_admission.hold(operation) + + def reopen_plugins(self) -> bool: + """为新应用生命周期解除运行时封口，仍活跃的 quiesce owner 禁止复用。""" + with self._plugin_quiesce_lock: + futures = ( + self._plugin_quiesce_future, + self._plugin_service_quiesce_future, + ) + if any(future is not None and not future.done() for future in futures): + logger.warning("插件后台服务仍在停止，无法开启新的应用生命周期") + return False + if self._plugin_runtime_closed and self._running_plugins: + logger.warning("上一应用生命周期仍持有插件实例，拒绝解除运行时封口") + return False + if not self._plugin_mutation_admission.reopen(): + logger.warning("上一应用生命周期仍有插件可变事务，拒绝解除运行时封口") + return False + self._plugin_runtime_closed = False + return True + + async def quiesce_plugins(self, timeout: float = 240.0) -> bool: + """封口变更事务并停用插件 handler，超时后保留 Future ownership。""" + if self._plugin_mutation_admission.is_held(): + logger.warning("插件可变事务不能等待自身收敛，拒绝在事务内执行停机") + return False + with self._plugin_quiesce_lock: + self._plugin_runtime_closed = True + self._plugin_mutation_admission.seal() + future = self._plugin_quiesce_future + if future is None or future.done(): + future = ThreadHelper().submit(self._quiesce_after_mutations) + self._plugin_quiesce_future = future + try: + result = await asyncio.wait_for( + asyncio.shield(asyncio.wrap_future(future)), + timeout=max(0.0, timeout), + ) + return bool(result) + except asyncio.TimeoutError: + logger.error(f"插件后台服务未在 {timeout:g} 秒内收敛") + return False + except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果 + logger.error(f"插件后台服务停止失败：{error}", exc_info=True) + return False + finally: + if future.done(): + with self._plugin_quiesce_lock: + if self._plugin_quiesce_future is future: + self._plugin_quiesce_future = None + + async def quiesce_plugin_services(self, timeout: float = 240.0) -> bool: + """在事件结算后执行旧插件停机 hook，并有界等待同步 owner。""" + with self._plugin_quiesce_lock: + prepare_future = self._plugin_quiesce_future + if prepare_future is not None and not prepare_future.done(): + logger.warning("插件事件入口仍在封口，拒绝提前关闭插件资源") + return False + if not self._plugin_runtime_closed: + logger.warning("插件运行时尚未封口，拒绝关闭插件资源") + return False + if self._plugin_mutation_admission.active_count: + logger.warning("插件可变事务仍在执行，拒绝关闭插件资源") + return False + future = self._plugin_service_quiesce_future + if future is None or future.done(): + future = ThreadHelper().submit( + self._plugin_lifecycle.quiesce_services, + ) + self._plugin_service_quiesce_future = future + try: + result = await asyncio.wait_for( + asyncio.shield(asyncio.wrap_future(future)), + timeout=max(0.0, timeout), + ) + return bool(result) + except asyncio.TimeoutError: + logger.error(f"插件旧停机 hook 未在 {timeout:g} 秒内收敛") + return False + except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果 + logger.error(f"插件旧停机 hook 执行失败：{error}", exc_info=True) + return False + finally: + if future.done(): + with self._plugin_quiesce_lock: + if self._plugin_service_quiesce_future is future: + self._plugin_service_quiesce_future = None + + def finalize_plugins(self) -> bool: + """确认 quiesce owner 已结束后禁用 handler 并卸载插件实例。""" + with self._plugin_quiesce_lock: + futures = ( + self._plugin_quiesce_future, + self._plugin_service_quiesce_future, + ) + if any(future is not None and not future.done() for future in futures): + logger.warning("插件后台服务仍在停止，拒绝释放运行实例") + return False + if self._plugin_mutation_admission.active_count: + logger.warning("插件可变事务仍在执行，拒绝释放运行实例") + return False + return self._plugin_lifecycle.finalize() + + def _quiesce_after_mutations(self) -> bool: + """等待已获准变更自然结束后，再停用插件事件入口。""" + self._plugin_mutation_admission.wait_until_idle() + return self._plugin_lifecycle.quiesce_handlers() @staticmethod def _load_selective_plugins(pid: Optional[str], installed_plugins: List[str], @@ -428,8 +587,10 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): """返回配置重载日志使用的功能名称。""" return "插件文件修改监测" - def start_monitor(self): - """按当前配置启动插件文件修改监测。""" + def start_monitor(self, *, reopen: bool = False) -> None: + """按当前配置启动监控；新生命周期可显式解除既有封口。""" + if reopen and not self._plugin_monitor.reopen(): + return if ( not self.is_plugin_settling() and (settings.DEV or settings.PLUGIN_AUTO_RELOAD) @@ -447,11 +608,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): ) ) - def stop_monitor(self): - """ - 停止监测插件文件修改监测 - """ - self._plugin_monitor.stop() + def stop_monitor(self, timeout: float = 5.0) -> bool: + """停止插件文件监控，并返回线程是否在预算内真正退出。""" + return self._plugin_monitor.stop(timeout=timeout) + + def close_monitor(self, timeout: float = 5.0) -> bool: + """封口当前生命周期的文件监控，并返回线程是否真正退出。""" + return self._plugin_monitor.close(timeout=timeout) def _run_file_watcher(self): """ @@ -504,25 +667,31 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): """ 已安装本地插件源码变化时，同步到运行目录 """ - return self._local_plugin_sync.sync(pid, candidate) + try: + with self.mutation("同步本地插件源码"): + return self._local_plugin_sync.sync(pid, candidate) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False @contextmanager def suppress_plugin_monitor(self, plugin_id: str): """在插件目录原子更新期间阻止文件监控抢先重载半成品。""" - normalized_id = plugin_id.lower() - with self._monitor_suppression_lock: - self._suppressed_monitor_plugins[normalized_id] = ( - self._suppressed_monitor_plugins.get(normalized_id, 0) + 1 - ) - try: - yield - finally: + with self.mutation("更新插件包"): + normalized_id = plugin_id.lower() with self._monitor_suppression_lock: - count = self._suppressed_monitor_plugins.get(normalized_id, 0) - if count <= 1: - self._suppressed_monitor_plugins.pop(normalized_id, None) - else: - self._suppressed_monitor_plugins[normalized_id] = count - 1 + self._suppressed_monitor_plugins[normalized_id] = ( + self._suppressed_monitor_plugins.get(normalized_id, 0) + 1 + ) + try: + yield + finally: + with self._monitor_suppression_lock: + count = self._suppressed_monitor_plugins.get(normalized_id, 0) + if count <= 1: + self._suppressed_monitor_plugins.pop(normalized_id, None) + else: + self._suppressed_monitor_plugins[normalized_id] = count - 1 def is_plugin_monitor_suppressed(self, plugin_id: str) -> bool: """判断指定插件是否处于安装或替换写入阶段。""" @@ -534,23 +703,43 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): 从内存中移除一个插件 :param plugin_id: 插件ID """ - self._plugin_lifecycle.stop(plugin_id) - self._plugin_registry.remove(plugin_id) + try: + with self.mutation("移除插件实例"): + with self._plugin_quiesce_lock: + self._plugin_lifecycle.stop(plugin_id) + self._plugin_registry.remove(plugin_id) + except PluginMutationRejectedError as error: + logger.warning(str(error)) def reload_plugin(self, plugin_id: str) -> PluginRuntimeStatus: """ 将一个插件重新加载到内存 :param plugin_id: 插件ID """ - return self._plugin_lifecycle.reload(plugin_id, EventType.PluginReload) + try: + with self.mutation("重新加载插件"): + with self._plugin_quiesce_lock: + return self._plugin_lifecycle.reload( + plugin_id, + EventType.PluginReload, + ) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return PluginRuntimeStatus.LOAD_FAILED def reload_plugin_tree(self, plugin_id: str) -> PluginRuntimeStatus: """重载源码插件，并同步刷新所有引用该源码的虚拟实例。""" - source_plugin_id = self.get_plugin_source_id(plugin_id) - status = self.reload_plugin(source_plugin_id) - for instance in self._plugin_instance_store.for_source(source_plugin_id): - self.reload_plugin(instance.instance_id) - return status + try: + with self.mutation("重载插件实例树"): + with self._plugin_quiesce_lock: + source_plugin_id = self.get_plugin_source_id(plugin_id) + status = self.reload_plugin(source_plugin_id) + for instance in self._plugin_instance_store.for_source(source_plugin_id): + self.reload_plugin(instance.instance_id) + return status + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return PluginRuntimeStatus.LOAD_FAILED def get_plugin_reload_targets(self, plugin_id: str) -> List[str]: """返回源码更新后需要刷新注册信息的源插件及其实例 ID。""" @@ -584,33 +773,40 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): 安装本地不存在或需要更新的插件 """ - return self._plugin_sync.sync() + with self.mutation("同步插件包"): + return self._plugin_sync.sync() @staticmethod def install_plugin_missing_dependencies() -> List[str]: """ 安装插件中缺失或不兼容的依赖项 """ - return PluginDependencyService( - system=get_plugin_system, - log=logger, - ).install_missing() + manager = PluginManager() + with manager.mutation("安装插件依赖"): + return PluginDependencyService( + system=get_plugin_system, + log=logger, + ).install_missing() @staticmethod def install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult: """安装插件缺失依赖并返回缺失项及安装成功状态。""" - return PluginDependencyService( - system=get_plugin_system, - log=logger, - ).install_missing_with_status() + manager = PluginManager() + with manager.mutation("安装插件依赖"): + return PluginDependencyService( + system=get_plugin_system, + log=logger, + ).install_missing_with_status() @staticmethod async def async_install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult: """在异步启动链中恢复插件依赖并保留取消语义。""" - return await PluginDependencyService( - system=get_plugin_system, - log=logger, - ).async_install_missing_with_status() + manager = PluginManager() + with manager.mutation("安装插件依赖"): + return await PluginDependencyService( + system=get_plugin_system, + log=logger, + ).async_install_missing_with_status() def classify_plugins(self) -> PluginDependencyClassification: """按源码依赖状态分类物理插件，并把结果映射到虚拟实例。""" @@ -706,7 +902,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): def delete_plugin_instance(self, plugin_id: str) -> bool: """删除虚拟实例描述；调用方仍负责停止实例和清理业务数据。""" - return self._plugin_instance_store.delete(plugin_id) + try: + with self.mutation("删除插件实例描述"): + return self._plugin_instance_store.delete(plugin_id) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False def save_plugin_config(self, pid: str, conf: dict, force: bool = False) -> bool: """ @@ -715,7 +916,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param conf: 配置 :param force: 强制保存 """ - return self._plugin_config_store.write(pid, conf, force) + try: + with self.mutation("保存插件配置"): + return self._plugin_config_store.write(pid, conf, force) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False async def async_save_plugin_config( self, pid: str, conf: dict, force: bool = False @@ -726,7 +932,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param conf: 配置 :param force: 强制保存 """ - return await self._plugin_config_store.async_write(pid, conf, force) + try: + with self.mutation("保存插件配置"): + return await self._plugin_config_store.async_write(pid, conf, force) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False def delete_plugin_config(self, pid: str, force: bool = False) -> bool: """ @@ -734,7 +945,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param pid: 插件ID :param force: 插件停止后仍允许按插件 ID 删除持久化配置 """ - return self._plugin_config_store.delete(pid, force) + try: + with self.mutation("删除插件配置"): + return self._plugin_config_store.delete(pid, force) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False def delete_plugin_data(self, pid: str, force: bool = False) -> bool: """ @@ -742,7 +958,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param pid: 插件ID :param force: 插件停止后仍允许按插件 ID 删除持久化数据 """ - return self._plugin_config_store.delete_data(pid, force) + try: + with self.mutation("删除插件数据"): + return self._plugin_config_store.delete_data(pid, force) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False def get_plugin_state(self, pid: str) -> bool: """ @@ -1130,14 +1351,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): :param icon: 自定义图标URL :return: (是否成功, 错误信息) """ - return self._plugin_clone.clone( - plugin_id=plugin_id, - suffix=suffix, - name=name, - description=description, - version=version, - icon=icon, - ) + try: + with self.mutation("创建插件分身"): + return self._plugin_clone.clone( + plugin_id=plugin_id, + suffix=suffix, + name=name, + description=description, + version=version, + icon=icon, + ) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False, str(error) def _modify_plugin_files(self, plugin_dir: Path, original_id: str, suffix: str, name: str, description: str, version: str = None, diff --git a/app/runtime/tasks.py b/app/runtime/tasks.py index 1d03523e6..56b644f2c 100644 --- a/app/runtime/tasks.py +++ b/app/runtime/tasks.py @@ -106,8 +106,8 @@ class TaskRegistry: } ) - async def shutdown(self, *, timeout_seconds: float = 10.0) -> None: - """停止接收并有限等待存量任务，超时任务保留登记并报告责任域。""" + async def shutdown(self, *, timeout_seconds: float = 10.0) -> bool: + """停止接收并有限等待存量任务，返回全部 owner 是否真实收敛。""" self._accepting = False records = self.records tasks = [record.task for record in records] @@ -139,6 +139,7 @@ class TaskRegistry: "timeout_seconds": timeout_seconds, } ) + return all(record.task.done() for record in records) _default_registry = TaskRegistry() diff --git a/app/schemas/exception.py b/app/schemas/exception.py index 062eed942..8e03e0ddd 100644 --- a/app/schemas/exception.py +++ b/app/schemas/exception.py @@ -4,6 +4,7 @@ __all__ = ( "APIRateLimitException", "RateLimitExceededException", "OperationInterrupted", + "PluginMutationRejectedError", "StorageQueryError", "TMDbException", ) @@ -49,6 +50,15 @@ class OperationInterrupted(KeyboardInterrupt): pass +class PluginMutationRejectedError(RuntimeError): + """表示插件运行时已封口，新的可变事务未获准执行。""" + + def __init__(self, operation: str) -> None: + """保存被拒绝的操作名称并生成稳定诊断消息。""" + self.operation = operation + super().__init__(f"插件运行时已进入停机阶段，拒绝{operation}") + + class StorageQueryError(Exception): """ 用于表示存储查询无法确认结果的异常类。 diff --git a/app/startup/agent_initializer.py b/app/startup/agent_initializer.py index dfe54f59f..d7a2d83ba 100644 --- a/app/startup/agent_initializer.py +++ b/app/startup/agent_initializer.py @@ -20,6 +20,9 @@ from app.runtime.log import logger from app.schemas.types import EventType +AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0 + + def _get_skill_catalog() -> Any: """按需返回 Agent 技能目录实现，供消息应用层消费端口。""" from app.agent.skills.registry import SkillHelper @@ -106,6 +109,7 @@ class AgentInitializer: self._initialized = False self._manager: Any = None self._compat_injected = False + self._shutdown_started = False self._shutdown_complete = False eventmanager.add_event_listener( EventType.ConfigChanged, @@ -117,6 +121,7 @@ class AgentInitializer: 初始化AI智能体管理器 """ try: + self._shutdown_started = False self._shutdown_complete = False if agent_manager is not None: if not settings.AI_AGENT_ENABLE: @@ -142,7 +147,12 @@ class AgentInitializer: async def handle_config_changed(self, event: Event) -> None: """仅在 manifest watch 命中时协调 service，关闭态保持 fail closed。""" changed_keys = _event_changed_keys(event) - if not changed_keys or self._compat_injected or self._shutdown_complete: + if ( + not changed_keys + or self._compat_injected + or self._shutdown_started + or self._shutdown_complete + ): return try: self._manager = await reconcile_agent_service( @@ -156,24 +166,25 @@ class AgentInitializer: self._initialized = False logger.debug(f"配置变更协调AI智能体失败: {error}") - async def cleanup(self) -> None: - """清理 initializer 引用；显式注入对象同时在此关闭。""" + async def cleanup(self) -> bool: + """清理 initializer 引用；未收敛的显式注入对象继续由本实例持有。""" try: manager = self._manager compat_injected = self._compat_injected if manager is None: - return - try: - if compat_injected: - await manager.close() - logger.info("AI智能体管理器已关闭") - finally: - self._initialized = False - self._manager = None - self._compat_injected = False + return True + if compat_injected and await manager.close() is False: + logger.error("AI智能体管理器仍有会话 owner 未收敛") + return False + logger.info("AI智能体管理器已关闭") + self._initialized = False + self._manager = None + self._compat_injected = False + return True except Exception as e: logger.debug(f"关闭AI智能体管理器时发生错误: {e}") + return False # 全局AI智能体初始化器实例 @@ -204,23 +215,56 @@ async def init_agent() -> bool: return False -async def stop_agent(): +async def stop_agent() -> bool: """ - 停止AI智能体（异步版本，用于在应用关闭时调用） + 停止AI智能体，并在全部会话和工具资源释放后返回 True。 """ + converged = True + close_blocking_executors = None + agent_initializer._shutdown_started = True + try: + if is_tool_factory_materialized(): + from app.agent.tools.base import ( + begin_blocking_executor_shutdown, + close_blocking_executors as close_executors, + ) + + # 必须在任何 manager await 之前封口，避免旧会话趁收尾窗口提交新同步调用。 + begin_blocking_executor_shutdown(cancel_futures=True) + close_blocking_executors = close_executors + except Exception as e: + logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}") + converged = False + try: if not agent_initializer._shutdown_complete: if agent_initializer._compat_injected: - await agent_initializer.cleanup() + service_converged = await agent_initializer.cleanup() else: - await begin_agent_shutdown() - await agent_initializer.cleanup() - agent_initializer._shutdown_complete = True - if is_tool_factory_materialized(): - from app.agent.tools.base import shutdown_blocking_executors - - shutdown_blocking_executors(wait=False, cancel_futures=True) + service_converged = await begin_agent_shutdown() + if service_converged is not False: + service_converged = await agent_initializer.cleanup() + converged = converged and service_converged is not False except Exception as e: logger.error(f"停止AI智能体时发生错误: {e}") - finally: - await close_materialized_terminal_sessions() + converged = False + + if close_blocking_executors is not None: + try: + blocking_converged = await close_blocking_executors( + timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS, + cancel_futures=True, + ) + converged = converged and blocking_converged + except Exception as e: + logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}") + converged = False + + if converged: + try: + await close_materialized_terminal_sessions() + except Exception as e: + logger.error(f"关闭AI智能体终端会话时发生错误: {e}") + converged = False + agent_initializer._shutdown_complete = converged + return converged diff --git a/app/startup/lifecycle/__init__.py b/app/startup/lifecycle/__init__.py index 73ed3efef..6c0bcf448 100644 --- a/app/startup/lifecycle/__init__.py +++ b/app/startup/lifecycle/__init__.py @@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper from app.runtime.state import SystemHelper from app.runtime.log import logger, LoggerManager from app.startup.command_initializer import init_command, stop_command, restart_command +from app.startup.agent_initializer import stop_agent from app.startup.domain_initializer import configure_domain_dependencies -from app.startup.modules_initializer import init_modules, stop_modules +from app.startup.modules_initializer import ( + drain_events, + init_modules, + settle_events, + stop_modules, +) from app.startup.monitor_initializer import stop_monitor, init_monitor from app.startup.plugins_initializer import ( configure_plugin_services, execute_task, + finalize_plugins, init_plugins, - stop_plugins, + quiesce_plugin_services, + quiesce_plugins, + stop_plugin_monitor, sync_plugins, ) from app.startup.routers_initializer import init_routers @@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import ( init_plugin_scheduler, ) from app.db.engine import check_connection_budget, get_engine, get_global_async_engine -from app.startup.transfer_initializer import replay_pending_transfers +from app.startup.transfer_initializer import ( + replay_pending_transfers, + stop_transfer_runtime, +) from app.startup.workflow_initializer import init_workflow, stop_workflow from app.startup.lifecycle.components import ( LifecycleComponent, + LifecycleFailurePolicy, LifecycleMode, lifecycle_manifest, ) @@ -101,8 +114,8 @@ async def run_shutdown_step( name: str, callback: Callable[[], object], timeout_seconds: float | None = None, -) -> None: - """在有限预算内执行关闭阶段，并保留未收敛任务的资源所有权。""" +) -> bool: + """在有限预算内执行关闭阶段，并返回资源 owner 是否已经收敛。""" try: result = callback() if inspect.isawaitable(result): @@ -120,16 +133,22 @@ async def run_shutdown_step( task.add_done_callback(_consume_shutdown_result) if timeout_seconds: try: - await asyncio.wait_for( + result = await asyncio.wait_for( asyncio.shield(task), timeout=timeout_seconds ) except asyncio.TimeoutError: logger.error("关闭%s超时，已请求取消并保留未收敛任务", name) task.cancel() + return False else: - await task + result = await task + if result is False: + logger.error("关闭%s未收敛，资源所有权保持不变", name) + return False + return True except Exception as err: logger.error(f"关闭{name}失败：{err}") + return False async def run_startup_step( @@ -152,6 +171,66 @@ async def run_startup_step( logger.info("启动%s完成，耗时=%.2fms", name, elapsed_ms) +async def stop_lifecycle_components( + components: tuple[LifecycleComponent, ...], +) -> bool: + """按声明顺序关闭组件，并在关键 owner 未收敛时停止释放依赖。""" + all_converged = True + for component in sorted( + (item for item in components if item.stop is not None), + key=lambda item: item.stop_order or 0, + ): + completed = await run_shutdown_step( + component.name, + component.stop, + component.stop_timeout_seconds, + ) + if completed: + continue + all_converged = False + if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST: + logger.error( + "关闭%s未收敛，停止释放其后续依赖", + component.name, + ) + break + return all_converged + + +def select_startup_cleanup_components( + components: tuple[LifecycleComponent, ...], + *, + started_names: set[str], + active_component: LifecycleComponent | None, +) -> tuple[LifecycleComponent, ...]: + """选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。""" + cleanup_names = set(started_names) + if active_component is not None: + cleanup_names.add(active_component.name) + + # stop-only owner 没有启动回调，按已激活依赖递归纳入，避免清理时反向 + # 实例化尚未触达的插件、模块或外部资源。 + changed = True + while changed: + changed = False + for component in components: + if ( + component.stop is None + or component.start is not None + or component.name in cleanup_names + or not set(component.dependencies).issubset(cleanup_names) + ): + continue + cleanup_names.add(component.name) + changed = True + + return tuple( + component + for component in components + if component.stop is not None and component.name in cleanup_names + ) + + async def initialize_modules_component(app: FastAPI) -> None: """启动模块并把其类型化运行时发布到当前 FastAPI AppState。""" try: @@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None: configure_task_registry(task_registry) -async def stop_task_registry(app: FastAPI) -> None: - """停止接收新后台任务，并取消、等待当前 lifespan 的存量任务。""" +async def stop_task_registry(app: FastAPI) -> bool: + """停止接收新后台任务，并把已封口登记器保留到下一次显式启动。""" task_registry = getattr(app.state, "task_registry", None) - try: - if isinstance(task_registry, TaskRegistry): - await task_registry.shutdown(timeout_seconds=30.0) - finally: + if not isinstance(task_registry, TaskRegistry): configure_task_registry(None) app.state.task_registry = None + return True + # 即使 owner 已收敛，也不能在后续插件/模块 stop hook 仍会运行时退回永久 + # accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。 + return await task_registry.shutdown(timeout_seconds=30.0) def prepare_plugin_restore() -> None: @@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]: stop_order=5, start_timeout_seconds=30, stop_timeout_seconds=60, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, ), LifecycleComponent( name="数据库准备", @@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]: dependencies=("插件备份恢复",), mode=LifecycleMode.NORMAL_ONLY, start=init_plugins, - stop=stop_plugins, + stop=finalize_plugins, start_order=90, stop_order=60, start_timeout_seconds=300, stop_timeout_seconds=300, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="插件变更监控", + dependencies=("插件",), + mode=LifecycleMode.NORMAL_ONLY, + stop=stop_plugin_monitor, + stop_order=8, + stop_timeout_seconds=10, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, ), LifecycleComponent( name="定时器", @@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]: stop_order=50, start_timeout_seconds=120, stop_timeout_seconds=120, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, ), LifecycleComponent( name="监控器", @@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]: stop_order=40, start_timeout_seconds=120, stop_timeout_seconds=120, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="整理后台服务", + dependencies=("模块服务",), + stop=stop_transfer_runtime, + stop_order=52, + stop_timeout_seconds=45, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="AI智能体会话", + dependencies=("模块服务",), + stop=stop_agent, + stop_order=51, + stop_timeout_seconds=300, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="插件事件入口", + dependencies=("插件",), + mode=LifecycleMode.NORMAL_ONLY, + stop=quiesce_plugins, + stop_order=53, + stop_timeout_seconds=300, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="事件尾任务结算", + dependencies=("模块服务", "插件"), + mode=LifecycleMode.NORMAL_ONLY, + stop=settle_events, + stop_order=54, + stop_timeout_seconds=120, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="插件后台服务", + dependencies=("插件",), + mode=LifecycleMode.NORMAL_ONLY, + stop=quiesce_plugin_services, + stop_order=55, + stop_timeout_seconds=300, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, + ), + LifecycleComponent( + name="事件投递屏障", + dependencies=("模块服务", "整理后台服务"), + stop=drain_events, + stop_order=58, + stop_timeout_seconds=120, + stop_failure=LifecycleFailurePolicy.FAIL_FAST, ), LifecycleComponent( name="待处理整理回放", - dependencies=("监控器",), + dependencies=("监控器", "整理后台服务"), mode=LifecycleMode.NORMAL_ONLY, start=replay_pending_transfers, start_order=120, @@ -369,6 +513,9 @@ async def lifespan(app: FastAPI): health = get_application_health(app) health.begin_startup() main_loop = asyncio.get_running_loop() + enabled_components: tuple[LifecycleComponent, ...] = () + started_component_names: set[str] = set() + active_start_component: LifecycleComponent | None = None try: validate_process_topology( workers=settings.API_WORKERS, @@ -390,11 +537,14 @@ async def lifespan(app: FastAPI): (item for item in enabled_components if item.start is not None), key=lambda item: item.start_order or 0, ): + active_start_component = component await run_startup_step( component.name, component.start, component.start_timeout_seconds, ) + started_component_names.add(component.name) + active_start_component = None if settings.MOVIEPILOT_SAFE_MODE: print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.") # 插件同步到本地 @@ -407,10 +557,15 @@ async def lifespan(app: FastAPI): except BaseException: # Uvicorn 在 lifespan 抛错时不会开始接流量；状态仍需供嵌入式入口和测试诊断。 health.mark_failed() + cleanup_components = select_startup_cleanup_components( + enabled_components, + started_names=started_component_names, + active_component=active_start_component, + ) try: - await stop_task_registry(app) + await stop_lifecycle_components(cleanup_components) except Exception as cleanup_error: - logger.error(f"启动失败后的后台任务清理失败：{cleanup_error}") + logger.error(f"启动失败后的生命周期清理失败：{cleanup_error}") finally: global_vars.clear_loop(main_loop) raise @@ -421,21 +576,10 @@ async def lifespan(app: FastAPI): health.mark_stopping() print("Shutting down...") global_vars.stop_system() - # 插件恢复会在线程池中修改源码与依赖，必须完成后再进入资源关闭阶段。 try: - await sync_plugins_task - except Exception as e: - print(str(e)) - try: - for component in sorted( - (item for item in enabled_components if item.stop is not None), - key=lambda item: item.stop_order or 0, - ): - await run_shutdown_step( - component.name, - component.stop, - component.stop_timeout_seconds, - ) + # 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner + # 在统一预算内取消/等待，不能在屏障之前无界 await 绕过停机预算。 + await stop_lifecycle_components(enabled_components) finally: try: # 日志最后关闭，确保其他组件的收尾信息已写入文件 diff --git a/app/startup/modules_initializer.py b/app/startup/modules_initializer.py index e69827f31..dc1becc12 100644 --- a/app/startup/modules_initializer.py +++ b/app/startup/modules_initializer.py @@ -129,7 +129,7 @@ from app.command import CommandChain from app.schemas.message import Message from app.schemas.message import MessageType from app.schemas.types import EventType, SystemConfigKey -from app.startup.agent_initializer import init_agent, stop_agent +from app.startup.agent_initializer import init_agent from app.startup.database import build_database_governance from app.startup.managed_resources_initializer import ( init_managed_resources, @@ -560,6 +560,22 @@ def close_browser_sessions() -> None: BrowserSessionHelper.close_all_sessions() +async def drain_events() -> bool: + """在插件卸载前等待已接收事件及其同步、异步处理器完成。""" + event_manager = EventManager.get_existing_instance() + if event_manager is None: + return True + return await event_manager.drain_async(seal=True) + + +async def settle_events() -> bool: + """在插件 handler 停用后结算在途事件，但保留停机 hook 的尾事件入口。""" + event_manager = EventManager.get_existing_instance() + if event_manager is None: + return True + return await event_manager.drain_async(seal=False) + + async def stop_modules(): """ 服务关闭 @@ -578,7 +594,6 @@ async def stop_modules(): logger.error(f"关闭{name}失败：{err}") return True - await run_step("AI智能体", stop_agent) await run_step("模块", lambda: ModuleManager().shutdown()) await run_step("事件消费", lambda: EventManager().stop_async()) await run_step("浏览器会话", close_browser_sessions) diff --git a/app/startup/monitor_initializer.py b/app/startup/monitor_initializer.py index b3aa3ddd2..4eb491ea6 100644 --- a/app/startup/monitor_initializer.py +++ b/app/startup/monitor_initializer.py @@ -1,15 +1,26 @@ from app.monitor import Monitor +from app.runtime.execution import run_in_threadpool_to_completion -def init_monitor(): - """ - 初始化监控器 - """ - Monitor() +def init_monitor() -> None: + """初始化监控器；复用单例时必须显式开启新的应用 lifespan。""" + monitor = Monitor.get_existing_instance() + if monitor is None: + Monitor() + return + if not monitor.lifecycle_closed: + return + if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT): + raise RuntimeError("旧目录监控 owner 未收敛，无法开启新生命周期") + if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT): + raise RuntimeError("目录监控初始化失败") -def stop_monitor(): - """ - 停止监控器 - """ - Monitor().stop() +async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool: + """在线程池里永久关闭监控器，取消后仍等待同步 owner 收敛到终态。""" + monitor = Monitor.get_existing_instance() + if monitor is None: + return True + return bool( + await run_in_threadpool_to_completion(monitor.close, timeout=timeout) + ) diff --git a/app/startup/plugins_initializer.py b/app/startup/plugins_initializer.py index f82a6f26b..83a29c234 100644 --- a/app/startup/plugins_initializer.py +++ b/app/startup/plugins_initializer.py @@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import ( configure_plugin_resource_import_preparer, configure_site_auth_level_provider, ) +from app.runtime.execution import run_in_threadpool_to_completion from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult from app.application.plugin.catalog import PluginCatalogService from app.application.plugin.data import DeletePluginDataCommand @@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork from app.runtime.log import logger from app.foundation.version import compare_version from app.schemas.plugin import PluginRuntimeStatus +from app.schemas.exception import PluginMutationRejectedError from app.schemas.types import SystemConfigKey @@ -142,56 +144,64 @@ async def sync_plugins() -> bool: """ plugin_manager = None try: - configure_plugin_services() loop = global_vars.loop plugin_manager = PluginManager() - plugin_manager.set_plugin_settling(True) - - sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地") - dependency_result = await ( - plugin_manager.async_install_plugin_missing_dependencies_with_status() - ) - if dependency_result is None: - return False - if not isinstance(dependency_result, PluginDependencyInstallResult): - logger.error("缺失依赖项安装返回了无效结果，跳过插件重新初始化") - return False - previous_statuses = plugin_manager.get_plugin_runtime_statuses() - classification = plugin_manager.classify_plugins() - plugin_manager.apply_plugin_dependency_classification(classification) - if not dependency_result.success: - logger.error("缺失依赖项安装未完成，将继续激活当前已就绪插件") - changed_ids = await execute_task( - loop, - lambda: _activate_ready_plugins( - plugin_manager, - classification.ready, - sync_result or [], - previous_statuses, - ), - "插件运行态激活", - ) - if changed_ids is None: - return False - - if not changed_ids: - logger.debug("没有新的插件进入可运行状态") - return False - - for plugin_id in changed_ids: - register_plugin_api(plugin_id) - if dependency_result.success: - logger.info(f"后台插件加载完成，共处理 {len(changed_ids)} 个插件") - else: - logger.warning( - f"缺失依赖项仍未全部恢复，已激活 {len(changed_ids)} 个就绪插件" - ) - return True + with plugin_manager.mutation("启动后同步插件"): + configure_plugin_services() + plugin_manager.set_plugin_settling(True) + return await _sync_plugins_admitted(plugin_manager, loop) + except PluginMutationRejectedError as error: + logger.warning(str(error)) + return False except Exception as e: logger.error(f"插件初始化过程中出现异常: {e}") return False +async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool: + """在一个 admission lease 内完成包、依赖、实例和动态路由同步。""" + sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地") + dependency_result = await ( + plugin_manager.async_install_plugin_missing_dependencies_with_status() + ) + if dependency_result is None: + return False + if not isinstance(dependency_result, PluginDependencyInstallResult): + logger.error("缺失依赖项安装返回了无效结果，跳过插件重新初始化") + return False + previous_statuses = plugin_manager.get_plugin_runtime_statuses() + classification = plugin_manager.classify_plugins() + plugin_manager.apply_plugin_dependency_classification(classification) + if not dependency_result.success: + logger.error("缺失依赖项安装未完成，将继续激活当前已就绪插件") + changed_ids = await execute_task( + loop, + lambda: _activate_ready_plugins( + plugin_manager, + classification.ready, + sync_result or [], + previous_statuses, + ), + "插件运行态激活", + ) + if changed_ids is None: + return False + + if not changed_ids: + logger.debug("没有新的插件进入可运行状态") + return False + + for plugin_id in changed_ids: + register_plugin_api(plugin_id) + if dependency_result.success: + logger.info(f"后台插件加载完成，共处理 {len(changed_ids)} 个插件") + else: + logger.warning( + f"缺失依赖项仍未全部恢复，已激活 {len(changed_ids)} 个就绪插件" + ) + return True + + def _activate_ready_plugins( plugin_manager: PluginManager, ready_ids: tuple[str, ...], @@ -217,12 +227,39 @@ def _activate_ready_plugins( return changed_ids +async def quiesce_plugins(timeout: float = 240.0) -> bool: + """封口插件变更并停用 handler，保留超时 Future 的运行所有权。""" + plugin_manager = PluginManager.get_existing_instance() + if plugin_manager is None: + return True + return await plugin_manager.quiesce_plugins(timeout=timeout) + + +async def quiesce_plugin_services(timeout: float = 240.0) -> bool: + """在事件结算后有界执行旧插件 close、stop_service hook。""" + plugin_manager = PluginManager.get_existing_instance() + if plugin_manager is None: + return True + return await plugin_manager.quiesce_plugin_services(timeout=timeout) + + +def finalize_plugins() -> bool: + """在事件屏障封口后卸载已停用 handler 的插件实例。""" + plugin_manager = PluginManager.get_existing_instance() + if plugin_manager is None: + return True + return bool(plugin_manager.finalize_plugins()) + + async def execute_task(loop, task_func, task_name): """ - 执行后台任务 + 执行后台任务；取消调用方时仍持有同步线程直到真实完成。 """ try: - result = await loop.run_in_executor(None, task_func) + # loop 参数属于既有调用 ABI；同步执行改由 completion-aware 适配器持有， + # 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。 + del loop + result = await run_in_threadpool_to_completion(task_func) if isinstance(result, PluginDependencyInstallResult): processed_count = len(result.missing) elif isinstance(result, list): @@ -245,13 +282,15 @@ def init_plugins(): """ configure_plugin_services() plugin_manager = PluginManager() + if not plugin_manager.reopen_plugins(): + raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛") classification = plugin_manager.classify_plugins() plugin_manager.apply_plugin_dependency_classification(classification) plugin_manager.set_plugin_settling(True) for plugin_id in classification.ready: plugin_manager.start(plugin_id) register_plugin_api() - plugin_manager.start_monitor() + plugin_manager.start_monitor(reopen=True) logger.info( "插件启动分类：立即加载=%s，等待依赖=%s，等待源码=%s", len(classification.ready), @@ -260,15 +299,30 @@ def init_plugins(): ) -def stop_plugins(): - """ - 停止插件 - """ +def stop_plugin_monitor(timeout: float = 5.0) -> bool: + """封口已创建管理器的文件监控线程，并返回是否完成收口。""" + plugin_manager = PluginManager.get_existing_instance() + if plugin_manager is None: + return True try: - plugin_manager = PluginManager() + return bool(plugin_manager.close_monitor(timeout=timeout)) + except Exception as e: + logger.error(f"停止插件文件监控时发生错误：{e}", exc_info=True) + return False + + +def stop_plugins() -> bool: + """停止已创建的插件监控和运行实例，不在停机阶段反向物化管理器。""" + try: + plugin_manager = PluginManager.get_existing_instance() + if plugin_manager is None: + return True + monitor_stopped = True try: - plugin_manager.stop_monitor() + monitor_stopped = plugin_manager.stop_monitor() finally: plugin_manager.stop() + return bool(monitor_stopped) except Exception as e: logger.error(f"停止插件时发生错误：{e}", exc_info=True) + return False diff --git a/app/startup/transfer_initializer.py b/app/startup/transfer_initializer.py index ede763510..6a242d6fe 100644 --- a/app/startup/transfer_initializer.py +++ b/app/startup/transfer_initializer.py @@ -11,3 +11,15 @@ def replay_pending_transfers(): 回放本身在后台线程执行，不阻塞启动流程。 """ TransferChain().replay_pending() + + +async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool: + """关闭已存在的整理后台 owner，且不在关停阶段创建新的整理链实例。 + + :param timeout_seconds: worker 与 pending 回放共享的最大等待秒数 + :return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True + """ + transfer_chain = TransferChain.get_existing_instance() + if transfer_chain is None: + return True + return await transfer_chain.close(timeout_seconds=timeout_seconds) diff --git a/docs/refactor/backend-architecture-next-stage.md b/docs/refactor/backend-architecture-next-stage.md index a8ececa83..fcf436e0f 100644 --- a/docs/refactor/backend-architecture-next-stage.md +++ b/docs/refactor/backend-architecture-next-stage.md @@ -6,7 +6,7 @@ > 审计范围：宿主后端；排除 `app/plugins/**` 运行时插件副本 > 规范优先级：`AGENTS.md` 与 `docs/rules/` 高于本文 > 相关文档：`docs/architecture-overview.md`、`docs/refactor/backend-architecture-governance.md`、`docs/refactor/backend-module-refactor-compatibility.md` -> 实施进度：阶段 0～6 的宿主架构能力已完成收口；API/Application 公共复杂度基线已清零，启动组合根的 SystemConfigOper 构造点已由 14 降至 1；API 进程内后台任务已完成首批统一登记，插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线；阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义。 +> 实施进度：阶段 0～6 的宿主架构能力已完成收口；API/Application 公共复杂度基线已清零，启动组合根的 SystemConfigOper 构造点已由 14 降至 1；API 进程内后台任务已完成首批统一登记，插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线；阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义；阶段 1b1 已收口整理 worker、pending 回放、失败通知、进程内 AI 重试、插件监控与事件投递的生命周期所有权。 ## 当前复核结论（2026-08-23） @@ -15,8 +15,8 @@ ### 长期整改阶段 0：治理门禁恢复（2026-08-23） -- 宿主依赖基线已审查 TaskRegistry 接入后的语义差异：当前为 `800` 个模块、`6479` 条内部导入边，12 组重点禁止边继续全部为 `0`，唯一非平凡 SCC 仍是隔离的 TMDB 移植包。 -- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistry；normal/safe 组件数分别为 `16`/`8`，CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序，不再把 Python/平台模块数量当作硬合同。 +- 宿主依赖基线已审查 TaskRegistry 与有界后台 owner 接入后的语义差异：当前为 `800` 个模块、`6482` 条内部导入边，12 组重点禁止边继续全部为 `0`，唯一非平凡 SCC 仍是隔离的 TMDB 移植包。 +- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistry；normal/safe 组件数分别为 `20`/`10`，CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序，不再把 Python/平台模块数量当作硬合同。 - 官方插件快照覆盖 `plugins.v3`、`plugins.v2` 以及 V3 实际会从 `package.json` 回退加载的 31 个默认实现；`app/plugins/**` 仍只是宿主运行副本，不进入扫描。 - SDK 快照以各模块显式 `__all__` 为公开合同，能够记录赋值别名；`typing`、`__future__` 等实现期导入不再被误冻结，既有数据库备份门面已补精确导出清单。 - async 阻塞实际债务已由 fixture 中的 10 项下降到 1 项并固化低水位；剩余项是 Scheduler Agent task 查询，后续阶段迁入异步查询边界后归零。 @@ -36,6 +36,36 @@ - 本子阶段仍只覆盖 TaskRegistry。Transfer worker/replay、Agent blocking executor、Event handler drain、通道线程和 E2/E3 durable 完成点继续作为阶段 1 后续切片，不能因 owner 门禁通过而宣称完成。 +### 长期整改阶段 1b1：整理后台生命周期所有权（2026-08-23） + +- `TransferChain` 为每一代整理 worker 使用独立停止信号；配置热更新只让旧代完成已经进入同步 I/O 的 + 工作，不再让旧线程重新领取新任务。超时旧线程继续由 `_retiring_threads` 持有，重复关闭可以继续等待， + 不把无法强制取消的文件操作伪装成已结束。生命周期锁等待与 worker/replay join 共用同一个 deadline， + 停止哨兵也不再被误算成真实队列任务而阻止最后一批进度结算。 +- pending 回放改为单一受管线程，启动重复调用不会并发扫描；关停信号会在查询、`stat` 和逐条回放边界 + 重新检查，尚未处理的 `TransferPending` 登记保持不变。关闭与 `queue.get()` 竞争时，尚未开始的任务会 + 原样放回队列并保持 `task_done()` 计数平衡。 +- 失败通知聚合器和 AI 重试 scheduler 现在拥有 timer、buffer 与 flush task 的显式 `close()`；分组使用 + generation 阻止旧 timer 在新静默窗口尚未 armed 时提前消费新批次。跨线程提交 AI 重试产生的 Future + 也会观察最终异常，不再只用提交调用外层的 `try/except` 假设异步执行成功。 +- 生命周期新增常驻“整理后台服务” owner：正常模式和安全模式都只关闭已存在的单例，不会在 shutdown + 反向创建 worker。插件文件监控、Scheduler、Agent 和整理 owner 先停止宿主生产任务；随后封口插件变更、 + 停用插件事件入口并结算全部在途 handler，最后才调用插件私有 timer、scheduler、watcher 的旧停机 hook。 + 任一阶段返回 `False` 或超时，`FAIL_FAST` 屏障都会停止释放仍可能被活线程使用的后续依赖。 +- EventManager 同时持有线程池同步 handler Future 和事件循环异步 handler completion；“事件投递屏障”会 + 等待队列、handler 及 handler 派生事件自然收敛，再原子封住新的广播提交。为兼容无法声明资源依赖的 + 旧插件，宿主先停用其新 handler 投递，再用非封口屏障等待已经开始的 handler 退出，之后按原顺序调用 + `close`/`stop_service`；hook 产生的尾事件仍可由其他宿主 handler 消费，最终屏障后才卸载插件实例。 + 事件消费、共享线程池和模块资源仍在插件之后关闭。 +- lifespan 启动中途失败会记录已完成及当前部分启动的组件，递归纳入已激活依赖对应的 stop-only owner， + 并复用正常停机的顺序、超时和 `FAIL_FAST` 策略。后段初始化失败不再只停止 TaskRegistry 后遗留 Scheduler、 + Transfer、Event、插件或 HTTP 资源。 +- 兼容边界没有变化：`do_transfer`、队列与手工整理公开签名、模块方法 kwargs、插件整理事件类型及 payload、 + 同步插件 ABI 和动态 API 原生返回结构均未改名或包裹；本阶段也没有 schema/Alembic 变更。 +- 本阶段只证明进程内 owner、取消、等待和依赖释放顺序。失败通知仍可能在业务提交后、消息接受前随进程 + 崩溃而丢失；五分钟 AI 重试缓冲仍未持久化；`TransferPending` 仍只有 `storage + src_path`，不能表达文件 + 副作用 checkpoint、lease 和未知完成状态。它们分别留给阶段 1b2、1b3、1b4，不能据此宣称 E2/E3 完成。 + ### 总体判断 当前架构总体合理，已经从跨层混合的遗留单体收敛为**边界清晰的模块化单体**： diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index 60aefa3e1..6df30e85c 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -13,8 +13,8 @@ "runtime_to_db": [], "workflow_to_db": [] }, - "edge_count": 6479, - "edge_sha256": "a65f8d4024e2299b37510359c7ffea91219f0aa0ed9673b74a3eb22d51d6c67f", + "edge_count": 6483, + "edge_sha256": "0a2f82ca189468d5e954b06a2c188c4ea5380a76c7c64039c7f6eb843cb60523", "edges": [ "app -> app.runtime", "app -> app.runtime.compat", @@ -5671,6 +5671,7 @@ "app.runtime.extensions.plugin_manager -> app.runtime.observability", "app.runtime.extensions.plugin_manager -> app.runtime.reload", "app.runtime.extensions.plugin_manager -> app.runtime.settings", + "app.runtime.extensions.plugin_manager -> app.runtime.thread", "app.runtime.extensions.plugin_manager -> app.schemas", "app.runtime.extensions.plugin_manager -> app.schemas.plugin", "app.runtime.extensions.plugin_manager -> app.schemas.types", @@ -6086,6 +6087,7 @@ "app.startup.lifecycle -> app.runtime.tasks", "app.startup.lifecycle -> app.runtime.topology", "app.startup.lifecycle -> app.startup", + "app.startup.lifecycle -> app.startup.agent_initializer", "app.startup.lifecycle -> app.startup.cache_initializer", "app.startup.lifecycle -> app.startup.command_initializer", "app.startup.lifecycle -> app.startup.database_initializer", @@ -6218,6 +6220,8 @@ "app.startup.modules_initializer -> app.startup.transaction", "app.startup.modules_initializer -> app.startup.workflow", "app.startup.monitor_initializer -> app.monitor", + "app.startup.monitor_initializer -> app.runtime", + "app.startup.monitor_initializer -> app.runtime.execution", "app.startup.outbox -> app.application", "app.startup.outbox -> app.application.outbox", "app.startup.outbox -> app.db", diff --git a/tests/fixtures/architecture/startup-performance-baseline.json b/tests/fixtures/architecture/startup-performance-baseline.json index fa0230eed..d0d0831b8 100644 --- a/tests/fixtures/architecture/startup-performance-baseline.json +++ b/tests/fixtures/architecture/startup-performance-baseline.json @@ -1,41 +1,41 @@ { "schema_version": 2, - "generated_at": "2026-08-23T08:11:40.406030+00:00", + "generated_at": "2026-08-23T11:06:42.858188+00:00", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "python": "3.14.3", "repeat": 3, "targets": { "app.startup.lifecycle": { "loaded_app_module_count": 362, - "max_ms": 944.347, - "median_ms": 935.035, - "min_ms": 933.625, + "max_ms": 1022.231, + "median_ms": 976.39, + "min_ms": 967.395, "samples_ms": [ - 944.347, - 935.035, - 933.625 + 1022.231, + 967.395, + 976.39 ] }, "app.factory": { "loaded_app_module_count": 374, - "max_ms": 950.655, - "median_ms": 950.429, - "min_ms": 947.964, + "max_ms": 989.21, + "median_ms": 980.605, + "min_ms": 951.564, "samples_ms": [ - 950.655, - 947.964, - 950.429 + 980.605, + 989.21, + 951.564 ] }, "app.main": { "loaded_app_module_count": 376, - "max_ms": 1150.269, - "median_ms": 1103.976, - "min_ms": 1097.978, + "max_ms": 1114.04, + "median_ms": 1088.251, + "min_ms": 1080.26, "samples_ms": [ - 1103.976, - 1097.978, - 1150.269 + 1080.26, + 1088.251, + 1114.04 ] } }, @@ -46,57 +46,26 @@ "samples": [ { "mode": "normal", - "enabled_component_count": 16, - "startup_ms": 0.721, - "full_lifespan_ms": 0.907, - "stage_ms": { - "后台任务登记器": 0.084, - "数据库准备": 0.038, - "HTTP 基础能力": 0.028, - "领域依赖装配": 0.027, - "数据库引擎预热": 0.024, - "数据库连接预算": 0.024, - "路由": 0.023, - "模块服务": 0.024, - "插件备份恢复": 0.023, - "插件": 0.029, - "定时器": 0.023, - "监控器": 0.024, - "待处理整理回放": 0.021, - "命令服务": 0.025, - "工作流": 0.02, - "插件同步与启动收尾": 0.031 - }, - "threads_before": 2, - "threads_started": 2, - "threads_after": 2, - "tasks_before": 1, - "tasks_started": 2, - "tasks_after": 1, - "database_connections_started": 0 - }, - { - "mode": "normal", - "enabled_component_count": 16, - "startup_ms": 0.614, - "full_lifespan_ms": 0.788, + "enabled_component_count": 21, + "startup_ms": 0.619, + "full_lifespan_ms": 0.806, "stage_ms": { "后台任务登记器": 0.075, "数据库准备": 0.037, - "HTTP 基础能力": 0.029, + "HTTP 基础能力": 0.031, "领域依赖装配": 0.027, - "数据库引擎预热": 0.023, - "数据库连接预算": 0.024, - "路由": 0.023, - "模块服务": 0.023, - "插件备份恢复": 0.023, - "插件": 0.024, - "定时器": 0.025, - "监控器": 0.023, - "待处理整理回放": 0.022, - "命令服务": 0.021, + "数据库引擎预热": 0.024, + "数据库连接预算": 0.025, + "路由": 0.022, + "模块服务": 0.024, + "插件备份恢复": 0.021, + "插件": 0.02, + "定时器": 0.023, + "监控器": 0.021, + "待处理整理回放": 0.019, + "命令服务": 0.022, "工作流": 0.022, - "插件同步与启动收尾": 0.03 + "插件同步与启动收尾": 0.034 }, "threads_before": 2, "threads_started": 2, @@ -108,26 +77,57 @@ }, { "mode": "normal", - "enabled_component_count": 16, - "startup_ms": 0.611, - "full_lifespan_ms": 0.785, + "enabled_component_count": 21, + "startup_ms": 0.628, + "full_lifespan_ms": 0.808, "stage_ms": { - "后台任务登记器": 0.072, - "数据库准备": 0.033, - "HTTP 基础能力": 0.026, + "后台任务登记器": 0.073, + "数据库准备": 0.037, + "HTTP 基础能力": 0.039, + "领域依赖装配": 0.027, + "数据库引擎预热": 0.025, + "数据库连接预算": 0.025, + "路由": 0.022, + "模块服务": 0.022, + "插件备份恢复": 0.021, + "插件": 0.02, + "定时器": 0.024, + "监控器": 0.022, + "待处理整理回放": 0.029, + "命令服务": 0.02, + "工作流": 0.021, + "插件同步与启动收尾": 0.032 + }, + "threads_before": 2, + "threads_started": 2, + "threads_after": 2, + "tasks_before": 1, + "tasks_started": 2, + "tasks_after": 1, + "database_connections_started": 0 + }, + { + "mode": "normal", + "enabled_component_count": 21, + "startup_ms": 0.664, + "full_lifespan_ms": 0.863, + "stage_ms": { + "后台任务登记器": 0.083, + "数据库准备": 0.047, + "HTTP 基础能力": 0.036, "领域依赖装配": 0.028, - "数据库引擎预热": 0.023, - "数据库连接预算": 0.022, - "路由": 0.024, - "模块服务": 0.02, - "插件备份恢复": 0.023, - "插件": 0.024, + "数据库引擎预热": 0.029, + "数据库连接预算": 0.025, + "路由": 0.026, + "模块服务": 0.025, + "插件备份恢复": 0.025, + "插件": 0.028, "定时器": 0.026, - "监控器": 0.024, - "待处理整理回放": 0.023, + "监控器": 0.021, + "待处理整理回放": 0.021, "命令服务": 0.024, - "工作流": 0.022, - "插件同步与启动收尾": 0.031 + "工作流": 0.021, + "插件同步与启动收尾": 0.036 }, "threads_before": 2, "threads_started": 2, @@ -138,9 +138,9 @@ "database_connections_started": 0 } ], - "median_startup_ms": 0.614, - "median_full_lifespan_ms": 0.788, - "enabled_component_count": 16, + "median_startup_ms": 0.628, + "median_full_lifespan_ms": 0.808, + "enabled_component_count": 21, "enabled_components": [ "后台任务登记器", "数据库准备", @@ -152,8 +152,13 @@ "模块服务", "插件备份恢复", "插件", + "插件变更监控", "定时器", "监控器", + "整理后台服务", + "AI智能体会话", + "插件后台服务", + "事件投递屏障", "待处理整理回放", "命令服务", "工作流", @@ -164,19 +169,19 @@ "samples": [ { "mode": "safe", - "enabled_component_count": 8, - "startup_ms": 0.463, - "full_lifespan_ms": 0.667, + "enabled_component_count": 11, + "startup_ms": 0.464, + "full_lifespan_ms": 0.637, "stage_ms": { - "后台任务登记器": 0.075, - "数据库准备": 0.035, + "后台任务登记器": 0.074, + "数据库准备": 0.038, "HTTP 基础能力": 0.03, - "领域依赖装配": 0.032, - "数据库引擎预热": 0.024, - "数据库连接预算": 0.022, - "路由": 0.025, - "模块服务": 0.022, - "插件同步与启动收尾": 0.058 + "领域依赖装配": 0.029, + "数据库引擎预热": 0.026, + "数据库连接预算": 0.025, + "路由": 0.024, + "模块服务": 0.024, + "插件同步与启动收尾": 0.033 }, "threads_before": 2, "threads_started": 2, @@ -188,43 +193,43 @@ }, { "mode": "safe", - "enabled_component_count": 8, - "startup_ms": 0.474, - "full_lifespan_ms": 0.68, + "enabled_component_count": 11, + "startup_ms": 0.451, + "full_lifespan_ms": 0.624, "stage_ms": { - "后台任务登记器": 0.076, - "数据库准备": 0.043, - "HTTP 基础能力": 0.029, - "领域依赖装配": 0.028, + "后台任务登记器": 0.073, + "数据库准备": 0.035, + "HTTP 基础能力": 0.028, + "领域依赖装配": 0.027, + "数据库引擎预热": 0.024, + "数据库连接预算": 0.023, + "路由": 0.023, + "模块服务": 0.02, + "插件同步与启动收尾": 0.031 + }, + "threads_before": 2, + "threads_started": 2, + "threads_after": 2, + "tasks_before": 1, + "tasks_started": 2, + "tasks_after": 1, + "database_connections_started": 0 + }, + { + "mode": "safe", + "enabled_component_count": 11, + "startup_ms": 0.477, + "full_lifespan_ms": 0.661, + "stage_ms": { + "后台任务登记器": 0.081, + "数据库准备": 0.05, + "HTTP 基础能力": 0.035, + "领域依赖装配": 0.03, "数据库引擎预热": 0.026, "数据库连接预算": 0.024, "路由": 0.024, - "模块服务": 0.024, - "插件同步与启动收尾": 0.057 - }, - "threads_before": 2, - "threads_started": 2, - "threads_after": 2, - "tasks_before": 1, - "tasks_started": 2, - "tasks_after": 1, - "database_connections_started": 0 - }, - { - "mode": "safe", - "enabled_component_count": 8, - "startup_ms": 0.454, - "full_lifespan_ms": 0.657, - "stage_ms": { - "后台任务登记器": 0.075, - "数据库准备": 0.036, - "HTTP 基础能力": 0.028, - "领域依赖装配": 0.028, - "数据库引擎预热": 0.025, - "数据库连接预算": 0.024, - "路由": 0.024, - "模块服务": 0.024, - "插件同步与启动收尾": 0.06 + "模块服务": 0.02, + "插件同步与启动收尾": 0.026 }, "threads_before": 2, "threads_started": 2, @@ -235,9 +240,9 @@ "database_connections_started": 0 } ], - "median_startup_ms": 0.463, - "median_full_lifespan_ms": 0.667, - "enabled_component_count": 8, + "median_startup_ms": 0.464, + "median_full_lifespan_ms": 0.637, + "enabled_component_count": 11, "enabled_components": [ "后台任务登记器", "数据库准备", @@ -246,7 +251,10 @@ "数据库引擎预热", "数据库连接预算", "路由", - "模块服务" + "模块服务", + "整理后台服务", + "AI智能体会话", + "事件投递屏障" ] } } diff --git a/tests/test_agent_lazy_initializer.py b/tests/test_agent_lazy_initializer.py index efa072d55..df8b9fd69 100644 --- a/tests/test_agent_lazy_initializer.py +++ b/tests/test_agent_lazy_initializer.py @@ -115,7 +115,7 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager( initializer._manager = manager initializer._initialized = True initializer._compat_injected = False - shutdown = AsyncMock() + shutdown = AsyncMock(return_value=True) monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown) monkeypatch.setattr(agent_initializer, "agent_initializer", initializer) monkeypatch.setattr( @@ -124,13 +124,63 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager( lambda: False, ) - await agent_initializer.stop_agent() + assert await agent_initializer.stop_agent() is True shutdown.assert_awaited_once_with() manager.close.assert_not_awaited() assert initializer._manager is None +@pytest.mark.anyio +async def test_production_stop_retains_manager_when_runtime_does_not_converge( + monkeypatch, +) -> None: + """Agent service 未收敛时不得释放 initializer 引用或下游工具资源。""" + manager = AsyncMock() + initializer = agent_initializer.AgentInitializer() + initializer._manager = manager + initializer._initialized = True + initializer._compat_injected = False + shutdown = AsyncMock(return_value=False) + executor_seal = MagicMock() + executor_close = AsyncMock(return_value=True) + monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown) + monkeypatch.setattr(agent_initializer, "agent_initializer", initializer) + monkeypatch.setattr( + agent_initializer, + "is_tool_factory_materialized", + lambda: True, + ) + fake_base = types.ModuleType("app.agent.tools.base") + fake_base.begin_blocking_executor_shutdown = executor_seal + fake_base.close_blocking_executors = executor_close + monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base) + + assert await agent_initializer.stop_agent() is False + + shutdown.assert_awaited_once_with() + assert initializer._manager is manager + assert initializer._shutdown_started is True + assert initializer._shutdown_complete is False + executor_seal.assert_called_once_with(cancel_futures=True) + executor_close.assert_awaited_once_with( + timeout_seconds=( + agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS + ), + cancel_futures=True, + ) + + event = agent_initializer.Event( + agent_initializer.EventType.ConfigChanged, + {"key": "AI_AGENT_ENABLE"}, + ) + reconcile = AsyncMock() + monkeypatch.setattr(agent_initializer, "reconcile_agent_service", reconcile) + await initializer.handle_config_changed(event) + reconcile.assert_not_awaited() + assert initializer._manager is manager + + @pytest.mark.anyio async def test_config_listener_delegates_watch_filter_to_runtime(monkeypatch) -> None: """配置监听器只转交 changed keys，不维护第二份启用开关。""" @@ -227,8 +277,10 @@ async def test_stop_closes_tool_executor_after_factory_materialization( ) -> None: """工具能力已解析时，应取消仍排队的阻塞工具任务。""" fake_base = types.ModuleType("app.agent.tools.base") - cleanup = MagicMock() - fake_base.shutdown_blocking_executors = cleanup + seal = MagicMock() + cleanup = AsyncMock(return_value=True) + fake_base.begin_blocking_executor_shutdown = seal + fake_base.close_blocking_executors = cleanup monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base) monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock()) monkeypatch.setattr( @@ -242,15 +294,27 @@ async def test_stop_closes_tool_executor_after_factory_materialization( agent_initializer.AgentInitializer(), ) - await agent_initializer.stop_agent() + assert await agent_initializer.stop_agent() is True - cleanup.assert_called_once_with(wait=False, cancel_futures=True) + seal.assert_called_once_with(cancel_futures=True) + cleanup.assert_awaited_once_with( + timeout_seconds=( + agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS + ), + cancel_futures=True, + ) @pytest.mark.anyio -async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None: - """应用关闭不得等待已经进入线程池且尚未返回的工具调用。""" - from app.agent.tools.base import MoviePilotTool +async def test_stop_retains_running_blocking_tool_until_retry(monkeypatch) -> None: + """阻塞工具超过预算时 stop 返回 False，真实结束后重试才释放 owner。""" + from app.agent.tools.base import ( + MoviePilotTool, + _blocking_futures, + _blocking_retiring_executors, + close_blocking_executors, + reopen_blocking_executors, + ) started = threading.Event() release = threading.Event() @@ -264,7 +328,21 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None MoviePilotTool.run_blocking("web", _blocking_call) ) assert await asyncio.wait_for(asyncio.to_thread(started.wait), timeout=1) - monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock()) + monkeypatch.setattr( + agent_initializer, + "begin_agent_shutdown", + AsyncMock(return_value=True), + ) + monkeypatch.setattr( + agent_initializer, + "close_materialized_terminal_sessions", + AsyncMock(), + ) + monkeypatch.setattr( + agent_initializer, + "AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS", + 0.01, + ) monkeypatch.setattr( agent_initializer, "is_tool_factory_materialized", @@ -277,8 +355,22 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None ) try: - await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) + assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is False assert worker.done() is False - finally: + assert _blocking_futures + assert _blocking_retiring_executors + + with pytest.raises(RuntimeError, match="正在关闭"): + await MoviePilotTool.run_blocking("web", lambda: "late") + release.set() assert await asyncio.wait_for(worker, timeout=1) == "done" + assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is True + assert not _blocking_futures + assert not _blocking_retiring_executors + finally: + release.set() + if not worker.done(): + await asyncio.wait_for(worker, timeout=1) + await close_blocking_executors(timeout_seconds=1, cancel_futures=True) + assert reopen_blocking_executors() is True diff --git a/tests/test_agent_lifecycle.py b/tests/test_agent_lifecycle.py index 666ed1121..e3195bed9 100644 --- a/tests/test_agent_lifecycle.py +++ b/tests/test_agent_lifecycle.py @@ -115,8 +115,8 @@ async def test_agent_manager_background_tasks_share_owner_loop(monkeypatch) -> N assert manager._idle_cleanup_task is idle_cleanup_task assert memory_manager.cleanup_task is memory_cleanup_task - await manager.close() - await manager.close() + assert await manager.close() is True + assert await manager.close() is True assert manager._idle_cleanup_task is None assert memory_manager.cleanup_task is None assert idle_cleanup_task.done() @@ -489,7 +489,8 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout( ) await asyncio.wait_for(started.wait(), timeout=1) - await manager.close() + assert await manager.close() is False + assert await manager.close() is False assert not cleanup_called.is_set() assert "close-timeout" in manager.active_agents assert manager._close_finalizer_task is not None @@ -503,6 +504,47 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout( break await asyncio.sleep(0) assert manager.active_agents == {} + assert await manager.close() is True + + +@pytest.mark.anyio +async def test_close_retains_agent_until_detached_subagent_converges( + monkeypatch, +) -> None: + """Agent cleanup 返回 False 时 manager 必须保留 agent 和共享记忆 owner。""" + manager = AgentManager() + memory_manager = MemoryManager() + monkeypatch.setattr(agent_module, "memory_manager", memory_manager) + + class PendingSubagentOwner: + """首次清理未收敛、第二次清理成功的最小 Agent 替身。""" + + def __init__(self) -> None: + """初始化封口计数与可重试清理结果。""" + self.seal_count = 0 + self.cleanup_results = iter((False, True)) + + def begin_shutdown(self) -> None: + """记录 manager 在首个 await 前封住了子代理提交。""" + self.seal_count += 1 + + async def cleanup(self) -> bool: + """按测试序列返回 detached owner 的收敛状态。""" + return next(self.cleanup_results) + + owner = PendingSubagentOwner() + await manager.initialize() + manager.active_agents["detached-owner"] = owner + + assert await manager.close() is False + assert manager.active_agents == {"detached-owner": owner} + assert owner.seal_count == 1 + assert memory_manager.cleanup_task is not None + + assert await manager.close() is True + assert manager.active_agents == {} + assert owner.seal_count == 2 + assert memory_manager.cleanup_task is None @pytest.mark.anyio diff --git a/tests/test_agent_plugin_tools.py b/tests/test_agent_plugin_tools.py index 9f94f4747..94a3ea85c 100644 --- a/tests/test_agent_plugin_tools.py +++ b/tests/test_agent_plugin_tools.py @@ -4,7 +4,12 @@ from types import SimpleNamespace from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch -from app.agent.tools.impl._plugin_tool_utils import install_plugin_runtime +import pytest + +from app.agent.tools.impl._plugin_tool_utils import ( + install_plugin_runtime, + uninstall_plugin_runtime, +) from app.agent.tools.impl.install_plugin import InstallPluginTool from app.agent.tools.impl.query_installed_plugins import QueryInstalledPluginsTool from app.agent.tools.impl.query_market_plugins import QueryMarketPluginsTool @@ -14,6 +19,10 @@ from app.agent.tools.impl.reload_plugin import ReloadPluginTool from app.schemas.plugin import PluginRuntimeStatus from app.agent.tools.impl.uninstall_plugin import UninstallPluginTool from app.agent.tools.impl.update_plugin_config import UpdatePluginConfigTool +from app.runtime.extensions.plugin.admission import ( + PluginMutationAdmission, + PluginMutationRejectedError, +) def _plugin_snapshot(state: bool = True) -> dict: @@ -334,8 +343,7 @@ def test_install_plugin_runtime_reloads_in_threadpool() -> None: ) assert len(calls) == 2 assert calls[0][0] == "plugin" - assert calls[0][1] == plugin_manager.reload_plugin_tree - assert calls[0][2] == ("DemoPlugin",) + assert calls[0][2] == (plugin_manager.reload_plugin_tree, "DemoPlugin") assert calls[0][3] == {} assert calls[1][0] == "plugin" assert calls[1][1] == refresh_registrations @@ -372,6 +380,32 @@ def test_uninstall_plugin_uninstalls_installed_candidate() -> None: uninstall_runtime.assert_awaited_once_with("DemoPlugin") +def test_sealed_agent_uninstall_rejects_before_persistence() -> None: + """Agent 卸载未获 admission 时不读取实例或修改安装清单。""" + admission = PluginMutationAdmission() + admission.seal() + plugin_manager = MagicMock() + plugin_manager.mutation.side_effect = admission.hold + config_oper = MagicMock() + + with ( + patch( + "app.agent.tools.impl._plugin_tool_utils.get_plugin_manager", + return_value=plugin_manager, + ), + patch( + "app.agent.tools.impl._plugin_tool_utils.SystemConfigOper", + return_value=config_oper, + ) as config_provider, + pytest.raises(PluginMutationRejectedError), + ): + asyncio.run(uninstall_plugin_runtime("DemoPlugin")) + + plugin_manager.get_plugin_instance.assert_not_called() + config_provider.assert_not_called() + config_oper.async_set.assert_not_called() + + def test_query_plugin_data_truncates_large_payload() -> None: """ 查询插件数据会截断超长内容并返回预览。 diff --git a/tests/test_agent_runtime_loader.py b/tests/test_agent_runtime_loader.py index 2cb7fa57a..cee62eead 100644 --- a/tests/test_agent_runtime_loader.py +++ b/tests/test_agent_runtime_loader.py @@ -42,6 +42,7 @@ class _FakeManager: self.fail_initialize = False self.initialize_entered: asyncio.Event | None = None self.initialize_release: asyncio.Event | None = None + self.close_converged: bool | None = None async def initialize(self) -> None: self.initialize_calls += 1 @@ -52,8 +53,42 @@ class _FakeManager: if self.fail_initialize: raise RuntimeError("service initialization failed") - async def close(self) -> None: + async def close(self) -> bool | None: self.close_calls += 1 + return self.close_converged + + +@pytest.mark.anyio +async def test_shutdown_retains_nonconverged_agent_service_for_retry( + runtime_loader, + monkeypatch, +) -> None: + """Manager 返回未收敛时 Runtime 必须保留 owner，后续关闭可继续等待。""" + manager = _FakeManager() + manager.close_converged = False + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + assert await runtime_loader.activate_agent_service() is manager + + assert await runtime_loader.begin_agent_shutdown() is False + snapshot = runtime_loader._agent_runtime.snapshot("agent.service") + assert snapshot.lifecycle is CapabilityLifecycleState.FAILED + assert snapshot.visible is False + + manager.close_converged = True + assert await runtime_loader.begin_agent_shutdown() is True + snapshot = runtime_loader._agent_runtime.snapshot("agent.service") + assert snapshot.lifecycle is CapabilityLifecycleState.STOPPED + assert manager.close_calls == 2 def _fake_agent_modules(manager: object | None = None) -> dict[str, types.ModuleType]: diff --git a/tests/test_agent_side_effect_boundaries.py b/tests/test_agent_side_effect_boundaries.py index 0ff7194a8..4b557fc82 100644 --- a/tests/test_agent_side_effect_boundaries.py +++ b/tests/test_agent_side_effect_boundaries.py @@ -284,7 +284,7 @@ async def test_agent_cleanup_closes_subagent_middlewares() -> None: agent._subagent_middlewares = (_Middleware(),) - await agent.cleanup() + assert await agent.cleanup() is True assert closed == [True] assert agent._subagent_middlewares == () @@ -345,16 +345,16 @@ def test_subagent_control_middleware_close_is_idempotent() -> None: middleware._tasks = {} async def _close() -> None: - await middleware.close() - await middleware.close() + assert await middleware.close() is True + assert await middleware.close() is True asyncio.run(_close()) assert middleware._tasks == {} @pytest.mark.anyio -async def test_subagent_close_has_bounded_cancel_wait() -> None: - """子代理忽略首次取消时，控制器关闭仍必须在上限内返回。""" +async def test_subagent_close_retains_stubborn_owner_until_retry() -> None: + """子代理忽略取消时 close 返回 False，任务结束后重试才清理记录。""" middleware = object.__new__(SubAgentTaskControlMiddleware) release = asyncio.Event() cancelled = asyncio.Event() @@ -383,14 +383,57 @@ async def test_subagent_close_has_bounded_cancel_wait() -> None: "app.agent.middleware.subagents.SUBAGENT_CANCEL_GRACE_SECONDS", 0.01, ): - await asyncio.wait_for(middleware.close(), timeout=0.2) + assert await asyncio.wait_for(middleware.close(), timeout=0.2) is False assert cancelled.is_set() assert task.done() is False - assert middleware._tasks == {} + assert middleware._tasks == {record.task_id: record} + assert task in middleware._close_cancel_requested release.set() await asyncio.wait_for(task, timeout=0.2) + assert await middleware.close() is True + assert middleware._tasks == {} + assert middleware._close_cancel_requested == set() + + +@pytest.mark.anyio +async def test_subagent_seal_rejects_new_detached_task() -> None: + """控制器封口后必须同步拒绝 start，且不能创建新的 asyncio Task。""" + middleware = object.__new__(SubAgentTaskControlMiddleware) + middleware._tasks = {} + middleware._accepting_tasks = True + middleware._close_cancel_requested = set() + + middleware.seal() + payload = await middleware._control_task( + action="start", + description="late task", + ) + + result = json.loads(payload) + assert result["success"] is False + assert "正在关闭" in result["error"] + assert middleware._tasks == {} + + +@pytest.mark.anyio +async def test_agent_cleanup_retains_nonconverged_subagent_middleware() -> None: + """Agent cleanup 必须保留返回 False 的 middleware，供重复调用继续收口。""" + agent = MoviePilotAgent(session_id="session-owner", user_id="user-owner") + middleware = SimpleNamespace( + seal=MagicMock(), + close=AsyncMock(side_effect=[False, True]), + ) + agent._subagent_middlewares = (middleware,) + + assert await agent.cleanup() is False + assert agent._subagent_middlewares == (middleware,) + + assert await agent.cleanup() is True + assert agent._subagent_middlewares == () + assert middleware.seal.call_count == 2 + assert middleware.close.await_count == 2 @pytest.mark.anyio diff --git a/tests/test_agent_tool_timeouts.py b/tests/test_agent_tool_timeouts.py index fb48b7f92..ed9c67119 100644 --- a/tests/test_agent_tool_timeouts.py +++ b/tests/test_agent_tool_timeouts.py @@ -8,6 +8,10 @@ from app.agent.tools.base import ( MoviePilotTool, ToolExecutionTimeoutError, _blocking_executors, + _blocking_futures, + _blocking_retiring_executors, + close_blocking_executors, + reopen_blocking_executors, shutdown_blocking_executors, ) from app.agent.tools.manager import MoviePilotToolsManager @@ -37,6 +41,15 @@ class BlockingAgentTool(MoviePilotTool): return "unused" +@pytest.fixture(autouse=True) +def _reset_blocking_executor_runtime(): + """每个用例前后恢复阻塞池门禁，避免进程级 owner 状态串扰。""" + assert reopen_blocking_executors() is True + yield + assert shutdown_blocking_executors(cancel_futures=True) is True + assert reopen_blocking_executors() is True + + def test_arun_raises_timeout_when_tool_exceeds_limit(): """底层工具入口应把超时交给宿主策略记录失败终态。""" tool = SlowAgentTool(session_id="session-1", user_id="10001") @@ -126,6 +139,8 @@ def test_shutdown_blocking_executors_clears_agent_tool_workers(): shutdown_blocking_executors() assert _blocking_executors == {} + assert _blocking_futures == {} + assert _blocking_retiring_executors == set() def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent(): @@ -159,10 +174,55 @@ def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent(): queued_future = asyncio.run(_run_scenario()) assert _blocking_executors == {} + assert _blocking_futures == {} + assert _blocking_retiring_executors == set() assert queued_future.cancelled() assert not queued_ran.is_set() +@pytest.mark.asyncio +async def test_close_blocking_executors_retains_owner_until_retry() -> None: + """同步调用超时后保留 Future/executor，完成后的重复 close 才成功。""" + started = threading.Event() + release = threading.Event() + + def _blocking_call() -> str: + """等待测试释放，稳定制造超过关停预算的运行 Future。""" + started.set() + release.wait() + return "done" + + task = asyncio.create_task( + MoviePilotTool.run_blocking("web", _blocking_call) + ) + assert await asyncio.to_thread(started.wait, 1) + + try: + assert await close_blocking_executors( + timeout_seconds=0.01, + cancel_futures=True, + ) is False + assert task.done() is False + assert _blocking_futures + assert _blocking_retiring_executors + + with pytest.raises(RuntimeError, match="正在关闭"): + await MoviePilotTool.run_blocking("web", lambda: "late") + + release.set() + assert await asyncio.wait_for(task, timeout=1) == "done" + assert await close_blocking_executors( + timeout_seconds=0.01, + cancel_futures=True, + ) is True + assert _blocking_futures == {} + assert _blocking_retiring_executors == set() + finally: + release.set() + if not task.done(): + await asyncio.wait_for(task, timeout=1) + + def test_create_agent_config_uses_llm_max_iterations(): """Agent 执行配置应把 LLM_MAX_ITERATIONS 传给 LangGraph recursion_limit。""" from app.agent.orchestrator import MoviePilotAgent diff --git a/tests/test_event_dispatch_snapshot.py b/tests/test_event_dispatch_snapshot.py index 4630b74f2..265fee96e 100644 --- a/tests/test_event_dispatch_snapshot.py +++ b/tests/test_event_dispatch_snapshot.py @@ -1,7 +1,9 @@ """事件调度订阅快照和生命周期回归测试。""" import asyncio +import concurrent.futures import threading +from queue import PriorityQueue import pytest @@ -16,12 +18,49 @@ class _ImmediateExecutor: @staticmethod def submit(func, *args, **kwargs): - return func(*args, **kwargs) + """立即执行调用并返回符合线程池接口的已完成 Future。""" + handle = concurrent.futures.Future() + try: + handle.set_result(func(*args, **kwargs)) + except Exception as err: + handle.set_exception(err) + return handle + + +class _DaemonThreadExecutor: + """以守护线程执行回调，让死锁回归测试能够有界失败。""" + + def __init__(self) -> None: + """初始化可观察的回调完成信号。""" + self.finished = threading.Event() + + def submit(self, func, *args, **kwargs): + """在线程中执行调用并返回符合线程池接口的 Future。""" + handle = concurrent.futures.Future() + + def run() -> None: + """执行回调并把结果或异常写入 Future。""" + if not handle.set_running_or_notify_cancel(): + return + try: + handle.set_result(func(*args, **kwargs)) + except BaseException as err: # pragma: no cover - 交由 Future 消费 + handle.set_exception(err) + finally: + self.finished.set() + + threading.Thread(target=run, daemon=True).start() + return handle @pytest.fixture def isolated_eventmanager(monkeypatch): """隔离全局事件总线的订阅表和广播执行器。""" + monkeypatch.setattr( + global_vars, + "CURRENT_EVENT_LOOP", + global_vars.CURRENT_EVENT_LOOP, + ) monkeypatch.setattr( eventmanager, "_EventManager__broadcast_subscribers", @@ -37,6 +76,16 @@ def isolated_eventmanager(monkeypatch): "_EventManager__handler_instance_resolvers", {}, ) + monkeypatch.setattr( + eventmanager, + "_EventManager__disabled_handlers", + set(), + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__disabled_classes", + set(), + ) monkeypatch.setattr( eventmanager, "_EventManager__executor", @@ -47,6 +96,11 @@ def isolated_eventmanager(monkeypatch): "_EventManager__event", threading.Event(), ) + monkeypatch.setattr( + eventmanager, + "_EventManager__event_queue", + PriorityQueue(), + ) monkeypatch.setattr( eventmanager, "_EventManager__consumer_threads", @@ -57,6 +111,11 @@ def isolated_eventmanager(monkeypatch): "_EventManager__lifecycle_state", "new", ) + monkeypatch.setattr( + eventmanager, + "_EventManager__sync_handles", + {}, + ) monkeypatch.setattr( eventmanager, "_EventManager__async_handles", @@ -68,6 +127,7 @@ def isolated_eventmanager(monkeypatch): def test_broadcast_dispatch_uses_subscription_snapshot(isolated_eventmanager): """广播事件中新增或移除的 handler 从下一个事件开始生效。""" calls = [] + isolated_eventmanager._EventManager__lifecycle_state = "running" def late_handler(_event): calls.append("late") @@ -276,6 +336,354 @@ async def test_async_broadcast_shutdown_waits_for_handler_cleanup( assert isolated_eventmanager._EventManager__lifecycle_state == "stopped" +@pytest.mark.asyncio +async def test_drain_waits_for_slow_sync_broadcast_handler( + isolated_eventmanager, + monkeypatch, +) -> None: + """同步广播处理器返回前 drain 不得提前宣称结算完成。""" + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + monkeypatch.setattr( + isolated_eventmanager, + "_EventManager__executor", + executor, + ) + started = threading.Event() + release = threading.Event() + + def handler(_event) -> None: + started.set() + release.wait(timeout=2) + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"drain-sync"}}, + ) + assert await asyncio.to_thread(started.wait, 1) + assert len(isolated_eventmanager._EventManager__sync_handles) == 1 + + drain_task = asyncio.create_task( + isolated_eventmanager.drain_async(timeout=1) + ) + await asyncio.sleep(0.02) + assert not drain_task.done() + + release.set() + assert await drain_task is True + assert isolated_eventmanager._EventManager__sync_handles == {} + assert ( + isolated_eventmanager._EventManager__event_queue.unfinished_tasks + == 0 + ) + finally: + release.set() + await isolated_eventmanager.stop_async() + executor.shutdown(wait=True) + + +@pytest.mark.asyncio +async def test_drain_timeout_does_not_cancel_async_broadcast_handler( + isolated_eventmanager, +) -> None: + """drain 超时只报告未收敛，不取消仍在执行的异步业务处理器。""" + global_vars.set_loop(asyncio.get_running_loop()) + started = asyncio.Event() + release = asyncio.Event() + cancelled = False + + async def handler(_event) -> None: + nonlocal cancelled + started.set() + try: + await release.wait() + except asyncio.CancelledError: + cancelled = True + raise + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"drain-async"}}, + ) + await asyncio.wait_for(started.wait(), timeout=1) + + assert await isolated_eventmanager.drain_async(timeout=0.01) is False + assert cancelled is False + assert isolated_eventmanager._EventManager__async_handles + + release.set() + assert await isolated_eventmanager.drain_async(timeout=1) is True + assert cancelled is False + finally: + release.set() + await isolated_eventmanager.stop_async() + + +@pytest.mark.asyncio +async def test_sealed_drain_waits_for_cascade_and_rejects_later_broadcasts( + isolated_eventmanager, +) -> None: + """封口 drain 应结算 handler 派生事件，并拒绝封口后的新广播。""" + calls = [] + + def first_handler(_event) -> None: + calls.append("first") + isolated_eventmanager.send_event(EventType.ModuleReload, {}) + + def cascaded_handler(_event) -> None: + calls.append("cascaded") + + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + first_handler, + ) + isolated_eventmanager.add_event_listener( + EventType.ModuleReload, + cascaded_handler, + ) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"cascade"}}, + ) + assert await isolated_eventmanager.drain_async( + timeout=1, + seal=True, + ) is True + assert calls == ["first", "cascaded"] + queue_size = isolated_eventmanager._EventManager__event_queue.qsize() + + isolated_eventmanager.send_event(EventType.ModuleReload, {}) + await asyncio.sleep(0.02) + + assert calls == ["first", "cascaded"] + assert ( + isolated_eventmanager._EventManager__event_queue.qsize() + == queue_size + ) + assert isolated_eventmanager._EventManager__lifecycle_state == "sealed" + finally: + await isolated_eventmanager.stop_async() + + +@pytest.mark.asyncio +async def test_drain_returns_false_when_stop_interleaves( + isolated_eventmanager, + monkeypatch, +) -> None: + """stop 开始后 drain 必须报告屏障失败，而非等待残留队列后返回成功。""" + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + monkeypatch.setattr( + isolated_eventmanager, + "_EventManager__executor", + executor, + ) + started = threading.Event() + release = threading.Event() + + def handler(_event) -> None: + started.set() + release.wait(timeout=2) + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"stop-race"}}, + ) + assert await asyncio.to_thread(started.wait, 1) + drain_task = asyncio.create_task(isolated_eventmanager.drain_async()) + stop_task = asyncio.create_task(isolated_eventmanager.stop_async()) + + assert await asyncio.wait_for(drain_task, timeout=1) is False + assert not stop_task.done() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + assert isolated_eventmanager._EventManager__sync_handles == {} + finally: + release.set() + if isolated_eventmanager._EventManager__lifecycle_state != "stopped": + await isolated_eventmanager.stop_async() + executor.shutdown(wait=True) + + +@pytest.mark.asyncio +async def test_repeated_async_stop_balances_stop_sentinels( + isolated_eventmanager, +) -> None: + """重复异步停止不得遗留哨兵或破坏队列 unfinished_tasks 计数。""" + isolated_eventmanager.start() + + await asyncio.gather( + isolated_eventmanager.stop_async(), + isolated_eventmanager.stop_async(), + ) + await isolated_eventmanager.stop_async() + + assert isolated_eventmanager._EventManager__event_queue.qsize() == 0 + assert isolated_eventmanager._EventManager__event_queue.unfinished_tasks == 0 + + +@pytest.mark.asyncio +async def test_sync_stop_waits_for_owned_sync_handler( + isolated_eventmanager, + monkeypatch, +) -> None: + """同步兼容停止入口也不得遗留仍运行的同步广播处理器。""" + executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + monkeypatch.setattr( + isolated_eventmanager, + "_EventManager__executor", + executor, + ) + started = threading.Event() + release = threading.Event() + + def handler(_event) -> None: + started.set() + release.wait(timeout=2) + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"sync-stop"}}, + ) + assert await asyncio.to_thread(started.wait, 1) + stop_task = asyncio.create_task( + asyncio.to_thread(isolated_eventmanager.stop) + ) + await asyncio.sleep(0.02) + assert not stop_task.done() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + assert isolated_eventmanager._EventManager__sync_handles == {} + assert ( + isolated_eventmanager._EventManager__event_queue.unfinished_tasks + == 0 + ) + finally: + release.set() + if isolated_eventmanager._EventManager__lifecycle_state != "stopped": + await isolated_eventmanager.stop_async() + executor.shutdown(wait=True) + + +def test_sync_broadcast_handler_can_call_legacy_stop_without_self_wait( + isolated_eventmanager, + monkeypatch, +) -> None: + """同步 handler 调用旧 stop() 时不得等待承载自己的 Future。""" + executor = _DaemonThreadExecutor() + monkeypatch.setattr( + isolated_eventmanager, + "_EventManager__executor", + executor, + ) + stopped = threading.Event() + + def handler(_event) -> None: + isolated_eventmanager.stop() + stopped.set() + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"handler-stop"}}, + ) + + assert stopped.wait(timeout=1) + assert executor.finished.wait(timeout=1) + assert isolated_eventmanager._EventManager__sync_handles == {} + assert isolated_eventmanager._EventManager__lifecycle_state == "stopped" + finally: + if isolated_eventmanager._EventManager__lifecycle_state == "running": + consumer_threads = isolated_eventmanager._EventManager__begin_stop() + for consumer_thread in consumer_threads: + consumer_thread.join(timeout=1) + isolated_eventmanager._EventManager__discard_stop_sentinels() + + +@pytest.mark.asyncio +async def test_async_broadcast_handler_can_await_stop_without_self_cancel( + isolated_eventmanager, +) -> None: + """异步 handler 调用 stop_async() 时不得取消或等待自身 completion。""" + global_vars.set_loop(asyncio.get_running_loop()) + stopped = asyncio.Event() + cancelled = False + + async def handler(_event) -> None: + nonlocal cancelled + try: + await isolated_eventmanager.stop_async() + stopped.set() + except asyncio.CancelledError: + cancelled = True + raise + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"async-handler-stop"}}, + ) + + await asyncio.wait_for(stopped.wait(), timeout=1) + + async def wait_for_owner_release() -> None: + """等待 handler 返回后的 completion 回调移除 owner。""" + while isolated_eventmanager._EventManager__async_handles: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_owner_release(), timeout=1) + assert cancelled is False + assert isolated_eventmanager._EventManager__lifecycle_state == "stopped" + + +@pytest.mark.asyncio +async def test_broadcast_handler_cannot_seal_while_own_completion_is_pending( + isolated_eventmanager, +) -> None: + """handler 内 drain 应立即失败，且不能把仍运行的事件总线伪装成已封口。""" + global_vars.set_loop(asyncio.get_running_loop()) + drain_results = [] + handler_done = asyncio.Event() + + async def handler(_event) -> None: + drain_results.append( + await isolated_eventmanager.drain_async(seal=True) + ) + handler_done.set() + + isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler) + isolated_eventmanager.start() + try: + isolated_eventmanager.send_event( + EventType.ConfigChanged, + {"key": {"handler-drain"}}, + ) + await asyncio.wait_for(handler_done.wait(), timeout=1) + + assert drain_results == [False] + assert isolated_eventmanager._EventManager__lifecycle_state == "running" + assert await isolated_eventmanager.drain_async(timeout=1) is True + finally: + await isolated_eventmanager.stop_async() + + @pytest.mark.asyncio async def test_async_broadcast_submission_is_registered_before_stop_snapshot( isolated_eventmanager, diff --git a/tests/test_lifecycle_shutdown.py b/tests/test_lifecycle_shutdown.py index 87e7fe811..056802c62 100644 --- a/tests/test_lifecycle_shutdown.py +++ b/tests/test_lifecycle_shutdown.py @@ -8,6 +8,7 @@ from fastapi import FastAPI from app.startup import lifecycle, modules_initializer from app.adapters.network import http as http_utils +from app.runtime.tasks import get_task_registry def _assert_completed_once(mock: MagicMock) -> None: @@ -59,11 +60,16 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict: shutdown_steps = { "backup_plugins": system_chain.backup_plugins, + "stop_plugin_monitor": MagicMock(return_value=True), "stop_workflow": MagicMock(), "stop_command": MagicMock(), "stop_monitor": MagicMock(), "stop_scheduler": MagicMock(), - "stop_plugins": MagicMock(), + "stop_agent": AsyncMock(return_value=True), + "stop_transfer": AsyncMock(return_value=True), + "quiesce_plugins": AsyncMock(return_value=True), + "drain_events": AsyncMock(return_value=True), + "finalize_plugins": MagicMock(return_value=True), "stop_modules": AsyncMock(), "close_http": AsyncMock(), } @@ -72,9 +78,22 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict: "stop_command", "stop_monitor", "stop_scheduler", - "stop_plugins", + "stop_plugin_monitor", + "finalize_plugins", ): monkeypatch.setattr(lifecycle, name, shutdown_steps[name]) + monkeypatch.setattr(lifecycle, "stop_agent", shutdown_steps["stop_agent"]) + monkeypatch.setattr( + lifecycle, + "stop_transfer_runtime", + shutdown_steps["stop_transfer"], + ) + monkeypatch.setattr( + lifecycle, + "quiesce_plugins", + shutdown_steps["quiesce_plugins"], + ) + monkeypatch.setattr(lifecycle, "drain_events", shutdown_steps["drain_events"]) monkeypatch.setattr(lifecycle, "stop_modules", shutdown_steps["stop_modules"]) monkeypatch.setattr( lifecycle, @@ -99,9 +118,6 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict: "backup_plugins", "stop_workflow", "stop_command", - "stop_monitor", - "stop_scheduler", - "stop_plugins", "stop_modules", "close_http", ], @@ -153,8 +169,258 @@ def test_lifespan_normal_mode_starts_full_runtime(monkeypatch): _assert_completed_once(step) -def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch): - """关停必须等待插件恢复线程结束，避免与备份和资源释放并发。""" +@pytest.mark.parametrize( + ("failing_step", "completed_steps", "blocked_steps"), + [ + ( + "stop_plugin_monitor", + ("stop_plugin_monitor",), + ( + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ), + ), + ( + "stop_monitor", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + ), + ( + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ), + ), + ( + "stop_scheduler", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + ), + ( + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ), + ), + ( + "stop_agent", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + ), + ( + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ), + ), + ( + "quiesce_plugins", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + ), + ( + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ), + ), + ( + "stop_transfer", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + ), + ("drain_events", "finalize_plugins", "stop_modules", "close_http"), + ), + ( + "drain_events", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + ), + ("finalize_plugins", "stop_modules", "close_http"), + ), + ( + "finalize_plugins", + ( + "stop_plugin_monitor", + "backup_plugins", + "stop_workflow", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + ), + ("stop_modules", "close_http"), + ), + ], +) +def test_lifespan_stops_releasing_dependencies_when_owner_does_not_converge( + monkeypatch, + failing_step, + completed_steps, + blocked_steps, +): + """关键 owner 未收敛时不得关闭仍被活任务使用的后续依赖。""" + shutdown_steps = _patch_lifespan(monkeypatch) + shutdown_steps[failing_step].return_value = False + + async def run_lifespan(): + """启动并关闭隔离后的应用生命周期。""" + async with lifecycle.lifespan(FastAPI()): + pass + + asyncio.run(run_lifespan()) + + for name in (*completed_steps, "logger"): + _assert_completed_once(shutdown_steps[name]) + for name in blocked_steps: + shutdown_steps[name].assert_not_called() + + +def test_task_registry_nonconvergence_blocks_all_dependency_release(monkeypatch): + """最前置任务 owner 超时后不得继续释放插件、模块或 HTTP 依赖。""" + shutdown_steps = _patch_lifespan(monkeypatch) + shutdown = AsyncMock(return_value=False) + monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown) + app = FastAPI() + + async def run_lifespan() -> None: + """运行后台登记器无法收敛的隔离生命周期。""" + async with lifecycle.lifespan(app): + pass + + asyncio.run(run_lifespan()) + + shutdown.assert_awaited_once_with(timeout_seconds=30.0) + for name, step in shutdown_steps.items(): + if name == "logger": + _assert_completed_once(step) + else: + step.assert_not_called() + assert isinstance(app.state.task_registry, lifecycle.TaskRegistry) + + +def test_closed_task_registry_rejects_late_shutdown_tasks(monkeypatch) -> None: + """首屏障完成后，后续 stop hook 的晚到任务不得落回默认登记器。""" + _patch_lifespan(monkeypatch) + app = FastAPI() + + async def run_lifespan() -> None: + """结束完整 lifespan 后验证当前发布的仍是已封口登记器。""" + async with lifecycle.lifespan(app): + pass + registry = get_task_registry() + assert registry is app.state.task_registry + with pytest.raises(RuntimeError, match="正在关闭"): + registry.create(asyncio.sleep(0), owner="shutdown.late_task") + + asyncio.run(run_lifespan()) + + +def test_plugin_settlement_cannot_bypass_task_registry_shutdown_budget( + monkeypatch, +) -> None: + """未收敛 settlement 必须交给首屏障判定，lifespan 不得提前无界等待。""" + shutdown_steps = _patch_lifespan(monkeypatch) + shutdown = AsyncMock(return_value=False) + monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown) + started = asyncio.Event() + release = asyncio.Event() + + async def settle_plugins() -> None: + """模拟停机时仍未结束的插件同步任务。""" + started.set() + await release.wait() + + lifecycle.init_extra.side_effect = settle_plugins + + async def run_lifespan() -> None: + """确认 context 能由 TaskRegistry 的失败结果立即结束。""" + async with lifecycle.lifespan(FastAPI()): + await started.wait() + release.set() + await asyncio.sleep(0) + + asyncio.run(asyncio.wait_for(run_lifespan(), timeout=0.5)) + + shutdown.assert_awaited_once_with(timeout_seconds=30.0) + for name, step in shutdown_steps.items(): + if name == "logger": + _assert_completed_once(step) + else: + step.assert_not_called() + + +def test_lifespan_waits_for_uncancellable_plugin_settlement_before_shutdown( + monkeypatch, +): + """已进入同步 I/O 的 settlement 必须真实结束，才能备份和释放资源。""" shutdown_steps = _patch_lifespan(monkeypatch) order = [] shutdown_steps["backup_plugins"].side_effect = lambda: order.append("backup") @@ -165,7 +431,12 @@ def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch): async def settle_plugins(): started.set() - await release.wait() + try: + await release.wait() + except asyncio.CancelledError: + # 模拟 run_in_threadpool_to_completion：外层取消只能封住新工作， + # 已进入同步插件源码/依赖修改的调用仍持有 owner 到真实终态。 + await release.wait() order.append("settled") lifecycle.init_extra.side_effect = settle_plugins @@ -225,14 +496,31 @@ def test_lifespan_safe_mode_skips_optional_runtime(monkeypatch): "stop_command", "stop_monitor", "stop_scheduler", - "stop_plugins", + "stop_plugin_monitor", + "quiesce_plugins", + "finalize_plugins", ): shutdown_steps[name].assert_not_called() _assert_completed_once(shutdown_steps["stop_modules"]) + _assert_completed_once(shutdown_steps["stop_agent"]) + _assert_completed_once(shutdown_steps["stop_transfer"]) + _assert_completed_once(shutdown_steps["drain_events"]) _assert_completed_once(shutdown_steps["close_http"]) _assert_completed_once(shutdown_steps["logger"]) +@pytest.mark.asyncio +async def test_event_drain_does_not_materialize_manager(monkeypatch) -> None: + """模块尚未创建事件总线时，停机屏障应直接收敛而不反向构造。""" + event_manager_type = MagicMock() + event_manager_type.get_existing_instance.return_value = None + monkeypatch.setattr(modules_initializer, "EventManager", event_manager_type) + + assert await modules_initializer.drain_events() is True + event_manager_type.get_existing_instance.assert_called_once_with() + event_manager_type.assert_not_called() + + def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None: """组件清单应显式冻结依赖、模式、启动/关闭顺序和超时预算。""" app = FastAPI() @@ -274,11 +562,16 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None: ] assert normal_stop == [ "后台任务登记器", + "插件变更监控", "插件备份", "工作流", "命令服务", "监控器", "定时器", + "AI智能体会话", + "插件后台服务", + "整理后台服务", + "事件投递屏障", "插件", "模块服务", "HTTP 基础能力", @@ -292,9 +585,42 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None: "数据库连接预算", "路由", "模块服务", + "AI智能体会话", + "整理后台服务", + "事件投递屏障", } assert all(item["start_failure"] == "fail_fast" for item in normal) - assert all(item["stop_failure"] == "continue" for item in normal) + assert { + item["name"] + for item in normal + if item["stop_failure"] == "fail_fast" + } == { + "插件变更监控", + "后台任务登记器", + "监控器", + "定时器", + "AI智能体会话", + "整理后台服务", + "插件后台服务", + "事件投递屏障", + "插件", + } + assert all( + item["stop_failure"] == "continue" + for item in normal + if item["name"] + not in { + "插件变更监控", + "后台任务登记器", + "监控器", + "定时器", + "AI智能体会话", + "整理后台服务", + "插件后台服务", + "事件投递屏障", + "插件", + } + ) assert all( item["start_timeout_seconds"] or item["stop_timeout_seconds"] for item in normal @@ -496,6 +822,80 @@ def test_lifespan_does_not_yield_after_migration_failure(monkeypatch): assert app.state.moviepilot_health.phase.value == "failed" +def test_lifespan_cleans_started_owners_after_late_startup_failure(monkeypatch): + """后段启动失败时应按同一停机策略回收已启动及部分启动的 owner。""" + shutdown_steps = _patch_lifespan(monkeypatch) + startup_error = RuntimeError("command startup failed") + lifecycle.init_command.side_effect = startup_error + app = FastAPI() + + async def run_lifespan() -> None: + """运行一个在命令服务阶段失败的隔离生命周期。""" + async with lifecycle.lifespan(app): + pytest.fail("命令服务启动失败后不应发布运行态") + + with pytest.raises(RuntimeError) as raised: + asyncio.run(run_lifespan()) + + assert raised.value is startup_error + lifecycle.global_vars.stop_system.assert_not_called() + for name in ( + "stop_plugin_monitor", + "backup_plugins", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ): + _assert_completed_once(shutdown_steps[name]) + shutdown_steps["stop_workflow"].assert_not_called() + shutdown_steps["logger"].assert_not_called() + assert isinstance(app.state.task_registry, lifecycle.TaskRegistry) + assert get_task_registry() is app.state.task_registry + assert app.state.moviepilot_health.phase.value == "failed" + + +def test_startup_failure_cleanup_honors_transfer_fail_fast(monkeypatch): + """启动失败清理中整理 owner 未收敛时也不得继续释放插件和模块。""" + shutdown_steps = _patch_lifespan(monkeypatch) + lifecycle.init_command.side_effect = RuntimeError("command startup failed") + shutdown_steps["stop_transfer"].return_value = False + + async def run_lifespan() -> None: + """运行后段失败且整理线程无法收敛的隔离生命周期。""" + async with lifecycle.lifespan(FastAPI()): + pytest.fail("命令服务启动失败后不应发布运行态") + + with pytest.raises(RuntimeError, match="command startup failed"): + asyncio.run(run_lifespan()) + + for name in ( + "stop_plugin_monitor", + "backup_plugins", + "stop_command", + "stop_monitor", + "stop_scheduler", + "stop_agent", + "quiesce_plugins", + "stop_transfer", + ): + _assert_completed_once(shutdown_steps[name]) + for name in ( + "drain_events", + "finalize_plugins", + "stop_modules", + "close_http", + ): + shutdown_steps[name].assert_not_called() + shutdown_steps["stop_workflow"].assert_not_called() + + def test_uvicorn_preserves_stop_requested_before_serve(monkeypatch): """Uvicorn 启动不能清除数据库初始化阶段已经发布的停止请求""" from app import main @@ -564,16 +964,13 @@ def test_command_restart_failure_does_not_publish_stop_request(monkeypatch): assert not stop_event.is_set() -def test_stop_modules_continues_after_internal_owner_failures(monkeypatch): - """模块关闭编排中的多个失败不能阻断其余清理""" - stop_agent = AsyncMock(side_effect=RuntimeError("agent failed")) - monkeypatch.setattr(modules_initializer, "stop_agent", stop_agent) +def test_stop_modules_continues_after_internal_owner_failure(monkeypatch): + """模块关闭编排中的单个失败不能阻断其余清理。""" dependencies = _patch_module_shutdown_dependencies(monkeypatch) dependencies["module"].side_effect = RuntimeError("module failed") asyncio.run(modules_initializer.stop_modules()) - stop_agent.assert_awaited_once_with() for dependency in dependencies.values(): _assert_completed_once(dependency) @@ -581,7 +978,6 @@ def test_stop_modules_continues_after_internal_owner_failures(monkeypatch): def test_stop_modules_drains_web_agent_tasks_before_persistence(monkeypatch): """关闭时先收口 Web Agent，再关闭持久化准入和数据库任务。""" order = [] - monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock()) dependencies = _patch_module_shutdown_dependencies(monkeypatch) monkeypatch.setattr( modules_initializer, @@ -619,7 +1015,6 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc started.set() await asyncio.Event().wait() - monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock()) _patch_module_shutdown_dependencies(monkeypatch) monkeypatch.setattr( modules_initializer, @@ -651,8 +1046,9 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc ) ) await started.wait() - await shutdown + completed = await shutdown + assert completed is False stop_database_worker.assert_awaited_once_with() @@ -683,9 +1079,10 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No ) ) await started.wait() - await shutdown + completed = await shutdown elapsed = asyncio.get_running_loop().time() - started_at + assert completed is False assert elapsed < 0.2 await asyncio.wait_for(cancel_requested.wait(), timeout=0.2) assert not settled.is_set() @@ -694,6 +1091,28 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No await asyncio.wait_for(settled.wait(), timeout=0.2) +@pytest.mark.asyncio +async def test_shutdown_step_reports_explicit_nonconvergence() -> None: + """同步和异步 owner 显式返回 False 时都必须向生命周期传播失败。""" + + async def async_nonconverging_shutdown() -> bool: + """模拟已经完成等待但仍持有资源的异步关闭入口。""" + return False + + assert await lifecycle.run_shutdown_step( + "同步 owner", + lambda: False, + ) is False + assert await lifecycle.run_shutdown_step( + "异步 owner", + async_nonconverging_shutdown, + ) is False + assert await lifecycle.run_shutdown_step( + "已收敛 owner", + lambda: None, + ) is True + + def _patch_module_shutdown_dependencies(monkeypatch) -> dict: """替换 stop_modules 的资源所有者，避免测试启动真实后台服务""" dependencies = {} @@ -749,7 +1168,6 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: def test_browser_sessions_close_before_managed_resources(monkeypatch) -> None: """显示等宿主资源必须晚于浏览器会话释放，避免存活上下文失去依赖。""" calls: list[str] = [] - monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock()) dependencies = _patch_module_shutdown_dependencies(monkeypatch) dependencies["close_browser_sessions"].side_effect = lambda: calls.append("browser") diff --git a/tests/test_monitor_gap_recovery.py b/tests/test_monitor_gap_recovery.py index 4c23abf4e..f3d28d878 100644 --- a/tests/test_monitor_gap_recovery.py +++ b/tests/test_monitor_gap_recovery.py @@ -1,14 +1,16 @@ import os +import threading import time from unittest.mock import MagicMock from watchfiles import Change from app.runtime.config import settings -from app.monitor import LocalDirectoryWatcher, Monitor from app.monitor.dispatcher import TransferDispatcher +from app.monitor.monitor import Monitor from app.monitor.recovery import RecoveryExecutor from app.monitor.syslimits import decide_monitor_mode +from app.monitor.watcher import LocalDirectoryWatcher from app.adapters.system.host import SystemUtils @@ -24,13 +26,24 @@ def _build_monitor(handle_file: MagicMock = None): if handle_file is not None: dispatcher.handle_file = handle_file monitor._dispatcher = dispatcher + monitor._lifecycle_lock = threading.RLock() + monitor._owner_lock = threading.Lock() + monitor._work_stop_event = threading.Event() + monitor._shutdown_event = threading.Event() + monitor._closed = False + monitor._compensation_threads = {} + monitor._scheduler_shutdown_thread = None + monitor._scheduler_shutdown_succeeded = False + monitor._scheduler = None monitor._watchers = [] + monitor._retired_watchers = [] monitor._watcher_lock = Lock() monitor._alerted_paths = {} monitor._restart_marks = {} monitor._stable_cycles = {} monitor._isolated = {} monitor._pending_rebuild = {} + monitor._pending_locals = [] monitor._recovery = RecoveryExecutor() return monitor, dispatcher diff --git a/tests/test_monitor_lifecycle.py b/tests/test_monitor_lifecycle.py new file mode 100644 index 000000000..5520d172c --- /dev/null +++ b/tests/test_monitor_lifecycle.py @@ -0,0 +1,345 @@ +"""目录监控 owner 的生命周期与停机屏障回归。""" + +import asyncio +import threading +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from app.monitor.monitor import Monitor +from app.monitor.recovery import RecoveryExecutor, RecoveryState +from app.foundation.singleton import SingletonClass +from app.startup.monitor_initializer import init_monitor, stop_monitor + + +def _build_monitor() -> Monitor: + """构造绕过单例初始化、但具备完整生命周期字段的 Monitor 骨架。""" + monitor = object.__new__(Monitor) + monitor._lifecycle_lock = threading.RLock() + monitor._owner_lock = threading.Lock() + monitor._work_stop_event = threading.Event() + monitor._shutdown_event = threading.Event() + monitor._closed = False + monitor._compensation_threads = {} + monitor._scheduler_shutdown_thread = None + monitor._scheduler_shutdown_succeeded = False + monitor._scheduler = None + monitor._watchers = [] + monitor._retired_watchers = [] + monitor._watcher_lock = threading.Lock() + monitor._pending_locals = [] + monitor._alerted_paths = {} + monitor._restart_marks = {} + monitor._stable_cycles = {} + monitor._isolated = {} + monitor._pending_rebuild = {} + monitor._recovery = RecoveryExecutor() + monitor._dispatcher = MagicMock() + return monitor + + +def _blocking_watcher(mon_path: Path): + """构造只有显式 release 后才退出的 watcher 与控制事件。""" + started = threading.Event() + release = threading.Event() + + def run() -> None: + """模拟无法被 stop event 唤醒的 FUSE watcher。""" + started.set() + release.wait() + + thread = threading.Thread(target=run, daemon=True, name="test-monitor-watcher") + thread.start() + assert started.wait(1) + watcher = MagicMock() + watcher.watch_path = mon_path + watcher.stop.side_effect = lambda: None + watcher.join.side_effect = thread.join + watcher.is_alive.side_effect = thread.is_alive + return watcher, thread, release + + +def test_close_budget_includes_lifecycle_lock_wait() -> None: + """停机 deadline 必须先于生命周期锁等待建立，锁竞争超时后立即返回。""" + monitor = _build_monitor() + lock_acquired = threading.Event() + release_lock = threading.Event() + + def hold_lifecycle_lock() -> None: + """占住生命周期锁，模拟配置重载阻塞在挂载访问中。""" + with monitor._lifecycle_lock: + lock_acquired.set() + release_lock.wait() + + holder = threading.Thread(target=hold_lifecycle_lock, daemon=True) + holder.start() + assert lock_acquired.wait(1) + + started_at = time.monotonic() + try: + assert monitor.close(timeout=0.02) is False + assert time.monotonic() - started_at < 0.5 + assert monitor.lifecycle_closed is True + finally: + release_lock.set() + holder.join(timeout=1) + assert holder.is_alive() is False + assert monitor.close(timeout=1) is True + + +def test_close_timeout_retains_watcher_and_seals_config_reload(tmp_path) -> None: + """挂死 watcher 超时时保留句柄，且配置变化不能重开已封口生命周期。""" + monitor = _build_monitor() + watcher, thread, release = _blocking_watcher(tmp_path) + monitor._watchers = [watcher] + reload_monitor = MagicMock() + monitor.init = reload_monitor + + try: + assert monitor.close(timeout=0.02) is False + assert monitor._watchers == [watcher] + monitor._dispatcher.clear_pending.assert_not_called() + + monitor.on_config_changed() + reload_monitor.assert_not_called() + finally: + release.set() + assert monitor.close(timeout=1) is True + thread.join(timeout=1) + assert thread.is_alive() is False + assert monitor._watchers == [] + monitor._dispatcher.clear_pending.assert_called_once_with() + + +def test_reopen_allows_only_an_explicit_new_lifespan() -> None: + """close 后普通 init 被拒绝，显式 reopen 收敛旧 owner 后才允许重新初始化。""" + monitor = _build_monitor() + initialize = MagicMock(return_value=True) + monitor._Monitor__initialize_monitors = initialize + + assert monitor.close(timeout=1) is True + assert monitor.init(timeout=0) is False + initialize.assert_not_called() + + assert monitor.reopen(timeout=1) is True + assert monitor.init(timeout=1) is True + initialize.assert_called_once_with() + assert monitor.lifecycle_closed is False + + +def test_close_tracks_recovery_and_blocks_post_seal_dispatch() -> None: + """恢复动作解冻后只能退出，不能越过封口继续派发整理重试。""" + monitor = _build_monitor() + entered = threading.Event() + release = threading.Event() + + def blocking_local_retry() -> None: + """模拟恢复线程阻塞在 FUSE 本地目录访问。""" + entered.set() + release.wait() + + monitor._Monitor__retry_pending_locals = blocking_local_retry + result = monitor._recovery.run( + {monitor.PENDING_KEY: monitor._Monitor__drive_pending}, timeout=0.01 + ) + assert entered.is_set() + assert result == {monitor.PENDING_KEY: RecoveryState.TIMEOUT} + + try: + assert monitor.close(timeout=0.02) is False + assert len(monitor._recovery.running_threads()) == 1 + monitor._dispatcher.retry_pending.assert_not_called() + finally: + release.set() + assert monitor.close(timeout=1) is True + assert monitor._recovery.running_threads() == () + monitor._dispatcher.retry_pending.assert_not_called() + + +def test_recovery_start_and_registration_are_atomic(monkeypatch) -> None: + """close 不得在恢复线程登记后、真正 start 前把它误判为已收敛。""" + executor = RecoveryExecutor() + start_entered = threading.Event() + allow_start = threading.Event() + close_results: list[bool] = [] + original_start = threading.Thread.start + + def delayed_recovery_start(thread: threading.Thread) -> None: + """只暂停恢复线程的 start，稳定放大登记与启动之间的竞态窗口。""" + if thread.name.startswith("MoviePilot-MonitorRecovery-"): + start_entered.set() + assert allow_start.wait(timeout=1) + original_start(thread) + + monkeypatch.setattr(threading.Thread, "start", delayed_recovery_start) + runner = threading.Thread( + target=lambda: executor.run({"atomic": lambda: None}, timeout=1), + name="recovery-submit-test", + ) + runner.start() + assert start_entered.wait(timeout=1) + + closer = threading.Thread( + target=lambda: close_results.append( + executor.close(deadline=time.monotonic() + 1) + ), + name="recovery-close-test", + ) + closer.start() + closer.join(timeout=0.05) + assert closer.is_alive(), "close 越过了尚未完成 start 的 owner 临界区" + + allow_start.set() + runner.join(timeout=1) + closer.join(timeout=1) + + assert not runner.is_alive() + assert not closer.is_alive() + assert close_results == [True] + assert executor.running_threads() == () + + +def test_close_tracks_compensation_and_blocks_post_seal_dispatch(tmp_path) -> None: + """补偿扫描解冻后不得在旧 lifespan 中继续把文件送入整理链。""" + monitor = _build_monitor() + entered = threading.Event() + release = threading.Event() + candidate = tmp_path / "late.mkv" + + def blocking_collect(_mon_path: Path): + """模拟补偿扫描阻塞在目录遍历。""" + entered.set() + release.wait() + return [(candidate, time.time(), 1)] + + monitor._Monitor__collect_compensation_files = blocking_collect + monitor._Monitor__start_compensation(mon_path=tmp_path, since=time.time()) + assert entered.wait(1) + + try: + assert monitor.close(timeout=0.02) is False + assert len(monitor._compensation_threads) == 1 + monitor._dispatcher.handle_file.assert_not_called() + finally: + release.set() + assert monitor.close(timeout=1) is True + assert monitor._compensation_threads == {} + monitor._dispatcher.handle_file.assert_not_called() + + +def test_close_tracks_scheduler_shutdown_thread() -> None: + """scheduler shutdown 阻塞时保留线程和 scheduler，恢复后才能清空句柄。""" + monitor = _build_monitor() + entered = threading.Event() + release = threading.Event() + scheduler = MagicMock() + scheduler.running = True + + def blocking_shutdown(*, wait: bool) -> None: + """模拟等待在途 APScheduler job 的同步 shutdown。""" + assert wait is True + entered.set() + release.wait() + + scheduler.shutdown.side_effect = blocking_shutdown + monitor._scheduler = scheduler + + try: + assert monitor.close(timeout=0.02) is False + assert entered.is_set() + assert monitor._scheduler is scheduler + assert monitor._scheduler_shutdown_thread is not None + assert monitor._scheduler_shutdown_thread.is_alive() + finally: + release.set() + assert monitor.close(timeout=1) is True + assert monitor._scheduler is None + assert monitor._scheduler_shutdown_thread is None + + +@pytest.mark.asyncio +async def test_stop_monitor_cancellation_waits_for_sync_close(monkeypatch) -> None: + """生命周期取消异步 stop 后，线程池调用仍由任务持有到同步 close 结束。""" + entered = threading.Event() + release = threading.Event() + + def blocking_close(timeout: float) -> bool: + """模拟仍在同步收尾的 Monitor.close。""" + assert timeout == 1 + entered.set() + release.wait() + return True + + monitor = SimpleNamespace(close=blocking_close) + monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor) + task = asyncio.create_task(stop_monitor(timeout=1)) + while not entered.is_set(): + await asyncio.sleep(0) + + task.cancel() + await asyncio.sleep(0.01) + assert task.done() is False + release.set() + with pytest.raises(asyncio.CancelledError): + await task + + +def test_init_monitor_explicitly_reopens_existing_lifespan(monkeypatch) -> None: + """同进程测试中的新 lifespan 必须显式 reopen，再初始化同一个 Monitor。""" + monitor = SimpleNamespace( + lifecycle_closed=True, + reopen=MagicMock(return_value=True), + init=MagicMock(return_value=True), + ) + monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor) + + init_monitor() + + monitor.reopen.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT) + monitor.init.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT) + + +@pytest.mark.asyncio +async def test_constructor_failure_publishes_started_watcher_to_cleanup( + monkeypatch, +) -> None: + """Monitor 构造中途失败后，stop-only 入口仍必须找到已启动 watcher owner。""" + instances = dict(SingletonClass._instances) + instances.pop(Monitor, None) + monkeypatch.setattr(SingletonClass, "_instances", instances) + watcher_started = threading.Event() + watcher_release = threading.Event() + watchers: list[threading.Thread] = [] + + def failing_init(monitor: Monitor) -> None: + """模拟 watcher 已启动、后续 scheduler 启动失败的构造过程。""" + watcher = threading.Thread( + target=lambda: (watcher_started.set(), watcher_release.wait()), + name="monitor-partial-construction", + daemon=True, + ) + watchers.append(watcher) + watcher.start() + + def close(timeout: float) -> bool: + """模拟真实 close 释放并等待半构造实例已经发布的 watcher。""" + watcher_release.set() + watcher.join(timeout=timeout) + return not watcher.is_alive() + + monitor.close = close + raise RuntimeError("scheduler start failed") + + monkeypatch.setattr(Monitor, "__init__", failing_init) + + with pytest.raises(RuntimeError, match="scheduler start failed"): + Monitor() + assert watcher_started.wait(timeout=1) + retained = Monitor.get_existing_instance() + assert retained is not None + + assert await stop_monitor(timeout=1) is True + assert watchers[0].is_alive() is False diff --git a/tests/test_monitor_mount_isolation.py b/tests/test_monitor_mount_isolation.py index 4206170b0..903f7d75b 100644 --- a/tests/test_monitor_mount_isolation.py +++ b/tests/test_monitor_mount_isolation.py @@ -18,8 +18,9 @@ from unittest.mock import MagicMock import pytest -from app.monitor import LocalDirectoryWatcher, Monitor +from app.monitor.monitor import Monitor from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path +from app.monitor.watcher import LocalDirectoryWatcher def _build_monitor(monkeypatch, put_recorder=None): @@ -33,7 +34,17 @@ def _build_monitor(monkeypatch, put_recorder=None): monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder)) monitor = object.__new__(Monitor) monitor._dispatcher = MagicMock() + monitor._lifecycle_lock = threading.RLock() + monitor._owner_lock = threading.Lock() + monitor._work_stop_event = threading.Event() + monitor._shutdown_event = threading.Event() + monitor._closed = False + monitor._compensation_threads = {} + monitor._scheduler_shutdown_thread = None + monitor._scheduler_shutdown_succeeded = False + monitor._scheduler = None monitor._watchers = [] + monitor._retired_watchers = [] monitor._watcher_lock = Lock() monitor._pending_locals = [] monitor._alerted_paths = {} @@ -114,7 +125,7 @@ def _stop_monitor_threads(monitor, timeout=10.0): :param monitor: Monitor 骨架 :param timeout: 每个线程的最长等待秒数 """ - for watcher in monitor._watchers: + for watcher in (*monitor._watchers, *monitor._retired_watchers): if isinstance(watcher, LocalDirectoryWatcher): watcher.stop() watcher.join(timeout=timeout) @@ -262,6 +273,10 @@ def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离" finally: _release_and_join(release, monitor._recovery._running.values()) + assert watcher in monitor._retired_watchers + # 重建后的旧 watcher 仍由 retired owner 表持有；测试替身不会自行切换 + # is_alive，释放模拟阻塞后显式推进到真实线程应有的退出终态。 + watcher._thread.is_alive.return_value = False _stop_monitor_threads(monitor) @@ -454,7 +469,7 @@ def test_stuck_transfer_does_not_block_other_files(monkeypatch): def test_recovery_executor_reports_timeout_without_blocking(): """ - 永不返回的动作只应消耗一次 timeout，执行器必须放弃它并如实报告。 + 永不返回的动作只应消耗一次 timeout，执行器必须结束本轮等待并如实报告。 """ executor = RecoveryExecutor() release = threading.Event() @@ -465,7 +480,7 @@ def test_recovery_executor_reports_timeout_without_blocking(): try: assert results == {"stuck": RecoveryState.TIMEOUT} - assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作" + assert elapsed < 3.0, "执行器没有在超时后结束本轮等待" finally: _release_and_join(release, executor._running.values()) diff --git a/tests/test_monitor_resilience.py b/tests/test_monitor_resilience.py index 715610af9..09d175ad9 100644 --- a/tests/test_monitor_resilience.py +++ b/tests/test_monitor_resilience.py @@ -1,8 +1,10 @@ +import threading from pathlib import Path from unittest.mock import MagicMock -from app.monitor import LocalDirectoryWatcher, Monitor +from app.monitor.monitor import Monitor from app.monitor.recovery import RecoveryExecutor +from app.monitor.watcher import LocalDirectoryWatcher def _build_watcher(tmp_path, force_polling): @@ -133,7 +135,17 @@ def _build_monitor(monkeypatch, put_recorder): monitor = object.__new__(Monitor) # 自动重启后健康检查会发起补偿扫描，骨架需要一个分发器替身 monitor._dispatcher = MagicMock() + monitor._lifecycle_lock = threading.RLock() + monitor._owner_lock = threading.Lock() + monitor._work_stop_event = threading.Event() + monitor._shutdown_event = threading.Event() + monitor._closed = False + monitor._compensation_threads = {} + monitor._scheduler_shutdown_thread = None + monitor._scheduler_shutdown_succeeded = False + monitor._scheduler = None monitor._watchers = [] + monitor._retired_watchers = [] monitor._watcher_lock = Lock() monitor._pending_locals = [] monitor._alerted_paths = {} diff --git a/tests/test_monitor_watchfiles.py b/tests/test_monitor_watchfiles.py index d67b3c91c..9819a4db1 100644 --- a/tests/test_monitor_watchfiles.py +++ b/tests/test_monitor_watchfiles.py @@ -1,11 +1,13 @@ +import threading from pathlib import Path from unittest.mock import MagicMock from watchfiles import Change -from app.monitor import DirectoryChangeEvent, LocalDirectoryWatcher, Monitor from app.monitor.dispatcher import TransferDispatcher -from app.schemas import TransferDirectoryConf +from app.monitor.monitor import Monitor +from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher +from app.schemas.system import TransferDirectoryConf from app.schemas.types import MediaType @@ -42,6 +44,15 @@ def _build_monitor_with_dispatcher(handle_file: MagicMock = None): if handle_file is not None: dispatcher.handle_file = handle_file monitor._dispatcher = dispatcher + monitor._lifecycle_lock = threading.RLock() + monitor._owner_lock = threading.Lock() + monitor._work_stop_event = threading.Event() + monitor._shutdown_event = threading.Event() + monitor._closed = False + monitor._compensation_threads = {} + monitor._scheduler_shutdown_thread = None + monitor._scheduler_shutdown_succeeded = False + monitor._scheduler = None return monitor, dispatcher diff --git a/tests/test_plugin_config_command.py b/tests/test_plugin_config_command.py index 34a296b10..ef6024e25 100644 --- a/tests/test_plugin_config_command.py +++ b/tests/test_plugin_config_command.py @@ -1,7 +1,15 @@ +from contextlib import nullcontext + from app.application.plugin.config import PluginConfigCommand +from app.runtime.extensions.plugin.admission import PluginMutationAdmission -def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCommand: +def _command( + calls: list[tuple], + *, + save_result: bool = True, + mutation=None, +) -> PluginConfigCommand: """构造记录端口调用顺序的插件配置用例。""" return PluginConfigCommand( save_config=lambda plugin_id, config, force: ( @@ -22,6 +30,7 @@ def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCom refresh_registrations=lambda plugin_id: calls.append( ("registrations", plugin_id) ), + mutation=mutation or (lambda _operation: nullcontext()), ) @@ -65,3 +74,22 @@ def test_reset_preserves_compensation_cleanup_and_reload_order() -> None: ("reload", "DemoPlugin"), ("registrations", "DemoPlugin"), ] + + +def test_sealed_config_command_rejects_before_first_side_effect() -> None: + """配置事务在 admission 封口后返回失败，且不得先发布 reset 事件。""" + calls: list[tuple] = [] + admission = PluginMutationAdmission() + admission.seal() + + update_result = _command(calls, mutation=admission.hold).update( + "DemoPlugin", + {"enabled": True}, + ) + reset_result = _command(calls, mutation=admission.hold).reset("DemoPlugin") + + assert update_result.success is False + assert reset_result.success is False + assert "停机阶段" in update_result.message + assert "停机阶段" in reset_result.message + assert calls == [] diff --git a/tests/test_plugin_endpoint.py b/tests/test_plugin_endpoint.py index c13a4d8b4..50da67f7d 100644 --- a/tests/test_plugin_endpoint.py +++ b/tests/test_plugin_endpoint.py @@ -1,4 +1,5 @@ import asyncio +from contextlib import nullcontext from unittest.mock import ANY, AsyncMock, MagicMock, patch from app.api.endpoints import plugin as plugin_endpoint @@ -13,6 +14,7 @@ from app.api.endpoints.plugin import uninstall_plugin from app.api.endpoints.system import sync_plugin_market_from_wiki from app.application.plugin.config import PluginConfigCommand from app.runtime.config import settings +from app.runtime.extensions.plugin.admission import PluginMutationAdmission from app.runtime.extensions.plugin_manager import PluginManager from app.runtime.tasks import TaskRegistry from app.schemas.event import PluginDataResetEventData @@ -495,6 +497,7 @@ def test_reset_plugin_sends_pre_reset_chain_event_before_deleting_data(): reload_runtime=plugin_manager.reload_plugin, publish_reset=publish_reset, refresh_registrations=lambda _plugin_id: None, + mutation=lambda _operation: nullcontext(), ) result = reset_plugin("SubscribeAssistantEnhanced", None, command) @@ -592,6 +595,82 @@ def test_uninstall_virtual_instance_never_removes_source_package(monkeypatch): plugin_manager.remove_plugin.assert_called_once_with("DemoPluginwork") +def test_sealed_http_uninstall_rejects_before_first_side_effect(monkeypatch): + """HTTP 卸载在封口后明确失败，且不读取或写入插件持久化状态。""" + admission = PluginMutationAdmission() + admission.seal() + plugin_manager = MagicMock() + plugin_manager.mutation.side_effect = admission.hold + config_provider = MagicMock() + remove_api = MagicMock() + remove_job = MagicMock() + monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager) + monkeypatch.setattr( + plugin_endpoint, + "get_configured_system_config", + config_provider, + ) + monkeypatch.setattr(plugin_endpoint, "remove_plugin_api", remove_api) + monkeypatch.setattr(plugin_endpoint, "remove_plugin_job", remove_job) + + result = uninstall_plugin("DemoPlugin", None) + + assert result.success is False + assert "停机阶段" in result.message + plugin_manager.get_plugin_instance.assert_not_called() + config_provider.assert_not_called() + remove_api.assert_not_called() + remove_job.assert_not_called() + + +def test_sealed_http_clone_rejects_before_runtime_and_registration(monkeypatch): + """HTTP 分身事务未获 admission 时不创建实例、不刷新注册或文件夹。""" + admission = PluginMutationAdmission() + admission.seal() + plugin_manager = MagicMock() + plugin_manager.mutation.side_effect = admission.hold + register = MagicMock() + add_to_folder = MagicMock() + monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager) + monkeypatch.setattr(plugin_endpoint, "register_plugin", register) + monkeypatch.setattr(plugin_endpoint, "_add_clone_to_plugin_folder", add_to_folder) + + result = plugin_endpoint.clone_plugin( + "DemoPlugin", + schemas.PluginCloneRequest(suffix="Work"), + None, + ) + + assert result.success is False + assert "停机阶段" in result.message + plugin_manager.clone_plugin.assert_not_called() + register.assert_not_called() + add_to_folder.assert_not_called() + + +def test_sealed_http_folder_update_rejects_before_config_access(monkeypatch): + """插件文件夹写入口在封口后不读取或改写持久化配置。""" + admission = PluginMutationAdmission() + admission.seal() + plugin_manager = MagicMock() + plugin_manager.mutation.side_effect = admission.hold + config_provider = MagicMock() + monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager) + monkeypatch.setattr( + plugin_endpoint, + "get_configured_system_config", + config_provider, + ) + + result = asyncio.run( + plugin_endpoint.update_folder_plugins("常用", ["DemoPlugin"], None) + ) + + assert result.success is False + assert "停机阶段" in result.message + config_provider.assert_not_called() + + def test_delete_plugin_data_can_force_delete_after_plugin_is_stopped(): """ 重置入口会先停止插件；插件数据删除不能依赖运行态注册仍存在。 diff --git a/tests/test_plugin_install_command.py b/tests/test_plugin_install_command.py index bcd2bdcb0..632298dc6 100644 --- a/tests/test_plugin_install_command.py +++ b/tests/test_plugin_install_command.py @@ -1,4 +1,5 @@ import asyncio +from contextlib import nullcontext from unittest.mock import AsyncMock, Mock, patch import pytest @@ -9,6 +10,7 @@ from app.schemas.exception import ( PersistenceUnavailableError, ) from app.application.plugin.install import PluginInstallCommand +from app.runtime.extensions.plugin.admission import PluginMutationAdmission def _command( @@ -24,6 +26,8 @@ def _command( checkpointer=None, committer=None, rollback=None, + mutation=None, + package_write_guard=None, ): """构造可观测每一步副作用的插件安装命令。""" return PluginInstallCommand( @@ -38,6 +42,9 @@ def _command( install_reporter=reporter or AsyncMock(), plugin_reloader=reloader or AsyncMock(), registration_refresher=refresher or AsyncMock(), + mutation=mutation or (lambda _operation: nullcontext()), + package_write_guard=package_write_guard + or (lambda _plugin_id: nullcontext()), ) @@ -74,6 +81,30 @@ async def test_install_failure_stops_before_report_persistence_and_reload(): reloader.assert_not_awaited() +@pytest.mark.asyncio +async def test_sealed_install_rejects_before_package_guard_and_checkpoint() -> None: + """安装事务在封口后不进入监控抑制，也不创建文件快照。""" + admission = PluginMutationAdmission() + admission.seal() + package_guard = Mock(return_value=nullcontext()) + checkpointer = AsyncMock() + + result = await _command( + mutation=admission.hold, + package_write_guard=package_guard, + checkpointer=checkpointer, + ).execute( + plugin_id="DemoPlugin", + repo_url="https://github.com/demo/plugins", + ) + + assert result.success is False + assert result.failure_stage == "admission" + assert "停机阶段" in result.message + package_guard.assert_not_called() + checkpointer.assert_not_awaited() + + @pytest.mark.asyncio async def test_success_records_completed_install_stages_in_order(): """成功安装在提交文件快照后再执行非关键远程上报。""" diff --git a/tests/test_plugin_lifecycle_status.py b/tests/test_plugin_lifecycle_status.py index 3da4e131a..47a0d2044 100644 --- a/tests/test_plugin_lifecycle_status.py +++ b/tests/test_plugin_lifecycle_status.py @@ -98,3 +98,226 @@ def test_lifecycle_records_load_failure_when_loader_returns_no_class(): assert result == {"DemoPlugin": PluginRuntimeStatus.LOAD_FAILED} assert running == {} assert statuses["DemoPlugin"] is PluginRuntimeStatus.LOAD_FAILED + + +def test_quiesce_keeps_instance_and_events_until_finalize(): + """第一阶段仅停止插件生产者，事件 handler 和实例留到屏障后释放。""" + order: list[str] = [] + + class DemoPlugin: + """记录旧插件 ABI hook 调用顺序的测试插件。""" + + plugin_name = "演示插件" + plugin_version = "1.0.0" + + def init_plugin(self, _config): + """接受宿主初始化配置。""" + + @staticmethod + def get_state(): + """保持插件事件 handler 启用。""" + return True + + def close(self): + """停止插件私有计时器和 watcher。""" + order.append("close") + + def stop_service(self): + """停止插件服务并允许尾事件继续投递。""" + order.append("stop_service") + + lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin]) + lifecycle.start("DemoPlugin") + lifecycle._disable_events.reset_mock() + lifecycle._clear_modules.reset_mock() + lifecycle._clear_tools.reset_mock() + + assert lifecycle.quiesce() is True + assert order == ["close", "stop_service"] + assert classes["DemoPlugin"] is DemoPlugin + assert isinstance(running["DemoPlugin"], DemoPlugin) + lifecycle._disable_events.assert_not_called() + lifecycle._clear_modules.assert_not_called() + lifecycle._clear_tools.assert_not_called() + + assert lifecycle.finalize() is True + lifecycle._disable_events.assert_called_once_with(DemoPlugin) + lifecycle._clear_modules.assert_called_once_with(None) + lifecycle._clear_tools.assert_called_once_with() + assert classes == {} + assert running == {} + + +def test_quiesce_runs_stop_service_after_close_failure_and_retries_missing_hook(): + """close 异常不能跳过 stop_service，重试时只补偿尚未成功的 hook。""" + order: list[str] = [] + close_fails = [True] + + class DemoPlugin: + """首次 close 失败、stop_service 正常完成的测试插件。""" + + plugin_name = "演示插件" + plugin_version = "1.0.0" + + def init_plugin(self, _config): + """接受宿主初始化配置。""" + + @staticmethod + def get_state(): + """保持插件为启用状态。""" + return True + + def close(self): + """首次调用模拟插件私有资源关闭异常。""" + order.append("close") + if close_fails[0]: + raise RuntimeError("close failed") + + def stop_service(self): + """记录旧 ABI 的第二个停机 hook。""" + order.append("stop_service") + + lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin]) + lifecycle.start("DemoPlugin") + + assert lifecycle.quiesce() is False + assert order == ["close", "stop_service"] + assert lifecycle.finalize() is False + assert "DemoPlugin" in classes + assert "DemoPlugin" in running + + close_fails[0] = False + assert lifecycle.quiesce() is True + assert order == ["close", "stop_service", "close"] + assert lifecycle.finalize() is True + assert classes == {} + assert running == {} + + +def test_legacy_stop_entry_remains_idempotent(): + """旧 stop 先解绑事件、保持 None 返回，且重复调用不重复执行 hook。""" + order: list[str] = [] + + class DemoPlugin: + """提供可计数停机 hook 的兼容测试插件。""" + + plugin_name = "演示插件" + plugin_version = "1.0.0" + + def init_plugin(self, _config): + """接受宿主初始化配置。""" + + @staticmethod + def get_state(): + """保持插件为启用状态。""" + return True + + def close(self): + """记录 close 调用。""" + order.append("close") + + def stop_service(self): + """记录 stop_service 调用。""" + order.append("stop_service") + + lifecycle, _classes, running, _statuses = _lifecycle(plugins=[DemoPlugin]) + lifecycle.start("DemoPlugin") + lifecycle._disable_events.reset_mock() + lifecycle._disable_events.side_effect = ( + lambda _plugin_type: order.append("disable_events") + ) + + assert lifecycle.stop() is None + assert lifecycle.stop() is None + assert order == ["disable_events", "close", "stop_service"] + lifecycle._disable_events.assert_called_once_with(DemoPlugin) + assert running == {} + + +def test_legacy_stop_force_finalizes_after_hook_failure(): + """旧 stop 即使 hook 报错也释放实例，并保持历史 None 返回 ABI。""" + order: list[str] = [] + + class DemoPlugin: + """close 失败但仍应完成兼容卸载的测试插件。""" + + plugin_name = "演示插件" + plugin_version = "1.0.0" + + def init_plugin(self, _config): + """接受宿主初始化配置。""" + + @staticmethod + def get_state(): + """保持插件为启用状态。""" + return True + + def close(self): + """模拟旧插件关闭异常。""" + order.append("close") + raise RuntimeError("close failed") + + def stop_service(self): + """证明 close 异常不会跳过后续 hook。""" + order.append("stop_service") + + lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin]) + lifecycle.start("DemoPlugin") + lifecycle._disable_events.reset_mock() + lifecycle._clear_modules.reset_mock() + + assert lifecycle.stop("DemoPlugin") is None + assert order == ["close", "stop_service"] + lifecycle._disable_events.assert_called_once_with(DemoPlugin) + lifecycle._clear_modules.assert_called_once_with("DemoPlugin") + assert classes == {} + assert running == {} + + +def test_legacy_reload_continues_after_old_instance_hook_failure(): + """旧实例 hook 失败只影响诊断，不得阻止 reload 创建新实例。""" + instances: list[object] = [] + stop_service = MagicMock() + + class DemoPlugin: + """记录每次构造并让 close 持续失败的重载测试插件。""" + + plugin_name = "演示插件" + plugin_version = "1.0.0" + + def __init__(self): + """记录新建的运行实例。""" + instances.append(self) + + def init_plugin(self, _config): + """接受宿主初始化配置。""" + + @staticmethod + def get_state(): + """保持插件为启用状态。""" + return True + + @staticmethod + def close(): + """模拟旧实例退出异常。""" + raise RuntimeError("close failed") + + def stop_service(self): + """记录旧 ABI 的第二个停机 hook。""" + stop_service() + + lifecycle, _classes, running, statuses = _lifecycle(plugins=[DemoPlugin]) + lifecycle.start("DemoPlugin") + previous = running["DemoPlugin"] + + result = lifecycle.reload("DemoPlugin", "plugin-reload") + + assert result is PluginRuntimeStatus.ACTIVE + assert statuses["DemoPlugin"] is PluginRuntimeStatus.ACTIVE + assert running["DemoPlugin"] is not previous + assert len(instances) == 2 + stop_service.assert_called_once_with() + lifecycle._event_sender.assert_called_once_with( + "plugin-reload", + data={"plugin_id": "DemoPlugin"}, + ) diff --git a/tests/test_plugin_monitor_lifecycle.py b/tests/test_plugin_monitor_lifecycle.py index eafe9db42..7be90756c 100644 --- a/tests/test_plugin_monitor_lifecycle.py +++ b/tests/test_plugin_monitor_lifecycle.py @@ -1,6 +1,7 @@ import asyncio import threading import time +from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -16,6 +17,7 @@ from app.runtime.extensions.plugin.monitor import ( PluginChangeMonitor, PluginMonitorController, ) +from app.runtime.extensions.plugin.admission import PluginMutationAdmission from app.runtime.extensions.plugin.system import reset_plugin_system from app.runtime.extensions.plugin_manager import PluginManager from app.schemas.plugin import PluginRuntimeStatus @@ -65,8 +67,9 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No missing_dependencies=("DependencyPending",), missing_source=("SourcePending",), ) + manager.reopen_plugins.side_effect = lambda: order.append("reopen") or True manager.start.side_effect = lambda plugin_id: order.append(f"plugin:{plugin_id}") - manager.start_monitor.side_effect = lambda: order.append("monitor") + manager.start_monitor.side_effect = lambda **_kwargs: order.append("monitor") monkeypatch.setattr( plugins_initializer, "configure_plugin_services", @@ -81,8 +84,16 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No plugins_initializer.init_plugins() - assert order == ["services", "plugin:ReadyPlugin", "routes", "monitor"] + assert order == [ + "services", + "reopen", + "plugin:ReadyPlugin", + "routes", + "monitor", + ] + manager.reopen_plugins.assert_called_once_with() manager.set_plugin_settling.assert_called_once_with(True) + manager.start_monitor.assert_called_once_with(reopen=True) def test_plugin_manager_projects_dependency_classification_to_runtime_status() -> None: @@ -155,6 +166,26 @@ def _patch_sync_plugins(monkeypatch, manager: MagicMock) -> MagicMock: return register +@pytest.mark.asyncio +async def test_sync_plugins_rejects_before_configuring_mutable_services( + monkeypatch, +) -> None: + """启动后同步在 admission 封口后不重装配服务、不写包或运行态。""" + admission = PluginMutationAdmission() + admission.seal() + manager = MagicMock() + manager.mutation.side_effect = admission.hold + configure = MagicMock() + monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager) + monkeypatch.setattr(plugins_initializer, "configure_plugin_services", configure) + + assert await plugins_initializer.sync_plugins() is False + + configure.assert_not_called() + manager.set_plugin_settling.assert_not_called() + manager.sync.assert_not_called() + + @pytest.mark.asyncio async def test_sync_plugins_activates_ready_plugins_when_dependencies_fail( monkeypatch, @@ -392,6 +423,303 @@ def test_plugin_monitor_waits_until_dependency_settlement(monkeypatch) -> None: _reset_plugin_manager() +def test_monitor_stop_keeps_thread_owned_until_it_really_exits() -> None: + """停止超时后保留活线程引用，使后续停机调用仍能继续等待。""" + started = threading.Event() + release = threading.Event() + + def runner() -> None: + """模拟暂时无法响应停止事件的文件监控循环。""" + started.set() + release.wait() + + controller = PluginMonitorController(runner=runner, log=MagicMock()) + controller.start() + assert started.wait(1) + owned_thread = controller._thread + + try: + assert controller.stop(timeout=0.01) is False + assert controller._thread is owned_thread + assert owned_thread is not None and owned_thread.is_alive() + finally: + release.set() + assert controller.stop(timeout=1) is True + + assert controller._thread is None + assert controller.stop(timeout=0) is True + + +def test_monitor_stop_counts_lifecycle_lock_wait_in_timeout() -> None: + """并发重建占用生命周期锁时，停止调用不得突破自身预算。""" + controller = PluginMonitorController(runner=lambda: None, log=MagicMock()) + lock_acquired = threading.Event() + release = threading.Event() + + def hold_lifecycle_lock() -> None: + """模拟配置重载在停止请求到达时仍持有生命周期锁。""" + with controller._lifecycle_lock: + lock_acquired.set() + release.wait() + + holder = threading.Thread(target=hold_lifecycle_lock, daemon=True) + holder.start() + assert lock_acquired.wait(1) + started_at = time.monotonic() + + try: + assert controller.stop(timeout=0.01) is False + assert time.monotonic() - started_at < 0.2 + finally: + release.set() + holder.join(timeout=1) + + assert holder.is_alive() is False + assert controller.stop(timeout=0) is True + + +def test_shutdown_seal_blocks_reload_between_stop_and_start(monkeypatch) -> None: + """停机封口落在热重载停启间隙时，旧重载不得再创建监控线程。""" + reload_stopped = threading.Event() + resume_reload = threading.Event() + runner_started = threading.Event() + controller = PluginMonitorController( + runner=lambda: runner_started.set(), + log=MagicMock(), + ) + original_stop = controller.stop + + def pause_after_stop(timeout: float = 5.0) -> bool: + """把配置热重载稳定暂停在旧线程已停、新线程未启的窗口。""" + stopped = original_stop(timeout=timeout) + reload_stopped.set() + resume_reload.wait() + return stopped + + monkeypatch.setattr(controller, "stop", pause_after_stop) + reload_thread = threading.Thread( + target=lambda: controller.reload(enabled=True), + daemon=True, + ) + reload_thread.start() + assert reload_stopped.wait(1) + + try: + assert controller.close(timeout=1) is True + resume_reload.set() + reload_thread.join(timeout=1) + assert reload_thread.is_alive() is False + assert runner_started.is_set() is False + assert controller._thread is None + finally: + resume_reload.set() + reload_thread.join(timeout=1) + + assert controller.reopen() is True + controller.start() + assert runner_started.wait(1) + assert controller.close(timeout=1) is True + + +def test_plugin_manager_stop_monitor_returns_controller_result(monkeypatch) -> None: + """管理器停止入口透传线程收口结果和调用预算。""" + _reset_plugin_manager() + reset_plugin_system() + manager = PluginManager() + stop = MagicMock(return_value=False) + manager._plugin_monitor.stop = stop + + try: + assert manager.stop_monitor(timeout=0.25) is False + stop.assert_called_once_with(timeout=0.25) + finally: + _reset_plugin_manager() + + +def test_plugin_manager_start_monitor_can_reopen_new_lifespan(monkeypatch) -> None: + """新应用生命周期可显式解除封口，再按运行配置启动监控。""" + _reset_plugin_manager() + reset_plugin_system() + monkeypatch.setattr( + "app.runtime.extensions.plugin_manager.settings", + SimpleNamespace( + DEV=True, + PLUGIN_AUTO_RELOAD=False, + ROOT_PATH=MagicMock(), + ), + ) + manager = PluginManager() + reopen = MagicMock(return_value=True) + start = MagicMock() + manager._plugin_monitor.reopen = reopen + manager._plugin_monitor.start = start + + try: + manager.start_monitor(reopen=True) + reopen.assert_called_once_with() + start.assert_called_once_with() + finally: + _reset_plugin_manager() + + +def test_stop_plugin_monitor_does_not_materialize_manager() -> None: + """插件运行时尚未创建时，独立停机入口直接视为已完成。""" + _reset_plugin_manager() + + assert plugins_initializer.stop_plugin_monitor(timeout=0) is True + assert PluginManager.get_existing_instance() is None + + +def test_stop_plugin_monitor_returns_existing_manager_result(monkeypatch) -> None: + """启动层入口只操作既有管理器，并透传超时失败。""" + manager = MagicMock() + manager.close_monitor.return_value = False + manager_type = SimpleNamespace(get_existing_instance=lambda: manager) + monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type) + + assert plugins_initializer.stop_plugin_monitor(timeout=0.25) is False + manager.close_monitor.assert_called_once_with(timeout=0.25) + + +@pytest.mark.asyncio +async def test_two_phase_plugin_shutdown_does_not_materialize_manager() -> None: + """两阶段入口在插件管理器尚未创建时都直接视为已收敛。""" + _reset_plugin_manager() + + assert await plugins_initializer.quiesce_plugins(timeout=0) is True + assert plugins_initializer.finalize_plugins() is True + assert PluginManager.get_existing_instance() is None + + +@pytest.mark.asyncio +async def test_quiesce_timeout_retains_future_owner_until_worker_finishes( + monkeypatch, +) -> None: + """同步插件 hook 超时后必须保留 Future，且未结束前拒绝卸载实例。""" + _reset_plugin_manager() + reset_plugin_system() + monkeypatch.setattr( + "app.runtime.extensions.plugin_manager.settings", + SimpleNamespace( + DEV=False, + PLUGIN_AUTO_RELOAD=False, + ROOT_PATH=MagicMock(), + ), + ) + manager = PluginManager() + started = threading.Event() + release = threading.Event() + + def blocking_quiesce() -> bool: + """模拟无法由 asyncio 取消的同步旧插件 hook。""" + started.set() + release.wait(timeout=2) + return True + + manager._plugin_lifecycle.quiesce = MagicMock(side_effect=blocking_quiesce) + manager._plugin_lifecycle.finalize = MagicMock(return_value=True) + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + thread_helper = SimpleNamespace(submit=executor.submit) + monkeypatch.setattr( + "app.runtime.extensions.plugin_manager.ThreadHelper", + lambda: thread_helper, + ) + + assert await manager.quiesce_plugins(timeout=0.01) is False + assert started.is_set() + owner = manager._plugin_quiesce_future + assert owner is not None + assert owner.done() is False + assert manager.finalize_plugins() is False + manager._plugin_lifecycle.finalize.assert_not_called() + + release.set() + assert await asyncio.wrap_future(owner) is True + assert manager.finalize_plugins() is True + manager._plugin_lifecycle.finalize.assert_called_once_with() + finally: + release.set() + _reset_plugin_manager() + + +@pytest.mark.asyncio +async def test_quiesce_seals_runtime_until_new_lifespan_reopens(monkeypatch) -> None: + """屏障前封口后 start/reload/config 不能重开 producer，新 lifespan 可显式恢复。""" + _reset_plugin_manager() + reset_plugin_system() + monkeypatch.setattr( + "app.runtime.extensions.plugin_manager.settings", + SimpleNamespace( + DEBUG=False, + DEV=False, + PLUGIN_AUTO_RELOAD=False, + ROOT_PATH=MagicMock(), + ), + ) + manager = PluginManager() + manager._plugin_lifecycle.quiesce = MagicMock(return_value=True) + manager._plugin_lifecycle.start = MagicMock( + return_value={"DemoPlugin": PluginRuntimeStatus.ACTIVE} + ) + manager._plugin_lifecycle.reload = MagicMock( + return_value=PluginRuntimeStatus.ACTIVE + ) + manager._plugin_lifecycle.stop = MagicMock(return_value=True) + manager._plugin_lifecycle.initialize = MagicMock() + manager._plugin_lifecycle._disable_events = MagicMock() + manager._plugin_registry.remove = MagicMock() + manager._plugin_registry.set_runtime_status = MagicMock() + manager.classify_plugins = MagicMock() + + class DemoPlugin: + """代表 quiesce 后仍由严格生命周期持有的运行实例。""" + + plugin_instance = DemoPlugin() + manager._plugins["DemoPlugin"] = DemoPlugin + manager._running_plugins["DemoPlugin"] = plugin_instance + + try: + assert await manager.quiesce_plugins(timeout=1) is True + + assert manager.start("DemoPlugin") == { + "DemoPlugin": PluginRuntimeStatus.LOAD_FAILED + } + assert manager.stop("DemoPlugin") is None + assert manager.remove_plugin("DemoPlugin") is None + assert ( + manager.reload_plugin("DemoPlugin") + is PluginRuntimeStatus.LOAD_FAILED + ) + manager.init_plugin("DemoPlugin", {}) + manager.init_config() + manager._plugin_lifecycle.start.assert_not_called() + manager._plugin_lifecycle.stop.assert_not_called() + manager._plugin_lifecycle.reload.assert_not_called() + manager._plugin_lifecycle.initialize.assert_not_called() + manager._plugin_lifecycle._disable_events.assert_not_called() + manager._plugin_registry.remove.assert_not_called() + manager._plugin_registry.set_runtime_status.assert_not_called() + manager.classify_plugins.assert_not_called() + assert manager._plugins["DemoPlugin"] is DemoPlugin + assert manager._running_plugins["DemoPlugin"] is plugin_instance + + assert manager.reopen_plugins() is False + manager._plugins.clear() + manager._running_plugins.clear() + assert manager.reopen_plugins() is True + assert manager.start("DemoPlugin") == { + "DemoPlugin": PluginRuntimeStatus.ACTIVE + } + assert ( + manager.reload_plugin("DemoPlugin") + is PluginRuntimeStatus.ACTIVE + ) + finally: + _reset_plugin_manager() + + def test_plugin_monitor_skips_installing_plugin_until_package_write_finishes(tmp_path) -> None: """安装替换目录期间，文件事件不得抢先导入未完成的插件包。""" reload_plugin = MagicMock() @@ -466,11 +794,12 @@ def test_stop_plugins_stops_monitor_before_plugin_runtime(monkeypatch) -> None: """关闭时先隔离文件变化，再停止插件实例。""" order: list[str] = [] manager = MagicMock() - manager.stop_monitor.side_effect = lambda: order.append("monitor") + manager.stop_monitor.side_effect = lambda: order.append("monitor") or True manager.stop.side_effect = lambda: order.append("plugins") - monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager) + manager_type = SimpleNamespace(get_existing_instance=lambda: manager) + monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type) - plugins_initializer.stop_plugins() + assert plugins_initializer.stop_plugins() is True assert order == ["monitor", "plugins"] @@ -479,8 +808,89 @@ def test_stop_plugins_still_stops_runtime_when_monitor_stop_fails(monkeypatch) - """监控线程停止异常不得阻止插件实例释放资源。""" manager = MagicMock() manager.stop_monitor.side_effect = RuntimeError("monitor stop failed") - monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager) + manager_type = SimpleNamespace(get_existing_instance=lambda: manager) + monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type) - plugins_initializer.stop_plugins() + assert plugins_initializer.stop_plugins() is False manager.stop.assert_called_once_with() + + +def test_stop_plugins_does_not_materialize_manager() -> None: + """插件初始化尚未创建管理器时，失败清理不得反向构造运行时。""" + _reset_plugin_manager() + + assert plugins_initializer.stop_plugins() is True + + assert PluginManager.get_existing_instance() is None + + +def test_stop_plugins_remains_idempotent(monkeypatch) -> None: + """兼容停机入口可重复调用，并保持每次先停监控再停插件。""" + order: list[str] = [] + manager = MagicMock() + manager.stop_monitor.side_effect = lambda: order.append("monitor") or True + manager.stop.side_effect = lambda: order.append("plugins") + manager_type = SimpleNamespace(get_existing_instance=lambda: manager) + monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type) + + assert plugins_initializer.stop_plugins() is True + assert plugins_initializer.stop_plugins() is True + + assert order == ["monitor", "plugins", "monitor", "plugins"] + + +def test_plugin_manager_legacy_stop_preserves_none_return() -> None: + """公共 PluginManager.stop 委托单阶段停机后保持历史 None 返回 ABI。""" + manager = object.__new__(PluginManager) + manager._plugin_quiesce_lock = threading.RLock() + manager._plugin_runtime_closed = False + manager._plugin_quiesce_future = None + manager._plugin_mutation_admission = PluginMutationAdmission() + manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=True)) + + assert PluginManager.stop(manager, "DemoPlugin") is None + + manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin") + + +def test_config_reload_continues_after_legacy_stop() -> None: + """配置热重载保持旧行为，stop 的 None 返回不得阻止重新分类和启动。""" + manager = object.__new__(PluginManager) + manager._plugin_quiesce_lock = threading.RLock() + manager._plugin_runtime_closed = False + manager._plugin_mutation_admission = PluginMutationAdmission() + manager.stop = MagicMock(return_value=None) + manager.classify_plugins = MagicMock( + return_value=PluginDependencyClassification( + ready=("ReadyPlugin",), + missing_dependencies=(), + missing_source=(), + ) + ) + manager.apply_plugin_dependency_classification = MagicMock() + manager.start = MagicMock() + + PluginManager.init_config(manager) + + manager.stop.assert_called_once_with() + manager.classify_plugins.assert_called_once_with() + manager.apply_plugin_dependency_classification.assert_called_once_with( + manager.classify_plugins.return_value + ) + manager.start.assert_called_once_with("ReadyPlugin") + + +def test_remove_plugin_clears_registry_after_legacy_stop() -> None: + """卸载路径保持忽略旧 stop 返回值并继续清理注册表。""" + manager = object.__new__(PluginManager) + manager._plugin_quiesce_lock = threading.RLock() + manager._plugin_runtime_closed = False + manager._plugin_mutation_admission = PluginMutationAdmission() + manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=None)) + manager._plugin_registry = SimpleNamespace(remove=MagicMock()) + + PluginManager.remove_plugin(manager, "DemoPlugin") + + manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin") + manager._plugin_registry.remove.assert_called_once_with("DemoPlugin") diff --git a/tests/test_plugin_mutation_admission.py b/tests/test_plugin_mutation_admission.py new file mode 100644 index 000000000..93b6e5b0e --- /dev/null +++ b/tests/test_plugin_mutation_admission.py @@ -0,0 +1,155 @@ +"""插件可变事务停机准入的确定性测试。""" + +import asyncio +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from app.foundation.singleton import Singleton +from app.runtime.extensions.plugin.admission import ( + PluginMutationAdmission, + PluginMutationRejectedError, +) +from app.runtime.extensions.plugin.system import reset_plugin_system +from app.runtime.extensions.plugin_manager import PluginManager +from app.schemas.plugin import PluginRuntimeStatus +from app.schemas.types import EventType + + +@pytest.fixture +def plugin_manager() -> Iterator[PluginManager]: + """构造隔离的插件管理器，并在用例结束后清除单例状态。""" + key = (PluginManager, (), frozenset()) + Singleton._instances.pop(key, None) + reset_plugin_system() + manager = PluginManager() + try: + yield manager + finally: + Singleton._instances.pop(key, None) + + +def test_seal_rejects_new_root_but_allows_propagated_nested_lease() -> None: + """封口拒绝无关调用，但已获准事务跨线程后的嵌套仍能完成。""" + admission = PluginMutationAdmission() + calls: list[str] = [] + + def mutate(label: str) -> None: + """在测试线程中尝试取得 lease 并记录副作用。""" + with admission.hold(label): + calls.append(label) + + with admission.hold("外层事务"): + propagated = copy_context() + assert admission.active_count == 1 + assert admission.seal() == 1 + with ThreadPoolExecutor(max_workers=2) as executor: + executor.submit(propagated.run, mutate, "嵌套事务").result(timeout=1) + rejected = executor.submit(mutate, "新事务") + with pytest.raises(PluginMutationRejectedError): + rejected.result(timeout=1) + + assert calls == ["嵌套事务"] + assert admission.active_count == 1 + + assert admission.active_count == 0 + assert admission.reopen() is True + + +@pytest.mark.asyncio +async def test_quiesce_timeout_retains_admitted_owner_and_nested_reload( + plugin_manager: PluginManager, + monkeypatch, +) -> None: + """停机超时保留 active owner，且封口前获准的事务可完成内部重载。""" + manager = plugin_manager + entered = asyncio.Event() + release = asyncio.Event() + manager._plugin_lifecycle.reload = MagicMock( + return_value=PluginRuntimeStatus.ACTIVE + ) + manager._plugin_lifecycle.quiesce = MagicMock(return_value=True) + manager._plugin_lifecycle.finalize = MagicMock(return_value=True) + + async def mutate() -> PluginRuntimeStatus: + """持有外层 lease，等待封口后再调用嵌套 Manager 写入口。""" + with manager.mutation("安装插件 DemoPlugin"): + entered.set() + await release.wait() + return manager.reload_plugin("DemoPlugin") + + with ThreadPoolExecutor(max_workers=1) as executor: + monkeypatch.setattr( + "app.runtime.extensions.plugin_manager.ThreadHelper", + lambda: SimpleNamespace(submit=executor.submit), + ) + mutation_task = asyncio.create_task(mutate()) + await entered.wait() + + assert await manager.quiesce_plugins(timeout=0.01) is False + owner = manager._plugin_quiesce_future + assert owner is not None + assert owner.done() is False + assert manager._plugin_mutation_admission.active_count == 1 + assert manager.finalize_plugins() is False + with pytest.raises(PluginMutationRejectedError): + with manager.mutation("新的配置写入"): + pass + + release.set() + assert await mutation_task is PluginRuntimeStatus.ACTIVE + assert await asyncio.wrap_future(owner) is True + + manager._plugin_lifecycle.reload.assert_called_once_with( + "DemoPlugin", + EventType.PluginReload, + ) + assert manager._plugin_mutation_admission.active_count == 0 + assert manager.finalize_plugins() is True + + +@pytest.mark.asyncio +async def test_quiesce_inside_mutation_fails_without_sealing( + plugin_manager: PluginManager, +) -> None: + """事务不能等待自身退出，快速失败时也不得误封口运行时。""" + with plugin_manager.mutation("配置插件"): + assert await plugin_manager.quiesce_plugins(timeout=0) is False + assert plugin_manager._plugin_mutation_admission.accepting is True + assert plugin_manager._plugin_runtime_closed is False + + +@pytest.mark.asyncio +async def test_sealed_manager_blocks_direct_persistent_mutations( + plugin_manager: PluginManager, +) -> None: + """Manager 单项兼容入口在封口后返回既有失败形状且不触发底层副作用。""" + plugin_manager._plugin_config_store = MagicMock() + plugin_manager._plugin_instance_store = MagicMock() + plugin_manager._plugin_sync = MagicMock() + plugin_manager._plugin_clone = MagicMock() + plugin_manager._plugin_mutation_admission.seal() + plugin_manager._plugin_runtime_closed = True + + assert plugin_manager.save_plugin_config("DemoPlugin", {}) is False + assert await plugin_manager.async_save_plugin_config("DemoPlugin", {}) is False + assert plugin_manager.delete_plugin_config("DemoPlugin") is False + assert plugin_manager.delete_plugin_data("DemoPlugin") is False + assert plugin_manager.delete_plugin_instance("DemoPlugin") is False + clone_result = plugin_manager.clone_plugin("DemoPlugin", "Work", "", "") + assert clone_result[0] is False + assert "停机阶段" in clone_result[1] + with pytest.raises(PluginMutationRejectedError): + plugin_manager.sync() + + plugin_manager._plugin_config_store.write.assert_not_called() + plugin_manager._plugin_config_store.async_write.assert_not_called() + plugin_manager._plugin_config_store.delete.assert_not_called() + plugin_manager._plugin_config_store.delete_data.assert_not_called() + plugin_manager._plugin_instance_store.delete.assert_not_called() + plugin_manager._plugin_sync.sync.assert_not_called() + plugin_manager._plugin_clone.clone.assert_not_called() diff --git a/tests/test_singleton.py b/tests/test_singleton.py index 824d9b32b..f8d53c22e 100644 --- a/tests/test_singleton.py +++ b/tests/test_singleton.py @@ -1,3 +1,5 @@ +import pytest + from app.foundation.singleton import Singleton, SingletonClass @@ -27,3 +29,49 @@ def test_parameterized_singleton_can_read_matching_instance_without_creating(mon instance = Example("first") assert Example.get_existing_instance("first") is instance assert Example.get_existing_instance("second") is None + + +def test_parameterized_singleton_can_retain_failed_lifecycle_owner(monkeypatch): + """构造中途失败时，可选 owner 单例必须仍能由启动失败清理读取。""" + + class Example(metaclass=Singleton): + """模拟在构造期已创建后台 owner 后抛错的参数化单例。""" + + _retain_failed_singleton = True + + def __init__(self) -> None: + """发布可观察 owner 后模拟后续启动失败。""" + self.owner_started = True + raise RuntimeError("startup failed") + + monkeypatch.setattr(Singleton, "_instances", {}) + + with pytest.raises(RuntimeError, match="startup failed"): + Example() + + retained = Example.get_existing_instance() + assert retained is not None + assert retained.owner_started is True + + +def test_class_singleton_can_retain_failed_lifecycle_owner(monkeypatch): + """按类 owner 单例也必须在 __init__ 抛错后保留可清理身份。""" + + class Example(metaclass=SingletonClass): + """模拟目录监控构造中途失败的按类单例。""" + + _retain_failed_singleton = True + + def __init__(self) -> None: + """发布可观察 owner 后模拟后续启动失败。""" + self.owner_started = True + raise RuntimeError("startup failed") + + monkeypatch.setattr(SingletonClass, "_instances", {}) + + with pytest.raises(RuntimeError, match="startup failed"): + Example() + + retained = Example.get_existing_instance() + assert retained is not None + assert retained.owner_started is True diff --git a/tests/test_task_registry.py b/tests/test_task_registry.py index 6719a8bf0..4ec9b0079 100644 --- a/tests/test_task_registry.py +++ b/tests/test_task_registry.py @@ -50,7 +50,7 @@ def test_task_registry_cancels_tasks_and_rejects_late_registration() -> None: task = registry.create(worker(), owner="test.shutdown") await started.wait() - await registry.shutdown(timeout_seconds=1.0) + assert await registry.shutdown(timeout_seconds=1.0) is True assert task.cancelled() assert cancelled.is_set() @@ -108,7 +108,7 @@ def test_task_registry_keeps_timed_out_sync_owner_until_real_completion() -> Non task = registry.create_sync(worker, owner="test.sync-timeout") assert await asyncio.to_thread(started.wait, 1.0) - await registry.shutdown(timeout_seconds=0.001) + assert await registry.shutdown(timeout_seconds=0.001) is False assert not task.done() assert [record.owner for record in registry.records] == [ @@ -155,7 +155,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None: task = registry.create(worker(), owner="test.stubborn") await started.wait() try: - await registry.shutdown(timeout_seconds=0.001) + assert await registry.shutdown(timeout_seconds=0.001) is False assert cleanup_started.is_set() assert not task.done() @@ -165,7 +165,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None: ] assert reports[-1]["owners"] == ("test.stubborn",) - await registry.shutdown(timeout_seconds=0.001) + assert await registry.shutdown(timeout_seconds=0.001) is False assert not task.done() assert cancellation_count == 1 assert len(reports) == 1 diff --git a/tests/test_transfer_failed_retry_scheduler.py b/tests/test_transfer_failed_retry_scheduler.py new file mode 100644 index 000000000..a273cda0c --- /dev/null +++ b/tests/test_transfer_failed_retry_scheduler.py @@ -0,0 +1,126 @@ +"""失败整理 AI 重试调度器的生命周期测试。""" + +import asyncio +from unittest.mock import Mock, patch + +import pytest + +from app.application.transfer import FailedRetryScheduler + + +def test_retry_scheduler_close_cancels_buffered_timer_and_rejects_new_work(): + """关闭应取消尚未触发的 timer、清空缓冲并拒绝新增记录。""" + + async def exercise() -> None: + """在独立事件循环内验证 timer 与关闭状态。""" + scheduler = FailedRetryScheduler() + scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 60 + await scheduler.schedule_retry(11, group_key="media:test") + timer = scheduler._retry_transfer_timers["media:test"] + + await scheduler.close() + await scheduler.close() + + assert timer.cancelled() + assert scheduler._retry_transfer_buffer == {} + assert scheduler._retry_transfer_timers == {} + with pytest.raises(RuntimeError, match="正在关闭"): + await scheduler.schedule_retry(12, group_key="media:test") + + asyncio.run(exercise()) + + +def test_retry_scheduler_close_cancels_and_waits_for_active_flush_task(): + """关闭返回前应等待已经启动的 flush 任务完成取消收尾。""" + + async def exercise() -> None: + """启动一个不会自行结束的 flush，并通过关闭流程取消它。""" + scheduler = FailedRetryScheduler() + scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0 + started = asyncio.Event() + stopped = asyncio.Event() + + async def blocking_flush(_group_key: str, _generation: int) -> None: + """等待取消信号，并在 finally 中证明收尾已经完成。""" + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + scheduler._flush_retry_transfer = blocking_flush + await scheduler.schedule_retry(11, group_key="media:test") + await asyncio.wait_for(started.wait(), timeout=1) + task = next(iter(scheduler._retry_transfer_tasks)) + + await scheduler.close() + + assert stopped.is_set() + assert task.cancelled() + assert task.get_name() == "transfer.failed_retry.flush" + assert scheduler._retry_transfer_tasks == set() + + asyncio.run(exercise()) + + +def test_retry_scheduler_observes_unexpected_background_task_error(): + """flush 协程越过自身防线的异常仍应由任务 owner 统一观察。""" + + async def exercise() -> None: + """让受管 flush 任务直接失败，并等待完成回调处理异常。""" + scheduler = FailedRetryScheduler() + scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0 + + async def failing_flush(_group_key: str, _generation: int) -> None: + """模拟 flush 外层出现未处理异常。""" + raise RuntimeError("flush failed") + + scheduler._flush_retry_transfer = failing_flush + with patch("app.application.transfer.logger.error", Mock()) as log_error: + await scheduler.schedule_retry(11, group_key="media:test") + for _ in range(5): + await asyncio.sleep(0) + if scheduler._retry_transfer_tasks: + break + assert scheduler._retry_transfer_tasks + for _ in range(5): + await asyncio.sleep(0) + if not scheduler._retry_transfer_tasks: + break + + assert scheduler._retry_transfer_tasks == set() + log_error.assert_called_once() + assert "flush failed" in log_error.call_args.args[0] + await scheduler.close() + + asyncio.run(exercise()) + + +def test_retry_scheduler_old_flush_cannot_consume_renewed_generation(): + """旧 timer 已建 task 后的新失败应续期，不能被旧 flush 提前取走。""" + + async def exercise() -> None: + """稳定复现 timer callback 与同组新 schedule 交错的窗口。""" + scheduler = FailedRetryScheduler() + scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 3600 + await scheduler.schedule_retry(11, group_key="media:test") + old_timer = scheduler._retry_transfer_timers["media:test"] + old_generation = scheduler._retry_transfer_generations["media:test"] + + # 模拟旧 timer callback 已进入事件循环，但 flush task 尚未取得分组锁。 + scheduler._start_retry_transfer_task("media:test", old_generation) + await scheduler.schedule_retry(12, group_key="media:test") + renewed_timer = scheduler._retry_transfer_timers["media:test"] + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert old_timer.cancelled() + assert renewed_timer.cancelled() is False + assert scheduler._retry_transfer_buffer["media:test"] == [11, 12] + assert scheduler._retry_transfer_timers["media:test"] is renewed_timer + assert scheduler._retry_transfer_tasks == set() + + await scheduler.close() + assert renewed_timer.cancelled() + + asyncio.run(exercise()) diff --git a/tests/test_transfer_failure_notification_aggregation.py b/tests/test_transfer_failure_notification_aggregation.py index 0e286f8e5..c19415389 100644 --- a/tests/test_transfer_failure_notification_aggregation.py +++ b/tests/test_transfer_failure_notification_aggregation.py @@ -1,6 +1,8 @@ from unittest.mock import Mock, patch from types import SimpleNamespace +import pytest + from app.chain import transfer as transfer_module from app.chain.transfer import TransferChain from app.application.transfer import ( @@ -59,6 +61,26 @@ class _Loop: return False +class _DeferredLoop(_Loop): + """延迟执行线程安全回调，用于覆盖关闭与入环之间的竞态。""" + + def __init__(self): + """初始化延迟回调和定时器清单。""" + super().__init__() + self.soon_callbacks = [] + + def call_soon_threadsafe(self, callback, *args): + """保存线程安全回调，直到测试显式执行。""" + self.soon_callbacks.append((callback, args)) + + def run_soon_callbacks(self): + """执行并清空已保存的线程安全回调。""" + callbacks = list(self.soon_callbacks) + self.soon_callbacks.clear() + for callback, args in callbacks: + callback(*args) + + def _task(*, episode: int, download_hash: str = "hash-1") -> TransferTask: """构造同一媒体不同剧集的整理任务。""" return TransferTask( @@ -122,6 +144,116 @@ def test_aggregator_debounces_same_group_and_flushes_once(): callback.assert_called_once_with(notices) +def test_aggregator_old_timer_cannot_flush_before_renewal_is_armed(): + """新通知已接收时，旧 timer 不得抢在事件循环重置静默窗前发送。""" + loop = _DeferredLoop() + aggregator = TransferFailureNotificationAggregator() + callback = Mock() + first = TransferFailureNotification( + "测试剧 (2026)", "S01E01", "原因A", 1, None, "tester" + ) + second = TransferFailureNotification( + "测试剧 (2026)", "S01E02", "原因B", 2, None, "tester" + ) + + aggregator.schedule( + group_key="media:test", + notification=first, + callback=callback, + loop=loop, + ) + loop.run_soon_callbacks() + old_timer = loop.timers[0] + + aggregator.schedule( + group_key="media:test", + notification=second, + callback=callback, + loop=loop, + ) + old_timer.callback(*old_timer.args) + callback.assert_not_called() + + loop.run_soon_callbacks() + renewed_timer = loop.timers[1] + assert old_timer.cancelled is True + renewed_timer.callback(*renewed_timer.args) + callback.assert_called_once_with([first, second]) + + +def test_aggregator_close_flushes_accepted_notification_before_timer_is_armed(): + """关闭应发送已接收但尚未进入事件循环的通知，且延迟回调不能重新建 timer。""" + loop = _DeferredLoop() + aggregator = TransferFailureNotificationAggregator() + callback = Mock() + notice = TransferFailureNotification( + "测试剧 (2026)", "S01E01", "原因A", 1, None, "tester" + ) + + aggregator.schedule( + group_key="media:test", + notification=notice, + callback=callback, + loop=loop, + ) + aggregator.close() + aggregator.close() + loop.run_soon_callbacks() + + callback.assert_called_once_with([notice]) + assert loop.timers == [] + + +def test_aggregator_close_cancels_timer_and_rejects_new_notification(): + """关闭应取消已建 timer，并让调用方明确感知后续投递被拒绝。""" + loop = _Loop() + aggregator = TransferFailureNotificationAggregator() + callback = Mock() + notice = TransferFailureNotification( + "测试剧 (2026)", "S01E01", "原因A", 1, None, "tester" + ) + aggregator.schedule( + group_key="media:test", + notification=notice, + callback=callback, + loop=loop, + ) + + aggregator.close() + + assert loop.timers[0].cancelled is True + callback.assert_called_once_with([notice]) + with pytest.raises(RuntimeError, match="正在关闭"): + aggregator.schedule( + group_key="media:test", + notification=notice, + callback=callback, + loop=loop, + ) + + +def test_aggregator_close_observes_flush_callback_error(): + """关闭阶段同步刷新失败时应记录异常而不是让通知静默丢失。""" + loop = _DeferredLoop() + aggregator = TransferFailureNotificationAggregator() + callback = Mock(side_effect=RuntimeError("send failed")) + notice = TransferFailureNotification( + "测试剧 (2026)", "S01E01", "原因A", 1, None, "tester" + ) + aggregator.schedule( + group_key="media:test", + notification=notice, + callback=callback, + loop=loop, + ) + + with patch("app.application.transfer.logger.error") as log_error: + aggregator.close() + + callback.assert_called_once_with([notice]) + log_error.assert_called_once() + + def test_aggregated_message_contains_count_reason_stats_and_batch_entry(): """聚合消息应给出失败数、原因统计、历史 ID 和批量处理入口。""" chain = object.__new__(TransferChain) diff --git a/tests/test_transfer_pending_replay.py b/tests/test_transfer_pending_replay.py index ba37b5ba9..a0cde2173 100644 --- a/tests/test_transfer_pending_replay.py +++ b/tests/test_transfer_pending_replay.py @@ -8,11 +8,12 @@ 这些测试固定三项不变量：入队即落盘登记、终态即注销、重启能回放。 """ from pathlib import Path +import threading from unittest.mock import MagicMock from app.chain.transfer import TransferChain -from app.schemas import FileItem from app.application.transfer import TransferTask +from app.schemas.file import FileItem def _build_chain(pendingoper) -> TransferChain: @@ -231,3 +232,32 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch): chain._TransferChain__replay_pending() assert handled == ["B.mkv"] + + +def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch): + """ + 宿主关闭后不得继续检查或注销下一条登记，未处理项留给下次启动回放。 + """ + first = tmp_path / "A.mkv" + first.write_bytes(b"x") + missing_second = tmp_path / "gone.mkv" + pendingoper = MagicMock() + pendingoper.list_all.return_value = [ + ("local", str(first)), + ("local", str(missing_second)), + ] + chain = _build_chain(pendingoper) + stop_event = threading.Event() + transferred = [] + + def transfer_first(**kwargs): + """首条回放送入整理链后模拟宿主发出关闭信号。""" + transferred.append(kwargs["fileitem"].path) + stop_event.set() + + monkeypatch.setattr(chain, "do_transfer", transfer_first) + + chain._TransferChain__replay_pending(stop_event) + + assert transferred == [first.as_posix()] + pendingoper.discard.assert_not_called() diff --git a/tests/test_transfer_worker_lifecycle.py b/tests/test_transfer_worker_lifecycle.py new file mode 100644 index 000000000..fe678fcc1 --- /dev/null +++ b/tests/test_transfer_worker_lifecycle.py @@ -0,0 +1,485 @@ +"""文件整理 worker 与 pending 回放的宿主生命周期测试。""" + +import asyncio +import queue +import threading +import time +from concurrent.futures import Future +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.chain.transfer import TransferChain +from app.foundation.singleton import Singleton +from app.runtime.config import global_vars +from app.application.transfer import TransferQueue, TransferTask +from app.schemas.file import FileItem +from app.startup import transfer_initializer + + +def _build_chain(*, transfer_threads: int = 0) -> TransferChain: + """构造只包含后台线程生命周期字段的 TransferChain 测试骨架。""" + chain = object.__new__(TransferChain) + chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads) + chain._queue = queue.Queue() + chain._transfer_interval = 0.1 + chain._threads = [] + chain._retiring_threads = [] + chain._queue_active = False + chain._worker_stop_event = threading.Event() + chain._worker_lifecycle_lock = threading.RLock() + chain._worker_state_lock = threading.RLock() + chain._closing = False + chain._replay_thread = None + chain._replay_stop_event = threading.Event() + return chain + + +def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None: + """热更新应等待旧 worker 收敛，再启动使用独立停止信号的新一代。""" + chain = _build_chain(transfer_threads=1) + started_workers: queue.Queue = queue.Queue() + + def run_worker(stop_event: threading.Event) -> None: + """记录 worker 代际并等待该代专属停止信号。""" + started_workers.put((threading.current_thread(), stop_event)) + stop_event.wait() + + chain._TransferChain__start_transfer = run_worker + assert chain._TransferChain__init() is True + first_thread, first_stop_event = started_workers.get(timeout=1) + + chain.on_config_changed() + + second_thread, second_stop_event = started_workers.get(timeout=1) + assert first_stop_event.is_set() is True + assert first_thread.is_alive() is False + assert second_thread is not first_thread + assert second_stop_event is not first_stop_event + assert second_stop_event.is_set() is False + + service = MagicMock() + service.put.return_value = True + chain._transfer_queue_service = MagicMock(return_value=service) + task = MagicMock() + assert chain.put_to_queue(task) is True + service.put.assert_called_once() + + assert chain.close_workers(timeout_seconds=1) is True + assert second_thread.is_alive() is False + + +def test_config_reload_hands_queue_to_new_generation_while_old_io_finishes() -> None: + """旧代同步 I/O 超时不应让后续队列永久失去 worker。""" + chain = _build_chain(transfer_threads=1) + chain._WORKER_RESTART_TIMEOUT_SECONDS = 0.01 + started_workers: queue.Queue = queue.Queue() + release_old_worker = threading.Event() + invocation_count = 0 + invocation_lock = threading.Lock() + + def run_worker(stop_event: threading.Event) -> None: + """首代模拟不可取消 I/O，后续代按各自停止信号正常收敛。""" + nonlocal invocation_count + with invocation_lock: + generation = invocation_count + invocation_count += 1 + started_workers.put((threading.current_thread(), stop_event)) + if generation == 0: + release_old_worker.wait() + else: + stop_event.wait() + + chain._TransferChain__start_transfer = run_worker + assert chain._TransferChain__init() is True + old_thread, old_stop_event = started_workers.get(timeout=1) + + chain.on_config_changed() + + new_thread, new_stop_event = started_workers.get(timeout=1) + assert old_stop_event.is_set() is True + assert old_thread.is_alive() is True + assert chain._retiring_threads == [old_thread] + assert chain._threads == [new_thread] + assert new_stop_event.is_set() is False + + release_old_worker.set() + assert chain.close_workers(timeout_seconds=1) is True + assert old_thread.is_alive() is False + assert new_thread.is_alive() is False + + +def test_close_workers_is_bounded_and_retains_nonconverging_owner() -> None: + """同步 I/O 线程超时后应保留句柄并报告失败，不能伪装成已取消。""" + chain = _build_chain() + release = threading.Event() + thread = threading.Thread( + target=release.wait, + name="transfer-blocked-test", + daemon=True, + ) + chain._threads = [thread] + thread.start() + + started_at = time.monotonic() + assert chain.close_workers(timeout_seconds=0.01) is False + assert time.monotonic() - started_at < 0.5 + assert chain._threads == [] + assert chain._retiring_threads == [thread] + assert thread.is_alive() is True + + service = MagicMock() + chain._transfer_queue_service = MagicMock(return_value=service) + assert chain.put_to_queue(MagicMock()) is False + service.put.assert_not_called() + + release.set() + assert chain.close_workers(timeout_seconds=1) is True + assert chain.close_workers(timeout_seconds=0) is True + assert chain._threads == [] + assert chain._retiring_threads == [] + + +def test_close_workers_lock_wait_uses_the_same_timeout_budget() -> None: + """生命周期锁竞争必须耗用关闭预算，超时返回后不得延迟修改 worker 状态。""" + chain = _build_chain() + lock_acquired = threading.Event() + release_lock = threading.Event() + + def hold_lifecycle_lock() -> None: + """在独立线程持锁，稳定制造无法重入的生命周期锁竞争。""" + with chain._worker_lifecycle_lock: + lock_acquired.set() + assert release_lock.wait(timeout=1) + + holder = threading.Thread(target=hold_lifecycle_lock, daemon=True) + holder.start() + assert lock_acquired.wait(timeout=1) + + started_at = time.monotonic() + assert chain.close_workers(timeout_seconds=0.01) is False + assert time.monotonic() - started_at < 0.5 + assert chain._closing is False + assert chain._worker_stop_event.is_set() is False + assert chain._queue.empty() is True + + release_lock.set() + holder.join(timeout=1) + assert holder.is_alive() is False + assert chain.close_workers(timeout_seconds=1) is True + + +def test_close_keeps_timer_dependencies_when_workers_do_not_converge() -> None: + """活跃整理线程超时后，通知和重试 owner 必须继续供线程使用。""" + chain = _build_chain() + chain.close_workers = MagicMock(return_value=False) + chain.failure_notification_aggregator = MagicMock() + chain.retry_scheduler = MagicMock(close=AsyncMock()) + + completed = asyncio.run(chain.close(timeout_seconds=0.01)) + + assert completed is False + chain.close_workers.assert_called_once_with(0.01) + chain.failure_notification_aggregator.close.assert_not_called() + chain.retry_scheduler.close.assert_not_awaited() + + +def test_close_releases_timer_dependencies_after_workers_converge() -> None: + """worker 和回放退出后，整理链应继续刷新通知并关闭 AI 重试。""" + chain = _build_chain() + chain.close_workers = MagicMock(return_value=True) + chain.failure_notification_aggregator = MagicMock() + chain.retry_scheduler = MagicMock(close=AsyncMock()) + + completed = asyncio.run(chain.close(timeout_seconds=0.01)) + + assert completed is True + chain.failure_notification_aggregator.close.assert_called_once_with() + chain.retry_scheduler.close.assert_awaited_once_with() + + +def test_stop_transfer_runtime_does_not_construct_chain(monkeypatch) -> None: + """关闭入口在整理链从未使用时应直接成功，不能因关停而启动 worker。""" + get_existing_instance = MagicMock(return_value=None) + monkeypatch.setattr( + transfer_initializer.TransferChain, + "get_existing_instance", + get_existing_instance, + ) + + completed = asyncio.run( + transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01) + ) + + assert completed is True + get_existing_instance.assert_called_once_with() + + +def test_stop_transfer_runtime_closes_existing_chain(monkeypatch) -> None: + """关闭入口应把超时预算和真实收敛结果原样传给既有整理链。""" + chain = MagicMock(close=AsyncMock(return_value=False)) + monkeypatch.setattr( + transfer_initializer.TransferChain, + "get_existing_instance", + MagicMock(return_value=chain), + ) + + completed = asyncio.run( + transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01) + ) + + assert completed is False + chain.close.assert_awaited_once_with(timeout_seconds=0.01) + + +def test_constructor_failure_publishes_started_worker_to_cleanup(monkeypatch) -> None: + """首个 worker 启动后构造失败时，stop-only 入口仍必须找到并等待它。""" + instances = dict(Singleton._instances) + instances.pop((TransferChain, (), frozenset()), None) + monkeypatch.setattr(Singleton, "_instances", instances) + worker_started = threading.Event() + worker_release = threading.Event() + workers: list[threading.Thread] = [] + + def failing_init(chain: TransferChain) -> None: + """模拟第二个 owner 启动失败前已经成功启动一个整理线程。""" + worker = threading.Thread( + target=lambda: (worker_started.set(), worker_release.wait()), + name="transfer-partial-construction", + daemon=True, + ) + workers.append(worker) + worker.start() + + async def close(*, timeout_seconds: float) -> bool: + """模拟真实 close 释放并等待半构造实例已经发布的 worker。""" + worker_release.set() + worker.join(timeout=timeout_seconds) + return not worker.is_alive() + + chain.close = close + raise RuntimeError("second worker failed") + + monkeypatch.setattr(TransferChain, "__init__", failing_init) + + with pytest.raises(RuntimeError, match="second worker failed"): + TransferChain() + assert worker_started.wait(timeout=1) + retained = TransferChain.get_existing_instance() + assert retained is not None + + assert asyncio.run( + transfer_initializer.stop_transfer_runtime(timeout_seconds=1) + ) is True + assert workers[0].is_alive() is False + + +def test_failed_retry_schedule_future_error_is_observed() -> None: + """跨线程调度协程的延迟异常必须被取回并写入日志。""" + future: Future[None] = Future() + future.set_exception(RuntimeError("scheduler closed")) + + with patch("app.chain.transfer.logger.error") as log_error: + TransferChain._observe_failed_retry_schedule(future) + + log_error.assert_called_once() + assert "scheduler closed" in log_error.call_args.args[0] + + +def test_failed_retry_schedule_registers_future_observer(monkeypatch) -> None: + """整理线程提交 AI 重试后应让 Future 持续连接到异常观察回调。""" + chain = _build_chain() + + async def schedule_retry(_history_id: int, *, group_key: str) -> None: + """提供不会实际执行的调度协程，供跨线程提交边界检查。""" + + chain.retry_scheduler = MagicMock(schedule_retry=schedule_retry) + future = MagicMock(spec=Future) + event_loop = MagicMock() + monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", event_loop) + + def submit(coroutine, loop): + """关闭测试协程并返回可检查的并发 Future。""" + assert loop is event_loop + coroutine.close() + return future + + with patch( + "app.chain.transfer.asyncio.run_coroutine_threadsafe", + side_effect=submit, + ): + chain._schedule_failed_transfer_retry(42, "media:test") + + future.add_done_callback.assert_called_once() + callback = future.add_done_callback.call_args.args[0] + assert callback is TransferChain._observe_failed_retry_schedule + + +def test_worker_requeues_item_taken_during_shutdown(monkeypatch) -> None: + """停止信号与 queue.get 竞态时，未开始处理的任务必须原样放回队列。""" + chain = _build_chain() + work_queue = MagicMock() + chain._queue = work_queue + entered_get = threading.Event() + release_get = threading.Event() + item = TransferQueue() + + def get_item(*_args, **_kwargs): + """让停止信号稳定落在阻塞取队列之后、任务处理之前。""" + entered_get.set() + assert release_get.wait(timeout=1) + return item + + work_queue.get.side_effect = get_item + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + stop_event = threading.Event() + thread = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + thread.start() + assert entered_get.wait(timeout=1) + + stop_event.set() + release_get.set() + thread.join(timeout=1) + + assert thread.is_alive() is False + work_queue.put.assert_called_once_with(item) + work_queue.task_done.assert_called_once_with() + + +def test_worker_settles_progress_when_only_stop_sentinel_remains(monkeypatch) -> None: + """真实任务完成时仅剩停止哨兵，仍应结束进度并重置本批计数。""" + chain = _build_chain() + task = TransferTask( + fileitem=FileItem( + storage="local", + path="/downloads/movie.mkv", + type="file", + name="movie.mkv", + basename="movie", + extension="mkv", + ) + ) + chain.jobview = MagicMock() + chain.jobview.pending_total.return_value = 1 + chain._progress = MagicMock() + chain._active_tasks = 0 + chain._processed_num = 0 + chain._fail_num = 0 + chain._total_num = 0 + task_started = threading.Event() + release_task = threading.Event() + + def handle_transfer(*_args, **_kwargs): + """阻塞真实任务，让测试能在其完成前稳定插入停止哨兵。""" + task_started.set() + assert release_task.wait(timeout=1) + return True, "" + + chain._TransferChain__handle_transfer = handle_transfer + chain._TransferChain__start_job_execution = MagicMock() + chain._TransferChain__finish_job_execution = MagicMock() + chain._queue.put(TransferQueue(task=task)) + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + stop_event = threading.Event() + worker = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + worker.start() + assert task_started.wait(timeout=1) + + stop_event.set() + chain._queue.put(chain._QUEUE_STOP_SENTINEL) + release_task.set() + worker.join(timeout=1) + + assert worker.is_alive() is False + chain._progress.end.assert_called_once_with() + assert chain._active_tasks == 0 + assert chain._total_num == 0 + assert chain._processed_num == 0 + assert chain._fail_num == 0 + with chain._queue.mutex: + assert list(chain._queue.queue) == [chain._QUEUE_STOP_SENTINEL] + + +def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None: + """其他 worker 已取走真实任务但尚未登记 active 时，当前批次不得提前结算。""" + chain = _build_chain() + task = TransferTask( + fileitem=FileItem( + storage="local", + path="/downloads/claimed.mkv", + type="file", + name="claimed.mkv", + basename="claimed", + extension="mkv", + ) + ) + chain._progress = MagicMock() + chain._active_tasks = 0 + chain._processed_num = 1 + chain._fail_num = 0 + chain._total_num = 2 + claimed = threading.Event() + release_claim = threading.Event() + + chain._queue.put(TransferQueue(task=task)) + + def hold_claimed_task() -> None: + """模拟 worker 已完成 queue.get、尚未取得 task_lock 登记 active 的窗口。""" + item = chain._queue.get(timeout=1) + assert item.task is task + claimed.set() + assert release_claim.wait(timeout=1) + chain._queue.task_done() + chain._TransferChain__settle_transfer_progress_if_idle() + + worker = threading.Thread(target=hold_claimed_task, daemon=True) + worker.start() + assert claimed.wait(timeout=1) + + chain._TransferChain__settle_transfer_progress_if_idle() + + chain._progress.end.assert_not_called() + assert chain._processed_num == 1 + + release_claim.set() + worker.join(timeout=1) + + assert worker.is_alive() is False + chain._progress.end.assert_called_once_with() + assert chain._total_num == 0 + assert chain._processed_num == 0 + + +def test_replay_has_single_owner_and_close_waits_for_it() -> None: + """重复回放只保留一个线程，关闭会通知并等待该线程退出。""" + chain = _build_chain() + replay_started = threading.Event() + replay_calls = [] + + def replay(stop_event: threading.Event) -> None: + """模拟可由逐项检查点收敛的 pending 回放。""" + replay_calls.append(stop_event) + replay_started.set() + stop_event.wait() + + chain._TransferChain__replay_pending = replay + chain.replay_pending() + assert replay_started.wait(timeout=1) + replay_thread = chain._replay_thread + + chain.replay_pending() + + assert chain._replay_thread is replay_thread + assert replay_calls == [chain._replay_stop_event] + assert chain.close_workers(timeout_seconds=1) is True + assert replay_thread.is_alive() is False + assert chain._replay_thread is None