refactor: own shutdown lifecycle boundaries

app/agent/tools/base.py

@@ -3,7 +3,7 @@ import inspect
 import json
 import threading
 from abc import ABCMeta, abstractmethod
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
 from contextvars import Context, copy_context
 from functools import partial
 from pathlib import Path
@@ -168,33 +168,130 @@ _blocking_semaphores = {
     for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
 }
 _blocking_executors: dict[str, ThreadPoolExecutor] = {}
-_blocking_executor_lock = threading.Lock()
+_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
+_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
+_blocking_executor_lock = threading.RLock()
+_blocking_executor_accepting = True
 
 
-def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
-    """按桶懒加载线程池，避免在导入阶段创建过多 worker。"""
+def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
+    """在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
     with _blocking_executor_lock:
+        executor = _blocking_futures.pop(future, None)
+        if executor is None or executor not in _blocking_retiring_executors:
+            return
+        if executor not in _blocking_futures.values():
+            _blocking_retiring_executors.discard(executor)
+
+
+def _submit_blocking_call(
+    bucket: str,
+    bound_call: Callable[[], Any],
+) -> ConcurrentFuture[Any]:
+    """在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
+    context = copy_context()
+    with _blocking_executor_lock:
+        if not _blocking_executor_accepting:
+            raise RuntimeError("Agent 工具阻塞执行器正在关闭，不能再提交新任务")
         executor = _blocking_executors.get(bucket)
-        if executor:
-            return executor
-
-        limit = _BLOCKING_BUCKET_LIMITS[bucket]
-        executor = ThreadPoolExecutor(
-            max_workers=limit,
-            thread_name_prefix=f"agent-tool-{bucket}",
-        )
-        _blocking_executors[bucket] = executor
-        return executor
+        if executor is None:
+            limit = _BLOCKING_BUCKET_LIMITS[bucket]
+            executor = ThreadPoolExecutor(
+                max_workers=limit,
+                thread_name_prefix=f"agent-tool-{bucket}",
+            )
+            _blocking_executors[bucket] = executor
+        # 长期 worker 保持空底层上下文，每个任务只在自己的调用快照内运行。
+        future = Context().run(executor.submit, context.run, bound_call)
+        _blocking_futures[future] = executor
+        future.add_done_callback(_discard_blocking_future)
+        return future
 
 
-def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
-    """关闭 Agent 工具阻塞线程池，释放长期运行进程或测试环境中的 worker。"""
+def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
+    """撤销活动 executor 的提交资格，并保留其运行 Future 对应的 owner。"""
     with _blocking_executor_lock:
-        executors = list(_blocking_executors.values())
+        executors = tuple(_blocking_executors.values())
         _blocking_executors.clear()
-
+        _blocking_retiring_executors.update(executors)
     for executor in executors:
-        executor.shutdown(wait=wait, cancel_futures=cancel_futures)
+        executor.shutdown(wait=False, cancel_futures=cancel_futures)
+    with _blocking_executor_lock:
+        owned_executors = set(_blocking_futures.values())
+        _blocking_retiring_executors.intersection_update(owned_executors)
+    return executors
+
+
+def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
+    """原子封住新阻塞工具提交，并请求取消尚未开始的同步调用。"""
+    global _blocking_executor_accepting
+    with _blocking_executor_lock:
+        _blocking_executor_accepting = False
+    _retire_blocking_executors(cancel_futures=cancel_futures)
+
+
+def reopen_blocking_executors() -> bool:
+    """仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
+    global _blocking_executor_accepting
+    with _blocking_executor_lock:
+        if _blocking_futures or _blocking_retiring_executors:
+            return False
+        _blocking_executor_accepting = True
+        return True
+
+
+async def close_blocking_executors(
+    *,
+    timeout_seconds: float,
+    cancel_futures: bool = True,
+) -> bool:
+    """有限等待全部阻塞工具 Future，超时保留 Future 与 executor owner。"""
+    begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
+    with _blocking_executor_lock:
+        futures = tuple(_blocking_futures)
+    wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
+    if wrapped_futures:
+        done, pending = await asyncio.wait(
+            wrapped_futures,
+            timeout=max(0.0, timeout_seconds),
+        )
+        if done:
+            await asyncio.gather(*done, return_exceptions=True)
+        for pending_future in pending:
+            pending_future.add_done_callback(
+                lambda completed: completed.exception()
+                if not completed.cancelled()
+                else None
+            )
+
+    with _blocking_executor_lock:
+        unfinished = tuple(
+            future for future in _blocking_futures if not future.done()
+        )
+        retiring_count = len(_blocking_retiring_executors)
+    if unfinished:
+        logger.error(
+            "Agent 阻塞工具未在 %.1f 秒内收敛：futures=%d，executors=%d",
+            max(0.0, timeout_seconds),
+            len(unfinished),
+            retiring_count,
+        )
+        return False
+    return True
+
+
+def shutdown_blocking_executors(
+    *,
+    wait: bool = True,
+    cancel_futures: bool = False,
+) -> bool:
+    """同步清理测试 owner；非等待模式下保留尚未收敛的 executor 句柄。"""
+    executors = _retire_blocking_executors(cancel_futures=cancel_futures)
+    for executor in executors:
+        if wait:
+            executor.shutdown(wait=True, cancel_futures=cancel_futures)
+    with _blocking_executor_lock:
+        return not _blocking_futures and not _blocking_retiring_executors
 
 
 class ToolExecutionTimeoutError(TimeoutError):
@@ -226,13 +323,7 @@ async def run_agent_blocking(
 
     await semaphore.acquire()
     try:
-        context = copy_context()
-        # 长期 worker 保持空底层上下文，每个任务只在自己的调用快照内运行。
-        future = Context().run(
-            _get_blocking_executor(bucket_name).submit,
-            context.run,
-            bound_call,
-        )
+        future = _submit_blocking_call(bucket_name, bound_call)
     except Exception:
         semaphore.release()
         raise

app/agent/tools/impl/_plugin_tool_utils.py

@@ -2,6 +2,7 @@
 
 import json
 import shutil
+from contextvars import copy_context
 from pathlib import Path
 from typing import Any, Optional
 
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
 
 def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
     """重载插件实例并重新注册其命令、定时任务和 API。"""
-    runtime_status = get_plugin_manager().reload_plugin(plugin_id)
-    refresh_plugin_registrations(plugin_id)
-    return runtime_status
+    plugin_manager = get_plugin_manager()
+    with plugin_manager.mutation(f"重载插件 {plugin_id}"):
+        runtime_status = plugin_manager.reload_plugin(plugin_id)
+        refresh_plugin_registrations(plugin_id)
+        return runtime_status
 
 
 def summarize_plugin(plugin: Any) -> dict[str, Any]:
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
 
     async def reload_runtime(target_id: str) -> object:
         """通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
+        mutation_context = copy_context()
         return await run_agent_blocking(
             "plugin",
+            mutation_context.run,
             plugin_manager.reload_plugin_tree,
             target_id,
         )
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
             )
         return result
 
-    with plugin_manager.suppress_plugin_monitor(plugin_id):
-        result = await PluginInstallCommand(
-            installed_plugins_reader=lambda: SystemConfigOper().get(
-                SystemConfigKey.UserInstalledPlugins
-            ) or [],
-            installed_plugins_writer=save_installed_plugins,
-            plugin_ids_provider=plugin_manager.get_plugin_ids,
-            compatibility_checker=skip_compatibility_check,
-            package_installer=install_package,
-            package_checkpointer=package_manager.async_checkpoint,
-            package_committer=package_manager.async_commit,
-            package_rollback=package_manager.async_rollback,
-            install_reporter=lambda target_id, target_repo: (
-                MoviePilotServerHelper.async_install_plugin_reg(
-                    plugin_id=target_id,
-                    repo_url=target_repo,
-                )
-            ),
-            plugin_reloader=reload_runtime,
-            registration_refresher=refresh_registrations,
-        ).execute(
-            plugin_id=plugin_id,
-            repo_url=repo_url,
-            force=force,
-        )
+    result = await PluginInstallCommand(
+        installed_plugins_reader=lambda: SystemConfigOper().get(
+            SystemConfigKey.UserInstalledPlugins
+        ) or [],
+        installed_plugins_writer=save_installed_plugins,
+        plugin_ids_provider=plugin_manager.get_plugin_ids,
+        compatibility_checker=skip_compatibility_check,
+        package_installer=install_package,
+        package_checkpointer=package_manager.async_checkpoint,
+        package_committer=package_manager.async_commit,
+        package_rollback=package_manager.async_rollback,
+        install_reporter=lambda target_id, target_repo: (
+            MoviePilotServerHelper.async_install_plugin_reg(
+                plugin_id=target_id,
+                repo_url=target_repo,
+            )
+        ),
+        plugin_reloader=reload_runtime,
+        registration_refresher=refresh_registrations,
+        mutation=plugin_manager.mutation,
+        package_write_guard=plugin_manager.suppress_plugin_monitor,
+    ).execute(
+        plugin_id=plugin_id,
+        repo_url=repo_url,
+        force=force,
+    )
     return result.success, result.message, result.refreshed_only
 
 
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
     from app.agent.tools.base import run_agent_blocking
 
     plugin_manager = get_plugin_manager()
-    virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
-    source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
-    if not virtual_instance and source_instances:
-        instance_ids = "、".join(item.instance_id for item in source_instances)
-        raise ValueError(f"请先卸载该插件的分身：{instance_ids}")
+    with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
+        virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
+        source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
+        if not virtual_instance and source_instances:
+            instance_ids = "、".join(item.instance_id for item in source_instances)
+            raise ValueError(f"请先卸载该插件的分身：{instance_ids}")
 
-    config_oper = SystemConfigOper()
-    install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
-    if plugin_id in install_plugins:
-        install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
-        await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
-
-    remove_plugin_api(plugin_id)
-    remove_plugin_job(plugin_id)
-
-    plugin_class = plugin_manager.plugins.get(plugin_id)
-    was_clone = bool(getattr(plugin_class, "is_clone", False))
-    clone_files_removed = False
-
-    if virtual_instance:
-        plugin_manager.delete_plugin_config(plugin_id, force=True)
-        plugin_manager.delete_plugin_data(plugin_id, force=True)
-        plugin_manager.delete_plugin_instance(plugin_id)
-    elif was_clone:
-        plugin_manager.delete_plugin_config(plugin_id)
-        plugin_manager.delete_plugin_data(plugin_id)
-        plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
-        try:
-            clone_files_removed = await run_agent_blocking(
-                "plugin",
-                _remove_plugin_directory,
-                plugin_base_dir,
+        config_oper = SystemConfigOper()
+        install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
+        if plugin_id in install_plugins:
+            install_plugins = [
+                plugin for plugin in install_plugins if plugin != plugin_id
+            ]
+            await config_oper.async_set(
+                SystemConfigKey.UserInstalledPlugins,
+                install_plugins,
             )
-            if clone_files_removed:
-                plugin_manager.plugins.pop(plugin_id, None)
-        except Exception:
-            clone_files_removed = False
 
-    remove_plugin_from_folders(plugin_id)
-    plugin_manager.remove_plugin(plugin_id)
+        remove_plugin_api(plugin_id)
+        remove_plugin_job(plugin_id)
 
-    return {
-        "was_clone": was_clone,
-        "clone_files_removed": clone_files_removed,
-    }
+        plugin_class = plugin_manager.plugins.get(plugin_id)
+        was_clone = bool(getattr(plugin_class, "is_clone", False))
+        clone_files_removed = False
+
+        if virtual_instance:
+            plugin_manager.delete_plugin_config(plugin_id, force=True)
+            plugin_manager.delete_plugin_data(plugin_id, force=True)
+            plugin_manager.delete_plugin_instance(plugin_id)
+        elif was_clone:
+            plugin_manager.delete_plugin_config(plugin_id)
+            plugin_manager.delete_plugin_data(plugin_id)
+            plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
+            try:
+                clone_files_removed = await run_agent_blocking(
+                    "plugin",
+                    _remove_plugin_directory,
+                    plugin_base_dir,
+                )
+                if clone_files_removed:
+                    plugin_manager.plugins.pop(plugin_id, None)
+            except Exception:
+                clone_files_removed = False
+
+        remove_plugin_from_folders(plugin_id)
+        plugin_manager.remove_plugin(plugin_id)
+
+        return {
+            "was_clone": was_clone,
+            "clone_files_removed": clone_files_removed,
+        }

app/agent/tools/impl/update_plugin_config.py

@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
             )
 
         plugin_manager = get_plugin_manager()
-        current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
+        with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
+            current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
 
-        # merge 模式以当前保存值为基准，replace 模式则从空配置开始重建。
-        next_config = {} if replace else dict(current_config)
-        if updates:
-            next_config.update(updates)
-        for key in remove_keys:
-            next_config.pop(key, None)
+            # merge 模式以当前保存值为基准，replace 模式则从空配置开始重建。
+            next_config = {} if replace else dict(current_config)
+            if updates:
+                next_config.update(updates)
+            for key in remove_keys:
+                next_config.pop(key, None)
 
-        changed_keys = sorted(
-            key
-            for key in set(current_config.keys()) | set(next_config.keys())
-            if current_config.get(key) != next_config.get(key)
-            or (key in current_config) != (key in next_config)
-        )
-
-        if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
-            return json.dumps(
-                {
-                    "success": False,
-                    "message": f"保存插件 {plugin_id} 配置失败",
-                },
-                ensure_ascii=False,
+            changed_keys = sorted(
+                key
+                for key in set(current_config.keys()) | set(next_config.keys())
+                if current_config.get(key) != next_config.get(key)
+                or (key in current_config) != (key in next_config)
             )
 
-        return json.dumps(
-            {
-                "success": True,
-                **plugin_info,
-                "message": "插件配置已保存，请调用 reload_plugin 使最新配置生效",
-                "replace": replace,
-                "changed_keys": changed_keys,
-                "removed_keys": remove_keys,
-                "config_requires_reload": True,
-                "previous_config": current_config,
-                "saved_config": next_config,
-            },
-            ensure_ascii=False,
-            indent=2,
-            default=str,
-        )
+            if not await plugin_manager.async_save_plugin_config(
+                plugin_id,
+                next_config,
+            ):
+                return json.dumps(
+                    {
+                        "success": False,
+                        "message": f"保存插件 {plugin_id} 配置失败",
+                    },
+                    ensure_ascii=False,
+                )
+
+            return json.dumps(
+                {
+                    "success": True,
+                    **plugin_info,
+                    "message": "插件配置已保存，请调用 reload_plugin 使最新配置生效",
+                    "replace": replace,
+                    "changed_keys": changed_keys,
+                    "removed_keys": remove_keys,
+                    "config_requires_reload": True,
+                    "previous_config": current_config,
+                    "saved_config": next_config,
+                },
+                ensure_ascii=False,
+                indent=2,
+                default=str,
+            )
 
     async def run(
         self,