refactor: own shutdown lifecycle boundaries

2026-09-05 23:47:41 +08:00 · 2026-08-23 20:20:26 +08:00
parent 59f020f226
commit 7f09927c47
59 changed files with 6393 additions and 958 deletions
@@ -0,0 +1,112 @@
+"""插件宿主可变事务的停机准入。"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+
+from app.schemas.exception import PluginMutationRejectedError
+
+
+@dataclass(slots=True)
+class _MutationContext:
+    """记录一个可跨协程和受控线程传播的事务上下文。"""
+
+    admission: "PluginMutationAdmission"
+    holders: int = 0
+    open: bool = True
+
+
+class PluginMutationAdmission:
+    """在停机封口与插件可变事务之间维护真实 owner 计数。"""
+
+    def __init__(self) -> None:
+        """初始化开放准入、活动 owner 计数和事务上下文。"""
+        self._condition = threading.Condition()
+        self._accepting = True
+        self._active_count = 0
+        self._current_context: ContextVar[_MutationContext | None] = ContextVar(
+            "plugin_mutation_context",
+            default=None,
+        )
+
+    @property
+    def active_count(self) -> int:
+        """返回尚未退出的 lease 数量，用于停机诊断和严格卸载判断。"""
+        with self._condition:
+            return self._active_count
+
+    @property
+    def accepting(self) -> bool:
+        """返回当前生命周期是否仍接纳新的根事务。"""
+        with self._condition:
+            return self._accepting
+
+    def is_held(self) -> bool:
+        """判断当前执行上下文是否持有仍有效的本 admission lease。"""
+        context = self._current_context.get()
+        return bool(
+            context
+            and context.admission is self
+            and context.open
+            and context.holders > 0
+        )
+
+    @contextmanager
+    def hold(self, operation: str) -> Iterator[None]:
+        """取得可变事务 lease；封口后仅允许已获准事务的嵌套调用。"""
+        context = self._current_context.get()
+        nested = bool(
+            context
+            and context.admission is self
+            and context.open
+            and context.holders > 0
+        )
+        context_token = None
+        if not nested:
+            context = _MutationContext(admission=self)
+            context_token = self._current_context.set(context)
+        assert context is not None
+
+        acquired = False
+        try:
+            with self._condition:
+                if not self._accepting and not nested:
+                    raise PluginMutationRejectedError(operation)
+                self._active_count += 1
+                context.holders += 1
+                acquired = True
+            yield
+        finally:
+            if acquired:
+                with self._condition:
+                    self._active_count -= 1
+                    context.holders -= 1
+                    if context.holders == 0:
+                        context.open = False
+                    self._condition.notify_all()
+            if context_token is not None:
+                self._current_context.reset(context_token)
+
+    def seal(self) -> int:
+        """原子停止接纳根事务，并返回封口瞬间的活动 lease 数量。"""
+        with self._condition:
+            self._accepting = False
+            return self._active_count
+
+    def wait_until_idle(self) -> None:
+        """自然等待全部已获准 lease 退出，不取消或遗失其 owner。"""
+        with self._condition:
+            while self._active_count:
+                self._condition.wait()
+
+    def reopen(self) -> bool:
+        """仅在没有遗留 lease 时为新的应用生命周期重新开放准入。"""
+        with self._condition:
+            if self._active_count:
+                return False
+            self._accepting = True
+            return True
@@ -5,6 +5,7 @@ from __future__ import annotations
 import traceback
 from collections.abc import Callable
 from functools import wraps
+import threading
 import time
 from typing import Any, Optional, ParamSpec, TypeVar, cast

@@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
            try:
                result = func(*args, **kwargs)
                statuses = result.values() if isinstance(result, dict) else (result,)
-                if PluginRuntimeStatus.LOAD_FAILED in statuses:
+                if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses:
                    outcome = "error"
                return result
            except BaseException:
@@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
 class PluginLifecycle:
    """管理插件发现、初始化、启停和热重载，不持有市场或 HTTP 路由职责。"""

+    _EVENT_HANDLERS_QUIESCED = "__event_handlers__"
+
    def __init__(
        self,
        *,
@@ -83,6 +86,8 @@ class PluginLifecycle:
        self._runtime_status_writer = runtime_status_writer
        self._logger = log
        self._event_sender = event_sender
+        self._lifecycle_lock = threading.RLock()
+        self._quiesced_hooks: dict[str, set[str]] = {}

    @observe_plugin_lifecycle("start")
    def start(
@@ -116,6 +121,7 @@ class PluginLifecycle:
                self._classes[current_id] = plugin
                instance = plugin()
                instance.init_plugin(self._plugin_config(current_id))
+                self._quiesced_hooks.pop(current_id, None)
                self._running[current_id] = instance
                self._logger.info(
                    f"加载插件：{current_id} 版本：{instance.plugin_version}"
@@ -156,31 +162,168 @@ class PluginLifecycle:

    @observe_plugin_lifecycle("stop")
    def stop(self, plugin_id: Optional[str] = None) -> None:
-        """停止指定插件或全部插件，并清理模块缓存。"""
+        """按旧单阶段 ABI 先解绑 handler，再停止并强制卸载插件。"""
+        with self._lifecycle_lock:
+            plugins = self._select_running_plugins(plugin_id)
+            self._quiesce_selected(plugins)
+            self._finalize(
+                plugin_id,
+                require_quiesced=False,
+                disable_events=not self._handlers_quiesced(plugins),
+            )
+
+    @observe_plugin_lifecycle("quiesce")
+    def quiesce(self, plugin_id: Optional[str] = None) -> bool:
+        """先解绑事件 handler，再按旧 hook 顺序停止生产者并保留实例。"""
+        with self._lifecycle_lock:
+            plugins = self._select_running_plugins(plugin_id)
+            return self._quiesce_selected(plugins)
+
+    @observe_plugin_lifecycle("quiesce_handlers")
+    def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool:
+        """禁止目标插件接收新事件，保留实例供在途 handler 和后续 hook 使用。"""
+        with self._lifecycle_lock:
+            plugins = self._select_running_plugins(plugin_id)
+            return self._disable_selected_handlers(plugins)
+
+    @observe_plugin_lifecycle("quiesce_services")
+    def quiesce_services(self, plugin_id: Optional[str] = None) -> bool:
+        """在事件结算屏障后执行旧 close、stop_service hook。"""
+        with self._lifecycle_lock:
+            plugins = self._select_running_plugins(plugin_id)
+            if not self._handlers_quiesced(plugins):
+                self._logger.warning("插件事件 handler 尚未全部停用，拒绝关闭插件资源")
+                return False
+            return self._quiesce_hooks(plugins)
+
+    def _quiesce_selected(self, plugins: dict[str, Any]) -> bool:
+        """兼容单阶段调用：先停用全部 handler，再执行稳定快照的旧 hooks。"""
+        if not self._disable_selected_handlers(plugins):
+            return False
+        return self._quiesce_hooks(plugins)
+
+    def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool:
+        """先停用稳定快照的全部事件入口，任一失败时不执行破坏性 hook。"""
+        all_converged = True
+        for current_id, plugin in plugins.items():
+            completed = self._quiesced_hooks.setdefault(current_id, set())
+            if self._EVENT_HANDLERS_QUIESCED in completed:
+                continue
+            try:
+                self._disable_events(type(plugin))
+            except Exception as error:  # noqa: BLE001  插件边界必须隔离
+                all_converged = False
+                self._logger.warning(
+                    f"停用插件 {current_id} 的事件 handler 时发生错误: {error}"
+                )
+                continue
+            completed.add(self._EVENT_HANDLERS_QUIESCED)
+        return all_converged
+
+    def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool:
+        """执行旧 ABI hooks，并只重试尚未成功的步骤。"""
+        all_converged = True
+        for current_id, plugin in plugins.items():
+            completed = self._quiesced_hooks.setdefault(current_id, set())
+            for hook_name in ("close", "stop_service"):
+                if hook_name in completed:
+                    continue
+                hook = getattr(plugin, hook_name, None)
+                if not callable(hook):
+                    completed.add(hook_name)
+                    continue
+                try:
+                    result = hook()
+                except Exception as error:  # noqa: BLE001  插件边界必须隔离
+                    all_converged = False
+                    self._logger.warning(
+                        f"停止插件 {current_id} 的 {hook_name} 时发生错误: {error}"
+                    )
+                    continue
+                if result is False:
+                    all_converged = False
+                    self._logger.warning(
+                        f"停止插件 {current_id} 的 {hook_name} 未收敛"
+                    )
+                    continue
+                completed.add(hook_name)
+        return all_converged
+
+    @observe_plugin_lifecycle("finalize")
+    def finalize(self, plugin_id: Optional[str] = None) -> bool:
+        """在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。"""
+        return self._finalize(
+            plugin_id,
+            require_quiesced=True,
+            disable_events=False,
+        )
+
+    def _finalize(
+        self,
+        plugin_id: Optional[str],
+        *,
+        require_quiesced: bool,
+        disable_events: bool = True,
+    ) -> bool:
+        """按严格或兼容策略卸载插件，并在清理失败时保留实例所有权。"""
+        with self._lifecycle_lock:
+            plugins = self._select_running_plugins(plugin_id)
+            if require_quiesced and any(
+                not self._is_quiesced(current_id, plugin)
+                for current_id, plugin in plugins.items()
+            ):
+                self._logger.warning("插件后台服务尚未全部收敛，拒绝卸载运行实例")
+                return False
+
+            try:
+                if disable_events:
+                    for plugin in plugins.values():
+                        self._disable_events(type(plugin))
+                self._clear_modules(plugin_id)
+                self._clear_tools()
+            except Exception as error:  # noqa: BLE001  保留实例所有权供后续重试
+                self._logger.warning(f"卸载插件运行实例时发生错误: {error}")
+                return False
+
+            if plugin_id:
+                self._classes.pop(plugin_id, None)
+                self._running.pop(plugin_id, None)
+                self._quiesced_hooks.pop(plugin_id, None)
+            else:
+                self._classes.clear()
+                self._running.clear()
+                self._quiesced_hooks.clear()
+            self._logger.info("插件停止完成")
+            return True
+
+    def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]:
+        """返回本阶段处理的稳定实例快照，并保持旧停机日志语义。"""
        if plugin_id:
            self._logger.info(f"正在停止插件 {plugin_id}...")
            plugin = self._running.get(plugin_id)
            plugins = {plugin_id: plugin} if plugin else {}
            if not plugin:
                self._logger.debug(f"插件 {plugin_id} 不存在或未加载")
-        else:
-            self._logger.info("正在停止所有插件...")
-            plugins = dict(self._running)
+            return plugins
+        self._logger.info("正在停止所有插件...")
+        return dict(self._running)

-        for current_id, plugin in plugins.items():
-            self._disable_events(type(plugin))
-            self._stop_plugin(plugin)
+    def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool:
+        """判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。"""
+        required = {self._EVENT_HANDLERS_QUIESCED} | {
+            hook_name
+            for hook_name in ("close", "stop_service")
+            if callable(getattr(plugin, hook_name, None))
+        }
+        return required.issubset(self._quiesced_hooks.get(plugin_id, set()))

-        if plugin_id:
-            self._classes.pop(plugin_id, None)
-            self._running.pop(plugin_id, None)
-            self._clear_modules(plugin_id)
-        else:
-            self._classes.clear()
-            self._running.clear()
-            self._clear_modules(None)
-        self._clear_tools()
-        self._logger.info("插件停止完成")
+    def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool:
+        """判断稳定快照中的全部插件是否已经停用事件入口。"""
+        return all(
+            self._EVENT_HANDLERS_QUIESCED
+            in self._quiesced_hooks.get(plugin_id, set())
+            for plugin_id in plugins
+        )

    @observe_plugin_lifecycle("reload")
    def reload(
@@ -194,14 +337,3 @@ class PluginLifecycle:
        status = self.start(plugin_id)[plugin_id]
        self._event_sender(reload_event, data={"plugin_id": plugin_id})
        return status
-
-    def _stop_plugin(self, plugin: Any) -> None:
-        """按插件旧 ABI 顺序关闭资源和服务。"""
-        try:
-            if hasattr(plugin, "close"):
-                plugin.close()
-            if hasattr(plugin, "stop_service"):
-                plugin.stop_service()
-        except Exception as error:  # noqa: BLE001
-            name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__
-            self._logger.warning(f"停止插件 {name} 时发生错误: {error}")
@@ -2,8 +2,8 @@

 from __future__ import annotations

-import time
 import threading
+import time
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any, Optional
@@ -27,6 +27,8 @@ class PluginMonitorController:
        self._logger = log
        self._thread: Optional[threading.Thread] = None
        self._stop_event = threading.Event()
+        self._lifecycle_lock = threading.RLock()
+        self._closed = False

    @property
    def stop_event(self) -> threading.Event:
@@ -35,32 +37,73 @@ class PluginMonitorController:

    def reload(self, enabled: bool) -> None:
        """按当前配置停止旧线程，并在启用时创建新线程。"""
-        self.stop()
-        if enabled:
+        stopped = self.stop()
+        if enabled and stopped:
            self.start()

    def start(self) -> None:
        """启动唯一的守护监控线程。"""
-        if self._thread and self._thread.is_alive():
-            self._logger.info("插件文件修改监测已经在运行中...")
-            return
-        self._logger.info("开始监测插件文件修改...")
-        self._stop_event.clear()
-        self._thread = threading.Thread(target=self._runner, daemon=True)
-        self._thread.start()
+        with self._lifecycle_lock:
+            if self._closed:
+                self._logger.info("插件文件修改监测已进入停机封口，跳过启动")
+                return
+            if self._thread and self._thread.is_alive():
+                self._logger.info("插件文件修改监测已经在运行中...")
+                return
+            self._logger.info("开始监测插件文件修改...")
+            self._stop_event.clear()
+            self._thread = threading.Thread(target=self._runner, daemon=True)
+            self._thread.start()

-    def stop(self) -> None:
-        """请求监控线程退出，并在限定时间内等待其清理。"""
-        if not self._thread or not self._thread.is_alive():
-            self._logger.info("未启用插件文件修改监测，无需停止")
-            return
-        self._logger.info("正在停止插件文件修改监测...")
+    def reopen(self) -> bool:
+        """为新的应用生命周期解除封口，仍有旧线程时拒绝重开。"""
+        with self._lifecycle_lock:
+            if self._thread and self._thread.is_alive():
+                self._logger.warning("旧插件文件监测线程仍在运行，无法开启新生命周期")
+                return False
+            self._thread = None
+            self._closed = False
+            return True
+
+    def stop(self, timeout: float = 5.0) -> bool:
+        """临时停止监控线程，并返回其是否在预算内真正退出。"""
+        return self._stop_with_budget(timeout=timeout, close=False)
+
+    def close(self, timeout: float = 5.0) -> bool:
+        """永久封口当前生命周期，并返回监控线程是否真正退出。"""
+        return self._stop_with_budget(timeout=timeout, close=True)
+
+    def _stop_with_budget(self, *, timeout: float, close: bool) -> bool:
+        """在同一预算内取得生命周期锁、设置封口并等待线程退出。"""
+        timeout = max(0.0, timeout)
+        deadline = time.monotonic() + timeout
        self._stop_event.set()
-        self._thread.join(timeout=5)
-        if self._thread.is_alive():
-            self._logger.warning("插件文件修改监测线程在5秒内未能正常停止。")
-        self._thread = None
-        self._logger.info("插件文件修改监测停止完成")
+        if not self._lifecycle_lock.acquire(timeout=timeout):
+            self._logger.warning(
+                f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权。"
+            )
+            return False
+        try:
+            if close:
+                self._closed = True
+            thread = self._thread
+            self._stop_event.set()
+            if not thread or not thread.is_alive():
+                self._thread = None
+                self._logger.info("未启用插件文件修改监测，无需停止")
+                return True
+            self._logger.info("正在停止插件文件修改监测...")
+            thread.join(timeout=max(0.0, deadline - time.monotonic()))
+            if thread.is_alive():
+                self._logger.warning(
+                    f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。"
+                )
+                return False
+            self._thread = None
+            self._logger.info("插件文件修改监测停止完成")
+            return True
+        finally:
+            self._lifecycle_lock.release()


 class PluginChangeMonitor: