fix(database): harden worker shutdown and overload propagation

app/application/plugin/install.py

@@ -6,6 +6,8 @@ from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import Any, Optional
 
+from app.application.database import DatabaseWorkerOverloadedError
+
 
 InstalledPluginsReader = Callable[[], list[str]]
 InstalledPluginsWriter = Callable[[list[str]], Awaitable[object]]
@@ -127,7 +129,7 @@ class PluginInstallCommand:
                 force,
             )
         except Exception as err:
-            return await self._failure(
+            result = await self._failure(
                 plugin_id=plugin_id,
                 original_plugins=installed_plugins,
                 checkpoint=checkpoint,
@@ -135,6 +137,9 @@ class PluginInstallCommand:
                 message=str(err),
                 package_installed=False,
             )
+            if isinstance(err, DatabaseWorkerOverloadedError):
+                raise
+            return result
         if not state:
             return await self._failure(
                 plugin_id=plugin_id,
@@ -152,7 +157,7 @@ class PluginInstallCommand:
                 await self._installed_plugins_writer(updated_plugins)
                 installed_list_persisted = True
             except Exception as err:
-                return await self._failure(
+                result = await self._failure(
                     plugin_id=plugin_id,
                     original_plugins=installed_plugins,
                     checkpoint=checkpoint,
@@ -160,11 +165,14 @@ class PluginInstallCommand:
                     message=str(err),
                     package_installed=True,
                 )
+                if isinstance(err, DatabaseWorkerOverloadedError):
+                    raise
+                return result
 
         try:
             await self._plugin_reloader(plugin_id)
         except Exception as err:
-            return await self._failure(
+            result = await self._failure(
                 plugin_id=plugin_id,
                 original_plugins=installed_plugins,
                 checkpoint=checkpoint,
@@ -174,11 +182,14 @@ class PluginInstallCommand:
                 installed_list_persisted=installed_list_persisted,
                 runtime_touched=True,
             )
+            if isinstance(err, DatabaseWorkerOverloadedError):
+                raise
+            return result
 
         try:
             await self._registration_refresher(plugin_id)
         except Exception as err:
-            return await self._failure(
+            result = await self._failure(
                 plugin_id=plugin_id,
                 original_plugins=installed_plugins,
                 checkpoint=checkpoint,
@@ -189,6 +200,9 @@ class PluginInstallCommand:
                 runtime_touched=True,
                 registrations_touched=True,
             )
+            if isinstance(err, DatabaseWorkerOverloadedError):
+                raise
+            return result
 
         checkpoint_cleanup_error = ""
         try:
@@ -262,7 +276,7 @@ class PluginInstallCommand:
                     registrations_restored = True
                 except Exception as rollback_err:
                     rollback_errors.append(f"路由和服务注册恢复失败：{rollback_err}")
-            return PluginInstallResult(
+            result = PluginInstallResult(
                 success=False,
                 message=f"刷新插件运行态失败：{err}",
                 refreshed_only=True,
@@ -275,6 +289,9 @@ class PluginInstallCommand:
                     errors=tuple(rollback_errors),
                 ),
             )
+            if isinstance(err, DatabaseWorkerOverloadedError):
+                raise
+            return result
 
         reported = False
         report_error = ""

app/db/worker.py

@@ -67,6 +67,8 @@ class DatabaseWorker:
         ] = {}
         self._queued = 0
         self._running = 0
+        self._reported_queued = 0
+        self._reported_running = 0
         self._rejected = 0
         self._closing = False
 
@@ -200,12 +202,19 @@ class DatabaseWorker:
         )
         self._record_depth()
 
-    async def _wait_until_done(self, future: asyncio.Future[object]) -> None:
-        """忽略后续取消请求，直到线程内事务结束。"""
+    async def _wait_until_done(
+        self,
+        future: asyncio.Future[object],
+        *,
+        interruptible: bool = False,
+    ) -> None:
+        """等待线程内事务结束，并按调用场景决定是否响应外层取消。"""
         while not future.done():
             try:
                 await asyncio.shield(future)
             except asyncio.CancelledError:
+                if interruptible:
+                    raise
                 continue
             except BaseException:
                 break
@@ -224,7 +233,8 @@ class DatabaseWorker:
             future.cancel()
         for future, (wrapped, _item) in futures:
             if not future.cancelled():
-                await self._wait_until_done(wrapped)
+                # 关停超时必须能返回并保留 owner；已开始的数据库事务继续由线程完成。
+                await self._wait_until_done(wrapped, interruptible=True)
         executor.shutdown(wait=True, cancel_futures=True)
         while self.snapshot().queued or self.snapshot().running:
             await asyncio.sleep(0)
@@ -232,7 +242,13 @@ class DatabaseWorker:
         self._record_depth()
 
     def _record_depth(self) -> None:
-        """记录当前排队量与运行量。"""
+        """以状态变化量记录队列和运行中的任务数量。"""
         stats = self.snapshot()
-        record_metric("db.worker.queue.depth", stats.queued)
-        record_metric("db.worker.active", stats.running)
+        queued_delta = stats.queued - self._reported_queued
+        running_delta = stats.running - self._reported_running
+        if queued_delta:
+            record_metric("db.worker.queue.depth", queued_delta)
+        if running_delta:
+            record_metric("db.worker.active", running_delta)
+        self._reported_queued = stats.queued
+        self._reported_running = stats.running

app/factory.py

@@ -79,6 +79,7 @@ def _native_ai_error_response(
         protocol: str,
         status_code: int,
         message: str,
+        headers: dict[str, str] | None = None,
 ) -> JSONResponse:
     """按 OpenAI 或 Anthropic 兼容协议构造原生错误响应。"""
     if protocol == "openai":
@@ -98,6 +99,7 @@ def _native_ai_error_response(
                     code=error_type,
                 )
             ).model_dump(mode="json"),
+            headers=headers,
         )
 
     error_type = (
@@ -112,6 +114,7 @@ def _native_ai_error_response(
         content=AnthropicErrorResponse(
             error=AnthropicErrorDetail(type=error_type, message=message)
         ).model_dump(mode="json"),
+        headers=headers,
     )
 
 
@@ -119,6 +122,7 @@ def _mcp_jsonrpc_error_response(
         status_code: int,
         code: int,
         message: str,
+        headers: dict[str, str] | None = None,
 ) -> JSONResponse:
     """构造带 HTTP 状态码的 MCP JSON-RPC 原生错误响应。"""
     return JSONResponse(
@@ -128,6 +132,7 @@ def _mcp_jsonrpc_error_response(
             id=None,
             error=McpJsonRpcErrorDetail(code=code, message=message),
         ).model_dump(mode="json"),
+        headers=headers,
     )
 
 
@@ -202,6 +207,7 @@ async def localized_http_exception_handler(
             protocol=native_ai_protocol,
             status_code=exc.status_code,
             message=message,
+            headers=exc.headers,
         )
     if _is_mcp_jsonrpc_request(request):
         error_codes = {
@@ -215,6 +221,7 @@ async def localized_http_exception_handler(
             status_code=exc.status_code,
             code=error_codes.get(exc.status_code, -32000),
             message=message,
+            headers=exc.headers,
         )
     return JSONResponse(
         status_code=exc.status_code,