mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-04 15:09:46 +08:00
refactor: unify host threadpool boundary
This commit is contained in:
jxxghp
@@ -3,8 +3,7 @@ import re
|
||||
import threading
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.agent.policy.sanitizer import sanitize_for_host
|
||||
from app.chain import ChainBase
|
||||
from app.runtime.log import logger
|
||||
|
||||
@@ -11,7 +11,6 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.messages import ( # noqa: F401
|
||||
HumanMessage,
|
||||
@@ -21,6 +20,7 @@ from langchain_core.messages import ( # noqa: F401
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.agent.callback import StreamingHandler
|
||||
from app.agent.contracts import ReplyMode, build_display_message
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
|
||||
@@ -14,9 +14,9 @@ from typing import Any, AsyncIterator, Awaitable, Callable, Optional, Union
|
||||
|
||||
import aiofiles
|
||||
from fastapi import Depends, File, Form, HTTPException, Request, UploadFile, status
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.schemas.agent import AgentChatDisplaySaveRequest as _SchemaAgentChatDisplaySaveRequest
|
||||
from app.schemas.agent import AgentChatSessionDetail as _SchemaAgentChatSessionDetail
|
||||
from app.schemas.agent import AgentChatSessionSummary as _SchemaAgentChatSessionSummary
|
||||
|
||||
@@ -2,8 +2,8 @@ from pathlib import Path
|
||||
from typing import Any, List, Optional, Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.schemas.dashboard import DashboardMemoryInfo as _SchemaDashboardMemoryInfo
|
||||
from app.schemas.dashboard import DashboardSystemInfo as _SchemaDashboardSystemInfo
|
||||
from app.schemas.dashboard import DownloaderInfo as _SchemaDownloaderInfo
|
||||
|
||||
@@ -6,10 +6,10 @@ from typing import Annotated, Any, Dict, List, Optional
|
||||
import aiofiles
|
||||
from anyio import Path as AsyncPath
|
||||
from fastapi import Depends, Header, HTTPException, Security
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from starlette import status
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.schemas.common import JsonObject as _SchemaJsonObject
|
||||
from app.schemas.plugin import Plugin as _SchemaPlugin
|
||||
from app.schemas.plugin import PluginDashboard as _SchemaPluginDashboard
|
||||
|
||||
@@ -7,8 +7,7 @@
|
||||
import copy
|
||||
from typing import Optional
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.application.configuration import get_configured_system_config
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
|
||||
+1
-2
@@ -3,8 +3,7 @@ from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.schemas.event import MediaRecognizeConvertEventData as _SchemaMediaRecognizeConvertEventData
|
||||
from app.chain import ChainBase
|
||||
from app.chain.acoustid import AcoustIdChain
|
||||
|
||||
+1
-2
@@ -10,8 +10,7 @@ from typing import AsyncIterator, Any, Dict, Iterable, Tuple
|
||||
from typing import List, Optional
|
||||
from unicodedata import normalize
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.chain import ChainBase
|
||||
from app.chain.media import MediaChain
|
||||
from app.runtime.config import global_vars
|
||||
|
||||
@@ -9,8 +9,7 @@ from pathlib import Path
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.runtime.settings import RuntimeSettingsCompat
|
||||
|
||||
settings = RuntimeSettingsCompat()
|
||||
|
||||
@@ -6,8 +6,8 @@ from typing import Optional, List, Dict, Any, Tuple, Union
|
||||
|
||||
import discord
|
||||
from discord import app_commands
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.runtime.settings import RuntimeSettingsCompat
|
||||
|
||||
settings = RuntimeSettingsCompat()
|
||||
|
||||
@@ -5,10 +5,10 @@ from typing import Any, Optional
|
||||
from typing import List
|
||||
from urllib.parse import quote, urlparse, parse_qs
|
||||
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from jinja2 import Template
|
||||
from pyquery import PyQuery
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.runtime.settings import RuntimeSettingsCompat
|
||||
|
||||
settings = RuntimeSettingsCompat()
|
||||
|
||||
@@ -104,6 +104,8 @@ Event Contract Registry 是 53 个事件的逐项机器清单。下表按相同
|
||||
的同步/异步调用合同。
|
||||
- 插件安装快照、取消补偿和临时约束文件清理统一复用 `runtime.execution.await_task_to_terminal`，连续取消
|
||||
不再由 Application 与 Adapter 各自维护近似循环；数据库 worker 的可中断队列等待仍保留独立职责。
|
||||
- canonical 宿主的同步函数异步桥接统一从 `runtime.execution.run_in_threadpool` 进入 AnyIO 线程池并传播
|
||||
context；FastAPI/Starlette 同名 helper 不再作为第二个导入入口，插件和精确兼容目录不受此门禁约束。
|
||||
|
||||
### Transfer pending / 文件整理
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
> 审计范围：宿主后端；排除 `app/plugins/**` 运行时插件副本
|
||||
> 规范优先级：`AGENTS.md` 与 `docs/rules/` 高于本文
|
||||
> 相关文档：`docs/architecture-overview.md`、`docs/refactor/backend-architecture-governance.md`、`docs/refactor/backend-module-refactor-compatibility.md`
|
||||
> 实施进度：阶段 0～6 的宿主架构能力已完成收口；API/Application 公共复杂度基线已清零，启动组合根的 SystemConfigOper 构造点已由 14 降至 1；API 进程内后台任务已完成首批统一登记，插件仓适配和 Outbox 外围扩展仍按风险切片推进。Model/Base 查询与写装饰器、legacy 隐式会话外壳均已清零，插件 SDK 也不再导出宿主 Model。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线；阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义；阶段 1b1 已收口整理 worker、pending 回放、失败通知、进程内 AI 重试、插件监控与事件投递的生命周期所有权；2026-08-24 的阶段 2 已将 212 个已观察宿主模块方法的 legacy aggregation 清零，并补齐可执行 fanout 与下载器文件 DTO 边界；阶段 3 已将消息交互和远程命令的订阅删除统一到 Application/UoW/outbox，宿主不再调用裸线程统计入口；阶段 4 已统一七种消息渠道的宿主回环与后台执行边界；阶段 5 已补齐事件窗口聚合任务的生命周期所有权；阶段 6 已统一插件文件操作的取消完成语义；阶段 7 已统一插件协程补偿的终态等待。
|
||||
> 实施进度：阶段 0～6 的宿主架构能力已完成收口；API/Application 公共复杂度基线已清零，启动组合根的 SystemConfigOper 构造点已由 14 降至 1；API 进程内后台任务已完成首批统一登记，插件仓适配和 Outbox 外围扩展仍按风险切片推进。Model/Base 查询与写装饰器、legacy 隐式会话外壳均已清零，插件 SDK 也不再导出宿主 Model。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线；阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义；阶段 1b1 已收口整理 worker、pending 回放、失败通知、进程内 AI 重试、插件监控与事件投递的生命周期所有权；2026-08-24 的阶段 2 已将 212 个已观察宿主模块方法的 legacy aggregation 清零，并补齐可执行 fanout 与下载器文件 DTO 边界；阶段 3 已将消息交互和远程命令的订阅删除统一到 Application/UoW/outbox，宿主不再调用裸线程统计入口；阶段 4 已统一七种消息渠道的宿主回环与后台执行边界；阶段 5 已补齐事件窗口聚合任务的生命周期所有权；阶段 6 已统一插件文件操作的取消完成语义；阶段 7 已统一插件协程补偿的终态等待；阶段 8 已统一宿主同步函数的异步线程池入口。
|
||||
|
||||
## 当前复核结论（2026-08-24）
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
### 长期整改阶段 0：治理门禁恢复（2026-08-23）
|
||||
|
||||
- 宿主依赖基线已审查 TaskRegistry、有界后台 owner 与插件变更准入接入后的语义差异：当前为 `806` 个模块、`6532` 条内部导入边，12 组重点禁止边继续全部为 `0`，唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
|
||||
- 宿主依赖基线已审查 TaskRegistry、有界后台 owner 与插件变更准入接入后的语义差异：当前为 `806` 个模块、`6544` 条内部导入边，12 组重点禁止边继续全部为 `0`，唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
|
||||
- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistry；normal/safe 组件数分别为 `23`/`11`，CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序，不再把 Python/平台模块数量当作硬合同。
|
||||
- 官方插件快照覆盖 `plugins.v3`、`plugins.v2` 以及 V3 实际会从 `package.json` 回退加载的 31 个默认实现；`app/plugins/**` 仍只是宿主运行副本，不进入扫描。
|
||||
- SDK 快照以各模块显式 `__all__` 为公开合同，能够记录赋值别名；`typing`、`__future__` 等实现期导入不再被误冻结，既有数据库备份门面已补精确导出清单。
|
||||
@@ -112,7 +112,7 @@
|
||||
|
||||
- 继续采用单进程控制面是正确选择，不建议现在拆成微服务；插件、调度器、工作流、事件和数据库共享进程内状态，拆分会放大部署、事务和兼容成本。
|
||||
- `foundation/domain/runtime/adapters/application/chain/api/startup` 的职责方向基本成立；宿主架构基线、复杂度 ratchet、异步阻塞 ratchet 当前均通过。
|
||||
- 依赖图当前为 `806` 个 Python 模块、`6532` 条内部导入边；唯一非平凡 SCC 位于隔离的 TMDB 第三方移植包内部，不应为了指标归零重写。
|
||||
- 依赖图当前为 `806` 个 Python 模块、`6544` 条内部导入边；唯一非平凡 SCC 位于隔离的 TMDB 第三方移植包内部，不应为了指标归零重写。
|
||||
- 当前主要风险已经从“目录和依赖失控”转移到运行时协议、后台副作用的可靠性和遗留兼容面。换言之，下一阶段重点应是**语义收口和可验证性**，而不是继续搬文件或机械拆大文件。
|
||||
|
||||
综合评价：架构方向可持续，生产可用性较高；可演进性仍处于中等水平。现阶段没有静态审计发现必须立即推倒重来的 P0 架构问题，但存在需要按 P1/P2 计划治理的真实债务。
|
||||
@@ -140,6 +140,8 @@
|
||||
文件 worker 到达终态后再传播，避免提前释放 mutation owner；旧模块内私有名称保留为 canonical 别名。
|
||||
插件安装快照、取消补偿与市场临时文件清理原有两套协程终态循环也已合并为
|
||||
`await_task_to_terminal`；数据库 worker 的 interruptible 等待属于队列 owner，未被机械合并。
|
||||
canonical 宿主原有 11 处 FastAPI `run_in_threadpool` 导入也已统一到 `runtime.execution`，AST 门禁
|
||||
防止再次出现框架直连；模块局部符号及调用参数不变，不改变插件运行和 monkeypatch 接缝。
|
||||
2. **Model/Base 的数据库装饰器和隐式会话 ABI 已全部清零。** 查询、写事务和 `legacy_*` 装饰器均为 `0`；所有 Model `db` 参数要求显式 Session，Base CRUD 仅在调用方事务内查询或 stage。可无会话构造的入口统一留在 Oper，经组合根事务执行器运行；插件 SDK 不再导出宿主 Model。后续重点转为减少 ORM 对象跨层流转，并保持 Model 隐式事务零回退。
|
||||
|
||||
Oper 内部的执行入口也已统一：最后一处 `AgentTaskOper` 直接 transaction runner 调用已迁入
|
||||
|
||||
+14
-2
@@ -13,8 +13,8 @@
|
||||
"runtime_to_db": [],
|
||||
"workflow_to_db": []
|
||||
},
|
||||
"edge_count": 6532,
|
||||
"edge_sha256": "3fdba2c6862f9ea28ab4f059c04dd6bb819b86fa9d9fb74d09a0cfcf69f6b711",
|
||||
"edge_count": 6544,
|
||||
"edge_sha256": "44812359c634235d327fdc499bc0ccf93e24923fba24bed4a29fc14adc26bf84",
|
||||
"edges": [
|
||||
"app -> app.runtime",
|
||||
"app -> app.runtime.compat",
|
||||
@@ -180,6 +180,7 @@
|
||||
"app.agent.callback -> app.agent.policy.sanitizer",
|
||||
"app.agent.callback -> app.chain",
|
||||
"app.agent.callback -> app.runtime",
|
||||
"app.agent.callback -> app.runtime.execution",
|
||||
"app.agent.callback -> app.runtime.log",
|
||||
"app.agent.callback -> app.schemas",
|
||||
"app.agent.callback -> app.schemas.message",
|
||||
@@ -363,6 +364,7 @@
|
||||
"app.agent.orchestrator -> app.foundation.identity",
|
||||
"app.agent.orchestrator -> app.runtime",
|
||||
"app.agent.orchestrator -> app.runtime.events",
|
||||
"app.agent.orchestrator -> app.runtime.execution",
|
||||
"app.agent.orchestrator -> app.runtime.log",
|
||||
"app.agent.orchestrator -> app.runtime.observability",
|
||||
"app.agent.orchestrator -> app.runtime.settings",
|
||||
@@ -1674,6 +1676,7 @@
|
||||
"app.api.endpoints.agent -> app.runtime",
|
||||
"app.api.endpoints.agent -> app.runtime.config",
|
||||
"app.api.endpoints.agent -> app.runtime.events",
|
||||
"app.api.endpoints.agent -> app.runtime.execution",
|
||||
"app.api.endpoints.agent -> app.runtime.localization",
|
||||
"app.api.endpoints.agent -> app.runtime.log",
|
||||
"app.api.endpoints.agent -> app.schemas",
|
||||
@@ -1760,6 +1763,8 @@
|
||||
"app.api.endpoints.dashboard -> app.chain",
|
||||
"app.api.endpoints.dashboard -> app.chain.dashboard",
|
||||
"app.api.endpoints.dashboard -> app.chain.storage",
|
||||
"app.api.endpoints.dashboard -> app.runtime",
|
||||
"app.api.endpoints.dashboard -> app.runtime.execution",
|
||||
"app.api.endpoints.dashboard -> app.schemas",
|
||||
"app.api.endpoints.dashboard -> app.schemas.dashboard",
|
||||
"app.api.endpoints.dashboard -> app.schemas.response",
|
||||
@@ -2095,6 +2100,7 @@
|
||||
"app.api.endpoints.plugin -> app.application.scheduling",
|
||||
"app.api.endpoints.plugin -> app.runtime",
|
||||
"app.api.endpoints.plugin -> app.runtime.cache",
|
||||
"app.api.endpoints.plugin -> app.runtime.execution",
|
||||
"app.api.endpoints.plugin -> app.runtime.extensions",
|
||||
"app.api.endpoints.plugin -> app.runtime.extensions.plugin",
|
||||
"app.api.endpoints.plugin -> app.runtime.extensions.plugin.contracts",
|
||||
@@ -2985,6 +2991,7 @@
|
||||
"app.chain._recognition -> app.runtime",
|
||||
"app.chain._recognition -> app.runtime.cache",
|
||||
"app.chain._recognition -> app.runtime.events",
|
||||
"app.chain._recognition -> app.runtime.execution",
|
||||
"app.chain._recognition -> app.runtime.log",
|
||||
"app.chain._recognition -> app.schemas",
|
||||
"app.chain._recognition -> app.schemas.media",
|
||||
@@ -3152,6 +3159,7 @@
|
||||
"app.chain.media -> app.runtime",
|
||||
"app.chain.media -> app.runtime.cache",
|
||||
"app.chain.media -> app.runtime.events",
|
||||
"app.chain.media -> app.runtime.execution",
|
||||
"app.chain.media -> app.runtime.log",
|
||||
"app.chain.media -> app.schemas",
|
||||
"app.chain.media -> app.schemas.event",
|
||||
@@ -3272,6 +3280,7 @@
|
||||
"app.chain.search -> app.runtime",
|
||||
"app.chain.search -> app.runtime.config",
|
||||
"app.chain.search -> app.runtime.events",
|
||||
"app.chain.search -> app.runtime.execution",
|
||||
"app.chain.search -> app.runtime.log",
|
||||
"app.chain.search -> app.runtime.progress",
|
||||
"app.chain.search -> app.schemas",
|
||||
@@ -3966,6 +3975,7 @@
|
||||
"app.modules.acoustid -> app.adapters.network.http",
|
||||
"app.modules.acoustid -> app.modules",
|
||||
"app.modules.acoustid -> app.runtime",
|
||||
"app.modules.acoustid -> app.runtime.execution",
|
||||
"app.modules.acoustid -> app.runtime.log",
|
||||
"app.modules.acoustid -> app.runtime.settings",
|
||||
"app.modules.acoustid -> app.schemas",
|
||||
@@ -4054,6 +4064,7 @@
|
||||
"app.modules.discord.discord -> app.foundation",
|
||||
"app.modules.discord.discord -> app.foundation.size",
|
||||
"app.modules.discord.discord -> app.runtime",
|
||||
"app.modules.discord.discord -> app.runtime.execution",
|
||||
"app.modules.discord.discord -> app.runtime.log",
|
||||
"app.modules.discord.discord -> app.runtime.settings",
|
||||
"app.modules.discord.discord -> app.schemas",
|
||||
@@ -4589,6 +4600,7 @@
|
||||
"app.modules.indexer.spider -> app.foundation.temporal",
|
||||
"app.modules.indexer.spider -> app.foundation.url",
|
||||
"app.modules.indexer.spider -> app.runtime",
|
||||
"app.modules.indexer.spider -> app.runtime.execution",
|
||||
"app.modules.indexer.spider -> app.runtime.log",
|
||||
"app.modules.indexer.spider -> app.runtime.settings",
|
||||
"app.modules.indexer.spider -> app.schemas",
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""运行时同步 worker 的取消与容量合同回归。"""
|
||||
|
||||
import asyncio
|
||||
import ast
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from anyio.to_thread import current_default_thread_limiter
|
||||
@@ -14,6 +16,31 @@ from app.runtime.execution import (
|
||||
)
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_host_uses_canonical_threadpool_boundary() -> None:
|
||||
"""canonical 宿主不得重新直连框架线程池 helper。"""
|
||||
violations: list[str] = []
|
||||
for path in sorted((PROJECT_ROOT / "app").rglob("*.py")):
|
||||
relative_path = path.relative_to(PROJECT_ROOT).as_posix()
|
||||
if relative_path.startswith(
|
||||
("app/plugins/", "app/runtime/compat/", "app/sdk/", "app/testing/")
|
||||
):
|
||||
continue
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.ImportFrom) or node.module not in {
|
||||
"fastapi.concurrency",
|
||||
"starlette.concurrency",
|
||||
}:
|
||||
continue
|
||||
if any(alias.name == "run_in_threadpool" for alias in node.names):
|
||||
violations.append(f"{relative_path}:{node.lineno}")
|
||||
|
||||
assert violations == []
|
||||
|
||||
|
||||
def test_plugin_file_adapters_share_runtime_completion_contract() -> None:
|
||||
"""市场与插件包适配器不得各自维护另一套线程取消实现。"""
|
||||
assert market_adapter._await_thread_operation is run_in_threadpool_to_completion
|
||||
|
||||
Reference in New Issue
Block a user