mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 07:27:15 +08:00
refactor: own shutdown lifecycle boundaries
This commit is contained in:
@@ -20,6 +20,9 @@ from app.runtime.log import logger
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
|
||||
def _get_skill_catalog() -> Any:
|
||||
"""按需返回 Agent 技能目录实现,供消息应用层消费端口。"""
|
||||
from app.agent.skills.registry import SkillHelper
|
||||
@@ -106,6 +109,7 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
self._manager: Any = None
|
||||
self._compat_injected = False
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
eventmanager.add_event_listener(
|
||||
EventType.ConfigChanged,
|
||||
@@ -117,6 +121,7 @@ class AgentInitializer:
|
||||
初始化AI智能体管理器
|
||||
"""
|
||||
try:
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
if agent_manager is not None:
|
||||
if not settings.AI_AGENT_ENABLE:
|
||||
@@ -142,7 +147,12 @@ class AgentInitializer:
|
||||
async def handle_config_changed(self, event: Event) -> None:
|
||||
"""仅在 manifest watch 命中时协调 service,关闭态保持 fail closed。"""
|
||||
changed_keys = _event_changed_keys(event)
|
||||
if not changed_keys or self._compat_injected or self._shutdown_complete:
|
||||
if (
|
||||
not changed_keys
|
||||
or self._compat_injected
|
||||
or self._shutdown_started
|
||||
or self._shutdown_complete
|
||||
):
|
||||
return
|
||||
try:
|
||||
self._manager = await reconcile_agent_service(
|
||||
@@ -156,24 +166,25 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
logger.debug(f"配置变更协调AI智能体失败: {error}")
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""清理 initializer 引用;显式注入对象同时在此关闭。"""
|
||||
async def cleanup(self) -> bool:
|
||||
"""清理 initializer 引用;未收敛的显式注入对象继续由本实例持有。"""
|
||||
try:
|
||||
manager = self._manager
|
||||
compat_injected = self._compat_injected
|
||||
if manager is None:
|
||||
return
|
||||
try:
|
||||
if compat_injected:
|
||||
await manager.close()
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
finally:
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
if compat_injected and await manager.close() is False:
|
||||
logger.error("AI智能体管理器仍有会话 owner 未收敛")
|
||||
return False
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"关闭AI智能体管理器时发生错误: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# 全局AI智能体初始化器实例
|
||||
@@ -204,23 +215,56 @@ async def init_agent() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def stop_agent():
|
||||
async def stop_agent() -> bool:
|
||||
"""
|
||||
停止AI智能体(异步版本,用于在应用关闭时调用)
|
||||
停止AI智能体,并在全部会话和工具资源释放后返回 True。
|
||||
"""
|
||||
converged = True
|
||||
close_blocking_executors = None
|
||||
agent_initializer._shutdown_started = True
|
||||
try:
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import (
|
||||
begin_blocking_executor_shutdown,
|
||||
close_blocking_executors as close_executors,
|
||||
)
|
||||
|
||||
# 必须在任何 manager await 之前封口,避免旧会话趁收尾窗口提交新同步调用。
|
||||
begin_blocking_executor_shutdown(cancel_futures=True)
|
||||
close_blocking_executors = close_executors
|
||||
except Exception as e:
|
||||
logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
try:
|
||||
if not agent_initializer._shutdown_complete:
|
||||
if agent_initializer._compat_injected:
|
||||
await agent_initializer.cleanup()
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
else:
|
||||
await begin_agent_shutdown()
|
||||
await agent_initializer.cleanup()
|
||||
agent_initializer._shutdown_complete = True
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import shutdown_blocking_executors
|
||||
|
||||
shutdown_blocking_executors(wait=False, cancel_futures=True)
|
||||
service_converged = await begin_agent_shutdown()
|
||||
if service_converged is not False:
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
converged = converged and service_converged is not False
|
||||
except Exception as e:
|
||||
logger.error(f"停止AI智能体时发生错误: {e}")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
converged = False
|
||||
|
||||
if close_blocking_executors is not None:
|
||||
try:
|
||||
blocking_converged = await close_blocking_executors(
|
||||
timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS,
|
||||
cancel_futures=True,
|
||||
)
|
||||
converged = converged and blocking_converged
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
if converged:
|
||||
try:
|
||||
await close_materialized_terminal_sessions()
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体终端会话时发生错误: {e}")
|
||||
converged = False
|
||||
agent_initializer._shutdown_complete = converged
|
||||
return converged
|
||||
|
||||
@@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.runtime.state import SystemHelper
|
||||
from app.runtime.log import logger, LoggerManager
|
||||
from app.startup.command_initializer import init_command, stop_command, restart_command
|
||||
from app.startup.agent_initializer import stop_agent
|
||||
from app.startup.domain_initializer import configure_domain_dependencies
|
||||
from app.startup.modules_initializer import init_modules, stop_modules
|
||||
from app.startup.modules_initializer import (
|
||||
drain_events,
|
||||
init_modules,
|
||||
settle_events,
|
||||
stop_modules,
|
||||
)
|
||||
from app.startup.monitor_initializer import stop_monitor, init_monitor
|
||||
from app.startup.plugins_initializer import (
|
||||
configure_plugin_services,
|
||||
execute_task,
|
||||
finalize_plugins,
|
||||
init_plugins,
|
||||
stop_plugins,
|
||||
quiesce_plugin_services,
|
||||
quiesce_plugins,
|
||||
stop_plugin_monitor,
|
||||
sync_plugins,
|
||||
)
|
||||
from app.startup.routers_initializer import init_routers
|
||||
@@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import (
|
||||
init_plugin_scheduler,
|
||||
)
|
||||
from app.db.engine import check_connection_budget, get_engine, get_global_async_engine
|
||||
from app.startup.transfer_initializer import replay_pending_transfers
|
||||
from app.startup.transfer_initializer import (
|
||||
replay_pending_transfers,
|
||||
stop_transfer_runtime,
|
||||
)
|
||||
from app.startup.workflow_initializer import init_workflow, stop_workflow
|
||||
from app.startup.lifecycle.components import (
|
||||
LifecycleComponent,
|
||||
LifecycleFailurePolicy,
|
||||
LifecycleMode,
|
||||
lifecycle_manifest,
|
||||
)
|
||||
@@ -101,8 +114,8 @@ async def run_shutdown_step(
|
||||
name: str,
|
||||
callback: Callable[[], object],
|
||||
timeout_seconds: float | None = None,
|
||||
) -> None:
|
||||
"""在有限预算内执行关闭阶段,并保留未收敛任务的资源所有权。"""
|
||||
) -> bool:
|
||||
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛。"""
|
||||
try:
|
||||
result = callback()
|
||||
if inspect.isawaitable(result):
|
||||
@@ -120,16 +133,22 @@ async def run_shutdown_step(
|
||||
task.add_done_callback(_consume_shutdown_result)
|
||||
if timeout_seconds:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(task), timeout=timeout_seconds
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
|
||||
task.cancel()
|
||||
return False
|
||||
else:
|
||||
await task
|
||||
result = await task
|
||||
if result is False:
|
||||
logger.error("关闭%s未收敛,资源所有权保持不变", name)
|
||||
return False
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return False
|
||||
|
||||
|
||||
async def run_startup_step(
|
||||
@@ -152,6 +171,66 @@ async def run_startup_step(
|
||||
logger.info("启动%s完成,耗时=%.2fms", name, elapsed_ms)
|
||||
|
||||
|
||||
async def stop_lifecycle_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
) -> bool:
|
||||
"""按声明顺序关闭组件,并在关键 owner 未收敛时停止释放依赖。"""
|
||||
all_converged = True
|
||||
for component in sorted(
|
||||
(item for item in components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
completed = await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
if completed:
|
||||
continue
|
||||
all_converged = False
|
||||
if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST:
|
||||
logger.error(
|
||||
"关闭%s未收敛,停止释放其后续依赖",
|
||||
component.name,
|
||||
)
|
||||
break
|
||||
return all_converged
|
||||
|
||||
|
||||
def select_startup_cleanup_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
*,
|
||||
started_names: set[str],
|
||||
active_component: LifecycleComponent | None,
|
||||
) -> tuple[LifecycleComponent, ...]:
|
||||
"""选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。"""
|
||||
cleanup_names = set(started_names)
|
||||
if active_component is not None:
|
||||
cleanup_names.add(active_component.name)
|
||||
|
||||
# stop-only owner 没有启动回调,按已激活依赖递归纳入,避免清理时反向
|
||||
# 实例化尚未触达的插件、模块或外部资源。
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for component in components:
|
||||
if (
|
||||
component.stop is None
|
||||
or component.start is not None
|
||||
or component.name in cleanup_names
|
||||
or not set(component.dependencies).issubset(cleanup_names)
|
||||
):
|
||||
continue
|
||||
cleanup_names.add(component.name)
|
||||
changed = True
|
||||
|
||||
return tuple(
|
||||
component
|
||||
for component in components
|
||||
if component.stop is not None and component.name in cleanup_names
|
||||
)
|
||||
|
||||
|
||||
async def initialize_modules_component(app: FastAPI) -> None:
|
||||
"""启动模块并把其类型化运行时发布到当前 FastAPI AppState。"""
|
||||
try:
|
||||
@@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None:
|
||||
configure_task_registry(task_registry)
|
||||
|
||||
|
||||
async def stop_task_registry(app: FastAPI) -> None:
|
||||
"""停止接收新后台任务,并取消、等待当前 lifespan 的存量任务。"""
|
||||
async def stop_task_registry(app: FastAPI) -> bool:
|
||||
"""停止接收新后台任务,并把已封口登记器保留到下一次显式启动。"""
|
||||
task_registry = getattr(app.state, "task_registry", None)
|
||||
try:
|
||||
if isinstance(task_registry, TaskRegistry):
|
||||
await task_registry.shutdown(timeout_seconds=30.0)
|
||||
finally:
|
||||
if not isinstance(task_registry, TaskRegistry):
|
||||
configure_task_registry(None)
|
||||
app.state.task_registry = None
|
||||
return True
|
||||
# 即使 owner 已收敛,也不能在后续插件/模块 stop hook 仍会运行时退回永久
|
||||
# accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。
|
||||
return await task_registry.shutdown(timeout_seconds=30.0)
|
||||
|
||||
|
||||
def prepare_plugin_restore() -> None:
|
||||
@@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=5,
|
||||
start_timeout_seconds=30,
|
||||
stop_timeout_seconds=60,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="数据库准备",
|
||||
@@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
dependencies=("插件备份恢复",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=init_plugins,
|
||||
stop=stop_plugins,
|
||||
stop=finalize_plugins,
|
||||
start_order=90,
|
||||
stop_order=60,
|
||||
start_timeout_seconds=300,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件变更监控",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=stop_plugin_monitor,
|
||||
stop_order=8,
|
||||
stop_timeout_seconds=10,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="定时器",
|
||||
@@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=50,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="监控器",
|
||||
@@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=40,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="整理后台服务",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_transfer_runtime,
|
||||
stop_order=52,
|
||||
stop_timeout_seconds=45,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="AI智能体会话",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_agent,
|
||||
stop_order=51,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件事件入口",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugins,
|
||||
stop_order=53,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件尾任务结算",
|
||||
dependencies=("模块服务", "插件"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=settle_events,
|
||||
stop_order=54,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件后台服务",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugin_services,
|
||||
stop_order=55,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件投递屏障",
|
||||
dependencies=("模块服务", "整理后台服务"),
|
||||
stop=drain_events,
|
||||
stop_order=58,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="待处理整理回放",
|
||||
dependencies=("监控器",),
|
||||
dependencies=("监控器", "整理后台服务"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=replay_pending_transfers,
|
||||
start_order=120,
|
||||
@@ -369,6 +513,9 @@ async def lifespan(app: FastAPI):
|
||||
health = get_application_health(app)
|
||||
health.begin_startup()
|
||||
main_loop = asyncio.get_running_loop()
|
||||
enabled_components: tuple[LifecycleComponent, ...] = ()
|
||||
started_component_names: set[str] = set()
|
||||
active_start_component: LifecycleComponent | None = None
|
||||
try:
|
||||
validate_process_topology(
|
||||
workers=settings.API_WORKERS,
|
||||
@@ -390,11 +537,14 @@ async def lifespan(app: FastAPI):
|
||||
(item for item in enabled_components if item.start is not None),
|
||||
key=lambda item: item.start_order or 0,
|
||||
):
|
||||
active_start_component = component
|
||||
await run_startup_step(
|
||||
component.name,
|
||||
component.start,
|
||||
component.start_timeout_seconds,
|
||||
)
|
||||
started_component_names.add(component.name)
|
||||
active_start_component = None
|
||||
if settings.MOVIEPILOT_SAFE_MODE:
|
||||
print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.")
|
||||
# 插件同步到本地
|
||||
@@ -407,10 +557,15 @@ async def lifespan(app: FastAPI):
|
||||
except BaseException:
|
||||
# Uvicorn 在 lifespan 抛错时不会开始接流量;状态仍需供嵌入式入口和测试诊断。
|
||||
health.mark_failed()
|
||||
cleanup_components = select_startup_cleanup_components(
|
||||
enabled_components,
|
||||
started_names=started_component_names,
|
||||
active_component=active_start_component,
|
||||
)
|
||||
try:
|
||||
await stop_task_registry(app)
|
||||
await stop_lifecycle_components(cleanup_components)
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"启动失败后的后台任务清理失败:{cleanup_error}")
|
||||
logger.error(f"启动失败后的生命周期清理失败:{cleanup_error}")
|
||||
finally:
|
||||
global_vars.clear_loop(main_loop)
|
||||
raise
|
||||
@@ -421,21 +576,10 @@ async def lifespan(app: FastAPI):
|
||||
health.mark_stopping()
|
||||
print("Shutting down...")
|
||||
global_vars.stop_system()
|
||||
# 插件恢复会在线程池中修改源码与依赖,必须完成后再进入资源关闭阶段。
|
||||
try:
|
||||
await sync_plugins_task
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
try:
|
||||
for component in sorted(
|
||||
(item for item in enabled_components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
# 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner
|
||||
# 在统一预算内取消/等待,不能在屏障之前无界 await 绕过停机预算。
|
||||
await stop_lifecycle_components(enabled_components)
|
||||
finally:
|
||||
try:
|
||||
# 日志最后关闭,确保其他组件的收尾信息已写入文件
|
||||
|
||||
@@ -129,7 +129,7 @@ from app.command import CommandChain
|
||||
from app.schemas.message import Message
|
||||
from app.schemas.message import MessageType
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
from app.startup.agent_initializer import init_agent, stop_agent
|
||||
from app.startup.agent_initializer import init_agent
|
||||
from app.startup.database import build_database_governance
|
||||
from app.startup.managed_resources_initializer import (
|
||||
init_managed_resources,
|
||||
@@ -560,6 +560,22 @@ def close_browser_sessions() -> None:
|
||||
BrowserSessionHelper.close_all_sessions()
|
||||
|
||||
|
||||
async def drain_events() -> bool:
|
||||
"""在插件卸载前等待已接收事件及其同步、异步处理器完成。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=True)
|
||||
|
||||
|
||||
async def settle_events() -> bool:
|
||||
"""在插件 handler 停用后结算在途事件,但保留停机 hook 的尾事件入口。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=False)
|
||||
|
||||
|
||||
async def stop_modules():
|
||||
"""
|
||||
服务关闭
|
||||
@@ -578,7 +594,6 @@ async def stop_modules():
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return True
|
||||
|
||||
await run_step("AI智能体", stop_agent)
|
||||
await run_step("模块", lambda: ModuleManager().shutdown())
|
||||
await run_step("事件消费", lambda: EventManager().stop_async())
|
||||
await run_step("浏览器会话", close_browser_sessions)
|
||||
|
||||
@@ -1,15 +1,26 @@
|
||||
from app.monitor import Monitor
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
|
||||
|
||||
def init_monitor():
|
||||
"""
|
||||
初始化监控器
|
||||
"""
|
||||
Monitor()
|
||||
def init_monitor() -> None:
|
||||
"""初始化监控器;复用单例时必须显式开启新的应用 lifespan。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
Monitor()
|
||||
return
|
||||
if not monitor.lifecycle_closed:
|
||||
return
|
||||
if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("旧目录监控 owner 未收敛,无法开启新生命周期")
|
||||
if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("目录监控初始化失败")
|
||||
|
||||
|
||||
def stop_monitor():
|
||||
"""
|
||||
停止监控器
|
||||
"""
|
||||
Monitor().stop()
|
||||
async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool:
|
||||
"""在线程池里永久关闭监控器,取消后仍等待同步 owner 收敛到终态。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
return True
|
||||
return bool(
|
||||
await run_in_threadpool_to_completion(monitor.close, timeout=timeout)
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import (
|
||||
configure_plugin_resource_import_preparer,
|
||||
configure_site_auth_level_provider,
|
||||
)
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult
|
||||
from app.application.plugin.catalog import PluginCatalogService
|
||||
from app.application.plugin.data import DeletePluginDataCommand
|
||||
@@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork
|
||||
from app.runtime.log import logger
|
||||
from app.foundation.version import compare_version
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
@@ -142,56 +144,64 @@ async def sync_plugins() -> bool:
|
||||
"""
|
||||
plugin_manager = None
|
||||
try:
|
||||
configure_plugin_services()
|
||||
loop = global_vars.loop
|
||||
plugin_manager = PluginManager()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
with plugin_manager.mutation("启动后同步插件"):
|
||||
configure_plugin_services()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
return await _sync_plugins_admitted(plugin_manager, loop)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"插件初始化过程中出现异常: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool:
|
||||
"""在一个 admission lease 内完成包、依赖、实例和动态路由同步。"""
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _activate_ready_plugins(
|
||||
plugin_manager: PluginManager,
|
||||
ready_ids: tuple[str, ...],
|
||||
@@ -217,12 +227,39 @@ def _activate_ready_plugins(
|
||||
return changed_ids
|
||||
|
||||
|
||||
async def quiesce_plugins(timeout: float = 240.0) -> bool:
|
||||
"""封口插件变更并停用 handler,保留超时 Future 的运行所有权。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugins(timeout=timeout)
|
||||
|
||||
|
||||
async def quiesce_plugin_services(timeout: float = 240.0) -> bool:
|
||||
"""在事件结算后有界执行旧插件 close、stop_service hook。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugin_services(timeout=timeout)
|
||||
|
||||
|
||||
def finalize_plugins() -> bool:
|
||||
"""在事件屏障封口后卸载已停用 handler 的插件实例。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return bool(plugin_manager.finalize_plugins())
|
||||
|
||||
|
||||
async def execute_task(loop, task_func, task_name):
|
||||
"""
|
||||
执行后台任务
|
||||
执行后台任务;取消调用方时仍持有同步线程直到真实完成。
|
||||
"""
|
||||
try:
|
||||
result = await loop.run_in_executor(None, task_func)
|
||||
# loop 参数属于既有调用 ABI;同步执行改由 completion-aware 适配器持有,
|
||||
# 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。
|
||||
del loop
|
||||
result = await run_in_threadpool_to_completion(task_func)
|
||||
if isinstance(result, PluginDependencyInstallResult):
|
||||
processed_count = len(result.missing)
|
||||
elif isinstance(result, list):
|
||||
@@ -245,13 +282,15 @@ def init_plugins():
|
||||
"""
|
||||
configure_plugin_services()
|
||||
plugin_manager = PluginManager()
|
||||
if not plugin_manager.reopen_plugins():
|
||||
raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛")
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
for plugin_id in classification.ready:
|
||||
plugin_manager.start(plugin_id)
|
||||
register_plugin_api()
|
||||
plugin_manager.start_monitor()
|
||||
plugin_manager.start_monitor(reopen=True)
|
||||
logger.info(
|
||||
"插件启动分类:立即加载=%s,等待依赖=%s,等待源码=%s",
|
||||
len(classification.ready),
|
||||
@@ -260,15 +299,30 @@ def init_plugins():
|
||||
)
|
||||
|
||||
|
||||
def stop_plugins():
|
||||
"""
|
||||
停止插件
|
||||
"""
|
||||
def stop_plugin_monitor(timeout: float = 5.0) -> bool:
|
||||
"""封口已创建管理器的文件监控线程,并返回是否完成收口。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
try:
|
||||
plugin_manager = PluginManager()
|
||||
return bool(plugin_manager.close_monitor(timeout=timeout))
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件文件监控时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def stop_plugins() -> bool:
|
||||
"""停止已创建的插件监控和运行实例,不在停机阶段反向物化管理器。"""
|
||||
try:
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
monitor_stopped = True
|
||||
try:
|
||||
plugin_manager.stop_monitor()
|
||||
monitor_stopped = plugin_manager.stop_monitor()
|
||||
finally:
|
||||
plugin_manager.stop()
|
||||
return bool(monitor_stopped)
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
@@ -11,3 +11,15 @@ def replay_pending_transfers():
|
||||
回放本身在后台线程执行,不阻塞启动流程。
|
||||
"""
|
||||
TransferChain().replay_pending()
|
||||
|
||||
|
||||
async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool:
|
||||
"""关闭已存在的整理后台 owner,且不在关停阶段创建新的整理链实例。
|
||||
|
||||
:param timeout_seconds: worker 与 pending 回放共享的最大等待秒数
|
||||
:return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True
|
||||
"""
|
||||
transfer_chain = TransferChain.get_existing_instance()
|
||||
if transfer_chain is None:
|
||||
return True
|
||||
return await transfer_chain.close(timeout_seconds=timeout_seconds)
|
||||
|
||||
Reference in New Issue
Block a user