mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 15:38:19 +08:00
refactor: own shutdown lifecycle boundaries
This commit is contained in:
@@ -155,7 +155,11 @@ class AgentServiceAdapter:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 必须返回 awaitable"
|
||||
)
|
||||
await result
|
||||
converged = await result
|
||||
if converged is False:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def cleanup(
|
||||
|
||||
@@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
|
||||
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
|
||||
self._accepting_tasks = True
|
||||
self._close_cancel_requested: set[asyncio.Task] = set()
|
||||
self.tools = [
|
||||
StructuredTool.from_function(
|
||||
coroutine=self._control_task,
|
||||
@@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
|
||||
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
|
||||
"""记录任务完成时间并取出异常避免未读取告警。"""
|
||||
cancel_requested = getattr(self, "_close_cancel_requested", None)
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.discard(task)
|
||||
record = self._tasks.get(task_id)
|
||||
if record:
|
||||
record.finished_at = datetime.now()
|
||||
@@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
@staticmethod
|
||||
async def _cancel_records(
|
||||
records: list[_SubAgentRuntimeTask],
|
||||
*,
|
||||
cancel_requested: Optional[set[asyncio.Task]] = None,
|
||||
) -> list[_SubAgentRuntimeTask]:
|
||||
"""取消一组任务,并返回等待上限内仍未收敛的记录。"""
|
||||
cancellable_tasks = [
|
||||
@@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
if cancellable_tasks:
|
||||
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
|
||||
for task in cancellable_tasks:
|
||||
if cancel_requested is not None and task in cancel_requested:
|
||||
continue
|
||||
task.cancel()
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.add(task)
|
||||
if not cancellable_tasks:
|
||||
return []
|
||||
|
||||
@@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
|
||||
return [record for record in records if record.task in pending]
|
||||
|
||||
async def close(self) -> None:
|
||||
"""取消脱离当前 Agent 回合的子代理任务。"""
|
||||
def seal(self) -> None:
|
||||
"""封住新的 detached 子代理提交,既有任务继续由记录表持有。"""
|
||||
self._accepting_tasks = False
|
||||
|
||||
async def close(self) -> bool:
|
||||
"""有限等待 detached 子代理;超时保留记录并返回 False。"""
|
||||
self.seal()
|
||||
if not hasattr(self, "_close_cancel_requested"):
|
||||
self._close_cancel_requested = set()
|
||||
unfinished_records = [
|
||||
record for record in self._tasks.values() if not record.task.done()
|
||||
]
|
||||
@@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(
|
||||
f"关闭子代理任务控制器,取消未完成任务: tasks={len(unfinished_records)}"
|
||||
)
|
||||
await self._cancel_records(unfinished_records)
|
||||
pending_records = await self._cancel_records(
|
||||
unfinished_records,
|
||||
cancel_requested=self._close_cancel_requested,
|
||||
)
|
||||
if pending_records:
|
||||
self._tasks = {
|
||||
record.task_id: record
|
||||
for record in pending_records
|
||||
}
|
||||
return False
|
||||
self._tasks.clear()
|
||||
self._close_cancel_requested.clear()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_description(
|
||||
@@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
|
||||
timeout = normalized_timeout_ms / 1000
|
||||
for step_index, spec in enumerate(specs, start=1):
|
||||
if not self._accepting_tasks:
|
||||
return records, "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
record = self._create_pipeline_record(spec)
|
||||
records.append(record)
|
||||
pipeline_description = self._pipeline_description(
|
||||
@@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
"""管理异步子代理任务。"""
|
||||
logger.info(f"收到子代理管控操作: action={action}")
|
||||
if action in {"start", "run", "pipeline"}:
|
||||
if not self._accepting_tasks:
|
||||
error = "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
return self._json_response({"success": False, "error": error})
|
||||
specs, error = self._normalize_specs(
|
||||
description=description,
|
||||
subagent_type=subagent_type,
|
||||
|
||||
+134
-33
@@ -436,6 +436,7 @@ class MoviePilotAgent:
|
||||
self._agent_started_at: Optional[datetime] = None
|
||||
self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None
|
||||
self._subagent_middlewares: tuple[Any, ...] = ()
|
||||
self._shutdown_started = False
|
||||
self._last_agent_cache_hit = False
|
||||
|
||||
# 流式token管理
|
||||
@@ -1751,10 +1752,25 @@ class MoviePilotAgent:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
def _seal_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> None:
|
||||
"""释放不再由 Agent 图持有的子代理控制器。"""
|
||||
"""同步封住子代理控制器的新任务入口,不等待既有任务退出。"""
|
||||
for middleware in middlewares:
|
||||
seal = getattr(middleware, "seal", None)
|
||||
if not callable(seal):
|
||||
continue
|
||||
try:
|
||||
seal()
|
||||
except Exception as error:
|
||||
logger.debug(f"封住子代理中间件失败: {error}")
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""关闭子代理控制器,并返回仍持有未收敛任务的实例。"""
|
||||
pending_middlewares = []
|
||||
for middleware in middlewares:
|
||||
close = getattr(middleware, "close", None)
|
||||
if not callable(close):
|
||||
@@ -1762,9 +1778,30 @@ class MoviePilotAgent:
|
||||
try:
|
||||
result = close()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
result = await result
|
||||
if result is False:
|
||||
pending_middlewares.append(middleware)
|
||||
except Exception as error:
|
||||
logger.debug(f"关闭子代理中间件失败: {error}")
|
||||
pending_middlewares.append(middleware)
|
||||
return tuple(pending_middlewares)
|
||||
|
||||
@staticmethod
|
||||
def _merge_subagent_middleware_owners(
|
||||
*groups: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""按对象身份合并当前图与延迟收敛控制器的 owner 集合。"""
|
||||
merged = []
|
||||
for group in groups:
|
||||
for middleware in group:
|
||||
if not any(middleware is existing for existing in merged):
|
||||
merged.append(middleware)
|
||||
return tuple(merged)
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""在任何异步等待前封住当前 Agent 的 detached 子代理提交。"""
|
||||
self._shutdown_started = True
|
||||
self._seal_subagent_middleware_instances(self._subagent_middlewares)
|
||||
|
||||
async def _cache_agent(
|
||||
self,
|
||||
@@ -1786,7 +1823,11 @@ class MoviePilotAgent:
|
||||
for replacement in subagent_middlewares
|
||||
)
|
||||
)
|
||||
await self._close_subagent_middleware_instances(previous_middlewares)
|
||||
if self._shutdown_started:
|
||||
self._seal_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
previous_middlewares
|
||||
)
|
||||
self._compiled_agent_bundle = _CompiledAgentBundle(
|
||||
signature=signature,
|
||||
agent=agent,
|
||||
@@ -1798,15 +1839,21 @@ class MoviePilotAgent:
|
||||
mcp_config_signature=mcp_config_signature,
|
||||
catalog_checked_at=datetime.now(),
|
||||
)
|
||||
self._subagent_middlewares = subagent_middlewares
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
pending_middlewares,
|
||||
subagent_middlewares,
|
||||
)
|
||||
return agent
|
||||
|
||||
async def _invalidate_cached_agent(self) -> None:
|
||||
"""使当前图失效,并释放只属于该图的子代理控制器。"""
|
||||
async def _invalidate_cached_agent(self) -> bool:
|
||||
"""使当前图失效,未收敛的子代理控制器继续由 Agent 持有。"""
|
||||
subagent_middlewares = self._subagent_middlewares
|
||||
self._subagent_middlewares = ()
|
||||
self._compiled_agent_bundle = None
|
||||
await self._close_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = pending_middlewares
|
||||
return not pending_middlewares
|
||||
|
||||
@staticmethod
|
||||
def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]:
|
||||
@@ -2003,9 +2050,13 @@ class MoviePilotAgent:
|
||||
if cached_agent:
|
||||
# 签名相同表示已编译图中的精确工具实例仍有效;新建快照仅用于复核。
|
||||
cached_bundle.catalog_checked_at = datetime.now()
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
temporary_subagent_middlewares = ()
|
||||
logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}")
|
||||
return cached_agent
|
||||
@@ -2123,14 +2174,22 @@ class MoviePilotAgent:
|
||||
temporary_subagent_middlewares = ()
|
||||
return cached_agent
|
||||
except asyncio.CancelledError:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
logger.error(f"创建 Agent 失败: {e}")
|
||||
raise
|
||||
|
||||
@@ -2539,14 +2598,20 @@ class MoviePilotAgent:
|
||||
)
|
||||
)
|
||||
|
||||
async def cleanup(self):
|
||||
async def cleanup(self) -> bool:
|
||||
"""
|
||||
清理智能体资源
|
||||
清理智能体资源;detached 子代理未收敛时保留 owner 并返回 False。
|
||||
"""
|
||||
await self._invalidate_cached_agent()
|
||||
self.begin_shutdown()
|
||||
if not await self._invalidate_cached_agent():
|
||||
logger.error(
|
||||
f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}"
|
||||
)
|
||||
return False
|
||||
self._pending_secret_confirmation = None
|
||||
self.protected_output_callback = None
|
||||
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2626,6 +2691,7 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {}
|
||||
self._session_cancel_requested: set[str] = set()
|
||||
self._close_finalizer_task: Optional[asyncio.Task] = None
|
||||
self._closed = False
|
||||
self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT
|
||||
# 接收门禁与队列写入共用一把锁,确保关闭开始后不会再创建 worker。
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
@@ -2693,22 +2759,33 @@ class AgentManager:
|
||||
async with self._lifecycle_lock:
|
||||
if self._accepting_tasks:
|
||||
return
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭")
|
||||
memory_manager.initialize()
|
||||
if not self._idle_cleanup_task or self._idle_cleanup_task.done():
|
||||
self._idle_cleanup_task = asyncio.create_task(
|
||||
self._cleanup_idle_sessions()
|
||||
)
|
||||
self._accepting_tasks = True
|
||||
self._closed = False
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> bool:
|
||||
"""
|
||||
关闭管理器
|
||||
关闭管理器,并诚实返回全部会话 owner 是否已经收敛。
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self._closed:
|
||||
return True
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
return
|
||||
return False
|
||||
# 门禁必须先关闭;锁内完成清理可阻止等待中的请求在收口期间重新入队。
|
||||
self._accepting_tasks = False
|
||||
# 子代理提交门禁同样必须在第一次 await 前关闭,否则 detached task
|
||||
# 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。
|
||||
for agent in self.active_agents.values():
|
||||
begin_shutdown = getattr(agent, "begin_shutdown", None)
|
||||
if callable(begin_shutdown):
|
||||
begin_shutdown()
|
||||
if self._idle_cleanup_task:
|
||||
self._idle_cleanup_task.cancel()
|
||||
try:
|
||||
@@ -2751,8 +2828,8 @@ class AgentManager:
|
||||
self._session_workers.pop(session_id, None)
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if session_id not in timed_out_session_ids:
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
logger.error(
|
||||
"AgentManager 关闭时仍有 worker 未收敛,"
|
||||
f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束"
|
||||
@@ -2760,13 +2837,21 @@ class AgentManager:
|
||||
self._close_finalizer_task = asyncio.create_task(
|
||||
self._finish_deferred_close(timed_out_workers)
|
||||
)
|
||||
return
|
||||
return False
|
||||
|
||||
self._session_workers.clear()
|
||||
for agent in list(self.active_agents.values()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.clear()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 仍有 %d 个 detached 子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return False
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
return True
|
||||
|
||||
def _record_session_activity(self, session_id: str, user_id: str) -> None:
|
||||
"""
|
||||
@@ -3072,7 +3157,10 @@ class AgentManager:
|
||||
and isinstance(task.agent_factory, type)
|
||||
and not isinstance(existing_agent, task.agent_factory)
|
||||
):
|
||||
await existing_agent.cleanup()
|
||||
if await existing_agent.cleanup() is False:
|
||||
raise AgentManagerUnavailableError(
|
||||
f"Agent 会话 {session_id} 仍有子代理任务在停止"
|
||||
)
|
||||
self.active_agents.pop(session_id, None)
|
||||
|
||||
if session_id not in self.active_agents:
|
||||
@@ -3249,7 +3337,11 @@ class AgentManager:
|
||||
# 清理agent
|
||||
if session_id in self.active_agents:
|
||||
agent = self.active_agents[session_id]
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 仍有子代理 owner 未收敛,保留会话与记忆"
|
||||
)
|
||||
return
|
||||
del self.active_agents[session_id]
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
@@ -3290,9 +3382,14 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks.pop(session_id, None)
|
||||
self._session_queue_rejections.pop(session_id, None)
|
||||
self._session_last_queue_wait_ms.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
agent = self.active_agents.get(session_id)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛"
|
||||
)
|
||||
return
|
||||
self.active_agents.pop(session_id, None)
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
|
||||
@@ -3310,15 +3407,19 @@ class AgentManager:
|
||||
for session_id, worker in workers:
|
||||
if self._session_workers.get(session_id) is worker:
|
||||
self._session_workers.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
self._session_shutdown_pending.clear()
|
||||
self._session_cancel_requested.clear()
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
finally:
|
||||
self._close_finalizer_task = None
|
||||
|
||||
|
||||
@@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import (
|
||||
build_agent_capability_registry,
|
||||
should_run_agent_service,
|
||||
)
|
||||
from app.runtime.capabilities.model import CapabilityMaterializationState
|
||||
from app.runtime.capabilities.model import (
|
||||
CapabilityLifecycleState,
|
||||
CapabilityMaterializationState,
|
||||
)
|
||||
from app.runtime.capabilities.runtime import CapabilityRuntime
|
||||
|
||||
|
||||
@@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None:
|
||||
await close()
|
||||
|
||||
|
||||
async def begin_agent_shutdown() -> None:
|
||||
"""不可逆关闭首用闸门,并等待全部同步及异步能力释放。"""
|
||||
try:
|
||||
await _ensure_runtime().shutdown_async(reason="application_shutdown")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
async def begin_agent_shutdown() -> bool:
|
||||
"""不可逆关闭首用闸门,并返回 Agent service 是否真实收敛。"""
|
||||
runtime = _ensure_runtime()
|
||||
await runtime.shutdown_async(reason="application_shutdown")
|
||||
return (
|
||||
runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle
|
||||
is CapabilityLifecycleState.STOPPED
|
||||
)
|
||||
|
||||
+117
-26
@@ -3,7 +3,7 @@ import inspect
|
||||
import json
|
||||
import threading
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
|
||||
from contextvars import Context, copy_context
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
@@ -168,33 +168,130 @@ _blocking_semaphores = {
|
||||
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
|
||||
}
|
||||
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.Lock()
|
||||
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
|
||||
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.RLock()
|
||||
_blocking_executor_accepting = True
|
||||
|
||||
|
||||
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
|
||||
"""按桶懒加载线程池,避免在导入阶段创建过多 worker。"""
|
||||
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
|
||||
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executor = _blocking_futures.pop(future, None)
|
||||
if executor is None or executor not in _blocking_retiring_executors:
|
||||
return
|
||||
if executor not in _blocking_futures.values():
|
||||
_blocking_retiring_executors.discard(executor)
|
||||
|
||||
|
||||
def _submit_blocking_call(
|
||||
bucket: str,
|
||||
bound_call: Callable[[], Any],
|
||||
) -> ConcurrentFuture[Any]:
|
||||
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
|
||||
context = copy_context()
|
||||
with _blocking_executor_lock:
|
||||
if not _blocking_executor_accepting:
|
||||
raise RuntimeError("Agent 工具阻塞执行器正在关闭,不能再提交新任务")
|
||||
executor = _blocking_executors.get(bucket)
|
||||
if executor:
|
||||
return executor
|
||||
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
return executor
|
||||
if executor is None:
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(executor.submit, context.run, bound_call)
|
||||
_blocking_futures[future] = executor
|
||||
future.add_done_callback(_discard_blocking_future)
|
||||
return future
|
||||
|
||||
|
||||
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
|
||||
"""关闭 Agent 工具阻塞线程池,释放长期运行进程或测试环境中的 worker。"""
|
||||
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
|
||||
"""撤销活动 executor 的提交资格,并保留其运行 Future 对应的 owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executors = list(_blocking_executors.values())
|
||||
executors = tuple(_blocking_executors.values())
|
||||
_blocking_executors.clear()
|
||||
|
||||
_blocking_retiring_executors.update(executors)
|
||||
for executor in executors:
|
||||
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
executor.shutdown(wait=False, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
owned_executors = set(_blocking_futures.values())
|
||||
_blocking_retiring_executors.intersection_update(owned_executors)
|
||||
return executors
|
||||
|
||||
|
||||
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
|
||||
"""原子封住新阻塞工具提交,并请求取消尚未开始的同步调用。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
_blocking_executor_accepting = False
|
||||
_retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
|
||||
|
||||
def reopen_blocking_executors() -> bool:
|
||||
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
if _blocking_futures or _blocking_retiring_executors:
|
||||
return False
|
||||
_blocking_executor_accepting = True
|
||||
return True
|
||||
|
||||
|
||||
async def close_blocking_executors(
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
cancel_futures: bool = True,
|
||||
) -> bool:
|
||||
"""有限等待全部阻塞工具 Future,超时保留 Future 与 executor owner。"""
|
||||
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
futures = tuple(_blocking_futures)
|
||||
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
|
||||
if wrapped_futures:
|
||||
done, pending = await asyncio.wait(
|
||||
wrapped_futures,
|
||||
timeout=max(0.0, timeout_seconds),
|
||||
)
|
||||
if done:
|
||||
await asyncio.gather(*done, return_exceptions=True)
|
||||
for pending_future in pending:
|
||||
pending_future.add_done_callback(
|
||||
lambda completed: completed.exception()
|
||||
if not completed.cancelled()
|
||||
else None
|
||||
)
|
||||
|
||||
with _blocking_executor_lock:
|
||||
unfinished = tuple(
|
||||
future for future in _blocking_futures if not future.done()
|
||||
)
|
||||
retiring_count = len(_blocking_retiring_executors)
|
||||
if unfinished:
|
||||
logger.error(
|
||||
"Agent 阻塞工具未在 %.1f 秒内收敛:futures=%d,executors=%d",
|
||||
max(0.0, timeout_seconds),
|
||||
len(unfinished),
|
||||
retiring_count,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def shutdown_blocking_executors(
|
||||
*,
|
||||
wait: bool = True,
|
||||
cancel_futures: bool = False,
|
||||
) -> bool:
|
||||
"""同步清理测试 owner;非等待模式下保留尚未收敛的 executor 句柄。"""
|
||||
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
for executor in executors:
|
||||
if wait:
|
||||
executor.shutdown(wait=True, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
return not _blocking_futures and not _blocking_retiring_executors
|
||||
|
||||
|
||||
class ToolExecutionTimeoutError(TimeoutError):
|
||||
@@ -226,13 +323,7 @@ async def run_agent_blocking(
|
||||
|
||||
await semaphore.acquire()
|
||||
try:
|
||||
context = copy_context()
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(
|
||||
_get_blocking_executor(bucket_name).submit,
|
||||
context.run,
|
||||
bound_call,
|
||||
)
|
||||
future = _submit_blocking_call(bucket_name, bound_call)
|
||||
except Exception:
|
||||
semaphore.release()
|
||||
raise
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from contextvars import copy_context
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
|
||||
|
||||
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载插件实例并重新注册其命令、定时任务和 API。"""
|
||||
runtime_status = get_plugin_manager().reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
plugin_manager = get_plugin_manager()
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
|
||||
|
||||
def summarize_plugin(plugin: Any) -> dict[str, Any]:
|
||||
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
|
||||
|
||||
async def reload_runtime(target_id: str) -> object:
|
||||
"""通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
|
||||
mutation_context = copy_context()
|
||||
return await run_agent_blocking(
|
||||
"plugin",
|
||||
mutation_context.run,
|
||||
plugin_manager.reload_plugin_tree,
|
||||
target_id,
|
||||
)
|
||||
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
|
||||
)
|
||||
return result
|
||||
|
||||
with plugin_manager.suppress_plugin_monitor(plugin_id):
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
mutation=plugin_manager.mutation,
|
||||
package_write_guard=plugin_manager.suppress_plugin_monitor,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
return result.success, result.message, result.refreshed_only
|
||||
|
||||
|
||||
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
|
||||
from app.agent.tools.base import run_agent_blocking
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
|
||||
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [
|
||||
plugin for plugin in install_plugins if plugin != plugin_id
|
||||
]
|
||||
await config_oper.async_set(
|
||||
SystemConfigKey.UserInstalledPlugins,
|
||||
install_plugins,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
|
||||
@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
|
||||
)
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
if not await plugin_manager.async_save_plugin_config(
|
||||
plugin_id,
|
||||
next_config,
|
||||
):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user