fix(agent): 收敛超时与取消后的副作用生命周期 (#6388)

This commit is contained in:
InfinityPacer
2026-08-22 12:11:15 +08:00
committed by GitHub
parent 7b6353337d
commit ed3c0541c7
16 changed files with 952 additions and 72 deletions
+2 -1
View File
@@ -377,7 +377,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
except ToolExecutionTimeoutError as e:
error_message = summarize_error(e)
logger.warning(error_message)
result = error_message
raise
except Exception as e:
error_message = f"工具执行异常: {summarize_error(e)}"
logger.error(f"Tool {self.name} execution failed: {summarize_error(e)}")
@@ -415,6 +415,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
except asyncio.TimeoutError as err:
raise ToolExecutionTimeoutError(
f"工具 {self.name} 执行超时(超过 {timeout:g} 秒),已停止等待结果。"
"若工具包含外部写操作,操作可能仍在继续,请先确认实际状态再重试。"
) from err
@staticmethod
+104 -8
View File
@@ -128,6 +128,11 @@ class _TerminalSessionManager:
"""初始化会话表和并发保护锁。"""
self._sessions: dict[str, _TerminalSession] = {}
self._lock = asyncio.Lock()
self._close_lock = asyncio.Lock()
self._closed = False
self._starting = 0
self._starts_idle = asyncio.Event()
self._starts_idle.set()
@staticmethod
def _normalize_bool(value: Any, default: bool = True) -> bool:
@@ -211,20 +216,55 @@ class _TerminalSessionManager:
should_use_pty = self._normalize_bool(use_pty, default=True) and os.name == "posix"
async with self._lock:
if self._closed:
raise RuntimeError("终端会话管理器已关闭")
self._cleanup_finished_sessions_locked()
if self._active_session_count_locked() >= TERMINAL_CONCURRENCY_LIMIT:
if (
self._active_session_count_locked() + self._starting
>= TERMINAL_CONCURRENCY_LIMIT
):
raise RuntimeError(
f"后台终端会话数已达到上限 {TERMINAL_CONCURRENCY_LIMIT}"
)
self._starting += 1
self._starts_idle.clear()
session = (
await self._start_pty_session(command, normalized_cwd, normalized_env)
if should_use_pty
else await self._start_pipe_session(command, normalized_cwd, normalized_env)
)
session: Optional[_TerminalSession] = None
reject_session = False
session_registered = False
session_released = False
try:
session = (
await self._start_pty_session(command, normalized_cwd, normalized_env)
if should_use_pty
else await self._start_pipe_session(
command, normalized_cwd, normalized_env
)
)
async with self._lock:
self._sessions[session.session_id] = session
async with self._lock:
reject_session = self._closed
if not reject_session:
self._sessions[session.session_id] = session
session_registered = True
if reject_session:
await self._terminate_session(session)
session_released = True
raise RuntimeError("终端会话管理器已关闭")
except BaseException:
if session is not None and not session_registered and not session_released:
cleanup_task = asyncio.create_task(self._terminate_session(session))
try:
await asyncio.shield(cleanup_task)
except asyncio.CancelledError:
await cleanup_task
raise
finally:
async with self._lock:
self._starting -= 1
if self._starting == 0:
self._starts_idle.set()
logger.info(
"启动后台终端会话: session_id=%s, pid=%s, use_pty=%s, command=%s",
@@ -473,6 +513,62 @@ class _TerminalSessionManager:
return self._session_payload(session, output="", output_truncated=False)
async def close(self) -> None:
"""停止所有后台终端会话并释放 PTY、读取任务和会话记录。"""
async with self._close_lock:
async with self._lock:
self._closed = True
await self._starts_idle.wait()
async with self._lock:
sessions = list(self._sessions.values())
await asyncio.gather(
*(self._terminate_session(session) for session in sessions),
return_exceptions=True,
)
async with self._lock:
for session in sessions:
session.close_pty()
self._sessions.clear()
async def _terminate_session(self, session: _TerminalSession) -> None:
"""以有限等待停止进程,并在必要时升级为 SIGKILL。"""
if session.status == "running":
session.kill_requested = True
self._send_signal(session, signal.SIGTERM)
wait_task = session.wait_task
if wait_task and not wait_task.done():
try:
await asyncio.wait_for(
asyncio.shield(wait_task),
timeout=TERMINAL_KILL_GRACE_SECONDS,
)
except asyncio.TimeoutError:
force_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
self._send_signal(session, force_signal)
try:
await asyncio.wait_for(
asyncio.shield(wait_task),
timeout=TERMINAL_KILL_GRACE_SECONDS,
)
except asyncio.TimeoutError:
logger.error(
"终端会话关闭超时: session_id=%s, pid=%s",
session.session_id,
session.pid,
)
for task in session.reader_tasks:
if not task.done():
task.cancel()
if session.reader_tasks:
await asyncio.gather(*session.reader_tasks, return_exceptions=True)
session.close_pty()
def get_session(self, session_id: str) -> _TerminalSession:
"""按 ID 获取会话,不存在时抛出清晰错误。"""
session = self._sessions.get(session_id)
+11 -5
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import json
import threading
import uuid
@@ -8,7 +9,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
from app.runtime.log import logger
if TYPE_CHECKING:
from app.agent.policy import AgentToolPolicyOrchestrator, ToolPolicyContext
from app.agent.policy.contracts import ToolPolicyContext
from app.agent.policy.orchestrator import AgentToolPolicyOrchestrator
from app.agent.tools.catalog import ToolCatalogSnapshot
@@ -57,7 +59,7 @@ class MoviePilotToolsManager:
@staticmethod
def _summarize_error(error: Exception) -> str:
"""仅在错误路径加载策略脱敏器,保持默认导入轻量。"""
from app.agent.policy import summarize_error
from app.agent.policy.sanitizer import summarize_error
return summarize_error(error)
@@ -143,13 +145,13 @@ class MoviePilotToolsManager:
if policy_orchestrator is not None and policy_context is not None:
return policy_orchestrator, policy_context
from app.agent.policy import (
DEFAULT_TOOL_POLICY_ORCHESTRATOR,
from app.agent.policy.contracts import (
AuthSource,
PrincipalType,
ToolOrigin,
ToolPolicyContext,
)
from app.agent.policy.orchestrator import DEFAULT_TOOL_POLICY_ORCHESTRATOR
if policy_orchestrator is None:
policy_orchestrator = DEFAULT_TOOL_POLICY_ORCHESTRATOR
@@ -381,7 +383,7 @@ class MoviePilotToolsManager:
)
return error_msg
from app.agent.policy import call_policy_hook
from app.agent.policy.orchestrator import call_policy_hook
from app.agent.tools.base import (
ToolExecutionTimeoutError,
format_tool_result_for_agent,
@@ -414,6 +416,10 @@ class MoviePilotToolsManager:
tool_name=tool_name,
max_chars=getattr(tool_instance, "result_max_chars", None),
)
except asyncio.CancelledError as e:
if observation is not None and policy_orchestrator is not None:
call_policy_hook("cancel", policy_orchestrator.fail, observation, e)
raise
except ToolExecutionTimeoutError as e:
if observation is not None and policy_orchestrator is not None:
call_policy_hook("fail", policy_orchestrator.fail, observation, e)