From 010899f369c2c55b7401308526319f47d4fd0fbd Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:48:14 +0800 Subject: [PATCH] feat(agent): add host policy foundation (#6273) --- app/agent/__init__.py | 54 +- app/agent/callback/__init__.py | 9 +- app/agent/middleware/activity_log.py | 55 +- app/agent/middleware/policy.py | 72 + app/agent/middleware/skills.py | 33 +- app/agent/middleware/subagents.py | 99 +- app/agent/policy/__init__.py | 66 + app/agent/policy/contracts.py | 247 +++ app/agent/policy/orchestrator.py | 191 ++ app/agent/policy/registry.py | 181 ++ app/agent/policy/sanitizer.py | 1056 ++++++++++ app/agent/policy/secret_fields.py | 69 + app/agent/tools/base.py | 32 +- app/agent/tools/impl/_system_setting_utils.py | 21 +- app/agent/tools/manager.py | 67 +- tests/test_agent_activity_log.py | 63 +- tests/test_agent_background_output.py | 3 + tests/test_agent_skills_middleware.py | 57 + tests/test_agent_subagents.py | 92 +- tests/test_agent_system_settings_tools.py | 43 + tests/test_agent_tool_policy.py | 744 +++++++ tests/test_agent_tool_result_policy.py | 1781 +++++++++++++++++ tests/test_agent_tool_streaming.py | 18 + 23 files changed, 4951 insertions(+), 102 deletions(-) create mode 100644 app/agent/middleware/policy.py create mode 100644 app/agent/policy/__init__.py create mode 100644 app/agent/policy/contracts.py create mode 100644 app/agent/policy/orchestrator.py create mode 100644 app/agent/policy/registry.py create mode 100644 app/agent/policy/sanitizer.py create mode 100644 app/agent/policy/secret_fields.py create mode 100644 tests/test_agent_tool_policy.py create mode 100644 tests/test_agent_tool_result_policy.py diff --git a/app/agent/__init__.py b/app/agent/__init__.py index d797a3f2d..6a90e1c21 100644 --- a/app/agent/__init__.py +++ b/app/agent/__init__.py @@ -39,6 +39,7 @@ from app.agent.middleware.jobs import ( ) from app.agent.middleware.memory import MemoryMiddleware from app.agent.middleware.patch_tool_calls import PatchToolCallsMiddleware +from app.agent.middleware.policy import AgentPolicyMiddleware from app.agent.middleware.runtime_config import RuntimeConfigMiddleware from app.agent.middleware.skills import SKILL_TOOL_NAME, SkillsMiddleware from app.agent.middleware.subagents import ( @@ -50,6 +51,12 @@ from app.agent.middleware.subagents import ( from app.agent.middleware.tool_selection import ToolSelectorMiddleware from app.agent.middleware.usage import UsageMiddleware from app.agent.prompt import prompt_manager +from app.agent.policy import ( + AuthSource, + PrincipalType, + ToolOrigin, + ToolPolicyContext, +) from app.agent.runtime import agent_runtime_manager from app.agent.mcp import agent_mcp_manager from app.agent.tools.factory import MoviePilotToolFactory @@ -727,6 +734,40 @@ class MoviePilotAgent: "original_chat_id": None if self.is_background else self.original_chat_id, } + def _build_policy_context(self) -> ToolPolicyContext: + """根据宿主入口建立模型参数无法伪造的策略上下文。""" + if not self.has_message_context: + origin = ToolOrigin.BACKGROUND + principal_type = PrincipalType.BACKGROUND + auth_source = AuthSource.INTERNAL + elif self.channel == MessageChannel.Web.value and self.source in { + "openai", + "openai.responses", + "anthropic", + }: + origin = ToolOrigin.AGENT_API + principal_type = PrincipalType.SYSTEM_ADMIN_INTEGRATION + auth_source = AuthSource.API_TOKEN + else: + origin = ToolOrigin.AGENT_INTERACTIVE + principal_type = PrincipalType.HUMAN + auth_source = ( + AuthSource.WEB_SESSION + if self.channel + in {MessageChannel.Web.value, MessageChannel.WebAgent.value} + else AuthSource.CHANNEL + ) + return ToolPolicyContext( + session_id=self.session_id, + user_id=str(self.user_id or self.username or principal_type.value), + origin=origin, + principal_type=principal_type, + auth_source=auth_source, + agent_context=self._tool_context, + channel=self.channel, + source=self.source, + ) + def _should_stream(self) -> bool: """ 判断是否应启用流式输出： @@ -1255,6 +1296,7 @@ class MoviePilotAgent: # LLM 模型（用于 agent 执行） agent_model = await self._initialize_llm(streaming=streaming) self._sync_model_profile(agent_model) + # 供应商原生工具不进入本地 ToolNode，宿主策略只覆盖 client-side tools。 server_tools = LLMHelper.get_server_tools(agent_model) use_local_web_search = LLMHelper.should_use_local_web_search(agent_model) @@ -1292,11 +1334,13 @@ class MoviePilotAgent: enabled=use_local_web_search, ) subagent_tools.extend(await self._initialize_subagent_mcp_tools()) + policy_context = self._build_policy_context() subagent_middlewares, subagent_task_tools = create_subagent_middlewares( model=non_streaming_model, tools=subagent_tools, server_tools=server_tools, stream_handler=self.stream_handler, + policy_context=policy_context.for_subagent(), ) max_tools = settings.LLM_MAX_TOOLS always_include_tools = ( @@ -1324,6 +1368,8 @@ class MoviePilotAgent: # 中间件 middlewares = [ + # 宿主策略必须位于最外层，确保插件覆盖工具基类也不能绕过。 + AgentPolicyMiddleware(context=policy_context), # Skills skills_middleware, # Jobs 任务管理 @@ -1334,6 +1380,8 @@ class MoviePilotAgent: RuntimeConfigMiddleware(), # 记忆管理 MemoryMiddleware(memory_dir=str(agent_runtime_manager.memory_dir)), + # 活动日志依赖记忆上下文，并应在摘要压缩前完成读取与记录。 + *([activity_log_middleware] if activity_log_middleware else []), # 上下文压缩 SummarizationMiddleware( model=non_streaming_model, trigger=("fraction", 0.85) @@ -1346,12 +1394,6 @@ class MoviePilotAgent: UsageMiddleware(on_usage=self._record_usage), ] - if self.has_message_context: - middlewares.insert( - 4, - activity_log_middleware, - ) - # 工具选择 if max_tools > 0: middlewares.append( diff --git a/app/agent/callback/__init__.py b/app/agent/callback/__init__.py index a091e7fd6..4a67366a9 100644 --- a/app/agent/callback/__init__.py +++ b/app/agent/callback/__init__.py @@ -4,6 +4,7 @@ from typing import Any, Optional, Tuple from fastapi.concurrency import run_in_threadpool +from app.agent.policy import sanitize_for_host from app.chain import ChainBase from app.log import logger from app.schemas import Notification @@ -256,10 +257,14 @@ class StreamingHandler: """ 记录一次工具调用，供非啰嗦模式下延迟汇总输出。 """ + recorded_message = sanitize_for_host(tool_message) if tool_message else tool_message + recorded_args = sanitize_for_host(tool_kwargs or {}) + if not isinstance(recorded_args, dict): + recorded_args = {} category, target = self._classify_tool_call( tool_name=tool_name, - tool_message=tool_message, - tool_kwargs=tool_kwargs or {}, + tool_message=recorded_message, + tool_kwargs=recorded_args, ) target_values = [] if isinstance(target, (list, tuple, set)): diff --git a/app/agent/middleware/activity_log.py b/app/agent/middleware/activity_log.py index 287f54be4..999f46a94 100644 --- a/app/agent/middleware/activity_log.py +++ b/app/agent/middleware/activity_log.py @@ -33,6 +33,7 @@ from langgraph.runtime import Runtime from pydantic import BaseModel, Field from app.agent.middleware.utils import append_to_system_message +from app.agent.policy import sanitize_for_host, summarize_error, summarize_result from app.agent.tools.tags import ToolTag from app.log import logger @@ -181,7 +182,9 @@ def load_activity_log_index(activity_dir: str, days: int = PROMPT_LOAD_DAYS) -> try: content = log_path.read_text(encoding="utf-8", errors="replace") except Exception as e: - logger.warning(f"读取活动日志索引失败 {log_path}: {e}") + logger.warning( + f"读取活动日志索引失败 {log_path}: {summarize_error(e)}" + ) continue entry_count = len(_parse_activity_entries(date_str, content)) if entry_count: @@ -245,7 +248,7 @@ def query_activity_logs( try: content = log_path.read_text(encoding="utf-8", errors="replace") except Exception as e: - logger.warning(f"读取活动日志失败 {log_path}: {e}") + logger.warning(f"读取活动日志失败 {log_path}: {summarize_error(e)}") continue for entry in _parse_activity_entries(date_str, content): if normalized_keyword and not _activity_summary_matches_keyword( @@ -287,14 +290,16 @@ class _ActivityLogToolProvider: limit: Optional[int] = DEFAULT_QUERY_LIMIT, ) -> str: """查询活动日志并返回 JSON 字符串。""" - logger.info( - "查询活动日志: keyword=%s, use_regex=%s, date=%s, days=%s, limit=%s", - keyword, - use_regex, - date, - days, - limit, + logged_args = sanitize_for_host( + { + "keyword": keyword, + "use_regex": use_regex, + "date": date, + "days": days, + "limit": limit, + } ) + logger.info(f"查询活动日志: args={logged_args}") try: payload = query_activity_logs( self._activity_dir, @@ -306,11 +311,12 @@ class _ActivityLogToolProvider: ) return json.dumps(payload, ensure_ascii=False, indent=2) except Exception as err: - logger.error(f"查询活动日志失败: {err}", exc_info=True) + error_summary = summarize_error(err) + logger.error(f"查询活动日志失败: {error_summary}") return json.dumps( { "success": False, - "message": f"查询活动日志时发生错误: {str(err)}", + "message": f"查询活动日志时发生错误: {error_summary}", }, ensure_ascii=False, ) @@ -454,7 +460,7 @@ async def _summarize_with_llm(conversation_text: str) -> Optional[str]: return None return summary if summary else None except Exception as e: - logger.debug(f"LLM 活动摘要生成失败: {e}") + logger.debug(f"LLM 活动摘要生成失败: {summarize_error(e)}") return None @@ -571,9 +577,9 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response else: with os.fdopen(fd, "w", encoding="utf-8") as stream: stream.write(header + entry) - logger.debug(f"Activity logged: {summary[:80]}") + logger.debug(f"Activity logged: {summarize_result(summary, max_chars=80)}") except Exception as e: - logger.warning(f"Failed to append activity log: {e}") + logger.warning(f"Failed to append activity log: {summarize_error(e)}") async def _cleanup_old_logs(self) -> None: """清理超过保留天数的旧日志文件。""" @@ -599,7 +605,9 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response except ValueError: continue except Exception as e: - logger.warning(f"Failed to cleanup old activity logs: {e}") + logger.warning( + f"Failed to cleanup old activity logs: {summarize_error(e)}" + ) def _schedule_activity_recording(self, messages: list) -> None: """提交后台活动记录任务，不阻塞当前 Agent 会话结束。""" @@ -615,7 +623,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response except asyncio.CancelledError: logger.debug("活动日志后台记录任务已取消") except Exception as err: - logger.warning(f"活动日志后台记录任务失败: {err}") + logger.warning(f"活动日志后台记录任务失败: {summarize_error(err)}") async def _record_activity(self, messages: list) -> None: """在后台生成本轮活动摘要并写入活动日志。""" @@ -637,7 +645,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response if summary: await self._append_activity(summary) except Exception as e: - logger.warning(f"Failed to record activity: {e}") + logger.warning(f"Failed to record activity: {summarize_error(e)}") async def abefore_agent( self, state: ActivityLogState, runtime: Runtime @@ -686,9 +694,12 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response tool_args = tool_call.get("args") or {} if not isinstance(tool_args, dict): tool_args = {} + logged_args = sanitize_for_host(tool_args) + if not isinstance(logged_args, dict): + logged_args = {} logger.info( - f"开始执行活动日志查询工具: keyword={tool_args.get('keyword') or '-'}, " - f"date={tool_args.get('date') or '-'}" + f"开始执行活动日志查询工具: keyword={logged_args.get('keyword') or '-'}, " + f"date={logged_args.get('date') or '-'}" ) if self.stream_handler and getattr(self.stream_handler, "is_streaming", False): self.stream_handler.record_tool_call( @@ -699,7 +710,9 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response try: result = await handler(request) except Exception as err: - logger.error(f"活动日志查询工具执行失败: error={err}") + logger.error( + f"活动日志查询工具执行失败: error={summarize_error(err)}" + ) raise logger.info("活动日志查询工具执行完成") return result @@ -714,7 +727,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response return None self._schedule_activity_recording(list(messages)) except Exception as e: - logger.warning(f"Failed to record activity: {e}") + logger.warning(f"Failed to record activity: {summarize_error(e)}") return None diff --git a/app/agent/middleware/policy.py b/app/agent/middleware/policy.py new file mode 100644 index 000000000..384885583 --- /dev/null +++ b/app/agent/middleware/policy.py @@ -0,0 +1,72 @@ +"""LangChain 工具调用的 MoviePilot 宿主策略中间件。""" + +from collections.abc import Awaitable, Callable +from typing import Any + +from langchain.agents.middleware import AgentMiddleware, ToolCallRequest + +from app.agent.policy import ( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + AgentToolPolicyOrchestrator, + ToolPolicyContext, + call_policy_hook, +) + + +class AgentPolicyMiddleware(AgentMiddleware): + """观测进入本地 ToolNode 的 client-side 工具调用和结果。 + + 模型供应商原生 server tools 在供应商侧执行，不经过本地 middleware， + 因而不具备这里生成的 start/finish/fail 回执。 + """ + + def __init__( + self, + *, + context: ToolPolicyContext, + orchestrator: AgentToolPolicyOrchestrator = DEFAULT_TOOL_POLICY_ORCHESTRATOR, + ) -> None: + """绑定宿主可信上下文和共享策略编排器。""" + self.context = context + self.orchestrator = orchestrator + + async def awrap_tool_call( + self, + request: ToolCallRequest, + handler: Callable[[ToolCallRequest], Awaitable[Any]], + ) -> Any: + """在 handler 外层生成 shadow 决策和 secret-safe 回执摘要。""" + tool_call = request.tool_call or {} + arguments = tool_call.get("args") or {} + if not isinstance(arguments, dict): + arguments = {} + observation = call_policy_hook( + "start", + self.orchestrator.start, + context=self.context, + tool=request.tool, + arguments=arguments, + invocation_id=tool_call.get("id"), + ) + try: + result = await handler(request) + except Exception as error: + if observation is not None: + call_policy_hook( + "fail", + self.orchestrator.fail, + observation, + error, + ) + raise + if observation is not None: + call_policy_hook( + "finish", + self.orchestrator.finish, + observation, + result, + ) + return result + + +__all__ = ["AgentPolicyMiddleware"] diff --git a/app/agent/middleware/skills.py b/app/agent/middleware/skills.py index b7424d460..311df42b8 100644 --- a/app/agent/middleware/skills.py +++ b/app/agent/middleware/skills.py @@ -24,6 +24,7 @@ from langgraph.runtime import Runtime from pydantic import BaseModel, Field from app.agent.middleware.utils import append_to_system_message +from app.agent.policy import sanitize_for_host, summarize_error from app.agent.tools.tags import ToolTag from app.log import logger @@ -124,7 +125,7 @@ def _parse_skill_metadata( # noqa: C901 try: frontmatter_data = yaml.safe_load(frontmatter_str) except yaml.YAMLError as e: - logger.warning("Invalid YAML in %s: %s", skill_path, e) + logger.warning("Invalid YAML in %s: %s", skill_path, summarize_error(e)) return None if not isinstance(frontmatter_data, dict): @@ -339,7 +340,7 @@ def _extract_version(skill_md: Path) -> int: try: content = skill_md.read_text(encoding="utf-8", errors="replace") except Exception as err: - logger.debug(f"读取技能版本失败: {err}") + logger.debug(f"读取技能版本失败: {summarize_error(err)}") return 0 match = re.match(r"^---\s*\n(.*?)\n---\s*\n", content, re.DOTALL) if not match: @@ -397,7 +398,11 @@ def _sync_bundled_skills(bundled_dir: Path, target_dir: Path) -> None: "已自动复制内置技能 '%s' -> '%s'", skill_src.name, skill_dst ) except Exception as e: - logger.warning("复制内置技能 '%s' 失败: %s", skill_src.name, e) + logger.warning( + "复制内置技能 '%s' 失败: %s", + sanitize_for_host(skill_src.name), + summarize_error(e), + ) continue # 目标已存在，比较版本号 @@ -424,7 +429,11 @@ def _sync_bundled_skills(bundled_dir: Path, target_dir: Path) -> None: bundled_version, ) except Exception as e: - logger.warning("更新内置技能 '%s' 失败: %s", skill_src.name, e) + logger.warning( + "更新内置技能 '%s' 失败: %s", + sanitize_for_host(skill_src.name), + summarize_error(e), + ) class _SkillToolProvider: @@ -519,7 +528,7 @@ class _SkillToolProvider: async def load_skill(self, name: str) -> str: """加载指定 Skill 的完整说明并返回 JSON 字符串。""" - logger.info(f"加载 Skill: name={name}") + logger.info(f"加载 Skill: name={sanitize_for_host(name)}") try: skill = await self._find_skill(name) if not skill: @@ -547,11 +556,12 @@ class _SkillToolProvider: } ) except Exception as err: - logger.error(f"加载 Skill 失败: {err}", exc_info=True) + error_summary = summarize_error(err) + logger.error(f"加载 Skill 失败: {error_summary}") return json.dumps( { "success": False, - "message": f"加载 Skill 时发生错误: {str(err)}", + "message": f"加载 Skill 时发生错误: {error_summary}", }, ensure_ascii=False, ) @@ -623,7 +633,7 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no try: _sync_bundled_skills(bundled, target) except Exception as e: - logger.warning("同步内置技能失败: %s", e) + logger.warning(f"同步内置技能失败: {summarize_error(e)}") def _load_skills_metadata(self) -> list[SkillMetadata]: """同步加载当前配置目录中的 Skill 元数据。""" @@ -728,8 +738,11 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no tool_args = tool_call.get("args") or {} if not isinstance(tool_args, dict): tool_args = {} + logged_args = sanitize_for_host(tool_args) + if not isinstance(logged_args, dict): + logged_args = {} logger.info( - f"开始执行 Skill 工具: name={tool_args.get('name') or '-'}" + f"开始执行 Skill 工具: name={logged_args.get('name') or '-'}" ) if self.stream_handler and getattr(self.stream_handler, "is_streaming", False): self.stream_handler.record_tool_call( @@ -740,7 +753,7 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no try: result = await handler(request) except Exception as err: - logger.error(f"Skill 工具执行失败: error={err}") + logger.error(f"Skill 工具执行失败: error={summarize_error(err)}") raise logger.info("Skill 工具执行完成") return result diff --git a/app/agent/middleware/subagents.py b/app/agent/middleware/subagents.py index 544ca66d6..3bec10f91 100644 --- a/app/agent/middleware/subagents.py +++ b/app/agent/middleware/subagents.py @@ -24,7 +24,16 @@ from langchain_core.tools import BaseTool, StructuredTool from pydantic import BaseModel, Field from app.agent.llm import LLMHelper +from app.agent.middleware.policy import AgentPolicyMiddleware from app.agent.middleware.utils import append_to_system_message +from app.agent.policy import ( + AuthSource, + PrincipalType, + ToolOrigin, + ToolPolicyContext, + sanitize_for_host, + summarize_error, +) from app.agent.runtime import SubAgentDefinition, agent_runtime_manager from app.agent.tools.tags import ToolTag from app.log import logger @@ -93,6 +102,36 @@ Requirements: """ +def _default_subagent_policy_context(tools: list[BaseTool]) -> ToolPolicyContext: + """从已注入工具继承会话归属，确保独立构造的子图也经过宿主策略。""" + for tool in tools: + session_id = getattr(tool, "_session_id", None) + user_id = getattr(tool, "_user_id", None) + if not session_id and not user_id: + continue + agent_context = getattr(tool, "_agent_context", None) + if not isinstance(agent_context, dict): + agent_context = {} + return ToolPolicyContext( + session_id=str(session_id or "subagent"), + user_id=str(user_id or "subagent"), + origin=ToolOrigin.SUBAGENT, + principal_type=PrincipalType.SUBAGENT, + auth_source=AuthSource.INTERNAL, + agent_context=agent_context, + channel=getattr(tool, "_channel", None), + source=getattr(tool, "_source", None), + ) + return ToolPolicyContext( + session_id="subagent", + user_id="subagent", + origin=ToolOrigin.SUBAGENT, + principal_type=PrincipalType.SUBAGENT, + auth_source=AuthSource.INTERNAL, + agent_context={}, + ) + + @dataclass(frozen=True) class _SubAgentProfile: """子代理运行时定义。""" @@ -378,12 +417,14 @@ class _SubAgentAgentProvider: profiles: tuple[_SubAgentProfile, ...], tools: list[BaseTool], server_tools: Optional[list[dict[str, Any]]] = None, + policy_context: Optional[ToolPolicyContext] = None, ) -> None: """初始化子代理执行器。""" self._model = model self._profiles = {profile.name: profile for profile in profiles} self._tools = tools self._server_tools = server_tools or [] + self._policy_context = policy_context or _default_subagent_policy_context(tools) self._agents = {} self._default_agent_name = "general-purpose" @@ -409,6 +450,7 @@ class _SubAgentAgentProvider: tools=[*subagent_tools, *self._server_tools], system_prompt=profile.prompt, name=profile.name, + middleware=[AgentPolicyMiddleware(context=self._policy_context)], ) self._agents[profile.name] = agent return profile.name, agent @@ -444,7 +486,7 @@ class _SubAgentAgentProvider: except Exception as err: logger.error( f"子代理调用失败: subagent_type={agent_name}, " - f"task_id={log_task_id}, error={err}" + f"task_id={log_task_id}, error={summarize_error(err)}" ) raise final_text = _extract_final_text(result) @@ -468,6 +510,7 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware): system_prompt: str = SUBAGENT_PARENT_PROMPT, task_description: str = SUBAGENT_TASK_DESCRIPTION, stream_handler: Any = None, + policy_context: Optional[ToolPolicyContext] = None, ) -> None: """初始化同步子代理中间件。""" self.system_prompt = system_prompt @@ -477,6 +520,7 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware): profiles=profiles, tools=tools, server_tools=server_tools, + policy_context=policy_context, ) self.tools = [ StructuredTool.from_function( @@ -527,9 +571,12 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware): return await handler(request) tool_args = _extract_tool_call_args(request) + logged_args = sanitize_for_host(tool_args) + if not isinstance(logged_args, dict): + logged_args = {} logger.info( f"开始执行子代理工具: tool_name={tool_name}, " - f"subagent_type={tool_args.get('subagent_type') or '-'}" + f"subagent_type={logged_args.get('subagent_type') or '-'}" ) _record_subagent_tool_call( stream_handler=self.stream_handler, @@ -539,7 +586,10 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware): try: result = await handler(request) except Exception as err: - logger.error(f"子代理工具执行失败: tool_name={tool_name}, error={err}") + logger.error( + f"子代理工具执行失败: tool_name={tool_name}, " + f"error={summarize_error(err)}" + ) raise logger.info(f"子代理工具执行完成: tool_name={tool_name}") return result @@ -557,6 +607,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): server_tools: Optional[list[dict[str, Any]]] = None, task_description: str = SUBAGENT_CONTROL_DESCRIPTION, stream_handler: Any = None, + policy_context: Optional[ToolPolicyContext] = None, ) -> None: """初始化异步子代理调度中间件。""" self.stream_handler = stream_handler @@ -565,6 +616,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): profiles=profiles, tools=tools, server_tools=server_tools, + policy_context=policy_context, ) self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS) self._tasks: dict[str, _SubAgentRuntimeTask] = {} @@ -628,7 +680,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): error = record.task.exception() if error: - payload["error"] = str(error) + payload["error"] = summarize_error(error) return payload result, result_truncated = _clip_text( @@ -733,7 +785,10 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): ) raise except Exception as err: - logger.error(f"子代理任务执行失败: task_id={record.task_id}, error={err}") + logger.error( + f"子代理任务执行失败: task_id={record.task_id}, " + f"error={summarize_error(err)}" + ) raise def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None: @@ -901,7 +956,10 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): ) raise except Exception as err: - logger.error(f"管道子代理任务执行失败: task_id={record.task_id}, error={err}") + logger.error( + f"管道子代理任务执行失败: task_id={record.task_id}, " + f"error={summarize_error(err)}" + ) raise @staticmethod @@ -975,8 +1033,13 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): ) return records, error except Exception as err: - error = f"第 {step_index} 个管道子代理任务执行失败: {err}" - logger.info(f"{error} task_id={record.task_id}") + error = ( + f"第 {step_index} 个管道子代理任务执行失败: " + f"{summarize_error(err)}" + ) + logger.info( + f"{error} task_id={record.task_id}" + ) return records, error previous_results.append((record, result)) @@ -1003,7 +1066,10 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): tasks=tasks, ) if error: - logger.info(f"子代理管控操作未启动任务: action={action}, error={error}") + logger.info( + f"子代理管控操作未启动任务: action={action}, " + f"error={sanitize_for_host(error)}" + ) return self._json_response({"success": False, "error": error}) logger.info(f"准备启动子代理任务: action={action}, tasks={len(specs)}") @@ -1095,10 +1161,13 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): return await handler(request) tool_args = _extract_tool_call_args(request) + logged_args = sanitize_for_host(tool_args) + if not isinstance(logged_args, dict): + logged_args = {} logger.info( f"开始执行子代理工具: tool_name={tool_name}, " - f"action={tool_args.get('action') or '-'}, " - f"subagent_type={tool_args.get('subagent_type') or '-'}" + f"action={logged_args.get('action') or '-'}, " + f"subagent_type={logged_args.get('subagent_type') or '-'}" ) _record_subagent_tool_call( stream_handler=self.stream_handler, @@ -1108,7 +1177,10 @@ class SubAgentTaskControlMiddleware(AgentMiddleware): try: result = await handler(request) except Exception as err: - logger.error(f"子代理工具执行失败: tool_name={tool_name}, error={err}") + logger.error( + f"子代理工具执行失败: tool_name={tool_name}, " + f"error={summarize_error(err)}" + ) raise logger.info(f"子代理工具执行完成: tool_name={tool_name}") return result @@ -1120,6 +1192,7 @@ def create_subagent_middlewares( tools: list[BaseTool], server_tools: Optional[list[dict[str, Any]]] = None, stream_handler: Any = None, + policy_context: Optional[ToolPolicyContext] = None, ) -> tuple[list[AgentMiddleware], list[BaseTool]]: """创建子代理中间件列表和任务工具列表。""" runtime_signature = agent_runtime_manager.current_signature() @@ -1130,6 +1203,7 @@ def create_subagent_middlewares( tools=tools, server_tools=server_tools or [], stream_handler=stream_handler, + policy_context=policy_context, ) control_middleware = SubAgentTaskControlMiddleware( model=model, @@ -1137,6 +1211,7 @@ def create_subagent_middlewares( tools=tools, server_tools=server_tools or [], stream_handler=stream_handler, + policy_context=policy_context, ) task_tools = [ diff --git a/app/agent/policy/__init__.py b/app/agent/policy/__init__.py new file mode 100644 index 000000000..393700215 --- /dev/null +++ b/app/agent/policy/__init__.py @@ -0,0 +1,66 @@ +"""MoviePilot Agent 宿主策略公共内部入口。""" + +from app.agent.policy.contracts import ( + ActionEffect, + ActionPolicy, + AuthSource, + ConfirmationMode, + ExecutionOutcome, + ExecutionReceipt, + MigrationState, + PolicyDecision, + PolicyObservation, + PolicyPrincipal, + PrincipalRole, + PrincipalType, + RecoveryMode, + ResultSensitivity, + ToolInvocation, + ToolOrigin, + ToolPolicyContext, +) +from app.agent.policy.orchestrator import ( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + AgentToolPolicyOrchestrator, + call_policy_hook, +) +from app.agent.policy.registry import DEFAULT_TOOL_POLICY_REGISTRY, ToolPolicyRegistry +from app.agent.policy.sanitizer import ( + REDACTED_VALUE, + sanitize_for_host, + stable_type_name, + summarize_error, + summarize_input, + summarize_result, +) + +__all__ = [ + "ActionEffect", + "ActionPolicy", + "AgentToolPolicyOrchestrator", + "AuthSource", + "ConfirmationMode", + "DEFAULT_TOOL_POLICY_ORCHESTRATOR", + "DEFAULT_TOOL_POLICY_REGISTRY", + "ExecutionOutcome", + "ExecutionReceipt", + "MigrationState", + "PolicyDecision", + "PolicyObservation", + "PolicyPrincipal", + "PrincipalRole", + "PrincipalType", + "REDACTED_VALUE", + "RecoveryMode", + "ResultSensitivity", + "ToolInvocation", + "ToolOrigin", + "ToolPolicyContext", + "ToolPolicyRegistry", + "call_policy_hook", + "sanitize_for_host", + "stable_type_name", + "summarize_error", + "summarize_input", + "summarize_result", +] diff --git a/app/agent/policy/contracts.py b/app/agent/policy/contracts.py new file mode 100644 index 000000000..1541cd08d --- /dev/null +++ b/app/agent/policy/contracts.py @@ -0,0 +1,247 @@ +"""MoviePilot Agent 宿主策略的内部契约。""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Mapping, MutableMapping, Optional + + +class ToolOrigin(str, Enum): + """工具调用的宿主可信入口。""" + + AGENT_INTERACTIVE = "agent_interactive" + AGENT_API = "agent_api" + OPERATOR_DIRECT = "operator_direct" + BACKGROUND = "background" + SUBAGENT = "subagent" + + +class PrincipalType(str, Enum): + """调用主体类型，用于区分人、管理员集成和内部运行时。""" + + HUMAN = "human" + SYSTEM_ADMIN_INTEGRATION = "system_admin_integration" + SCOPED_AGENT = "scoped_agent" + BACKGROUND = "background" + SUBAGENT = "subagent" + + +class AuthSource(str, Enum): + """主体身份的宿主认证来源。""" + + CHANNEL = "channel" + WEB_SESSION = "web_session" + API_TOKEN = "api_token" + INTERNAL = "internal" + AGENT_TOKEN = "agent_token" + + +class PrincipalRole(str, Enum): + """策略授权使用的角色层级。""" + + USER = "user" + CHANNEL_ADMIN = "channel_admin" + SYSTEM_ADMIN = "system_admin" + SYSTEM_INTERNAL = "system_internal" + + +class ActionEffect(str, Enum): + """工具调用的实际副作用类别。""" + + SAFE_READ = "safe_read" + SENSITIVE_READ = "sensitive_read" + REVERSIBLE_WRITE = "reversible_write" + DESTRUCTIVE_WRITE = "destructive_write" + EXTERNAL_SIDE_EFFECT = "external_side_effect" + ARBITRARY_EXECUTION = "arbitrary_execution" + UNKNOWN = "unknown" + + +class ConfirmationMode(str, Enum): + """动作在完成授权后所需的确认方式。""" + + NONE = "none" + REQUIRED = "required" + UNSUPPORTED = "unsupported" + + +class RecoveryMode(str, Enum): + """动作可提供的执行恢复保证。""" + + NONE = "none" + TRANSACTION = "transaction" + BEFORE_STATE = "before_state" + RECOVERABLE_DELETE = "recoverable_delete" + IDEMPOTENT = "idempotent" + RECONCILE = "reconcile" + MANUAL_ONLY = "manual_only" + + +class ResultSensitivity(str, Enum): + """工具结果进入模型、记忆和日志时的敏感等级。""" + + NORMAL = "normal" + PRIVATE = "private" + SECRET = "secret" + UNKNOWN = "unknown" + + +class MigrationState(str, Enum): + """工具策略从兼容观测迁移到宿主执行的状态。""" + + ENFORCED = "enforced" + LEGACY_SHADOW = "legacy_shadow" + + +class ExecutionOutcome(str, Enum): + """P1-G1 handler 生命周期终态；成功不代表业务授权或副作用已完成。""" + + SUCCEEDED = "succeeded" + FAILED = "failed" + + +@dataclass(frozen=True) +class PolicyPrincipal: + """由可信入口建立、不可由工具参数覆盖的调用主体。""" + + principal_id: str + principal_type: PrincipalType + auth_source: AuthSource + role: PrincipalRole + scopes: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ToolInvocation: + """一次进入宿主策略层的规范化工具调用。""" + + invocation_id: str + tool_name: str + arguments: Mapping[str, Any] + principal: PolicyPrincipal + session_id: str + origin: ToolOrigin + channel: Optional[str] = None + source: Optional[str] = None + + +@dataclass(frozen=True) +class ActionPolicy: + """参数级动作策略及其兼容迁移状态。""" + + effect: ActionEffect + required_role: PrincipalRole + confirmation: ConfirmationMode + recovery: RecoveryMode + result_sensitivity: ResultSensitivity + migration_state: MigrationState + policy_version: str = "p1-g1-v1" + interactive_allowed: bool = True + machine_allowed: bool = True + background_allowed: bool = True + subagent_allowed: bool = True + + +@dataclass(frozen=True) +class PolicyDecision: + """宿主策略层决定；shadow allow 仅表示新策略不拦截，旧门禁仍是授权事实源。""" + + allowed: bool + confirmation_required: bool + shadow: bool + reason_code: str + + +@dataclass(frozen=True) +class PolicyObservation: + """调用开始时生成、供完成或失败回执复用的观测对象。""" + + invocation: ToolInvocation + policy: ActionPolicy + decision: PolicyDecision + input_summary: str + started_at: float + + +@dataclass(frozen=True) +class ExecutionReceipt: + """P1-G1 的非持久化脱敏回执 envelope。""" + + invocation_id: str + tool_name: str + origin: ToolOrigin + decision: PolicyDecision + outcome: ExecutionOutcome + input_summary: str + result_summary: Optional[str] = None + error_summary: Optional[str] = None + duration_ms: int = 0 + + +@dataclass(frozen=True) +class ToolPolicyContext: + """宿主入口上下文；管理员状态引用会随缓存图的每轮执行刷新。""" + + session_id: str + user_id: str + origin: ToolOrigin + principal_type: PrincipalType + auth_source: AuthSource + agent_context: MutableMapping[str, Any] = field(repr=False, compare=False) + channel: Optional[str] = None + source: Optional[str] = None + + @property + def principal(self) -> PolicyPrincipal: + """根据当前宿主上下文生成本次调用主体。""" + if self.principal_type in {PrincipalType.BACKGROUND, PrincipalType.SUBAGENT}: + default_role = PrincipalRole.SYSTEM_INTERNAL + else: + default_role = PrincipalRole.USER + role = ( + PrincipalRole.SYSTEM_ADMIN + if bool(self.agent_context.get("is_admin")) + else default_role + ) + raw_scopes = self.agent_context.get("policy_scopes") or () + scopes = tuple(str(scope) for scope in raw_scopes if scope) + return PolicyPrincipal( + principal_id=str(self.user_id or self.principal_type.value), + principal_type=self.principal_type, + auth_source=self.auth_source, + role=role, + scopes=scopes, + ) + + def for_subagent(self) -> "ToolPolicyContext": + """保留用户与会话归属，并切换为子代理可信来源。""" + return ToolPolicyContext( + session_id=self.session_id, + user_id=self.user_id, + origin=ToolOrigin.SUBAGENT, + principal_type=PrincipalType.SUBAGENT, + auth_source=AuthSource.INTERNAL, + agent_context=self.agent_context, + channel=self.channel, + source=self.source, + ) + + +__all__ = [ + "ActionEffect", + "ActionPolicy", + "AuthSource", + "ConfirmationMode", + "ExecutionOutcome", + "ExecutionReceipt", + "MigrationState", + "PolicyDecision", + "PolicyObservation", + "PolicyPrincipal", + "PrincipalRole", + "PrincipalType", + "RecoveryMode", + "ResultSensitivity", + "ToolInvocation", + "ToolOrigin", + "ToolPolicyContext", +] diff --git a/app/agent/policy/orchestrator.py b/app/agent/policy/orchestrator.py new file mode 100644 index 000000000..2756a18ce --- /dev/null +++ b/app/agent/policy/orchestrator.py @@ -0,0 +1,191 @@ +"""Agent 工具策略观测、脱敏回执与共享执行边界。""" + +import time +import uuid +from collections.abc import Callable +from typing import Any, Mapping, Optional, TypeVar + +from langchain_core.messages import ToolMessage +from pydantic import ValidationError + +from app.agent.policy.contracts import ( + ExecutionOutcome, + ExecutionReceipt, + MigrationState, + PolicyDecision, + PolicyObservation, + ToolInvocation, + ToolPolicyContext, +) +from app.agent.policy.registry import DEFAULT_TOOL_POLICY_REGISTRY, ToolPolicyRegistry +from app.agent.policy.sanitizer import ( + stable_type_name, + summarize_error, + summarize_input, + summarize_result, +) +from app.log import logger + + +_HookResult = TypeVar("_HookResult") + + +def call_policy_hook( + phase: str, + hook: Callable[..., _HookResult], + *args: Any, + **kwargs: Any, +) -> Optional[_HookResult]: + """以 fail-open 方式调用 P1-G1 观测 hook，故障只记录稳定类型。""" + try: + return hook(*args, **kwargs) + except Exception as error: + try: + logger.warning( + f"Agent工具策略观测失败: phase={phase}, " + f"error_type={stable_type_name(error)}" + ) + except Exception: + pass + return None + + +def _normalize_policy_arguments(tool: Any, arguments: Mapping[str, Any]) -> dict[str, Any]: + """为策略生成 Pydantic 规范化副本，不改变真实执行参数。""" + raw_arguments = dict(arguments or {}) + args_schema = getattr(tool, "args_schema", None) + if not args_schema: + return raw_arguments + try: + validated = args_schema.model_validate(raw_arguments) + return validated.model_dump(mode="json") + except (AttributeError, TypeError, ValueError, ValidationError): + # 实际 handler 仍负责既有参数错误语义；策略观测按原始值保守处理。 + return raw_arguments + + +def _result_payload(result: Any) -> Any: + """从 LangChain 工具消息中提取模型可见结果供脱敏摘要使用。""" + if isinstance(result, ToolMessage): + return result.content + return result + + +class AgentToolPolicyOrchestrator: + """让 Agent middleware 与 direct manager 复用同一策略生命周期。""" + + def __init__(self, registry: ToolPolicyRegistry = DEFAULT_TOOL_POLICY_REGISTRY) -> None: + """绑定固定工具迁移注册表。""" + self.registry = registry + + def start( + self, + *, + context: ToolPolicyContext, + tool: Any, + arguments: Mapping[str, Any], + invocation_id: Optional[str] = None, + ) -> PolicyObservation: + """解析调用策略，并创建不影响现有 allow 行为的观测对象。""" + tool_name = str(getattr(tool, "name", None) or "unknown_tool") + normalized_arguments = _normalize_policy_arguments(tool, arguments) + policy = self.registry.resolve( + tool_name=tool_name, + arguments=normalized_arguments, + requires_admin=bool(getattr(tool, "_require_admin", False)), + ) + if policy.migration_state is MigrationState.LEGACY_SHADOW: + decision = PolicyDecision( + allowed=True, + confirmation_required=False, + shadow=True, + reason_code="legacy_shadow_allow", + ) + else: + decision = PolicyDecision( + allowed=True, + confirmation_required=False, + shadow=False, + reason_code="safe_read_allow", + ) + invocation = ToolInvocation( + invocation_id=invocation_id or uuid.uuid4().hex, + tool_name=tool_name, + arguments=normalized_arguments, + principal=context.principal, + session_id=context.session_id, + origin=context.origin, + channel=context.channel, + source=context.source, + ) + input_summary = summarize_input(normalized_arguments) + observation = PolicyObservation( + invocation=invocation, + policy=policy, + decision=decision, + input_summary=input_summary, + started_at=time.monotonic(), + ) + logger.debug( + f"Agent工具策略: tool={tool_name}, origin={context.origin.value}, " + f"decision={decision.reason_code}, input={input_summary}" + ) + return observation + + @staticmethod + def finish(observation: PolicyObservation, result: Any) -> ExecutionReceipt: + """生成成功回执 envelope，并只记录脱敏结果摘要。""" + result_summary = summarize_result(_result_payload(result)) + receipt = ExecutionReceipt( + invocation_id=observation.invocation.invocation_id, + tool_name=observation.invocation.tool_name, + origin=observation.invocation.origin, + decision=observation.decision, + outcome=ExecutionOutcome.SUCCEEDED, + input_summary=observation.input_summary, + result_summary=result_summary, + duration_ms=max( + 0, + int((time.monotonic() - observation.started_at) * 1000), + ), + ) + logger.info( + f"Agent工具执行完成: tool={receipt.tool_name}, " + f"origin={receipt.origin.value}, shadow={receipt.decision.shadow}, " + f"duration_ms={receipt.duration_ms}, result={result_summary}" + ) + return receipt + + @staticmethod + def fail(observation: PolicyObservation, error: BaseException) -> ExecutionReceipt: + """生成失败回执 envelope，不把异常中的凭据写入日志。""" + error_summary = summarize_error(error) + receipt = ExecutionReceipt( + invocation_id=observation.invocation.invocation_id, + tool_name=observation.invocation.tool_name, + origin=observation.invocation.origin, + decision=observation.decision, + outcome=ExecutionOutcome.FAILED, + input_summary=observation.input_summary, + error_summary=error_summary, + duration_ms=max( + 0, + int((time.monotonic() - observation.started_at) * 1000), + ), + ) + logger.error( + f"Agent工具执行失败: tool={receipt.tool_name}, " + f"origin={receipt.origin.value}, shadow={receipt.decision.shadow}, " + f"duration_ms={receipt.duration_ms}, error={error_summary}" + ) + return receipt + + +DEFAULT_TOOL_POLICY_ORCHESTRATOR = AgentToolPolicyOrchestrator() + + +__all__ = [ + "AgentToolPolicyOrchestrator", + "DEFAULT_TOOL_POLICY_ORCHESTRATOR", + "call_policy_hook", +] diff --git a/app/agent/policy/registry.py b/app/agent/policy/registry.py new file mode 100644 index 000000000..a43d8396e --- /dev/null +++ b/app/agent/policy/registry.py @@ -0,0 +1,181 @@ +"""固定工具迁移注册表与参数级策略解析。""" + +from typing import Any, Mapping + +from app.agent.policy.contracts import ( + ActionEffect, + ActionPolicy, + ConfirmationMode, + MigrationState, + PrincipalRole, + RecoveryMode, + ResultSensitivity, +) + + +# 这些读取已具备清晰的无副作用语义，用于证明新宿主边界不会改变正常结果。 +SAFE_READ_TOOL_NAMES = frozenset( + { + "list_slash_commands", + "query_installed_plugins", + "query_personas", + "query_schedulers", + "query_workflows", + } +) + + +# 其余固定工具先显式处于兼容观测状态，待领域叶子 Goal 逐个迁移。 +LEGACY_SHADOW_TOOL_NAMES = frozenset( + { + "add_custom_filter_rule", + "add_download_tasks", + "add_rule_group", + "add_subscribe", + "ask_user_choice", + "browse_webpage", + "create_agent_task", + "delete_agent_task", + "delete_custom_filter_rule", + "delete_download_history", + "delete_download_tasks", + "delete_rule_group", + "delete_subscribe", + "delete_transfer_history", + "edit_file", + "execute_command", + "get_recommendations", + "get_search_results", + "install_plugin", + "list_directory", + "query_agent_tasks", + "query_builtin_filter_rules", + "query_custom_filter_rules", + "query_custom_identifiers", + "query_directory_settings", + "query_doctor_report", + "query_download_tasks", + "query_downloaders", + "query_episode_schedule", + "query_library_exists", + "query_library_latest", + "query_market_plugins", + "query_media_detail", + "query_plugin_capabilities", + "query_plugin_config", + "query_plugin_data", + "query_popular_subscribes", + "query_rule_groups", + "query_site_userdata", + "query_sites", + "query_subscribe_history", + "query_subscribe_shares", + "query_subscribes", + "query_system_settings", + "query_transfer_history", + "read_file", + "recognize_captcha", + "recognize_media", + "reload_plugin", + "run_agent_task", + "run_scheduler", + "run_slash_command", + "run_workflow", + "scrape_metadata", + "search_media", + "search_person", + "search_person_credits", + "search_subscribe", + "search_torrents", + "search_web", + "send_local_file", + "send_message", + "send_voice_message", + "switch_persona", + "test_site", + "transfer_file", + "uninstall_plugin", + "update_agent_task", + "update_custom_filter_rule", + "update_custom_identifiers", + "update_download_tasks", + "update_persona_definition", + "update_plugin_config", + "update_rule_group", + "update_site", + "update_site_cookie", + "update_subscribe", + "update_system_settings", + "write_file", + } +) + + +class ToolPolicyRegistry: + """解析固定和动态工具的 P1-G1 迁移策略。""" + + def __init__( + self, + *, + safe_read_tool_names: frozenset[str] = SAFE_READ_TOOL_NAMES, + legacy_shadow_tool_names: frozenset[str] = LEGACY_SHADOW_TOOL_NAMES, + ) -> None: + """建立互斥的固定工具迁移表。""" + overlap = safe_read_tool_names & legacy_shadow_tool_names + if overlap: + raise ValueError(f"工具策略迁移表存在重复项: {sorted(overlap)}") + self._safe_read_tool_names = safe_read_tool_names + self._legacy_shadow_tool_names = legacy_shadow_tool_names + + @property + def builtin_tool_names(self) -> set[str]: + """返回注册表覆盖的全部固定工具名。""" + return set(self._safe_read_tool_names | self._legacy_shadow_tool_names) + + def resolve( + self, + *, + tool_name: str, + arguments: Mapping[str, Any], + requires_admin: bool, + ) -> ActionPolicy: + """根据工具名和宿主权限声明解析当前迁移策略。""" + del arguments # 参数级迁移由后续领域 Goal 逐项加入。 + required_role = ( + PrincipalRole.SYSTEM_ADMIN if requires_admin else PrincipalRole.USER + ) + if tool_name in self._safe_read_tool_names: + return ActionPolicy( + effect=ActionEffect.SAFE_READ, + required_role=required_role, + confirmation=ConfirmationMode.NONE, + recovery=RecoveryMode.NONE, + result_sensitivity=ResultSensitivity.NORMAL, + # 角色门禁仍可能异步识别渠道管理员；G1 不复制旧授权事实源。 + migration_state=( + MigrationState.LEGACY_SHADOW + if requires_admin + else MigrationState.ENFORCED + ), + ) + + # 固定未迁移工具和动态工具都保持现有执行能力，但不得被视为安全读取。 + return ActionPolicy( + effect=ActionEffect.UNKNOWN, + required_role=required_role, + confirmation=ConfirmationMode.REQUIRED, + recovery=RecoveryMode.MANUAL_ONLY, + result_sensitivity=ResultSensitivity.UNKNOWN, + migration_state=MigrationState.LEGACY_SHADOW, + ) + + +DEFAULT_TOOL_POLICY_REGISTRY = ToolPolicyRegistry() + + +__all__ = [ + "DEFAULT_TOOL_POLICY_REGISTRY", + "LEGACY_SHADOW_TOOL_NAMES", + "SAFE_READ_TOOL_NAMES", + "ToolPolicyRegistry", +] diff --git a/app/agent/policy/sanitizer.py b/app/agent/policy/sanitizer.py new file mode 100644 index 000000000..465441500 --- /dev/null +++ b/app/agent/policy/sanitizer.py @@ -0,0 +1,1056 @@ +"""宿主工具输入、结果与异常的递归脱敏摘要。""" + +import base64 +import binascii +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import fields +from typing import Any + +from pydantic import AliasChoices, AliasPath, BaseModel, ValidationError + +from app.agent.policy.secret_fields import is_secret_setting_key + + +REDACTED_VALUE = "***" +_MAX_DEPTH = 8 +_MAX_ITEMS = 100 +_MAX_KEY_SCAN_CHARS = 256 +_MAX_TEXT_CHARS = 16 * 1024 +_MAX_WORK_ITEMS = 1000 +_SECRET_KEYS = { + "access_token", + "api_key", + "api_token", + "authorization", + "client_secret", + "cookie", + "passkey", + "passwd", + "password", + "private_key", + "pwd", + "refresh_token", + "secret", + "secret_access_key", + "token", +} +_ACRONYM_BOUNDARY_PATTERN = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") +_CAMEL_CASE_BOUNDARY_PATTERN = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_COMPACT_SECRET_SUFFIXES = ( + "apikey", + "authorization", + "cookie", + "passkey", + "passwd", + "password", + "privatekey", + "pwd", + "secret", + "secretaccesskey", + "secretkey", + "token", +) +_SECRET_CONTAINER_SUFFIXES = ( + "auth", + "authentication", + "credential", + "credentials", +) +_SECRET_CONTAINER_ENDINGS = tuple( + f"_{suffix}" for suffix in _SECRET_CONTAINER_SUFFIXES +) +_COMPACT_SECRET_CONTAINERS = frozenset(("oauth", "oauth2")) +_SECRET_IDENTITY_FIELDS = frozenset( + { + "config_key", + "field_name", + "key", + "name", + "property_name", + "setting_key", + } +) +_SECRET_IDENTITY_VALUE_FIELDS = frozenset( + { + "current", + "current_value", + "default", + "default_value", + "new_value", + "old_value", + "previous", + "previous_value", + "saved_value", + "value", + "value_preview", + } +) +_BEARER_PATTERN = re.compile(r"(?i)(\bbearer\s+)[^\s,;]+") +_BASIC_AUTH_PATTERN = re.compile( + r"(?i)(\bbasic\s+)([A-Za-z0-9+/]+={0,2})(?![A-Za-z0-9+/=])" +) +_SENSITIVE_HEADER_PATTERN = re.compile( + r"(?i)(\b(?:authorization|proxy-authorization|cookie|set-cookie|" + r"x-api-key|api[_-]?key|api[_-]?token)\s*[:=]\s*)[^\r\n]+" +) +_OPENAI_KEY_PATTERN = re.compile(r"\bsk-[A-Za-z0-9_-]{8,}\b") +_PRIVATE_KEY_PATTERN = re.compile( + r"-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", + re.DOTALL, +) +_PRIVATE_KEY_OPEN_PATTERN = re.compile( + r"-----BEGIN [^-]*PRIVATE KEY-----.*", + re.DOTALL, +) +_CONTAINER_PAIRS = {"[": "]", "{": "}", "(": ")"} +_CONTAINER_CLOSERS = frozenset(_CONTAINER_PAIRS.values()) + + +def _normalize_key(key: Any) -> str: + """把结构化字段名转换为可比较的 snake_case。""" + if type(key) is not str: + return "" + text = key + # 凭据判定只依赖完整短字段或字段尾部，固定尾窗可保留后缀语义并限制同步扫描成本。 + if len(text) > _MAX_KEY_SCAN_CHARS: + text = text[-_MAX_KEY_SCAN_CHARS:] + text = _ACRONYM_BOUNDARY_PATTERN.sub("_", text.strip()) + text = _CAMEL_CASE_BOUNDARY_PATTERN.sub("_", text) + return re.sub(r"[^a-z0-9]+", "_", text.strip().lower()).strip("_") + + +def _is_secret_key(key: Any) -> bool: + """判断字段是否承载凭据原值，避免误伤 token_count 等统计字段。""" + normalized = _normalize_key(key) + if normalized in _SECRET_KEYS: + return True + compact = normalized.replace("_", "") + if compact in _COMPACT_SECRET_CONTAINERS: + return True + if normalized in _SECRET_CONTAINER_SUFFIXES or normalized.endswith( + _SECRET_CONTAINER_ENDINGS + ): + return True + return compact.endswith(_COMPACT_SECRET_SUFFIXES) + + +def _mapping_has_secret_identity( + entries: list[tuple[str, str, bool, Any]], +) -> bool: + """判断同一结构是否声明了凭据设置身份，不把语义传播到嵌套对象。""" + return any( + _normalize_key(key_text) in _SECRET_IDENTITY_FIELDS + and type(item) is str + and is_secret_setting_key(item) + for key_text, _output_key, _secret_key, item in entries + ) + + +def _advance_quote_context( + value: str, + start: int, + end: int, + active_quote: str, +) -> str: + """线性跟踪文本片段内尚未闭合的单双引号。""" + cursor = start + while cursor < end: + char = value[cursor] + if char == "\\": + cursor += 2 + continue + if active_quote: + if char == active_quote: + active_quote = "" + elif char in ("\"", "'"): + active_quote = char + cursor += 1 + return active_quote + + +def _is_assignment_key_start(char: str) -> bool: + """判断字符是否可作为 ASCII 赋值字段名的起点。""" + return char == "_" or char.isascii() and char.isalnum() + + +def _is_assignment_key_char(char: str) -> bool: + """判断字符是否属于常见日志或配置字段名。""" + return _is_assignment_key_start(char) or char in (".", "-") + + +def _is_quoted_assignment_key_char(char: str) -> bool: + """quoted key 额外允许横向空白，普通文本字段名仍保持窄边界。""" + return _is_assignment_key_char(char) or char in (" ", "\t") + + +def _quote_wrapper_at(value: str, start: int) -> tuple[str, int, int]: + """识别引号包装，并返回引号、反斜杠数量和已扫描位置。""" + quote_at = start + while quote_at < len(value) and value[quote_at] == "\\": + quote_at += 1 + if quote_at >= len(value) or value[quote_at] not in ("\"", "'"): + return "", quote_at - start, quote_at + return value[quote_at], quote_at - start, quote_at + 1 + + +def _quote_wrapper_end( + value: str, + content_start: int, + quote: str, + slash_count: int, +) -> int: + """查找同层引号结束位置，更深层转义引号视为值内容。""" + cursor = content_start + while cursor < len(value): + if value[cursor] == "\\": + slash_start = cursor + while cursor < len(value) and value[cursor] == "\\": + cursor += 1 + if ( + cursor < len(value) + and value[cursor] == quote + and cursor - slash_start >= slash_count + and (cursor - slash_start - slash_count) + % (2 * (slash_count + 1)) + == 0 + ): + return cursor + 1 + if cursor < len(value): + cursor += 1 + continue + if slash_count == 0 and value[cursor] == quote: + return cursor + 1 + cursor += 1 + return len(value) + + +def _unquoted_assignment_value_end( + value: str, + value_start: int, + delimiters: str, + outer_quote: str, +) -> int: + """线性扫描未引号值，仅在容器外识别字段分隔符。""" + stack = [] + cursor = value_start + while cursor < len(value): + char = value[cursor] + if not stack and outer_quote and char == outer_quote: + return cursor + + if char == "\\": + slash_end = cursor + while slash_end < len(value) and value[slash_end] == "\\": + slash_end += 1 + if slash_end >= len(value): + return len(value) + + slash_count = slash_end - cursor + next_char = value[slash_end] + if next_char in ("\"", "'"): + cursor = _quote_wrapper_end( + value, + slash_end + 1, + next_char, + slash_count, + ) + continue + if next_char in _CONTAINER_PAIRS: + stack.append(_CONTAINER_PAIRS[next_char]) + cursor = slash_end + 1 + continue + cursor = slash_end + 1 if slash_count % 2 else slash_end + continue + if char in ("\"", "'"): + cursor = _quote_wrapper_end( + value, + cursor + 1, + char, + 0, + ) + continue + if char in _CONTAINER_PAIRS: + stack.append(_CONTAINER_PAIRS[char]) + elif char in _CONTAINER_CLOSERS: + if not stack: + return cursor if char in delimiters else len(value) + if char != stack[-1]: + return len(value) + stack.pop() + elif not stack and char in delimiters: + return cursor + cursor += 1 + + # 未闭合结构无法可靠区分后续字段，按敏感尾部处理。 + return len(value) + + +def _find_assignment_header( + value: str, + search_from: int, +) -> tuple[int, str, int] | None: + """单向查找下一个赋值头，返回起点、字段名和值起点。""" + cursor = search_from + while cursor < len(value): + match_start = cursor + if match_start > 0 and value[match_start - 1].isascii() and value[ + match_start - 1 + ].isalnum(): + cursor += 1 + continue + + quote, slash_count, key_start = _quote_wrapper_at(value, cursor) + if not quote: + if slash_count: + cursor = key_start + continue + key_start = cursor + else: + while key_start < len(value) and value[key_start] in (" ", "\t"): + key_start += 1 + if key_start >= len(value) or not _is_assignment_key_start( + value[key_start] + ): + cursor += 1 + continue + + key_end = key_start + 1 + key_char_predicate = ( + _is_quoted_assignment_key_char if quote else _is_assignment_key_char + ) + while key_end < len(value) and key_char_predicate(value[key_end]): + key_end += 1 + + header_end = key_end + if quote: + closing_quote_at = header_end + while ( + closing_quote_at < len(value) + and value[closing_quote_at] == "\\" + ): + closing_quote_at += 1 + if ( + closing_quote_at >= len(value) + or value[closing_quote_at] != quote + or closing_quote_at - header_end != slash_count + ): + # 引号不是字段名的一部分时，从引号内首字符重试一次。 + cursor = key_start + continue + header_end = closing_quote_at + 1 + while header_end < len(value) and value[header_end].isspace(): + header_end += 1 + if header_end >= len(value) or value[header_end] not in (":", "="): + cursor = key_end + continue + header_end += 1 + while header_end < len(value) and value[header_end].isspace(): + header_end += 1 + return match_start, value[key_start:key_end], header_end + return None + + +def _assignment_value_end( + value: str, + match_start: int, + value_start: int, + active_quote: str, +) -> int: + """返回凭据值的安全消费边界，并保留外层文本结构。""" + if value_start >= len(value): + return value_start + + quote, slash_count, content_start = _quote_wrapper_at(value, value_start) + if quote and slash_count == 0 and quote == active_quote: + return value_start + if quote: + return _quote_wrapper_end( + value, + content_start, + quote, + slash_count, + ) + + delimiters = ",;}\r\n" + if match_start > 0 and value[match_start - 1] in "?&#": + delimiters += "&#" + + return _unquoted_assignment_value_end( + value, + value_start, + delimiters, + active_quote, + ) + + +def _sanitize_assignments( + value: str, + *, + redact_identity_values: bool = False, +) -> str: + """按结构化字段语义清理文本赋值与身份不明的通用值。""" + fragments = [] + emitted_until = 0 + search_from = 0 + context_from = 0 + active_quote = "" + while header := _find_assignment_header(value, search_from): + match_start, key, value_start = header + active_quote = _advance_quote_context( + value, + context_from, + match_start, + active_quote, + ) + secret_value = _is_secret_key(key) or ( + redact_identity_values + and _normalize_key(key) in _SECRET_IDENTITY_VALUE_FIELDS + ) + if secret_value: + fragments.append(value[emitted_until:match_start]) + fragments.append(value[match_start:value_start]) + fragments.append(REDACTED_VALUE) + emitted_until = _assignment_value_end( + value, + match_start, + value_start, + active_quote, + ) + search_from = emitted_until + active_quote = _advance_quote_context( + value, + match_start, + emitted_until, + active_quote, + ) + context_from = emitted_until + continue + + # 只消费字段头，值内部的 URL query 或嵌套诊断仍会继续进入扫描。 + search_from = value_start + active_quote = _advance_quote_context( + value, + match_start, + search_from, + active_quote, + ) + context_from = search_from + + if not fragments: + return value + fragments.append(value[emitted_until:]) + return "".join(fragments) + + +def _is_uri_scheme_char(char: str) -> bool: + """判断字符是否属于 RFC 3986 scheme 的 ASCII 字符集。""" + return char.isascii() and char.isalnum() or char in ("+", "-", ".") + + +def _uri_authority_start(value: str, scheme_end: int) -> int | None: + """返回字面量或 slash-escaped 双斜杠后的 authority 起点。""" + if scheme_end >= len(value) or value[scheme_end] != ":": + return None + cursor = scheme_end + 1 + while cursor < len(value) and value[cursor] == "\\": + cursor += 1 + if cursor >= len(value) or value[cursor] != "/": + return None + cursor += 1 + while cursor < len(value) and value[cursor] == "\\": + cursor += 1 + if cursor >= len(value) or value[cursor] != "/": + return None + return cursor + 1 + + +def _starts_uri_scheme(value: str, start: int) -> bool: + """判断指定位置是否以完整的 URI scheme 与 authority 开始。""" + if ( + start >= len(value) + or not value[start].isascii() + or not value[start].isalpha() + ): + return False + scheme_end = start + 1 + while scheme_end < len(value) and _is_uri_scheme_char(value[scheme_end]): + scheme_end += 1 + return _uri_authority_start(value, scheme_end) is not None + + +def _find_uri_authority(value: str, search_from: int) -> tuple[int, int] | None: + """单向查找下一个 URI scheme 及其 authority 起点。""" + while (scheme_end := value.find(":", search_from)) >= 0: + scheme_start = scheme_end + while scheme_start > 0 and _is_uri_scheme_char(value[scheme_start - 1]): + scheme_start -= 1 + authority_start = _uri_authority_start(value, scheme_end) + if ( + scheme_start < scheme_end + and value[scheme_start].isascii() + and value[scheme_start].isalpha() + and authority_start is not None + ): + return scheme_end, authority_start + search_from = scheme_end + 1 + return None + + +def _sanitize_uri_userinfo(value: str, *, truncated: bool = False) -> str: + """清理 URI authority 中 `@` 前的 userinfo 凭据。""" + fragments = [] + emitted_until = 0 + search_from = 0 + while authority := _find_uri_authority(value, search_from): + _, authority_start = authority + authority_end = authority_start + seen_userinfo = False + while authority_end < len(value): + char = value[authority_end] + if char.isspace() or char in ("/", "?", "#"): + break + if char == "@": + seen_userinfo = True + elif char in (",", ";", "|") and ( + seen_userinfo or _starts_uri_scheme(value, authority_end + 1) + ): + break + authority_end += 1 + if truncated and authority_end == len(value): + fragments.append(value[emitted_until:authority_start]) + fragments.append(REDACTED_VALUE) + emitted_until = authority_end + search_from = authority_end + continue + + userinfo_end = value.rfind("@", authority_start, authority_end) + if userinfo_end < 0: + search_from = max(authority_end, authority_start) + continue + + fragments.append(value[emitted_until:authority_start]) + fragments.append(f"{REDACTED_VALUE}@") + emitted_until = userinfo_end + 1 + search_from = max(authority_end, emitted_until) + + if not fragments: + return value + fragments.append(value[emitted_until:]) + return "".join(fragments) + + +def _redact_basic_auth( + match: re.Match[str], + *, + truncated: bool = False, +) -> str: + """仅遮蔽可解码为 `user:password` 的 Basic token。""" + if truncated and match.end(2) == len(match.string): + return f"{match.group(1)}{REDACTED_VALUE}" + token = match.group(2) + if len(token) % 4 == 1: + return match.group(0) + padded_token = token + "=" * (-len(token) % 4) + try: + decoded = base64.b64decode(padded_token, validate=True) + except (binascii.Error, ValueError): + return match.group(0) + if b":" not in decoded: + return match.group(0) + return f"{match.group(1)}{REDACTED_VALUE}" + + +def _sanitize_text(value: str, *, truncated_input: bool = False) -> str: + """清理非结构化文本中的常见凭据表达。""" + truncated = truncated_input or len(value) > _MAX_TEXT_CHARS + bounded_value = value[:_MAX_TEXT_CHARS] + truncated_json = truncated and bounded_value.lstrip().startswith(("{", "[")) + sanitized = _PRIVATE_KEY_PATTERN.sub(REDACTED_VALUE, bounded_value) + sanitized = _PRIVATE_KEY_OPEN_PATTERN.sub(REDACTED_VALUE, sanitized) + sanitized = _SENSITIVE_HEADER_PATTERN.sub(r"\1***", sanitized) + sanitized = _BEARER_PATTERN.sub(r"\1***", sanitized) + sanitized = _BASIC_AUTH_PATTERN.sub( + lambda match: _redact_basic_auth(match, truncated=truncated), + sanitized, + ) + sanitized = _sanitize_uri_userinfo(sanitized, truncated=truncated) + sanitized = _sanitize_assignments( + sanitized, + redact_identity_values=truncated_json, + ) + sanitized = _OPENAI_KEY_PATTERN.sub(REDACTED_VALUE, sanitized) + return f"{sanitized}" if truncated else sanitized + + +def stable_type_name(value: Any) -> str: + """绕过自定义 metaclass 协议，返回仅用于诊断的稳定类型名。""" + try: + name = type.__getattribute__(type(value), "__name__") + except Exception: + return "unknown" + return name if type(name) is str and name else "unknown" + + +def _unavailable(value: Any) -> str: + """在对象协议异常时返回不含对象文本的稳定占位。""" + return f"" + + +def _named_tuple_fields(value: Any) -> tuple[str, ...] | None: + """返回合法命名元组的字段契约，普通 tuple 仍按序列处理。""" + value_type = type(value) + if not issubclass(value_type, tuple): + return None + try: + field_names = type.__getattribute__(value_type, "_fields") + except Exception: + return None + if type(field_names) is not tuple or tuple.__len__(field_names) != tuple.__len__( + value + ): + return None + bounded_names = field_names[:_MAX_ITEMS] + if not all(type(field_name) is str for field_name in bounded_names): + return None + return field_names + + +def _bounded_mapping_key_text(key: Any) -> tuple[str, bool] | None: + """为受信内建 key 生成有界快照，未知对象不执行字符串协议。""" + if type(key) is str: + return key, False + if type(key) in (bool, int, float, type(None)): + return str(key), False + if type(key) is not tuple or len(key) > _MAX_ITEMS: + return None + + fragments = [] + remaining = _MAX_TEXT_CHARS - 2 + truncated = False + for index, part in enumerate(key): + if type(part) is not str: + return None + separator = ", " if index else "" + if remaining <= len(separator): + truncated = True + break + fragments.append(separator) + remaining -= len(separator) + if len(part) > remaining: + fragments.append(part[:remaining]) + truncated = True + remaining = 0 + break + fragments.append(part) + remaining -= len(part) + text = f"({''.join(fragments)}" + return (text, True) if truncated else (f"{text})", False) + + +def _is_dataclass_type(value_type: type) -> bool: + """绕过自定义 metaclass 协议检查 dataclass 类型标记。""" + try: + mro = type.__getattribute__(value_type, "__mro__") + return any( + "__dataclass_fields__" + in type.__getattribute__(candidate, "__dict__") + for candidate in mro + ) + except Exception: + return False + + +def _pydantic_alias_names( + alias: Any, + *, + _budget: list[int] | None = None, +) -> tuple[str, ...] | None: + """提取有界的 Pydantic 外部字段名；未知结构要求调用方保守处理。""" + if alias is None: + return () + budget = _budget if _budget is not None else [_MAX_ITEMS] + if type(alias) is AliasChoices: + choices = alias.choices + if type(choices) is not list: + return None + else: + choices = (alias,) + if len(choices) > budget[0]: + return None + + names = [] + for choice in choices: + if type(choice) is str: + if budget[0] <= 0: + return None + budget[0] -= 1 + names.append(choice) + elif type(choice) is AliasPath: + path = choice.path + if type(path) is not list or len(path) > budget[0]: + return None + budget[0] -= len(path) + for part in path: + if type(part) is str: + names.append(part) + elif type(part) is not int: + return None + else: + return None + return tuple(names) + + +def _pydantic_field_is_secret(field_name: str, field_info: Any) -> bool: + """按 Python 字段名及 Pydantic 的输入输出别名共同判定凭据字段。""" + if _is_secret_key(field_name): + return True + if field_info is None: + return True + try: + aliases = ( + field_info.alias, + field_info.validation_alias, + field_info.serialization_alias, + ) + except Exception: + return True + alias_budget = [_MAX_ITEMS] + for alias in aliases: + alias_names = _pydantic_alias_names(alias, _budget=alias_budget) + if alias_names is None or any( + _is_secret_key(alias_name) for alias_name in alias_names + ): + return True + return False + + +def _validation_error_details(error: ValidationError) -> dict[str, int | str]: + """提取不含任何动态错误文本的校验计数。""" + try: + error_count = error.error_count() + except Exception: + return {"error_count": "unavailable"} + return {"error_count": error_count} + + +def _consume_work_item(budget: list[int]) -> bool: + """从全调用预算消费一个递归节点或容器输出项。""" + if budget[0] <= 0: + return False + budget[0] -= 1 + return True + + +def sanitize_for_host( + value: Any, + *, + _depth: int = 0, + _seen: set[int] | None = None, + _budget: list[int] | None = None, +) -> Any: + """递归清理宿主日志/回执使用的数据，不修改调用方原对象。""" + try: + budget = _budget if _budget is not None else [_MAX_WORK_ITEMS] + if not _consume_work_item(budget): + return "" + if _depth >= _MAX_DEPTH: + return "" + value_type = type(value) + if value is None or value_type in (bool, int, float): + return value + if value_type is str: + truncated = len(value) > _MAX_TEXT_CHARS + if not truncated and value.strip().startswith(("{", "[")): + try: + parsed = json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + pass + else: + sanitized_json = sanitize_for_host( + parsed, + _depth=_depth + 1, + _seen=_seen, + _budget=budget, + ) + return json.dumps(sanitized_json, ensure_ascii=False) + return _sanitize_text(value) + if value_type in (bytes, bytearray, memoryview): + return f"" + if issubclass(value_type, ValidationError): + return _validation_error_details(value) + + seen = _seen if _seen is not None else set() + is_exception = issubclass(value_type, BaseException) + is_model = issubclass(value_type, BaseModel) + is_mapping = issubclass(value_type, Mapping) + is_set = issubclass(value_type, (set, frozenset)) + is_sequence = issubclass(value_type, Sequence) and not issubclass( + value_type, + (str, bytes, bytearray), + ) + is_dataclass_value = _is_dataclass_type(value_type) + track_identity = any( + ( + is_exception, + is_model, + is_mapping, + is_set, + is_sequence, + is_dataclass_value, + ) + ) + value_id = id(value) + if track_identity: + if value_id in seen: + return "" + seen.add(value_id) + + try: + if is_exception: + try: + error_args = BaseException.__getattribute__(value, "args") + except Exception: + return _unavailable(value) + return sanitize_for_host( + error_args, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + + if is_model: + sanitized = {} + model_fields = getattr(value_type, "model_fields", {}) + for index, field_name in enumerate(model_fields): + if index >= _MAX_ITEMS: + sanitized[""] = "more fields" + break + if not _consume_work_item(budget): + sanitized[""] = "more fields" + break + try: + item = getattr(value, field_name) + except Exception: + item = _unavailable(value) + sanitized[field_name] = ( + REDACTED_VALUE + if _pydantic_field_is_secret( + field_name, + model_fields.get(field_name), + ) + else sanitize_for_host( + item, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + ) + return sanitized + + if is_dataclass_value: + sanitized = {} + pydantic_fields = getattr( + value_type, + "__pydantic_fields__", + None, + ) + for index, field_info in enumerate(fields(value_type)): + if index >= _MAX_ITEMS: + sanitized[""] = "more fields" + break + if not _consume_work_item(budget): + sanitized[""] = "more fields" + break + try: + item = getattr(value, field_info.name) + except Exception: + item = _unavailable(value) + if pydantic_fields is None: + secret_field = _is_secret_key(field_info.name) + elif isinstance(pydantic_fields, Mapping): + secret_field = _pydantic_field_is_secret( + field_info.name, + pydantic_fields.get(field_info.name), + ) + else: + secret_field = True + sanitized[field_info.name] = ( + REDACTED_VALUE + if secret_field + else sanitize_for_host( + item, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + ) + return sanitized + + named_tuple_fields = _named_tuple_fields(value) + if named_tuple_fields is not None: + sanitized = {} + for index, field_name in enumerate( + named_tuple_fields[:_MAX_ITEMS] + ): + if not _consume_work_item(budget): + sanitized[""] = "more fields" + break + output_key = _sanitize_text(field_name) + item = tuple.__getitem__(value, index) + sanitized[output_key] = ( + REDACTED_VALUE + if _is_secret_key(field_name) + else sanitize_for_host( + item, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + ) + if tuple.__len__(named_tuple_fields) > _MAX_ITEMS: + sanitized[""] = "more fields" + return sanitized + + if is_mapping: + sanitized = {} + items = iter(value.items()) + entries: list[tuple[str, str, bool, Any]] = [] + identity_scan_complete = True + for index in range(_MAX_ITEMS + 1): + if not _consume_work_item(budget): + identity_scan_complete = False + entries.append( + ("", "", False, "more items") + ) + break + try: + key, item = next(items) + except StopIteration: + break + if index >= _MAX_ITEMS: + identity_scan_complete = False + entries.append( + ("", "", False, "more items") + ) + break + try: + key_snapshot = _bounded_mapping_key_text(key) + if key_snapshot is None: + raise TypeError("unsupported mapping key") + key_text, key_truncated = key_snapshot + output_key = _sanitize_text( + key_text, + truncated_input=key_truncated, + ) + secret_key = _is_secret_key(key_text) + except Exception: + identity_scan_complete = False + key_text = "" + output_key = f"" + secret_key = True + entries.append((key_text, output_key, secret_key, item)) + + secret_identity = _mapping_has_secret_identity(entries) + for key_text, output_key, secret_key, item in entries: + identity_value_field = ( + _normalize_key(key_text) + in _SECRET_IDENTITY_VALUE_FIELDS + ) + contextual_secret = ( + identity_value_field + and (secret_identity or not identity_scan_complete) + ) + sanitized[output_key] = ( + REDACTED_VALUE + if secret_key or contextual_secret + else sanitize_for_host( + item, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + ) + return sanitized + + if is_set or is_sequence: + items = iter(value) + sanitized_items = [] + for index in range(_MAX_ITEMS + 1): + if not _consume_work_item(budget): + sanitized_items.append("") + break + try: + item = next(items) + except StopIteration: + break + if index >= _MAX_ITEMS: + sanitized_items.append("") + break + sanitized_items.append( + sanitize_for_host( + item, + _depth=_depth + 1, + _seen=seen, + _budget=budget, + ) + ) + return sanitized_items + + return _unavailable(value) + finally: + if track_identity: + seen.discard(value_id) + except Exception: + return _unavailable(value) + + +def _bounded_summary(value: Any, *, max_chars: int) -> str: + """把脱敏结构转换为不超过指定长度的稳定文本。""" + sanitized = sanitize_for_host(value) + if isinstance(sanitized, str): + text = sanitized + else: + # 保留调用方字段顺序，避免排序后由大型低价值字段挤掉前部诊断信息。 + try: + text = json.dumps(sanitized, ensure_ascii=False) + except Exception: + text = _unavailable(sanitized) + if max_chars <= 0: + return "" + if len(text) <= max_chars: + return text + suffix = "..." + if max_chars <= len(suffix): + return suffix[:max_chars] + return f"{text[:max_chars - len(suffix)]}{suffix}" + + +def summarize_input(value: Any, *, max_chars: int = 500) -> str: + """生成工具输入的 secret-safe 有界摘要。""" + return _bounded_summary(value, max_chars=max_chars) + + +def summarize_result(value: Any, *, max_chars: int = 500) -> str: + """生成工具结果的 secret-safe 有界摘要。""" + return _bounded_summary(value, max_chars=max_chars) + + +def summarize_error(error: BaseException, *, max_chars: int = 500) -> str: + """生成不回显异常凭据的类型化摘要。""" + error_text = sanitize_for_host(error) + return _bounded_summary( + f"{stable_type_name(error)}: {error_text}", + max_chars=max_chars, + ) + + +__all__ = [ + "REDACTED_VALUE", + "sanitize_for_host", + "stable_type_name", + "summarize_error", + "summarize_input", + "summarize_result", +] diff --git a/app/agent/policy/secret_fields.py b/app/agent/policy/secret_fields.py new file mode 100644 index 000000000..97762356b --- /dev/null +++ b/app/agent/policy/secret_fields.py @@ -0,0 +1,69 @@ +"""Agent 设置工具与宿主回执共用的敏感字段身份判定。""" + +import re +from typing import Any + + +_MAX_FIELD_NAME_CHARS = 256 +_ACRONYM_BOUNDARY_PATTERN = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") +_CAMEL_CASE_BOUNDARY_PATTERN = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_SECRET_FIELD_NAMES = frozenset( + { + "access_token", + "api_key", + "apikey", + "api_token", + "auth_header", + "authorization", + "client_secret", + "cookie", + "passkey", + "passwd", + "password", + "private_key", + "pwd", + "refresh_token", + "secret", + "secret_access_key", + "secret_key", + "token", + } +) +_SECRET_FIELD_ENDINGS = tuple(f"_{name}" for name in _SECRET_FIELD_NAMES) +_SECRET_SETTING_NAMES = frozenset( + { + # CookieCloud 的用户 key 没有类型后缀，但与密码共同构成端到端加密凭据。 + "cookiecloud_key", + } +) +_SECRET_SETTING_ENDINGS = ( + "_encrypt_key", +) + + +def _normalize_field_name(value: Any) -> str: + """将短字段名规范化为 snake_case，非字符串不参与身份推导。""" + if type(value) is not str: + return "" + text = value.strip() + if len(text) > _MAX_FIELD_NAME_CHARS: + text = text[-_MAX_FIELD_NAME_CHARS:] + text = _ACRONYM_BOUNDARY_PATTERN.sub("_", text) + text = _CAMEL_CASE_BOUNDARY_PATTERN.sub("_", text) + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def is_secret_setting_key(key: Any) -> bool: + """按完整字段或类型后缀识别凭据，避免误伤 token 统计与过期配置。""" + normalized = _normalize_field_name(key) + if not normalized: + return False + return ( + normalized in _SECRET_FIELD_NAMES + or normalized in _SECRET_SETTING_NAMES + or normalized.endswith(_SECRET_FIELD_ENDINGS) + or normalized.endswith(_SECRET_SETTING_ENDINGS) + ) + + +__all__ = ["is_secret_setting_key"] diff --git a/app/agent/tools/base.py b/app/agent/tools/base.py index bed361aed..dbd6b44a5 100644 --- a/app/agent/tools/base.py +++ b/app/agent/tools/base.py @@ -11,6 +11,11 @@ from langchain_core.tools import BaseTool from pydantic import PrivateAttr from app.agent import StreamingHandler +from app.agent.policy.sanitizer import ( + summarize_error, + summarize_input, + summarize_result, +) from app.agent.tools.tags import ToolTag from app.chain import ChainBase from app.core.config import settings @@ -39,7 +44,9 @@ def serialize_tool_result_for_agent(result: Any) -> str: try: return json.dumps(result, ensure_ascii=False, indent=2, default=str) except Exception as e: - logger.warning(f"工具结果转换为JSON失败: {e}, 使用字符串表示") + logger.warning( + f"工具结果转换为JSON失败: {summarize_error(e)}, 使用字符串表示" + ) return str(result) @@ -303,27 +310,26 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta): # 未启用流式传输，不发送任何工具消息内容 pass - logger.debug(f"Executing tool {self.name} with args: {kwargs}") + logger.debug( + f"Executing tool {self.name} with input summary: {summarize_input(kwargs)}" + ) # 执行具体工具逻辑 try: result = await self.run_with_timeout(**kwargs) - # 记录工具执行结果摘要日志 - str_result = serialize_tool_result_for_agent(result) - if len(str_result) > 500: - summary = str_result[:500] + f"...(已截断，总长度: {len(str_result)})" - else: - summary = str_result - logger.info(f"Agent工具 {self.name} 执行完成，结果摘要: {summary}") + logger.info( + f"Agent工具 {self.name} 执行完成，" + f"结果摘要: {summarize_result(result)}" + ) except ToolExecutionTimeoutError as e: - error_message = str(e) + error_message = summarize_error(e) logger.warning(error_message) result = error_message except Exception as e: - error_message = f"工具执行异常 ({type(e).__name__}): {str(e)}" - logger.error(f"Tool {self.name} execution failed: {e}", exc_info=True) + error_message = f"工具执行异常: {summarize_error(e)}" + logger.error(f"Tool {self.name} execution failed: {summarize_error(e)}") result = error_message return format_tool_result_for_agent( @@ -625,7 +631,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta): return False except Exception as e: - logger.error(f"检查权限失败: {e}") + logger.error(f"检查权限失败: {summarize_error(e)}") return False diff --git a/app/agent/tools/impl/_system_setting_utils.py b/app/agent/tools/impl/_system_setting_utils.py index c6476396f..ddae82ae3 100644 --- a/app/agent/tools/impl/_system_setting_utils.py +++ b/app/agent/tools/impl/_system_setting_utils.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import Any, Optional +from app.agent.policy.secret_fields import is_secret_setting_key from app.core.config import Settings from app.schemas.types import SystemConfigKey @@ -368,26 +369,6 @@ def get_default_list_match_field(setting_key: str) -> Optional[str]: return LIST_ITEM_MATCH_FIELD_DEFAULTS.get(setting_key) -SECRET_KEYWORDS = ( - "api_key", - "apikey", - "token", - "secret", - "password", - "passwd", - "cookie", - "authorization", - "refresh_token", - "access_token", -) - - -def is_secret_setting_key(key: str) -> bool: - """判断设置键名是否疑似敏感字段。""" - normalized = _normalize_token(key) - return any(keyword in normalized for keyword in SECRET_KEYWORDS) - - def redact_secret_value(value: Any, *, redact_scalar: bool = False) -> Any: """递归脱敏配置值中的密钥、Cookie、Token 等敏感字段。""" if isinstance(value, dict): diff --git a/app/agent/tools/manager.py b/app/agent/tools/manager.py index 95320cbfa..3249ff64b 100644 --- a/app/agent/tools/manager.py +++ b/app/agent/tools/manager.py @@ -3,6 +3,16 @@ import threading import uuid from typing import Any, Dict, List, Optional +from app.agent.policy import ( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + AgentToolPolicyOrchestrator, + AuthSource, + PrincipalType, + ToolOrigin, + ToolPolicyContext, + call_policy_hook, + summarize_error, +) from app.agent.tools.base import ToolExecutionTimeoutError, format_tool_result_for_agent from app.agent.tools.factory import MoviePilotToolFactory from app.core.plugin import PluginManager @@ -30,6 +40,7 @@ class MoviePilotToolsManager: user_id: str = "api_user", session_id: str = uuid.uuid4(), is_admin: bool = True, + policy_orchestrator: Optional[AgentToolPolicyOrchestrator] = None, ): """ 初始化工具管理器 @@ -41,6 +52,19 @@ class MoviePilotToolsManager: self.user_id = user_id self.session_id = session_id self.is_admin = is_admin + self.policy_orchestrator = ( + policy_orchestrator or DEFAULT_TOOL_POLICY_ORCHESTRATOR + ) + self._policy_context = ToolPolicyContext( + session_id=session_id, + user_id=user_id, + origin=ToolOrigin.OPERATOR_DIRECT, + principal_type=PrincipalType.SYSTEM_ADMIN_INTEGRATION, + auth_source=AuthSource.API_TOKEN, + channel=None, + source="api", + agent_context={"is_admin": is_admin}, + ) self.tools: List[Any] = [] self._tools_lock = threading.Lock() self._plugin_agent_tools_revision = -1 @@ -74,7 +98,7 @@ class MoviePilotToolsManager: self._plugin_agent_tools_revision = plugin_tools_revision logger.info(f"成功加载 {len(self.tools)} 个工具") except Exception as e: - logger.error(f"加载工具失败: {e}", exc_info=True) + logger.error(f"加载工具失败: {summarize_error(e)}") self.tools = [] self._plugin_agent_tools_revision = -1 @@ -231,7 +255,7 @@ class MoviePilotToolsManager: schema = args_schema.model_json_schema() properties = schema.get("properties", {}) except Exception as e: - logger.warning(f"获取工具schema失败: {e}") + logger.warning(f"获取工具schema失败: {summarize_error(e)}") return arguments # 规范化参数 @@ -286,6 +310,7 @@ class MoviePilotToolsManager: ) return error_msg + observation = None try: permission_error = self._check_tool_permission(tool_instance) if permission_error: @@ -293,38 +318,50 @@ class MoviePilotToolsManager: # 规范化参数类型 normalized_arguments = self._normalize_arguments(tool_instance, arguments) + self._policy_context.agent_context["is_admin"] = self.is_admin + observation = call_policy_hook( + "start", + self.policy_orchestrator.start, + context=self._policy_context, + tool=tool_instance, + arguments=normalized_arguments, + ) # 调用工具的run方法。HTTP/MCP 工具调用不会经过 BaseTool._arun， # 因此这里也必须复用同一套返回值格式化和兜底截断逻辑。 result = await tool_instance.run_with_timeout(**normalized_arguments) - - # 记录工具执行结果摘要日志 str_result = format_tool_result_for_agent( result, tool_name=tool_name, max_chars=getattr(tool_instance, "result_max_chars", None), ) - if len(str_result) > 500: - summary = str_result[:500] + f"...(已截断，总长度: {len(str_result)})" - else: - summary = str_result - logger.info(f"Agent工具 {tool_name} 执行完成，结果摘要: {summary}") - - return str_result except ToolExecutionTimeoutError as e: - logger.warning(str(e)) + if observation: + call_policy_hook("fail", self.policy_orchestrator.fail, observation, e) + logger.warning(summarize_error(e)) return format_tool_result_for_agent( - str(e), + summarize_error(e), tool_name=tool_name, max_chars=getattr(tool_instance, "result_max_chars", None), ) except Exception as e: - logger.error(f"调用工具 {tool_name} 时发生错误: {e}", exc_info=True) + if observation: + call_policy_hook("fail", self.policy_orchestrator.fail, observation, e) + error_summary = summarize_error(e) + logger.error(f"调用工具 {tool_name} 时发生错误: {error_summary}") error_msg = json.dumps( - {"error": f"调用工具 '{tool_name}' 时发生错误: {str(e)}"}, + {"error": f"调用工具 '{tool_name}' 时发生错误: {error_summary}"}, ensure_ascii=False, ) return error_msg + if observation: + call_policy_hook( + "finish", + self.policy_orchestrator.finish, + observation, + str_result, + ) + return str_result @staticmethod def _convert_to_json_schema(args_schema: Any) -> Dict[str, Any]: diff --git a/tests/test_agent_activity_log.py b/tests/test_agent_activity_log.py index 26e9e3139..f76836595 100644 --- a/tests/test_agent_activity_log.py +++ b/tests/test_agent_activity_log.py @@ -2,7 +2,7 @@ import asyncio import json from datetime import datetime from types import SimpleNamespace -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage @@ -418,6 +418,67 @@ def test_activity_log_tool_call_records_streaming_summary(tmp_path): ] +def test_activity_log_middleware_sanitizes_its_own_logs(tmp_path): + """活动日志中间件读取参数和异常写日志时必须脱敏。""" + + async def _run_test(): + secret_marker = "activity-secret-marker-6825" + stream_handler = SimpleNamespace( + is_streaming=True, + record_tool_call=MagicMock(), + ) + middleware = ActivityLogMiddleware( + activity_dir=str(tmp_path), + stream_handler=stream_handler, + ) + request = SimpleNamespace( + tool=SimpleNamespace(name=QUERY_ACTIVITY_LOG_TOOL_NAME), + tool_call={"args": {"keyword": f"token={secret_marker}"}}, + ) + mock_logger = MagicMock() + + async def _failing_handler(_request): + raise RuntimeError(f"Authorization: Bearer {secret_marker}") + + with patch("app.agent.middleware.activity_log.logger", mock_logger): + try: + await middleware.awrap_tool_call(request, _failing_handler) + except RuntimeError: + pass + else: + raise AssertionError("middleware should re-raise handler errors") + + return secret_marker, mock_logger + + secret_marker, mock_logger = asyncio.run(_run_test()) + + assert secret_marker not in str(mock_logger.method_calls) + assert "***" in str(mock_logger.method_calls) + + +def test_activity_log_provider_error_does_not_echo_secret(tmp_path): + """活动日志 provider 内部异常不能进入日志或模型错误结果。""" + secret_marker = "activity-provider-secret-3584" + middleware = ActivityLogMiddleware(activity_dir=str(tmp_path)) + mock_logger = MagicMock() + + with ( + patch( + "app.agent.middleware.activity_log.query_activity_logs", + side_effect=RuntimeError(f"OPENAI_API_KEY={secret_marker}"), + ), + patch("app.agent.middleware.activity_log.logger", mock_logger), + ): + result = asyncio.run( + middleware._tool_provider.query_activity_log(keyword="visible") + ) + + assert secret_marker not in result + assert secret_marker not in str(mock_logger.method_calls) + assert "***" in result + assert "***" in str(mock_logger.method_calls) + + def test_factory_does_not_register_activity_log_tool(): """活动日志查询工具应由中间件注册，不应进入全局工具工厂。""" with patch( diff --git a/tests/test_agent_background_output.py b/tests/test_agent_background_output.py index 617583810..a4f4a1809 100644 --- a/tests/test_agent_background_output.py +++ b/tests/test_agent_background_output.py @@ -431,6 +431,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): self.assertEqual( [ + "AgentPolicyMiddleware", "skills", "jobs", "runtime", @@ -546,6 +547,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): self.assertEqual( [ + "AgentPolicyMiddleware", "skills", "jobs", "runtime", @@ -749,6 +751,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): self.assertEqual( [ + "AgentPolicyMiddleware", "skills", "jobs", "runtime", diff --git a/tests/test_agent_skills_middleware.py b/tests/test_agent_skills_middleware.py index 7dceb8456..5c044b776 100644 --- a/tests/test_agent_skills_middleware.py +++ b/tests/test_agent_skills_middleware.py @@ -1,5 +1,6 @@ import json from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch import pytest from anyio import Path as AsyncPath @@ -171,3 +172,59 @@ async def test_skill_tool_call_records_streaming_summary(tmp_path): }, } ] + + +@pytest.mark.anyio +async def test_skill_middleware_sanitizes_its_own_logs(tmp_path): + """Skill 中间件读取参数和异常写日志时必须脱敏。""" + secret_marker = "skill-secret-marker-2471" + stream_handler = SimpleNamespace( + is_streaming=True, + record_tool_call=MagicMock(), + ) + middleware = SkillsMiddleware( + sources=[str(tmp_path)], + stream_handler=stream_handler, + ) + request = SimpleNamespace( + tool=SimpleNamespace(name=SKILL_TOOL_NAME), + tool_call={"args": {"name": f"api_key={secret_marker}"}}, + ) + mock_logger = MagicMock() + + async def _failing_handler(_request): + raise RuntimeError(f"Authorization: Bearer {secret_marker}") + + with ( + patch("app.agent.middleware.skills.logger", mock_logger), + pytest.raises(RuntimeError), + ): + await middleware.awrap_tool_call(request, _failing_handler) + + assert secret_marker not in str(mock_logger.method_calls) + assert "***" in str(mock_logger.method_calls) + + +@pytest.mark.anyio +async def test_skill_provider_error_does_not_echo_secret(tmp_path): + """Skill provider 内部捕获的异常不能进入日志或模型错误结果。""" + secret_marker = "skill-provider-secret-6518" + middleware = SkillsMiddleware(sources=[str(tmp_path)]) + mock_logger = MagicMock() + + with ( + patch.object( + middleware._skill_provider, + "_find_skill", + new=AsyncMock( + side_effect=RuntimeError(f"DATABASE_PASSWORD={secret_marker}") + ), + ), + patch("app.agent.middleware.skills.logger", mock_logger), + ): + result = await middleware._skill_provider.load_skill("visible-skill") + + assert secret_marker not in result + assert secret_marker not in str(mock_logger.method_calls) + assert "***" in result + assert "***" in str(mock_logger.method_calls) diff --git a/tests/test_agent_subagents.py b/tests/test_agent_subagents.py index e9156dbcd..537dd21b0 100644 --- a/tests/test_agent_subagents.py +++ b/tests/test_agent_subagents.py @@ -2,11 +2,12 @@ import asyncio import json from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch from langchain_core.language_models.fake_chat_models import FakeListChatModel import app.agent.middleware.subagents as subagent_module +from app.agent.middleware.policy import AgentPolicyMiddleware from app.agent.middleware.subagents import ( MoviePilotSubAgentMiddleware, SUBAGENT_CONTROL_TOOL_NAME, @@ -14,6 +15,7 @@ from app.agent.middleware.subagents import ( SubAgentTaskControlMiddleware, create_subagent_middlewares, ) +from app.agent.policy import AuthSource, PrincipalType, ToolOrigin, ToolPolicyContext from app.agent.tools.tags import ToolTag @@ -77,6 +79,40 @@ def test_subagent_tools_are_selected_by_tags(): assert [tool.name for tool in captured["tools"]] == ["custom_media_lookup"] +def test_subagent_graph_registers_policy_middleware_as_outermost(): + """懒加载的子代理图必须继承宿主上下文并先经过 policy middleware。""" + model = FakeListChatModel(responses=["ok"]) + context = ToolPolicyContext( + session_id="subagent-session", + user_id="user-1", + origin=ToolOrigin.SUBAGENT, + principal_type=PrincipalType.SUBAGENT, + auth_source=AuthSource.INTERNAL, + agent_context={"is_admin": True}, + channel="Telegram", + source="telegram", + ) + captured = {} + + def _fake_create_agent(**kwargs): + captured.update(kwargs) + return kwargs + + middleware = MoviePilotSubAgentMiddleware( + model=model, + profiles=subagent_module._builtin_subagent_profiles(), + tools=[], + policy_context=context, + ) + + with patch.object(subagent_module, "create_agent", side_effect=_fake_create_agent): + middleware._get_agent("general-purpose") + + assert isinstance(captured["middleware"][0], AgentPolicyMiddleware) + assert captured["middleware"][0].context is context + assert captured["middleware"][0].context.origin is ToolOrigin.SUBAGENT + + def test_moviepilot_explorer_selects_code_and_settings_tools(): """MoviePilot 探索子代理应能读取代码、目录、设置和命令诊断工具。""" model = FakeListChatModel(responses=["ok"]) @@ -187,6 +223,51 @@ def test_task_tool_call_records_streaming_summary(): ] +def test_task_middleware_sanitizes_its_own_logs(): + """子代理中间件读取任务参数和异常写日志时必须脱敏。""" + + async def _run_test(): + secret_marker = "subagent-secret-marker-7316" + stream_handler = SimpleNamespace( + is_streaming=True, + record_tool_call=MagicMock(), + ) + middleware = MoviePilotSubAgentMiddleware( + model=FakeListChatModel(responses=["ok"]), + profiles=subagent_module._builtin_subagent_profiles(), + tools=[], + stream_handler=stream_handler, + ) + request = SimpleNamespace( + tool=SimpleNamespace(name=SUBAGENT_TASK_TOOL_NAME), + tool_call={ + "args": { + "description": f"password={secret_marker}", + "subagent_type": "media-researcher", + } + }, + ) + mock_logger = MagicMock() + + async def _failing_handler(_request): + raise RuntimeError(f"Authorization: Bearer {secret_marker}") + + with patch.object(subagent_module, "logger", mock_logger): + try: + await middleware.awrap_tool_call(request, _failing_handler) + except RuntimeError: + pass + else: + raise AssertionError("middleware should re-raise handler errors") + + return secret_marker, mock_logger + + secret_marker, mock_logger = asyncio.run(_run_test()) + + assert secret_marker not in str(mock_logger.method_calls) + assert "***" in str(mock_logger.method_calls) + + def test_control_tool_call_records_streaming_summary(): """subagent_task 子代理工具执行时应记录流式聚合摘要。""" @@ -392,11 +473,14 @@ def test_control_tool_pipeline_stops_after_failed_step(): tools=[], ) calls = [] + secret_marker = "subagent-runtime-secret-9042" async def _fake_run_task(self, *, description, subagent_type, task_id=None): calls.append(subagent_type) if subagent_type == "download-diagnostician": - raise RuntimeError("下载器不可用") + raise RuntimeError( + f"下载器不可用 DATABASE_PASSWORD={secret_marker}" + ) return f"{subagent_type}:ok" with patch.object( @@ -433,6 +517,10 @@ def test_control_tool_pipeline_stops_after_failed_step(): "failed", ] assert "下载器不可用" in payload["tasks"][1]["error"] + assert secret_marker not in payload["error"] + assert secret_marker not in payload["tasks"][1]["error"] + assert "***" in payload["error"] + assert "***" in payload["tasks"][1]["error"] asyncio.run(_run_test()) diff --git a/tests/test_agent_system_settings_tools.py b/tests/test_agent_system_settings_tools.py index 76104cab6..8e4b2f578 100644 --- a/tests/test_agent_system_settings_tools.py +++ b/tests/test_agent_system_settings_tools.py @@ -3,6 +3,8 @@ import json import unittest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from app.agent.tools.impl._system_setting_utils import list_setting_specs from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool from app.agent.tools.impl.update_system_settings import UpdateSystemSettingsTool @@ -295,3 +297,44 @@ class TestAgentSystemSettingsTools(unittest.TestCase): payload = json.loads(result) self.assertIn("error", payload) self.assertIn("系统管理员", payload["error"]) + + +@pytest.mark.parametrize( + ("setting_key", "should_redact"), + [ + ("API_TOKEN", True), + ("LLM_API_KEY", True), + ("COOKIECLOUD_KEY", True), + ("COOKIECLOUD_AUTH_HEADER", True), + ("SUPERUSER_PASSWORD", True), + ("DB_POSTGRESQL_PASSWORD", True), + ("GITHUB_TOKEN", True), + ("FEISHU_VERIFICATION_TOKEN", True), + ("SECRET_KEY", True), + ("RESOURCE_SECRET_KEY", True), + ("PROJECT_NAME", False), + ("ACCESS_TOKEN_EXPIRE_MINUTES", False), + ("LLM_MAX_CONTEXT_TOKENS", False), + ("COOKIECLOUD_INTERVAL", False), + ], +) +def test_query_system_settings_uses_precise_secret_identity_matrix( + setting_key: str, + should_redact: bool, +) -> None: + """设置查询应隐藏真实凭据，同时保留仅名称相似的普通设置。""" + marker = "credential-marker" if should_redact else "visible-marker" + tool = QuerySystemSettingsTool(session_id="session-1", user_id="10001") + + with patch.object( + QuerySystemSettingsTool, + "_load_setting_value", + return_value=marker, + ): + payload = json.loads(asyncio.run(tool.run(setting_key=setting_key))) + + item = payload["settings"][0] + assert item["redacted"] is should_redact + assert item["value"] == ("***" if should_redact else marker) + if should_redact: + assert marker not in json.dumps(payload) diff --git a/tests/test_agent_tool_policy.py b/tests/test_agent_tool_policy.py new file mode 100644 index 000000000..32ec5fbed --- /dev/null +++ b/tests/test_agent_tool_policy.py @@ -0,0 +1,744 @@ +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from langchain.agents.middleware import SummarizationMiddleware +from langchain_core.messages import ToolMessage +from pydantic import BaseModel, Field + +import app.agent as agent_module +from app.agent.middleware.activity_log import ActivityLogMiddleware +from app.agent.middleware.memory import MemoryMiddleware +from app.agent.middleware.policy import AgentPolicyMiddleware +from app.agent.policy import ( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + DEFAULT_TOOL_POLICY_REGISTRY, + ActionEffect, + AuthSource, + MigrationState, + PolicyPrincipal, + PrincipalRole, + PrincipalType, + ResultSensitivity, + ToolOrigin, + ToolPolicyContext, + call_policy_hook, +) +from app.agent.tools.base import MoviePilotTool +from app.agent.tools.factory import MoviePilotToolFactory +from app.agent.tools.impl.ask_user_choice import AskUserChoiceTool +from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool +from app.agent.tools.impl.send_local_file import SendLocalFileTool +from app.agent.tools.impl.send_voice_message import SendVoiceMessageTool +from app.agent.tools.manager import MoviePilotToolsManager +from app.schemas.types import MessageChannel + + +class _EchoInput(BaseModel): + """策略测试工具的输入契约。""" + + query: str = Field(description="待回显文本") + + +class _EchoTool(MoviePilotTool): + """返回输入文本的测试工具。""" + + name: str = "policy_echo" + description: str = "Echo policy test input." + args_schema: type[BaseModel] = _EchoInput + + async def run(self, query: str) -> str: + """返回输入文本。""" + return query + + +class _OverrideArunTool(_EchoTool): + """模拟插件覆盖宿主工具基类 `_arun` 的实现。""" + + name: str = "plugin_override_arun" + + def __init__(self, events: list[str], **kwargs): + super().__init__(**kwargs) + self._events = events + + async def _arun(self, *args, **kwargs) -> str: + """绕过基类实现并记录真实执行顺序。""" + self._events.append("tool") + return str(kwargs.get("query") or "ok") + + +class _FailingTool(_EchoTool): + """抛出固定异常以验证 direct manager 的 observation fail-open。""" + + name: str = "policy_failure" + + async def run(self, query: str) -> str: + """模拟真实工具失败。""" + raise ValueError(f"tool-error:{query}") + + +class _AdminSafeReadTool(_EchoTool): + """复用 safe-read 名称并保留旧 require_admin 门禁的测试工具。""" + + name: str = "list_slash_commands" + require_admin: bool = True + + def __init__(self, events: list[str], **kwargs): + super().__init__(**kwargs) + self._events = events + + async def run(self, query: str) -> str: + """记录实际执行，区分旧门禁放行与拒绝。""" + self._events.append("run") + return query + + +def _tool_class_name(tool_class: type[MoviePilotTool]) -> str: + """从 Pydantic 字段默认值读取工具类的稳定名称。""" + return str(tool_class.model_fields["name"].default) + + +def _interactive_context(*, is_admin: bool = True) -> ToolPolicyContext: + """构造会随本轮管理员上下文刷新的交互式策略上下文。""" + return ToolPolicyContext( + session_id="session-1", + user_id="user-1", + origin=ToolOrigin.AGENT_INTERACTIVE, + principal_type=PrincipalType.HUMAN, + auth_source=AuthSource.CHANNEL, + channel="telegram", + source="user", + agent_context={"is_admin": is_admin}, + ) + + +def test_builtin_policy_registry_covers_every_fixed_tool() -> None: + """固定内置工具必须全部具有显式 migration registry 条目。""" + fixed_tool_names = { + _tool_class_name(tool_class) + for tool_class in MoviePilotToolFactory.BUILTIN_TOOL_CLASSES + } + fixed_tool_names.update( + { + _tool_class_name(AskUserChoiceTool), + _tool_class_name(SendLocalFileTool), + _tool_class_name(SendVoiceMessageTool), + } + ) + + assert DEFAULT_TOOL_POLICY_REGISTRY.builtin_tool_names == fixed_tool_names + + +def test_registry_separates_safe_read_from_legacy_shadow() -> None: + """少量安全读取可直接迁移，其余高影响工具保持 shadow allow。""" + safe_policy = DEFAULT_TOOL_POLICY_REGISTRY.resolve( + tool_name="query_personas", + arguments={}, + requires_admin=False, + ) + admin_safe_policy = DEFAULT_TOOL_POLICY_REGISTRY.resolve( + tool_name="list_slash_commands", + arguments={}, + requires_admin=True, + ) + shadow_policy = DEFAULT_TOOL_POLICY_REGISTRY.resolve( + tool_name="update_system_settings", + arguments={"updates": []}, + requires_admin=True, + ) + dynamic_policy = DEFAULT_TOOL_POLICY_REGISTRY.resolve( + tool_name="plugin_unknown_tool", + arguments={"action": "custom"}, + requires_admin=False, + ) + + assert safe_policy.effect is ActionEffect.SAFE_READ + assert safe_policy.result_sensitivity is ResultSensitivity.NORMAL + assert safe_policy.migration_state is MigrationState.ENFORCED + + assert admin_safe_policy.effect is ActionEffect.SAFE_READ + assert admin_safe_policy.required_role is PrincipalRole.SYSTEM_ADMIN + assert admin_safe_policy.migration_state is MigrationState.LEGACY_SHADOW + + assert shadow_policy.migration_state is MigrationState.LEGACY_SHADOW + assert shadow_policy.effect is ActionEffect.UNKNOWN + assert shadow_policy.required_role is PrincipalRole.SYSTEM_ADMIN + + assert dynamic_policy.migration_state is MigrationState.LEGACY_SHADOW + assert dynamic_policy.effect is ActionEffect.UNKNOWN + assert dynamic_policy.result_sensitivity is ResultSensitivity.UNKNOWN + + +def test_legacy_shadow_decision_allows_without_claiming_enforcement() -> None: + """G1 的 shadow 决策只能观测，不能拒绝或要求确认。""" + context = _interactive_context() + tool = _EchoTool(session_id="session-1", user_id="user-1") + + observation = DEFAULT_TOOL_POLICY_ORCHESTRATOR.start( + context=context, + tool=tool, + arguments={"query": "hello"}, + ) + + assert observation.decision.allowed is True + assert observation.decision.shadow is True + assert observation.decision.reason_code == "legacy_shadow_allow" + + +def test_policy_context_reads_mutable_admin_state_without_model_fields() -> None: + """缓存图复用时权限取当前宿主状态，模型参数不能伪造 principal。""" + context = _interactive_context(is_admin=False) + forged_arguments = { + "query": "hello", + "origin": "operator_direct", + "principal_type": "system_admin_integration", + "is_admin": True, + } + + assert context.principal.role is PrincipalRole.USER + context.agent_context["is_admin"] = True + assert context.principal.role is PrincipalRole.SYSTEM_ADMIN + assert "origin" not in PolicyPrincipal.__dataclass_fields__ + assert forged_arguments["origin"] != context.origin.value + + +def test_policy_context_maps_trusted_host_origins() -> None: + """各入口必须由宿主稳定映射 origin、主体类型和认证来源。""" + cases = [ + ( + {"channel": MessageChannel.Web.value, "source": "openai"}, + ToolOrigin.AGENT_API, + PrincipalType.SYSTEM_ADMIN_INTEGRATION, + AuthSource.API_TOKEN, + ), + ( + {"channel": MessageChannel.Web.value, "source": "openai.responses"}, + ToolOrigin.AGENT_API, + PrincipalType.SYSTEM_ADMIN_INTEGRATION, + AuthSource.API_TOKEN, + ), + ( + {"channel": MessageChannel.Web.value, "source": "anthropic"}, + ToolOrigin.AGENT_API, + PrincipalType.SYSTEM_ADMIN_INTEGRATION, + AuthSource.API_TOKEN, + ), + ( + {"channel": MessageChannel.Web.value, "source": "browser"}, + ToolOrigin.AGENT_INTERACTIVE, + PrincipalType.HUMAN, + AuthSource.WEB_SESSION, + ), + ( + {"channel": MessageChannel.WebAgent.value, "source": "web-agent"}, + ToolOrigin.AGENT_INTERACTIVE, + PrincipalType.HUMAN, + AuthSource.WEB_SESSION, + ), + ( + {"channel": MessageChannel.Telegram.value, "source": "telegram"}, + ToolOrigin.AGENT_INTERACTIVE, + PrincipalType.HUMAN, + AuthSource.CHANNEL, + ), + ( + {"channel": MessageChannel.Feishu.value, "source": "feishu"}, + ToolOrigin.AGENT_INTERACTIVE, + PrincipalType.HUMAN, + AuthSource.CHANNEL, + ), + ( + {"channel": None, "source": None, "output_callback": lambda _text: None}, + ToolOrigin.BACKGROUND, + PrincipalType.BACKGROUND, + AuthSource.INTERNAL, + ), + ] + + for kwargs, expected_origin, expected_principal, expected_auth_source in cases: + context = agent_module.MoviePilotAgent( + session_id="origin-session", + user_id="user-1", + **kwargs, + )._build_policy_context() + + assert context.origin is expected_origin + assert context.principal_type is expected_principal + assert context.auth_source is expected_auth_source + + subagent_context = agent_module.MoviePilotAgent( + session_id="subagent-session", + user_id="user-1", + channel=MessageChannel.Telegram.value, + source="telegram", + )._build_policy_context().for_subagent() + assert subagent_context.origin is ToolOrigin.SUBAGENT + assert subagent_context.principal_type is PrincipalType.SUBAGENT + assert subagent_context.auth_source is AuthSource.INTERNAL + + +def test_host_middleware_observes_plugin_before_overridden_arun() -> None: + """插件覆盖 `_arun` 时，宿主 middleware 仍必须先执行策略。""" + events: list[str] = [] + tool = _OverrideArunTool( + events, + session_id="session-1", + user_id="user-1", + ) + middleware = AgentPolicyMiddleware(context=_interactive_context()) + request = SimpleNamespace( + tool=tool, + tool_call={ + "id": "call-1", + "name": tool.name, + "args": {"query": "ok"}, + }, + ) + original_start = DEFAULT_TOOL_POLICY_ORCHESTRATOR.start + + def _record_start(**kwargs): + events.append("policy") + return original_start(**kwargs) + + async def _handler(_request): + result = await tool._arun(query="ok") + return ToolMessage(content=result, tool_call_id="call-1") + + with patch.object( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + "start", + side_effect=_record_start, + ): + result = asyncio.run(middleware.awrap_tool_call(request, _handler)) + + assert result.content == "ok" + assert events == ["policy", "tool"] + + +@pytest.mark.parametrize("failed_phase", ["start", "finish"]) +def test_middleware_observation_failure_does_not_replace_success( + failed_phase: str, +) -> None: + """shadow start/finish 故障不能阻止 handler 或替换成功结果。""" + orchestrator = MagicMock() + orchestrator.start.return_value = object() + getattr(orchestrator, failed_phase).side_effect = RuntimeError( + f"policy-{failed_phase}-failure" + ) + middleware = AgentPolicyMiddleware( + context=_interactive_context(), + orchestrator=orchestrator, + ) + request = SimpleNamespace( + tool=_EchoTool(session_id="session-1", user_id="user-1"), + tool_call={"id": "call-1", "args": {"query": "same"}}, + ) + expected = ToolMessage(content="same", tool_call_id="call-1") + handler_called = False + + async def _handler(_request): + nonlocal handler_called + handler_called = True + return expected + + result = asyncio.run(middleware.awrap_tool_call(request, _handler)) + + assert handler_called is True + assert result is expected + + +def test_middleware_fail_observation_does_not_mask_tool_error() -> None: + """shadow fail hook 故障后仍必须抛出原始工具异常。""" + orchestrator = MagicMock() + orchestrator.start.return_value = object() + orchestrator.fail.side_effect = RuntimeError("policy-fail-hook-failure") + middleware = AgentPolicyMiddleware( + context=_interactive_context(), + orchestrator=orchestrator, + ) + request = SimpleNamespace( + tool=_EchoTool(session_id="session-1", user_id="user-1"), + tool_call={"id": "call-1", "args": {"query": "same"}}, + ) + tool_error = ValueError("original-tool-failure") + + async def _handler(_request): + raise tool_error + + with pytest.raises(ValueError) as error_info: + asyncio.run(middleware.awrap_tool_call(request, _handler)) + + assert error_info.value is tool_error + + +def test_policy_hook_failure_logs_only_stable_type_information() -> None: + """fail-open 诊断只记录阶段和异常类型，不读取可能含凭据的异常文本。""" + mock_logger = MagicMock() + + def _fail() -> None: + raise RuntimeError("DATABASE_PASSWORD=policy-secret-marker") + + with patch("app.agent.policy.orchestrator.logger", mock_logger): + result = call_policy_hook("start", _fail) + + logged = "\n".join(str(call) for call in mock_logger.warning.call_args_list) + assert result is None + assert "phase=start" in logged + assert "RuntimeError" in logged + assert "policy-secret-marker" not in logged + + +def test_policy_hook_hostile_error_type_is_fail_open() -> None: + """异常类型名协议不可信时，观测故障仍不得逃出 fail-open 边界。""" + secret_marker = "hostile-policy-type-secret-6274" + + class _HostileMeta(type): + def __getattribute__(cls, name): + if name == "__name__": + raise RuntimeError(f"DATABASE_PASSWORD={secret_marker}") + return super().__getattribute__(name) + + class _HostilePolicyError(RuntimeError, metaclass=_HostileMeta): + pass + + mock_logger = MagicMock() + + def _fail() -> None: + raise _HostilePolicyError("visible policy failure") + + escaped = False + with patch("app.agent.policy.orchestrator.logger", mock_logger): + try: + result = call_policy_hook("start", _fail) + except BaseException: + escaped = True + result = None + + logged = "\n".join(str(call) for call in mock_logger.warning.call_args_list) + assert escaped is False + assert result is None + assert "phase=start" in logged + assert secret_marker not in logged + + +@pytest.mark.parametrize( + ("legacy_admin", "expected_result", "expected_run_count"), + [ + (True, "same", 1), + ( + False, + ( + "抱歉，您没有执行此工具的权限。" + "只有渠道管理员或系统管理员才能执行工具操作。" + "如需执行工具，请联系管理员将您的用户ID添加到渠道管理员列表中（设定 -> 通知 -> 对应渠道配置 -> 管理员名单），" + "或联系系统管理员为您设置管理员权限。" + ), + 0, + ), + ], +) +def test_agent_admin_safe_read_keeps_legacy_authorization_authority( + legacy_admin: bool, + expected_result: str, + expected_run_count: int, +) -> None: + """渠道管理员与普通用户结果保持旧门禁语义，policy 只做 shadow passthrough。""" + events: list[str] = [] + observations = [] + original_start = DEFAULT_TOOL_POLICY_ORCHESTRATOR.start + tool = _AdminSafeReadTool( + events, + session_id="session-1", + user_id="user-1", + ) + tool.set_message_attr( + channel=MessageChannel.Telegram.value, + source="user", + username="member", + ) + tool.set_agent_context({"is_admin": False}) + middleware = AgentPolicyMiddleware(context=_interactive_context(is_admin=False)) + request = SimpleNamespace( + tool=tool, + tool_call={"id": "call-1", "args": {"query": "same"}}, + ) + + def _capture_start(**kwargs): + observation = original_start(**kwargs) + observations.append(observation) + return observation + + async def _handler(_request): + content = await tool._arun(query="same") + return ToolMessage(content=content, tool_call_id="call-1") + + with ( + patch.object( + _AdminSafeReadTool, + "is_admin_user", + new=AsyncMock(return_value=legacy_admin), + ), + patch.object( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + "start", + side_effect=_capture_start, + ), + ): + result = asyncio.run(middleware.awrap_tool_call(request, _handler)) + + assert result.content == expected_result + assert events.count("run") == expected_run_count + assert len(observations) == 1 + assert observations[0].policy.effect is ActionEffect.SAFE_READ + assert observations[0].policy.migration_state is MigrationState.LEGACY_SHADOW + assert observations[0].decision.allowed is True + assert observations[0].decision.shadow is True + assert observations[0].decision.reason_code == "legacy_shadow_allow" + + +def test_direct_non_admin_safe_read_rejects_before_policy_or_schema() -> None: + """direct 未授权请求保持原 JSON 拒绝格式且不提前触发 policy/schema。""" + events: list[str] = [] + orchestrator = MagicMock() + tool = _AdminSafeReadTool( + events, + session_id="session-1", + user_id="user-1", + ) + manager = MoviePilotToolsManager( + is_admin=False, + policy_orchestrator=orchestrator, + ) + manager.tools = [tool] + + result = json.loads( + asyncio.run(manager.call_tool(tool.name, {"query": "same"})) + ) + + assert result == { + "error": "抱歉，您没有执行此工具的权限。只有系统管理员才能执行工具操作。" + } + assert events == [] + orchestrator.start.assert_not_called() + + +@pytest.mark.parametrize("failed_phase", ["start", "finish"]) +def test_direct_manager_observation_failure_does_not_replace_success( + failed_phase: str, +) -> None: + """direct manager 的 shadow start/finish 故障不能改写真实返回值。""" + orchestrator = MagicMock() + orchestrator.start.return_value = object() + getattr(orchestrator, failed_phase).side_effect = RuntimeError( + f"policy-{failed_phase}-failure" + ) + tool = _EchoTool(session_id="session-1", user_id="user-1") + manager = MoviePilotToolsManager( + is_admin=True, + policy_orchestrator=orchestrator, + ) + manager.tools = [tool] + + result = asyncio.run(manager.call_tool(tool.name, {"query": "same"})) + + assert result == "same" + + +def test_direct_manager_fail_observation_does_not_mask_tool_error() -> None: + """direct manager 的 fail hook 故障不能替换既有工具错误格式。""" + orchestrator = MagicMock() + orchestrator.start.return_value = object() + orchestrator.fail.side_effect = RuntimeError("policy-fail-hook-failure") + tool = _FailingTool(session_id="session-1", user_id="user-1") + manager = MoviePilotToolsManager( + is_admin=True, + policy_orchestrator=orchestrator, + ) + manager.tools = [tool] + + result = asyncio.run(manager.call_tool(tool.name, {"query": "same"})) + + assert "ValueError" in result + assert "tool-error:same" in result + assert "policy-fail-hook-failure" not in result + + +def test_direct_manager_and_agent_middleware_share_policy_resolution() -> None: + """相同工具参数在 Agent 与 direct 入口应获得相同动作策略。""" + observations = [] + original_start = DEFAULT_TOOL_POLICY_ORCHESTRATOR.start + + def _capture_start(**kwargs): + observation = original_start(**kwargs) + observations.append(observation) + return observation + + tool = _EchoTool(session_id="session-1", user_id="user-1") + middleware = AgentPolicyMiddleware(context=_interactive_context()) + request = SimpleNamespace( + tool=tool, + tool_call={ + "id": "call-agent", + "name": tool.name, + "args": {"query": "same"}, + }, + ) + + async def _handler(_request): + return ToolMessage(content="same", tool_call_id="call-agent") + + manager = MoviePilotToolsManager( + user_id="api-user", + session_id="api-session", + is_admin=True, + policy_orchestrator=DEFAULT_TOOL_POLICY_ORCHESTRATOR, + ) + manager.tools = [tool] + + async def _run_both(): + await middleware.awrap_tool_call(request, _handler) + return await manager.call_tool(tool.name, {"query": "same"}) + + with patch.object( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + "start", + side_effect=_capture_start, + ): + direct_result = asyncio.run(_run_both()) + + assert direct_result == "same" + assert [item.invocation.origin for item in observations] == [ + ToolOrigin.AGENT_INTERACTIVE, + ToolOrigin.OPERATOR_DIRECT, + ] + assert observations[0].policy == observations[1].policy + + +def test_agent_middleware_secret_setting_result_stays_out_of_receipt_logs() -> None: + """Agent ToolNode 可接收管理员请求的原值，但策略回执不得记录该值。""" + secret_marker = "middleware-secret-setting-marker" + tool = QuerySystemSettingsTool(session_id="session-1", user_id="admin") + tool.set_agent_context({"is_admin": True}) + middleware = AgentPolicyMiddleware(context=_interactive_context()) + request = SimpleNamespace( + tool=tool, + tool_call={ + "id": "call-secret-setting", + "name": tool.name, + "args": {"setting_key": "COOKIECLOUD_KEY", "show_secrets": True}, + }, + ) + mock_logger = MagicMock() + + async def _handler(_request): + result = await tool._arun( + setting_key="COOKIECLOUD_KEY", + show_secrets=True, + ) + return ToolMessage(content=result, tool_call_id="call-secret-setting") + + with ( + patch.object( + QuerySystemSettingsTool, + "_load_setting_value", + return_value=secret_marker, + ), + patch("app.agent.policy.orchestrator.logger", mock_logger), + ): + result = asyncio.run(middleware.awrap_tool_call(request, _handler)) + + assert secret_marker in result.content + logged = "\n".join( + str(call) + for call in ( + mock_logger.debug.call_args_list + mock_logger.info.call_args_list + ) + ) + assert secret_marker not in logged + assert '"value": "***"' in logged + assert '"value_preview": "***"' in logged + + +def test_main_agent_registers_policy_middleware_as_outermost() -> None: + """主 Agent 必须把宿主策略中间件放在 middleware 链最外层。""" + agent = agent_module.MoviePilotAgent(session_id="session-1", user_id="user-1") + fake_llm = SimpleNamespace( + _llm_type="openai-chat", + model="fake", + profile={"max_input_tokens": 64000}, + ) + captured = {} + + def _fake_create_agent(**kwargs): + captured.update(kwargs) + return object() + + with ( + patch.object(agent, "_initialize_llm", return_value=fake_llm), + patch.object(agent, "_initialize_tools", return_value=[]), + patch.object(agent_module.prompt_manager, "get_agent_prompt", return_value="prompt"), + patch.object(agent_module, "create_subagent_middlewares", return_value=([], [])), + patch.object(agent_module, "create_agent", side_effect=_fake_create_agent), + patch.object(agent_module.settings, "LLM_MAX_TOOLS", 0), + ): + asyncio.run(agent._create_agent(streaming=False)) + + assert isinstance(captured["middleware"][0], AgentPolicyMiddleware) + + +def test_main_agent_preserves_activity_log_middleware_order() -> None: + """策略层加入后，ActivityLog 仍应位于 Memory 后、摘要前。""" + agent = agent_module.MoviePilotAgent( + session_id="session-1", + user_id="user-1", + channel=MessageChannel.WebAgent.value, + source="web-agent", + ) + fake_llm = SimpleNamespace( + _llm_type="openai-chat", + model="fake", + profile={"max_input_tokens": 64000}, + ) + captured = {} + + def _fake_create_agent(**kwargs): + captured.update(kwargs) + return object() + + with ( + patch.object(agent, "_initialize_llm", return_value=fake_llm), + patch.object(agent, "_initialize_tools", return_value=[]), + patch.object(agent_module.prompt_manager, "get_agent_prompt", return_value="prompt"), + patch.object(agent_module, "create_subagent_middlewares", return_value=([], [])), + patch.object(agent_module, "create_agent", side_effect=_fake_create_agent), + patch.object(agent_module.settings, "LLM_MAX_TOOLS", 0), + ): + asyncio.run(agent._create_agent(streaming=False)) + + middlewares = captured["middleware"] + policy_index = next( + index + for index, middleware in enumerate(middlewares) + if isinstance(middleware, AgentPolicyMiddleware) + ) + memory_index = next( + index + for index, middleware in enumerate(middlewares) + if isinstance(middleware, MemoryMiddleware) + ) + activity_index = next( + index + for index, middleware in enumerate(middlewares) + if isinstance(middleware, ActivityLogMiddleware) + ) + summary_index = next( + index + for index, middleware in enumerate(middlewares) + if isinstance(middleware, SummarizationMiddleware) + ) + + assert policy_index == 0 + assert activity_index == memory_index + 1 + assert summary_index == activity_index + 1 diff --git a/tests/test_agent_tool_result_policy.py b/tests/test_agent_tool_result_policy.py new file mode 100644 index 000000000..c6f6d54b9 --- /dev/null +++ b/tests/test_agent_tool_result_policy.py @@ -0,0 +1,1781 @@ +import asyncio +import base64 +import json +from statistics import median +from time import perf_counter +from typing import Annotated, NamedTuple +from unittest.mock import MagicMock, patch + +import pytest +from langgraph.types import Command +from pydantic import ( + AliasChoices, + AliasPath, + BaseModel, + Field, + ValidationError, + field_validator, +) +from pydantic_core import PydanticCustomError +from pydantic.dataclasses import dataclass as pydantic_dataclass + +import app.agent.policy.sanitizer as sanitizer_module +from app.agent.policy import sanitize_for_host, summarize_error, summarize_input, summarize_result +from app.agent.tools.base import MoviePilotTool, serialize_tool_result_for_agent +from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool +from app.agent.tools.manager import MoviePilotToolsManager + + +SECRET_MARKER = "nested-secret-marker-8472" + + +class _SecretInput(BaseModel): + """日志脱敏测试工具的输入契约。""" + + payload: dict = Field(description="包含嵌套值的测试载荷") + + +class _InvalidSecretInput(BaseModel): + """用于验证 Pydantic 输入错误不会回显凭据原值。""" + + api_key: int + + +class _DynamicSecretLocationInput(BaseModel): + """用于验证动态 Mapping key 不会通过错误位置泄漏。""" + + payload: dict[str, int] + + +class _CustomSecretValidationInput(BaseModel): + """用于验证自定义错误类型、消息和上下文不会进入宿主摘要。""" + + value: str + + @field_validator("value") + @classmethod + def reject_value(cls, value: str) -> str: + """构造携带输入值的第三方自定义校验错误。""" + raise PydanticCustomError( + f"custom_{value}", + f"rejected {value}", + {"rejected_value": value}, + ) + + +class _NamedTupleCredential(NamedTuple): + """模拟插件或第三方 SDK 返回的命名元组。""" + + api_key: str + label: str + + +class _AliasedCredentialResult(BaseModel): + """模拟外部凭据名与 Python 属性名不同的结构化结果。""" + + credential: str = Field(alias="apiKey") + + +class _ValidationAliasedCredentialResult(BaseModel): + """模拟仅通过 validation alias 接受凭据的 Pydantic 字段。""" + + credential: str = Field(validation_alias="apiToken") + + +class _ChoiceAliasedCredentialResult(BaseModel): + """模拟敏感名称位于后续 choice 的 Pydantic 字段。""" + + credential: str = Field( + validation_alias=AliasChoices("credentialLabel", "apiKey") + ) + + +class _PathAliasedCredentialResult(BaseModel): + """模拟通过带整数索引的嵌套路径接收凭据的 Pydantic 字段。""" + + credential: str = Field( + validation_alias=AliasPath("payload", 0, "clientSecret") + ) + + +class _ChoicePathAliasedCredentialResult(BaseModel): + """模拟后续 choice 使用嵌套路径的 Pydantic 凭据字段。""" + + credential: str = Field( + validation_alias=AliasChoices( + "credentialLabel", + AliasPath("payload", "refreshToken"), + ) + ) + + +class _SerializationAliasedCredentialResult(BaseModel): + """模拟仅在序列化契约中使用凭据名称的结构化结果。""" + + credential: str = Field(serialization_alias="clientSecret") + + +class _AliasedMetadataResult(BaseModel): + """模拟外部 metadata 名称与 Python 属性名不同的结构化结果。""" + + count: int = Field(alias="tokenCount") + + +class _DisguisedSecretAlias(str): + """保存敏感底层值但通过字符串协议伪装成 metadata 名称。""" + + def __str__(self) -> str: + return "tokenCount" + + +class _HostileAliasedCredentialResult(BaseModel): + """模拟使用 hostile str 子类作为直接别名的 Pydantic 字段。""" + + credential: str = Field(alias=_DisguisedSecretAlias("apiKey")) + + +class _HostilePathAliasedCredentialResult(BaseModel): + """模拟 AliasPath 中包含 hostile str 子类的 Pydantic 字段。""" + + credential: str = Field( + validation_alias=AliasPath( + "payload", + _DisguisedSecretAlias("clientSecret"), + ) + ) + + +@pydantic_dataclass +class _AliasedCredentialDataclass: + """模拟通过赋值形式声明外部凭据名的 Pydantic dataclass。""" + + credential: str = Field(alias="apiKey") + + +@pydantic_dataclass +class _AnnotatedAliasedCredentialDataclass: + """模拟通过 Annotated 声明外部凭据名的 Pydantic dataclass。""" + + credential: Annotated[str, Field(alias="refreshToken")] + + +class _SecretResultTool(MoviePilotTool): + """返回嵌套敏感值的测试工具。""" + + name: str = "secret_result_tool" + description: str = "Return a nested secret test payload." + args_schema: type[BaseModel] = _SecretInput + + async def run(self, payload: dict) -> dict: + """返回输入载荷，验证 shadow 模式不改变工具结果。""" + return { + "ok": True, + "nested": [payload, {"authorization": f"Bearer {SECRET_MARKER}"}], + } + + +class _SecretErrorTool(_SecretResultTool): + """抛出包含敏感值异常的测试工具。""" + + name: str = "secret_error_tool" + + async def run(self, payload: dict) -> dict: + """抛出测试异常。""" + raise RuntimeError(f"api_key={SECRET_MARKER}") + + +def _logged_text(mock_logger: MagicMock) -> str: + """汇总 mock logger 收到的全部消息文本。""" + calls = [] + for method_name in ("debug", "info", "warning", "error"): + method = getattr(mock_logger, method_name) + calls.extend(str(call) for call in method.call_args_list) + return "\n".join(calls) + + +def test_recursive_sanitizer_redacts_nested_structures_and_json_text() -> None: + """嵌套 mapping/sequence 与 JSON 字符串都不能保留 secret marker。""" + payload = { + "name": "normal-name", + "items": [ + {"api_key": SECRET_MARKER}, + {"headers": {"Authorization": f"Bearer {SECRET_MARKER}"}}, + '{"cookie":"' + SECRET_MARKER + '","count":2}', + ], + "token_count": 12, + } + + sanitized = sanitize_for_host(payload) + serialized = str(sanitized) + + assert SECRET_MARKER not in serialized + assert "normal-name" in serialized + assert sanitized["token_count"] == 12 + assert "***" in serialized + + +@pytest.mark.parametrize("as_json_text", [False, True]) +def test_recursive_sanitizer_redacts_values_identified_by_secret_setting_key( + as_json_text: bool, +) -> None: + """设置项身份为凭据时，同一结构中的通用值字段也必须脱敏。""" + payload = { + "settings": [ + { + "setting_key": "API_TOKEN", + "value": SECRET_MARKER, + "value_preview": SECRET_MARKER, + "metadata": {"value": "visible-nested-value"}, + } + ], + "value": "visible-outer-value", + } + source = json.dumps(payload, ensure_ascii=False) if as_json_text else payload + + sanitized = sanitize_for_host(source) + if as_json_text: + sanitized = json.loads(sanitized) + + setting = sanitized["settings"][0] + assert setting["value"] == "***" + assert setting["value_preview"] == "***" + assert setting["metadata"]["value"] == "visible-nested-value" + assert sanitized["value"] == "visible-outer-value" + assert SECRET_MARKER not in json.dumps(sanitized, ensure_ascii=False) + + +def test_recursive_sanitizer_preserves_values_for_nonsecret_setting_key() -> None: + """普通设置的 value 字段仍应保留可诊断内容。""" + payload = { + "setting_key": "PROJECT_NAME", + "value": "MoviePilot", + "value_preview": "MoviePilot", + } + + sanitized = sanitize_for_host(payload) + + assert sanitized == payload + + +@pytest.mark.parametrize( + "setting_key", + [ + "API_TOKEN", + "LLM_API_KEY", + "COOKIECLOUD_KEY", + "COOKIECLOUD_AUTH_HEADER", + "SUPERUSER_PASSWORD", + "DB_POSTGRESQL_PASSWORD", + "GITHUB_TOKEN", + "FEISHU_VERIFICATION_TOKEN", + "SECRET_KEY", + "RESOURCE_SECRET_KEY", + ], +) +def test_recursive_sanitizer_redacts_shared_secret_setting_identities( + setting_key: str, +) -> None: + """宿主回执必须与系统设置工具共享敏感设置身份语义。""" + payload = { + "setting_key": setting_key, + "value": SECRET_MARKER, + "value_preview": SECRET_MARKER, + } + + sanitized = sanitize_for_host(payload) + + assert sanitized["value"] == "***" + assert sanitized["value_preview"] == "***" + + +@pytest.mark.parametrize( + "setting_key", + [ + "PROJECT_NAME", + "ACCESS_TOKEN_EXPIRE_MINUTES", + "LLM_MAX_CONTEXT_TOKENS", + "COOKIECLOUD_INTERVAL", + ], +) +def test_recursive_sanitizer_preserves_shared_nonsecret_setting_identities( + setting_key: str, +) -> None: + """名称中提及凭据概念的普通设置仍应保留诊断值。""" + payload = { + "setting_key": setting_key, + "value": "visible-value", + "value_preview": "visible-value", + } + + assert sanitize_for_host(payload) == payload + + +def test_recursive_sanitizer_fails_closed_when_setting_identity_is_truncated() -> None: + """设置身份扫描不完整时，已捕获的通用值字段不能按明文放行。""" + payload = {"value": SECRET_MARKER} + payload.update({f"padding_{index}": index for index in range(100)}) + payload["setting_key"] = "API_TOKEN" + + sanitized = sanitize_for_host(payload) + + assert sanitized["value"] == "***" + assert sanitized[""] == "more items" + assert SECRET_MARKER not in json.dumps(sanitized, ensure_ascii=False) + + +@pytest.mark.parametrize( + "field_name", + [ + "auth", + "basicAuth", + "authentication", + "httpAuthentication", + "credential", + "credentials", + "serviceCredentials", + ], +) +def test_recursive_sanitizer_redacts_credential_containers( + field_name: str, +) -> None: + """认证与凭据容器必须在读取内部用户名或密码前整体遮蔽。""" + payload = { + field_name: ("alice", SECRET_MARKER), + "authEnabled": True, + "credentialCount": 1, + } + + sanitized = sanitize_for_host(payload) + + assert sanitized == { + field_name: "***", + "authEnabled": True, + "credentialCount": 1, + } + assert SECRET_MARKER not in str(sanitized) + + +@pytest.mark.parametrize("field_name", ["oauth", "OAuth", "oauth2", "OAuth2"]) +def test_recursive_sanitizer_redacts_oauth_credential_containers( + field_name: str, +) -> None: + """OAuth 容器判敏必须与字段大小写及数字分词无关。""" + payload = { + field_name: ("alice", SECRET_MARKER), + "oauthEnabled": True, + "OAuthVersion": 2, + } + + sanitized = sanitize_for_host(payload) + + assert sanitized == { + field_name: "***", + "oauthEnabled": True, + "OAuthVersion": 2, + } + assert SECRET_MARKER not in str(sanitized) + + +def test_recursive_sanitizer_redacts_named_tuple_secret_fields() -> None: + """命名元组必须保留字段语义并按字段名脱敏。""" + payload = _NamedTupleCredential( + api_key=SECRET_MARKER, + label="visible-label", + ) + + sanitized = sanitize_for_host(payload) + + assert sanitized == {"api_key": "***", "label": "visible-label"} + assert SECRET_MARKER not in str(sanitized) + + +def test_sanitizer_rejects_hostile_named_tuple_metadata_without_protocols() -> None: + """伪造的 `_fields` 与 tuple 覆盖协议不得参与 named-tuple 分类。""" + calls = [] + + class _HostileFields(tuple): + def __len__(self) -> int: + calls.append("fields.__len__") + raise AssertionError("hostile fields length executed") + + def __getitem__(self, index): + calls.append("fields.__getitem__") + raise AssertionError("hostile fields item executed") + + class _TupleLike(tuple): + _fields = _HostileFields(("label",)) + + def __len__(self) -> int: + calls.append("value.__len__") + raise AssertionError("hostile value length executed") + + def __getitem__(self, index): + calls.append("value.__getitem__") + raise AssertionError("hostile value item executed") + + sanitized = sanitize_for_host(_TupleLike(("visible",))) + + assert calls == [] + assert sanitized == ["visible"] + + +@pytest.mark.parametrize( + "value", + [ + _AliasedCredentialResult(apiKey=SECRET_MARKER), + _ValidationAliasedCredentialResult(apiToken=SECRET_MARKER), + _ChoiceAliasedCredentialResult(apiKey=SECRET_MARKER), + _PathAliasedCredentialResult( + payload=[{"clientSecret": SECRET_MARKER}] + ), + _ChoicePathAliasedCredentialResult( + payload={"refreshToken": SECRET_MARKER} + ), + _SerializationAliasedCredentialResult(credential=SECRET_MARKER), + _HostileAliasedCredentialResult.model_validate( + {"apiKey": SECRET_MARKER} + ), + _HostilePathAliasedCredentialResult.model_validate( + {"payload": {"clientSecret": SECRET_MARKER}} + ), + ], +) +def test_recursive_sanitizer_redacts_pydantic_secret_aliases( + value: BaseModel, +) -> None: + """Pydantic 字段的输入、路径及输出别名均参与凭据判定。""" + sanitized = sanitize_for_host(value) + + assert sanitized == {"credential": "***"} + assert SECRET_MARKER not in str(sanitized) + + +def test_recursive_sanitizer_preserves_pydantic_metadata_alias() -> None: + """非敏感 Pydantic 外部别名不应遮蔽 metadata 值。""" + assert sanitize_for_host(_AliasedMetadataResult(tokenCount=12)) == { + "count": 12 + } + + +@pytest.mark.parametrize( + "value", + [ + _AliasedCredentialDataclass(apiKey=SECRET_MARKER), + _AnnotatedAliasedCredentialDataclass(refreshToken=SECRET_MARKER), + ], +) +def test_recursive_sanitizer_redacts_pydantic_dataclass_secret_aliases( + value: object, +) -> None: + """Pydantic dataclass 的解析后别名元数据同样参与凭据判定。""" + sanitized = sanitize_for_host(value) + + assert sanitized == {"credential": "***"} + assert SECRET_MARKER not in str(sanitized) + + +def test_pydantic_alias_path_limit_applies_before_iteration() -> None: + """AliasPath 必须先验证长度边界，再读取或复制任何 path part。""" + + class _TrackingPath(list): + """记录 alias path 实际向 sanitizer 交付的 part 数量。""" + + yielded_parts = 0 + + def __iter__(self): + for part in super().__iter__(): + type(self).yielded_parts += 1 + yield part + + alias = AliasPath("placeholder") + alias.path = _TrackingPath( + ["metadata"] * (sanitizer_module._MAX_ITEMS + 1) + ) + + assert sanitizer_module._pydantic_alias_names(alias) is None + assert _TrackingPath.yielded_parts == 0 + + +def test_pydantic_alias_choices_share_one_part_budget() -> None: + """AliasPath 与普通 choice 共用额度，耗尽后必须 fail-closed。""" + alias = AliasChoices( + AliasPath(*(["metadata"] * sanitizer_module._MAX_ITEMS)), + "metadataTail", + ) + budget = [sanitizer_module._MAX_ITEMS] + + assert sanitizer_module._pydantic_alias_names( + alias, + _budget=budget, + ) is None + assert budget == [0] + + class _OversizedAliasResult(BaseModel): + """模拟 alias part 总量超过宿主固定额度的第三方模型。""" + + credential: str = Field(validation_alias=alias) + + sanitized = sanitize_for_host( + _OversizedAliasResult.model_construct(credential=SECRET_MARKER) + ) + + assert sanitized == {"credential": "***"} + assert SECRET_MARKER not in str(sanitized) + + +@pytest.mark.parametrize( + "field_name", + [ + "accessToken", + "refreshToken", + "apiKey", + "APIKey", + "apikey", + "APIKEY", + "authToken", + "clientSecret", + "appSecret", + "proxyAuthorization", + "dbPwd", + "passKey", + "secretKey", + "SECRETKEY", + "AccessKeySecret", + "awsSecretAccessKey", + ], +) +def test_recursive_sanitizer_redacts_camel_case_secret_fields( + field_name: str, +) -> None: + """结构化第三方载荷的驼峰凭据字段必须脱敏，统计字段保持可见。""" + payload = { + field_name: SECRET_MARKER, + "tokenCount": 12, + } + + sanitized = sanitize_for_host(payload) + + assert sanitized[field_name] == "***" + assert sanitized["tokenCount"] == 12 + assert SECRET_MARKER not in str(sanitized) + + +@pytest.mark.parametrize( + ("source", "secret_parts"), + [ + ( + 'password="quoted-secret-alpha quoted-secret-beta"', + ("quoted-secret-alpha", "quoted-secret-beta"), + ), + ( + "password='single-secret-alpha single-secret-beta'", + ("single-secret-alpha", "single-secret-beta"), + ), + ( + f"DATABASE_PASSWORD={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"OPENAI_API_KEY={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"MOVIEPILOT_API_TOKEN={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"authToken={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"dbPassword={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"secretKey={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"proxyAuthorization={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"awsSecretAccessKey={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"passKey={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"url=https://example.invalid/callback?authToken={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + f"{'x' * 300}AuthToken={SECRET_MARKER}", + (SECRET_MARKER,), + ), + ( + "password=unquoted-secret-alpha unquoted-secret-beta; status=failed", + ("unquoted-secret-alpha", "unquoted-secret-beta"), + ), + ( + "DATABASE_PASSWORD=correct horse battery staple, retry=off", + ("correct", "horse", "battery", "staple"), + ), + ( + f"DATABASE_PASSWORD={SECRET_MARKER}#password-tail&more, retry=off", + (SECRET_MARKER, "password-tail", "more"), + ), + ], +) +def test_sanitizer_redacts_secret_assignments( + source: str, + secret_parts: tuple[str, ...], +) -> None: + """常见字段拼写、业务前缀和多词凭据都必须完整脱敏。""" + sanitized = str(sanitize_for_host(source)) + + assert "***" in sanitized + for secret_part in secret_parts: + assert secret_part not in sanitized + + if "; status=failed" in source: + assert "; status=failed" in sanitized + if ", retry=off" in source: + assert ", retry=off" in sanitized + + +@pytest.mark.parametrize( + "source", + [ + "tokenCount=12", + "tokenType=usage", + "secretVersion=2", + "apiKeyId=public-id", + "accessTokenExpiresAt=2030-01-01T00:00:00Z", + "passwordHash=sha256:diagnostic", + "url=https://example.invalid/callback?apiKeyId=public-id", + ], +) +def test_sanitizer_preserves_metadata_assignments(source: str) -> None: + """带凭据词根但以 metadata 语义结尾的字段保持诊断价值。""" + assert sanitize_for_host(source) == source + + +@pytest.mark.parametrize( + "field_name", + ["api key", "access token", "client secret", "refresh token"], +) +@pytest.mark.parametrize("quote", ['"', "'"]) +@pytest.mark.parametrize("escape_layers", range(4)) +def test_sanitizer_redacts_escaped_quoted_secret_keys_with_spaces( + field_name: str, + quote: str, + escape_layers: int, +) -> None: + """转义 JSON 片段中的空格分隔凭据名必须复用结构化判敏语义。""" + wrapper = "\\" * escape_layers + quote + source = ( + f"payload={{{wrapper}{field_name}{wrapper}:" + f"{wrapper}{SECRET_MARKER}{wrapper}}}" + ) + + sanitized = str(sanitize_for_host(source)) + + assert SECRET_MARKER not in sanitized + assert "***" in sanitized + assert field_name in sanitized + + +@pytest.mark.parametrize("quote", ['"', "'"]) +@pytest.mark.parametrize("escape_layers", range(7)) +@pytest.mark.parametrize("leading_whitespace", [" ", "\t", " \t"]) +def test_sanitizer_redacts_quoted_secret_keys_with_leading_whitespace( + quote: str, + escape_layers: int, + leading_whitespace: str, +) -> None: + """quoted key 的前导横向空白不得绕过凭据名识别。""" + wrapper = "\\" * escape_layers + quote + source = ( + f"payload={{{wrapper}{leading_whitespace}api key{wrapper}:" + f"{wrapper}{SECRET_MARKER}{wrapper}}}" + ) + + outputs = ( + str(sanitize_for_host(source)), + summarize_input(source), + summarize_result(source), + summarize_error(RuntimeError(source)), + ) + + assert all(SECRET_MARKER not in output for output in outputs) + assert all("***" in output for output in outputs) + + +def test_sanitizer_preserves_escaped_quoted_metadata_key_with_spaces() -> None: + """空格分隔的 metadata key 不应因 quoted-key 支持而被误判。""" + source = r'payload=\"{\\\"token count\\\":12}\"' + + assert sanitize_for_host(source) == source + + +@pytest.mark.parametrize("header", ["Authorization", "Proxy-Authorization"]) +def test_sanitizer_redacts_basic_auth_in_builtin_tuple_key( + header: str, +) -> None: + """内建 tuple key 中的 Basic Auth 与 URI userinfo 都不得进入输出 key。""" + basic_token = "YWxpY2U6c3ludGhldGljLXBhc3N3b3Jk" + payload = { + ( + header, + f"Basic {basic_token} https://alice:{SECRET_MARKER}@example.invalid", + ): "ok" + } + + sanitized = sanitize_for_host(payload) + output_key = next(iter(sanitized)) + + assert sanitized[output_key] == "ok" + assert basic_token not in output_key + assert SECRET_MARKER not in output_key + assert "Basic ***" in output_key + assert "https://***@example.invalid" in output_key + + +@pytest.mark.parametrize("scheme", ["Basic", "basic", "BASIC"]) +@pytest.mark.parametrize( + "basic_token", + [ + "dTpw", + "YWxpY2U6cA==", + "YWxpY2U6c3ludGhldGljLXBhc3N3b3Jk", + ], +) +def test_sanitizer_redacts_basic_auth_across_host_summaries( + scheme: str, + basic_token: str, +) -> None: + """裸 Basic Auth token 在全部宿主摘要入口复用中央文本脱敏。""" + source = f"upstream returned {scheme} {basic_token}. status=failed" + + outputs = ( + str(sanitize_for_host(source)), + summarize_input(source), + summarize_result(source), + summarize_error(RuntimeError(source)), + ) + + assert all(basic_token not in output for output in outputs) + assert all(f"{scheme} ***" in output for output in outputs) + assert all("status=failed" in output for output in outputs) + + +@pytest.mark.parametrize( + "source", + [ + "transport uses basic mode", + "scheme=Basic dG9rZW4=", + ], +) +def test_sanitizer_preserves_noncredential_basic_metadata(source: str) -> None: + """普通 basic 文案及不含 user:password 的 Base64 metadata 保持可见。""" + assert sanitize_for_host(source) == source + + +def test_sanitizer_fails_closed_for_truncated_basic_auth_token() -> None: + """Basic token 在文本上限内未闭合时遮蔽整个已保留前缀。""" + prefix = "log: Basic " + token = base64.b64encode( + b"alice:" + b"x" * sanitizer_module._MAX_TEXT_CHARS + ).decode() + source = prefix + token + + sanitized = str(sanitize_for_host(source)) + + assert sanitized == f"{prefix}***" + assert token[:100] not in sanitized + + +def test_sanitizer_fails_closed_for_truncated_basic_auth_in_tuple_key() -> None: + """tuple renderer 的内部截断事实必须传给 Basic token 脱敏。""" + token = base64.b64encode( + b"alice:" + b"x" * sanitizer_module._MAX_TEXT_CHARS + ).decode() + payload = {("Authorization", f"Basic {token}"): "ok"} + + sanitized = sanitize_for_host(payload) + output_key = next(iter(sanitized)) + + assert sanitized[output_key] == "ok" + assert "Basic ***" in output_key + assert "" in output_key + assert token[:100] not in output_key + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + f"url=https://example.invalid/cb?authToken={SECRET_MARKER}&status=ok#done", + "url=https://example.invalid/cb?authToken=***&status=ok#done", + ), + ( + f"url=https://example.invalid/cb?authToken={SECRET_MARKER}" + f"&refreshToken={SECRET_MARKER}#done", + "url=https://example.invalid/cb?authToken=***&refreshToken=***#done", + ), + ( + f'message="authToken={SECRET_MARKER}"; status=failed', + 'message="authToken=***"; status=failed', + ), + ( + 'message="authToken="; status=ok', + 'message="authToken=***"; status=ok', + ), + ( + "message='authToken='; status=ok", + "message='authToken=***'; status=ok", + ), + ( + 'message="prefix authToken="; status=ok', + 'message="prefix authToken=***"; status=ok', + ), + ( + 'authToken=""', + 'authToken=***', + ), + ( + 'url=https://example.invalid/cb?authToken=&status=ok', + 'url=https://example.invalid/cb?authToken=***&status=ok', + ), + ( + 'authToken="unterminated', + 'authToken=***', + ), + ], +) +def test_sanitizer_preserves_nested_assignment_boundaries( + source: str, + expected: str, +) -> None: + """内层凭据脱敏后保留 URL 分段与外层引号结构。""" + assert sanitize_for_host(source) == expected + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + f"database_url=postgresql://alice:{SECRET_MARKER}" + "@example.invalid/media", + "database_url=postgresql://***@example.invalid/media", + ), + ( + f"endpoint=https://{SECRET_MARKER}@example.invalid/path", + "endpoint=https://***@example.invalid/path", + ), + ( + f"dsn=postgresql://alice:{SECRET_MARKER}%40tail" + "@[2001:db8::1]:5432/media?sslmode=require", + "dsn=postgresql://***@[2001:db8::1]:5432/media?sslmode=require", + ), + ( + f"primary=https://alice:{SECRET_MARKER}@one.invalid/a " + f"secondary=redis://:{SECRET_MARKER}-two@two.invalid/0", + "primary=https://***@one.invalid/a " + "secondary=redis://***@two.invalid/0", + ), + ( + f"https://alice:{SECRET_MARKER}@one.invalid," + f"redis://:{SECRET_MARKER}-two@two.invalid/0", + "https://***@one.invalid,redis://***@two.invalid/0", + ), + ( + f"https://alice:{SECRET_MARKER}@one.invalid;" + f"redis://:{SECRET_MARKER}-two@two.invalid/0", + "https://***@one.invalid;redis://***@two.invalid/0", + ), + ( + f"https://alice:{SECRET_MARKER}@one.invalid|" + f"redis://:{SECRET_MARKER}-two@two.invalid/0", + "https://***@one.invalid|redis://***@two.invalid/0", + ), + ], +) +def test_sanitizer_redacts_uri_userinfo(source: str, expected: str) -> None: + """URI authority 中的 userinfo 不得进入宿主摘要。""" + assert sanitize_for_host(source) == expected + + +@pytest.mark.parametrize("escape_layers", [1, 2]) +def test_sanitizer_redacts_slash_escaped_uri_userinfo( + escape_layers: int, +) -> None: + """嵌入诊断文本中的 slash-escaped URI 仍须清理 userinfo。""" + separator = ":" + "\\" * escape_layers + "/" + "\\" * escape_layers + "/" + source = ( + r'payload={\"dsn\":\"postgresql' + f"{separator}alice:{SECRET_MARKER}@example.invalid/media" + r'\"}' + ) + + sanitized = str(sanitize_for_host(source)) + + assert SECRET_MARKER not in sanitized + assert f"postgresql{separator}***@example.invalid/media" in sanitized + + +def test_sanitizer_preserves_slash_escaped_uri_without_userinfo() -> None: + """slash-escaped URI 没有 userinfo 时保持原始诊断文本。""" + source = r"url=https:\/\/example.invalid/path?email=user@example.invalid" + + assert sanitize_for_host(source) == source + + +def test_sanitizer_redacts_truncated_uri_with_unresolved_userinfo() -> None: + """截断点前无法确认 authority 结束时按敏感内容处理。""" + source = ( + f"dsn=postgresql://alice:{SECRET_MARKER}" + f"{'x' * (16 * 1024)}@example.invalid/media" + ) + + summaries = ( + summarize_input(source), + summarize_result(source), + summarize_error(RuntimeError(source)), + ) + + assert all(SECRET_MARKER not in summary for summary in summaries) + assert all("***" in summary for summary in summaries) + assert all("" in summary for summary in summaries) + + +def test_sanitizer_redacts_truncated_uri_after_early_at_sign() -> None: + """截断 authority 内的早期 `@` 不能证明 userinfo 已完整结束。""" + trailing_secret = "truncated-uri-tail-secret-5931" + prefix = f"https://user:{SECRET_MARKER}@{trailing_secret}" + source = ( + prefix + + "x" * (16 * 1024 - len(prefix)) + + "@example.invalid/media" + ) + + summaries = ( + summarize_input(source), + summarize_result(source), + summarize_error(RuntimeError(source)), + ) + + assert all(SECRET_MARKER not in summary for summary in summaries) + assert all(trailing_secret not in summary for summary in summaries) + assert all("***" in summary for summary in summaries) + assert all("" in summary for summary in summaries) + + +@pytest.mark.parametrize( + "source", + [ + 'payload="{\\"apiKey\\":\\"' + SECRET_MARKER + '\\"}"', + rf'payload=\"{{\\\"apiKey\\\":\\\"{SECRET_MARKER}\\\"}}\"', + ], +) +def test_sanitizer_redacts_escaped_json_secret_fields(source: str) -> None: + """普通文本内多层转义的 JSON 凭据字段仍须脱敏。""" + sanitized = str(sanitize_for_host(source)) + + assert SECRET_MARKER not in sanitized + assert "***" in sanitized + assert "}" in sanitized + + +def test_sanitizer_preserves_tail_after_escaped_json_secret() -> None: + """转义 JSON 凭据中的分隔符不应截断脱敏或吞掉后续字段。""" + source = ( + 'payload="{\\"apiKey\\":\\"' + f"{SECRET_MARKER},still-secret" + '\\",\\"status\\":\\"ok\\"}"' + ) + + sanitized = str(sanitize_for_host(source)) + + assert SECRET_MARKER not in sanitized + assert "still-secret" not in sanitized + assert "status" in sanitized + assert "ok" in sanitized + + +@pytest.mark.parametrize("escape_layers", [0, 1, 2]) +def test_sanitizer_handles_trailing_backslashes_before_secret_quote( + escape_layers: int, +) -> None: + """凭据值末尾的 literal backslash 不得吞掉后续敏感字段。""" + payload = ( + '{"authToken":"first-secret\\\\",' + '"refreshToken":"second-secret","status":"ok"}' + ) + for _ in range(escape_layers): + escaped_payload = payload.replace("\\", "\\\\").replace('"', '\\"') + payload = f'"{escaped_payload}"' + source = f"payload={payload}" + + sanitized = str(sanitize_for_host(source)) + + assert "first-secret" not in sanitized + assert "second-secret" not in sanitized + assert sanitized.count("***") == 2 + assert "status" in sanitized + assert "ok" in sanitized + + +def test_sanitizer_preserves_escaped_json_metadata_fields() -> None: + """转义 JSON 中的 metadata 字段保持诊断值。""" + source = 'payload="{\\"apiKeyId\\":\\"public-id\\"}"' + + assert sanitize_for_host(source) == source + + +def test_sanitizer_preserves_uri_without_userinfo() -> None: + """不含 userinfo 的 URL 及 query 邮箱保持原始诊断信息。""" + source = "url=https://example.invalid/path?email=user@example.invalid" + + assert sanitize_for_host(source) == source + + +@pytest.mark.parametrize( + "unit", + ["a=", "a.", "a://host/", "\\", "\\\""], +) +def test_sanitizer_assignment_scan_scales_at_text_limit(unit: str) -> None: + """赋值链和无头字段链在宿主文本上限内保持近似线性扫描。""" + + def median_duration(size: int) -> float: + source = (unit * (size // len(unit) + 1))[:size] + durations = [] + for _ in range(3): + started_at = perf_counter() + assert sanitize_for_host(source) == source + durations.append(perf_counter() - started_at) + return median(durations) + + small_duration = median_duration(4 * 1024) + max_duration = median_duration(16 * 1024) + + # 4x 输入允许 10x 时间与 20ms 调度余量，同时约束同步宿主观测的延迟增长。 + assert max_duration <= small_duration * 10 + 0.02 + + +def test_secret_assignment_slash_run_scales_at_text_limit() -> None: + """凭据值中的连续反斜杠必须单向扫描，不能重复遍历同一后缀。""" + + def median_duration(slash_count: int) -> float: + source = "password=" + "\\" * slash_count + "tail" + durations = [] + for _ in range(3): + started_at = perf_counter() + sanitized = str(sanitize_for_host(source)) + durations.append(perf_counter() - started_at) + assert sanitized.startswith("password=***") + return median(durations) + + small_duration = median_duration(4 * 1024) + max_duration = median_duration(sanitizer_module._MAX_TEXT_CHARS) + + # 4x 输入允许 10x 时间与 50ms 调度余量，同时排除平方级同步扫描。 + assert max_duration <= small_duration * 10 + 0.05 + + +def test_sanitizer_bounds_oversized_mapping_key_normalization() -> None: + """超长结构化字段只允许固定窗口进入凭据名规范化。""" + + class _TrackingPattern: + """记录正则收到的最大文本长度并复用真实匹配行为。""" + + def __init__(self, pattern) -> None: + self.pattern = pattern + self.max_chars = 0 + + def sub(self, replacement: str, value: str) -> str: + self.max_chars = max(self.max_chars, len(value)) + return self.pattern.sub(replacement, value) + + padding = "x" * (2 * 1024 * 1024) + secret_pattern = _TrackingPattern( + sanitizer_module._ACRONYM_BOUNDARY_PATTERN + ) + camel_pattern = _TrackingPattern( + sanitizer_module._CAMEL_CASE_BOUNDARY_PATTERN + ) + payload = { + f"secret-prefix-{padding}AuthToken": SECRET_MARKER, + f"metadata-prefix-{padding}tokenCount": 12, + } + + with ( + patch.object( + sanitizer_module, + "_ACRONYM_BOUNDARY_PATTERN", + secret_pattern, + ), + patch.object( + sanitizer_module, + "_CAMEL_CASE_BOUNDARY_PATTERN", + camel_pattern, + ), + ): + sanitized = sanitize_for_host(payload) + + assert SECRET_MARKER not in str(sanitized) + assert "***" in sanitized.values() + assert 12 in sanitized.values() + assert secret_pattern.max_chars <= 1024 + assert camel_pattern.max_chars <= 1024 + + +@pytest.mark.parametrize("value", [SECRET_MARKER, 12]) +def test_sanitizer_does_not_stringify_dynamic_mapping_key( + value: object, +) -> None: + """动态 Mapping key 不执行字符串协议，值按未知字段保守遮蔽。""" + + class _StatefulKey: + """通过字符串协议伪装字段语义的第三方 key。""" + + def __init__(self) -> None: + self.calls = 0 + + def __str__(self) -> str: + self.calls += 1 + return "api_key" + + key = _StatefulKey() + + sanitized = sanitize_for_host({key: value}) + + assert key.calls == 0 + assert sanitized == {"": "***"} + assert SECRET_MARKER not in str(sanitized) + + +def test_sanitizer_redacts_value_for_uninspectable_mapping_key() -> None: + """无法取得稳定名称的 Mapping key 按敏感字段处理。""" + + class _UninspectableKey: + """模拟字符串协议故障的第三方 key。""" + + def __str__(self) -> str: + raise RuntimeError(f"unavailable key {SECRET_MARKER}") + + sanitized = sanitize_for_host({_UninspectableKey(): SECRET_MARKER}) + + assert sanitized == {"": "***"} + assert SECRET_MARKER not in str(sanitized) + + +def test_sanitizer_bounds_shared_reference_expansion() -> None: + """整个净化调用共享工作预算，重复引用不能按分支指数展开。""" + + class _CountingValue: + """记录共享叶节点被字符串化的次数。""" + + def __init__(self) -> None: + self.calls = 0 + + def __str__(self) -> str: + self.calls += 1 + return "visible-leaf" + + leaf = _CountingValue() + payload = leaf + for _ in range(7): + payload = [payload] * 5 + + sanitized = sanitize_for_host(payload) + + assert leaf.calls <= 2000 + assert "" in str(sanitized) + + +def test_sanitizer_budget_bounds_container_item_expansion() -> None: + """共享 DAG 的容器读取和输出项总量必须受全调用预算约束。""" + + class _CountingMapping(dict): + """统计 Mapping iterator 实际交付给 sanitizer 的项数。""" + + yielded_items = 0 + + def items(self): + for item in super().items(): + type(self).yielded_items += 1 + yield item + + def count_entries(value: object) -> int: + """统计 sanitizer 结果中实际生成的容器项数。""" + if isinstance(value, dict): + return len(value) + sum(count_entries(item) for item in value.values()) + if isinstance(value, list): + return len(value) + sum(count_entries(item) for item in value) + return 0 + + shared: object = {"label": "visible"} + for level in range(7): + node = _CountingMapping( + { + f"field{level}_{index}ApiKey": SECRET_MARKER + for index in range(97) + } + ) + node.update({f"child{index}": shared for index in range(3)}) + shared = node + + sanitized = sanitize_for_host(shared) + max_emitted_items = sanitizer_module._MAX_WORK_ITEMS + 16 + + assert _CountingMapping.yielded_items <= max_emitted_items + assert count_entries(sanitized) <= max_emitted_items + assert "" in str(sanitized) + + +def test_camel_case_secret_assignment_is_redacted_from_host_summaries() -> None: + """输入、结果和异常摘要共享非结构化凭据赋值的脱敏契约。""" + source = f"authToken={SECRET_MARKER}" + + summaries = ( + summarize_input(source), + summarize_result(source), + summarize_error(RuntimeError(source)), + ) + + assert all(SECRET_MARKER not in summary for summary in summaries) + assert all("***" in summary for summary in summaries) + + +@pytest.mark.parametrize( + "container_value", + [ + f"['{SECRET_MARKER}', 'second-list-secret']", + ( + "{'primary': '" + f"{SECRET_MARKER}', 'nested': ['second-dict-secret', {{'ok': true}}]}}" + ), + f"('{SECRET_MARKER}', ('second-tuple-secret', 2))", + ], +) +def test_sanitizer_redacts_complete_unquoted_secret_container_assignment( + container_value: str, +) -> None: + """未加引号的嵌套容器凭据值必须整体遮蔽并保留后续字段。""" + source = f"password={container_value}, operation=connect" + + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***, operation=connect" + assert SECRET_MARKER not in sanitized + assert "second-" not in sanitized + + +def test_sanitizer_fails_closed_for_unclosed_secret_container_assignment() -> None: + """未闭合的凭据容器无法确认边界时遮蔽剩余文本。""" + source = ( + f"password=['{SECRET_MARKER}', 'unclosed-container-secret', " + "operation=connect" + ) + + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***" + assert SECRET_MARKER not in sanitized + assert "unclosed-container-secret" not in sanitized + + +def test_sanitizer_redacts_secret_tail_after_closed_assignment_container() -> None: + """容器闭合符不代表凭据值结束，尾随内容也必须遮蔽。""" + source = ( + f"password=['{SECRET_MARKER}']tail-container-secret, " + "operation=connect" + ) + + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***, operation=connect" + assert "tail-container-secret" not in sanitized + + +@pytest.mark.parametrize( + "source", + [ + f"password=[first\\], {SECRET_MARKER}], status=ok", + f"password=(first\\), {SECRET_MARKER}), status=ok", + "password={first\\}, " + SECRET_MARKER + "}, status=ok", + ], +) +def test_sanitizer_ignores_escaped_assignment_container_closers( + source: str, +) -> None: + """未引号容器中的转义闭合符不得提前结束凭据扫描。""" + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***, status=ok" + assert SECRET_MARKER not in sanitized + + +@pytest.mark.parametrize( + "source", + [ + f"password=prefix[{SECRET_MARKER}, second-prefix-secret], status=ok", + f"password=call({SECRET_MARKER}, second-call-secret), status=ok", + ( + "password=\\[" + f"{SECRET_MARKER}, second-escaped-open-secret], status=ok" + ), + ], +) +def test_sanitizer_tracks_containers_after_unquoted_value_prefix( + source: str, +) -> None: + """未引号值任意位置的容器均须屏蔽其内部字段分隔符。""" + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***, status=ok" + assert SECRET_MARKER not in sanitized + assert "second-" not in sanitized + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + f'password=prefix"{SECRET_MARKER}, second-double-secret", status=ok', + "password=***, status=ok", + ), + ( + f"password=prefix'{SECRET_MARKER}, second-single-secret', status=ok", + "password=***, status=ok", + ), + ( + f'message="password=prefix\'{SECRET_MARKER}, second-inner-secret\'"; ' + "status=ok", + 'message="password=***"; status=ok', + ), + ], +) +def test_sanitizer_tracks_quoted_fragments_inside_unquoted_secret_value( + source: str, + expected: str, +) -> None: + """值中途的 quoted fragment 不得让内部逗号提前结束脱敏。""" + sanitized = str(sanitize_for_host(source)) + + assert sanitized == expected + assert SECRET_MARKER not in sanitized + assert "second-" not in sanitized + + +@pytest.mark.parametrize("quote", ['"', "'"]) +@pytest.mark.parametrize("escape_layers", [1, 2, 3]) +def test_sanitizer_tracks_escaped_quoted_fragments_inside_secret_value( + quote: str, + escape_layers: int, +) -> None: + """多层 slash-escaped quoted fragment 的内部逗号仍属于凭据值。""" + wrapper = "\\" * escape_layers + quote + source = ( + f"password=prefix{wrapper}{SECRET_MARKER}, second-escaped-secret" + f"{wrapper}, status=ok" + ) + + sanitized = str(sanitize_for_host(source)) + + assert sanitized == "password=***, status=ok" + assert SECRET_MARKER not in sanitized + assert "second-escaped-secret" not in sanitized + + +def test_sanitizer_redacts_unquoted_multiword_secret_in_error_summary() -> None: + """异常中的无引号多词凭据必须净化到可靠分隔符。""" + summary = summarize_error( + RuntimeError("password=alpha beta; operation=connect") + ) + + assert "alpha" not in summary + assert "beta" not in summary + assert "operation=connect" in summary + + +def test_pydantic_validation_error_is_safe_across_host_entry_points() -> None: + """Pydantic 原始输入不得从递归 sanitizer 或任一摘要入口回显。""" + with pytest.raises(ValidationError) as exc_info: + _InvalidSecretInput(api_key=SECRET_MARKER) + + error = exc_info.value + sanitized = sanitize_for_host(error) + outputs = ( + str(sanitized), + str(sanitize_for_host({"error": error})), + summarize_input(error), + summarize_result({"error": error}), + summarize_error(error), + ) + + assert all(SECRET_MARKER not in output for output in outputs) + assert sanitized == {"error_count": 1} + assert "ValidationError" in outputs[-1] + + +def test_pydantic_validation_error_excludes_dynamic_metadata() -> None: + """动态错误位置、类型、消息和上下文均不得成为宿主诊断文本。""" + with pytest.raises(ValidationError) as location_exc_info: + _DynamicSecretLocationInput( + payload={SECRET_MARKER: "not-an-integer"} + ) + with pytest.raises(ValidationError) as custom_exc_info: + _CustomSecretValidationInput(value=SECRET_MARKER) + + outputs = [] + for error in (location_exc_info.value, custom_exc_info.value): + outputs.extend( + ( + str(sanitize_for_host(error)), + str(sanitize_for_host({"error": error})), + summarize_result({"error": error}), + summarize_error(error), + ) + ) + + assert all(SECRET_MARKER not in output for output in outputs) + assert all("error_count" in output for output in outputs) + + +def test_pydantic_validation_error_count_does_not_expand_details() -> None: + """校验错误计数不得构造完整 errors 明细。""" + with pytest.raises(ValidationError) as exc_info: + _InvalidSecretInput(api_key=SECRET_MARKER) + + with patch.object( + ValidationError, + "errors", + side_effect=AssertionError("validation details must not be expanded"), + ) as mock_errors: + sanitized = sanitize_for_host(exc_info.value) + + mock_errors.assert_not_called() + assert sanitized == {"error_count": 1} + + +def test_sanitizer_type_fallback_ignores_hostile_metaclass() -> None: + """对象协议与类型名读取同时失败时仍应返回稳定占位。""" + secret_marker = "hostile-type-secret-4381" + + class _HostileMeta(type): + def __getattribute__(cls, name): + if name == "__name__": + raise RuntimeError(f"DATABASE_PASSWORD={secret_marker}") + return super().__getattribute__(name) + + class _HostileValue(metaclass=_HostileMeta): + def __str__(self) -> str: + raise RuntimeError("string conversion failed") + + escaped = False + try: + sanitized = sanitize_for_host(_HostileValue()) + except BaseException: + escaped = True + sanitized = "" + + assert escaped is False + assert str(sanitized).startswith(" None: + """未知叶对象只输出固定类型占位，不能执行无界字符串协议。""" + + class _UnsupportedLeaf: + """记录 sanitizer 是否调用第三方字符串协议。""" + + calls = 0 + class_reads = 0 + + def __getattribute__(self, name: str): + if name == "__class__": + type(self).class_reads += 1 + return object.__getattribute__(self, name) + + def __str__(self) -> str: + type(self).calls += 1 + return SECRET_MARKER + + leaf = _UnsupportedLeaf() + + sanitized = sanitize_for_host(leaf) + summary = summarize_result(leaf) + + assert _UnsupportedLeaf.calls == 0 + assert _UnsupportedLeaf.class_reads == 0 + assert sanitized == "" + assert summary == "" + assert SECRET_MARKER not in summary + + +def test_sanitizer_does_not_query_hostile_metaclass_for_dataclass_marker() -> None: + """未知叶对象的 dataclass 分派不得执行自定义 metaclass 属性协议。""" + + class _HostileMeta(type): + dataclass_reads = 0 + + def __getattribute__(cls, name: str): + if name == "__dataclass_fields__": + reads = type.__getattribute__(_HostileMeta, "dataclass_reads") + type.__setattr__(_HostileMeta, "dataclass_reads", reads + 1) + raise RuntimeError(SECRET_MARKER) + return type.__getattribute__(cls, name) + + class _UnsupportedLeaf(metaclass=_HostileMeta): + pass + + sanitized = sanitize_for_host(_UnsupportedLeaf()) + + assert type.__getattribute__(_HostileMeta, "dataclass_reads") == 0 + assert sanitized == "" + assert SECRET_MARKER not in sanitized + + +def test_sanitizer_reads_exception_args_without_custom_string_protocol() -> None: + """异常摘要保留安全参数，但不得调用异常子类的自定义字符串协议。""" + + class _HostileError(RuntimeError): + """通过字符串协议回显凭据的第三方异常。""" + + calls = 0 + class_reads = 0 + + def __getattribute__(self, name: str): + if name in ("__class__", "args"): + type(self).class_reads += 1 + return RuntimeError.__getattribute__(self, name) + + def __str__(self) -> str: + type(self).calls += 1 + return SECRET_MARKER + + error = _HostileError("operation=connect") + + summary = summarize_error(error) + + assert _HostileError.calls == 0 + assert _HostileError.class_reads == 0 + assert "operation=connect" in summary + assert SECRET_MARKER not in summary + + +def test_sanitizer_bounds_json_shaped_text_before_parsing() -> None: + """超过文本上限的 JSON 外形输入不得触发完整解析。""" + secret_marker = "oversized-json-secret-9056" + source = ( + '{"password":"' + secret_marker + '","padding":"' + "x" * 20000 + '"}' + ) + + with patch( + "app.agent.policy.sanitizer.json.loads", + side_effect=AssertionError("oversized JSON must not be parsed"), + ) as mock_loads: + sanitized = str(sanitize_for_host(source)) + + mock_loads.assert_not_called() + assert secret_marker not in sanitized + assert sanitized.endswith("") + assert len(sanitized) < 17000 + + +def test_sanitizer_fails_closed_for_oversized_json_identity_values() -> None: + """超长 JSON 无法确认对象身份时，窗口内通用值字段必须脱敏。""" + secret_marker = "oversized-setting-secret-marker" + source = json.dumps( + { + "setting_key": "API_TOKEN", + "value_preview": secret_marker, + "value": secret_marker + "x" * sanitizer_module._MAX_TEXT_CHARS, + } + ) + + with patch( + "app.agent.policy.sanitizer.json.loads", + side_effect=AssertionError("oversized JSON must not be parsed"), + ) as mock_loads: + sanitized = str(sanitize_for_host(source)) + + mock_loads.assert_not_called() + assert secret_marker not in sanitized + assert '"value_preview": ***' in sanitized + assert '"value": ***' in sanitized + assert sanitized.endswith("") + + +def test_sanitizer_handles_cyclic_command_without_raising() -> None: + """循环 LangGraph Command 必须生成有界摘要而不是破坏工具成功结果。""" + cycle = [] + command = Command(update={"state": cycle}) + cycle.append(command) + + summary = summarize_result(command, max_chars=240) + + assert len(summary) <= 240 + assert summary + + +def test_tool_result_json_fallback_warning_is_sanitized() -> None: + """结果序列化 fallback 不能把第三方异常中的凭据写入 warning。""" + + class _FallbackResult: + def __init__(self) -> None: + self.calls = 0 + + def __str__(self) -> str: + self.calls += 1 + if self.calls == 1: + raise RuntimeError(f"DATABASE_PASSWORD={SECRET_MARKER}") + return "fallback-result" + + mock_logger = MagicMock() + with patch("app.agent.tools.base.logger", mock_logger): + result = serialize_tool_result_for_agent(_FallbackResult()) + + assert result == "fallback-result" + logged = _logged_text(mock_logger) + assert SECRET_MARKER not in logged + assert "RuntimeError" in logged + + +def test_summary_helpers_bound_output_without_losing_normal_context() -> None: + """输入、结果与异常摘要应有界且保留非敏感诊断上下文。""" + payload = { + "query": "MoviePilot", + "password": SECRET_MARKER, + "body": "x" * 2000, + } + + input_summary = summarize_input(payload, max_chars=240) + result_summary = summarize_result(payload, max_chars=240) + error_summary = summarize_error( + RuntimeError(f"Authorization: Bearer {SECRET_MARKER}"), + max_chars=240, + ) + + for summary in (input_summary, result_summary, error_summary): + assert len(summary) <= 240 + assert SECRET_MARKER not in summary + assert "MoviePilot" in input_summary + + +def test_agent_tool_logs_are_sanitized_but_shadow_result_is_unchanged() -> None: + """G1 只净化宿主日志，shadow 工具返回值仍保持兼容。""" + tool = _SecretResultTool(session_id="session-1", user_id="user-1") + payload = {"token": SECRET_MARKER, "label": "visible"} + mock_logger = MagicMock() + + with patch("app.agent.tools.base.logger", mock_logger): + result = asyncio.run(tool._arun(payload=payload)) + + assert SECRET_MARKER in result + logged = _logged_text(mock_logger) + assert SECRET_MARKER not in logged + assert "visible" in logged + + +def test_direct_manager_logs_are_sanitized_but_result_is_unchanged() -> None: + """HTTP/MCP/CLI manager 与 Agent 路径使用同一 secret-safe 日志语义。""" + tool = _SecretResultTool(session_id="session-1", user_id="user-1") + manager = MoviePilotToolsManager(is_admin=True) + manager.tools = [tool] + payload = {"cookie": SECRET_MARKER, "label": "visible"} + mock_logger = MagicMock() + + with ( + patch("app.agent.tools.manager.logger", mock_logger), + patch("app.agent.policy.orchestrator.logger", mock_logger), + ): + result = asyncio.run(manager.call_tool(tool.name, {"payload": payload})) + + assert SECRET_MARKER in result + logged = _logged_text(mock_logger) + assert SECRET_MARKER not in logged + assert "visible" in logged + + +def test_direct_secret_setting_result_is_returned_without_entering_policy_logs() -> None: + """管理员显式读取凭据时，原值只返回调用方，不进入宿主策略日志。""" + tool = QuerySystemSettingsTool(session_id="session-1", user_id="admin") + tool.set_agent_context({"is_admin": True}) + manager = MoviePilotToolsManager(is_admin=True) + manager.tools = [tool] + mock_logger = MagicMock() + + with ( + patch.object( + QuerySystemSettingsTool, + "_load_setting_value", + return_value=SECRET_MARKER, + ), + patch("app.agent.tools.manager.logger", mock_logger), + patch("app.agent.policy.orchestrator.logger", mock_logger), + ): + result = asyncio.run( + manager.call_tool( + tool.name, + {"setting_key": "API_TOKEN", "show_secrets": True}, + ) + ) + + assert SECRET_MARKER in result + logged = _logged_text(mock_logger) + assert SECRET_MARKER not in logged + assert '"value": "***"' in logged + assert '"value_preview": "***"' in logged + + +def test_oversized_direct_secret_setting_result_stays_out_of_policy_logs() -> None: + """超长管理员读取结果仍只返回调用方，不进入 direct 策略回执。""" + secret_marker = "oversized-direct-secret-marker" + secret_value = secret_marker + "x" * sanitizer_module._MAX_TEXT_CHARS + tool = QuerySystemSettingsTool(session_id="session-1", user_id="admin") + tool.set_agent_context({"is_admin": True}) + manager = MoviePilotToolsManager(is_admin=True) + manager.tools = [tool] + mock_logger = MagicMock() + + with ( + patch.object( + QuerySystemSettingsTool, + "_load_setting_value", + return_value=secret_value, + ), + patch("app.agent.tools.manager.logger", mock_logger), + patch("app.agent.policy.orchestrator.logger", mock_logger), + ): + result = asyncio.run( + manager.call_tool( + tool.name, + {"setting_key": "API_TOKEN", "show_secrets": True}, + ) + ) + + assert secret_marker in result + logged = _logged_text(mock_logger) + assert secret_marker not in logged + assert '"value_preview": ***' in logged + + +def test_tool_error_does_not_echo_secret_to_logs_or_result() -> None: + """异常消息中的凭据既不能进日志，也不能回显给模型或 direct 调用方。""" + tool = _SecretErrorTool(session_id="session-1", user_id="user-1") + payload = {"token": SECRET_MARKER} + mock_logger = MagicMock() + + with patch("app.agent.tools.base.logger", mock_logger): + result = asyncio.run(tool._arun(payload=payload)) + + assert SECRET_MARKER not in result + assert SECRET_MARKER not in _logged_text(mock_logger) + assert "***" in result diff --git a/tests/test_agent_tool_streaming.py b/tests/test_agent_tool_streaming.py index fd665b846..0ffb127cd 100644 --- a/tests/test_agent_tool_streaming.py +++ b/tests/test_agent_tool_streaming.py @@ -149,6 +149,24 @@ class TestAgentToolStreaming: assert buffered_message == "处理中：\n\n（执行了 2 次搜索，读取了 2 个文件）\n\n继续分析" + def test_tool_summary_does_not_retain_secret_arguments(self): + """流式聚合状态只能保留递归脱敏后的工具参数。""" + secret_marker = "stream-secret-marker-1947" + handler = StreamingHandler() + + handler.record_tool_call( + tool_name="execute_command", + tool_message=f"Authorization: Bearer {secret_marker}", + tool_kwargs={ + "command": f"curl -H 'Authorization: Bearer {secret_marker}'", + "headers": {"X-API-Key": secret_marker}, + }, + ) + + pending_state = str(handler._pending_tool_stats) + assert secret_marker not in pending_state + assert "***" in pending_state + def test_non_verbose_tool_summary_counts_subagents(self): """校验非详细模式统计子代理调用次数。""" async def _run():