feat(agent): mark and propagate voice input metadata in agent messages; clarify terminal tool usage in prompts

- Add `has_audio_input` flag to agent message handling and propagate through processing pipeline
- Structure agent input payloads to include `input.mode` and `input.transcribed` for voice messages
- Update prompts and tool descriptions to clarify that `send_voice_message` and `ask_user_choice` are terminal tools and should not be followed by redundant text replies
- Enhance tests to cover voice input metadata propagation and prompt updates
This commit is contained in:
jxxghp
2026-05-31 18:04:02 +08:00
parent 13b2163788
commit 855681ff35
11 changed files with 139 additions and 10 deletions

View File

@@ -242,6 +242,7 @@ class AgentImageSupportTest(unittest.TestCase):
handle_ai_message.assert_called_once()
self.assertEqual(handle_ai_message.call_args.kwargs["text"], "帮我推荐一部电影")
self.assertTrue(handle_ai_message.call_args.kwargs["has_audio_input"])
self.assertNotIn("reply_with_voice", handle_ai_message.call_args.kwargs)
def test_file_message_routes_to_agent_even_when_global_agent_is_disabled(self):
@@ -390,8 +391,36 @@ class AgentImageSupportTest(unittest.TestCase):
self.assertIsInstance(content, list)
payload = json.loads(content[0]["text"])
self.assertEqual(payload["message"], "帮我总结这个文件")
self.assertEqual(payload["input"]["mode"], "text")
self.assertFalse(payload["input"]["transcribed"])
self.assertEqual(payload["files"][0]["local_path"], "/tmp/report.txt")
def test_agent_process_marks_voice_input_in_structured_json(self):
"""语音输入应在结构化消息中标记为转写来源。"""
agent = MoviePilotAgent(
session_id="session-1",
user_id="user-1",
channel=MessageChannel.Telegram.value,
source="telegram-test",
username="tester",
)
with patch(
"app.agent.memory.memory_manager.get_agent_messages", return_value=[]
), patch.object(agent, "_execute_agent", new_callable=AsyncMock) as execute_agent:
asyncio.run(
agent.process(
"帮我推荐一部电影",
has_audio_input=True,
)
)
messages = execute_agent.await_args.args[0]
payload = json.loads(messages[-1].content[0]["text"])
self.assertEqual(payload["message"], "帮我推荐一部电影")
self.assertEqual(payload["input"]["mode"], "voice")
self.assertTrue(payload["input"]["transcribed"])
def test_llm_supports_image_input_respects_explicit_override(self):
with patch.object(settings, "LLM_SUPPORT_IMAGE_INPUT", False):
self.assertFalse(LLMHelper.supports_image_input())
@@ -447,6 +476,29 @@ class AgentImageSupportTest(unittest.TestCase):
"/tmp/image_1.jpg",
)
def test_handle_ai_message_forwards_voice_input_to_agent_manager(self):
"""AI消息入队时应保留语音输入标记。"""
chain = MessageChain()
with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object(
chain, "_get_or_create_session_id", return_value="session-1"
), patch(
"app.chain.message.agent_manager.process_message", new_callable=AsyncMock
) as process_message, patch(
"app.chain.message.asyncio.run_coroutine_threadsafe",
side_effect=lambda coro, _loop: (coro.close(), Mock())[1],
):
chain._handle_ai_message(
text="帮我推荐一部电影",
channel=MessageChannel.Telegram,
source="telegram-test",
userid="10001",
username="tester",
has_audio_input=True,
)
self.assertTrue(process_message.call_args.kwargs["has_audio_input"])
def test_slack_images_use_authenticated_data_url_download(self):
chain = MessageChain()