GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-07-08 00:01:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(agent): add voice message support with TTS/STT for Telegram and WeChat

Browse Source 
- Integrate voice message handling: detect and extract audio references from Telegram and WeChat messages, route to agent with voice reply preference.
- Add voice provider abstraction and OpenAI-based TTS/STT implementation.
- Implement agent tool `send_voice_message` for generating and sending voice replies, with fallback to text if voice is unavailable.
- Extend agent prompt and context to support voice reply instructions.
- Update notification and message schemas to support audio fields.
- Add Telegram and WeChat voice sending logic, including audio file conversion and temporary media upload for WeChat.
- Add tests for voice helper and agent voice routing.
This commit is contained in:
jxxghp
2026-04-12 12:30:02 +08:00
parent 9dababbcfd
commit e5f97cd299
17 changed files with 945 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/agent/__init__.py
View File

@@ -129,6 +129,8 @@ class MoviePilotAgent:
self.channel = channel
self.source = source
self.username = username
self.reply_with_voice = False
self._tool_context: Dict[str, object] = {}
# 流式token管理
self.stream_handler = StreamingHandler()
@@ -151,6 +153,8 @@ class MoviePilotAgent:
"""
if self.is_background:
return False
if self.reply_with_voice:
return False
# 啰嗦模式下始终需要流式输出来捕获工具调用前的 Agent 文字
if settings.AI_AGENT_VERBOSE:
return True
@@ -214,6 +218,7 @@ class MoviePilotAgent:
source=self.source,
username=self.username,
stream_handler=self.stream_handler,
agent_context=self._tool_context,
)
def _create_agent(self, streaming: bool = False):
@@ -223,7 +228,10 @@ class MoviePilotAgent:
"""
try:
# 系统提示词
system_prompt = prompt_manager.get_agent_prompt(channel=self.channel)
system_prompt = prompt_manager.get_agent_prompt(
channel=self.channel,
prefer_voice_reply=self.reply_with_voice,
)
# LLM 模型（用于 agent 执行）
llm = self._initialize_llm(streaming=streaming)
@@ -281,6 +289,11 @@ class MoviePilotAgent:
logger.info(
f"Agent推理: session_id={self.session_id}, input={message}, images={len(images) if images else 0}"
)
self._tool_context = {
"incoming_voice": self.reply_with_voice,
"user_reply_sent": False,
"reply_mode": None,
}
# 获取历史消息
messages = memory_manager.get_agent_messages(
@@ -417,7 +430,7 @@ class MoviePilotAgent:
# 流式输出未能发送全部内容（发送失败等）
# 通过常规方式发送剩余内容
remaining_text = await self.stream_handler.take()
if remaining_text:
if remaining_text and not self._tool_context.get("user_reply_sent"):
await self.send_agent_message(remaining_text)
elif streamed_text:
# 流式输出已发送全部内容，但未记录到数据库，补充保存消息记录
@@ -447,7 +460,7 @@ class MoviePilotAgent:
final_text = text.strip()
break
if final_text:
if final_text and not self._tool_context.get("user_reply_sent"):
if self.is_background:
# 后台任务仅广播最终回复，带标题
await self.send_agent_message(
@@ -534,6 +547,7 @@ class _MessageTask:
channel: Optional[str] = None
source: Optional[str] = None
username: Optional[str] = None
reply_with_voice: bool = False
class AgentManager:
@@ -599,6 +613,7 @@ class AgentManager:
channel: str = None,
source: str = None,
username: str = None,
reply_with_voice: bool = False,
) -> str:
"""
处理用户消息：将消息放入会话队列，按顺序依次处理。
@@ -612,6 +627,7 @@ class AgentManager:
channel=channel,
source=source,
username=username,
reply_with_voice=reply_with_voice,
)
# 获取或创建会话队列
@@ -709,6 +725,7 @@ class AgentManager:
agent.source = task.source
if task.username:
agent.username = task.username
agent.reply_with_voice = task.reply_with_voice
return await agent.process(task.message, images=task.images)

1
app/agent/prompt/Agent Prompt.txt
View File

@@ -21,6 +21,7 @@ Core Capabilities:
- Include key details (year, rating, resolution) but do NOT over-explain.
- Do not stop for approval on read-only operations. Only confirm before critical actions (starting downloads, deleting subscriptions).
- If the current channel supports image sending and an image would materially help, you may use the `send_message` tool with `image_url` to send it.
- Voice replies: {voice_reply_spec}
- NOT a coding assistant. Do not offer code snippets.
- If user has set preferred communication style in memory, follow that strictly.
</communication>

23
app/agent/prompt/__init__.py
View File

@@ -50,10 +50,13 @@ class PromptManager:
logger.error(f"加载提示词失败: {prompt_name}, 错误: {e}")
raise
def get_agent_prompt(self, channel: str = None) -> str:
def get_agent_prompt(
self, channel: str = None, prefer_voice_reply: bool = False
) -> str:
"""
获取智能体提示词
:param channel: 消息渠道（Telegram、微信、Slack等）
:param prefer_voice_reply: 是否优先使用语音回复
:return: 提示词内容
"""
# 基础提示词
@@ -87,12 +90,16 @@ class PromptManager:
# MoviePilot系统信息
moviepilot_info = self._get_moviepilot_info()
voice_reply_spec = self._generate_voice_reply_instructions(
prefer_voice_reply=prefer_voice_reply
)
# 始终替换占位符，避免后续 .format() 时因残留花括号报 KeyError
base_prompt = base_prompt.format(
markdown_spec=markdown_spec,
verbose_spec=verbose_spec,
moviepilot_info=moviepilot_info,
voice_reply_spec=voice_reply_spec,
)
return base_prompt
@@ -166,6 +173,20 @@ class PromptManager:
instructions.append("- Links: Paste URLs directly as text.")
return "\n".join(instructions)
@staticmethod
def _generate_voice_reply_instructions(prefer_voice_reply: bool) -> str:
if not prefer_voice_reply:
return (
"- Voice replies: Use normal text replies by default. "
"Only call `send_voice_message` when spoken playback is clearly better than plain text."
)
return (
"- Current message context: The user sent a voice message.\n"
"- Reply preference: Prioritize calling `send_voice_message` for the main user-facing reply.\n"
"- Fallback: If voice is unavailable on the current channel, `send_voice_message` will fall back to text.\n"
"- Do not repeat the same full reply again after calling `send_voice_message`."
)
def clear_cache(self):
"""
清空缓存

7
app/agent/tools/base.py
View File

@@ -31,6 +31,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
_username: Optional[str] = PrivateAttr(default=None)
_stream_handler: Optional[StreamingHandler] = PrivateAttr(default=None)
_require_admin: bool = PrivateAttr(default=False)
_agent_context: dict = PrivateAttr(default_factory=dict)
def __init__(self, session_id: str, user_id: str, **kwargs):
super().__init__(**kwargs)
@@ -142,6 +143,12 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
"""
self._stream_handler = stream_handler
def set_agent_context(self, agent_context: Optional[dict]):
"""
设置与当前 Agent 共享的上下文。
"""
self._agent_context = agent_context or {}
async def _check_permission(self) -> Optional[str]:
"""
检查用户权限：

5
app/agent/tools/factory.py
View File

@@ -30,6 +30,7 @@ from app.agent.tools.impl.search_torrents import SearchTorrentsTool
from app.agent.tools.impl.get_search_results import GetSearchResultsTool
from app.agent.tools.impl.search_web import SearchWebTool
from app.agent.tools.impl.send_message import SendMessageTool
from app.agent.tools.impl.send_voice_message import SendVoiceMessageTool
from app.agent.tools.impl.query_schedulers import QuerySchedulersTool
from app.agent.tools.impl.run_scheduler import RunSchedulerTool
from app.agent.tools.impl.query_workflows import QueryWorkflowsTool
@@ -72,6 +73,7 @@ class MoviePilotToolFactory:
source: str = None,
username: str = None,
stream_handler: Callable = None,
agent_context: dict = None,
) -> List[MoviePilotTool]:
"""
创建MoviePilot工具列表
@@ -117,6 +119,7 @@ class MoviePilotToolFactory:
QueryTransferHistoryTool,
TransferFileTool,
SendMessageTool,
SendVoiceMessageTool,
QuerySchedulersTool,
RunSchedulerTool,
QueryWorkflowsTool,
@@ -138,6 +141,7 @@ class MoviePilotToolFactory:
tool = ToolClass(session_id=session_id, user_id=user_id)
tool.set_message_attr(channel=channel, source=source, username=username)
tool.set_stream_handler(stream_handler=stream_handler)
tool.set_agent_context(agent_context=agent_context)
tools.append(tool)
# 加载插件提供的工具
@@ -161,6 +165,7 @@ class MoviePilotToolFactory:
channel=channel, source=source, username=username
)
tool.set_stream_handler(stream_handler=stream_handler)
tool.set_agent_context(agent_context=agent_context)
tools.append(tool)
plugin_tools_count += 1
logger.debug(

96
app/agent/tools/impl/send_voice_message.py Normal file
View File

@@ -0,0 +1,96 @@
"""发送语音消息工具。"""
import asyncio
from typing import Optional, Type
from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool, ToolChain
from app.core.config import settings
from app.helper.voice import VoiceHelper
from app.helper.service import ServiceConfigHelper
from app.log import logger
from app.schemas import Notification, NotificationType
from app.schemas.types import MessageChannel
class SendVoiceMessageInput(BaseModel):
"""发送语音消息工具输入。"""
explanation: str = Field(
...,
description="Clear explanation of why a voice reply is the best fit in the current context",
)
message: str = Field(
...,
description="The spoken content to send back to the user",
)
class SendVoiceMessageTool(MoviePilotTool):
name: str = "send_voice_message"
description: str = (
"Send a voice reply to the current user. Prefer this when the user sent a voice message "
"or when spoken playback is more natural. On channels without voice support or when TTS "
"is unavailable, it automatically falls back to sending the same content as plain text."
)
args_schema: Type[BaseModel] = SendVoiceMessageInput
require_admin: bool = False
def get_tool_message(self, **kwargs) -> Optional[str]:
message = kwargs.get("message") or ""
if len(message) > 40:
message = message[:40] + "..."
return f"正在发送语音回复: {message}"
def _supports_real_voice_reply(self) -> bool:
channel = self._channel or ""
if channel == MessageChannel.Telegram.value:
return True
if channel != MessageChannel.Wechat.value:
return False
for config in ServiceConfigHelper.get_notification_configs():
if config.name != self._source:
continue
return (config.config or {}).get("WECHAT_MODE", "app") != "bot"
return False
async def run(self, message: str, **kwargs) -> str:
if not message:
return "语音回复内容不能为空"
voice_path = None
used_voice = False
channel = self._channel or ""
if self._supports_real_voice_reply() and VoiceHelper.is_available("tts"):
voice_file = await asyncio.to_thread(VoiceHelper.synthesize_speech, message)
if voice_file:
voice_path = str(voice_file)
used_voice = True
logger.info(
"执行工具: %s, channel=%s, use_voice=%s, text_len=%s",
self.name,
channel,
used_voice,
len(message),
)
await ToolChain().async_post_message(
Notification(
channel=self._channel,
source=self._source,
mtype=NotificationType.Agent,
userid=self._user_id,
username=self._username,
text=message,
voice_path=voice_path,
voice_caption=message if settings.AI_VOICE_REPLY_WITH_TEXT else None,
)
)
self._agent_context["user_reply_sent"] = True
self._agent_context["reply_mode"] = "voice" if used_voice else "text_fallback"
if used_voice:
return "语音回复已发送"
return "当前未使用语音通道，已自动回退为文字回复"