mirror of
https://github.com/qingchencloud/clawpanel.git
synced 2026-05-29 04:10:00 +08:00
feat(hermes): Batch 3 §K - 多模态图片上传(chat attach + 拖拽 + 粘贴 + base64)
Hermes Agent 已支持 OpenAI 多模态格式(tests/gateway/test_api_server_multimodal.py),
ClawPanel 前端补 attach UI 即可对接 Claude 3.5 / GPT-4o / Gemini 等视觉模型。
## 后端
- HermesAttachment 结构:{ kind, mime, name?, data_base64 }
- build_multimodal_input(text, attachments) 把 text 转成
[{type:"text",text}, {type:"image_url",image_url:{url:"data:..;base64,.."}}, ...]
- hermes_agent_run 加 attachments 参数(向后兼容:未传时走原 string input)
## 前端
- tauri-api.js: hermesAgentRun 加 attachments 参数
- chat-store sendMessage:
· 允许 text 空但有 attachments 也能发
· user 消息记录 attachments[].dataUrl 用于气泡内渲染
· isTauriRuntime 时 await api.hermesAgentRun(...,atts)
- chat.js 渲染层:
· renderMessage 在内容前插入 .hm-chat-msg-attachments + .hm-chat-msg-image(点击放大)
· 输入栏前面增加 .hm-chat-attach-preview 预览条(缩略图 + 文件名 + × 移除)
· 输入框左侧加 paperclip attach 按钮 + 隐藏 <input type="file" accept="image/*" multiple>
· 发送按钮 disabled 条件改为 (!text && !attachments)
- chat.js 交互:
· fileToBase64 用 FileReader 转纯 base64
· addAttachmentFromFile 校验 image/* + 10MB + 最多 5 张
· attach 按钮 click → 触发文件选择器
· 拖拽到输入区 → dragover 高亮 + drop 加附件
· 粘贴图片 → clipboardData 读 image 文件
· 移除按钮 splice
- chat-store.sendMessage 后清 pendingAttachments
## 限制
- 单图最大 10 MB(base64 后约 13 MB)
- 一次最多 5 张
- 超限 toast 友好提示
- 非图片格式拒收
## CSS
- .hm-chat-attach-btn / hover / disabled
- .hm-chat-attach-preview / chip / chip-name / chip-remove
- .hm-chat-input-wrap--dragover(拖拽虚线高亮)
- .hm-chat-msg-attachments / msg-image / msg-image--zoom(点击放大模式)
## i18n
- engine.chatAttach / chatAttachRemove / chatAttachOnlyImage
- chatAttachTooBig / chatAttachTooMany / chatAttachReadFailed
- 3 语言(zh-CN/en/zh-TW)
## 注意
- Web 模式(dev-api 走 SSE)暂不支持 attachments 透传(hermes_agent_run_stream 没改),
原因:Web 模式当前 chat 是 stream-only 路径,需要单独改 dev-api 的 hermes_agent_run_stream handler
- 桌面端 Tauri 模式开箱可用
- 累计变动:6 个文件,~120 行新代码,6 个 i18n 键
- cargo check ✓ + npm build ✓
This commit is contained in:
@@ -3832,6 +3832,36 @@ pub async fn hermes_session_export(session_id: String) -> Result<Value, String>
|
||||
resp.json::<Value>().await.map_err(|e| format!("解析 JSON 失败: {e}"))
|
||||
}
|
||||
|
||||
/// Batch 3 §K: 多模态附件结构
|
||||
///
|
||||
/// 前端传过来的附件描述(图片用 base64 直传)。
|
||||
/// 支持 kind="image"(暂时只接图片,文件附件留作后续)。
|
||||
#[derive(serde::Deserialize, Clone)]
|
||||
pub struct HermesAttachment {
|
||||
pub kind: String,
|
||||
pub mime: String,
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
/// base64 编码的内容(不含 data:image/...,base64, 前缀,仅纯 base64)
|
||||
pub data_base64: String,
|
||||
}
|
||||
|
||||
/// 构造 OpenAI 多模态 content:[{type:"text"}, {type:"image_url"}, ...]
|
||||
fn build_multimodal_input(text: &str, attachments: &[HermesAttachment]) -> Value {
|
||||
let mut parts: Vec<Value> = Vec::new();
|
||||
parts.push(serde_json::json!({ "type": "text", "text": text }));
|
||||
for a in attachments {
|
||||
if a.kind == "image" {
|
||||
let url = format!("data:{};base64,{}", a.mime, a.data_base64);
|
||||
parts.push(serde_json::json!({
|
||||
"type": "image_url",
|
||||
"image_url": { "url": url },
|
||||
}));
|
||||
}
|
||||
}
|
||||
Value::Array(parts)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn hermes_agent_run(
|
||||
app: tauri::AppHandle,
|
||||
@@ -3839,6 +3869,7 @@ pub async fn hermes_agent_run(
|
||||
session_id: Option<String>,
|
||||
conversation_history: Option<Value>,
|
||||
instructions: Option<String>,
|
||||
attachments: Option<Vec<HermesAttachment>>,
|
||||
) -> Result<String, String> {
|
||||
let gw_url = hermes_gateway_url();
|
||||
let runs_url = format!("{gw_url}/v1/runs");
|
||||
@@ -3862,7 +3893,12 @@ pub async fn hermes_agent_run(
|
||||
key
|
||||
};
|
||||
|
||||
let mut payload = serde_json::json!({ "input": input });
|
||||
// Batch 3 §K: 有 attachments 时 input 改成多模态格式
|
||||
let mut payload = if let Some(atts) = attachments.as_ref().filter(|v| !v.is_empty()) {
|
||||
serde_json::json!({ "input": build_multimodal_input(&input, atts) })
|
||||
} else {
|
||||
serde_json::json!({ "input": input })
|
||||
};
|
||||
if let Some(sid) = &session_id {
|
||||
payload["session_id"] = Value::String(sid.clone());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user