feat: 新增图文上下文同步

app/services/chat/message_converter.py

@@ -1,9 +1,11 @@
 # app/services/chat/message_converter.py
 
 from abc import ABC, abstractmethod
+import re
 from typing import Any, Dict, List, Optional
 
 SUPPORTED_ROLES = ["user", "model", "system"]
+IMAGE_URL_PATTERN = r'(https://img\.picgo\.net/[^\s)\]\"\']+)'
 
 
 class MessageConverter(ABC):
@@ -29,6 +31,58 @@ def _convert_image(image_url: str) -> Dict[str, Any]:
     }
 
 
+def _convert_image_to_base64(url: str) -> str:
+    """
+    将图片URL转换为base64编码
+    Args:
+        url: 图片URL
+    Returns:
+        str: base64编码的图片数据
+    """
+    import requests
+    import base64
+    response = requests.get(url)
+    if response.status_code == 200:
+        # 将图片内容转换为base64
+        img_data = base64.b64encode(response.content).decode('utf-8')
+        return img_data
+    else:
+        raise Exception(f"Failed to fetch image: {response.status_code}")
+
+
+def _process_text_with_image(text: str) -> List[Dict[str, Any]]:
+    """
+    处理可能包含图片URL的文本，提取图片并转换为base64
+
+    Args:
+        text: 可能包含图片URL的文本
+
+    Returns:
+        List[Dict[str, Any]]: 包含文本和图片的部分列表
+    """
+    parts = []
+    img_url_match = re.search(IMAGE_URL_PATTERN, text)
+    if img_url_match:
+        # 提取URL
+        img_url = img_url_match.group(1)
+        # 将URL对应的图片转换为base64
+        try:
+            base64_data = _convert_image_to_base64(img_url)
+            parts.append({
+                "inlineData": {
+                    "mimeType": "image/jpeg",
+                    "data": base64_data
+                }
+            })
+        except Exception as e:
+            # 如果转换失败，回退到文本模式
+            parts.append({"text": text})
+    else:
+        # 没有图片URL，作为纯文本处理
+        parts.append({"text": text})
+    return parts
+
+
 class OpenAIMessageConverter(MessageConverter):
     """OpenAI消息格式转换器"""
 
@@ -49,9 +103,18 @@ class OpenAIMessageConverter(MessageConverter):
                         role = "model"
 
             parts = []
-            if isinstance(msg["content"], str) and msg["content"]:
+            # 特别处理assistant的消息，按\n\n分割
+            if role == "assistant" and isinstance(msg["content"], str) and msg["content"]:
+                # 按\n\n分割消息
+                content_parts = msg["content"].split("\n\n")
+                for part in content_parts:
+                    if not part.strip():  # 跳过空内容
+                        continue
+                    # 处理可能包含图片的文本
+                    parts.extend(_process_text_with_image(part))
+            elif isinstance(msg["content"], str) and msg["content"]:
                 # 请求 gemini 接口时如果包含 content 字段但内容为空时会返回 400 错误，所以需要判断是否为空并移除
-                parts.append({"text": msg["content"]})
+                parts.extend(_process_text_with_image(msg["content"]))
             elif isinstance(msg["content"], list):
                 for content in msg["content"]:
                     if isinstance(content, str) and content:
@@ -76,4 +139,4 @@ class OpenAIMessageConverter(MessageConverter):
                 "parts": system_instruction_parts,
             }
         )
-        return converted_messages, system_instruction
+        return converted_messages, system_instruction

app/services/chat/response_handler.py

@@ -207,7 +207,7 @@ def _extract_image_data(part: dict) -> str:
     bytes_data = base64.b64decode(base64_data)
     upload_response = image_uploader.upload(bytes_data,filename)
     if upload_response.success:
-        text = f"\n![image]({upload_response.data.url})\n"
+        text = f"![image]({upload_response.data.url})"
     else:
         text = ""
     return text