From ecca5da8abee085cd221ebd3860c1d04ecef9a8d Mon Sep 17 00:00:00 2001 From: JOJO <1498581755@qq.com> Date: Mon, 27 Apr 2026 20:11:21 +0800 Subject: [PATCH] fix: harden multimodal handling and sanitize mcp action results --- prompts/main_system.txt | 1 + server/chat_flow_tool_loop.py | 15 ++++- static/src/stores/model.ts | 9 +++ test/test_api_client_multimodal_sanitize.py | 75 +++++++++++++++++++++ utils/api_client.py | 52 +++++++++++--- 5 files changed, 141 insertions(+), 11 deletions(-) diff --git a/prompts/main_system.txt b/prompts/main_system.txt index 5fb4a44..81ccfb3 100644 --- a/prompts/main_system.txt +++ b/prompts/main_system.txt @@ -122,6 +122,7 @@ - 当你不确定当前有哪些 MCP 工具可用时，先调用 `list_mcp_servers` - 需要刷新远程工具缓存时，可使用 `list_mcp_servers` 并传 `refresh=true` - 调用具体 MCP 工具时，使用工具列表中的 `mcp__...` 别名 +- **纯文本模型硬性约束**：调用工具（尤其是 MCP 工具）时，禁止调用任何可能返回多模态结果（图片/视频/音频）的工具；若用户需求依赖此类结果，先明确告知当前模型无法查看，并建议切换到支持多模态的模型后再执行。 --- diff --git a/server/chat_flow_tool_loop.py b/server/chat_flow_tool_loop.py index eab2e8f..2a04a4a 100644 --- a/server/chat_flow_tool_loop.py +++ b/server/chat_flow_tool_loop.py @@ -673,10 +673,20 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie 'entries': after_entries } + mcp_parsed = None + update_result_data = result_data + if isinstance(function_name, str) and function_name.startswith("mcp__") and isinstance(result_data, dict): + try: + mcp_parsed = extract_mcp_content_for_context(result_data) + update_result_data = mcp_parsed.get("sanitized_payload") or result_data + except Exception: + mcp_parsed = None + update_result_data = result_data + update_payload = { 'id': tool_display_id, 'status': action_status, - 'result': result_data, + 'result': update_result_data, 'preparing_id': tool_call_id, 'conversation_id': conversation_id } @@ -702,7 +712,8 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie tool_media_refs = None if isinstance(result_data, dict): if isinstance(function_name, str) and function_name.startswith("mcp__"): - mcp_parsed = extract_mcp_content_for_context(result_data) + if not isinstance(mcp_parsed, dict): + mcp_parsed = extract_mcp_content_for_context(result_data) tool_result_content = ( mcp_parsed.get("text") or format_tool_result_for_context(function_name, result_data, tool_result) diff --git a/static/src/stores/model.ts b/static/src/stores/model.ts index 3059863..ee5fbb3 100644 --- a/static/src/stores/model.ts +++ b/static/src/stores/model.ts @@ -40,6 +40,9 @@ export const useModelStore = defineStore('model', { key: 'kimi', label: 'Kimi-k2', description: '综合能力较强', + multimodal: 'none', + supportsImage: false, + supportsVideo: false, fastOnly: false, supportsThinking: true }, @@ -47,6 +50,9 @@ export const useModelStore = defineStore('model', { key: 'deepseek', label: 'Deepseek-V3.2', description: '数学能力较强', + multimodal: 'none', + supportsImage: false, + supportsVideo: false, fastOnly: false, supportsThinking: true }, @@ -64,6 +70,9 @@ export const useModelStore = defineStore('model', { key: 'minimax-m2.5', label: 'MiniMax-M2.5', description: '仅深度思考，超长上下文', + multimodal: 'none', + supportsImage: false, + supportsVideo: false, fastOnly: false, supportsThinking: true, deepOnly: true diff --git a/test/test_api_client_multimodal_sanitize.py b/test/test_api_client_multimodal_sanitize.py index cfecd9f..fdeffe4 100644 --- a/test/test_api_client_multimodal_sanitize.py +++ b/test/test_api_client_multimodal_sanitize.py @@ -6,6 +6,28 @@ from utils.api_client import DeepSeekClient class APIClientMultimodalSanitizeTest(unittest.TestCase): + def test_text_only_model_adds_notice_when_only_media_parts(self): + client = DeepSeekClient(web_mode=True) + client.model_key = "DeepSeek-V4-Pro High" # custom_models.json: multimodal=none + + messages = [ + { + "role": "tool", + "name": "mcp__playwright__browser_take_screenshot", + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}, + ], + } + ] + + sanitized = client._sanitize_messages_for_model_capability(messages) + self.assertEqual(len(sanitized), 1) + self.assertEqual( + sanitized[0].get("content"), + "当前模型无查看图片/视频能力，无法返回结果", + sanitized[0], + ) + def test_text_only_model_strips_image_parts_to_text(self): client = DeepSeekClient(web_mode=True) client.model_key = "DeepSeek-V4-Pro High" # custom_models.json: multimodal=none @@ -25,6 +47,35 @@ class APIClientMultimodalSanitizeTest(unittest.TestCase): self.assertEqual(len(sanitized), 1) self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0]) self.assertIn("截图成功", sanitized[0].get("content") or "", sanitized[0]) + self.assertIn( + "当前模型无查看图片/视频能力，无法返回结果", + sanitized[0].get("content") or "", + sanitized[0], + ) + + def test_builtin_text_only_model_strips_media_parts(self): + client = DeepSeekClient(web_mode=True) + client.model_key = "deepseek" # 内置模型: multimodal=none + + messages = [ + { + "role": "tool", + "name": "view_image", + "content": [ + {"type": "text", "text": "已尝试查看图片"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}, + ], + } + ] + + sanitized = client._sanitize_messages_for_model_capability(messages) + self.assertEqual(len(sanitized), 1) + self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0]) + self.assertIn( + "当前模型无查看图片/视频能力，无法返回结果", + sanitized[0].get("content") or "", + sanitized[0], + ) def test_multimodal_model_keeps_image_parts(self): client = DeepSeekClient(web_mode=True) @@ -49,6 +100,30 @@ class APIClientMultimodalSanitizeTest(unittest.TestCase): sanitized[0], ) + def test_profile_multimodal_fallback_works_when_model_key_missing(self): + client = DeepSeekClient(web_mode=True) + client.model_key = "" # 模拟未知/缺失 key，仅走 profile 兜底 + client.model_multimodal = "none" + + messages = [ + { + "role": "tool", + "content": [ + {"type": "text", "text": "测试"}, + {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,abc"}}, + ], + } + ] + + sanitized = client._sanitize_messages_for_model_capability(messages) + self.assertEqual(len(sanitized), 1) + self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0]) + self.assertIn( + "当前模型无查看图片/视频能力，无法返回结果", + sanitized[0].get("content") or "", + sanitized[0], + ) + if __name__ == "__main__": unittest.main() diff --git a/utils/api_client.py b/utils/api_client.py index 8160bb9..9ecb330 100644 --- a/utils/api_client.py +++ b/utils/api_client.py @@ -69,6 +69,7 @@ class DeepSeekClient: self.api_key = self.fast_api_config["api_key"] self.model_id = self.fast_api_config["model_id"] self.model_key = None # 由宿主终端注入，便于做模型兼容处理 + self.model_multimodal = "none" # 由模型 profile 注入，model_key 不可用时用于能力兜底 self.project_path: Optional[str] = None # 每个任务的独立状态 self.current_task_first_call = True # 当前任务是否是第一次调用 @@ -317,6 +318,15 @@ class DeepSeekClient: } return [merged] + messages[idx:] + @staticmethod + def _normalize_multimodal_capability(value: Any) -> str: + text = str(value or "none").strip().lower() + if text == "video,image": + return "image,video" + if text in {"none", "image", "video", "image,video"}: + return text + return "none" + @staticmethod def _sanitize_content_for_capability( content: Any, @@ -330,6 +340,8 @@ class DeepSeekClient: kept_parts: List[Dict[str, Any]] = [] text_segments: List[str] = [] + dropped_media = False + unsupported_media_notice = "当前模型无查看图片/视频能力，无法返回结果" for part in content: if not isinstance(part, dict): @@ -348,10 +360,14 @@ class DeepSeekClient: if ctype == "image_url": if supports_image: kept_parts.append(part) + else: + dropped_media = True continue if ctype == "video_url": if supports_video: kept_parts.append(part) + else: + dropped_media = True continue # 其他未知/暂不支持类型统一忽略（纯文本模型下只保留 text） @@ -359,7 +375,14 @@ class DeepSeekClient: # 纯文本模型：统一回退为字符串，规避供应商对 content part 类型的严格校验 if not supports_image and not supports_video: - return "\n".join(seg for seg in text_segments if seg).strip() + text_payload = "\n".join(seg for seg in text_segments if seg).strip() + if dropped_media: + if text_payload: + if unsupported_media_notice in text_payload: + return text_payload + return f"{text_payload}\n\n{unsupported_media_notice}" + return unsupported_media_notice + return text_payload # 半多模态模型（例如只支持图片）：尽量保留可支持的 part if kept_parts: @@ -370,16 +393,25 @@ class DeepSeekClient: def _sanitize_messages_for_model_capability(self, messages: List[Dict]) -> List[Dict]: model_key = str(self.model_key or "").strip() - if not model_key: - return messages + supports_image = False + supports_video = False + capability_source = "profile_fallback" - try: - from config.model_profiles import model_supports_image, model_supports_video - except Exception: - return messages + if model_key: + try: + from config.model_profiles import get_model_capabilities + caps = get_model_capabilities(model_key) + supports_image = bool(caps.get("supports_image")) + supports_video = bool(caps.get("supports_video")) + capability_source = "model_key" + except Exception: + capability_source = "profile_fallback" + + if capability_source != "model_key": + multimodal = self._normalize_multimodal_capability(self.model_multimodal) + supports_image = multimodal in {"image", "image,video"} + supports_video = multimodal in {"video", "image,video"} - supports_image = bool(model_supports_image(model_key)) - supports_video = bool(model_supports_video(model_key)) if supports_image and supports_video: return messages @@ -405,6 +437,7 @@ class DeepSeekClient: { "event": "sanitize_messages_for_model_capability", "model_key": model_key, + "capability_source": capability_source, "supports_image": supports_image, "supports_video": supports_video, "changed_messages": changed, @@ -515,6 +548,7 @@ class DeepSeekClient: self.thinking_max_tokens = thinking.get("max_tokens") self.fast_extra_params = fast.get("extra_params") or {} self.thinking_extra_params = thinking.get("extra_params") or {} + self.model_multimodal = self._normalize_multimodal_capability(profile.get("multimodal")) self.default_context_window = profile.get("context_window") or fast.get("context_window") # 同步旧字段 self.api_base_url = self.fast_api_config["base_url"]