JOJO/agent-Specialization
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: harden multimodal handling and sanitize mcp action results

Browse Source
This commit is contained in:
JOJO 2026-04-27 20:11:21 +08:00
parent dcf5de5d03
commit ecca5da8ab
5 changed files with 141 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
prompts/main_system.txt
View File

@ -122,6 +122,7 @@
- 当你不确定当前有哪些 MCP 工具可用时，先调用 `list_mcp_servers`
- 需要刷新远程工具缓存时，可使用 `list_mcp_servers` 并传 `refresh=true`
- 调用具体 MCP 工具时，使用工具列表中的 `mcp__...` 别名
- **纯文本模型硬性约束**：调用工具（尤其是 MCP 工具）时，禁止调用任何可能返回多模态结果（图片/视频/音频）的工具；若用户需求依赖此类结果，先明确告知当前模型无法查看，并建议切换到支持多模态的模型后再执行。
---

15
server/chat_flow_tool_loop.py
View File

@ -673,10 +673,20 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie
'entries': after_entries
}
mcp_parsed = None
update_result_data = result_data
if isinstance(function_name, str) and function_name.startswith("mcp__") and isinstance(result_data, dict):
try:
mcp_parsed = extract_mcp_content_for_context(result_data)
update_result_data = mcp_parsed.get("sanitized_payload") or result_data
except Exception:
mcp_parsed = None
update_result_data = result_data
update_payload = {
'id': tool_display_id,
'status': action_status,
'result': result_data,
'result': update_result_data,
'preparing_id': tool_call_id,
'conversation_id': conversation_id
}
@ -702,7 +712,8 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie
tool_media_refs = None
if isinstance(result_data, dict):
if isinstance(function_name, str) and function_name.startswith("mcp__"):
mcp_parsed = extract_mcp_content_for_context(result_data)
if not isinstance(mcp_parsed, dict):
mcp_parsed = extract_mcp_content_for_context(result_data)
tool_result_content = (
mcp_parsed.get("text")
or format_tool_result_for_context(function_name, result_data, tool_result)

9
static/src/stores/model.ts
View File

@ -40,6 +40,9 @@ export const useModelStore = defineStore('model', {
key: 'kimi',
label: 'Kimi-k2',
description: '综合能力较强',
multimodal: 'none',
supportsImage: false,
supportsVideo: false,
fastOnly: false,
supportsThinking: true
},
@ -47,6 +50,9 @@ export const useModelStore = defineStore('model', {
key: 'deepseek',
label: 'Deepseek-V3.2',
description: '数学能力较强',
multimodal: 'none',
supportsImage: false,
supportsVideo: false,
fastOnly: false,
supportsThinking: true
},
@ -64,6 +70,9 @@ export const useModelStore = defineStore('model', {
key: 'minimax-m2.5',
label: 'MiniMax-M2.5',
description: '仅深度思考，超长上下文',
multimodal: 'none',
supportsImage: false,
supportsVideo: false,
fastOnly: false,
supportsThinking: true,
deepOnly: true

75
test/test_api_client_multimodal_sanitize.py
View File

@ -6,6 +6,28 @@ from utils.api_client import DeepSeekClient
class APIClientMultimodalSanitizeTest(unittest.TestCase):
def test_text_only_model_adds_notice_when_only_media_parts(self):
client = DeepSeekClient(web_mode=True)
client.model_key = "DeepSeek-V4-Pro High" # custom_models.json: multimodal=none
messages = [
{
"role": "tool",
"name": "mcp__playwright__browser_take_screenshot",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
],
}
]
sanitized = client._sanitize_messages_for_model_capability(messages)
self.assertEqual(len(sanitized), 1)
self.assertEqual(
sanitized[0].get("content"),
"当前模型无查看图片/视频能力，无法返回结果",
sanitized[0],
)
def test_text_only_model_strips_image_parts_to_text(self):
client = DeepSeekClient(web_mode=True)
client.model_key = "DeepSeek-V4-Pro High" # custom_models.json: multimodal=none
@ -25,6 +47,35 @@ class APIClientMultimodalSanitizeTest(unittest.TestCase):
self.assertEqual(len(sanitized), 1)
self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0])
self.assertIn("截图成功", sanitized[0].get("content") or "", sanitized[0])
self.assertIn(
"当前模型无查看图片/视频能力，无法返回结果",
sanitized[0].get("content") or "",
sanitized[0],
)
def test_builtin_text_only_model_strips_media_parts(self):
client = DeepSeekClient(web_mode=True)
client.model_key = "deepseek" # 内置模型: multimodal=none
messages = [
{
"role": "tool",
"name": "view_image",
"content": [
{"type": "text", "text": "已尝试查看图片"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
],
}
]
sanitized = client._sanitize_messages_for_model_capability(messages)
self.assertEqual(len(sanitized), 1)
self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0])
self.assertIn(
"当前模型无查看图片/视频能力，无法返回结果",
sanitized[0].get("content") or "",
sanitized[0],
)
def test_multimodal_model_keeps_image_parts(self):
client = DeepSeekClient(web_mode=True)
@ -49,6 +100,30 @@ class APIClientMultimodalSanitizeTest(unittest.TestCase):
sanitized[0],
)
def test_profile_multimodal_fallback_works_when_model_key_missing(self):
client = DeepSeekClient(web_mode=True)
client.model_key = "" # 模拟未知/缺失 key，仅走 profile 兜底
client.model_multimodal = "none"
messages = [
{
"role": "tool",
"content": [
{"type": "text", "text": "测试"},
{"type": "video_url", "video_url": {"url": "data:video/mp4;base64,abc"}},
],
}
]
sanitized = client._sanitize_messages_for_model_capability(messages)
self.assertEqual(len(sanitized), 1)
self.assertIsInstance(sanitized[0].get("content"), str, sanitized[0])
self.assertIn(
"当前模型无查看图片/视频能力，无法返回结果",
sanitized[0].get("content") or "",
sanitized[0],
)
if __name__ == "__main__":
unittest.main()

52
utils/api_client.py
View File

@ -69,6 +69,7 @@ class DeepSeekClient:
self.api_key = self.fast_api_config["api_key"]
self.model_id = self.fast_api_config["model_id"]
self.model_key = None # 由宿主终端注入，便于做模型兼容处理
self.model_multimodal = "none" # 由模型 profile 注入，model_key 不可用时用于能力兜底
self.project_path: Optional[str] = None
# 每个任务的独立状态
self.current_task_first_call = True # 当前任务是否是第一次调用
@ -317,6 +318,15 @@ class DeepSeekClient:
}
return [merged] + messages[idx:]
@staticmethod
def _normalize_multimodal_capability(value: Any) -> str:
text = str(value or "none").strip().lower()
if text == "video,image":
return "image,video"
if text in {"none", "image", "video", "image,video"}:
return text
return "none"
@staticmethod
def _sanitize_content_for_capability(
content: Any,
@ -330,6 +340,8 @@ class DeepSeekClient:
kept_parts: List[Dict[str, Any]] = []
text_segments: List[str] = []
dropped_media = False
unsupported_media_notice = "当前模型无查看图片/视频能力，无法返回结果"
for part in content:
if not isinstance(part, dict):
@ -348,10 +360,14 @@ class DeepSeekClient:
if ctype == "image_url":
if supports_image:
kept_parts.append(part)
else:
dropped_media = True
continue
if ctype == "video_url":
if supports_video:
kept_parts.append(part)
else:
dropped_media = True
continue
# 其他未知/暂不支持类型统一忽略（纯文本模型下只保留 text）
@ -359,7 +375,14 @@ class DeepSeekClient:
# 纯文本模型：统一回退为字符串，规避供应商对 content part 类型的严格校验
if not supports_image and not supports_video:
return "\n".join(seg for seg in text_segments if seg).strip()
text_payload = "\n".join(seg for seg in text_segments if seg).strip()
if dropped_media:
if text_payload:
if unsupported_media_notice in text_payload:
return text_payload
return f"{text_payload}\n\n{unsupported_media_notice}"
return unsupported_media_notice
return text_payload
# 半多模态模型（例如只支持图片）：尽量保留可支持的 part
if kept_parts:
@ -370,16 +393,25 @@ class DeepSeekClient:
def _sanitize_messages_for_model_capability(self, messages: List[Dict]) -> List[Dict]:
model_key = str(self.model_key or "").strip()
if not model_key:
return messages
supports_image = False
supports_video = False
capability_source = "profile_fallback"
try:
from config.model_profiles import model_supports_image, model_supports_video
except Exception:
return messages
if model_key:
try:
from config.model_profiles import get_model_capabilities
caps = get_model_capabilities(model_key)
supports_image = bool(caps.get("supports_image"))
supports_video = bool(caps.get("supports_video"))
capability_source = "model_key"
except Exception:
capability_source = "profile_fallback"
if capability_source != "model_key":
multimodal = self._normalize_multimodal_capability(self.model_multimodal)
supports_image = multimodal in {"image", "image,video"}
supports_video = multimodal in {"video", "image,video"}
supports_image = bool(model_supports_image(model_key))
supports_video = bool(model_supports_video(model_key))
if supports_image and supports_video:
return messages
@ -405,6 +437,7 @@ class DeepSeekClient:
{
"event": "sanitize_messages_for_model_capability",
"model_key": model_key,
"capability_source": capability_source,
"supports_image": supports_image,
"supports_video": supports_video,
"changed_messages": changed,
@ -515,6 +548,7 @@ class DeepSeekClient:
self.thinking_max_tokens = thinking.get("max_tokens")
self.fast_extra_params = fast.get("extra_params") or {}
self.thinking_extra_params = thinking.get("extra_params") or {}
self.model_multimodal = self._normalize_multimodal_capability(profile.get("multimodal"))
self.default_context_window = profile.get("context_window") or fast.get("context_window")
# 同步旧字段
self.api_base_url = self.fast_api_config["base_url"]