fix: persist media via store and preserve MCP content

This commit is contained in:
JOJO 2026-04-27 12:21:45 +08:00
parent dd334549c7
commit 727cafc370
8 changed files with 1201 additions and 83 deletions

View File

@ -73,7 +73,10 @@ from modules.container_monitor import collect_stats, inspect_state
from core.tool_config import TOOL_CATEGORIES
from utils.api_client import DeepSeekClient
from utils.context_manager import ContextManager
from utils.tool_result_formatter import format_tool_result_for_context
from utils.tool_result_formatter import (
extract_mcp_content_for_context,
format_tool_result_for_context,
)
from utils.logger import setup_logger
from config.model_profiles import (
get_model_profile,
@ -226,7 +229,37 @@ class MainTerminalCommandMixin:
result_data = parsed if isinstance(parsed, dict) else {}
except Exception:
result_data = {}
tool_result_content = format_tool_result_for_context(tool_name, result_data, result)
tool_metadata = {"tool_payload": result_data} if result_data else None
if isinstance(tool_name, str) and tool_name.startswith("mcp__") and isinstance(result_data, dict):
mcp_parsed = extract_mcp_content_for_context(result_data)
tool_result_content = (
mcp_parsed.get("text")
or format_tool_result_for_context(tool_name, result_data, result)
)
tool_metadata = {"tool_payload": mcp_parsed.get("sanitized_payload") or result_data}
mcp_media_items = mcp_parsed.get("media_items") or []
tool_media_refs = []
for item in mcp_media_items:
if not isinstance(item, dict):
continue
tool_media_refs.append(
{
"kind": item.get("kind"),
"mime_type": item.get("mime_type"),
"data_base64": item.get("data_base64"),
"source": "mcp_content",
"item_type": item.get("item_type"),
"label": item.get("label"),
"name": item.get("name"),
"title": item.get("title"),
"uri": item.get("uri"),
"url": item.get("url"),
"index": item.get("index"),
}
)
else:
tool_result_content = format_tool_result_for_context(tool_name, result_data, result)
tool_media_refs = None
tool_images = None
tool_videos = None
if (
@ -250,8 +283,10 @@ class MainTerminalCommandMixin:
"system_message": result_data.get("system_message") if isinstance(result_data, dict) else None,
"task_id": result_data.get("task_id") if isinstance(result_data, dict) else None,
"raw_result_data": result_data if result_data else None,
"metadata": tool_metadata,
"images": tool_images,
"videos": tool_videos,
"media_refs": tool_media_refs,
})
return result
@ -290,8 +325,10 @@ class MainTerminalCommandMixin:
tool_result["content"],
tool_call_id=tool_result["tool_call_id"],
name=tool_result["name"],
metadata=tool_result.get("metadata"),
images=tool_result.get("images"),
videos=tool_result.get("videos")
videos=tool_result.get("videos"),
media_refs=tool_result.get("media_refs"),
)
system_message = tool_result.get("system_message")
if system_message:

View File

@ -313,11 +313,17 @@ class MainTerminalContextMixin:
# Tool消息需要保留完整结构
images = conv.get("images") or metadata.get("images") or []
videos = conv.get("videos") or metadata.get("videos") or []
media_refs = conv.get("media_refs") or metadata.get("media_refs") or []
content_value = conv.get("content")
if isinstance(content_value, list):
content_payload = content_value
elif images or videos:
content_payload = self.context_manager._build_content_with_images(content_value, images, videos)
elif images or videos or media_refs:
content_payload = self.context_manager._build_content_with_images(
content_value,
images,
videos,
media_refs=media_refs,
)
else:
content_payload = content_value
message = {
@ -338,9 +344,15 @@ class MainTerminalContextMixin:
# User 或普通 System 消息
images = conv.get("images") or metadata.get("images") or []
videos = conv.get("videos") or metadata.get("videos") or []
media_refs = conv.get("media_refs") or metadata.get("media_refs") or []
content_payload = (
self.context_manager._build_content_with_images(conv["content"], images, videos)
if (images or videos) else conv["content"]
self.context_manager._build_content_with_images(
conv["content"],
images,
videos,
media_refs=media_refs,
)
if (images or videos or media_refs) else conv["content"]
)
# 调试:记录所有 system 消息
if conv["role"] == "system":

View File

@ -893,11 +893,47 @@ class MCPClientManager:
def _build_call_message(result: Dict[str, Any]) -> str:
content = result.get("content")
if isinstance(content, list):
for item in content:
lines: List[str] = []
for idx, item in enumerate(content, start=1):
if isinstance(item, dict) and item.get("type") == "text":
text = str(item.get("text") or "").strip()
if text:
return text[:500]
lines.append(text)
continue
if not isinstance(item, dict):
continue
ctype = str(item.get("type") or "").strip().lower()
if ctype in {"image", "audio"}:
mime = str(item.get("mimeType") or item.get("mime_type") or "").strip()
if not mime:
mime = "image/png" if ctype == "image" else "audio/wav"
data_len = len(str(item.get("data") or ""))
lines.append(f"[MCP {ctype} #{idx}] {mime} ({data_len} base64 chars)")
continue
if ctype in {"resource_link", "resourcelink"}:
uri = str(item.get("uri") or item.get("url") or "").strip()
name = str(item.get("name") or item.get("title") or "").strip()
if uri and name:
lines.append(f"[MCP 资源] {name} -> {uri}")
elif uri:
lines.append(f"[MCP 资源] {uri}")
continue
if ctype == "resource":
resource = item.get("resource")
if isinstance(resource, dict):
text = str(resource.get("text") or "").strip()
if text:
lines.append(text)
continue
uri = str(resource.get("uri") or "").strip()
if uri:
lines.append(f"[MCP 资源] {uri}")
continue
uri = str(item.get("uri") or item.get("url") or "").strip()
if uri:
lines.append(f"[MCP 资源] {uri}")
if lines:
return "\n".join(lines)[:500]
structured = result.get("structuredContent")
if structured is not None:
try:

View File

@ -13,7 +13,10 @@ from .monitor import cache_monitor_snapshot
from .security import compact_web_search_result
from .chat_flow_helpers import detect_tool_failure
from .chat_flow_runner_helpers import resolve_monitor_path, resolve_monitor_memory, capture_monitor_snapshot
from utils.tool_result_formatter import format_tool_result_for_context
from utils.tool_result_formatter import (
extract_mcp_content_for_context,
format_tool_result_for_context,
)
from utils.context_manager import AUTO_SHALLOW_PLACEHOLDER
from config import TOOL_CALL_COOLDOWN
from modules.personalization_manager import load_personalization_config, resolve_context_compression_settings
@ -696,16 +699,48 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie
metadata_payload = None
tool_images = None
tool_videos = None
tool_media_refs = None
if isinstance(result_data, dict):
# 特殊处理 web_search保留可供前端渲染的精简结构以便历史记录复现搜索结果
if function_name == "web_search":
try:
tool_result_content = json.dumps(compact_web_search_result(result_data), ensure_ascii=False)
except Exception:
tool_result_content = tool_result
if isinstance(function_name, str) and function_name.startswith("mcp__"):
mcp_parsed = extract_mcp_content_for_context(result_data)
tool_result_content = (
mcp_parsed.get("text")
or format_tool_result_for_context(function_name, result_data, tool_result)
)
mcp_media_items = mcp_parsed.get("media_items") or []
if mcp_media_items:
tool_media_refs = []
for item in mcp_media_items:
if not isinstance(item, dict):
continue
tool_media_refs.append(
{
"kind": item.get("kind"),
"mime_type": item.get("mime_type"),
"data_base64": item.get("data_base64"),
"source": "mcp_content",
"item_type": item.get("item_type"),
"label": item.get("label"),
"name": item.get("name"),
"title": item.get("title"),
"uri": item.get("uri"),
"url": item.get("url"),
"index": item.get("index"),
}
)
metadata_payload = {
"tool_payload": mcp_parsed.get("sanitized_payload") or result_data
}
else:
tool_result_content = format_tool_result_for_context(function_name, result_data, tool_result)
metadata_payload = {"tool_payload": result_data}
# 特殊处理 web_search保留可供前端渲染的精简结构以便历史记录复现搜索结果
if function_name == "web_search":
try:
tool_result_content = json.dumps(compact_web_search_result(result_data), ensure_ascii=False)
except Exception:
tool_result_content = tool_result
else:
tool_result_content = format_tool_result_for_context(function_name, result_data, tool_result)
metadata_payload = {"tool_payload": result_data}
else:
tool_result_content = tool_result
tool_message_content = tool_result_content
@ -748,27 +783,32 @@ async def execute_tool_calls(*, web_terminal, tool_calls, sender, messages, clie
'content': f'系统已记录视频路径(不再附带二进制数据): {video_path}'
})
# 将工具结果即时组装为多模态消息,确保同一轮 tool loop 的下一次模型请求能真正看到图片/视频
if tool_images or tool_videos:
try:
tool_message_content = web_terminal.context_manager._build_content_with_images(
str(tool_result_content or ""),
tool_images or [],
tool_videos or []
)
except Exception:
tool_message_content = tool_result_content
# 立即保存工具结果
web_terminal.context_manager.add_conversation(
saved_tool_message = web_terminal.context_manager.add_conversation(
"tool",
tool_result_content,
tool_call_id=tool_call_id,
name=function_name,
metadata=metadata_payload,
images=tool_images,
videos=tool_videos
videos=tool_videos,
media_refs=tool_media_refs,
)
saved_media_refs = []
if isinstance(saved_tool_message, dict):
saved_media_refs = saved_tool_message.get("media_refs") or []
# 将工具结果即时组装为多模态消息,确保同一轮 tool loop 的下一次模型请求能真正看到媒体
if tool_images or tool_videos or saved_media_refs:
try:
tool_message_content = web_terminal.context_manager._build_content_with_images(
str(tool_result_content or ""),
tool_images or [],
tool_videos or [],
media_refs=saved_media_refs,
)
except Exception:
tool_message_content = tool_result_content
try:
workspace_data_dir = getattr(workspace, "data_dir", None) if workspace else None
personal_config = load_personalization_config(workspace_data_dir) if workspace_data_dir else {}

View File

@ -0,0 +1,130 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from utils.context_manager import ContextManager
from utils.tool_result_formatter import (
extract_mcp_content_for_context,
format_tool_result_for_context,
)
class MediaStoreAndMCPContentTest(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
root = Path(self.temp_dir.name)
self.project_dir = root / "project"
self.project_dir.mkdir(parents=True, exist_ok=True)
self.data_dir = root / "data"
self.data_dir.mkdir(parents=True, exist_ok=True)
def tearDown(self):
self.temp_dir.cleanup()
def test_media_store_survives_source_file_deletion(self):
image_path = self.project_dir / "demo.png"
image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake-image-data")
ctx = ContextManager(project_path=str(self.project_dir), data_dir=str(self.data_dir))
conv_id = ctx.start_new_conversation(project_path=str(self.project_dir), thinking_mode=False)
ctx.add_conversation("user", "看图", images=["demo.png"])
message = ctx.conversation_history[-1]
refs = message.get("media_refs") or []
self.assertTrue(refs, message)
self.assertEqual(refs[0].get("kind"), "image")
image_path.unlink()
payload = ctx._build_content_with_images(
message.get("content") or "",
message.get("images") or [],
message.get("videos") or [],
media_refs=refs,
)
self.assertIsInstance(payload, list, payload)
image_parts = [item for item in payload if isinstance(item, dict) and item.get("type") == "image_url"]
self.assertTrue(image_parts, payload)
self.assertTrue(str(image_parts[0].get("image_url", {}).get("url", "")).startswith("data:image/"))
index_file = self.data_dir / "conversations" / "media_store" / "index.json"
self.assertTrue(index_file.exists())
index_data = json.loads(index_file.read_text(encoding="utf-8"))
self.assertIn(conv_id, index_data.get("message_map", {}))
self.assertIn(message.get("message_id"), (index_data.get("message_map", {}).get(conv_id) or {}))
def test_add_tool_message_with_inline_mcp_media_refs(self):
ctx = ContextManager(project_path=str(self.project_dir), data_dir=str(self.data_dir))
ctx.start_new_conversation(project_path=str(self.project_dir), thinking_mode=False)
saved = ctx.add_conversation(
"tool",
"截图已记录",
tool_call_id="call_1",
name="mcp__browsermcp__browser_screenshot",
media_refs=[
{
"kind": "image",
"mime_type": "image/png",
"data_base64": "aGVsbG8=",
"source": "mcp_content",
}
],
)
refs = (saved or {}).get("media_refs") or []
self.assertTrue(refs, saved)
self.assertTrue(str(refs[0].get("media_id") or "").startswith("sha256:"))
payload = ctx._build_content_with_images(
saved.get("content") or "",
[],
[],
media_refs=refs,
)
self.assertIsInstance(payload, list, payload)
image_parts = [item for item in payload if isinstance(item, dict) and item.get("type") == "image_url"]
self.assertTrue(image_parts, payload)
self.assertTrue(str(image_parts[0].get("image_url", {}).get("url", "")).startswith("data:image/png;base64,"))
def test_mcp_formatter_keeps_non_text_information(self):
result_data = {
"success": True,
"server_id": "browsermcp",
"tool_alias": "mcp__browsermcp__browser_screenshot",
"tool_name": "browser_screenshot",
"message": "MCP 工具调用完成",
"content": [
{
"type": "image",
"mimeType": "image/png",
"data": "aGVsbG8=",
}
],
"raw_result": {
"content": [
{
"type": "image",
"mimeType": "image/png",
"data": "aGVsbG8=",
}
]
},
}
text = format_tool_result_for_context("mcp__browsermcp__browser_screenshot", result_data, "")
self.assertIn("MCP image #1", text)
parsed = extract_mcp_content_for_context(result_data)
self.assertTrue(parsed.get("media_items"))
sanitized = parsed.get("sanitized_payload") or {}
content = (sanitized.get("content") or [{}])[0]
self.assertNotIn("data", content)
self.assertTrue(content.get("data_omitted"))
if __name__ == "__main__":
unittest.main()

View File

@ -5,6 +5,7 @@ import json
import base64
import mimetypes
import io
import uuid
import platform
import shutil
import subprocess
@ -54,6 +55,7 @@ except ImportError:
model_supports_video,
)
from utils.conversation_manager import ConversationManager
from utils.media_store import MediaStore
AUTO_SHALLOW_PLACEHOLDER = "过早的工具结果已经被自动压缩"
AUTO_SHALLOW_TOOL_WHITELIST = {
@ -95,6 +97,7 @@ class ContextManager:
# 新增:对话持久化管理器
self.conversation_manager = ConversationManager(base_dir=self.data_dir)
self.media_store = MediaStore(self.data_dir)
self.current_conversation_id: Optional[str] = None
self.auto_save_enabled = True
self.main_terminal = None # 由宿主终端在初始化后回填,用于工具定义访问
@ -837,6 +840,24 @@ class ContextManager:
model_key = metadata.get("model_key")
self.has_images = metadata.get("has_images", False)
self.has_videos = metadata.get("has_videos", False)
if not self.has_images or not self.has_videos:
for msg in self.conversation_history:
if not isinstance(msg, dict):
continue
images = msg.get("images") or []
videos = msg.get("videos") or []
if images:
self.has_images = True
if videos:
self.has_videos = True
for ref in (msg.get("media_refs") or []):
kind = str((ref or {}).get("kind") or "").strip().lower()
if kind == "image":
self.has_images = True
elif kind == "video":
self.has_videos = True
if self.has_images and self.has_videos:
break
if self.main_terminal:
try:
if model_key:
@ -1329,6 +1350,189 @@ class ContextManager:
print(f"[Debug] 广播token更新失败: {e}")
import traceback
traceback.print_exc()
def _generate_message_id(self) -> str:
"""生成消息唯一 ID用于 media_store 映射)。"""
return f"msg_{uuid.uuid4().hex}"
def _resolve_media_input_path(self, raw_path: str) -> Optional[Path]:
path_text = str(raw_path or "").strip()
if not path_text:
return None
try:
candidate = Path(path_text).expanduser()
if candidate.is_absolute():
return candidate.resolve()
return (Path(self.project_path) / candidate).resolve()
except Exception:
return None
def _store_media_item_from_candidate(self, candidate: Dict[str, Any]) -> Optional[Dict[str, Any]]:
if not isinstance(candidate, dict):
return None
kind = str(candidate.get("kind") or "").strip().lower() or "binary"
mime_type = candidate.get("mime_type")
source_path = candidate.get("source_path")
base_ref: Optional[Dict[str, Any]] = None
try:
media_id = str(candidate.get("media_id") or "").strip()
if media_id:
data_url = self.media_store.to_data_url(candidate)
if data_url:
base_ref = {
"media_id": media_id,
"kind": kind,
"mime_type": mime_type,
"store_path": candidate.get("store_path"),
"source_path": source_path,
}
else:
base_ref = None
elif candidate.get("data_url"):
stored = self.media_store.store_data_url(
str(candidate.get("data_url")),
kind=kind,
source_path=source_path,
)
base_ref = dict(stored)
elif candidate.get("data_base64") or candidate.get("data"):
payload = str(candidate.get("data_base64") or candidate.get("data") or "")
if payload:
stored = self.media_store.store_base64_data(
payload,
kind=kind,
mime_type=mime_type,
source_path=source_path,
)
base_ref = dict(stored)
elif candidate.get("path"):
abs_path = self._resolve_media_input_path(str(candidate.get("path")))
if abs_path and abs_path.exists() and abs_path.is_file():
stored = self.media_store.store_file(
abs_path,
kind=kind,
mime_type=mime_type,
source_path=source_path or str(candidate.get("path")),
)
base_ref = dict(stored)
except Exception:
base_ref = None
if not base_ref:
return None
# 保留调用侧附加信息(例如视频 fps、MCP item 类型等)
passthrough_keys = {
"label",
"fps",
"source",
"origin",
"item_type",
"name",
"title",
"uri",
"url",
"index",
}
for key in passthrough_keys:
if key in candidate and candidate.get(key) is not None:
base_ref[key] = candidate.get(key)
if base_ref.get("source_path") is None and source_path:
base_ref["source_path"] = source_path
return base_ref
def _prepare_media_candidates(
self,
images: Optional[List[Any]] = None,
videos: Optional[List[Any]] = None,
media_refs: Optional[List[Dict[str, Any]]] = None,
) -> List[Dict[str, Any]]:
candidates: List[Dict[str, Any]] = []
for item in media_refs or []:
if not isinstance(item, dict):
continue
kind = str(item.get("kind") or "").strip().lower()
if not kind:
mime_hint = str(item.get("mime_type") or item.get("mimeType") or "").strip()
if mime_hint.startswith("image/"):
kind = "image"
elif mime_hint.startswith("video/"):
kind = "video"
elif mime_hint.startswith("audio/"):
kind = "audio"
candidate = dict(item)
if kind:
candidate["kind"] = kind
if "mimeType" in candidate and "mime_type" not in candidate:
candidate["mime_type"] = candidate.get("mimeType")
candidates.append(candidate)
for path in images or []:
if isinstance(path, dict):
candidate = dict(path)
candidate.setdefault("kind", "image")
if "mimeType" in candidate and "mime_type" not in candidate:
candidate["mime_type"] = candidate.get("mimeType")
if "path" not in candidate and path.get("url"):
candidate["path"] = path.get("url")
candidates.append(candidate)
continue
path_text = str(path or "").strip()
if not path_text:
continue
candidates.append(
{
"kind": "image",
"path": path_text,
"source_path": path_text,
"source": "message_images",
}
)
for item in videos or []:
if isinstance(item, dict):
path_text = str(item.get("path") or "").strip()
if not path_text:
continue
candidate = {
"kind": "video",
"path": path_text,
"source_path": path_text,
"fps": item.get("fps"),
"source": item.get("source") or "message_videos",
}
if item.get("mime_type"):
candidate["mime_type"] = item.get("mime_type")
elif item.get("mimeType"):
candidate["mime_type"] = item.get("mimeType")
candidates.append(candidate)
else:
path_text = str(item or "").strip()
if not path_text:
continue
candidates.append(
{
"kind": "video",
"path": path_text,
"source_path": path_text,
"source": "message_videos",
}
)
return candidates
@staticmethod
def _dedupe_media_refs(media_refs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
deduped: List[Dict[str, Any]] = []
seen = set()
for item in media_refs:
if not isinstance(item, dict):
continue
media_id = str(item.get("media_id") or "").strip()
source_path = str(item.get("source_path") or "").strip()
key = (media_id, source_path, str(item.get("kind") or ""))
if key in seen:
continue
seen.add(key)
deduped.append(item)
return deduped
def add_conversation(
self,
@ -1339,33 +1543,61 @@ class ContextManager:
name: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
reasoning_content: Optional[str] = None,
images: Optional[List[str]] = None,
videos: Optional[List[str]] = None
images: Optional[List[Any]] = None,
videos: Optional[List[Any]] = None,
media_refs: Optional[List[Dict[str, Any]]] = None,
):
"""添加对话记录(改进版:集成自动保存 + 智能token统计"""
timestamp = datetime.now().isoformat()
message_id = self._generate_message_id()
if role == "assistant":
message = {
"role": role,
"reasoning_content": reasoning_content if reasoning_content is not None else "",
"content": content or "",
"timestamp": timestamp
"timestamp": timestamp,
"message_id": message_id,
}
else:
message = {
"role": role,
"content": content,
"timestamp": timestamp
"timestamp": timestamp,
"message_id": message_id,
}
if metadata:
message["metadata"] = metadata
message["metadata"] = dict(metadata)
if images:
message["images"] = images
self.has_images = True
if videos:
message["videos"] = videos
self.has_videos = True
prepared_candidates = self._prepare_media_candidates(images=images, videos=videos, media_refs=media_refs)
stored_media_refs: List[Dict[str, Any]] = []
for candidate in prepared_candidates:
stored_ref = self._store_media_item_from_candidate(candidate)
if stored_ref:
stored_media_refs.append(stored_ref)
stored_media_refs = self._dedupe_media_refs(stored_media_refs)
if stored_media_refs:
message["media_refs"] = stored_media_refs
message.setdefault("metadata", {})
message["metadata"]["media_refs"] = stored_media_refs
if any(str(item.get("kind") or "").strip().lower() == "image" for item in stored_media_refs):
self.has_images = True
if any(str(item.get("kind") or "").strip().lower() == "video" for item in stored_media_refs):
self.has_videos = True
try:
self.media_store.link_message_media(
self.current_conversation_id,
message_id,
stored_media_refs,
)
except Exception:
pass
# 记录当前助手回复所用模型,便于回放时查看
if role == "assistant":
@ -1406,6 +1638,7 @@ class ContextManager:
print(f"[Debug] 添加{role}消息后广播token更新")
self.safe_broadcast_token_update()
return message
def add_tool_result(self, tool_call_id: str, function_name: str, result: str):
"""添加工具调用结果(保留方法以兼容)"""
@ -1866,65 +2099,140 @@ class ContextManager:
except Exception:
return None
def _build_content_with_images(self, text: str, images: List[str], videos: Optional[List[Any]] = None) -> Any:
"""将文本与图片/视频路径组合成多模态content图片转换为data URI视频转换为 data URL。"""
def _build_content_with_images(
self,
text: str,
images: List[Any],
videos: Optional[List[Any]] = None,
media_refs: Optional[List[Dict[str, Any]]] = None,
) -> Any:
"""将文本与媒体组合成多模态 content。
优先使用 media_refs来自 media_store回退到历史路径字段images/videos
"""
videos = videos or []
if not images and not videos:
media_refs = media_refs or []
if not images and not videos and not media_refs:
return text
parts: List[Dict[str, Any]] = []
extra_videos: List[Any] = []
extra_notes: List[str] = []
current_model = getattr(getattr(self, "main_terminal", None), "model_key", None)
supports_video_fps = current_model == "qwen3-vl-plus"
qwen_video_fps = 2
if text:
parts.append({"type": "text", "text": text})
for path in images:
try:
abs_path = Path(self.project_path) / path
if not abs_path.exists() or not abs_path.is_file():
media_payload_attached = False
if media_refs:
for ref in media_refs:
if not isinstance(ref, dict):
continue
mime, _ = mimetypes.guess_type(abs_path.name)
if mime and mime.startswith("video/"):
extra_videos.append(path)
continue
if mime and not mime.startswith("image/"):
continue
data_url = self._compress_image_if_needed(abs_path)
kind = str(ref.get("kind") or "").strip().lower()
data_url = self.media_store.to_data_url(ref)
if not data_url:
continue
if kind == "image":
parts.append({"type": "image_url", "image_url": {"url": data_url}})
media_payload_attached = True
continue
if kind == "video":
payload: Dict[str, Any] = {"type": "video_url", "video_url": {"url": data_url}}
fps_value = ref.get("fps")
if supports_video_fps and fps_value is None:
fps_value = qwen_video_fps
if fps_value is not None:
payload["fps"] = fps_value
parts.append(payload)
media_payload_attached = True
continue
if kind == "audio":
mime = str(ref.get("mime_type") or "audio/*")
size = ref.get("size")
label = f"[已附加音频 {mime}"
if isinstance(size, int):
label += f", {size} bytes"
label += "]"
extra_notes.append(label)
media_payload_attached = True
# 若已通过 media_refs 成功附加媒体,避免再次从路径重复注入
should_use_legacy_paths = not media_payload_attached
if should_use_legacy_paths:
for item in images:
try:
if isinstance(item, dict):
path = str(item.get("path") or "").strip()
else:
path = str(item or "").strip()
if not path:
continue
abs_path = self._resolve_media_input_path(path)
if not abs_path or not abs_path.exists() or not abs_path.is_file():
continue
mime, _ = mimetypes.guess_type(abs_path.name)
if mime and mime.startswith("video/"):
extra_videos.append(item)
continue
if mime and not mime.startswith("image/"):
continue
data_url = self._compress_image_if_needed(abs_path)
if not data_url:
if not mime:
mime = "image/png"
data = abs_path.read_bytes()
b64 = base64.b64encode(data).decode("utf-8")
data_url = f"data:{mime};base64,{b64}"
parts.append({"type": "image_url", "image_url": {"url": data_url}})
except Exception:
continue
for item in [*videos, *extra_videos]:
try:
if isinstance(item, dict):
path = item.get("path") or ""
else:
path = item
path_text = str(path or "").strip()
if not path_text:
continue
abs_path = self._resolve_media_input_path(path_text)
if not abs_path or not abs_path.exists() or not abs_path.is_file():
continue
if abs_path.stat().st_size > 50 * 1024 * 1024:
continue
mime, _ = mimetypes.guess_type(abs_path.name)
if not mime:
mime = "image/png"
mime = "video/mp4"
data = abs_path.read_bytes()
b64 = base64.b64encode(data).decode("utf-8")
data_url = f"data:{mime};base64,{b64}"
parts.append({"type": "image_url", "image_url": {"url": data_url}})
except Exception:
continue
for item in [*videos, *extra_videos]:
try:
if isinstance(item, dict):
path = item.get("path") or ""
else:
path = item
if not path:
payload = {"type": "video_url", "video_url": {"url": data_url}}
fps_value = item.get("fps") if isinstance(item, dict) else None
if supports_video_fps and fps_value is None:
fps_value = qwen_video_fps
if fps_value is not None:
payload["fps"] = fps_value
parts.append(payload)
except Exception:
continue
abs_path = Path(self.project_path) / path
if not abs_path.exists() or not abs_path.is_file():
continue
if abs_path.stat().st_size > 50 * 1024 * 1024:
continue
mime, _ = mimetypes.guess_type(abs_path.name)
if not mime:
mime = "video/mp4"
data = abs_path.read_bytes()
b64 = base64.b64encode(data).decode("utf-8")
data_url = f"data:{mime};base64,{b64}"
payload: Dict[str, Any] = {"type": "video_url", "video_url": {"url": data_url}}
if supports_video_fps:
payload["fps"] = qwen_video_fps
parts.append(payload)
except Exception:
continue
return parts if parts else text
text_segments = []
if text:
text_segments.append(text)
if extra_notes:
text_segments.extend(extra_notes)
merged_text = "\n".join(segment for segment in text_segments if segment)
if merged_text:
parts.insert(0, {"type": "text", "text": merged_text})
# 只有纯文本时保持字符串,避免无意义地返回 list[{"type":"text"}]
if not parts:
return text
if len(parts) == 1 and parts[0].get("type") == "text":
return parts[0].get("text", "")
return parts
def _build_workspace_system_message(self, context: Dict) -> Optional[str]:
"""构建独立的工作区系统消息,根据运行模式动态展示环境与资源信息。"""
@ -1971,5 +2279,3 @@ class ContextManager:
)
return content

369
utils/media_store.py Normal file
View File

@ -0,0 +1,369 @@
"""对话媒体存储:将图片/视频/音频二进制与对话 JSON 解耦。"""
from __future__ import annotations
import base64
import hashlib
import json
import mimetypes
import os
import tempfile
import threading
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
DEFAULT_MIME_BY_KIND: Dict[str, str] = {
"image": "image/png",
"video": "video/mp4",
"audio": "audio/wav",
}
def _normalize_kind(value: Optional[str], mime_type: Optional[str] = None) -> str:
kind = str(value or "").strip().lower()
if kind in {"image", "video", "audio"}:
return kind
mime = str(mime_type or "").strip().lower()
if mime.startswith("image/"):
return "image"
if mime.startswith("video/"):
return "video"
if mime.startswith("audio/"):
return "audio"
return "binary"
def _normalize_mime(value: Optional[str], kind: str, fallback_path: Optional[str] = None) -> str:
mime = str(value or "").strip().lower()
if mime:
return mime
if fallback_path:
guessed, _ = mimetypes.guess_type(str(fallback_path))
if guessed:
return guessed
return DEFAULT_MIME_BY_KIND.get(kind, "application/octet-stream")
def _guess_extension(mime_type: str, kind: str) -> str:
ext = mimetypes.guess_extension(mime_type or "")
if ext:
return ext
if kind == "image":
return ".png"
if kind == "video":
return ".mp4"
if kind == "audio":
return ".wav"
return ".bin"
def _decode_base64_payload(payload: str) -> bytes:
data = "".join(str(payload or "").strip().split())
if not data:
raise ValueError("base64 数据为空")
missing_padding = len(data) % 4
if missing_padding:
data += "=" * (4 - missing_padding)
return base64.b64decode(data.encode("utf-8"), validate=False)
def _parse_data_url(data_url: str) -> Tuple[str, str]:
raw = str(data_url or "").strip()
if not raw.startswith("data:") or "," not in raw:
raise ValueError("非法 data URL")
header, payload = raw.split(",", 1)
mime = "application/octet-stream"
if ";" in header:
mime = header[5:].split(";", 1)[0] or mime
elif len(header) > 5:
mime = header[5:]
if ";base64" not in header.lower():
raise ValueError("仅支持 base64 data URL")
return mime, payload
class MediaStore:
"""媒体存储中心data/conversations/media_store"""
INDEX_VERSION = 1
def __init__(self, data_dir: Path):
self.data_dir = Path(data_dir).expanduser().resolve()
self.conversations_dir = self.data_dir / "conversations"
self.root_dir = self.conversations_dir / "media_store"
self.blobs_dir = self.root_dir / "blobs"
self.index_file = self.root_dir / "index.json"
self._lock = threading.RLock()
self._ensure_dirs()
def _ensure_dirs(self) -> None:
self.conversations_dir.mkdir(parents=True, exist_ok=True)
self.root_dir.mkdir(parents=True, exist_ok=True)
self.blobs_dir.mkdir(parents=True, exist_ok=True)
if not self.index_file.exists():
self._save_index(
{
"version": self.INDEX_VERSION,
"media": {},
"message_map": {},
"updated_at": datetime.now().isoformat(),
}
)
def _atomic_write_json(self, target_file: Path, payload: Dict[str, Any]) -> None:
target_file.parent.mkdir(parents=True, exist_ok=True)
temp_path: Optional[Path] = None
try:
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=str(target_file.parent),
prefix=f".{target_file.name}.",
suffix=".tmp",
delete=False,
) as fh:
temp_path = Path(fh.name)
json.dump(payload, fh, ensure_ascii=False, indent=2)
fh.flush()
os.fsync(fh.fileno())
os.replace(str(temp_path), str(target_file))
finally:
if temp_path and temp_path.exists():
try:
temp_path.unlink()
except Exception:
pass
def _load_index(self) -> Dict[str, Any]:
try:
if not self.index_file.exists():
return {
"version": self.INDEX_VERSION,
"media": {},
"message_map": {},
"updated_at": datetime.now().isoformat(),
}
data = json.loads(self.index_file.read_text(encoding="utf-8") or "{}")
if not isinstance(data, dict):
raise ValueError("index.json 不是对象")
data.setdefault("version", self.INDEX_VERSION)
data.setdefault("media", {})
data.setdefault("message_map", {})
data.setdefault("updated_at", datetime.now().isoformat())
if not isinstance(data.get("media"), dict):
data["media"] = {}
if not isinstance(data.get("message_map"), dict):
data["message_map"] = {}
return data
except Exception:
return {
"version": self.INDEX_VERSION,
"media": {},
"message_map": {},
"updated_at": datetime.now().isoformat(),
}
def _save_index(self, payload: Dict[str, Any]) -> None:
payload["updated_at"] = datetime.now().isoformat()
self._atomic_write_json(self.index_file, payload)
@staticmethod
def _build_media_id(sha256_hex: str) -> str:
return f"sha256:{sha256_hex}"
def _blob_path_for(self, sha256_hex: str, ext: str) -> Tuple[Path, str]:
rel = Path("blobs") / sha256_hex[:2] / f"{sha256_hex}{ext}"
return self.root_dir / rel, rel.as_posix()
def _write_blob_if_missing(self, blob_path: Path, payload: bytes) -> None:
if blob_path.exists():
return
blob_path.parent.mkdir(parents=True, exist_ok=True)
temp_path: Optional[Path] = None
try:
with tempfile.NamedTemporaryFile(
mode="wb",
dir=str(blob_path.parent),
prefix=f".{blob_path.name}.",
suffix=".tmp",
delete=False,
) as fh:
temp_path = Path(fh.name)
fh.write(payload)
fh.flush()
os.fsync(fh.fileno())
os.replace(str(temp_path), str(blob_path))
finally:
if temp_path and temp_path.exists():
try:
temp_path.unlink()
except Exception:
pass
def store_bytes(
self,
payload: bytes,
*,
kind: Optional[str] = None,
mime_type: Optional[str] = None,
source_path: Optional[str] = None,
) -> Dict[str, Any]:
if not isinstance(payload, (bytes, bytearray)) or len(payload) == 0:
raise ValueError("媒体二进制为空")
sha256_hex = hashlib.sha256(payload).hexdigest()
media_id = self._build_media_id(sha256_hex)
normalized_kind = _normalize_kind(kind, mime_type)
normalized_mime = _normalize_mime(mime_type, normalized_kind, fallback_path=source_path)
ext = _guess_extension(normalized_mime, normalized_kind)
blob_path, blob_rel = self._blob_path_for(sha256_hex, ext)
with self._lock:
self._write_blob_if_missing(blob_path, bytes(payload))
index = self._load_index()
media = index.setdefault("media", {})
entry = media.get(media_id) if isinstance(media.get(media_id), dict) else {}
entry.update(
{
"media_id": media_id,
"sha256": sha256_hex,
"kind": normalized_kind,
"mime_type": normalized_mime,
"size": len(payload),
"blob_rel_path": blob_rel,
"created_at": entry.get("created_at") or datetime.now().isoformat(),
"updated_at": datetime.now().isoformat(),
}
)
media[media_id] = entry
self._save_index(index)
return {
"media_id": media_id,
"kind": normalized_kind,
"mime_type": normalized_mime,
"size": len(payload),
"sha256": sha256_hex,
"store_path": f"media_store/{blob_rel}",
"source_path": source_path,
}
def store_file(
self,
file_path: Path,
*,
kind: Optional[str] = None,
mime_type: Optional[str] = None,
source_path: Optional[str] = None,
) -> Dict[str, Any]:
path = Path(file_path).expanduser().resolve()
payload = path.read_bytes()
return self.store_bytes(
payload,
kind=kind,
mime_type=mime_type,
source_path=source_path or str(file_path),
)
def store_base64_data(
self,
data_base64: str,
*,
kind: Optional[str] = None,
mime_type: Optional[str] = None,
source_path: Optional[str] = None,
) -> Dict[str, Any]:
payload = _decode_base64_payload(data_base64)
return self.store_bytes(
payload,
kind=kind,
mime_type=mime_type,
source_path=source_path,
)
def store_data_url(
self,
data_url: str,
*,
kind: Optional[str] = None,
source_path: Optional[str] = None,
) -> Dict[str, Any]:
mime_type, base64_payload = _parse_data_url(data_url)
return self.store_base64_data(
base64_payload,
kind=kind,
mime_type=mime_type,
source_path=source_path,
)
def _resolve_blob_path(self, media_ref: Dict[str, Any]) -> Optional[Tuple[Path, Dict[str, Any]]]:
media_id = str(media_ref.get("media_id") or "").strip()
with self._lock:
index = self._load_index()
media = index.get("media") if isinstance(index.get("media"), dict) else {}
entry = media.get(media_id) if media_id else None
if isinstance(entry, dict):
rel = str(entry.get("blob_rel_path") or "").strip()
if rel:
path = self.root_dir / rel
if path.exists() and path.is_file():
return path, entry
store_path = str(media_ref.get("store_path") or "").strip()
if store_path.startswith("media_store/"):
candidate = self.conversations_dir / store_path
elif store_path:
candidate = self.root_dir / store_path
else:
candidate = None
if candidate and candidate.exists() and candidate.is_file():
if not isinstance(entry, dict):
entry = {}
return candidate, entry
return None
def load_bytes(self, media_ref: Dict[str, Any]) -> Optional[bytes]:
resolved = self._resolve_blob_path(media_ref or {})
if not resolved:
return None
blob_path, _entry = resolved
try:
return blob_path.read_bytes()
except Exception:
return None
def to_data_url(self, media_ref: Dict[str, Any]) -> Optional[str]:
payload = self.load_bytes(media_ref)
if payload is None:
return None
mime = _normalize_mime(
media_ref.get("mime_type"),
_normalize_kind(media_ref.get("kind"), media_ref.get("mime_type")),
)
b64 = base64.b64encode(payload).decode("utf-8")
return f"data:{mime};base64,{b64}"
def link_message_media(
self,
conversation_id: Optional[str],
message_id: Optional[str],
media_refs: List[Dict[str, Any]],
) -> None:
conv = str(conversation_id or "").strip()
msg = str(message_id or "").strip()
if not conv or not msg or not media_refs:
return
media_ids = [str((item or {}).get("media_id") or "").strip() for item in media_refs]
media_ids = [item for item in media_ids if item]
if not media_ids:
return
with self._lock:
index = self._load_index()
message_map = index.setdefault("message_map", {})
conv_map = message_map.get(conv) if isinstance(message_map.get(conv), dict) else {}
conv_map[msg] = {
"media_ids": media_ids,
"updated_at": datetime.now().isoformat(),
}
message_map[conv] = conv_map
self._save_index(index)

View File

@ -2,9 +2,191 @@
from __future__ import annotations
import json
from typing import Any, Dict, List, Optional, Tuple
def _looks_like_mcp_payload(function_name: str, result_data: Any) -> bool:
if isinstance(function_name, str) and function_name.startswith("mcp__"):
return True
if not isinstance(result_data, dict):
return False
if result_data.get("tool_alias") or result_data.get("server_id"):
return True
content = result_data.get("content")
if isinstance(content, list) and any(isinstance(item, dict) and item.get("type") for item in content):
return True
return False
def _format_mcp_resource_reference(item: Dict[str, Any]) -> str:
uri = str(item.get("uri") or item.get("url") or "").strip()
title = str(item.get("title") or item.get("name") or "").strip()
if uri and title:
return f"[MCP 资源] {title} -> {uri}"
if uri:
return f"[MCP 资源] {uri}"
if title:
return f"[MCP 资源] {title}"
return ""
def _sanitize_mcp_content_item(item: Dict[str, Any]) -> Dict[str, Any]:
sanitized = dict(item)
ctype = str(sanitized.get("type") or "").strip().lower()
if ctype in {"image", "audio"}:
data = str(sanitized.pop("data", "") or "")
if data:
sanitized["data_omitted"] = True
sanitized["data_length"] = len(data)
if ctype == "resource":
resource = sanitized.get("resource")
if isinstance(resource, dict) and "blob" in resource:
blob_data = str(resource.get("blob") or "")
resource_copy = dict(resource)
resource_copy.pop("blob", None)
if blob_data:
resource_copy["blob_omitted"] = True
resource_copy["blob_length"] = len(blob_data)
sanitized["resource"] = resource_copy
return sanitized
def extract_mcp_content_for_context(result_data: Dict[str, Any]) -> Dict[str, Any]:
"""提取 MCP content返回可读文本 + 可落盘媒体列表 + 去二进制后的 payload。"""
if not isinstance(result_data, dict):
return {
"text": "",
"media_items": [],
"sanitized_payload": result_data,
}
content_items = result_data.get("content")
if not isinstance(content_items, list):
raw_result = result_data.get("raw_result")
if isinstance(raw_result, dict) and isinstance(raw_result.get("content"), list):
content_items = raw_result.get("content")
else:
content_items = []
text_lines: List[str] = []
media_lines: List[str] = []
media_items: List[Dict[str, Any]] = []
sanitized_content: List[Dict[str, Any]] = []
for index, item in enumerate(content_items, start=1):
if not isinstance(item, dict):
continue
ctype = str(item.get("type") or "").strip().lower()
if ctype == "text":
text_value = str(item.get("text") or "").strip()
if text_value:
text_lines.append(text_value)
sanitized_content.append(_sanitize_mcp_content_item(item))
continue
if ctype in {"image", "audio"}:
kind = "image" if ctype == "image" else "audio"
mime_type = str(item.get("mimeType") or item.get("mime_type") or "").strip().lower()
if not mime_type:
mime_type = "image/png" if kind == "image" else "audio/wav"
data_base64 = str(item.get("data") or "")
if data_base64:
media_items.append(
{
"index": index,
"item_type": ctype,
"kind": kind,
"mime_type": mime_type,
"data_base64": data_base64,
"label": item.get("label") or f"{kind}_{index}",
"name": item.get("name"),
"title": item.get("title"),
"uri": item.get("uri"),
"url": item.get("url"),
}
)
media_lines.append(
f"[MCP {kind} #{index}] {mime_type}{len(data_base64)} base64 chars"
)
else:
media_lines.append(f"[MCP {kind} #{index}] 未包含数据")
sanitized_content.append(_sanitize_mcp_content_item(item))
continue
if ctype in {"resource_link", "resourcelink"}:
line = _format_mcp_resource_reference(item)
if line:
text_lines.append(line)
sanitized_content.append(_sanitize_mcp_content_item(item))
continue
if ctype == "resource":
resource = item.get("resource")
resource_line = _format_mcp_resource_reference(item)
if isinstance(resource, dict):
resource_line = _format_mcp_resource_reference(resource) or resource_line
resource_text = str(resource.get("text") or "").strip()
if resource_text:
text_lines.append(resource_text)
elif resource_line:
text_lines.append(resource_line)
elif resource_line:
text_lines.append(resource_line)
sanitized_content.append(_sanitize_mcp_content_item(item))
continue
fallback_line = _format_mcp_resource_reference(item)
if fallback_line:
text_lines.append(fallback_line)
else:
text_lines.append(f"[MCP content:{ctype or 'unknown'}]")
sanitized_content.append(_sanitize_mcp_content_item(item))
message = str(result_data.get("message") or "").strip()
if message and message != "MCP 工具调用完成":
if message not in text_lines:
text_lines.insert(0, message)
if media_lines:
text_lines.extend(media_lines)
if not text_lines:
structured = result_data.get("structured_content")
if structured is None:
structured = result_data.get("structuredContent")
if structured is not None:
try:
text_lines.append(json.dumps(structured, ensure_ascii=False))
except Exception:
text_lines.append(str(structured))
if not text_lines and message:
text_lines.append(message)
if not text_lines:
text_lines.append("MCP 工具调用完成")
sanitized_payload: Dict[str, Any] = dict(result_data)
if isinstance(result_data.get("content"), list):
sanitized_payload["content"] = sanitized_content
raw_result = result_data.get("raw_result")
if isinstance(raw_result, dict):
raw_sanitized = dict(raw_result)
raw_content = raw_result.get("content")
if isinstance(raw_content, list):
raw_sanitized["content"] = [
_sanitize_mcp_content_item(item) if isinstance(item, dict) else item
for item in raw_content
]
sanitized_payload["raw_result"] = raw_sanitized
return {
"text": "\n".join(line for line in text_lines if line),
"media_items": media_items,
"sanitized_payload": sanitized_payload,
}
def format_read_file_result(result_data: Dict[str, Any]) -> str:
"""格式化 read_file 工具的输出,兼容读取/搜索/抽取模式。"""
if not isinstance(result_data, dict):
@ -83,6 +265,12 @@ def format_tool_result_for_context(function_name: str, result_data: Any, raw_tex
parts.append(str(output))
return "\n".join(parts).strip() or raw_text
if _looks_like_mcp_payload(function_name, result_data):
parsed = extract_mcp_content_for_context(result_data if isinstance(result_data, dict) else {})
text = str(parsed.get("text") or "").strip()
if text:
return text
if function_name == "read_file" and isinstance(result_data, dict):
return format_read_file_result(result_data)