fix: render user media previews from media store refs

This commit is contained in:
JOJO 2026-04-27 12:57:13 +08:00
parent 727cafc370
commit 9dfb1567d2
8 changed files with 250 additions and 20 deletions

View File

@ -752,13 +752,27 @@ async def handle_task_with_sender(
if auto_user_message_event: if auto_user_message_event:
user_message_metadata["is_auto_generated"] = True user_message_metadata["is_auto_generated"] = True
user_message_metadata["auto_message_type"] = "completion_notice" user_message_metadata["auto_message_type"] = "completion_notice"
web_terminal.context_manager.add_conversation( saved_user_message = web_terminal.context_manager.add_conversation(
"user", "user",
message, message,
images=images, images=images,
videos=videos, videos=videos,
metadata=user_message_metadata metadata=user_message_metadata
) )
if not auto_user_message_event:
try:
sender(
'user_message',
{
"message": message,
"images": (saved_user_message or {}).get("images") or images or [],
"videos": (saved_user_message or {}).get("videos") or videos or [],
"media_refs": (saved_user_message or {}).get("media_refs") or [],
"conversation_id": conversation_id,
},
)
except Exception as exc:
debug_log(f"[TaskFlow] 发送 user_message 回显失败: {exc}")
try: try:
user_message_index = len(getattr(web_terminal.context_manager, "conversation_history", []) or []) - 1 user_message_index = len(getattr(web_terminal.context_manager, "conversation_history", []) or []) - 1
except Exception: except Exception:

View File

@ -8,9 +8,10 @@ import asyncio, json, time, re, os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from collections import defaultdict, Counter, deque from collections import defaultdict, Counter, deque
from io import BytesIO
from typing import Dict, Any, Optional, List, Tuple from typing import Dict, Any, Optional, List, Tuple
from flask import Blueprint, request, jsonify, session from flask import Blueprint, request, jsonify, session, send_file
from flask_socketio import emit from flask_socketio import emit
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
import zipfile import zipfile
@ -542,6 +543,44 @@ def get_conversation_messages(conversation_id, terminal: WebTerminal, workspace:
}), 500 }), 500
@conversation_bp.route('/api/conversations/media/<path:media_id>', methods=['GET'])
@api_login_required
@with_terminal
def download_conversation_media(media_id, terminal: WebTerminal, workspace: UserWorkspace, username: str):
"""按 media_store 中的 media_id 下载会话媒体。"""
try:
ctx = getattr(terminal, "context_manager", None)
media_store = getattr(ctx, "media_store", None)
if media_store is None:
return jsonify({"success": False, "error": "media_store 不可用"}), 503
target_id = str(media_id or "").strip()
if not target_id:
return jsonify({"success": False, "error": "media_id 不能为空"}), 400
entry = media_store.get_media_entry(target_id)
if not isinstance(entry, dict):
return jsonify({"success": False, "error": "媒体不存在"}), 404
payload = media_store.load_bytes_by_media_id(target_id)
if payload is None:
return jsonify({"success": False, "error": "媒体文件不存在"}), 404
mime_type = str(entry.get("mime_type") or "application/octet-stream").strip() or "application/octet-stream"
blob_name = str(entry.get("blob_rel_path") or "")
filename = Path(blob_name).name if blob_name else target_id.replace(":", "_")
return send_file(
BytesIO(payload),
mimetype=mime_type,
as_attachment=False,
download_name=filename,
conditional=True,
etag=True,
max_age=86400,
)
except Exception as exc:
return jsonify({"success": False, "error": str(exc)}), 500
@conversation_bp.route('/api/conversations/<conversation_id>/versioning', methods=['GET']) @conversation_bp.route('/api/conversations/<conversation_id>/versioning', methods=['GET'])
@api_login_required @api_login_required
@with_terminal @with_terminal

View File

@ -152,8 +152,12 @@ export const historyMethods = {
} }
const messagesData = await messagesResponse.json(); const messagesData = await messagesResponse.json();
const rawMessages = Array.isArray(messagesData?.data?.messages) ? messagesData.data.messages : []; const rawMessages = Array.isArray(messagesData?.data?.messages)
const lastAssistantRaw = [...rawMessages].reverse().find((m: any) => m?.role === 'assistant'); ? messagesData.data.messages
: [];
const lastAssistantRaw = [...rawMessages]
.reverse()
.find((m: any) => m?.role === 'assistant');
restoreDebugLog('history:fetch:response', { restoreDebugLog('history:fetch:response', {
loadSeq, loadSeq,
success: !!messagesData?.success, success: !!messagesData?.success,
@ -272,6 +276,8 @@ export const historyMethods = {
} }
const images = message.images || (message.metadata && message.metadata.images) || []; const images = message.images || (message.metadata && message.metadata.images) || [];
const videos = message.videos || (message.metadata && message.metadata.videos) || []; const videos = message.videos || (message.metadata && message.metadata.videos) || [];
const mediaRefs =
message.media_refs || (message.metadata && message.metadata.media_refs) || [];
if (Array.isArray(images) && images.length) { if (Array.isArray(images) && images.length) {
historyHasImages = true; historyHasImages = true;
} }
@ -283,6 +289,7 @@ export const historyMethods = {
content: message.content || '', content: message.content || '',
images, images,
videos, videos,
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
metadata: message.metadata || {} metadata: message.metadata || {}
}); });
debugLog('添加用户消息:', message.content?.substring(0, 50) + '...'); debugLog('添加用户消息:', message.content?.substring(0, 50) + '...');

View File

@ -994,7 +994,11 @@ export const taskPollingMethods = {
if (targetAction.tool && targetAction.tool.name === 'manage_personalization') { if (targetAction.tool && targetAction.tool.name === 'manage_personalization') {
let result = data.result; let result = data.result;
if (typeof result === 'string') { if (typeof result === 'string') {
try { result = JSON.parse(result); } catch (e) { /* ignore */ } try {
result = JSON.parse(result);
} catch (e) {
/* ignore */
}
} }
// 处理主题变更 // 处理主题变更
if (result?.theme_changed === true && result.new_theme) { if (result?.theme_changed === true && result.new_theme) {
@ -1029,7 +1033,11 @@ export const taskPollingMethods = {
// 待办工具执行完成后主动刷新左侧待办列表(网页端不再依赖 websocket todo_updated // 待办工具执行完成后主动刷新左侧待办列表(网页端不再依赖 websocket todo_updated
if (data.status === 'completed') { if (data.status === 'completed') {
const toolName = String(targetAction?.tool?.name || '').toLowerCase(); const toolName = String(targetAction?.tool?.name || '').toLowerCase();
if (toolName.startsWith('todo_') || toolName === 'todo_create' || toolName === 'todo_update_task') { if (
toolName.startsWith('todo_') ||
toolName === 'todo_create' ||
toolName === 'todo_update_task'
) {
this.scheduleTodoListRefresh(80); this.scheduleTodoListRefresh(80);
} }
} }
@ -1269,7 +1277,33 @@ export const taskPollingMethods = {
}); });
const isAutoUserMessage = isSystemAutoUserMessagePayload(data); const isAutoUserMessage = isSystemAutoUserMessagePayload(data);
if (!isAutoUserMessage) { if (!isAutoUserMessage) {
this.chatAddUserMessage(message, data?.images || [], data?.videos || []); const incomingImages = Array.isArray(data?.images) ? data.images : [];
const incomingVideos = Array.isArray(data?.videos) ? data.videos : [];
const incomingMediaRefs = Array.isArray(data?.media_refs)
? data.media_refs
: Array.isArray(data?.mediaRefs)
? data.mediaRefs
: [];
const last =
Array.isArray(this.messages) && this.messages.length > 0
? this.messages[this.messages.length - 1]
: null;
const isLikelyOptimisticEcho = !!(
last &&
last.role === 'user' &&
String(last.content || '').trim() === message &&
JSON.stringify(last.images || []) === JSON.stringify(incomingImages || []) &&
JSON.stringify(last.videos || []) === JSON.stringify(incomingVideos || [])
);
if (isLikelyOptimisticEcho) {
last.media_refs = incomingMediaRefs;
last.metadata = {
...(last.metadata || {}),
media_refs: incomingMediaRefs
};
} else {
this.chatAddUserMessage(message, incomingImages, incomingVideos, incomingMediaRefs);
}
this.taskInProgress = true; this.taskInProgress = true;
this.streamingMessage = false; this.streamingMessage = false;
this.stopRequested = false; this.stopRequested = false;
@ -1317,7 +1351,8 @@ export const taskPollingMethods = {
userMDebug('taskPolling.handleSystemMessage:after-append', { userMDebug('taskPolling.handleSystemMessage:after-append', {
currentMessagesLength: Array.isArray(this.messages) ? this.messages.length : -1, currentMessagesLength: Array.isArray(this.messages) ? this.messages.length : -1,
lastRole: this.messages?.[this.messages.length - 1]?.role || null, lastRole: this.messages?.[this.messages.length - 1]?.role || null,
lastActions: this.messages?.[this.messages.length - 1]?.actions?.map((a: any) => a?.type) || [] lastActions:
this.messages?.[this.messages.length - 1]?.actions?.map((a: any) => a?.type) || []
}); });
}, },
@ -1358,8 +1393,7 @@ export const taskPollingMethods = {
const errorMessage = data.message || '未知错误'; const errorMessage = data.message || '未知错误';
const errorType = data.error_type || 'unknown'; const errorType = data.error_type || 'unknown';
const isToolArgumentParseError = const isToolArgumentParseError =
errorType === 'parameter_format_error' || errorType === 'parameter_format_error' || /工具参数解析失败/.test(String(errorMessage || ''));
/工具参数解析失败/.test(String(errorMessage || ''));
// 工具参数解析失败属于“单个工具调用失败”,后端会继续执行主任务。 // 工具参数解析失败属于“单个工具调用失败”,后端会继续执行主任务。
// 这里不能停止轮询,否则会出现“后端继续跑、前端不再更新”的假死状态。 // 这里不能停止轮询,否则会出现“后端继续跑、前端不再更新”的假死状态。

View File

@ -18,18 +18,26 @@
<div class="message-text user-bubble-text"> <div class="message-text user-bubble-text">
<div v-if="msg.content" class="bubble-text">{{ msg.content }}</div> <div v-if="msg.content" class="bubble-text">{{ msg.content }}</div>
<div v-if="msg.images && msg.images.length" class="image-inline-row"> <div v-if="msg.images && msg.images.length" class="image-inline-row">
<div class="image-thumbnail-wrapper" v-for="img in msg.images" :key="img"> <div
class="image-thumbnail-wrapper"
v-for="(img, imgIndex) in msg.images"
:key="mediaPreviewKey(msg, img, imgIndex)"
>
<img <img
:src="getPreviewUrl(img)" :src="getPreviewUrl(msg, img, 'image')"
:alt="formatImageName(img)" :alt="formatImageName(img)"
class="image-thumbnail" class="image-thumbnail"
/> />
</div> </div>
</div> </div>
<div v-if="msg.videos && msg.videos.length" class="image-inline-row video-inline-row"> <div v-if="msg.videos && msg.videos.length" class="image-inline-row video-inline-row">
<div class="image-thumbnail-wrapper" v-for="video in msg.videos" :key="video"> <div
class="image-thumbnail-wrapper"
v-for="(video, videoIndex) in msg.videos"
:key="mediaPreviewKey(msg, video, videoIndex)"
>
<img <img
:src="getPreviewUrl(video)" :src="getPreviewUrl(msg, video, 'video')"
:alt="formatImageName(video)" :alt="formatImageName(video)"
class="image-thumbnail" class="image-thumbnail"
/> />
@ -1054,17 +1062,86 @@ function iconStyleSafe(key: string, size?: string) {
return {}; return {};
} }
function formatImageName(path: string): string { function normalizeMediaPath(input: any): string {
if (!path) return ''; if (!input) return '';
if (typeof input === 'string') {
return input;
}
if (typeof input?.path === 'string') {
return input.path;
}
if (typeof input?.source_path === 'string') {
return input.source_path;
}
return '';
}
function formatImageName(input: any): string {
const path = normalizeMediaPath(input);
if (!path) {
if (typeof input?.name === 'string' && input.name) {
return input.name;
}
if (typeof input?.title === 'string' && input.title) {
return input.title;
}
if (typeof input?.media_id === 'string' && input.media_id) {
return input.media_id;
}
return '';
}
const parts = path.split(/[/\\]/); const parts = path.split(/[/\\]/);
return parts[parts.length - 1] || path; return parts[parts.length - 1] || path;
} }
function getPreviewUrl(path: string): string { function resolveMessageMediaRef(message: any, input: any, kind: 'image' | 'video'): any | null {
const refs = message?.media_refs || message?.metadata?.media_refs || [];
if (!Array.isArray(refs) || !refs.length) {
return null;
}
const mediaId = typeof input?.media_id === 'string' ? input.media_id : '';
if (mediaId) {
const matched = refs.find((item: any) => item?.media_id === mediaId);
if (matched) return matched;
}
const sourcePath = normalizeMediaPath(input);
if (sourcePath) {
const matched = refs.find(
(item: any) =>
item &&
item.kind === kind &&
typeof item.source_path === 'string' &&
item.source_path === sourcePath
);
if (matched) return matched;
}
const firstSameKind = refs.find((item: any) => item && item.kind === kind);
return firstSameKind || null;
}
function getPreviewUrl(message: any, input: any, kind: 'image' | 'video'): string {
const mediaRef = resolveMessageMediaRef(message, input, kind);
const mediaId = typeof mediaRef?.media_id === 'string' ? mediaRef.media_id : '';
if (mediaId) {
return `/api/conversations/media/${encodeURIComponent(mediaId)}`;
}
const path = normalizeMediaPath(input);
if (!path) return ''; if (!path) return '';
return `/api/gui/files/download?path=${encodeURIComponent(path)}`; return `/api/gui/files/download?path=${encodeURIComponent(path)}`;
} }
function mediaPreviewKey(message: any, input: any, index: number): string {
const mediaRef =
resolveMessageMediaRef(message, input, 'image') ||
resolveMessageMediaRef(message, input, 'video');
if (typeof mediaRef?.media_id === 'string' && mediaRef.media_id) {
return `${mediaRef.media_id}-${index}`;
}
const path = normalizeMediaPath(input);
if (path) return `${path}-${index}`;
return `media-${index}`;
}
function formatDurationMs(durationMs: number): string { function formatDurationMs(durationMs: number): string {
const totalSeconds = Math.max(0, Math.floor((durationMs || 0) / 1000)); const totalSeconds = Math.max(0, Math.floor((durationMs || 0) / 1000));
const hours = Math.floor(totalSeconds / 3600); const hours = Math.floor(totalSeconds / 3600);

View File

@ -903,7 +903,33 @@ export async function initializeLegacySocket(ctx: any) {
if (!message) { if (!message) {
return; return;
} }
ctx.chatAddUserMessage(message, data.images || [], data.videos || []); const incomingImages = Array.isArray(data.images) ? data.images : [];
const incomingVideos = Array.isArray(data.videos) ? data.videos : [];
const incomingMediaRefs = Array.isArray(data.media_refs)
? data.media_refs
: Array.isArray(data.mediaRefs)
? data.mediaRefs
: [];
const last =
Array.isArray(ctx.messages) && ctx.messages.length
? ctx.messages[ctx.messages.length - 1]
: null;
const isLikelyOptimisticEcho = !!(
last &&
last.role === 'user' &&
String(last.content || '').trim() === message &&
JSON.stringify(last.images || []) === JSON.stringify(incomingImages || []) &&
JSON.stringify(last.videos || []) === JSON.stringify(incomingVideos || [])
);
if (isLikelyOptimisticEcho) {
last.media_refs = incomingMediaRefs;
last.metadata = {
...(last.metadata || {}),
media_refs: incomingMediaRefs
};
} else {
ctx.chatAddUserMessage(message, incomingImages, incomingVideos, incomingMediaRefs);
}
ctx.taskInProgress = true; ctx.taskInProgress = true;
ctx.streamingMessage = false; ctx.streamingMessage = false;
ctx.stopRequested = false; ctx.stopRequested = false;

View File

@ -174,14 +174,21 @@ export const useChatStore = defineStore('chat', {
this.currentMessageIndex = this.messages.length - 1; this.currentMessageIndex = this.messages.length - 1;
return message; return message;
}, },
addUserMessage(content: string, images: string[] = [], videos: string[] = []) { addUserMessage(
content: string,
images: string[] = [],
videos: string[] = [],
mediaRefs: Array<Record<string, any>> = []
) {
const startedAt = new Date().toISOString(); const startedAt = new Date().toISOString();
this.messages.push({ this.messages.push({
role: 'user', role: 'user',
content, content,
images, images,
videos, videos,
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
metadata: { metadata: {
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
work_timer: { work_timer: {
status: 'working', status: 'working',
started_at: startedAt started_at: startedAt

View File

@ -321,6 +321,18 @@ class MediaStore:
return candidate, entry return candidate, entry
return None return None
def get_media_entry(self, media_id: str) -> Optional[Dict[str, Any]]:
target_id = str(media_id or "").strip()
if not target_id:
return None
with self._lock:
index = self._load_index()
media = index.get("media") if isinstance(index.get("media"), dict) else {}
entry = media.get(target_id)
if isinstance(entry, dict):
return dict(entry)
return None
def load_bytes(self, media_ref: Dict[str, Any]) -> Optional[bytes]: def load_bytes(self, media_ref: Dict[str, Any]) -> Optional[bytes]:
resolved = self._resolve_blob_path(media_ref or {}) resolved = self._resolve_blob_path(media_ref or {})
if not resolved: if not resolved:
@ -331,6 +343,21 @@ class MediaStore:
except Exception: except Exception:
return None return None
def load_bytes_by_media_id(self, media_id: str) -> Optional[bytes]:
target_id = str(media_id or "").strip()
if not target_id:
return None
entry = self.get_media_entry(target_id)
if not isinstance(entry, dict):
return None
ref = {
"media_id": target_id,
"store_path": entry.get("blob_rel_path"),
"kind": entry.get("kind"),
"mime_type": entry.get("mime_type"),
}
return self.load_bytes(ref)
def to_data_url(self, media_ref: Dict[str, Any]) -> Optional[str]: def to_data_url(self, media_ref: Dict[str, Any]) -> Optional[str]:
payload = self.load_bytes(media_ref) payload = self.load_bytes(media_ref)
if payload is None: if payload is None:
@ -366,4 +393,3 @@ class MediaStore:
} }
message_map[conv] = conv_map message_map[conv] = conv_map
self._save_index(index) self._save_index(index)