JOJO/agent-Specialization
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: render user media previews from media store refs

Browse Source
This commit is contained in:
JOJO 2026-04-27 12:57:13 +08:00
parent 727cafc370
commit 9dfb1567d2
8 changed files with 250 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
server/chat_flow_task_main.py
View File

@ -752,13 +752,27 @@ async def handle_task_with_sender(
if auto_user_message_event:
user_message_metadata["is_auto_generated"] = True
user_message_metadata["auto_message_type"] = "completion_notice"
web_terminal.context_manager.add_conversation(
saved_user_message = web_terminal.context_manager.add_conversation(
"user",
message,
images=images,
videos=videos,
metadata=user_message_metadata
)
if not auto_user_message_event:
try:
sender(
'user_message',
{
"message": message,
"images": (saved_user_message or {}).get("images") or images or [],
"videos": (saved_user_message or {}).get("videos") or videos or [],
"media_refs": (saved_user_message or {}).get("media_refs") or [],
"conversation_id": conversation_id,
},
)
except Exception as exc:
debug_log(f"[TaskFlow] 发送 user_message 回显失败: {exc}")
try:
user_message_index = len(getattr(web_terminal.context_manager, "conversation_history", []) or []) - 1
except Exception:

41
server/conversation.py
View File

@ -8,9 +8,10 @@ import asyncio, json, time, re, os
from datetime import datetime, timedelta
from pathlib import Path
from collections import defaultdict, Counter, deque
from io import BytesIO
from typing import Dict, Any, Optional, List, Tuple
from flask import Blueprint, request, jsonify, session
from flask import Blueprint, request, jsonify, session, send_file
from flask_socketio import emit
from werkzeug.utils import secure_filename
import zipfile
@ -542,6 +543,44 @@ def get_conversation_messages(conversation_id, terminal: WebTerminal, workspace:
}), 500
@conversation_bp.route('/api/conversations/media/<path:media_id>', methods=['GET'])
@api_login_required
@with_terminal
def download_conversation_media(media_id, terminal: WebTerminal, workspace: UserWorkspace, username: str):
"""按 media_store 中的 media_id 下载会话媒体。"""
try:
ctx = getattr(terminal, "context_manager", None)
media_store = getattr(ctx, "media_store", None)
if media_store is None:
return jsonify({"success": False, "error": "media_store 不可用"}), 503
target_id = str(media_id or "").strip()
if not target_id:
return jsonify({"success": False, "error": "media_id 不能为空"}), 400
entry = media_store.get_media_entry(target_id)
if not isinstance(entry, dict):
return jsonify({"success": False, "error": "媒体不存在"}), 404
payload = media_store.load_bytes_by_media_id(target_id)
if payload is None:
return jsonify({"success": False, "error": "媒体文件不存在"}), 404
mime_type = str(entry.get("mime_type") or "application/octet-stream").strip() or "application/octet-stream"
blob_name = str(entry.get("blob_rel_path") or "")
filename = Path(blob_name).name if blob_name else target_id.replace(":", "_")
return send_file(
BytesIO(payload),
mimetype=mime_type,
as_attachment=False,
download_name=filename,
conditional=True,
etag=True,
max_age=86400,
)
except Exception as exc:
return jsonify({"success": False, "error": str(exc)}), 500
@conversation_bp.route('/api/conversations/<conversation_id>/versioning', methods=['GET'])
@api_login_required
@with_terminal

11
static/src/app/methods/history.ts
View File

@ -152,8 +152,12 @@ export const historyMethods = {
}
const messagesData = await messagesResponse.json();
const rawMessages = Array.isArray(messagesData?.data?.messages) ? messagesData.data.messages : [];
const lastAssistantRaw = [...rawMessages].reverse().find((m: any) => m?.role === 'assistant');
const rawMessages = Array.isArray(messagesData?.data?.messages)
? messagesData.data.messages
: [];
const lastAssistantRaw = [...rawMessages]
.reverse()
.find((m: any) => m?.role === 'assistant');
restoreDebugLog('history:fetch:response', {
loadSeq,
success: !!messagesData?.success,
@ -272,6 +276,8 @@ export const historyMethods = {
}
const images = message.images || (message.metadata && message.metadata.images) || [];
const videos = message.videos || (message.metadata && message.metadata.videos) || [];
const mediaRefs =
message.media_refs || (message.metadata && message.metadata.media_refs) || [];
if (Array.isArray(images) && images.length) {
historyHasImages = true;
}
@ -283,6 +289,7 @@ export const historyMethods = {
content: message.content || '',
images,
videos,
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
metadata: message.metadata || {}
});
debugLog('添加用户消息:', message.content?.substring(0, 50) + '...');

46
static/src/app/methods/taskPolling.ts
View File

@ -994,7 +994,11 @@ export const taskPollingMethods = {
if (targetAction.tool && targetAction.tool.name === 'manage_personalization') {
let result = data.result;
if (typeof result === 'string') {
try { result = JSON.parse(result); } catch (e) { /* ignore */ }
try {
result = JSON.parse(result);
} catch (e) {
/* ignore */
}
}
// 处理主题变更
if (result?.theme_changed === true && result.new_theme) {
@ -1029,7 +1033,11 @@ export const taskPollingMethods = {
// 待办工具执行完成后主动刷新左侧待办列表（网页端不再依赖 websocket todo_updated）
if (data.status === 'completed') {
const toolName = String(targetAction?.tool?.name || '').toLowerCase();
if (toolName.startsWith('todo_') || toolName === 'todo_create' || toolName === 'todo_update_task') {
if (
toolName.startsWith('todo_') ||
toolName === 'todo_create' ||
toolName === 'todo_update_task'
) {
this.scheduleTodoListRefresh(80);
}
}
@ -1269,7 +1277,33 @@ export const taskPollingMethods = {
});
const isAutoUserMessage = isSystemAutoUserMessagePayload(data);
if (!isAutoUserMessage) {
this.chatAddUserMessage(message, data?.images || [], data?.videos || []);
const incomingImages = Array.isArray(data?.images) ? data.images : [];
const incomingVideos = Array.isArray(data?.videos) ? data.videos : [];
const incomingMediaRefs = Array.isArray(data?.media_refs)
? data.media_refs
: Array.isArray(data?.mediaRefs)
? data.mediaRefs
: [];
const last =
Array.isArray(this.messages) && this.messages.length > 0
? this.messages[this.messages.length - 1]
: null;
const isLikelyOptimisticEcho = !!(
last &&
last.role === 'user' &&
String(last.content || '').trim() === message &&
JSON.stringify(last.images || []) === JSON.stringify(incomingImages || []) &&
JSON.stringify(last.videos || []) === JSON.stringify(incomingVideos || [])
);
if (isLikelyOptimisticEcho) {
last.media_refs = incomingMediaRefs;
last.metadata = {
...(last.metadata || {}),
media_refs: incomingMediaRefs
};
} else {
this.chatAddUserMessage(message, incomingImages, incomingVideos, incomingMediaRefs);
}
this.taskInProgress = true;
this.streamingMessage = false;
this.stopRequested = false;
@ -1317,7 +1351,8 @@ export const taskPollingMethods = {
userMDebug('taskPolling.handleSystemMessage:after-append', {
currentMessagesLength: Array.isArray(this.messages) ? this.messages.length : -1,
lastRole: this.messages?.[this.messages.length - 1]?.role || null,
lastActions: this.messages?.[this.messages.length - 1]?.actions?.map((a: any) => a?.type) || []
lastActions:
this.messages?.[this.messages.length - 1]?.actions?.map((a: any) => a?.type) || []
});
},
@ -1358,8 +1393,7 @@ export const taskPollingMethods = {
const errorMessage = data.message || '未知错误';
const errorType = data.error_type || 'unknown';
const isToolArgumentParseError =
errorType === 'parameter_format_error' ||
/工具参数解析失败/.test(String(errorMessage || ''));
errorType === 'parameter_format_error' || /工具参数解析失败/.test(String(errorMessage || ''));
// 工具参数解析失败属于“单个工具调用失败”，后端会继续执行主任务。
// 这里不能停止轮询，否则会出现“后端继续跑、前端不再更新”的假死状态。

91
static/src/components/chat/ChatArea.vue
View File

@ -18,18 +18,26 @@
<div class="message-text user-bubble-text">
<div v-if="msg.content" class="bubble-text">{{ msg.content }}</div>
<div v-if="msg.images && msg.images.length" class="image-inline-row">
<div class="image-thumbnail-wrapper" v-for="img in msg.images" :key="img">
<div
class="image-thumbnail-wrapper"
v-for="(img, imgIndex) in msg.images"
:key="mediaPreviewKey(msg, img, imgIndex)"
>
<img
:src="getPreviewUrl(img)"
:src="getPreviewUrl(msg, img, 'image')"
:alt="formatImageName(img)"
class="image-thumbnail"
/>
</div>
</div>
<div v-if="msg.videos && msg.videos.length" class="image-inline-row video-inline-row">
<div class="image-thumbnail-wrapper" v-for="video in msg.videos" :key="video">
<div
class="image-thumbnail-wrapper"
v-for="(video, videoIndex) in msg.videos"
:key="mediaPreviewKey(msg, video, videoIndex)"
>
<img
:src="getPreviewUrl(video)"
:src="getPreviewUrl(msg, video, 'video')"
:alt="formatImageName(video)"
class="image-thumbnail"
/>
@ -1054,17 +1062,86 @@ function iconStyleSafe(key: string, size?: string) {
return {};
}
function formatImageName(path: string): string {
if (!path) return '';
function normalizeMediaPath(input: any): string {
if (!input) return '';
if (typeof input === 'string') {
return input;
}
if (typeof input?.path === 'string') {
return input.path;
}
if (typeof input?.source_path === 'string') {
return input.source_path;
}
return '';
}
function formatImageName(input: any): string {
const path = normalizeMediaPath(input);
if (!path) {
if (typeof input?.name === 'string' && input.name) {
return input.name;
}
if (typeof input?.title === 'string' && input.title) {
return input.title;
}
if (typeof input?.media_id === 'string' && input.media_id) {
return input.media_id;
}
return '';
}
const parts = path.split(/[/\\]/);
return parts[parts.length - 1] || path;
}
function getPreviewUrl(path: string): string {
function resolveMessageMediaRef(message: any, input: any, kind: 'image' | 'video'): any | null {
const refs = message?.media_refs || message?.metadata?.media_refs || [];
if (!Array.isArray(refs) || !refs.length) {
return null;
}
const mediaId = typeof input?.media_id === 'string' ? input.media_id : '';
if (mediaId) {
const matched = refs.find((item: any) => item?.media_id === mediaId);
if (matched) return matched;
}
const sourcePath = normalizeMediaPath(input);
if (sourcePath) {
const matched = refs.find(
(item: any) =>
item &&
item.kind === kind &&
typeof item.source_path === 'string' &&
item.source_path === sourcePath
);
if (matched) return matched;
}
const firstSameKind = refs.find((item: any) => item && item.kind === kind);
return firstSameKind || null;
}
function getPreviewUrl(message: any, input: any, kind: 'image' | 'video'): string {
const mediaRef = resolveMessageMediaRef(message, input, kind);
const mediaId = typeof mediaRef?.media_id === 'string' ? mediaRef.media_id : '';
if (mediaId) {
return `/api/conversations/media/${encodeURIComponent(mediaId)}`;
}
const path = normalizeMediaPath(input);
if (!path) return '';
return `/api/gui/files/download?path=${encodeURIComponent(path)}`;
}
function mediaPreviewKey(message: any, input: any, index: number): string {
const mediaRef =
resolveMessageMediaRef(message, input, 'image') ||
resolveMessageMediaRef(message, input, 'video');
if (typeof mediaRef?.media_id === 'string' && mediaRef.media_id) {
return `${mediaRef.media_id}-${index}`;
}
const path = normalizeMediaPath(input);
if (path) return `${path}-${index}`;
return `media-${index}`;
}
function formatDurationMs(durationMs: number): string {
const totalSeconds = Math.max(0, Math.floor((durationMs || 0) / 1000));
const hours = Math.floor(totalSeconds / 3600);

28
static/src/composables/useLegacySocket.ts
View File

@ -903,7 +903,33 @@ export async function initializeLegacySocket(ctx: any) {
if (!message) {
return;
}
ctx.chatAddUserMessage(message, data.images || [], data.videos || []);
const incomingImages = Array.isArray(data.images) ? data.images : [];
const incomingVideos = Array.isArray(data.videos) ? data.videos : [];
const incomingMediaRefs = Array.isArray(data.media_refs)
? data.media_refs
: Array.isArray(data.mediaRefs)
? data.mediaRefs
: [];
const last =
Array.isArray(ctx.messages) && ctx.messages.length
? ctx.messages[ctx.messages.length - 1]
: null;
const isLikelyOptimisticEcho = !!(
last &&
last.role === 'user' &&
String(last.content || '').trim() === message &&
JSON.stringify(last.images || []) === JSON.stringify(incomingImages || []) &&
JSON.stringify(last.videos || []) === JSON.stringify(incomingVideos || [])
);
if (isLikelyOptimisticEcho) {
last.media_refs = incomingMediaRefs;
last.metadata = {
...(last.metadata || {}),
media_refs: incomingMediaRefs
};
} else {
ctx.chatAddUserMessage(message, incomingImages, incomingVideos, incomingMediaRefs);
}
ctx.taskInProgress = true;
ctx.streamingMessage = false;
ctx.stopRequested = false;

9
static/src/stores/chat.ts
View File

@ -174,14 +174,21 @@ export const useChatStore = defineStore('chat', {
this.currentMessageIndex = this.messages.length - 1;
return message;
},
addUserMessage(content: string, images: string[] = [], videos: string[] = []) {
addUserMessage(
content: string,
images: string[] = [],
videos: string[] = [],
mediaRefs: Array<Record<string, any>> = []
) {
const startedAt = new Date().toISOString();
this.messages.push({
role: 'user',
content,
images,
videos,
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
metadata: {
media_refs: Array.isArray(mediaRefs) ? mediaRefs : [],
work_timer: {
status: 'working',
started_at: startedAt

28
utils/media_store.py
View File

@ -321,6 +321,18 @@ class MediaStore:
return candidate, entry
return None
def get_media_entry(self, media_id: str) -> Optional[Dict[str, Any]]:
target_id = str(media_id or "").strip()
if not target_id:
return None
with self._lock:
index = self._load_index()
media = index.get("media") if isinstance(index.get("media"), dict) else {}
entry = media.get(target_id)
if isinstance(entry, dict):
return dict(entry)
return None
def load_bytes(self, media_ref: Dict[str, Any]) -> Optional[bytes]:
resolved = self._resolve_blob_path(media_ref or {})
if not resolved:
@ -331,6 +343,21 @@ class MediaStore:
except Exception:
return None
def load_bytes_by_media_id(self, media_id: str) -> Optional[bytes]:
target_id = str(media_id or "").strip()
if not target_id:
return None
entry = self.get_media_entry(target_id)
if not isinstance(entry, dict):
return None
ref = {
"media_id": target_id,
"store_path": entry.get("blob_rel_path"),
"kind": entry.get("kind"),
"mime_type": entry.get("mime_type"),
}
return self.load_bytes(ref)
def to_data_url(self, media_ref: Dict[str, Any]) -> Optional[str]:
payload = self.load_bytes(media_ref)
if payload is None:
@ -366,4 +393,3 @@ class MediaStore:
}
message_map[conv] = conv_map
self._save_index(index)