diff --git a/core/main_terminal.py b/core/main_terminal.py index 59f19d7..a6aaa55 100644 --- a/core/main_terminal.py +++ b/core/main_terminal.py @@ -119,6 +119,7 @@ class MainTerminal: self.terminal_ops = TerminalOperator(project_path, container_session=container_session) self.ocr_client = OCRClient(project_path, self.file_manager) self.pending_image_view = None # 供 view_image 工具使用，保存一次性图片插入请求 + self.pending_video_view = None # 供 view_video 工具使用，保存一次性视频插入请求 # 新增：终端管理器 self.terminal_manager = TerminalManager( @@ -1826,6 +1827,24 @@ class MainTerminal: } } }) + if getattr(self, "model_key", None) == "kimi-k2.5": + tools.append({ + "type": "function", + "function": { + "name": "view_video", + "description": "将指定本地视频插入到对话中（系统代发一条包含视频的消息），便于模型查看视频内容。", + "parameters": { + "type": "object", + "properties": self._inject_intent({ + "path": { + "type": "string", + "description": "项目内的视频相对路径（不要以 /workspace 开头），支持 mp4/mov/mkv/avi/webm。" + } + }), + "required": ["path"] + } + } + }) # 附加自定义工具（仅管理员可见） custom_tools = self._build_custom_tools() if custom_tools: @@ -1923,6 +1942,26 @@ class MainTerminal: "path": str(path) } result = {"success": True, "message": "图片已请求插入到对话中，将在后续消息中呈现。", "path": path} + elif tool_name == "view_video": + path = (arguments.get("path") or "").strip() + if not path: + return json.dumps({"success": False, "error": "path 不能为空"}, ensure_ascii=False) + if path.startswith("/workspace"): + return json.dumps({"success": False, "error": "非法路径，超出项目根目录，请使用相对路径"}, ensure_ascii=False) + abs_path = (Path(self.context_manager.project_path) / path).resolve() + try: + abs_path.relative_to(Path(self.context_manager.project_path).resolve()) + except Exception: + return json.dumps({"success": False, "error": "非法路径，超出项目根目录，请使用相对路径"}, ensure_ascii=False) + if not abs_path.exists() or not abs_path.is_file(): + return json.dumps({"success": False, "error": f"视频不存在: {path}"}, ensure_ascii=False) + allowed_ext = {".mp4", ".mov", ".mkv", ".avi", ".webm"} + if abs_path.suffix.lower() not in allowed_ext: + return json.dumps({"success": False, "error": f"不支持的视频格式: {abs_path.suffix}"}, ensure_ascii=False) + if abs_path.stat().st_size > 50 * 1024 * 1024: + return json.dumps({"success": False, "error": "视频过大，需 <= 50MB"}, ensure_ascii=False) + self.pending_video_view = {"path": str(path)} + result = {"success": True, "message": "视频已请求插入到对话中，将在后续消息中呈现。", "path": path} # 终端会话管理工具 elif tool_name == "terminal_session": @@ -2511,9 +2550,10 @@ class MainTerminal: else: # User 或普通 System 消息 images = conv.get("images") or metadata.get("images") or [] + videos = conv.get("videos") or metadata.get("videos") or [] content_payload = ( - self.context_manager._build_content_with_images(conv["content"], images) - if images else conv["content"] + self.context_manager._build_content_with_images(conv["content"], images, videos) + if (images or videos) else conv["content"] ) messages.append({ "role": conv["role"], @@ -2686,6 +2726,8 @@ class MainTerminal: profile = get_model_profile(model_key) if getattr(self.context_manager, "has_images", False) and model_key not in {"qwen3-vl-plus", "kimi-k2.5"}: raise ValueError("当前对话包含图片，仅支持 Qwen-VL 或 Kimi-k2.5") + if getattr(self.context_manager, "has_videos", False) and model_key != "kimi-k2.5": + raise ValueError("当前对话包含视频，仅支持 Kimi-k2.5") self.model_key = model_key self.model_profile = profile # 将模型标识传递给底层 API 客户端，便于按模型做兼容处理 diff --git a/core/web_terminal.py b/core/web_terminal.py index 82a04be..003a43a 100644 --- a/core/web_terminal.py +++ b/core/web_terminal.py @@ -321,6 +321,7 @@ class WebTerminal(MainTerminal): "run_mode": self.run_mode, "model_key": getattr(self, "model_key", None), "has_images": getattr(self.context_manager, "has_images", False), + "has_videos": getattr(self.context_manager, "has_videos", False), "context": { "usage_percent": context_status['usage_percent'], "total_size": context_status['sizes']['total'], diff --git a/server/chat.py b/server/chat.py index df96652..e4f5f19 100644 --- a/server/chat.py +++ b/server/chat.py @@ -69,7 +69,9 @@ def update_thinking_mode(terminal: WebTerminal, workspace: UserWorkspace, userna todo_list=ctx.todo_list, thinking_mode=terminal.thinking_mode, run_mode=terminal.run_mode, - model_key=getattr(terminal, "model_key", None) + model_key=getattr(terminal, "model_key", None), + has_images=getattr(ctx, "has_images", False), + has_videos=getattr(ctx, "has_videos", False) ) except Exception as exc: print(f"[API] 保存思考模式到对话失败: {exc}") @@ -134,7 +136,8 @@ def update_model(terminal: WebTerminal, workspace: UserWorkspace, username: str) thinking_mode=terminal.thinking_mode, run_mode=terminal.run_mode, model_key=terminal.model_key, - has_images=getattr(ctx, "has_images", False) + has_images=getattr(ctx, "has_images", False), + has_videos=getattr(ctx, "has_videos", False) ) except Exception as exc: print(f"[API] 保存模型到对话失败: {exc}") diff --git a/server/chat_flow.py b/server/chat_flow.py index cca96e6..ccc8647 100644 --- a/server/chat_flow.py +++ b/server/chat_flow.py @@ -380,14 +380,15 @@ def detect_tool_failure(result_data: Any) -> bool: return False -def process_message_task(terminal: WebTerminal, message: str, images, sender, client_sid, workspace: UserWorkspace, username: str): +def process_message_task(terminal: WebTerminal, message: str, images, sender, client_sid, workspace: UserWorkspace, username: str, videos=None): """在后台处理消息任务""" + videos = videos or [] try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) # 创建可取消的任务 - task = loop.create_task(handle_task_with_sender(terminal, workspace, message, images, sender, client_sid, username)) + task = loop.create_task(handle_task_with_sender(terminal, workspace, message, images, sender, client_sid, username, videos)) entry = get_stop_flag(client_sid, username) if not isinstance(entry, dict): @@ -462,10 +463,11 @@ def detect_malformed_tool_call(text): return False -async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspace, message, images, sender, client_sid, username: str): +async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspace, message, images, sender, client_sid, username: str, videos=None): """处理任务并发送消息 - 集成token统计版本""" web_terminal = terminal conversation_id = getattr(web_terminal.context_manager, "current_conversation_id", None) + videos = videos or [] # 如果是思考模式，重置状态 if web_terminal.thinking_mode: @@ -478,7 +480,7 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac # 添加到对话历史 history_len_before = len(getattr(web_terminal.context_manager, "conversation_history", []) or []) is_first_user_message = history_len_before == 0 - web_terminal.context_manager.add_conversation("user", message, images=images) + web_terminal.context_manager.add_conversation("user", message, images=images, videos=videos) if is_first_user_message and getattr(web_terminal, "context_manager", None): try: @@ -1178,6 +1180,7 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac full_response = "" tool_calls = [] + video_injections = [] current_thinking = "" detected_tools = {} last_usage_payload = None @@ -2281,17 +2284,28 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac "content": tool_result_content }) - # 收集图片注入请求，延后统一追加 - if ( + # 收集图片/视频注入请求，延后统一追加 + if ( function_name == "view_image" and getattr(web_terminal, "pending_image_view", None) and not tool_failed and (isinstance(result_data, dict) and result_data.get("success") is not False) - ): - inj = web_terminal.pending_image_view - web_terminal.pending_image_view = None - if inj and inj.get("path"): - image_injections.append(inj["path"]) + ): + inj = web_terminal.pending_image_view + web_terminal.pending_image_view = None + if inj and inj.get("path"): + image_injections.append(inj["path"]) + + if ( + function_name == "view_video" + and getattr(web_terminal, "pending_video_view", None) + and not tool_failed + and (isinstance(result_data, dict) and result_data.get("success") is not False) + ): + inj = web_terminal.pending_video_view + web_terminal.pending_video_view = None + if inj and inj.get("path"): + video_injections.append(inj["path"]) if function_name not in {'write_file', 'edit_file'}: await process_sub_agent_updates(messages, inline=True, after_tool_call_id=tool_call_id) @@ -2304,7 +2318,7 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac # 标记不再是第一次迭代 is_first_iteration = False - # 统一附加图片消息，保证所有 tool 响应先完成 + # 统一附加图片/视频消息，保证所有 tool 响应先完成 if image_injections: for img_path in image_injections: injected_text = "这是一条系统控制发送的信息，并非用户主动发送，目的是返回你需要查看的图片。" @@ -2326,6 +2340,29 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac sender('system_message', { 'content': f'系统已按模型请求插入图片: {img_path}' }) + + if video_injections: + for video_path in video_injections: + injected_text = "这是一条系统控制发送的信息，并非用户主动发送，目的是返回你需要查看的视频。" + web_terminal.context_manager.add_conversation( + "user", + injected_text, + videos=[video_path], + metadata={"system_injected_video": True} + ) + content_payload = web_terminal.context_manager._build_content_with_images( + injected_text, + [], + [video_path] + ) + messages.append({ + "role": "user", + "content": content_payload, + "metadata": {"system_injected_video": True} + }) + sender('system_message', { + 'content': f'系统已按模型请求插入视频: {video_path}' + }) # 最终统计 debug_log(f"\n{'='*40}") @@ -2345,7 +2382,7 @@ async def handle_task_with_sender(terminal: WebTerminal, workspace: UserWorkspac # === 统一对外入口 === -def start_chat_task(terminal, message: str, images: Any, sender, client_sid: str, workspace, username: str): +def start_chat_task(terminal, message: str, images: Any, sender, client_sid: str, workspace, username: str, videos: Any = None): """在线程模式下启动对话任务，供 Socket 事件调用。""" return socketio.start_background_task( process_message_task, @@ -2356,9 +2393,10 @@ def start_chat_task(terminal, message: str, images: Any, sender, client_sid: str client_sid, workspace, username, + videos ) -def run_chat_task_sync(terminal, message: str, images: Any, sender, client_sid: str, workspace, username: str): +def run_chat_task_sync(terminal, message: str, images: Any, sender, client_sid: str, workspace, username: str, videos: Any = None): """同步执行（测试/CLI 使用）。""" - return process_message_task(terminal, message, images, sender, client_sid, workspace, username) + return process_message_task(terminal, message, images, sender, client_sid, workspace, username, videos) diff --git a/server/socket_handlers.py b/server/socket_handlers.py index 810f0c8..d5dd466 100644 --- a/server/socket_handlers.py +++ b/server/socket_handlers.py @@ -217,12 +217,19 @@ def handle_message(data): message = (data.get('message') or '').strip() images = data.get('images') or [] - if not message and not images: + videos = data.get('videos') or [] + if not message and not images and not videos: emit('error', {'message': '消息不能为空'}) return if images and getattr(terminal, "model_key", None) not in {"qwen3-vl-plus", "kimi-k2.5"}: emit('error', {'message': '当前模型不支持图片，请切换到 Qwen-VL 或 Kimi-k2.5'}) return + if videos and getattr(terminal, "model_key", None) != "kimi-k2.5": + emit('error', {'message': '当前模型不支持视频，请切换到 Kimi-k2.5'}) + return + if images and videos: + emit('error', {'message': '图片和视频请分开发送'}) + return print(f"[WebSocket] 收到消息: {message}") debug_log(f"\n{'='*80}\n新任务开始: {message}\n{'='*80}") @@ -285,7 +292,8 @@ def handle_message(data): # 传递客户端ID images = data.get('images') or [] - start_chat_task(terminal, message, images, send_with_activity, client_sid, workspace, username) + videos = data.get('videos') or [] + start_chat_task(terminal, message, images, send_with_activity, client_sid, workspace, username, videos) @socketio.on('client_chunk_log') diff --git a/static/src/App.vue b/static/src/App.vue index 753acfa..8d191f4 100644 --- a/static/src/App.vue +++ b/static/src/App.vue @@ -251,6 +251,7 @@ :icon-style="iconStyle" :tool-category-icon="toolCategoryIcon" :selected-images="selectedImages" + :selected-videos="selectedVideos" :block-upload="policyUiBlocks.block_upload" :block-tool-toggle="policyUiBlocks.block_tool_toggle" :block-realtime-terminal="policyUiBlocks.block_realtime_terminal" @@ -279,7 +280,9 @@ @compress-conversation="handleCompressConversationClick" @file-selected="handleFileSelected" @pick-images="openImagePicker" + @pick-video="openVideoPicker" @remove-image="handleRemoveImage" + @remove-video="handleRemoveVideo" @open-review="openReviewDialog" /> @@ -309,6 +312,17 @@ @confirm="handleImagesConfirmed" /> + + + import appOptions from './app'; import VirtualMonitorSurface from './components/chat/VirtualMonitorSurface.vue'; +import VideoPicker from './components/overlay/VideoPicker.vue'; const mobilePanelIcon = new URL('../icons/align-left.svg', import.meta.url).href; const mobileMenuIcons = { diff --git a/static/src/app.ts b/static/src/app.ts index 731466d..9f7e218 100644 --- a/static/src/app.ts +++ b/static/src/app.ts @@ -285,7 +285,10 @@ const appOptions = { modelMenuOpen: false, imageEntries: [], imageLoading: false, + videoEntries: [], + videoLoading: false, conversationHasImages: false, + conversationHasVideos: false, conversationListRequestSeq: 0, conversationListRefreshToken: 0, @@ -429,7 +432,9 @@ const appOptions = { 'toolMenuOpen', 'settingsOpen', 'imagePickerOpen', - 'selectedImages' + 'videoPickerOpen', + 'selectedImages', + 'selectedVideos' ]), resolvedRunMode() { const allowed = ['fast', 'thinking', 'deep']; @@ -819,7 +824,11 @@ const appOptions = { inputSetImagePickerOpen: 'setImagePickerOpen', inputSetSelectedImages: 'setSelectedImages', inputClearSelectedImages: 'clearSelectedImages', - inputRemoveSelectedImage: 'removeSelectedImage' + inputRemoveSelectedImage: 'removeSelectedImage', + inputSetVideoPickerOpen: 'setVideoPickerOpen', + inputSetSelectedVideos: 'setSelectedVideos', + inputClearSelectedVideos: 'clearSelectedVideos', + inputRemoveSelectedVideo: 'removeSelectedVideo' }), ...mapActions(useToolStore, { toolRegisterAction: 'registerToolAction', @@ -1553,6 +1562,9 @@ const appOptions = { if (status && typeof status.has_images !== 'undefined') { this.conversationHasImages = !!status.has_images; } + if (status && typeof status.has_videos !== 'undefined') { + this.conversationHasVideos = !!status.has_videos; + } }, updateContainerStatus(status) { @@ -1879,12 +1891,13 @@ const appOptions = { let currentAssistantMessage = null; let historyHasImages = false; + let historyHasVideos = false; historyMessages.forEach((message, index) => { debugLog(`处理消息 ${index + 1}/${historyMessages.length}:`, message.role, message); const meta = message.metadata || {}; - if (message.role === 'user' && meta.system_injected_image) { - debugLog('跳过系统代发的图片消息（仅用于模型查看，不在前端展示）'); + if (message.role === 'user' && (meta.system_injected_image || meta.system_injected_video)) { + debugLog('跳过系统代发的图片/视频消息（仅用于模型查看，不在前端展示）'); return; } @@ -1895,13 +1908,18 @@ const appOptions = { currentAssistantMessage = null; } const images = message.images || (message.metadata && message.metadata.images) || []; + const videos = message.videos || (message.metadata && message.metadata.videos) || []; if (Array.isArray(images) && images.length) { historyHasImages = true; } + if (Array.isArray(videos) && videos.length) { + historyHasVideos = true; + } this.messages.push({ role: 'user', content: message.content || '', - images + images, + videos }); debugLog('添加用户消息:', message.content?.substring(0, 50) + '...'); @@ -2106,6 +2124,7 @@ const appOptions = { } this.conversationHasImages = historyHasImages; + this.conversationHasVideos = historyHasVideos; debugLog(`历史消息渲染完成，共 ${this.messages.length} 条消息`); this.logMessageState('renderHistoryMessages:after-render'); @@ -2410,10 +2429,12 @@ const appOptions = { const text = (this.inputMessage || '').trim(); const images = Array.isArray(this.selectedImages) ? this.selectedImages.slice(0, 9) : []; + const videos = Array.isArray(this.selectedVideos) ? this.selectedVideos.slice(0, 1) : []; const hasText = text.length > 0; const hasImages = images.length > 0; + const hasVideos = videos.length > 0; - if (!hasText && !hasImages) { + if (!hasText && !hasImages && !hasVideos) { return; } @@ -2432,12 +2453,31 @@ const appOptions = { return; } + if (hasVideos && this.currentModelKey !== 'kimi-k2.5') { + this.uiPushToast({ + title: '当前模型不支持视频', + message: '请切换到 Kimi-k2.5 后再发送视频', + type: 'error' + }); + return; + } + + if (hasVideos && hasImages) { + this.uiPushToast({ + title: '请勿同时发送', + message: '视频与图片需分开发送，每条仅包含一种媒体', + type: 'warning' + }); + return; + } + const message = text; - const isCommand = hasText && !hasImages && message.startsWith('/'); + const isCommand = hasText && !hasImages && !hasVideos && message.startsWith('/'); if (isCommand) { this.socket.emit('send_command', { command: message }); this.inputClearMessage(); this.inputClearSelectedImages(); + this.inputClearSelectedVideos(); this.autoResizeInput(); return; } @@ -2454,18 +2494,25 @@ const appOptions = { // 标记任务进行中，直到任务完成或用户手动停止 this.taskInProgress = true; - this.chatAddUserMessage(message, images); - this.socket.emit('send_message', { message: message, images, conversation_id: this.currentConversationId }); + this.chatAddUserMessage(message, images, videos); + this.socket.emit('send_message', { message: message, images, videos, conversation_id: this.currentConversationId }); if (typeof this.monitorShowPendingReply === 'function') { this.monitorShowPendingReply(); } this.inputClearMessage(); this.inputClearSelectedImages(); + this.inputClearSelectedVideos(); this.inputSetImagePickerOpen(false); + this.inputSetVideoPickerOpen(false); this.inputSetLineCount(1); this.inputSetMultiline(false); if (hasImages) { this.conversationHasImages = true; + this.conversationHasVideos = false; + } + if (hasVideos) { + this.conversationHasVideos = true; + this.conversationHasImages = false; } if (this.autoScrollEnabled) { this.scrollToBottom(); @@ -2669,6 +2716,24 @@ const appOptions = { this.inputSetImagePickerOpen(false); }, + async openVideoPicker() { + if (this.currentModelKey !== 'kimi-k2.5') { + this.uiPushToast({ + title: '当前模型不支持视频', + message: '请切换到 Kimi-k2.5 后再发送视频', + type: 'error' + }); + return; + } + this.closeQuickMenu(); + this.inputSetVideoPickerOpen(true); + await this.loadWorkspaceVideos(); + }, + + closeVideoPicker() { + this.inputSetVideoPickerOpen(false); + }, + async loadWorkspaceImages() { this.imageLoading = true; try { @@ -2746,6 +2811,83 @@ const appOptions = { return results; }, + async fetchAllVideoEntries(startPath = '') { + const queue: string[] = [startPath || '']; + const visited = new Set(); + const results: Array<{ name: string; path: string }> = []; + const exts = new Set(['.mp4', '.mov', '.mkv', '.avi', '.webm']); + const maxFolders = 120; + + while (queue.length && visited.size < maxFolders) { + const path = queue.shift() || ''; + if (visited.has(path)) { + continue; + } + visited.add(path); + try { + const resp = await fetch(`/api/gui/files/entries?path=${encodeURIComponent(path)}`, { + method: 'GET', + credentials: 'include', + headers: { Accept: 'application/json' } + }); + const data = await resp.json().catch(() => null); + if (!data?.success) { + continue; + } + const items = Array.isArray(data?.data?.items) ? data.data.items : []; + for (const item of items) { + const rawPath = + item?.path || + [path, item?.name].filter(Boolean).join('/').replace(/\\/g, '/').replace(/\/{2,}/g, '/'); + const type = String(item?.type || '').toLowerCase(); + if (type === 'directory' || type === 'folder') { + queue.push(rawPath); + continue; + } + const ext = + String(item?.extension || '').toLowerCase() || + (rawPath.includes('.') ? `.${rawPath.split('.').pop()?.toLowerCase()}` : ''); + if (exts.has(ext)) { + results.push({ + name: item?.name || rawPath.split('/').pop() || rawPath, + path: rawPath + }); + if (results.length >= 200) { + return results; + } + } + } + } catch (error) { + console.warn('遍历文件夹失败', path, error); + } + } + return results; + }, + + async loadWorkspaceVideos() { + this.videoLoading = true; + try { + const entries = await this.fetchAllVideoEntries(''); + this.videoEntries = entries; + if (!entries.length) { + this.uiPushToast({ + title: '未找到视频', + message: '工作区内没有可用的视频文件', + type: 'info' + }); + } + } catch (error) { + console.error('加载视频列表失败', error); + this.uiPushToast({ + title: '加载视频失败', + message: error?.message || '请稍后重试', + type: 'error' + }); + } finally { + this.videoLoading = false; + } + }, + handleImagesConfirmed(list) { this.inputSetSelectedImages(Array.isArray(list) ? list : []); this.inputSetImagePickerOpen(false); @@ -2753,6 +2895,17 @@ const appOptions = { handleRemoveImage(path) { this.inputRemoveSelectedImage(path); }, + handleVideosConfirmed(list) { + const arr = Array.isArray(list) ? list.slice(0, 1) : []; + this.inputSetSelectedVideos(arr); + this.inputSetVideoPickerOpen(false); + if (arr.length) { + this.inputClearSelectedImages(); + } + }, + handleRemoveVideo(path) { + this.inputRemoveSelectedVideo(path); + }, handleQuickUpload() { if (this.uploading || !this.isConnected) { diff --git a/static/src/components/chat/ChatArea.vue b/static/src/components/chat/ChatArea.vue index 256e1d4..704b3c8 100644 --- a/static/src/components/chat/ChatArea.vue +++ b/static/src/components/chat/ChatArea.vue @@ -12,6 +12,9 @@
{{ formatImageName(img) }}
+
+ {{ formatImageName(video) }} +
diff --git a/static/src/components/input/InputComposer.vue b/static/src/components/input/InputComposer.vue index abac9c8..3a1c4ec 100644 --- a/static/src/components/input/InputComposer.vue +++ b/static/src/components/input/InputComposer.vue @@ -18,6 +18,12 @@
+
+ + {{ formatImageName(video) }} + + +
+