From f07a72de2b4f450b9a07d79ef9e9594d62c77692 Mon Sep 17 00:00:00 2001 From: JOJO <1498581755@qq.com> Date: Thu, 30 Apr 2026 21:09:08 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E5=A4=9A=E6=A8=A1=E6=80=81?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E8=BF=87=E6=BB=A4=20+=20memory=20=E6=96=B0?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=20+=20UI=20=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 根据模型 multimodal 字段过滤 read_mediafile 工具 - prompt 新增模型多模态能力说明 - 记忆新增 machine 类型（本机路径/脚本/配置） - 记忆内容统一用「用户」指代，禁止用名字 - done 后按 Fn 不再清空之前内容 - 设置窗口 checkbox 文字与其他控件对齐 - 菜单和设置窗口去掉所有 emoji --- Agent/doc/tools.json | 6 +++--- Agent/prompts/system.txt | 8 ++++++++ Agent/src/bridge.js | 26 ++++++++++++++++++++++++-- Agent/src/core/context.js | 1 + Agent/src/tools/remember.js | 2 +- Sources/App/AppDelegate.swift | 16 +++++++++++----- Sources/App/MenuBarController.swift | 20 ++++++++++---------- Sources/UI/SettingsWindow.swift | 8 ++++---- 8 files changed, 62 insertions(+), 25 deletions(-) diff --git a/Agent/doc/tools.json b/Agent/doc/tools.json index 17ea9b1..29ed939 100644 --- a/Agent/doc/tools.json +++ b/Agent/doc/tools.json @@ -287,8 +287,8 @@ }, "type": { "type": "string", - "enum": ["preference", "habit", "event", "todo", "identity", "relationship", "other"], - "description": "记忆类型。preference=偏好喜欢什么不喜欢什么，habit=用户的生活习惯，event=过去需要记住的重要事件，todo=未来需要完成的任务，identity=用户的身份信息，relationship=用户和其他人的关联，other=其他" + "enum": ["preference", "habit", "event", "todo", "identity", "relationship", "machine", "other"], + "description": "记忆类型。preference=偏好喜欢什么不喜欢什么，habit=用户的生活习惯，event=过去需要记住的重要事件，todo=未来需要完成的任务，identity=用户的身份信息，relationship=用户和其他人的关联，machine=本机电脑上的信息（如特定路径、脚本、配置等），other=其他" }, "content": { "type": "string", @@ -319,7 +319,7 @@ "types": { "type": "array", "items": { "type": "string" }, - "description": "限定搜索的记忆类型，可选值：preference, habit, event, todo, identity, relationship, other。不提供则搜索所有类型" + "description": "限定搜索的记忆类型，可选值：preference, habit, event, todo, identity, relationship, machine, other。不提供则搜索所有类型" }, "query": { "type": "string", diff --git a/Agent/prompts/system.txt b/Agent/prompts/system.txt index ced516f..eafae5b 100644 --- a/Agent/prompts/system.txt +++ b/Agent/prompts/system.txt @@ -87,6 +87,11 @@ set_identity 和 remember 的本质区别： 1) 何时记录记忆（remember 工具）： 以下情况应调用 remember 工具 add 操作： + +⚠️ 内容格式：所有记忆的 content 字段中，一律使用「用户」指代用户本人，禁止使用用户的名字、昵称或称呼。 + 正确：「用户喜欢喝咖啡」「用户每天早上8点起床」 + 错误：「jojo喜欢喝咖啡」「张总每天早上8点起床」 + - 用户表达了个人偏好（喜欢/不喜欢/觉得什么好用/讨厌什么） → type=preference，content=具体偏好，keywords=相关关键词 - 用户透露了生活习惯（如「我每天早上8点喝咖啡」） @@ -100,6 +105,8 @@ set_identity 和 remember 的本质区别： → type=identity - 用户提及了人际关系（如「张三是我同事」） → type=relationship +- 用户提到了本机电脑上的信息（如特定路径、脚本位置、配置文件等值得记住的内容） + → type=machine - 其他值得记住的信息 → type=other @@ -148,6 +155,7 @@ set_identity 和 remember 的本质区别： 【系统信息】 - 当前时间：{current_time} - 当前模型：{model_id} +- 模型多模态能力：{multimodal_ability} - 工作区路径：{path} - 系统信息：{system} - 终端类型：{terminal} diff --git a/Agent/src/bridge.js b/Agent/src/bridge.js index 2aad809..f3edaa2 100644 --- a/Agent/src/bridge.js +++ b/Agent/src/bridge.js @@ -22,7 +22,7 @@ const fs = require('fs'); const path = require('path'); const readline = require('readline'); -const { ensureConfig } = require('./config'); +const { ensureConfig, getModelByKey } = require('./config'); const { createState } = require('./core/state'); const { buildSystemPrompt } = require('./core/context'); const { streamChat } = require('./model/client'); @@ -53,6 +53,17 @@ const TOOLS_JSON = path.join(__dirname, '..', 'doc', 'tools.json'); const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8'); const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8')); +// 根据模型多模态能力过滤工具（不支持图片/视频则移除 read_mediafile） +function filterToolsByModel(modelKey) { + const model = getModelByKey(config, modelKey); + if (!model) return tools; + const mm = model.multimodal || 'none'; + if (mm === 'none' || mm === null) { + return tools.filter(t => t.function?.name !== 'read_mediafile'); + } + return tools; +} + // --- 输出辅助 --- function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); @@ -67,10 +78,18 @@ function resolveWorkspace(raw) { // --- 构建消息 --- function buildApiMessages(state, workspace) { + // 获取当前模型的多模态能力描述 + const model = getModelByKey(config, state.modelKey); + const mm = model?.multimodal || 'none'; + let multimodalAbility = '不可查看图片，不可查看视频'; + if (mm === 'image' || mm === 'image+video') multimodalAbility = '可以查看图片，不可查看视频'; + if (mm === 'image+video') multimodalAbility = '可以查看图片，可以查看视频'; + const system = buildSystemPrompt(systemPrompt, { workspace, allowMode: state.allowMode, modelId: state.modelId || state.modelKey, + multimodal_ability: multimodalAbility, }); const messages = [{ role: 'system', content: system }]; for (const msg of state.messages) { @@ -151,6 +170,9 @@ async function runAgent(input) { let lastAssistantContent = ''; let accumulatedUsage = { prompt: 0, completion: 0, total: 0 }; + // 根据当前模型多模态能力过滤工具 + const activeTools = filterToolsByModel(state.modelKey); + while (continueLoop) { let assistantContent = ''; let fullThinkingBuffer = ''; @@ -166,7 +188,7 @@ async function runAgent(input) { config, modelKey: state.modelKey, messages, - tools, + tools: activeTools, thinkingMode: state.thinkingMode, currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0, abortSignal: streamController.signal, diff --git a/Agent/src/core/context.js b/Agent/src/core/context.js index 7ce6431..1f5711c 100644 --- a/Agent/src/core/context.js +++ b/Agent/src/core/context.js @@ -73,6 +73,7 @@ function buildSystemPrompt(basePrompt, opts) { permissions: allowModeValue, git: getGitInfo(opts.workspace), model_id: opts.modelId || '', + multimodal_ability: opts.multimodal_ability || '不可查看图片，不可查看视频', }; for (const [key, value] of Object.entries(replacements)) { const token = `{${key}}`; diff --git a/Agent/src/tools/remember.js b/Agent/src/tools/remember.js index 0ddc503..5274541 100644 --- a/Agent/src/tools/remember.js +++ b/Agent/src/tools/remember.js @@ -8,7 +8,7 @@ const crypto = require('crypto'); const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory'); const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json'); -const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'other']; +const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'machine', 'other']; function ensureDir() { if (!fs.existsSync(MEMORY_DIR)) { diff --git a/Sources/App/AppDelegate.swift b/Sources/App/AppDelegate.swift index b977eae..7bdd240 100644 --- a/Sources/App/AppDelegate.swift +++ b/Sources/App/AppDelegate.swift @@ -172,11 +172,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDel // 智能体模式 → 统一显示刘海弹窗 if Config.shared.triggerMode == .agent { - agentTracker.resetForInput() - agentTracker.isTextInputMode = true - agentTracker.toolsMode = Config.shared.agentToolsMode - agentTracker.newConversationMode = Config.shared.agentNewConversationMode - showAgentNotchInternal(isTextInput: true) + // done 后再次按 Fn：不清空内容，只显示输入框 + if agentTracker.mode == "done" { + agentTracker.isTextInputMode = true + showAgentNotchInternal(isTextInput: true) + } else { + agentTracker.resetForInput() + agentTracker.isTextInputMode = true + agentTracker.toolsMode = Config.shared.agentToolsMode + agentTracker.newConversationMode = Config.shared.agentNewConversationMode + showAgentNotchInternal(isTextInput: true) + } // 打字输入：不录音，等待用户输入 if Config.shared.agentInputMode == "text" { diff --git a/Sources/App/MenuBarController.swift b/Sources/App/MenuBarController.swift index 5589eea..93d78c0 100644 --- a/Sources/App/MenuBarController.swift +++ b/Sources/App/MenuBarController.swift @@ -48,12 +48,12 @@ final class MenuBarController: NSObject { keyEquivalent: "" ) agentVoiceInputItem = NSMenuItem( - title: "🎤 语音输入", + title: "语音输入", action: #selector(selectAgentVoiceInput), keyEquivalent: "" ) agentTextInputItem = NSMenuItem( - title: "⌨️ 打字输入", + title: "打字输入", action: #selector(selectAgentTextInput), keyEquivalent: "" ) @@ -118,7 +118,7 @@ final class MenuBarController: NSObject { // ===== 模式切换 ===== let modeMenu = NSMenu() let inputModeItem = NSMenuItem( - title: "🎤 语音输入", + title: "语音输入", action: #selector(selectMode(_:)), keyEquivalent: "" ) @@ -129,7 +129,7 @@ final class MenuBarController: NSObject { modeMenu.addItem(inputModeItem) let assistModeItem = NSMenuItem( - title: "💬 AI 助手", + title: "AI 助手", action: #selector(selectMode(_:)), keyEquivalent: "" ) @@ -140,7 +140,7 @@ final class MenuBarController: NSObject { modeMenu.addItem(assistModeItem) let agentModeItem = NSMenuItem( - title: "🤖 智能体", + title: "智能体", action: #selector(selectMode(_:)), keyEquivalent: "" ) @@ -180,7 +180,7 @@ final class MenuBarController: NSObject { inputLLMSubmenu.addItem(inputSettingsItem) let inputSubmenuItem = NSMenuItem() - inputSubmenuItem.title = "🎤 Input LLM" + inputSubmenuItem.title = "Input LLM" inputSubmenuItem.submenu = inputLLMSubmenu menu.addItem(inputSubmenuItem) @@ -201,7 +201,7 @@ final class MenuBarController: NSObject { assistLLMSubmenu.addItem(assistSettingsItem) let assistSubmenuItem = NSMenuItem() - assistSubmenuItem.title = "💬 Assist LLM" + assistSubmenuItem.title = "Assist LLM" assistSubmenuItem.submenu = assistLLMSubmenu menu.addItem(assistSubmenuItem) @@ -235,7 +235,7 @@ final class MenuBarController: NSObject { agentLLMSubmenu.addItem(agentSettingsItem) let agentSubmenuItem = NSMenuItem() - agentSubmenuItem.title = "🤖 Agent LLM" + agentSubmenuItem.title = "Agent LLM" agentSubmenuItem.submenu = agentLLMSubmenu menu.addItem(agentSubmenuItem) @@ -243,7 +243,7 @@ final class MenuBarController: NSObject { // 助手设置 let assistantSettingsItem = NSMenuItem( - title: "⚙️ 助手设置...", + title: "助手设置...", action: #selector(openAgentSettings), keyEquivalent: "" ) @@ -252,7 +252,7 @@ final class MenuBarController: NSObject { // 记忆管理 let memoryItem = NSMenuItem( - title: "🧠 记忆管理...", + title: "记忆管理...", action: #selector(openMemoryManager), keyEquivalent: "" ) diff --git a/Sources/UI/SettingsWindow.swift b/Sources/UI/SettingsWindow.swift index 4425c49..74627bb 100644 --- a/Sources/UI/SettingsWindow.swift +++ b/Sources/UI/SettingsWindow.swift @@ -445,14 +445,14 @@ final class SettingsWindow: NSWindow { let inputLabel = makeLabel("输入方式:") inputModePopup.translatesAutoresizingMaskIntoConstraints = false - inputModePopup.addItems(withTitles: ["🎤 语音输入", "⌨️ 打字输入"]) + inputModePopup.addItems(withTitles: ["语音输入", "打字输入"]) inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0) inputModePopup.target = self inputModePopup.action = #selector(agentInputModeChanged) let thinkLabel = makeLabel("思考模式:") thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false - thinkingModePopup.addItems(withTitles: ["⚡ 快速", "🧠 思考"]) + thinkingModePopup.addItems(withTitles: ["快速", "思考"]) thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0) thinkingModePopup.target = self thinkingModePopup.action = #selector(agentThinkingModeChanged) @@ -683,10 +683,10 @@ final class SettingsWindow: NSWindow { autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor), autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100), - agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20), + agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113), agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10), - voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20), + voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113), voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6), // 助手名字