fix(agent): 多模态工具过滤 + memory 新类型 + UI 优化

- 根据模型 multimodal 字段过滤 read_mediafile 工具
- prompt 新增模型多模态能力说明
- 记忆新增 machine 类型(本机路径/脚本/配置)
- 记忆内容统一用「用户」指代,禁止用名字
- done 后按 Fn 不再清空之前内容
- 设置窗口 checkbox 文字与其他控件对齐
- 菜单和设置窗口去掉所有 emoji
This commit is contained in:
JOJO 2026-04-30 21:09:08 +08:00
parent 76fcda7210
commit f07a72de2b
8 changed files with 62 additions and 25 deletions

View File

@ -287,8 +287,8 @@
}, },
"type": { "type": {
"type": "string", "type": "string",
"enum": ["preference", "habit", "event", "todo", "identity", "relationship", "other"], "enum": ["preference", "habit", "event", "todo", "identity", "relationship", "machine", "other"],
"description": "记忆类型。preference=偏好喜欢什么不喜欢什么habit=用户的生活习惯event=过去需要记住的重要事件todo=未来需要完成的任务identity=用户的身份信息relationship=用户和其他人的关联,other=其他" "description": "记忆类型。preference=偏好喜欢什么不喜欢什么habit=用户的生活习惯event=过去需要记住的重要事件todo=未来需要完成的任务identity=用户的身份信息relationship=用户和其他人的关联,machine=本机电脑上的信息(如特定路径、脚本、配置等),other=其他"
}, },
"content": { "content": {
"type": "string", "type": "string",
@ -319,7 +319,7 @@
"types": { "types": {
"type": "array", "type": "array",
"items": { "type": "string" }, "items": { "type": "string" },
"description": "限定搜索的记忆类型可选值preference, habit, event, todo, identity, relationship, other。不提供则搜索所有类型" "description": "限定搜索的记忆类型可选值preference, habit, event, todo, identity, relationship, machine, other。不提供则搜索所有类型"
}, },
"query": { "query": {
"type": "string", "type": "string",

View File

@ -87,6 +87,11 @@ set_identity 和 remember 的本质区别:
1) 何时记录记忆remember 工具): 1) 何时记录记忆remember 工具):
以下情况应调用 remember 工具 add 操作: 以下情况应调用 remember 工具 add 操作:
⚠️ 内容格式:所有记忆的 content 字段中,一律使用「用户」指代用户本人,禁止使用用户的名字、昵称或称呼。
正确「用户喜欢喝咖啡」「用户每天早上8点起床」
错误「jojo喜欢喝咖啡」「张总每天早上8点起床」
- 用户表达了个人偏好(喜欢/不喜欢/觉得什么好用/讨厌什么) - 用户表达了个人偏好(喜欢/不喜欢/觉得什么好用/讨厌什么)
→ type=preferencecontent=具体偏好keywords=相关关键词 → type=preferencecontent=具体偏好keywords=相关关键词
- 用户透露了生活习惯如「我每天早上8点喝咖啡」 - 用户透露了生活习惯如「我每天早上8点喝咖啡」
@ -100,6 +105,8 @@ set_identity 和 remember 的本质区别:
→ type=identity → type=identity
- 用户提及了人际关系(如「张三是我同事」) - 用户提及了人际关系(如「张三是我同事」)
→ type=relationship → type=relationship
- 用户提到了本机电脑上的信息(如特定路径、脚本位置、配置文件等值得记住的内容)
→ type=machine
- 其他值得记住的信息 - 其他值得记住的信息
→ type=other → type=other
@ -148,6 +155,7 @@ set_identity 和 remember 的本质区别:
【系统信息】 【系统信息】
- 当前时间:{current_time} - 当前时间:{current_time}
- 当前模型:{model_id} - 当前模型:{model_id}
- 模型多模态能力:{multimodal_ability}
- 工作区路径:{path} - 工作区路径:{path}
- 系统信息:{system} - 系统信息:{system}
- 终端类型:{terminal} - 终端类型:{terminal}

View File

@ -22,7 +22,7 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const readline = require('readline'); const readline = require('readline');
const { ensureConfig } = require('./config'); const { ensureConfig, getModelByKey } = require('./config');
const { createState } = require('./core/state'); const { createState } = require('./core/state');
const { buildSystemPrompt } = require('./core/context'); const { buildSystemPrompt } = require('./core/context');
const { streamChat } = require('./model/client'); const { streamChat } = require('./model/client');
@ -53,6 +53,17 @@ const TOOLS_JSON = path.join(__dirname, '..', 'doc', 'tools.json');
const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8'); const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8');
const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8')); const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8'));
// 根据模型多模态能力过滤工具(不支持图片/视频则移除 read_mediafile
function filterToolsByModel(modelKey) {
const model = getModelByKey(config, modelKey);
if (!model) return tools;
const mm = model.multimodal || 'none';
if (mm === 'none' || mm === null) {
return tools.filter(t => t.function?.name !== 'read_mediafile');
}
return tools;
}
// --- 输出辅助 --- // --- 输出辅助 ---
function emit(obj) { function emit(obj) {
process.stdout.write(JSON.stringify(obj) + '\n'); process.stdout.write(JSON.stringify(obj) + '\n');
@ -67,10 +78,18 @@ function resolveWorkspace(raw) {
// --- 构建消息 --- // --- 构建消息 ---
function buildApiMessages(state, workspace) { function buildApiMessages(state, workspace) {
// 获取当前模型的多模态能力描述
const model = getModelByKey(config, state.modelKey);
const mm = model?.multimodal || 'none';
let multimodalAbility = '不可查看图片,不可查看视频';
if (mm === 'image' || mm === 'image+video') multimodalAbility = '可以查看图片,不可查看视频';
if (mm === 'image+video') multimodalAbility = '可以查看图片,可以查看视频';
const system = buildSystemPrompt(systemPrompt, { const system = buildSystemPrompt(systemPrompt, {
workspace, workspace,
allowMode: state.allowMode, allowMode: state.allowMode,
modelId: state.modelId || state.modelKey, modelId: state.modelId || state.modelKey,
multimodal_ability: multimodalAbility,
}); });
const messages = [{ role: 'system', content: system }]; const messages = [{ role: 'system', content: system }];
for (const msg of state.messages) { for (const msg of state.messages) {
@ -151,6 +170,9 @@ async function runAgent(input) {
let lastAssistantContent = ''; let lastAssistantContent = '';
let accumulatedUsage = { prompt: 0, completion: 0, total: 0 }; let accumulatedUsage = { prompt: 0, completion: 0, total: 0 };
// 根据当前模型多模态能力过滤工具
const activeTools = filterToolsByModel(state.modelKey);
while (continueLoop) { while (continueLoop) {
let assistantContent = ''; let assistantContent = '';
let fullThinkingBuffer = ''; let fullThinkingBuffer = '';
@ -166,7 +188,7 @@ async function runAgent(input) {
config, config,
modelKey: state.modelKey, modelKey: state.modelKey,
messages, messages,
tools, tools: activeTools,
thinkingMode: state.thinkingMode, thinkingMode: state.thinkingMode,
currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0, currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0,
abortSignal: streamController.signal, abortSignal: streamController.signal,

View File

@ -73,6 +73,7 @@ function buildSystemPrompt(basePrompt, opts) {
permissions: allowModeValue, permissions: allowModeValue,
git: getGitInfo(opts.workspace), git: getGitInfo(opts.workspace),
model_id: opts.modelId || '', model_id: opts.modelId || '',
multimodal_ability: opts.multimodal_ability || '不可查看图片,不可查看视频',
}; };
for (const [key, value] of Object.entries(replacements)) { for (const [key, value] of Object.entries(replacements)) {
const token = `{${key}}`; const token = `{${key}}`;

View File

@ -8,7 +8,7 @@ const crypto = require('crypto');
const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory'); const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory');
const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json'); const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json');
const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'other']; const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'machine', 'other'];
function ensureDir() { function ensureDir() {
if (!fs.existsSync(MEMORY_DIR)) { if (!fs.existsSync(MEMORY_DIR)) {

View File

@ -172,11 +172,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDel
// //
if Config.shared.triggerMode == .agent { if Config.shared.triggerMode == .agent {
agentTracker.resetForInput() // done Fn
agentTracker.isTextInputMode = true if agentTracker.mode == "done" {
agentTracker.toolsMode = Config.shared.agentToolsMode agentTracker.isTextInputMode = true
agentTracker.newConversationMode = Config.shared.agentNewConversationMode showAgentNotchInternal(isTextInput: true)
showAgentNotchInternal(isTextInput: true) } else {
agentTracker.resetForInput()
agentTracker.isTextInputMode = true
agentTracker.toolsMode = Config.shared.agentToolsMode
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
showAgentNotchInternal(isTextInput: true)
}
// //
if Config.shared.agentInputMode == "text" { if Config.shared.agentInputMode == "text" {

View File

@ -48,12 +48,12 @@ final class MenuBarController: NSObject {
keyEquivalent: "" keyEquivalent: ""
) )
agentVoiceInputItem = NSMenuItem( agentVoiceInputItem = NSMenuItem(
title: "🎤 语音输入", title: "语音输入",
action: #selector(selectAgentVoiceInput), action: #selector(selectAgentVoiceInput),
keyEquivalent: "" keyEquivalent: ""
) )
agentTextInputItem = NSMenuItem( agentTextInputItem = NSMenuItem(
title: "⌨️ 打字输入", title: "打字输入",
action: #selector(selectAgentTextInput), action: #selector(selectAgentTextInput),
keyEquivalent: "" keyEquivalent: ""
) )
@ -118,7 +118,7 @@ final class MenuBarController: NSObject {
// ===== ===== // ===== =====
let modeMenu = NSMenu() let modeMenu = NSMenu()
let inputModeItem = NSMenuItem( let inputModeItem = NSMenuItem(
title: "🎤 语音输入", title: "语音输入",
action: #selector(selectMode(_:)), action: #selector(selectMode(_:)),
keyEquivalent: "" keyEquivalent: ""
) )
@ -129,7 +129,7 @@ final class MenuBarController: NSObject {
modeMenu.addItem(inputModeItem) modeMenu.addItem(inputModeItem)
let assistModeItem = NSMenuItem( let assistModeItem = NSMenuItem(
title: "💬 AI 助手", title: "AI 助手",
action: #selector(selectMode(_:)), action: #selector(selectMode(_:)),
keyEquivalent: "" keyEquivalent: ""
) )
@ -140,7 +140,7 @@ final class MenuBarController: NSObject {
modeMenu.addItem(assistModeItem) modeMenu.addItem(assistModeItem)
let agentModeItem = NSMenuItem( let agentModeItem = NSMenuItem(
title: "🤖 智能体", title: "智能体",
action: #selector(selectMode(_:)), action: #selector(selectMode(_:)),
keyEquivalent: "" keyEquivalent: ""
) )
@ -180,7 +180,7 @@ final class MenuBarController: NSObject {
inputLLMSubmenu.addItem(inputSettingsItem) inputLLMSubmenu.addItem(inputSettingsItem)
let inputSubmenuItem = NSMenuItem() let inputSubmenuItem = NSMenuItem()
inputSubmenuItem.title = "🎤 Input LLM" inputSubmenuItem.title = "Input LLM"
inputSubmenuItem.submenu = inputLLMSubmenu inputSubmenuItem.submenu = inputLLMSubmenu
menu.addItem(inputSubmenuItem) menu.addItem(inputSubmenuItem)
@ -201,7 +201,7 @@ final class MenuBarController: NSObject {
assistLLMSubmenu.addItem(assistSettingsItem) assistLLMSubmenu.addItem(assistSettingsItem)
let assistSubmenuItem = NSMenuItem() let assistSubmenuItem = NSMenuItem()
assistSubmenuItem.title = "💬 Assist LLM" assistSubmenuItem.title = "Assist LLM"
assistSubmenuItem.submenu = assistLLMSubmenu assistSubmenuItem.submenu = assistLLMSubmenu
menu.addItem(assistSubmenuItem) menu.addItem(assistSubmenuItem)
@ -235,7 +235,7 @@ final class MenuBarController: NSObject {
agentLLMSubmenu.addItem(agentSettingsItem) agentLLMSubmenu.addItem(agentSettingsItem)
let agentSubmenuItem = NSMenuItem() let agentSubmenuItem = NSMenuItem()
agentSubmenuItem.title = "🤖 Agent LLM" agentSubmenuItem.title = "Agent LLM"
agentSubmenuItem.submenu = agentLLMSubmenu agentSubmenuItem.submenu = agentLLMSubmenu
menu.addItem(agentSubmenuItem) menu.addItem(agentSubmenuItem)
@ -243,7 +243,7 @@ final class MenuBarController: NSObject {
// //
let assistantSettingsItem = NSMenuItem( let assistantSettingsItem = NSMenuItem(
title: "⚙️ 助手设置...", title: "助手设置...",
action: #selector(openAgentSettings), action: #selector(openAgentSettings),
keyEquivalent: "" keyEquivalent: ""
) )
@ -252,7 +252,7 @@ final class MenuBarController: NSObject {
// //
let memoryItem = NSMenuItem( let memoryItem = NSMenuItem(
title: "🧠 记忆管理...", title: "记忆管理...",
action: #selector(openMemoryManager), action: #selector(openMemoryManager),
keyEquivalent: "" keyEquivalent: ""
) )

View File

@ -445,14 +445,14 @@ final class SettingsWindow: NSWindow {
let inputLabel = makeLabel("输入方式:") let inputLabel = makeLabel("输入方式:")
inputModePopup.translatesAutoresizingMaskIntoConstraints = false inputModePopup.translatesAutoresizingMaskIntoConstraints = false
inputModePopup.addItems(withTitles: ["🎤 语音输入", "⌨️ 打字输入"]) inputModePopup.addItems(withTitles: ["语音输入", "打字输入"])
inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0) inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0)
inputModePopup.target = self inputModePopup.target = self
inputModePopup.action = #selector(agentInputModeChanged) inputModePopup.action = #selector(agentInputModeChanged)
let thinkLabel = makeLabel("思考模式:") let thinkLabel = makeLabel("思考模式:")
thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false
thinkingModePopup.addItems(withTitles: ["快速", "🧠 思考"]) thinkingModePopup.addItems(withTitles: ["快速", "思考"])
thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0) thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0)
thinkingModePopup.target = self thinkingModePopup.target = self
thinkingModePopup.action = #selector(agentThinkingModeChanged) thinkingModePopup.action = #selector(agentThinkingModeChanged)
@ -683,10 +683,10 @@ final class SettingsWindow: NSWindow {
autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor), autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor),
autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100), autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100),
agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20), agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10), agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10),
voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20), voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6), voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6),
// //