fix(agent): 多模态工具过滤 + memory 新类型 + UI 优化
- 根据模型 multimodal 字段过滤 read_mediafile 工具 - prompt 新增模型多模态能力说明 - 记忆新增 machine 类型（本机路径/脚本/配置） - 记忆内容统一用「用户」指代，禁止用名字 - done 后按 Fn 不再清空之前内容 - 设置窗口 checkbox 文字与其他控件对齐 - 菜单和设置窗口去掉所有 emoji
This commit is contained in:
parent
76fcda7210
commit
f07a72de2b
@ -287,8 +287,8 @@
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["preference", "habit", "event", "todo", "identity", "relationship", "other"],
|
||||
"description": "记忆类型。preference=偏好喜欢什么不喜欢什么，habit=用户的生活习惯，event=过去需要记住的重要事件，todo=未来需要完成的任务，identity=用户的身份信息，relationship=用户和其他人的关联，other=其他"
|
||||
"enum": ["preference", "habit", "event", "todo", "identity", "relationship", "machine", "other"],
|
||||
"description": "记忆类型。preference=偏好喜欢什么不喜欢什么，habit=用户的生活习惯，event=过去需要记住的重要事件，todo=未来需要完成的任务，identity=用户的身份信息，relationship=用户和其他人的关联，machine=本机电脑上的信息（如特定路径、脚本、配置等），other=其他"
|
||||
},
|
||||
"content": {
|
||||
"type": "string",
|
||||
@ -319,7 +319,7 @@
|
||||
"types": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"description": "限定搜索的记忆类型，可选值：preference, habit, event, todo, identity, relationship, other。不提供则搜索所有类型"
|
||||
"description": "限定搜索的记忆类型，可选值：preference, habit, event, todo, identity, relationship, machine, other。不提供则搜索所有类型"
|
||||
},
|
||||
"query": {
|
||||
"type": "string",
|
||||
|
||||
@ -87,6 +87,11 @@ set_identity 和 remember 的本质区别：
|
||||
|
||||
1) 何时记录记忆（remember 工具）：
|
||||
以下情况应调用 remember 工具 add 操作：
|
||||
|
||||
⚠️ 内容格式：所有记忆的 content 字段中，一律使用「用户」指代用户本人，禁止使用用户的名字、昵称或称呼。
|
||||
正确：「用户喜欢喝咖啡」「用户每天早上8点起床」
|
||||
错误：「jojo喜欢喝咖啡」「张总每天早上8点起床」
|
||||
|
||||
- 用户表达了个人偏好（喜欢/不喜欢/觉得什么好用/讨厌什么）
|
||||
→ type=preference，content=具体偏好，keywords=相关关键词
|
||||
- 用户透露了生活习惯（如「我每天早上8点喝咖啡」）
|
||||
@ -100,6 +105,8 @@ set_identity 和 remember 的本质区别：
|
||||
→ type=identity
|
||||
- 用户提及了人际关系（如「张三是我同事」）
|
||||
→ type=relationship
|
||||
- 用户提到了本机电脑上的信息（如特定路径、脚本位置、配置文件等值得记住的内容）
|
||||
→ type=machine
|
||||
- 其他值得记住的信息
|
||||
→ type=other
|
||||
|
||||
@ -148,6 +155,7 @@ set_identity 和 remember 的本质区别：
|
||||
【系统信息】
|
||||
- 当前时间：{current_time}
|
||||
- 当前模型：{model_id}
|
||||
- 模型多模态能力：{multimodal_ability}
|
||||
- 工作区路径：{path}
|
||||
- 系统信息：{system}
|
||||
- 终端类型：{terminal}
|
||||
|
||||
@ -22,7 +22,7 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
|
||||
const { ensureConfig } = require('./config');
|
||||
const { ensureConfig, getModelByKey } = require('./config');
|
||||
const { createState } = require('./core/state');
|
||||
const { buildSystemPrompt } = require('./core/context');
|
||||
const { streamChat } = require('./model/client');
|
||||
@ -53,6 +53,17 @@ const TOOLS_JSON = path.join(__dirname, '..', 'doc', 'tools.json');
|
||||
const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8');
|
||||
const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8'));
|
||||
|
||||
// 根据模型多模态能力过滤工具（不支持图片/视频则移除 read_mediafile）
|
||||
function filterToolsByModel(modelKey) {
|
||||
const model = getModelByKey(config, modelKey);
|
||||
if (!model) return tools;
|
||||
const mm = model.multimodal || 'none';
|
||||
if (mm === 'none' || mm === null) {
|
||||
return tools.filter(t => t.function?.name !== 'read_mediafile');
|
||||
}
|
||||
return tools;
|
||||
}
|
||||
|
||||
// --- 输出辅助 ---
|
||||
function emit(obj) {
|
||||
process.stdout.write(JSON.stringify(obj) + '\n');
|
||||
@ -67,10 +78,18 @@ function resolveWorkspace(raw) {
|
||||
|
||||
// --- 构建消息 ---
|
||||
function buildApiMessages(state, workspace) {
|
||||
// 获取当前模型的多模态能力描述
|
||||
const model = getModelByKey(config, state.modelKey);
|
||||
const mm = model?.multimodal || 'none';
|
||||
let multimodalAbility = '不可查看图片，不可查看视频';
|
||||
if (mm === 'image' || mm === 'image+video') multimodalAbility = '可以查看图片，不可查看视频';
|
||||
if (mm === 'image+video') multimodalAbility = '可以查看图片，可以查看视频';
|
||||
|
||||
const system = buildSystemPrompt(systemPrompt, {
|
||||
workspace,
|
||||
allowMode: state.allowMode,
|
||||
modelId: state.modelId || state.modelKey,
|
||||
multimodal_ability: multimodalAbility,
|
||||
});
|
||||
const messages = [{ role: 'system', content: system }];
|
||||
for (const msg of state.messages) {
|
||||
@ -151,6 +170,9 @@ async function runAgent(input) {
|
||||
let lastAssistantContent = '';
|
||||
let accumulatedUsage = { prompt: 0, completion: 0, total: 0 };
|
||||
|
||||
// 根据当前模型多模态能力过滤工具
|
||||
const activeTools = filterToolsByModel(state.modelKey);
|
||||
|
||||
while (continueLoop) {
|
||||
let assistantContent = '';
|
||||
let fullThinkingBuffer = '';
|
||||
@ -166,7 +188,7 @@ async function runAgent(input) {
|
||||
config,
|
||||
modelKey: state.modelKey,
|
||||
messages,
|
||||
tools,
|
||||
tools: activeTools,
|
||||
thinkingMode: state.thinkingMode,
|
||||
currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0,
|
||||
abortSignal: streamController.signal,
|
||||
|
||||
@ -73,6 +73,7 @@ function buildSystemPrompt(basePrompt, opts) {
|
||||
permissions: allowModeValue,
|
||||
git: getGitInfo(opts.workspace),
|
||||
model_id: opts.modelId || '',
|
||||
multimodal_ability: opts.multimodal_ability || '不可查看图片，不可查看视频',
|
||||
};
|
||||
for (const [key, value] of Object.entries(replacements)) {
|
||||
const token = `{${key}}`;
|
||||
|
||||
@ -8,7 +8,7 @@ const crypto = require('crypto');
|
||||
const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory');
|
||||
const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json');
|
||||
|
||||
const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'other'];
|
||||
const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'machine', 'other'];
|
||||
|
||||
function ensureDir() {
|
||||
if (!fs.existsSync(MEMORY_DIR)) {
|
||||
|
||||
@ -172,11 +172,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDel
|
||||
|
||||
// 智能体模式 → 统一显示刘海弹窗
|
||||
if Config.shared.triggerMode == .agent {
|
||||
agentTracker.resetForInput()
|
||||
agentTracker.isTextInputMode = true
|
||||
agentTracker.toolsMode = Config.shared.agentToolsMode
|
||||
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
|
||||
showAgentNotchInternal(isTextInput: true)
|
||||
// done 后再次按 Fn：不清空内容，只显示输入框
|
||||
if agentTracker.mode == "done" {
|
||||
agentTracker.isTextInputMode = true
|
||||
showAgentNotchInternal(isTextInput: true)
|
||||
} else {
|
||||
agentTracker.resetForInput()
|
||||
agentTracker.isTextInputMode = true
|
||||
agentTracker.toolsMode = Config.shared.agentToolsMode
|
||||
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
|
||||
showAgentNotchInternal(isTextInput: true)
|
||||
}
|
||||
|
||||
// 打字输入：不录音，等待用户输入
|
||||
if Config.shared.agentInputMode == "text" {
|
||||
|
||||
@ -48,12 +48,12 @@ final class MenuBarController: NSObject {
|
||||
keyEquivalent: ""
|
||||
)
|
||||
agentVoiceInputItem = NSMenuItem(
|
||||
title: "🎤 语音输入",
|
||||
title: "语音输入",
|
||||
action: #selector(selectAgentVoiceInput),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
agentTextInputItem = NSMenuItem(
|
||||
title: "⌨️ 打字输入",
|
||||
title: "打字输入",
|
||||
action: #selector(selectAgentTextInput),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
@ -118,7 +118,7 @@ final class MenuBarController: NSObject {
|
||||
// ===== 模式切换 =====
|
||||
let modeMenu = NSMenu()
|
||||
let inputModeItem = NSMenuItem(
|
||||
title: "🎤 语音输入",
|
||||
title: "语音输入",
|
||||
action: #selector(selectMode(_:)),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
@ -129,7 +129,7 @@ final class MenuBarController: NSObject {
|
||||
modeMenu.addItem(inputModeItem)
|
||||
|
||||
let assistModeItem = NSMenuItem(
|
||||
title: "💬 AI 助手",
|
||||
title: "AI 助手",
|
||||
action: #selector(selectMode(_:)),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
@ -140,7 +140,7 @@ final class MenuBarController: NSObject {
|
||||
modeMenu.addItem(assistModeItem)
|
||||
|
||||
let agentModeItem = NSMenuItem(
|
||||
title: "🤖 智能体",
|
||||
title: "智能体",
|
||||
action: #selector(selectMode(_:)),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
@ -180,7 +180,7 @@ final class MenuBarController: NSObject {
|
||||
inputLLMSubmenu.addItem(inputSettingsItem)
|
||||
|
||||
let inputSubmenuItem = NSMenuItem()
|
||||
inputSubmenuItem.title = "🎤 Input LLM"
|
||||
inputSubmenuItem.title = "Input LLM"
|
||||
inputSubmenuItem.submenu = inputLLMSubmenu
|
||||
menu.addItem(inputSubmenuItem)
|
||||
|
||||
@ -201,7 +201,7 @@ final class MenuBarController: NSObject {
|
||||
assistLLMSubmenu.addItem(assistSettingsItem)
|
||||
|
||||
let assistSubmenuItem = NSMenuItem()
|
||||
assistSubmenuItem.title = "💬 Assist LLM"
|
||||
assistSubmenuItem.title = "Assist LLM"
|
||||
assistSubmenuItem.submenu = assistLLMSubmenu
|
||||
menu.addItem(assistSubmenuItem)
|
||||
|
||||
@ -235,7 +235,7 @@ final class MenuBarController: NSObject {
|
||||
agentLLMSubmenu.addItem(agentSettingsItem)
|
||||
|
||||
let agentSubmenuItem = NSMenuItem()
|
||||
agentSubmenuItem.title = "🤖 Agent LLM"
|
||||
agentSubmenuItem.title = "Agent LLM"
|
||||
agentSubmenuItem.submenu = agentLLMSubmenu
|
||||
menu.addItem(agentSubmenuItem)
|
||||
|
||||
@ -243,7 +243,7 @@ final class MenuBarController: NSObject {
|
||||
|
||||
// 助手设置
|
||||
let assistantSettingsItem = NSMenuItem(
|
||||
title: "⚙️ 助手设置...",
|
||||
title: "助手设置...",
|
||||
action: #selector(openAgentSettings),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
@ -252,7 +252,7 @@ final class MenuBarController: NSObject {
|
||||
|
||||
// 记忆管理
|
||||
let memoryItem = NSMenuItem(
|
||||
title: "🧠 记忆管理...",
|
||||
title: "记忆管理...",
|
||||
action: #selector(openMemoryManager),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
|
||||
@ -445,14 +445,14 @@ final class SettingsWindow: NSWindow {
|
||||
|
||||
let inputLabel = makeLabel("输入方式:")
|
||||
inputModePopup.translatesAutoresizingMaskIntoConstraints = false
|
||||
inputModePopup.addItems(withTitles: ["🎤 语音输入", "⌨️ 打字输入"])
|
||||
inputModePopup.addItems(withTitles: ["语音输入", "打字输入"])
|
||||
inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0)
|
||||
inputModePopup.target = self
|
||||
inputModePopup.action = #selector(agentInputModeChanged)
|
||||
|
||||
let thinkLabel = makeLabel("思考模式:")
|
||||
thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false
|
||||
thinkingModePopup.addItems(withTitles: ["⚡ 快速", "🧠 思考"])
|
||||
thinkingModePopup.addItems(withTitles: ["快速", "思考"])
|
||||
thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0)
|
||||
thinkingModePopup.target = self
|
||||
thinkingModePopup.action = #selector(agentThinkingModeChanged)
|
||||
@ -683,10 +683,10 @@ final class SettingsWindow: NSWindow {
|
||||
autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor),
|
||||
autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100),
|
||||
|
||||
agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20),
|
||||
agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
|
||||
agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10),
|
||||
|
||||
voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20),
|
||||
voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
|
||||
voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6),
|
||||
|
||||
// 助手名字
|
||||
|
||||
Loading…
Reference in New Issue
Block a user