fix(agent): 多模态工具过滤 + memory 新类型 + UI 优化
- 根据模型 multimodal 字段过滤 read_mediafile 工具 - prompt 新增模型多模态能力说明 - 记忆新增 machine 类型(本机路径/脚本/配置) - 记忆内容统一用「用户」指代,禁止用名字 - done 后按 Fn 不再清空之前内容 - 设置窗口 checkbox 文字与其他控件对齐 - 菜单和设置窗口去掉所有 emoji
This commit is contained in:
parent
76fcda7210
commit
f07a72de2b
@ -287,8 +287,8 @@
|
|||||||
},
|
},
|
||||||
"type": {
|
"type": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["preference", "habit", "event", "todo", "identity", "relationship", "other"],
|
"enum": ["preference", "habit", "event", "todo", "identity", "relationship", "machine", "other"],
|
||||||
"description": "记忆类型。preference=偏好喜欢什么不喜欢什么,habit=用户的生活习惯,event=过去需要记住的重要事件,todo=未来需要完成的任务,identity=用户的身份信息,relationship=用户和其他人的关联,other=其他"
|
"description": "记忆类型。preference=偏好喜欢什么不喜欢什么,habit=用户的生活习惯,event=过去需要记住的重要事件,todo=未来需要完成的任务,identity=用户的身份信息,relationship=用户和其他人的关联,machine=本机电脑上的信息(如特定路径、脚本、配置等),other=其他"
|
||||||
},
|
},
|
||||||
"content": {
|
"content": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@ -319,7 +319,7 @@
|
|||||||
"types": {
|
"types": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": { "type": "string" },
|
"items": { "type": "string" },
|
||||||
"description": "限定搜索的记忆类型,可选值:preference, habit, event, todo, identity, relationship, other。不提供则搜索所有类型"
|
"description": "限定搜索的记忆类型,可选值:preference, habit, event, todo, identity, relationship, machine, other。不提供则搜索所有类型"
|
||||||
},
|
},
|
||||||
"query": {
|
"query": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
|||||||
@ -87,6 +87,11 @@ set_identity 和 remember 的本质区别:
|
|||||||
|
|
||||||
1) 何时记录记忆(remember 工具):
|
1) 何时记录记忆(remember 工具):
|
||||||
以下情况应调用 remember 工具 add 操作:
|
以下情况应调用 remember 工具 add 操作:
|
||||||
|
|
||||||
|
⚠️ 内容格式:所有记忆的 content 字段中,一律使用「用户」指代用户本人,禁止使用用户的名字、昵称或称呼。
|
||||||
|
正确:「用户喜欢喝咖啡」「用户每天早上8点起床」
|
||||||
|
错误:「jojo喜欢喝咖啡」「张总每天早上8点起床」
|
||||||
|
|
||||||
- 用户表达了个人偏好(喜欢/不喜欢/觉得什么好用/讨厌什么)
|
- 用户表达了个人偏好(喜欢/不喜欢/觉得什么好用/讨厌什么)
|
||||||
→ type=preference,content=具体偏好,keywords=相关关键词
|
→ type=preference,content=具体偏好,keywords=相关关键词
|
||||||
- 用户透露了生活习惯(如「我每天早上8点喝咖啡」)
|
- 用户透露了生活习惯(如「我每天早上8点喝咖啡」)
|
||||||
@ -100,6 +105,8 @@ set_identity 和 remember 的本质区别:
|
|||||||
→ type=identity
|
→ type=identity
|
||||||
- 用户提及了人际关系(如「张三是我同事」)
|
- 用户提及了人际关系(如「张三是我同事」)
|
||||||
→ type=relationship
|
→ type=relationship
|
||||||
|
- 用户提到了本机电脑上的信息(如特定路径、脚本位置、配置文件等值得记住的内容)
|
||||||
|
→ type=machine
|
||||||
- 其他值得记住的信息
|
- 其他值得记住的信息
|
||||||
→ type=other
|
→ type=other
|
||||||
|
|
||||||
@ -148,6 +155,7 @@ set_identity 和 remember 的本质区别:
|
|||||||
【系统信息】
|
【系统信息】
|
||||||
- 当前时间:{current_time}
|
- 当前时间:{current_time}
|
||||||
- 当前模型:{model_id}
|
- 当前模型:{model_id}
|
||||||
|
- 模型多模态能力:{multimodal_ability}
|
||||||
- 工作区路径:{path}
|
- 工作区路径:{path}
|
||||||
- 系统信息:{system}
|
- 系统信息:{system}
|
||||||
- 终端类型:{terminal}
|
- 终端类型:{terminal}
|
||||||
|
|||||||
@ -22,7 +22,7 @@ const fs = require('fs');
|
|||||||
const path = require('path');
|
const path = require('path');
|
||||||
const readline = require('readline');
|
const readline = require('readline');
|
||||||
|
|
||||||
const { ensureConfig } = require('./config');
|
const { ensureConfig, getModelByKey } = require('./config');
|
||||||
const { createState } = require('./core/state');
|
const { createState } = require('./core/state');
|
||||||
const { buildSystemPrompt } = require('./core/context');
|
const { buildSystemPrompt } = require('./core/context');
|
||||||
const { streamChat } = require('./model/client');
|
const { streamChat } = require('./model/client');
|
||||||
@ -53,6 +53,17 @@ const TOOLS_JSON = path.join(__dirname, '..', 'doc', 'tools.json');
|
|||||||
const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8');
|
const systemPrompt = fs.readFileSync(path.join(PROMPTS_DIR, 'system.txt'), 'utf8');
|
||||||
const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8'));
|
const tools = JSON.parse(fs.readFileSync(TOOLS_JSON, 'utf8'));
|
||||||
|
|
||||||
|
// 根据模型多模态能力过滤工具(不支持图片/视频则移除 read_mediafile)
|
||||||
|
function filterToolsByModel(modelKey) {
|
||||||
|
const model = getModelByKey(config, modelKey);
|
||||||
|
if (!model) return tools;
|
||||||
|
const mm = model.multimodal || 'none';
|
||||||
|
if (mm === 'none' || mm === null) {
|
||||||
|
return tools.filter(t => t.function?.name !== 'read_mediafile');
|
||||||
|
}
|
||||||
|
return tools;
|
||||||
|
}
|
||||||
|
|
||||||
// --- 输出辅助 ---
|
// --- 输出辅助 ---
|
||||||
function emit(obj) {
|
function emit(obj) {
|
||||||
process.stdout.write(JSON.stringify(obj) + '\n');
|
process.stdout.write(JSON.stringify(obj) + '\n');
|
||||||
@ -67,10 +78,18 @@ function resolveWorkspace(raw) {
|
|||||||
|
|
||||||
// --- 构建消息 ---
|
// --- 构建消息 ---
|
||||||
function buildApiMessages(state, workspace) {
|
function buildApiMessages(state, workspace) {
|
||||||
|
// 获取当前模型的多模态能力描述
|
||||||
|
const model = getModelByKey(config, state.modelKey);
|
||||||
|
const mm = model?.multimodal || 'none';
|
||||||
|
let multimodalAbility = '不可查看图片,不可查看视频';
|
||||||
|
if (mm === 'image' || mm === 'image+video') multimodalAbility = '可以查看图片,不可查看视频';
|
||||||
|
if (mm === 'image+video') multimodalAbility = '可以查看图片,可以查看视频';
|
||||||
|
|
||||||
const system = buildSystemPrompt(systemPrompt, {
|
const system = buildSystemPrompt(systemPrompt, {
|
||||||
workspace,
|
workspace,
|
||||||
allowMode: state.allowMode,
|
allowMode: state.allowMode,
|
||||||
modelId: state.modelId || state.modelKey,
|
modelId: state.modelId || state.modelKey,
|
||||||
|
multimodal_ability: multimodalAbility,
|
||||||
});
|
});
|
||||||
const messages = [{ role: 'system', content: system }];
|
const messages = [{ role: 'system', content: system }];
|
||||||
for (const msg of state.messages) {
|
for (const msg of state.messages) {
|
||||||
@ -151,6 +170,9 @@ async function runAgent(input) {
|
|||||||
let lastAssistantContent = '';
|
let lastAssistantContent = '';
|
||||||
let accumulatedUsage = { prompt: 0, completion: 0, total: 0 };
|
let accumulatedUsage = { prompt: 0, completion: 0, total: 0 };
|
||||||
|
|
||||||
|
// 根据当前模型多模态能力过滤工具
|
||||||
|
const activeTools = filterToolsByModel(state.modelKey);
|
||||||
|
|
||||||
while (continueLoop) {
|
while (continueLoop) {
|
||||||
let assistantContent = '';
|
let assistantContent = '';
|
||||||
let fullThinkingBuffer = '';
|
let fullThinkingBuffer = '';
|
||||||
@ -166,7 +188,7 @@ async function runAgent(input) {
|
|||||||
config,
|
config,
|
||||||
modelKey: state.modelKey,
|
modelKey: state.modelKey,
|
||||||
messages,
|
messages,
|
||||||
tools,
|
tools: activeTools,
|
||||||
thinkingMode: state.thinkingMode,
|
thinkingMode: state.thinkingMode,
|
||||||
currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0,
|
currentContextTokens: normalizeTokenUsage(state.tokenUsage).total || 0,
|
||||||
abortSignal: streamController.signal,
|
abortSignal: streamController.signal,
|
||||||
|
|||||||
@ -73,6 +73,7 @@ function buildSystemPrompt(basePrompt, opts) {
|
|||||||
permissions: allowModeValue,
|
permissions: allowModeValue,
|
||||||
git: getGitInfo(opts.workspace),
|
git: getGitInfo(opts.workspace),
|
||||||
model_id: opts.modelId || '',
|
model_id: opts.modelId || '',
|
||||||
|
multimodal_ability: opts.multimodal_ability || '不可查看图片,不可查看视频',
|
||||||
};
|
};
|
||||||
for (const [key, value] of Object.entries(replacements)) {
|
for (const [key, value] of Object.entries(replacements)) {
|
||||||
const token = `{${key}}`;
|
const token = `{${key}}`;
|
||||||
|
|||||||
@ -8,7 +8,7 @@ const crypto = require('crypto');
|
|||||||
const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory');
|
const MEMORY_DIR = path.join(os.homedir(), 'Library', 'Application Support', 'VoiceInput', 'memory');
|
||||||
const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json');
|
const MEMORY_FILE = path.join(MEMORY_DIR, 'memory.json');
|
||||||
|
|
||||||
const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'other'];
|
const VALID_TYPES = ['preference', 'habit', 'event', 'todo', 'identity', 'relationship', 'machine', 'other'];
|
||||||
|
|
||||||
function ensureDir() {
|
function ensureDir() {
|
||||||
if (!fs.existsSync(MEMORY_DIR)) {
|
if (!fs.existsSync(MEMORY_DIR)) {
|
||||||
|
|||||||
@ -172,11 +172,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDel
|
|||||||
|
|
||||||
// 智能体模式 → 统一显示刘海弹窗
|
// 智能体模式 → 统一显示刘海弹窗
|
||||||
if Config.shared.triggerMode == .agent {
|
if Config.shared.triggerMode == .agent {
|
||||||
agentTracker.resetForInput()
|
// done 后再次按 Fn:不清空内容,只显示输入框
|
||||||
agentTracker.isTextInputMode = true
|
if agentTracker.mode == "done" {
|
||||||
agentTracker.toolsMode = Config.shared.agentToolsMode
|
agentTracker.isTextInputMode = true
|
||||||
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
|
showAgentNotchInternal(isTextInput: true)
|
||||||
showAgentNotchInternal(isTextInput: true)
|
} else {
|
||||||
|
agentTracker.resetForInput()
|
||||||
|
agentTracker.isTextInputMode = true
|
||||||
|
agentTracker.toolsMode = Config.shared.agentToolsMode
|
||||||
|
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
|
||||||
|
showAgentNotchInternal(isTextInput: true)
|
||||||
|
}
|
||||||
|
|
||||||
// 打字输入:不录音,等待用户输入
|
// 打字输入:不录音,等待用户输入
|
||||||
if Config.shared.agentInputMode == "text" {
|
if Config.shared.agentInputMode == "text" {
|
||||||
|
|||||||
@ -48,12 +48,12 @@ final class MenuBarController: NSObject {
|
|||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
agentVoiceInputItem = NSMenuItem(
|
agentVoiceInputItem = NSMenuItem(
|
||||||
title: "🎤 语音输入",
|
title: "语音输入",
|
||||||
action: #selector(selectAgentVoiceInput),
|
action: #selector(selectAgentVoiceInput),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
agentTextInputItem = NSMenuItem(
|
agentTextInputItem = NSMenuItem(
|
||||||
title: "⌨️ 打字输入",
|
title: "打字输入",
|
||||||
action: #selector(selectAgentTextInput),
|
action: #selector(selectAgentTextInput),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
@ -118,7 +118,7 @@ final class MenuBarController: NSObject {
|
|||||||
// ===== 模式切换 =====
|
// ===== 模式切换 =====
|
||||||
let modeMenu = NSMenu()
|
let modeMenu = NSMenu()
|
||||||
let inputModeItem = NSMenuItem(
|
let inputModeItem = NSMenuItem(
|
||||||
title: "🎤 语音输入",
|
title: "语音输入",
|
||||||
action: #selector(selectMode(_:)),
|
action: #selector(selectMode(_:)),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
@ -129,7 +129,7 @@ final class MenuBarController: NSObject {
|
|||||||
modeMenu.addItem(inputModeItem)
|
modeMenu.addItem(inputModeItem)
|
||||||
|
|
||||||
let assistModeItem = NSMenuItem(
|
let assistModeItem = NSMenuItem(
|
||||||
title: "💬 AI 助手",
|
title: "AI 助手",
|
||||||
action: #selector(selectMode(_:)),
|
action: #selector(selectMode(_:)),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
@ -140,7 +140,7 @@ final class MenuBarController: NSObject {
|
|||||||
modeMenu.addItem(assistModeItem)
|
modeMenu.addItem(assistModeItem)
|
||||||
|
|
||||||
let agentModeItem = NSMenuItem(
|
let agentModeItem = NSMenuItem(
|
||||||
title: "🤖 智能体",
|
title: "智能体",
|
||||||
action: #selector(selectMode(_:)),
|
action: #selector(selectMode(_:)),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
@ -180,7 +180,7 @@ final class MenuBarController: NSObject {
|
|||||||
inputLLMSubmenu.addItem(inputSettingsItem)
|
inputLLMSubmenu.addItem(inputSettingsItem)
|
||||||
|
|
||||||
let inputSubmenuItem = NSMenuItem()
|
let inputSubmenuItem = NSMenuItem()
|
||||||
inputSubmenuItem.title = "🎤 Input LLM"
|
inputSubmenuItem.title = "Input LLM"
|
||||||
inputSubmenuItem.submenu = inputLLMSubmenu
|
inputSubmenuItem.submenu = inputLLMSubmenu
|
||||||
menu.addItem(inputSubmenuItem)
|
menu.addItem(inputSubmenuItem)
|
||||||
|
|
||||||
@ -201,7 +201,7 @@ final class MenuBarController: NSObject {
|
|||||||
assistLLMSubmenu.addItem(assistSettingsItem)
|
assistLLMSubmenu.addItem(assistSettingsItem)
|
||||||
|
|
||||||
let assistSubmenuItem = NSMenuItem()
|
let assistSubmenuItem = NSMenuItem()
|
||||||
assistSubmenuItem.title = "💬 Assist LLM"
|
assistSubmenuItem.title = "Assist LLM"
|
||||||
assistSubmenuItem.submenu = assistLLMSubmenu
|
assistSubmenuItem.submenu = assistLLMSubmenu
|
||||||
menu.addItem(assistSubmenuItem)
|
menu.addItem(assistSubmenuItem)
|
||||||
|
|
||||||
@ -235,7 +235,7 @@ final class MenuBarController: NSObject {
|
|||||||
agentLLMSubmenu.addItem(agentSettingsItem)
|
agentLLMSubmenu.addItem(agentSettingsItem)
|
||||||
|
|
||||||
let agentSubmenuItem = NSMenuItem()
|
let agentSubmenuItem = NSMenuItem()
|
||||||
agentSubmenuItem.title = "🤖 Agent LLM"
|
agentSubmenuItem.title = "Agent LLM"
|
||||||
agentSubmenuItem.submenu = agentLLMSubmenu
|
agentSubmenuItem.submenu = agentLLMSubmenu
|
||||||
menu.addItem(agentSubmenuItem)
|
menu.addItem(agentSubmenuItem)
|
||||||
|
|
||||||
@ -243,7 +243,7 @@ final class MenuBarController: NSObject {
|
|||||||
|
|
||||||
// 助手设置
|
// 助手设置
|
||||||
let assistantSettingsItem = NSMenuItem(
|
let assistantSettingsItem = NSMenuItem(
|
||||||
title: "⚙️ 助手设置...",
|
title: "助手设置...",
|
||||||
action: #selector(openAgentSettings),
|
action: #selector(openAgentSettings),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
@ -252,7 +252,7 @@ final class MenuBarController: NSObject {
|
|||||||
|
|
||||||
// 记忆管理
|
// 记忆管理
|
||||||
let memoryItem = NSMenuItem(
|
let memoryItem = NSMenuItem(
|
||||||
title: "🧠 记忆管理...",
|
title: "记忆管理...",
|
||||||
action: #selector(openMemoryManager),
|
action: #selector(openMemoryManager),
|
||||||
keyEquivalent: ""
|
keyEquivalent: ""
|
||||||
)
|
)
|
||||||
|
|||||||
@ -445,14 +445,14 @@ final class SettingsWindow: NSWindow {
|
|||||||
|
|
||||||
let inputLabel = makeLabel("输入方式:")
|
let inputLabel = makeLabel("输入方式:")
|
||||||
inputModePopup.translatesAutoresizingMaskIntoConstraints = false
|
inputModePopup.translatesAutoresizingMaskIntoConstraints = false
|
||||||
inputModePopup.addItems(withTitles: ["🎤 语音输入", "⌨️ 打字输入"])
|
inputModePopup.addItems(withTitles: ["语音输入", "打字输入"])
|
||||||
inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0)
|
inputModePopup.selectItem(at: Config.shared.agentInputMode == "text" ? 1 : 0)
|
||||||
inputModePopup.target = self
|
inputModePopup.target = self
|
||||||
inputModePopup.action = #selector(agentInputModeChanged)
|
inputModePopup.action = #selector(agentInputModeChanged)
|
||||||
|
|
||||||
let thinkLabel = makeLabel("思考模式:")
|
let thinkLabel = makeLabel("思考模式:")
|
||||||
thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false
|
thinkingModePopup.translatesAutoresizingMaskIntoConstraints = false
|
||||||
thinkingModePopup.addItems(withTitles: ["⚡ 快速", "🧠 思考"])
|
thinkingModePopup.addItems(withTitles: ["快速", "思考"])
|
||||||
thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0)
|
thinkingModePopup.selectItem(at: Config.shared.agentThinkingMode ? 1 : 0)
|
||||||
thinkingModePopup.target = self
|
thinkingModePopup.target = self
|
||||||
thinkingModePopup.action = #selector(agentThinkingModeChanged)
|
thinkingModePopup.action = #selector(agentThinkingModeChanged)
|
||||||
@ -683,10 +683,10 @@ final class SettingsWindow: NSWindow {
|
|||||||
autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor),
|
autoNewTokenThresholdField.centerYAnchor.constraint(equalTo: thresholdLabel.centerYAnchor),
|
||||||
autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100),
|
autoNewTokenThresholdField.widthAnchor.constraint(equalToConstant: 100),
|
||||||
|
|
||||||
agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20),
|
agentSpeakCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
|
||||||
agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10),
|
agentSpeakCheckbox.topAnchor.constraint(equalTo: thresholdLabel.bottomAnchor, constant: 10),
|
||||||
|
|
||||||
voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 20),
|
voiceAutoSendCheckbox.leadingAnchor.constraint(equalTo: content.leadingAnchor, constant: 113),
|
||||||
voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6),
|
voiceAutoSendCheckbox.topAnchor.constraint(equalTo: agentSpeakCheckbox.bottomAnchor, constant: 6),
|
||||||
|
|
||||||
// 助手名字
|
// 助手名字
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user