VoiceInput/Sources/App/AppDelegate.swift
JOJO 9f3ea09b06 feat(agent): Better Siri 改造 — 个人助手 + 记忆系统 + 统一弹窗 + 多项优化
=== 记忆系统 ===
- 新增 remember(记录)和 recall(搜索)工具,持久化到 ~/Library/Application Support/VoiceInput/memory/memory.json
- 新增 set_identity 工具,独立管理助手名字/用户称呼/语气
- prompt 重写为个人助手身份,注入 macOS 能力 + 记忆规则
- prompt 中积极调用记忆工具,按需搜索、自然融入回答
- 禁止相对时间,待办必须绝对日期

=== 对话持久化 ===
- 集成 conversation_store,done 前自动写入对话文件
- token 记录:prompt/completion 累加,total 覆盖为最新上下文长度
- 启动时加载最新对话继续(使用 eagent /resume 逻辑)
- 去重:末尾连续 assistant 消息自动合并
- 修复消息加载顺序 bug(旧对话覆盖用户新消息)

=== 新建对话 ===
- 手动/自动两种模式(设置窗口可切换)
- 自动模式:total > 阈值时隐藏按钮,下次输入自动新建
- 自动阈值在设置窗口可配置(默认 40000)

=== 弹窗 UI 统一 ===
- 语音输入按 Fn 即显示刘海弹窗(和打字输入统一)
- 语音识别内容自动填入并发送(可关闭自动发送)
- 工作期间(thinking/running)隐藏输入框,idle/done 时显示
- 工具显示三种模式:全部显示/最后隐藏/一直不显示

=== 设置窗口增强 ===
- 助手名字 / 用户称呼 / 语气可配置
- 工具显示模式 / 新建对话模式 / 自动阈值
- 语音输入后自动发送开关(默认开)
- 记忆管理窗口可视化编辑(Dock 可访问)
- 窗口高度 840→860

=== 菜单增强 ===
- Agent LLM 二级菜单切换语音/打字输入
- 助手设置和记忆管理入口(Dock 可访问)

=== 修复 ===
- Fn 按下时停止正在进行的朗读
- 弹窗已存在时不再覆盖 isTextInputMode
- voiceInputText 加 @Published 以触发 UI 更新
2026-04-30 18:16:58 +08:00

558 lines
22 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import Cocoa
import AVFoundation
import Speech
import SwiftUI
import DynamicNotchKit
/// Fn Control
final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDelegate {
/// ARC NSApplication.delegate weak
private static var retainedInstance: AppDelegate?
private let keyMonitor = KeyMonitor()
private let speechRecognizer = SpeechRecognizer()
private let audioMonitor = AudioLevelMonitor()
private let recordingPanel = RecordingPanel()
private let menuBarController = MenuBarController()
private let llmRefiner = LLMRefiner()
private let speechSynthesizer = NSSpeechSynthesizer()
///
private var speakingMode: LLMMode? = nil
///
private var speakAutoHideWorkItem: DispatchWorkItem? = nil
private let agentBridge: AgentBridge
private let agentTracker = AgentProgressTracker()
private var agentNotch: DynamicNotch<AgentNotchView>?
private var isRecording = false
private var finalTranscription: String = ""
private var isTextInputActive = false
private var agentActive = false
private var agentIsTextMode = false
private var agentExitingCleanly = false // 退
private var autoHideWorkItem: DispatchWorkItem?
override init() {
agentBridge = AgentBridge(agentDir: Config.shared.agentDir)
super.init()
Self.retainedInstance = self
speechSynthesizer.delegate = self
}
func applicationDidFinishLaunching(_ notification: Notification) {
NSApp.setActivationPolicy(.accessory)
beginPermissionFlow()
print("[App] Ready")
}
// MARK: - Permissions
private func beginPermissionFlow() {
let hasAccessibility = CGPreflightListenEventAccess()
let hasInputMonitoring = canCreateHIDEventTapSimple()
// OK
if hasAccessibility && hasInputMonitoring {
startKeyMonitor()
return
}
//
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security")!)
//
var missing: [String] = []
if !hasAccessibility { missing.append("辅助功能") }
if !hasInputMonitoring { missing.append("输入监控") }
let alert = NSAlert()
alert.messageText = "需要系统权限"
alert.informativeText = "VoiceInput 需要以下权限才能正常工作:\n\n\(missing.joined(separator: "\n"))\n\n请在左侧列表中找到对应项目,点击 + 添加 VoiceInput然后打开开关。"
alert.alertStyle = .informational
alert.addButton(withTitle: "已授权,继续")
alert.addButton(withTitle: "退出")
guard alert.runModal() == .alertFirstButtonReturn else { NSApp.terminate(nil); return }
//
Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { [weak self] timer in
guard let self = self else { return }
let ok1 = CGPreflightListenEventAccess()
let ok2 = self.canCreateHIDEventTapSimple()
if ok1 && ok2 {
timer.invalidate()
DispatchQueue.main.async {
print("[Permission] ✅ All permissions granted")
self.startKeyMonitor()
self.showReadyAlert()
}
}
}
}
/// KeyMonitor HIDSession 退
private func canCreateHIDEventTapSimple() -> Bool {
let mask = (1 << CGEventType.flagsChanged.rawValue)
guard let tap = CGEvent.tapCreate(tap: .cghidEventTap, place: .headInsertEventTap,
options: .defaultTap, eventsOfInterest: CGEventMask(mask),
callback: { _, _, _, _ in return nil }, userInfo: nil) else { return false }
CFMachPortInvalidate(tap)
return true
}
private func showReadyAlert() {
let alert = NSAlert()
alert.messageText = "权限已就绪"
alert.informativeText = "Voice Input 正在运行。首次使用时还需授权麦克风和语音识别。"
alert.alertStyle = .informational
alert.runModal()
}
private func showMicrophonePermissionAlert() {
let alert = NSAlert()
alert.messageText = "需要麦克风权限"
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 麦克风,开启 VoiceInput 的权限。"
alert.alertStyle = .warning
alert.addButton(withTitle: "打开系统设置")
alert.addButton(withTitle: "取消")
if alert.runModal() == .alertFirstButtonReturn {
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone")!)
}
}
private func showSpeechPermissionAlert() {
let alert = NSAlert()
alert.messageText = "需要语音识别权限"
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 语音识别,开启 VoiceInput 的权限。"
alert.alertStyle = .warning
alert.addButton(withTitle: "打开系统设置")
alert.addButton(withTitle: "取消")
if alert.runModal() == .alertFirstButtonReturn {
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition")!)
}
}
// MARK: - Key Monitor
private func startKeyMonitor() {
guard keyMonitor.start() else { print("[App] Failed to start key monitor"); return }
keyMonitor.onFnDown = { [weak self] in self?.startRecording() }
keyMonitor.onFnUp = { [weak self] in self?.stopRecording() }
keyMonitor.onControlDoubleClick = { [weak self] in self?.toggleModeViaDoubleClick() }
}
private func toggleModeViaDoubleClick() {
let newMode: TriggerMode
switch Config.shared.triggerMode {
case .input: newMode = .assist
case .assist: newMode = .agent
case .agent: newMode = .input
}
Config.shared.triggerMode = newMode
menuBarController.refreshModeMenu()
print("[App] Control double-click: mode switched to \(newMode)")
}
// MARK: - Recording
private func startRecording() {
guard !isRecording else { return }
//
if speechSynthesizer.isSpeaking {
speechSynthesizer.stopSpeaking()
speakingMode = nil
speakAutoHideWorkItem?.cancel()
}
//
if agentNotch != nil && Config.shared.triggerMode != .agent {
hideAgentNotch()
}
//
if Config.shared.triggerMode == .agent {
agentTracker.resetForInput()
agentTracker.isTextInputMode = true
agentTracker.toolsMode = Config.shared.agentToolsMode
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
showAgentNotchInternal(isTextInput: true)
//
if Config.shared.agentInputMode == "text" {
isTextInputActive = true
recordingPanel.updateText("Type your question...")
recordingPanel.showAnimated()
return
}
//
}
isRecording = true
finalTranscription = ""
let micStatus = AVCaptureDevice.authorizationStatus(for: .audio)
if micStatus == .denied || micStatus == .restricted { showMicrophonePermissionAlert(); isRecording = false; return }
let speechStatus = SFSpeechRecognizer.authorizationStatus()
if speechStatus == .denied || speechStatus == .restricted { showSpeechPermissionAlert(); isRecording = false; return }
if speechStatus == .notDetermined {
SFSpeechRecognizer.requestAuthorization { [weak self] status in
DispatchQueue.main.async {
self?.isRecording = false
guard status == .authorized else { self?.showSpeechPermissionAlert(); return }
self?.startRecording()
}
}
return
}
recordingPanel.showAnimated()
let promptText: String = {
switch Config.shared.triggerMode {
case .input: return "Listening..."
case .assist: return "Ask AI..."
case .agent: return "Ask Agent..."
}
}()
recordingPanel.updateText(promptText)
let language = Config.shared.language.rawValue
Task { @MainActor in
let audioOK = audioMonitor.start()
if audioOK {
audioMonitor.onLevelUpdate = { [weak self] level in
self?.recordingPanel.waveformView.updateLevel(level)
}
}
let speechOK = await speechRecognizer.start(language: language)
if !speechOK {
recordingPanel.updateText("Recognition error")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in
self?.cancelRecording()
}
return
}
speechRecognizer.onResult = { [weak self] text in
guard let self, isRecording else { return }
finalTranscription = text
recordingPanel.updateText(text)
}
speechRecognizer.onError = { error in
let ns = error as NSError
if ns.domain == "kAFAssistantErrorDomain", ns.code == 1110 { return }
print("[App] Speech error: \(error)")
}
}
}
private func stopRecording() {
//
if isTextInputActive {
isTextInputActive = false
recordingPanel.hideAnimated()
return
}
guard isRecording else { return }
isRecording = false
audioMonitor.stop()
_ = speechRecognizer.stop()
let text = finalTranscription.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else { recordingPanel.hideAnimated(); finalTranscription = ""; return }
switch Config.shared.triggerMode {
case .input: handleInputMode(text: text, config: Config.shared.inputLLM)
case .assist: handleAssistMode(text: text, config: Config.shared.assistLLM)
case .agent:
if Config.shared.agentVoiceAutoSend {
handleTextInput(text)
} else {
recordingPanel.hideAnimated()
agentTracker.voiceInputText = text
}
}
finalTranscription = ""
}
private func cancelRecording() {
audioMonitor.stop()
speechRecognizer.cancel()
recordingPanel.hideAnimated()
isRecording = false
finalTranscription = ""
}
// MARK: - Input Mode
private func handleInputMode(text: String, config: LLMConfig) {
guard config.enabled, config.isConfigured else {
recordingPanel.hideAnimated { TextInjector.inject(text) }
return
}
recordingPanel.showRefining()
Task { @MainActor in
do {
let result = try await llmRefiner.refine(text: text, config: config)
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
} catch {
print("[App] LLM refine error: \(error)")
recordingPanel.hideAnimated { TextInjector.inject(text) }
}
}
}
// MARK: - Assist Mode
private func handleAssistMode(text: String, config: LLMConfig) {
guard config.enabled, config.isConfigured else {
recordingPanel.updateText("AI 助手未配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
return
}
recordingPanel.showThinking()
Task { @MainActor in
do {
let result = try await llmRefiner.chat(text: text, config: config)
if Config.shared.assistOutputMode == .notch {
recordingPanel.hideAnimated()
if Config.shared.assistSpeakEnabled {
NotchDisplayManager.shared.show(text: result.text, autoHideAfter: nil)
speakingMode = .assist
speechSynthesizer.startSpeaking(result.text)
} else {
NotchDisplayManager.shared.show(text: result.text)
}
} else {
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
}
} catch {
print("[App] LLM chat error: \(error)")
recordingPanel.updateText("请求失败,请检查配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in self?.recordingPanel.hideAnimated() }
}
}
}
// MARK: - Agent Mode
private func handleAgentMode(text: String) {
speakAutoHideWorkItem?.cancel()
speechSynthesizer.stopSpeaking()
speakingMode = nil
let modelsFile = Config.shared.loadAgentModels()
let validModels = modelsFile.models.filter { !$0.url.isEmpty && !$0.name.isEmpty && !$0.apikey.isEmpty && !$0.modes.isEmpty }
let defaultModel = modelsFile.defaultModel.isEmpty ? validModels.first?.name : modelsFile.defaultModel
guard !validModels.isEmpty, defaultModel != nil else {
recordingPanel.updateText("智能体未配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
return
}
recordingPanel.hideAnimated()
//
let needsNewConversation = agentTracker.autoNewTriggered || agentTracker.manualNewRequested
if needsNewConversation {
agentBridge.stop()
agentActive = false
agentTracker.autoNewTriggered = false
agentTracker.manualNewRequested = false
}
//
if agentActive {
autoHideWorkItem?.cancel()
agentTracker.start()
showAgentNotchInternal(isTextInput: agentIsTextMode)
agentBridge.sendContinue(message: text)
return
}
agentActive = true
agentIsTextMode = Config.shared.agentInputMode == "text"
agentTracker.start()
showAgentNotchInternal(isTextInput: agentIsTextMode)
agentBridge.onEvent = { [weak self] event in
DispatchQueue.main.async { self?.handleAgentEvent(event) }
}
agentBridge.onExit = { [weak self] code in
DispatchQueue.main.async {
guard let self = self else { return }
self.agentLog("[App] onExit code=\(code) currentMode=\(self.agentTracker.mode)")
self.agentActive = false
self.agentIsTextMode = false
if !self.agentExitingCleanly && code != 0 && self.agentTracker.mode != "done" {
self.agentTracker.fail("进程异常退出 (code \(code))")
}
self.agentExitingCleanly = false
}
}
guard agentBridge.start() else {
agentActive = false
agentTracker.fail("无法启动智能体进程")
return
}
agentBridge.send(message: text, workspace: Config.shared.agentWorkspace,
allowMode: Config.shared.agentAllowMode,
modelKey: defaultModel!,
newConversation: needsNewConversation)
}
private func hideAgentNotch() {
agentNotch?.hide(ignoreMouse: true)
agentNotch = nil
}
private func handleAgentEvent(_ event: AgentEvent) {
switch event {
case .status: break // no-op toolPhaseActive
case .toolStart(let tool, let display):
agentLog("[App] handleEvent toolStart tool=\(tool)")
agentTracker.addTool(tool, display: display)
case .toolEnd(let tool, let resultLines):
agentLog("[App] handleEvent toolEnd tool=\(tool) lines=\(resultLines.count)")
agentTracker.updateToolResult(tool, resultLines: resultLines)
case .assistantChunk(let content):
agentTracker.appendChunk(content)
case .done(let content, _, let totalTokens):
agentLog("[App] handleEvent done contentLen=\(content.count) totalTokens=\(totalTokens)")
agentTracker.finish(content: content, totalTokens: totalTokens)
let willSpeak = Config.shared.agentSpeakEnabled && !content.isEmpty
if willSpeak {
speakingMode = .agent
speechSynthesizer.startSpeaking(content)
}
autoHideWorkItem?.cancel()
//
if !agentIsTextMode && !willSpeak {
let workItem = DispatchWorkItem { [weak self] in self?.hideAgentNotch() }
autoHideWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 20, execute: workItem)
}
case .error(let message):
agentLog("[App] handleEvent error message=\(message)")
agentTracker.fail(message)
}
}
private func agentLog(_ msg: String) {
let logPath = (Bundle.main.bundlePath as NSString).deletingLastPathComponent + "/../../../agent_debug.log"
let ts = ISO8601DateFormatter().string(from: Date())
let line = "[\(ts)] \(msg)\n"
if let handle = FileHandle(forWritingAtPath: logPath) {
handle.seekToEndOfFile()
handle.write(line.data(using: .utf8)!)
handle.closeFile()
}
}
private func showAgentNotch() {
showAgentNotchInternal(isTextInput: false)
}
private func showTextInputNotch() {
// thinking
agentTracker.resetForInput()
agentIsTextMode = true
showAgentNotchInternal(isTextInput: true)
}
private func showAgentNotchInternal(isTextInput: Bool) {
//
if agentNotch != nil {
agentNotch?.show()
return
}
agentTracker.isTextInputMode = isTextInput
agentTracker.toolsMode = Config.shared.agentToolsMode
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
//
let view = AgentNotchView(
tracker: agentTracker,
onCopy: {
NSPasteboard.general.clearContents()
NSPasteboard.general.setString(self.agentTracker.finalContent, forType: .string)
},
onStop: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.agentExitingCleanly = true
self?.agentBridge.stop()
self?.hideAgentNotch()
},
onClose: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.hideAgentNotch()
},
onNewConversation: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.agentExitingCleanly = true
self?.agentBridge.stop()
self?.agentActive = false
self?.agentIsTextMode = false
self?.agentTracker.resetForInput()
self?.agentTracker.manualNewRequested = true
},
onSendText: { [weak self] text in
self?.handleTextInput(text)
},
onContentSizeChanged: { [weak self] in
self?.agentNotch?.notifyContentChanged()
}
)
agentNotch = DynamicNotch(style: .notch) { view }
agentNotch?.show()
}
private func handleTextInput(_ text: String) {
handleAgentMode(text: text)
}
func applicationWillTerminate(_ notification: Notification) {
speechSynthesizer.stopSpeaking()
keyMonitor.stop()
audioMonitor.stop()
speechRecognizer.cancel()
recordingPanel.close()
}
// MARK: - NSSpeechSynthesizerDelegate
func speechSynthesizer(_ sender: NSSpeechSynthesizer, didFinishSpeaking finishedSpeaking: Bool) {
guard finishedSpeaking else { return }
DispatchQueue.main.async { [weak self] in
guard let self, let mode = self.speakingMode else { return }
self.speakingMode = nil
// 5
let workItem = DispatchWorkItem { [weak self] in
switch mode {
case .assist:
NotchDisplayManager.shared.extendAutoHide(to: 0.1)
case .agent:
//
if self?.agentIsTextMode != true {
self?.agentNotch?.show(for: 0.1)
}
default:
break
}
}
self.speakAutoHideWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 5, execute: workItem)
}
}
}