VoiceInput/Sources/App/AppDelegate.swift
JOJO f07a72de2b fix(agent): 多模态工具过滤 + memory 新类型 + UI 优化
- 根据模型 multimodal 字段过滤 read_mediafile 工具
- prompt 新增模型多模态能力说明
- 记忆新增 machine 类型(本机路径/脚本/配置)
- 记忆内容统一用「用户」指代,禁止用名字
- done 后按 Fn 不再清空之前内容
- 设置窗口 checkbox 文字与其他控件对齐
- 菜单和设置窗口去掉所有 emoji
2026-04-30 21:09:08 +08:00

564 lines
22 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import Cocoa
import AVFoundation
import Speech
import SwiftUI
import DynamicNotchKit
/// Fn Control
final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDelegate {
/// ARC NSApplication.delegate weak
private static var retainedInstance: AppDelegate?
private let keyMonitor = KeyMonitor()
private let speechRecognizer = SpeechRecognizer()
private let audioMonitor = AudioLevelMonitor()
private let recordingPanel = RecordingPanel()
private let menuBarController = MenuBarController()
private let llmRefiner = LLMRefiner()
private let speechSynthesizer = NSSpeechSynthesizer()
///
private var speakingMode: LLMMode? = nil
///
private var speakAutoHideWorkItem: DispatchWorkItem? = nil
private let agentBridge: AgentBridge
private let agentTracker = AgentProgressTracker()
private var agentNotch: DynamicNotch<AgentNotchView>?
private var isRecording = false
private var finalTranscription: String = ""
private var isTextInputActive = false
private var agentActive = false
private var agentIsTextMode = false
private var agentExitingCleanly = false // 退
private var autoHideWorkItem: DispatchWorkItem?
override init() {
agentBridge = AgentBridge(agentDir: Config.shared.agentDir)
super.init()
Self.retainedInstance = self
speechSynthesizer.delegate = self
}
func applicationDidFinishLaunching(_ notification: Notification) {
NSApp.setActivationPolicy(.accessory)
beginPermissionFlow()
print("[App] Ready")
}
// MARK: - Permissions
private func beginPermissionFlow() {
let hasAccessibility = CGPreflightListenEventAccess()
let hasInputMonitoring = canCreateHIDEventTapSimple()
// OK
if hasAccessibility && hasInputMonitoring {
startKeyMonitor()
return
}
//
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security")!)
//
var missing: [String] = []
if !hasAccessibility { missing.append("辅助功能") }
if !hasInputMonitoring { missing.append("输入监控") }
let alert = NSAlert()
alert.messageText = "需要系统权限"
alert.informativeText = "VoiceInput 需要以下权限才能正常工作:\n\n\(missing.joined(separator: "\n"))\n\n请在左侧列表中找到对应项目,点击 + 添加 VoiceInput然后打开开关。"
alert.alertStyle = .informational
alert.addButton(withTitle: "已授权,继续")
alert.addButton(withTitle: "退出")
guard alert.runModal() == .alertFirstButtonReturn else { NSApp.terminate(nil); return }
//
Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { [weak self] timer in
guard let self = self else { return }
let ok1 = CGPreflightListenEventAccess()
let ok2 = self.canCreateHIDEventTapSimple()
if ok1 && ok2 {
timer.invalidate()
DispatchQueue.main.async {
print("[Permission] ✅ All permissions granted")
self.startKeyMonitor()
self.showReadyAlert()
}
}
}
}
/// KeyMonitor HIDSession 退
private func canCreateHIDEventTapSimple() -> Bool {
let mask = (1 << CGEventType.flagsChanged.rawValue)
guard let tap = CGEvent.tapCreate(tap: .cghidEventTap, place: .headInsertEventTap,
options: .defaultTap, eventsOfInterest: CGEventMask(mask),
callback: { _, _, _, _ in return nil }, userInfo: nil) else { return false }
CFMachPortInvalidate(tap)
return true
}
private func showReadyAlert() {
let alert = NSAlert()
alert.messageText = "权限已就绪"
alert.informativeText = "Voice Input 正在运行。首次使用时还需授权麦克风和语音识别。"
alert.alertStyle = .informational
alert.runModal()
}
private func showMicrophonePermissionAlert() {
let alert = NSAlert()
alert.messageText = "需要麦克风权限"
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 麦克风,开启 VoiceInput 的权限。"
alert.alertStyle = .warning
alert.addButton(withTitle: "打开系统设置")
alert.addButton(withTitle: "取消")
if alert.runModal() == .alertFirstButtonReturn {
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone")!)
}
}
private func showSpeechPermissionAlert() {
let alert = NSAlert()
alert.messageText = "需要语音识别权限"
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 语音识别,开启 VoiceInput 的权限。"
alert.alertStyle = .warning
alert.addButton(withTitle: "打开系统设置")
alert.addButton(withTitle: "取消")
if alert.runModal() == .alertFirstButtonReturn {
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition")!)
}
}
// MARK: - Key Monitor
private func startKeyMonitor() {
guard keyMonitor.start() else { print("[App] Failed to start key monitor"); return }
keyMonitor.onFnDown = { [weak self] in self?.startRecording() }
keyMonitor.onFnUp = { [weak self] in self?.stopRecording() }
keyMonitor.onControlDoubleClick = { [weak self] in self?.toggleModeViaDoubleClick() }
}
private func toggleModeViaDoubleClick() {
let newMode: TriggerMode
switch Config.shared.triggerMode {
case .input: newMode = .assist
case .assist: newMode = .agent
case .agent: newMode = .input
}
Config.shared.triggerMode = newMode
menuBarController.refreshModeMenu()
print("[App] Control double-click: mode switched to \(newMode)")
}
// MARK: - Recording
private func startRecording() {
guard !isRecording else { return }
//
if speechSynthesizer.isSpeaking {
speechSynthesizer.stopSpeaking()
speakingMode = nil
speakAutoHideWorkItem?.cancel()
}
//
if agentNotch != nil && Config.shared.triggerMode != .agent {
hideAgentNotch()
}
//
if Config.shared.triggerMode == .agent {
// done Fn
if agentTracker.mode == "done" {
agentTracker.isTextInputMode = true
showAgentNotchInternal(isTextInput: true)
} else {
agentTracker.resetForInput()
agentTracker.isTextInputMode = true
agentTracker.toolsMode = Config.shared.agentToolsMode
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
showAgentNotchInternal(isTextInput: true)
}
//
if Config.shared.agentInputMode == "text" {
isTextInputActive = true
recordingPanel.updateText("Type your question...")
recordingPanel.showAnimated()
return
}
//
}
isRecording = true
finalTranscription = ""
let micStatus = AVCaptureDevice.authorizationStatus(for: .audio)
if micStatus == .denied || micStatus == .restricted { showMicrophonePermissionAlert(); isRecording = false; return }
let speechStatus = SFSpeechRecognizer.authorizationStatus()
if speechStatus == .denied || speechStatus == .restricted { showSpeechPermissionAlert(); isRecording = false; return }
if speechStatus == .notDetermined {
SFSpeechRecognizer.requestAuthorization { [weak self] status in
DispatchQueue.main.async {
self?.isRecording = false
guard status == .authorized else { self?.showSpeechPermissionAlert(); return }
self?.startRecording()
}
}
return
}
recordingPanel.showAnimated()
let promptText: String = {
switch Config.shared.triggerMode {
case .input: return "Listening..."
case .assist: return "Ask AI..."
case .agent: return "Ask Agent..."
}
}()
recordingPanel.updateText(promptText)
let language = Config.shared.language.rawValue
Task { @MainActor in
let audioOK = audioMonitor.start()
if audioOK {
audioMonitor.onLevelUpdate = { [weak self] level in
self?.recordingPanel.waveformView.updateLevel(level)
}
}
let speechOK = await speechRecognizer.start(language: language)
if !speechOK {
recordingPanel.updateText("Recognition error")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in
self?.cancelRecording()
}
return
}
speechRecognizer.onResult = { [weak self] text in
guard let self, isRecording else { return }
finalTranscription = text
recordingPanel.updateText(text)
}
speechRecognizer.onError = { error in
let ns = error as NSError
if ns.domain == "kAFAssistantErrorDomain", ns.code == 1110 { return }
print("[App] Speech error: \(error)")
}
}
}
private func stopRecording() {
//
if isTextInputActive {
isTextInputActive = false
recordingPanel.hideAnimated()
return
}
guard isRecording else { return }
isRecording = false
audioMonitor.stop()
_ = speechRecognizer.stop()
let text = finalTranscription.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else { recordingPanel.hideAnimated(); finalTranscription = ""; return }
switch Config.shared.triggerMode {
case .input: handleInputMode(text: text, config: Config.shared.inputLLM)
case .assist: handleAssistMode(text: text, config: Config.shared.assistLLM)
case .agent:
if Config.shared.agentVoiceAutoSend {
handleTextInput(text)
} else {
recordingPanel.hideAnimated()
agentTracker.voiceInputText = text
}
}
finalTranscription = ""
}
private func cancelRecording() {
audioMonitor.stop()
speechRecognizer.cancel()
recordingPanel.hideAnimated()
isRecording = false
finalTranscription = ""
}
// MARK: - Input Mode
private func handleInputMode(text: String, config: LLMConfig) {
guard config.enabled, config.isConfigured else {
recordingPanel.hideAnimated { TextInjector.inject(text) }
return
}
recordingPanel.showRefining()
Task { @MainActor in
do {
let result = try await llmRefiner.refine(text: text, config: config)
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
} catch {
print("[App] LLM refine error: \(error)")
recordingPanel.hideAnimated { TextInjector.inject(text) }
}
}
}
// MARK: - Assist Mode
private func handleAssistMode(text: String, config: LLMConfig) {
guard config.enabled, config.isConfigured else {
recordingPanel.updateText("AI 助手未配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
return
}
recordingPanel.showThinking()
Task { @MainActor in
do {
let result = try await llmRefiner.chat(text: text, config: config)
if Config.shared.assistOutputMode == .notch {
recordingPanel.hideAnimated()
if Config.shared.assistSpeakEnabled {
NotchDisplayManager.shared.show(text: result.text, autoHideAfter: nil)
speakingMode = .assist
speechSynthesizer.startSpeaking(result.text)
} else {
NotchDisplayManager.shared.show(text: result.text)
}
} else {
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
}
} catch {
print("[App] LLM chat error: \(error)")
recordingPanel.updateText("请求失败,请检查配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in self?.recordingPanel.hideAnimated() }
}
}
}
// MARK: - Agent Mode
private func handleAgentMode(text: String) {
speakAutoHideWorkItem?.cancel()
speechSynthesizer.stopSpeaking()
speakingMode = nil
let modelsFile = Config.shared.loadAgentModels()
let validModels = modelsFile.models.filter { !$0.url.isEmpty && !$0.name.isEmpty && !$0.apikey.isEmpty && !$0.modes.isEmpty }
let defaultModel = modelsFile.defaultModel.isEmpty ? validModels.first?.name : modelsFile.defaultModel
guard !validModels.isEmpty, defaultModel != nil else {
recordingPanel.updateText("智能体未配置")
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
return
}
recordingPanel.hideAnimated()
//
let needsNewConversation = agentTracker.autoNewTriggered || agentTracker.manualNewRequested
if needsNewConversation {
agentBridge.stop()
agentActive = false
agentTracker.autoNewTriggered = false
agentTracker.manualNewRequested = false
}
//
if agentActive {
autoHideWorkItem?.cancel()
agentTracker.start()
showAgentNotchInternal(isTextInput: agentIsTextMode)
agentBridge.sendContinue(message: text)
return
}
agentActive = true
agentIsTextMode = Config.shared.agentInputMode == "text"
agentTracker.start()
showAgentNotchInternal(isTextInput: agentIsTextMode)
agentBridge.onEvent = { [weak self] event in
DispatchQueue.main.async { self?.handleAgentEvent(event) }
}
agentBridge.onExit = { [weak self] code in
DispatchQueue.main.async {
guard let self = self else { return }
self.agentLog("[App] onExit code=\(code) currentMode=\(self.agentTracker.mode)")
self.agentActive = false
self.agentIsTextMode = false
if !self.agentExitingCleanly && code != 0 && self.agentTracker.mode != "done" {
self.agentTracker.fail("进程异常退出 (code \(code))")
}
self.agentExitingCleanly = false
}
}
guard agentBridge.start() else {
agentActive = false
agentTracker.fail("无法启动智能体进程")
return
}
agentBridge.send(message: text, workspace: Config.shared.agentWorkspace,
allowMode: Config.shared.agentAllowMode,
modelKey: defaultModel!,
newConversation: needsNewConversation)
}
private func hideAgentNotch() {
agentNotch?.hide(ignoreMouse: true)
agentNotch = nil
}
private func handleAgentEvent(_ event: AgentEvent) {
switch event {
case .status: break // no-op toolPhaseActive
case .toolStart(let tool, let display):
agentLog("[App] handleEvent toolStart tool=\(tool)")
agentTracker.addTool(tool, display: display)
case .toolEnd(let tool, let resultLines):
agentLog("[App] handleEvent toolEnd tool=\(tool) lines=\(resultLines.count)")
agentTracker.updateToolResult(tool, resultLines: resultLines)
case .assistantChunk(let content):
agentTracker.appendChunk(content)
case .done(let content, _, let totalTokens):
agentLog("[App] handleEvent done contentLen=\(content.count) totalTokens=\(totalTokens)")
agentTracker.finish(content: content, totalTokens: totalTokens)
let willSpeak = Config.shared.agentSpeakEnabled && !content.isEmpty
if willSpeak {
speakingMode = .agent
speechSynthesizer.startSpeaking(content)
}
autoHideWorkItem?.cancel()
//
if !agentIsTextMode && !willSpeak {
let workItem = DispatchWorkItem { [weak self] in self?.hideAgentNotch() }
autoHideWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 20, execute: workItem)
}
case .error(let message):
agentLog("[App] handleEvent error message=\(message)")
agentTracker.fail(message)
}
}
private func agentLog(_ msg: String) {
let logPath = (Bundle.main.bundlePath as NSString).deletingLastPathComponent + "/../../../agent_debug.log"
let ts = ISO8601DateFormatter().string(from: Date())
let line = "[\(ts)] \(msg)\n"
if let handle = FileHandle(forWritingAtPath: logPath) {
handle.seekToEndOfFile()
handle.write(line.data(using: .utf8)!)
handle.closeFile()
}
}
private func showAgentNotch() {
showAgentNotchInternal(isTextInput: false)
}
private func showTextInputNotch() {
// thinking
agentTracker.resetForInput()
agentIsTextMode = true
showAgentNotchInternal(isTextInput: true)
}
private func showAgentNotchInternal(isTextInput: Bool) {
//
if agentNotch != nil {
agentNotch?.show()
return
}
agentTracker.isTextInputMode = isTextInput
agentTracker.toolsMode = Config.shared.agentToolsMode
agentTracker.newConversationMode = Config.shared.agentNewConversationMode
//
let view = AgentNotchView(
tracker: agentTracker,
onCopy: {
NSPasteboard.general.clearContents()
NSPasteboard.general.setString(self.agentTracker.finalContent, forType: .string)
},
onStop: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.agentExitingCleanly = true
self?.agentBridge.stop()
self?.hideAgentNotch()
},
onClose: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.hideAgentNotch()
},
onNewConversation: { [weak self] in
self?.speakingMode = nil
self?.speakAutoHideWorkItem?.cancel()
self?.speechSynthesizer.stopSpeaking()
self?.autoHideWorkItem?.cancel()
self?.agentExitingCleanly = true
self?.agentBridge.stop()
self?.agentActive = false
self?.agentIsTextMode = false
self?.agentTracker.resetForInput()
self?.agentTracker.manualNewRequested = true
},
onSendText: { [weak self] text in
self?.handleTextInput(text)
},
onContentSizeChanged: { [weak self] in
self?.agentNotch?.notifyContentChanged()
}
)
agentNotch = DynamicNotch(style: .notch) { view }
agentNotch?.show()
}
private func handleTextInput(_ text: String) {
handleAgentMode(text: text)
}
func applicationWillTerminate(_ notification: Notification) {
speechSynthesizer.stopSpeaking()
keyMonitor.stop()
audioMonitor.stop()
speechRecognizer.cancel()
recordingPanel.close()
}
// MARK: - NSSpeechSynthesizerDelegate
func speechSynthesizer(_ sender: NSSpeechSynthesizer, didFinishSpeaking finishedSpeaking: Bool) {
guard finishedSpeaking else { return }
DispatchQueue.main.async { [weak self] in
guard let self, let mode = self.speakingMode else { return }
self.speakingMode = nil
// 5
let workItem = DispatchWorkItem { [weak self] in
switch mode {
case .assist:
NotchDisplayManager.shared.extendAutoHide(to: 0.1)
case .agent:
//
if self?.agentIsTextMode != true {
self?.agentNotch?.show(for: 0.1)
}
default:
break
}
}
self.speakAutoHideWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 5, execute: workItem)
}
}
}