JOJO
76741877a8
- Assist/Agent 模式新增朗读开关 + NSSpeechSynthesizer 语音合成 - 朗读完成后弹窗延迟5秒带动画关闭（不再写死15/20秒） - 打字模式下不自动关闭弹窗 - 二次输入时取消旧朗读/旧延迟关闭，避免竞争问题 - Agent 进度计时器第二次输入时正确归零 - DMG 打包自动清空 models.json API Key - Node.js 运行时打包进 Agent/bin/，用户无需安装 - Agent system prompt 更新为 VoiceInput 身份 + 语音容错说明 - AGENTS.md 全面更新，补全 Agent 模式文档
519 lines
20 KiB
Swift
519 lines
20 KiB
Swift
import Cocoa
|
||
import AVFoundation
|
||
import Speech
|
||
import SwiftUI
|
||
import DynamicNotchKit
|
||
|
||
/// 应用主控制器，协调 Fn 键（长按录音）和 Control 键（双击切换模式）
|
||
final class AppDelegate: NSObject, NSApplicationDelegate, NSSpeechSynthesizerDelegate {
|
||
|
||
/// 防止 ARC 优化释放（NSApplication.delegate 是 weak）
|
||
private static var retainedInstance: AppDelegate?
|
||
|
||
private let keyMonitor = KeyMonitor()
|
||
private let speechRecognizer = SpeechRecognizer()
|
||
private let audioMonitor = AudioLevelMonitor()
|
||
private let recordingPanel = RecordingPanel()
|
||
private let menuBarController = MenuBarController()
|
||
private let llmRefiner = LLMRefiner()
|
||
private let speechSynthesizer = NSSpeechSynthesizer()
|
||
/// 当前正在朗读的模式，用于朗读完成后关闭对应弹窗
|
||
private var speakingMode: LLMMode? = nil
|
||
/// 朗读完成后的延迟关闭任务，重新输入时取消
|
||
private var speakAutoHideWorkItem: DispatchWorkItem? = nil
|
||
private let agentBridge: AgentBridge
|
||
private let agentTracker = AgentProgressTracker()
|
||
private var agentNotch: DynamicNotch<AgentNotchView>?
|
||
|
||
private var isRecording = false
|
||
private var finalTranscription: String = ""
|
||
private var isTextInputActive = false
|
||
private var agentActive = false
|
||
private var agentIsTextMode = false
|
||
private var agentExitingCleanly = false // 主动关闭标记，避免误报退出错误
|
||
private var autoHideWorkItem: DispatchWorkItem?
|
||
|
||
override init() {
|
||
agentBridge = AgentBridge(agentDir: Config.shared.agentDir)
|
||
super.init()
|
||
Self.retainedInstance = self
|
||
speechSynthesizer.delegate = self
|
||
}
|
||
|
||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||
NSApp.setActivationPolicy(.accessory)
|
||
beginPermissionFlow()
|
||
print("[App] Ready")
|
||
}
|
||
|
||
// MARK: - Permissions
|
||
|
||
private func beginPermissionFlow() {
|
||
guard CGPreflightListenEventAccess() else {
|
||
openPrefsAndWait(panel: "Privacy_Accessibility", label: "辅助功能") { [weak self] in
|
||
self?.beginPermissionFlow()
|
||
}
|
||
return
|
||
}
|
||
guard canCreateHIDEventTap() else {
|
||
openPrefsAndWait(panel: "Privacy_InputMonitoring", label: "输入监控") { [weak self] in
|
||
self?.beginPermissionFlow()
|
||
}
|
||
return
|
||
}
|
||
startKeyMonitor()
|
||
showReadyAlert()
|
||
}
|
||
|
||
private func openPrefsAndWait(panel: String, label: String, onGranted: @escaping () -> Void) {
|
||
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?\(panel)")!)
|
||
let alert = NSAlert()
|
||
alert.messageText = "需要\(label)权限"
|
||
alert.informativeText = "点击 + 号，选择 VoiceInput 应用，点击打开。"
|
||
alert.alertStyle = .informational
|
||
alert.addButton(withTitle: "继续")
|
||
alert.addButton(withTitle: "退出")
|
||
guard alert.runModal() == .alertFirstButtonReturn else { NSApp.terminate(nil); return }
|
||
let check: () -> Bool = {
|
||
panel == "Privacy_Accessibility" ? CGPreflightListenEventAccess() : self.canCreateHIDEventTap()
|
||
}
|
||
Timer.scheduledTimer(withTimeInterval: 2.0, repeats: true) { timer in
|
||
guard check() else { return }
|
||
timer.invalidate()
|
||
DispatchQueue.main.async {
|
||
print("[Permission] ✅ \(label) granted")
|
||
onGranted()
|
||
}
|
||
}
|
||
}
|
||
|
||
private func canCreateHIDEventTap() -> Bool {
|
||
let mask = (1 << CGEventType.flagsChanged.rawValue)
|
||
guard let tap = CGEvent.tapCreate(tap: .cghidEventTap, place: .headInsertEventTap,
|
||
options: .defaultTap, eventsOfInterest: CGEventMask(mask),
|
||
callback: { _, _, _, _ in return nil }, userInfo: nil) else { return false }
|
||
CFMachPortInvalidate(tap)
|
||
return true
|
||
}
|
||
|
||
private func showReadyAlert() {
|
||
let alert = NSAlert()
|
||
alert.messageText = "权限已就绪"
|
||
alert.informativeText = "Voice Input 正在运行。首次使用时还需授权麦克风和语音识别。"
|
||
alert.alertStyle = .informational
|
||
alert.runModal()
|
||
}
|
||
|
||
private func showMicrophonePermissionAlert() {
|
||
let alert = NSAlert()
|
||
alert.messageText = "需要麦克风权限"
|
||
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 麦克风，开启 VoiceInput 的权限。"
|
||
alert.alertStyle = .warning
|
||
alert.addButton(withTitle: "打开系统设置")
|
||
alert.addButton(withTitle: "取消")
|
||
if alert.runModal() == .alertFirstButtonReturn {
|
||
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone")!)
|
||
}
|
||
}
|
||
|
||
private func showSpeechPermissionAlert() {
|
||
let alert = NSAlert()
|
||
alert.messageText = "需要语音识别权限"
|
||
alert.informativeText = "请前往 系统设置 > 隐私与安全性 > 语音识别，开启 VoiceInput 的权限。"
|
||
alert.alertStyle = .warning
|
||
alert.addButton(withTitle: "打开系统设置")
|
||
alert.addButton(withTitle: "取消")
|
||
if alert.runModal() == .alertFirstButtonReturn {
|
||
NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition")!)
|
||
}
|
||
}
|
||
|
||
// MARK: - Key Monitor
|
||
|
||
private func startKeyMonitor() {
|
||
guard keyMonitor.start() else { print("[App] Failed to start key monitor"); return }
|
||
keyMonitor.onFnDown = { [weak self] in self?.startRecording() }
|
||
keyMonitor.onFnUp = { [weak self] in self?.stopRecording() }
|
||
keyMonitor.onControlDoubleClick = { [weak self] in self?.toggleModeViaDoubleClick() }
|
||
}
|
||
|
||
private func toggleModeViaDoubleClick() {
|
||
let newMode: TriggerMode
|
||
switch Config.shared.triggerMode {
|
||
case .input: newMode = .assist
|
||
case .assist: newMode = .agent
|
||
case .agent: newMode = .input
|
||
}
|
||
Config.shared.triggerMode = newMode
|
||
menuBarController.refreshModeMenu()
|
||
print("[App] Control double-click: mode switched to \(newMode)")
|
||
}
|
||
|
||
// MARK: - Recording
|
||
|
||
private func startRecording() {
|
||
guard !isRecording else { return }
|
||
|
||
// 如果当前模式不是智能体但刘海弹窗还在 → 关闭
|
||
if agentNotch != nil && Config.shared.triggerMode != .agent {
|
||
hideAgentNotch()
|
||
}
|
||
|
||
// 智能体 + 打字输入模式 → 直接显示文本输入刘海
|
||
if Config.shared.triggerMode == .agent && Config.shared.agentInputMode == "text" {
|
||
isTextInputActive = true
|
||
recordingPanel.updateText("Type your question...")
|
||
recordingPanel.showAnimated()
|
||
showTextInputNotch()
|
||
return
|
||
}
|
||
|
||
isRecording = true
|
||
finalTranscription = ""
|
||
|
||
let micStatus = AVCaptureDevice.authorizationStatus(for: .audio)
|
||
if micStatus == .denied || micStatus == .restricted { showMicrophonePermissionAlert(); isRecording = false; return }
|
||
|
||
let speechStatus = SFSpeechRecognizer.authorizationStatus()
|
||
if speechStatus == .denied || speechStatus == .restricted { showSpeechPermissionAlert(); isRecording = false; return }
|
||
if speechStatus == .notDetermined {
|
||
SFSpeechRecognizer.requestAuthorization { [weak self] status in
|
||
DispatchQueue.main.async {
|
||
self?.isRecording = false
|
||
guard status == .authorized else { self?.showSpeechPermissionAlert(); return }
|
||
self?.startRecording()
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
recordingPanel.showAnimated()
|
||
let promptText: String = {
|
||
switch Config.shared.triggerMode {
|
||
case .input: return "Listening..."
|
||
case .assist: return "Ask AI..."
|
||
case .agent: return "Ask Agent..."
|
||
}
|
||
}()
|
||
recordingPanel.updateText(promptText)
|
||
|
||
let language = Config.shared.language.rawValue
|
||
Task { @MainActor in
|
||
let audioOK = audioMonitor.start()
|
||
if audioOK {
|
||
audioMonitor.onLevelUpdate = { [weak self] level in
|
||
self?.recordingPanel.waveformView.updateLevel(level)
|
||
}
|
||
}
|
||
let speechOK = await speechRecognizer.start(language: language)
|
||
if !speechOK {
|
||
recordingPanel.updateText("Recognition error")
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in
|
||
self?.cancelRecording()
|
||
}
|
||
return
|
||
}
|
||
speechRecognizer.onResult = { [weak self] text in
|
||
guard let self, isRecording else { return }
|
||
finalTranscription = text
|
||
recordingPanel.updateText(text)
|
||
}
|
||
speechRecognizer.onError = { error in
|
||
let ns = error as NSError
|
||
if ns.domain == "kAFAssistantErrorDomain", ns.code == 1110 { return }
|
||
print("[App] Speech error: \(error)")
|
||
}
|
||
}
|
||
}
|
||
|
||
private func stopRecording() {
|
||
// 打字模式：仅隐藏录音胶囊，刘海保持打开等待输入
|
||
if isTextInputActive {
|
||
isTextInputActive = false
|
||
recordingPanel.hideAnimated()
|
||
return
|
||
}
|
||
|
||
guard isRecording else { return }
|
||
isRecording = false
|
||
audioMonitor.stop()
|
||
_ = speechRecognizer.stop()
|
||
|
||
let text = finalTranscription.trimmingCharacters(in: .whitespacesAndNewlines)
|
||
guard !text.isEmpty else { recordingPanel.hideAnimated(); finalTranscription = ""; return }
|
||
|
||
switch Config.shared.triggerMode {
|
||
case .input: handleInputMode(text: text, config: Config.shared.inputLLM)
|
||
case .assist: handleAssistMode(text: text, config: Config.shared.assistLLM)
|
||
case .agent: handleAgentMode(text: text)
|
||
}
|
||
finalTranscription = ""
|
||
}
|
||
|
||
private func cancelRecording() {
|
||
audioMonitor.stop()
|
||
speechRecognizer.cancel()
|
||
recordingPanel.hideAnimated()
|
||
isRecording = false
|
||
finalTranscription = ""
|
||
}
|
||
|
||
// MARK: - Input Mode
|
||
|
||
private func handleInputMode(text: String, config: LLMConfig) {
|
||
guard config.enabled, config.isConfigured else {
|
||
recordingPanel.hideAnimated { TextInjector.inject(text) }
|
||
return
|
||
}
|
||
recordingPanel.showRefining()
|
||
Task { @MainActor in
|
||
do {
|
||
let result = try await llmRefiner.refine(text: text, config: config)
|
||
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
|
||
} catch {
|
||
print("[App] LLM refine error: \(error)")
|
||
recordingPanel.hideAnimated { TextInjector.inject(text) }
|
||
}
|
||
}
|
||
}
|
||
|
||
// MARK: - Assist Mode
|
||
|
||
private func handleAssistMode(text: String, config: LLMConfig) {
|
||
guard config.enabled, config.isConfigured else {
|
||
recordingPanel.updateText("AI 助手未配置")
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
|
||
return
|
||
}
|
||
recordingPanel.showThinking()
|
||
Task { @MainActor in
|
||
do {
|
||
let result = try await llmRefiner.chat(text: text, config: config)
|
||
if Config.shared.assistOutputMode == .notch {
|
||
recordingPanel.hideAnimated()
|
||
if Config.shared.assistSpeakEnabled {
|
||
NotchDisplayManager.shared.show(text: result.text, autoHideAfter: nil)
|
||
speakingMode = .assist
|
||
speechSynthesizer.startSpeaking(result.text)
|
||
} else {
|
||
NotchDisplayManager.shared.show(text: result.text)
|
||
}
|
||
} else {
|
||
recordingPanel.hideAnimated { TextInjector.inject(result.text) }
|
||
}
|
||
} catch {
|
||
print("[App] LLM chat error: \(error)")
|
||
recordingPanel.updateText("请求失败，请检查配置")
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in self?.recordingPanel.hideAnimated() }
|
||
}
|
||
}
|
||
}
|
||
|
||
// MARK: - Agent Mode
|
||
|
||
private func handleAgentMode(text: String) {
|
||
speakAutoHideWorkItem?.cancel()
|
||
speechSynthesizer.stopSpeaking()
|
||
speakingMode = nil
|
||
let modelsFile = Config.shared.loadAgentModels()
|
||
let validModels = modelsFile.models.filter { !$0.url.isEmpty && !$0.name.isEmpty && !$0.apikey.isEmpty && !$0.modes.isEmpty }
|
||
let defaultModel = modelsFile.defaultModel.isEmpty ? validModels.first?.name : modelsFile.defaultModel
|
||
guard !validModels.isEmpty, defaultModel != nil else {
|
||
recordingPanel.updateText("智能体未配置")
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { [weak self] in self?.recordingPanel.hideAnimated() }
|
||
return
|
||
}
|
||
recordingPanel.hideAnimated()
|
||
|
||
// 如果对话已在进行中 → 追加消息
|
||
if agentActive {
|
||
autoHideWorkItem?.cancel()
|
||
agentTracker.start()
|
||
showAgentNotchInternal(isTextInput: agentIsTextMode)
|
||
agentBridge.sendContinue(message: text)
|
||
return
|
||
}
|
||
|
||
agentActive = true
|
||
agentIsTextMode = Config.shared.agentInputMode == "text"
|
||
agentTracker.start()
|
||
showAgentNotchInternal(isTextInput: agentIsTextMode)
|
||
|
||
agentBridge.onEvent = { [weak self] event in
|
||
DispatchQueue.main.async { self?.handleAgentEvent(event) }
|
||
}
|
||
agentBridge.onExit = { [weak self] code in
|
||
DispatchQueue.main.async {
|
||
guard let self = self else { return }
|
||
self.agentLog("[App] onExit code=\(code) currentMode=\(self.agentTracker.mode)")
|
||
self.agentActive = false
|
||
self.agentIsTextMode = false
|
||
if !self.agentExitingCleanly && code != 0 && self.agentTracker.mode != "done" {
|
||
self.agentTracker.fail("进程异常退出 (code \(code))")
|
||
}
|
||
self.agentExitingCleanly = false
|
||
}
|
||
}
|
||
guard agentBridge.start() else {
|
||
agentActive = false
|
||
agentTracker.fail("无法启动智能体进程")
|
||
return
|
||
}
|
||
agentBridge.send(message: text, workspace: Config.shared.agentWorkspace,
|
||
allowMode: Config.shared.agentAllowMode,
|
||
modelKey: defaultModel!)
|
||
}
|
||
|
||
private func hideAgentNotch() {
|
||
agentNotch?.hide(ignoreMouse: true)
|
||
agentNotch = nil
|
||
}
|
||
|
||
private func handleAgentEvent(_ event: AgentEvent) {
|
||
switch event {
|
||
case .status: break // no-op，内容切换由 toolPhaseActive 控制
|
||
case .toolStart(let tool, let display):
|
||
agentLog("[App] handleEvent toolStart tool=\(tool)")
|
||
agentTracker.addTool(tool, display: display)
|
||
case .toolEnd(let tool, let resultLines):
|
||
agentLog("[App] handleEvent toolEnd tool=\(tool) lines=\(resultLines.count)")
|
||
agentTracker.updateToolResult(tool, resultLines: resultLines)
|
||
case .assistantChunk(let content):
|
||
agentTracker.appendChunk(content)
|
||
case .done(let content, _):
|
||
agentLog("[App] handleEvent done contentLen=\(content.count)")
|
||
agentTracker.finish(content: content)
|
||
let willSpeak = Config.shared.agentSpeakEnabled && !content.isEmpty
|
||
if willSpeak {
|
||
speakingMode = .agent
|
||
speechSynthesizer.startSpeaking(content)
|
||
}
|
||
autoHideWorkItem?.cancel()
|
||
// 打字模式下，输入栏有内容时不自动收回；朗读模式下由朗读完成回调处理
|
||
if !agentIsTextMode && !willSpeak {
|
||
let workItem = DispatchWorkItem { [weak self] in self?.hideAgentNotch() }
|
||
autoHideWorkItem = workItem
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 20, execute: workItem)
|
||
}
|
||
case .error(let message):
|
||
agentLog("[App] handleEvent error message=\(message)")
|
||
agentTracker.fail(message)
|
||
}
|
||
}
|
||
|
||
private func agentLog(_ msg: String) {
|
||
let logPath = (Bundle.main.bundlePath as NSString).deletingLastPathComponent + "/../../../agent_debug.log"
|
||
let ts = ISO8601DateFormatter().string(from: Date())
|
||
let line = "[\(ts)] \(msg)\n"
|
||
if let handle = FileHandle(forWritingAtPath: logPath) {
|
||
handle.seekToEndOfFile()
|
||
handle.write(line.data(using: .utf8)!)
|
||
handle.closeFile()
|
||
}
|
||
}
|
||
|
||
private func showAgentNotch() {
|
||
showAgentNotchInternal(isTextInput: false)
|
||
}
|
||
|
||
private func showTextInputNotch() {
|
||
// 重置显示状态，但不进入 thinking（等用户发送后再开始）
|
||
agentTracker.resetForInput()
|
||
agentIsTextMode = true
|
||
showAgentNotchInternal(isTextInput: true)
|
||
}
|
||
|
||
private func showAgentNotchInternal(isTextInput: Bool) {
|
||
agentTracker.isTextInputMode = isTextInput
|
||
|
||
// 弹窗已存在 → 复用，只刷新显示
|
||
if agentNotch != nil {
|
||
agentNotch?.show()
|
||
return
|
||
}
|
||
|
||
// 创建新弹窗
|
||
let view = AgentNotchView(
|
||
tracker: agentTracker,
|
||
onCopy: {
|
||
NSPasteboard.general.clearContents()
|
||
NSPasteboard.general.setString(self.agentTracker.finalContent, forType: .string)
|
||
},
|
||
onStop: { [weak self] in
|
||
self?.speakingMode = nil
|
||
self?.speakAutoHideWorkItem?.cancel()
|
||
self?.speechSynthesizer.stopSpeaking()
|
||
self?.autoHideWorkItem?.cancel()
|
||
self?.agentExitingCleanly = true
|
||
self?.agentBridge.stop()
|
||
self?.hideAgentNotch()
|
||
},
|
||
onClose: { [weak self] in
|
||
self?.speakingMode = nil
|
||
self?.speakAutoHideWorkItem?.cancel()
|
||
self?.speechSynthesizer.stopSpeaking()
|
||
self?.autoHideWorkItem?.cancel()
|
||
self?.hideAgentNotch()
|
||
},
|
||
onNewConversation: { [weak self] in
|
||
self?.speakingMode = nil
|
||
self?.speakAutoHideWorkItem?.cancel()
|
||
self?.speechSynthesizer.stopSpeaking()
|
||
self?.autoHideWorkItem?.cancel()
|
||
self?.agentExitingCleanly = true
|
||
self?.agentBridge.stop()
|
||
self?.agentActive = false
|
||
self?.agentIsTextMode = false
|
||
self?.agentTracker.resetForInput()
|
||
},
|
||
onSendText: { [weak self] text in
|
||
self?.handleTextInput(text)
|
||
},
|
||
onContentSizeChanged: { [weak self] in
|
||
self?.agentNotch?.notifyContentChanged()
|
||
}
|
||
)
|
||
agentNotch = DynamicNotch(style: .notch) { view }
|
||
agentNotch?.show()
|
||
}
|
||
|
||
private func handleTextInput(_ text: String) {
|
||
handleAgentMode(text: text)
|
||
}
|
||
|
||
func applicationWillTerminate(_ notification: Notification) {
|
||
speechSynthesizer.stopSpeaking()
|
||
keyMonitor.stop()
|
||
audioMonitor.stop()
|
||
speechRecognizer.cancel()
|
||
recordingPanel.close()
|
||
}
|
||
|
||
// MARK: - NSSpeechSynthesizerDelegate
|
||
|
||
func speechSynthesizer(_ sender: NSSpeechSynthesizer, didFinishSpeaking finishedSpeaking: Bool) {
|
||
guard finishedSpeaking else { return }
|
||
DispatchQueue.main.async { [weak self] in
|
||
guard let self, let mode = self.speakingMode else { return }
|
||
self.speakingMode = nil
|
||
|
||
// 朗读完成后等待 5 秒再带动画关闭弹窗
|
||
let workItem = DispatchWorkItem { [weak self] in
|
||
switch mode {
|
||
case .assist:
|
||
NotchDisplayManager.shared.extendAutoHide(to: 0.1)
|
||
case .agent:
|
||
// 打字模式下不自动关闭，用户可能还在输入框中
|
||
if self?.agentIsTextMode != true {
|
||
self?.agentNotch?.show(for: 0.1)
|
||
}
|
||
default:
|
||
break
|
||
}
|
||
}
|
||
self.speakAutoHideWorkItem = workItem
|
||
DispatchQueue.main.asyncAfter(deadline: .now() + 5, execute: workItem)
|
||
}
|
||
}
|
||
}
|