JOJO
d24e00b9b3
基于 Sherpa-ONNX + SenseVoice int8 实现 APK 端侧语音识别： - VoiceBridge: AudioRecord 录音 + 整段识别 + JS Bridge 回调 - ModelManager: 模型下载管理（自有服务器），支持断点续传/校验/删除 - 前端：语音按钮仅 APK 环境显示，识别结果回填 ProseMirror 编辑器 - 调试：文件日志 + /api/voice_debug 接收路由 - demo/sense_voice_demo.py: Python 端测 Demo versionCode 24→36, versionName 1.0.22→1.0.34
409 lines
15 KiB
Kotlin
409 lines
15 KiB
Kotlin
package com.cyjai.agent
|
||
|
||
import android.Manifest
|
||
import android.annotation.SuppressLint
|
||
import android.content.Intent
|
||
import android.content.pm.PackageManager
|
||
import android.graphics.Color
|
||
import android.net.Uri
|
||
import android.os.Build
|
||
import android.os.Bundle
|
||
import android.util.Log
|
||
import android.view.ViewGroup
|
||
import android.view.View
|
||
import android.webkit.CookieManager
|
||
import android.webkit.JavascriptInterface
|
||
import android.webkit.PermissionRequest
|
||
import android.webkit.ValueCallback
|
||
import android.webkit.WebChromeClient
|
||
import android.webkit.WebResourceRequest
|
||
import android.webkit.WebSettings
|
||
import android.webkit.WebView
|
||
import android.webkit.WebViewClient
|
||
import androidx.activity.OnBackPressedCallback
|
||
import androidx.activity.result.ActivityResultLauncher
|
||
import androidx.activity.result.contract.ActivityResultContracts
|
||
import androidx.appcompat.app.AppCompatActivity
|
||
import androidx.core.app.ActivityCompat
|
||
import androidx.core.content.ContextCompat
|
||
import androidx.core.view.ViewCompat
|
||
import androidx.core.view.WindowInsetsCompat
|
||
import androidx.core.view.WindowCompat
|
||
import androidx.lifecycle.lifecycleScope
|
||
import kotlinx.coroutines.Dispatchers
|
||
import kotlinx.coroutines.launch
|
||
import kotlinx.coroutines.withContext
|
||
import java.util.Locale
|
||
|
||
class MainActivity : AppCompatActivity() {
|
||
|
||
private lateinit var webView: WebView
|
||
private var filePathCallback: ValueCallback<Array<Uri>>? = null
|
||
private var voiceBridge: VoiceBridge? = null
|
||
|
||
companion object {
|
||
private const val TAG = "MainActivity"
|
||
private const val REQUEST_RECORD_AUDIO = 300
|
||
private const val HOME_URL = "https://agent.cyjai.com"
|
||
private const val WEB_ASSET_VERSION = "20260406_4"
|
||
}
|
||
|
||
private val fileChooserLauncher: ActivityResultLauncher<Intent> = registerForActivityResult(
|
||
ActivityResultContracts.StartActivityForResult()
|
||
) { result ->
|
||
val results = WebChromeClient.FileChooserParams.parseResult(result.resultCode, result.data)
|
||
filePathCallback?.onReceiveValue(results)
|
||
filePathCallback = null
|
||
}
|
||
|
||
@SuppressLint("SetJavaScriptEnabled")
|
||
override fun onCreate(savedInstanceState: Bundle?) {
|
||
super.onCreate(savedInstanceState)
|
||
|
||
webView = WebView(this)
|
||
webView.layoutParams = ViewGroup.LayoutParams(
|
||
ViewGroup.LayoutParams.MATCH_PARENT,
|
||
ViewGroup.LayoutParams.MATCH_PARENT
|
||
)
|
||
webView.isVerticalScrollBarEnabled = false
|
||
webView.isHorizontalScrollBarEnabled = false
|
||
webView.overScrollMode = View.OVER_SCROLL_NEVER
|
||
setContentView(webView)
|
||
|
||
ViewCompat.setOnApplyWindowInsetsListener(webView) { view, insets ->
|
||
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
|
||
view.setPadding(0, 0, 0, systemBars.bottom)
|
||
insets
|
||
}
|
||
ViewCompat.requestApplyInsets(webView)
|
||
|
||
val cookieManager = CookieManager.getInstance()
|
||
cookieManager.setAcceptCookie(true)
|
||
cookieManager.setAcceptThirdPartyCookies(webView, true)
|
||
|
||
webView.settings.apply {
|
||
javaScriptEnabled = true
|
||
domStorageEnabled = true
|
||
databaseEnabled = true
|
||
mediaPlaybackRequiresUserGesture = false
|
||
// 你的 Nginx 对静态资源设置了 immutable 长缓存，这里对 App 侧禁用缓存，确保样式更新及时生效
|
||
cacheMode = WebSettings.LOAD_NO_CACHE
|
||
useWideViewPort = true
|
||
loadWithOverviewMode = true
|
||
setSupportZoom(false)
|
||
builtInZoomControls = false
|
||
displayZoomControls = false
|
||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||
safeBrowsingEnabled = true
|
||
}
|
||
}
|
||
webView.clearCache(true)
|
||
|
||
webView.addJavascriptInterface(ThemeBridge(), "AndroidThemeBridge")
|
||
|
||
// 语音识别桥接（立即注册，前端始终可检测到；引擎在首次使用时懒初始化）
|
||
voiceBridge = VoiceBridge(this@MainActivity, webView)
|
||
webView.addJavascriptInterface(voiceBridge!!, "AndroidVoiceBridge")
|
||
|
||
// 后台检查并下载模型
|
||
lifecycleScope.launch(Dispatchers.IO) {
|
||
ModelManager.init(this@MainActivity)
|
||
if (!ModelManager.isModelReady(this@MainActivity)) {
|
||
Log.i(TAG, "首次启动，开始下载语音模型...")
|
||
withContext(Dispatchers.Main) {
|
||
injectVoiceStatus("downloading")
|
||
}
|
||
val success = ModelManager.downloadModels(this@MainActivity) { pct, msg ->
|
||
Log.i(TAG, "模型下载: ${pct}% - $msg")
|
||
runOnUiThread {
|
||
injectVoiceDownloadProgress(pct, msg)
|
||
}
|
||
}
|
||
if (!success) {
|
||
Log.e(TAG, "模型下载失败")
|
||
withContext(Dispatchers.Main) {
|
||
injectVoiceStatus("error")
|
||
}
|
||
return@launch
|
||
}
|
||
}
|
||
Log.i(TAG, "模型就绪")
|
||
withContext(Dispatchers.Main) {
|
||
injectVoiceStatus("ready")
|
||
// 模型就绪后立即预初始化识别引擎，用户点击时直接可用
|
||
voiceBridge?.ensureEngine()
|
||
}
|
||
}
|
||
|
||
// 请求录音权限
|
||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
|
||
!= PackageManager.PERMISSION_GRANTED
|
||
) {
|
||
ActivityCompat.requestPermissions(
|
||
this,
|
||
arrayOf(Manifest.permission.RECORD_AUDIO),
|
||
REQUEST_RECORD_AUDIO
|
||
)
|
||
}
|
||
|
||
webView.webViewClient = object : WebViewClient() {
|
||
override fun onPageFinished(view: WebView?, url: String?) {
|
||
super.onPageFinished(view, url)
|
||
injectThemeObserver()
|
||
injectMobileOverlayWidthPatch()
|
||
injectNoPageScrollPatch()
|
||
}
|
||
|
||
override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
|
||
val url = request?.url?.toString() ?: return false
|
||
if (isApkUrl(url)) {
|
||
val intent = Intent(Intent.ACTION_VIEW, Uri.parse(url))
|
||
startActivity(intent)
|
||
return true
|
||
}
|
||
return !url.startsWith(HOME_URL)
|
||
}
|
||
}
|
||
|
||
webView.webChromeClient = object : WebChromeClient() {
|
||
override fun onPermissionRequest(request: PermissionRequest) {
|
||
// 按需放行媒体权限（例如文件上传触发的媒体访问）
|
||
request.grant(request.resources)
|
||
}
|
||
|
||
override fun onShowFileChooser(
|
||
webView: WebView?,
|
||
filePathCallback: ValueCallback<Array<Uri>>?,
|
||
fileChooserParams: FileChooserParams?
|
||
): Boolean {
|
||
this@MainActivity.filePathCallback?.onReceiveValue(null)
|
||
this@MainActivity.filePathCallback = filePathCallback
|
||
|
||
val intent = fileChooserParams?.createIntent() ?: Intent(Intent.ACTION_GET_CONTENT).apply {
|
||
addCategory(Intent.CATEGORY_OPENABLE)
|
||
type = "*/*"
|
||
}
|
||
fileChooserLauncher.launch(intent)
|
||
return true
|
||
}
|
||
}
|
||
|
||
onBackPressedDispatcher.addCallback(this, object : OnBackPressedCallback(true) {
|
||
override fun handleOnBackPressed() {
|
||
if (webView.canGoBack()) {
|
||
webView.goBack()
|
||
} else {
|
||
finish()
|
||
}
|
||
}
|
||
})
|
||
|
||
if (savedInstanceState == null) {
|
||
webView.loadUrl(buildHomeUrl(), mapOf(
|
||
"Cache-Control" to "no-cache, no-store, must-revalidate",
|
||
"Pragma" to "no-cache"
|
||
))
|
||
} else {
|
||
webView.restoreState(savedInstanceState)
|
||
}
|
||
}
|
||
|
||
override fun onSaveInstanceState(outState: Bundle) {
|
||
webView.saveState(outState)
|
||
super.onSaveInstanceState(outState)
|
||
}
|
||
|
||
override fun onDestroy() {
|
||
voiceBridge?.destroy()
|
||
webView.destroy()
|
||
super.onDestroy()
|
||
}
|
||
|
||
private fun applySystemBarTheme(rawTheme: String?) {
|
||
val theme = (rawTheme ?: "").lowercase(Locale.getDefault())
|
||
val isDark = theme == "dark"
|
||
val bgColor = if (isDark) Color.BLACK else Color.WHITE
|
||
|
||
window.statusBarColor = bgColor
|
||
window.navigationBarColor = bgColor
|
||
WindowCompat.getInsetsController(window, window.decorView)?.let { controller ->
|
||
controller.isAppearanceLightStatusBars = !isDark
|
||
controller.isAppearanceLightNavigationBars = !isDark
|
||
}
|
||
}
|
||
|
||
private fun injectThemeObserver() {
|
||
val script = """
|
||
(function() {
|
||
function readTheme() {
|
||
var t = document.documentElement.getAttribute('data-theme')
|
||
|| document.body.getAttribute('data-theme')
|
||
|| localStorage.getItem('agents_ui_theme')
|
||
|| 'claude';
|
||
return String(t).toLowerCase();
|
||
}
|
||
function notify() {
|
||
if (window.AndroidThemeBridge && window.AndroidThemeBridge.onThemeChanged) {
|
||
window.AndroidThemeBridge.onThemeChanged(readTheme());
|
||
}
|
||
}
|
||
notify();
|
||
if (!window.__androidThemeObserverInstalled) {
|
||
window.__androidThemeObserverInstalled = true;
|
||
var ob = new MutationObserver(function() { notify(); });
|
||
ob.observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] });
|
||
if (document.body) {
|
||
ob.observe(document.body, { attributes: true, attributeFilter: ['data-theme'] });
|
||
}
|
||
window.addEventListener('storage', function(e){
|
||
if (e && e.key === 'agents_ui_theme') notify();
|
||
});
|
||
}
|
||
})();
|
||
""".trimIndent()
|
||
webView.evaluateJavascript(script, null)
|
||
}
|
||
|
||
private fun injectMobileOverlayWidthPatch() {
|
||
val script = """
|
||
(function() {
|
||
var styleId = '__android_mobile_overlay_width_patch';
|
||
var css = [
|
||
'.mobile-panel-sheet.mobile-panel-sheet--conversation {',
|
||
' width: min(var(--conversation-expanded-width, 306px), 100vw) !important;',
|
||
' max-width: 100vw !important;',
|
||
' padding-top: 0 !important;',
|
||
'}',
|
||
'.mobile-panel-sheet--conversation .mobile-overlay-content {',
|
||
' height: 100% !important;',
|
||
'}',
|
||
'.mobile-panel-sheet.mobile-panel-sheet--workspace {',
|
||
' width: fit-content !important;',
|
||
' min-width: min(420px, 60vw) !important;',
|
||
' max-width: 100vw !important;',
|
||
'}'
|
||
].join('\n');
|
||
var existing = document.getElementById(styleId);
|
||
if (existing) {
|
||
existing.textContent = css;
|
||
return;
|
||
}
|
||
var style = document.createElement('style');
|
||
style.id = styleId;
|
||
style.type = 'text/css';
|
||
style.textContent = css;
|
||
(document.head || document.documentElement).appendChild(style);
|
||
})();
|
||
""".trimIndent()
|
||
webView.evaluateJavascript(script, null)
|
||
}
|
||
|
||
private fun injectNoPageScrollPatch() {
|
||
val script = """
|
||
(function() {
|
||
var styleId = '__android_no_page_scroll_patch';
|
||
var css = [
|
||
'html, body {',
|
||
' overflow: hidden !important;',
|
||
' height: 100% !important;',
|
||
' max-height: 100% !important;',
|
||
'}',
|
||
'body {',
|
||
' position: fixed !important;',
|
||
' width: 100% !important;',
|
||
' inset: 0 !important;',
|
||
'}',
|
||
'* {',
|
||
' scrollbar-width: none !important;',
|
||
'}',
|
||
'*::-webkit-scrollbar {',
|
||
' width: 0 !important;',
|
||
' height: 0 !important;',
|
||
' display: none !important;',
|
||
'}',
|
||
'#app, .app-root, .chat-container {',
|
||
' max-height: 100% !important;',
|
||
' overflow: hidden !important;',
|
||
'}'
|
||
].join('\n');
|
||
var existing = document.getElementById(styleId);
|
||
if (existing) {
|
||
existing.textContent = css;
|
||
return;
|
||
}
|
||
var style = document.createElement('style');
|
||
style.id = styleId;
|
||
style.type = 'text/css';
|
||
style.textContent = css;
|
||
(document.head || document.documentElement).appendChild(style);
|
||
})();
|
||
""".trimIndent()
|
||
webView.evaluateJavascript(script, null)
|
||
}
|
||
|
||
// ═══════════════════ 语音状态通知（JS 注入） ═══════════════════
|
||
|
||
private fun injectVoiceStatus(status: String) {
|
||
val script = """
|
||
(function() {
|
||
if (window.__onVoiceStatus) window.__onVoiceStatus('$status');
|
||
if (window.dispatchEvent) {
|
||
window.dispatchEvent(new CustomEvent('voicebridge:status', { detail: '$status' }));
|
||
}
|
||
})();
|
||
""".trimIndent()
|
||
webView.evaluateJavascript(script, null)
|
||
}
|
||
|
||
private fun injectVoiceDownloadProgress(pct: Int, msg: String) {
|
||
val safeMsg = msg.replace("'", "\\'")
|
||
val script = """
|
||
(function() {
|
||
if (window.__onVoiceDownloadProgress) {
|
||
window.__onVoiceDownloadProgress($pct, '$safeMsg');
|
||
}
|
||
})();
|
||
""".trimIndent()
|
||
webView.evaluateJavascript(script, null)
|
||
}
|
||
|
||
inner class ThemeBridge {
|
||
@JavascriptInterface
|
||
fun onThemeChanged(theme: String?) {
|
||
runOnUiThread {
|
||
applySystemBarTheme(theme)
|
||
}
|
||
}
|
||
|
||
@JavascriptInterface
|
||
fun getAppVersionCode(): String {
|
||
return getInstalledVersionCode().toString()
|
||
}
|
||
|
||
@JavascriptInterface
|
||
fun getAppVersionName(): String {
|
||
return getInstalledVersionName()
|
||
}
|
||
}
|
||
|
||
private fun buildHomeUrl(): String {
|
||
val vc = getInstalledVersionCode()
|
||
val vn = Uri.encode(getInstalledVersionName())
|
||
return "$HOME_URL/?app_shell=$WEB_ASSET_VERSION&app_vc=$vc&app_vn=$vn"
|
||
}
|
||
|
||
private fun getInstalledVersionCode(): Long {
|
||
val pkgInfo = packageManager.getPackageInfo(packageName, 0)
|
||
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) pkgInfo.longVersionCode else @Suppress("DEPRECATION") pkgInfo.versionCode.toLong()
|
||
}
|
||
|
||
private fun getInstalledVersionName(): String {
|
||
val pkgInfo = packageManager.getPackageInfo(packageName, 0)
|
||
return pkgInfo.versionName ?: "unknown"
|
||
}
|
||
|
||
private fun isApkUrl(url: String): Boolean {
|
||
return url.lowercase(Locale.getDefault()).endsWith(".apk") || url.contains("/api/app/apk/latest")
|
||
}
|
||
}
|