feat(android-agent): Audio-Fokus — ARIA redet nicht mehr ueber den laufenden Film
Bisher wurde KEIN Audio-Fokus angefordert: Android mischt dann beide Tonspuren,
d.h. Netflix/Jellyfin/Spotify liefen in voller Lautstaerke weiter und ARIA war
kaum zu verstehen — und beim Zuhoeren nahm das Mikrofon den Filmton mit auf.
AudioFocusHelper mit drei Modi:
- speak/notify: AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK ('duck', Standard) -> Medien
werden leiser und danach automatisch wieder lauter. Per focus-Param auch
'pause' (Medien anhalten) oder 'none'.
- listen: AUDIOFOCUS_GAIN_TRANSIENT ('pause') -> Medien halten an, solange das
Mikrofon offen ist; danach laufen sie weiter.
- Fokus wird in allen Pfaden zurueckgegeben (Completion, Fehler, stop, shutdown),
sonst blieben die Medien dauerhaft leise.
Brain: focus-Parameter in host_notify/host_speak. Android-Version 0.0.1.4.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+4
-2
@@ -1523,6 +1523,7 @@ META_TOOLS = [
|
|||||||
"title": {"type": "string", "description": "Ueberschrift (Default 'ARIA')."},
|
"title": {"type": "string", "description": "Ueberschrift (Default 'ARIA')."},
|
||||||
"seconds": {"type": "integer", "description": "Anzeigedauer (Default 12)."},
|
"seconds": {"type": "integer", "description": "Anzeigedauer (Default 12)."},
|
||||||
"mode": {"type": "string", "description": "overlay | speech | both"},
|
"mode": {"type": "string", "description": "overlay | speech | both"},
|
||||||
|
"focus": {"type": "string", "description": "Bei Sprachausgabe: 'duck' (laufende Medien werden leiser, Standard) | 'pause' (Film/Musik anhalten) | 'none'."},
|
||||||
},
|
},
|
||||||
"required": ["host", "text"],
|
"required": ["host", "text"],
|
||||||
},
|
},
|
||||||
@@ -1542,6 +1543,7 @@ META_TOOLS = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||||
"text": {"type": "string", "description": "Was gesprochen werden soll."},
|
"text": {"type": "string", "description": "Was gesprochen werden soll."},
|
||||||
|
"focus": {"type": "string", "description": "'duck' (Medien leiser, Standard) | 'pause' (Film/Musik anhalten) | 'none'."},
|
||||||
},
|
},
|
||||||
"required": ["host", "text"],
|
"required": ["host", "text"],
|
||||||
},
|
},
|
||||||
@@ -2925,8 +2927,8 @@ class Agent:
|
|||||||
"host_ui_swipe": ("ui_swipe", ["x1", "y1", "x2", "y2", "duration_ms"]),
|
"host_ui_swipe": ("ui_swipe", ["x1", "y1", "x2", "y2", "duration_ms"]),
|
||||||
"host_ui_key": ("ui_key", ["key"]),
|
"host_ui_key": ("ui_key", ["key"]),
|
||||||
"host_app_launch": ("app_launch", ["package", "query", "command"]),
|
"host_app_launch": ("app_launch", ["package", "query", "command"]),
|
||||||
"host_notify": ("notify", ["text", "title", "seconds", "mode"]),
|
"host_notify": ("notify", ["text", "title", "seconds", "mode", "focus"]),
|
||||||
"host_speak": ("speak", ["text"]),
|
"host_speak": ("speak", ["text", "focus"]),
|
||||||
"host_volume": ("volume", ["percent"]),
|
"host_volume": ("volume", ["percent"]),
|
||||||
}
|
}
|
||||||
if name in _UI_ACTIONS:
|
if name in _UI_ACTIONS:
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package de.hackersoft.ariaagent
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.media.AudioAttributes
|
||||||
|
import android.media.AudioFocusRequest
|
||||||
|
import android.media.AudioManager
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Audio-Fokus — damit ARIA nicht einfach ueber den laufenden Film redet.
|
||||||
|
*
|
||||||
|
* Ohne Fokus mischt Android beide Tonspuren: Netflix/Jellyfin/Spotify laufen in
|
||||||
|
* voller Lautstaerke weiter und ARIA ist kaum zu verstehen. Mit Fokus verhaelt
|
||||||
|
* sie sich wie ein normaler Sprachassistent:
|
||||||
|
*
|
||||||
|
* "duck" (Standard) laufende Medien werden LEISER, ARIA spricht darueber,
|
||||||
|
* danach geht die Lautstaerke von selbst wieder hoch.
|
||||||
|
* "pause" laufende Medien werden PAUSIERT (Player reagieren auf
|
||||||
|
* den Fokusverlust) — sinnvoll beim Zuhoeren, damit das
|
||||||
|
* Mikrofon nicht den Filmton aufnimmt.
|
||||||
|
* "none" kein Fokus (alte Mischung).
|
||||||
|
*/
|
||||||
|
object AudioFocusHelper {
|
||||||
|
|
||||||
|
private var request: AudioFocusRequest? = null
|
||||||
|
private var holder: String = ""
|
||||||
|
|
||||||
|
/** Fordert den Fokus an. [mode] = duck | pause | none. */
|
||||||
|
@Synchronized
|
||||||
|
fun request(ctx: Context, mode: String, who: String): Boolean {
|
||||||
|
if (mode == "none") return true
|
||||||
|
abandon(ctx)
|
||||||
|
val gain = if (mode == "pause") AudioManager.AUDIOFOCUS_GAIN_TRANSIENT
|
||||||
|
else AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK
|
||||||
|
val usage = if (who == "listen") AudioAttributes.USAGE_ASSISTANCE_SONIFICATION
|
||||||
|
else AudioAttributes.USAGE_ASSISTANT
|
||||||
|
return try {
|
||||||
|
val am = ctx.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||||
|
val req = AudioFocusRequest.Builder(gain)
|
||||||
|
.setAudioAttributes(
|
||||||
|
AudioAttributes.Builder()
|
||||||
|
.setUsage(usage)
|
||||||
|
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
// Kurze Unterbrechungen nicht als "verloren" behandeln.
|
||||||
|
.setWillPauseWhenDucked(false)
|
||||||
|
.setOnAudioFocusChangeListener { }
|
||||||
|
.build()
|
||||||
|
request = req
|
||||||
|
holder = who
|
||||||
|
am.requestAudioFocus(req) == AudioManager.AUDIOFOCUS_REQUEST_GRANTED
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Gibt den Fokus zurueck — die Medien werden wieder lauter / laufen weiter. */
|
||||||
|
@Synchronized
|
||||||
|
fun abandon(ctx: Context) {
|
||||||
|
val req = request ?: return
|
||||||
|
request = null
|
||||||
|
holder = ""
|
||||||
|
try {
|
||||||
|
val am = ctx.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||||
|
am.abandonAudioFocusRequest(req)
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -336,12 +336,14 @@ class RvsClient(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Audio bevorzugt (ARIAs echte Stimme), sonst Geraete-TTS. */
|
/** Audio bevorzugt (ARIAs echte Stimme), sonst Geraete-TTS.
|
||||||
|
* focus: duck (Medien leiser, Standard) | pause (Medien anhalten) | none. */
|
||||||
private fun speakOut(p: JSONObject, text: String): Result<Unit> {
|
private fun speakOut(p: JSONObject, text: String): Result<Unit> {
|
||||||
val audio = p.optString("audio_base64")
|
val audio = p.optString("audio_base64")
|
||||||
|
val focus = p.optString("focus").ifBlank { "duck" }
|
||||||
return if (audio.isNotBlank())
|
return if (audio.isNotBlank())
|
||||||
Speaker.play(appCtx, audio, p.optString("audio_format", "wav"))
|
Speaker.play(appCtx, audio, p.optString("audio_format", "wav"), focus)
|
||||||
else Speaker.speak(appCtx, text)
|
else Speaker.speak(appCtx, text, focus)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun doVolume(p: JSONObject): JSONObject {
|
private fun doVolume(p: JSONObject): JSONObject {
|
||||||
|
|||||||
@@ -24,28 +24,38 @@ object Speaker {
|
|||||||
private var ttsReady = false
|
private var ttsReady = false
|
||||||
private var player: MediaPlayer? = null
|
private var player: MediaPlayer? = null
|
||||||
|
|
||||||
/** Spielt fertiges Audio (Base64, z.B. WAV aus dem ARIA-TTS) ueber die Lautsprecher. */
|
/** Spielt fertiges Audio (Base64, z.B. WAV aus dem ARIA-TTS) ueber die Lautsprecher.
|
||||||
fun play(ctx: Context, audioB64: String, format: String = "wav"): Result<Unit> = runCatching {
|
* [focus] = duck (Medien leiser, Standard) | pause (Medien anhalten) | none. */
|
||||||
|
fun play(ctx: Context, audioB64: String, format: String = "wav",
|
||||||
|
focus: String = "duck"): Result<Unit> = runCatching {
|
||||||
stop()
|
stop()
|
||||||
val bytes = Base64.decode(audioB64, Base64.DEFAULT)
|
val bytes = Base64.decode(audioB64, Base64.DEFAULT)
|
||||||
val f = File(ctx.cacheDir, "aria_speak.${format.ifBlank { "wav" }}")
|
val f = File(ctx.cacheDir, "aria_speak.${format.ifBlank { "wav" }}")
|
||||||
f.writeBytes(bytes)
|
f.writeBytes(bytes)
|
||||||
|
// Erst Fokus holen, damit Netflix & Co. leiser werden (sonst redet ARIA
|
||||||
|
// einfach mittendrin mit und man versteht beides nicht).
|
||||||
|
AudioFocusHelper.request(ctx, focus, "speak")
|
||||||
player = MediaPlayer().apply {
|
player = MediaPlayer().apply {
|
||||||
setAudioAttributes(
|
setAudioAttributes(
|
||||||
AudioAttributes.Builder()
|
AudioAttributes.Builder()
|
||||||
.setUsage(AudioAttributes.USAGE_MEDIA)
|
.setUsage(AudioAttributes.USAGE_ASSISTANT)
|
||||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||||
.build()
|
.build()
|
||||||
)
|
)
|
||||||
setDataSource(f.absolutePath)
|
setDataSource(f.absolutePath)
|
||||||
setOnCompletionListener { mp -> try { mp.release() } catch (_: Exception) {}; player = null }
|
setOnCompletionListener { mp ->
|
||||||
|
try { mp.release() } catch (_: Exception) {}
|
||||||
|
player = null
|
||||||
|
AudioFocusHelper.abandon(ctx) // Medien wieder laut / weiterlaufen
|
||||||
|
}
|
||||||
|
setOnErrorListener { _, _, _ -> AudioFocusHelper.abandon(ctx); false }
|
||||||
prepare()
|
prepare()
|
||||||
start()
|
start()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Rueckfallebene: geraeteeigene Sprachausgabe. */
|
/** Rueckfallebene: geraeteeigene Sprachausgabe. */
|
||||||
fun speak(ctx: Context, text: String): Result<Unit> = runCatching {
|
fun speak(ctx: Context, text: String, focus: String = "duck"): Result<Unit> = runCatching {
|
||||||
if (tts == null) {
|
if (tts == null) {
|
||||||
tts = TextToSpeech(ctx.applicationContext) { status ->
|
tts = TextToSpeech(ctx.applicationContext) { status ->
|
||||||
ttsReady = status == TextToSpeech.SUCCESS
|
ttsReady = status == TextToSpeech.SUCCESS
|
||||||
@@ -56,6 +66,13 @@ object Speaker {
|
|||||||
while (!ttsReady && waited < 3000) { Thread.sleep(100); waited += 100 }
|
while (!ttsReady && waited < 3000) { Thread.sleep(100); waited += 100 }
|
||||||
}
|
}
|
||||||
if (!ttsReady) throw IllegalStateException("Keine Sprachausgabe verfuegbar (TTS-Engine fehlt)")
|
if (!ttsReady) throw IllegalStateException("Keine Sprachausgabe verfuegbar (TTS-Engine fehlt)")
|
||||||
|
AudioFocusHelper.request(ctx, focus, "speak")
|
||||||
|
tts?.setOnUtteranceProgressListener(object : android.speech.tts.UtteranceProgressListener() {
|
||||||
|
override fun onStart(id: String?) {}
|
||||||
|
override fun onDone(id: String?) { AudioFocusHelper.abandon(ctx) }
|
||||||
|
@Deprecated("Deprecated in Java")
|
||||||
|
override fun onError(id: String?) { AudioFocusHelper.abandon(ctx) }
|
||||||
|
})
|
||||||
tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, "aria")
|
tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, "aria")
|
||||||
Unit
|
Unit
|
||||||
}
|
}
|
||||||
@@ -68,9 +85,10 @@ object Speaker {
|
|||||||
am.setStreamVolume(AudioManager.STREAM_MUSIC, v, AudioManager.FLAG_SHOW_UI)
|
am.setStreamVolume(AudioManager.STREAM_MUSIC, v, AudioManager.FLAG_SHOW_UI)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun stop() {
|
fun stop(ctx: Context? = null) {
|
||||||
try { player?.stop(); player?.release() } catch (_: Exception) {}
|
try { player?.stop(); player?.release() } catch (_: Exception) {}
|
||||||
player = null
|
player = null
|
||||||
try { tts?.stop() } catch (_: Exception) {}
|
try { tts?.stop() } catch (_: Exception) {}
|
||||||
|
ctx?.let { AudioFocusHelper.abandon(it) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,6 +100,9 @@ class VoiceSession(
|
|||||||
|
|
||||||
rec = r
|
rec = r
|
||||||
recording = true
|
recording = true
|
||||||
|
// Beim Zuhoeren laufende Medien ANHALTEN — sonst nimmt das Mikrofon den
|
||||||
|
// Filmton mit auf und die Spracherkennung versteht nur Kauderwelsch.
|
||||||
|
AudioFocusHelper.request(appCtx, "pause", "listen")
|
||||||
|
|
||||||
send("stt_stream_start", JSONObject().apply {
|
send("stt_stream_start", JSONObject().apply {
|
||||||
put("requestId", sid)
|
put("requestId", sid)
|
||||||
@@ -154,6 +157,9 @@ class VoiceSession(
|
|||||||
try { aec?.release() } catch (_: Exception) {}
|
try { aec?.release() } catch (_: Exception) {}
|
||||||
try { ns?.release() } catch (_: Exception) {}
|
try { ns?.release() } catch (_: Exception) {}
|
||||||
rec = null; aec = null; ns = null
|
rec = null; aec = null; ns = null
|
||||||
|
// Aufnahme vorbei -> Medien laufen weiter (die Antwort holt sich gleich
|
||||||
|
// selbst wieder Fokus, sobald ARIAs Audio kommt).
|
||||||
|
AudioFocusHelper.abandon(appCtx)
|
||||||
sessionId?.let {
|
sessionId?.let {
|
||||||
send("stt_stream_end", JSONObject().apply {
|
send("stt_stream_end", JSONObject().apply {
|
||||||
put("requestId", it); put("reason", reason); put("targetInstance", "")
|
put("requestId", it); put("reason", reason); put("targetInstance", "")
|
||||||
@@ -245,6 +251,8 @@ class VoiceSession(
|
|||||||
try { track?.stop() } catch (_: Exception) {}
|
try { track?.stop() } catch (_: Exception) {}
|
||||||
try { track?.release() } catch (_: Exception) {}
|
try { track?.release() } catch (_: Exception) {}
|
||||||
track = null; playingMessageId = null
|
track = null; playingMessageId = null
|
||||||
|
// Fertig gesprochen -> Medien wieder hochfahren/weiterlaufen lassen.
|
||||||
|
AudioFocusHelper.abandon(appCtx)
|
||||||
onEvent("bereit")
|
onEvent("bereit")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -253,6 +261,8 @@ class VoiceSession(
|
|||||||
val pcm = try { Base64.decode(b64, Base64.DEFAULT) } catch (_: Exception) { return }
|
val pcm = try { Base64.decode(b64, Base64.DEFAULT) } catch (_: Exception) { return }
|
||||||
if (track == null || playingMessageId != mid) {
|
if (track == null || playingMessageId != mid) {
|
||||||
try { track?.release() } catch (_: Exception) {}
|
try { track?.release() } catch (_: Exception) {}
|
||||||
|
// ARIA antwortet -> laufende Medien leiser (duck), bis sie fertig ist.
|
||||||
|
AudioFocusHelper.request(appCtx, "duck", "speak")
|
||||||
val min = AudioTrack.getMinBufferSize(rate, AudioFormat.CHANNEL_OUT_MONO,
|
val min = AudioTrack.getMinBufferSize(rate, AudioFormat.CHANNEL_OUT_MONO,
|
||||||
AudioFormat.ENCODING_PCM_16BIT)
|
AudioFormat.ENCODING_PCM_16BIT)
|
||||||
track = AudioTrack.Builder()
|
track = AudioTrack.Builder()
|
||||||
@@ -276,5 +286,6 @@ class VoiceSession(
|
|||||||
stopListening("shutdown")
|
stopListening("shutdown")
|
||||||
try { track?.stop(); track?.release() } catch (_: Exception) {}
|
try { track?.stop(); track?.release() } catch (_: Exception) {}
|
||||||
track = null
|
track = null
|
||||||
|
AudioFocusHelper.abandon(appCtx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user