feat(android-agent): Fire TV als ARIA-Endpunkt — Sprache, Overlay, Ausgabe-Modus
Der Stick wird vom gesteuerten Geraet zum Gespraechspartner: - VoiceSession: Mikro -> stt_stream_start/stt_audio_chunk(pcm,16k mono s16le, 6400B)/stt_stream_end (exakt das App-Protokoll, keine Server-Aenderung noetig); Antwort via chat(sender=aria) + audio_pcm(base64, 24k) in AudioTrack. AEC/NS an. TURN-OWNERSHIP: der RVS broadcastet an alle — es wird nur ausgegeben, was aus der EIGENEN Frage entstand (audioRequestId -> messageId), sonst reden Handy und Stick gleichzeitig. - OverlayView: TYPE_APPLICATION_OVERLAY, Einblendung ueber dem laufenden Bild (Text + optional Bild), nicht fokussierbar (Fernbedienung bleibt frei). - Speaker: ARIAs echte Stimme (fertiges Audio) bevorzugt, Geraete-TTS als Fallback, Lautstaerke-Steuerung. - Neue Aktionen: notify (Einblendung/Sprache, mode ueberschreibbar), speak, volume, listen/listen_stop. - Einstellung replyMode (overlay|speech) in der App, pro Anfrage ueberschreibbar. - Sprech-Taste in der UI (Fire TV: OK-Taste) + Mikro-/Overlay-Berechtigungen. Version 0.0.1.0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,8 +11,8 @@ android {
|
|||||||
applicationId 'de.hackersoft.ariaagent'
|
applicationId 'de.hackersoft.ariaagent'
|
||||||
minSdk 26
|
minSdk 26
|
||||||
targetSdk 33 // 33 vermeidet die Foreground-Service-Typ-Pflicht von 34
|
targetSdk 33 // 33 vermeidet die Foreground-Service-Typ-Pflicht von 34
|
||||||
versionCode 8
|
versionCode 10
|
||||||
versionName '0.0.0.8'
|
versionName '0.0.1.0'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fester Signaturschlüssel: jeder Build signiert mit DEMSELBEN Key, damit
|
// Fester Signaturschlüssel: jeder Build signiert mit DEMSELBEN Key, damit
|
||||||
|
|||||||
@@ -11,6 +11,13 @@
|
|||||||
<uses-permission android:name="android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS" />
|
<uses-permission android:name="android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS" />
|
||||||
<!-- app_launch: Start-Intents/App-Labels sind ab Android 11 sonst unsichtbar. -->
|
<!-- app_launch: Start-Intents/App-Labels sind ab Android 11 sonst unsichtbar. -->
|
||||||
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
|
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
|
||||||
|
<!-- Overlay: ARIA blendet Text/Bild UEBER das laufende Programm ein (Fire TV).
|
||||||
|
Muss vom Nutzer einmalig erlaubt werden (Spezialberechtigung). -->
|
||||||
|
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
|
||||||
|
<!-- Sprache: Mikrofon (Fire-TV-Fernbedienung / USB-Mikro) fuer ARIA. -->
|
||||||
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||||
|
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
|
||||||
|
<uses-feature android:name="android.hardware.microphone" android:required="false" />
|
||||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||||
<!-- Fire TV / Android TV: dieselbe APK laeuft auf Stick UND Handy.
|
<!-- Fire TV / Android TV: dieselbe APK laeuft auf Stick UND Handy.
|
||||||
touchscreen/faketouch NICHT erforderlich (Fire TV hat keinen), und
|
touchscreen/faketouch NICHT erforderlich (Fire TV hat keinen), und
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ data class AgentConfig(
|
|||||||
var token: String = "",
|
var token: String = "",
|
||||||
var name: String = "",
|
var name: String = "",
|
||||||
var controlEnabled: Boolean = false,
|
var controlEnabled: Boolean = false,
|
||||||
|
/** Wie antwortet ARIA auf diesem Geraet standardmaessig: "overlay" oder "speech".
|
||||||
|
* Pro Anfrage ueberschreibbar ("...bitte als Sprache"). */
|
||||||
|
var replyMode: String = "overlay",
|
||||||
) {
|
) {
|
||||||
companion object {
|
companion object {
|
||||||
private const val PREFS = "aria_agent"
|
private const val PREFS = "aria_agent"
|
||||||
@@ -24,6 +27,7 @@ data class AgentConfig(
|
|||||||
token = p.getString("token", "") ?: "",
|
token = p.getString("token", "") ?: "",
|
||||||
name = p.getString("name", "") ?: "",
|
name = p.getString("name", "") ?: "",
|
||||||
controlEnabled = p.getBoolean("control", false),
|
controlEnabled = p.getBoolean("control", false),
|
||||||
|
replyMode = p.getString("replyMode", "overlay") ?: "overlay",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -36,6 +40,7 @@ data class AgentConfig(
|
|||||||
putString("token", token)
|
putString("token", token)
|
||||||
putString("name", name)
|
putString("name", name)
|
||||||
putBoolean("control", controlEnabled)
|
putBoolean("control", controlEnabled)
|
||||||
|
putString("replyMode", replyMode)
|
||||||
apply()
|
apply()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ class AgentService : Service() {
|
|||||||
private const val NOTIF_ID = 1
|
private const val NOTIF_ID = 1
|
||||||
const val ACTION_STATUS = "de.hackersoft.ariaagent.STATUS"
|
const val ACTION_STATUS = "de.hackersoft.ariaagent.STATUS"
|
||||||
const val ACTION_PROJECTION = "de.hackersoft.ariaagent.PROJECTION"
|
const val ACTION_PROJECTION = "de.hackersoft.ariaagent.PROJECTION"
|
||||||
|
const val ACTION_TALK = "de.hackersoft.ariaagent.TALK"
|
||||||
const val EXTRA_RESULT_CODE = "resultCode"
|
const val EXTRA_RESULT_CODE = "resultCode"
|
||||||
const val EXTRA_RESULT_DATA = "resultData"
|
const val EXTRA_RESULT_DATA = "resultData"
|
||||||
|
|
||||||
@@ -67,6 +68,13 @@ class AgentService : Service() {
|
|||||||
// zuerst startForeground aufrufen (idempotent).
|
// zuerst startForeground aufrufen (idempotent).
|
||||||
startForegroundDataSync(status)
|
startForegroundDataSync(status)
|
||||||
|
|
||||||
|
if (intent?.action == ACTION_TALK) {
|
||||||
|
// Sprech-Taste: laeuft gerade eine Aufnahme -> beenden, sonst starten.
|
||||||
|
val v = rvs?.voice
|
||||||
|
if (v == null) startRvs()
|
||||||
|
else if (v.isRecording) v.stopListening("user") else v.startListening()
|
||||||
|
return START_STICKY
|
||||||
|
}
|
||||||
if (intent?.action == ACTION_PROJECTION) {
|
if (intent?.action == ACTION_PROJECTION) {
|
||||||
val code = intent.getIntExtra(EXTRA_RESULT_CODE, 0)
|
val code = intent.getIntExtra(EXTRA_RESULT_CODE, 0)
|
||||||
@Suppress("DEPRECATION")
|
@Suppress("DEPRECATION")
|
||||||
@@ -119,6 +127,7 @@ class AgentService : Service() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun onDestroy() {
|
override fun onDestroy() {
|
||||||
|
try { rvs?.voice?.shutdown() } catch (_: Exception) {}
|
||||||
rvs?.stop()
|
rvs?.stop()
|
||||||
ScreenCapturer.stop()
|
ScreenCapturer.stop()
|
||||||
connected = false
|
connected = false
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import android.os.Bundle
|
|||||||
import android.provider.Settings
|
import android.provider.Settings
|
||||||
import android.widget.Button
|
import android.widget.Button
|
||||||
import android.widget.EditText
|
import android.widget.EditText
|
||||||
|
import android.widget.RadioButton
|
||||||
import android.widget.TextView
|
import android.widget.TextView
|
||||||
import android.widget.Toast
|
import android.widget.Toast
|
||||||
import androidx.activity.result.contract.ActivityResultContracts
|
import androidx.activity.result.contract.ActivityResultContracts
|
||||||
@@ -41,6 +42,12 @@ class MainActivity : AppCompatActivity() {
|
|||||||
}
|
}
|
||||||
private val notifPerm = registerForActivityResult(ActivityResultContracts.RequestPermission()) { }
|
private val notifPerm = registerForActivityResult(ActivityResultContracts.RequestPermission()) { }
|
||||||
|
|
||||||
|
private val micPerm = registerForActivityResult(
|
||||||
|
ActivityResultContracts.RequestPermission()
|
||||||
|
) { granted ->
|
||||||
|
if (granted) talkToAria() else toast("Ohne Mikrofon-Berechtigung kein Sprachdialog")
|
||||||
|
}
|
||||||
|
|
||||||
private val projection = registerForActivityResult(
|
private val projection = registerForActivityResult(
|
||||||
ActivityResultContracts.StartActivityForResult()
|
ActivityResultContracts.StartActivityForResult()
|
||||||
) { res ->
|
) { res ->
|
||||||
@@ -90,6 +97,22 @@ class MainActivity : AppCompatActivity() {
|
|||||||
val mpm = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
val mpm = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||||
projection.launch(mpm.createScreenCaptureIntent())
|
projection.launch(mpm.createScreenCaptureIntent())
|
||||||
}
|
}
|
||||||
|
findViewById<Button>(R.id.btnTalk).setOnClickListener {
|
||||||
|
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO)
|
||||||
|
== PackageManager.PERMISSION_GRANTED) talkToAria()
|
||||||
|
else micPerm.launch(Manifest.permission.RECORD_AUDIO)
|
||||||
|
}
|
||||||
|
findViewById<Button>(R.id.btnOverlayPerm).setOnClickListener {
|
||||||
|
if (OverlayView.canDraw(this)) { toast("Einblendung ist bereits erlaubt"); return@setOnClickListener }
|
||||||
|
try {
|
||||||
|
startActivity(Intent(Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
|
||||||
|
android.net.Uri.parse("package:$packageName")))
|
||||||
|
toast("'ARIA Host-Agent' erlauben")
|
||||||
|
} catch (_: Exception) {
|
||||||
|
try { startActivity(Intent(Settings.ACTION_MANAGE_OVERLAY_PERMISSION)) }
|
||||||
|
catch (_: Exception) { toast("Einstellung nicht gefunden") }
|
||||||
|
}
|
||||||
|
}
|
||||||
findViewById<Button>(R.id.btnAccessibility).setOnClickListener {
|
findViewById<Button>(R.id.btnAccessibility).setOnClickListener {
|
||||||
try {
|
try {
|
||||||
startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS))
|
startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS))
|
||||||
@@ -156,6 +179,9 @@ class MainActivity : AppCompatActivity() {
|
|||||||
name.setText(c.name)
|
name.setText(c.name)
|
||||||
tls.isChecked = c.tls
|
tls.isChecked = c.tls
|
||||||
control.isChecked = c.controlEnabled
|
control.isChecked = c.controlEnabled
|
||||||
|
findViewById<RadioButton>(
|
||||||
|
if (c.replyMode == "speech") R.id.modeSpeech else R.id.modeOverlay
|
||||||
|
).isChecked = true
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun saveAndConnect() {
|
private fun saveAndConnect() {
|
||||||
@@ -166,6 +192,7 @@ class MainActivity : AppCompatActivity() {
|
|||||||
token = token.text.toString().trim(),
|
token = token.text.toString().trim(),
|
||||||
name = name.text.toString().trim(),
|
name = name.text.toString().trim(),
|
||||||
controlEnabled = control.isChecked,
|
controlEnabled = control.isChecked,
|
||||||
|
replyMode = if (findViewById<RadioButton>(R.id.modeSpeech).isChecked) "speech" else "overlay",
|
||||||
)
|
)
|
||||||
if (!c.isValid()) {
|
if (!c.isValid()) {
|
||||||
toast("Host und Token sind Pflicht")
|
toast("Host und Token sind Pflicht")
|
||||||
@@ -182,6 +209,14 @@ class MainActivity : AppCompatActivity() {
|
|||||||
statusView.text = "Status: ${AgentService.status} $dot"
|
statusView.text = "Status: ${AgentService.status} $dot"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Sprech-Taste: startet/stoppt die Aufnahme im laufenden Dienst. */
|
||||||
|
private fun talkToAria() {
|
||||||
|
if (!AgentConfig.load(this).isValid()) { toast("Erst verbinden"); return }
|
||||||
|
ContextCompat.startForegroundService(this,
|
||||||
|
Intent(this, AgentService::class.java).setAction(AgentService.ACTION_TALK))
|
||||||
|
toast("Sprich jetzt — nochmal drücken beendet die Aufnahme")
|
||||||
|
}
|
||||||
|
|
||||||
/** Fire TV / Android TV (kein Touchscreen, Bedienung per Fernbedienung). */
|
/** Fire TV / Android TV (kein Touchscreen, Bedienung per Fernbedienung). */
|
||||||
private fun isTvDevice(): Boolean = try {
|
private fun isTvDevice(): Boolean = try {
|
||||||
packageManager.hasSystemFeature("android.software.leanback") ||
|
packageManager.hasSystemFeature("android.software.leanback") ||
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package de.hackersoft.ariaagent
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.graphics.Color
|
||||||
|
import android.graphics.drawable.GradientDrawable
|
||||||
|
import android.os.Build
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.Looper
|
||||||
|
import android.provider.Settings
|
||||||
|
import android.util.Base64
|
||||||
|
import android.view.Gravity
|
||||||
|
import android.view.View
|
||||||
|
import android.view.WindowManager
|
||||||
|
import android.widget.ImageView
|
||||||
|
import android.widget.LinearLayout
|
||||||
|
import android.widget.TextView
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Einblendung ueber dem laufenden Bild (Fire TV / Android TV).
|
||||||
|
*
|
||||||
|
* ARIA kann Text (und optional ein Bild) UEBER Netflix & Co. anzeigen, ohne die
|
||||||
|
* Wiedergabe zu stoppen — fuer Antworten, Hinweise, Tuerklingel-Schnappschuesse.
|
||||||
|
* Technisch ein TYPE_APPLICATION_OVERLAY-Fenster; der Nutzer muss die
|
||||||
|
* Spezialberechtigung "Ueber anderen Apps anzeigen" einmalig erteilen.
|
||||||
|
*
|
||||||
|
* Nicht fokussierbar (FLAG_NOT_FOCUSABLE) — die Fernbedienung steuert weiter die
|
||||||
|
* App darunter, die Einblendung schluckt keine Tastendruecke.
|
||||||
|
*/
|
||||||
|
object OverlayView {
|
||||||
|
|
||||||
|
private val main = Handler(Looper.getMainLooper())
|
||||||
|
private var root: LinearLayout? = null
|
||||||
|
private var hideRunnable: Runnable? = null
|
||||||
|
|
||||||
|
/** Hat der Nutzer "Ueber anderen Apps anzeigen" erlaubt? */
|
||||||
|
fun canDraw(ctx: Context): Boolean =
|
||||||
|
if (Build.VERSION.SDK_INT >= 23) Settings.canDrawOverlays(ctx) else true
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zeigt eine Einblendung. [seconds] = Anzeigedauer (0 = bis zum naechsten
|
||||||
|
* hide()). [imageB64] optional ein PNG/JPEG als Base64.
|
||||||
|
* Muss NICHT vom Main-Thread gerufen werden.
|
||||||
|
*/
|
||||||
|
fun show(ctx: Context, title: String, text: String, seconds: Int, imageB64: String?) {
|
||||||
|
main.post {
|
||||||
|
try {
|
||||||
|
hideNow(ctx)
|
||||||
|
val wm = ctx.getSystemService(Context.WINDOW_SERVICE) as WindowManager
|
||||||
|
|
||||||
|
val box = LinearLayout(ctx).apply {
|
||||||
|
orientation = LinearLayout.VERTICAL
|
||||||
|
setPadding(36, 28, 36, 28)
|
||||||
|
background = GradientDrawable().apply {
|
||||||
|
cornerRadius = 22f
|
||||||
|
setColor(Color.parseColor("#E60D0D1A")) // dunkel, leicht transparent
|
||||||
|
setStroke(3, Color.parseColor("#3FFF9F"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (title.isNotBlank()) {
|
||||||
|
box.addView(TextView(ctx).apply {
|
||||||
|
setText(title)
|
||||||
|
setTextColor(Color.parseColor("#3FFF9F"))
|
||||||
|
textSize = 22f
|
||||||
|
setPadding(0, 0, 0, 10)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (text.isNotBlank()) {
|
||||||
|
box.addView(TextView(ctx).apply {
|
||||||
|
setText(text)
|
||||||
|
setTextColor(Color.parseColor("#E0E0F0"))
|
||||||
|
textSize = 19f
|
||||||
|
maxLines = 8
|
||||||
|
})
|
||||||
|
}
|
||||||
|
imageB64?.let { b64 ->
|
||||||
|
try {
|
||||||
|
val bytes = Base64.decode(b64, Base64.DEFAULT)
|
||||||
|
val bmp = android.graphics.BitmapFactory
|
||||||
|
.decodeByteArray(bytes, 0, bytes.size)
|
||||||
|
if (bmp != null) {
|
||||||
|
box.addView(ImageView(ctx).apply {
|
||||||
|
setImageBitmap(bmp)
|
||||||
|
adjustViewBounds = true
|
||||||
|
maxHeight = 520
|
||||||
|
setPadding(0, 14, 0, 0)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (_: Exception) { /* Bild optional */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
val type = if (Build.VERSION.SDK_INT >= 26)
|
||||||
|
WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY
|
||||||
|
else
|
||||||
|
@Suppress("DEPRECATION") WindowManager.LayoutParams.TYPE_SYSTEM_ALERT
|
||||||
|
val lp = WindowManager.LayoutParams(
|
||||||
|
WindowManager.LayoutParams.WRAP_CONTENT,
|
||||||
|
WindowManager.LayoutParams.WRAP_CONTENT,
|
||||||
|
type,
|
||||||
|
WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE or
|
||||||
|
WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE or
|
||||||
|
WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN,
|
||||||
|
android.graphics.PixelFormat.TRANSLUCENT,
|
||||||
|
).apply {
|
||||||
|
gravity = Gravity.TOP or Gravity.END
|
||||||
|
x = 48
|
||||||
|
y = 48
|
||||||
|
width = (ctx.resources.displayMetrics.widthPixels * 0.42f).toInt()
|
||||||
|
}
|
||||||
|
|
||||||
|
wm.addView(box, lp)
|
||||||
|
root = box
|
||||||
|
|
||||||
|
if (seconds > 0) {
|
||||||
|
hideRunnable = Runnable { hideNow(ctx) }
|
||||||
|
main.postDelayed(hideRunnable!!, seconds * 1000L)
|
||||||
|
}
|
||||||
|
} catch (_: Exception) {
|
||||||
|
root = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Blendet eine laufende Einblendung aus. */
|
||||||
|
fun hide(ctx: Context) {
|
||||||
|
main.post { hideNow(ctx) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun hideNow(ctx: Context) {
|
||||||
|
hideRunnable?.let { main.removeCallbacks(it) }
|
||||||
|
hideRunnable = null
|
||||||
|
val v: View? = root
|
||||||
|
root = null
|
||||||
|
if (v != null) {
|
||||||
|
try {
|
||||||
|
(ctx.getSystemService(Context.WINDOW_SERVICE) as WindowManager).removeView(v)
|
||||||
|
} catch (_: Exception) { /* schon weg */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,13 @@ class RvsClient(
|
|||||||
@Volatile private var running = false
|
@Volatile private var running = false
|
||||||
private var pingThread: Thread? = null
|
private var pingThread: Thread? = null
|
||||||
|
|
||||||
|
/** Sprachdialog mit ARIA (Mikro rein, Sprache/Einblendung raus). */
|
||||||
|
val voice: VoiceSession by lazy {
|
||||||
|
VoiceSession(appCtx, config,
|
||||||
|
send = { t, p -> ws?.let { send(it, t, p) } },
|
||||||
|
onEvent = { msg -> onStatus(true, msg) })
|
||||||
|
}
|
||||||
|
|
||||||
/** Fire TV / Android TV? Dann gibt es keinen Touchscreen, dafuer D-Pad. */
|
/** Fire TV / Android TV? Dann gibt es keinen Touchscreen, dafuer D-Pad. */
|
||||||
private val isTv: Boolean by lazy {
|
private val isTv: Boolean by lazy {
|
||||||
try {
|
try {
|
||||||
@@ -46,7 +53,11 @@ class RvsClient(
|
|||||||
|
|
||||||
private val caps: List<String> by lazy {
|
private val caps: List<String> by lazy {
|
||||||
val base = mutableListOf("info", "screenshot", "ui_dump",
|
val base = mutableListOf("info", "screenshot", "ui_dump",
|
||||||
"ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch")
|
"ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch",
|
||||||
|
// Ausgabe auf dem Geraet: Einblendung ueber dem Bild + Sprachausgabe.
|
||||||
|
"notify", "speak", "volume",
|
||||||
|
// Sprachdialog: Mikro oeffnen/schliessen (ARIA kann es auch anstossen).
|
||||||
|
"listen", "listen_stop")
|
||||||
if (isTv) base += "tv" // Kennzeichen: dieses Geraet ist ein Fernseher/Stick
|
if (isTv) base += "tv" // Kennzeichen: dieses Geraet ist ein Fernseher/Stick
|
||||||
base
|
base
|
||||||
}
|
}
|
||||||
@@ -134,7 +145,17 @@ class RvsClient(
|
|||||||
|
|
||||||
private fun handle(webSocket: WebSocket, text: String) {
|
private fun handle(webSocket: WebSocket, text: String) {
|
||||||
val msg = try { JSONObject(text) } catch (_: Exception) { return }
|
val msg = try { JSONObject(text) } catch (_: Exception) { return }
|
||||||
if (msg.optString("type") != "host_command") return
|
val type = msg.optString("type")
|
||||||
|
// Sprachdialog: chat/audio_pcm/stt_* laufen ueber dieselbe Verbindung.
|
||||||
|
// (Der RVS broadcastet alles im Raum — VoiceSession filtert auf den
|
||||||
|
// eigenen Turn, damit nicht auch Handy und Stick gleichzeitig antworten.)
|
||||||
|
if (type != "host_command") {
|
||||||
|
if (type in VOICE_TYPES) {
|
||||||
|
try { voice.handle(type, msg.optJSONObject("payload") ?: JSONObject()) }
|
||||||
|
catch (_: Throwable) {}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
val payload = msg.optJSONObject("payload") ?: JSONObject()
|
val payload = msg.optJSONObject("payload") ?: JSONObject()
|
||||||
|
|
||||||
val target = payload.optString("host").ifBlank { payload.optString("hostId") }
|
val target = payload.optString("host").ifBlank { payload.optString("hostId") }
|
||||||
@@ -164,7 +185,12 @@ class RvsClient(
|
|||||||
action == "ui_swipe" -> doUiSwipe(params)
|
action == "ui_swipe" -> doUiSwipe(params)
|
||||||
action == "ui_key" -> doUiKey(params)
|
action == "ui_key" -> doUiKey(params)
|
||||||
action == "app_launch" -> doAppLaunch(params)
|
action == "app_launch" -> doAppLaunch(params)
|
||||||
action in listOf("app_list", "notify") ->
|
action == "notify" -> doNotify(params)
|
||||||
|
action == "speak" -> doSpeak(params)
|
||||||
|
action == "volume" -> doVolume(params)
|
||||||
|
action == "listen" -> voice.startListening()
|
||||||
|
action == "listen_stop" -> voice.stopListening("remote")
|
||||||
|
action == "app_list" ->
|
||||||
err("Aktion '$action' kommt spaeter (noch nicht implementiert).")
|
err("Aktion '$action' kommt spaeter (noch nicht implementiert).")
|
||||||
else -> err("Aktion '$action' unbekannt.")
|
else -> err("Aktion '$action' unbekannt.")
|
||||||
}
|
}
|
||||||
@@ -180,6 +206,10 @@ class RvsClient(
|
|||||||
/** Steuer-Aktionen, die ein entsperrtes Gerät brauchen. */
|
/** Steuer-Aktionen, die ein entsperrtes Gerät brauchen. */
|
||||||
private val CONTROL_ACTIONS = setOf("ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch")
|
private val CONTROL_ACTIONS = setOf("ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch")
|
||||||
|
|
||||||
|
/** Message-Typen des ARIA-Sprachdialogs (kein host_command). */
|
||||||
|
private val VOICE_TYPES = setOf(
|
||||||
|
"chat", "audio_pcm", "stt_endpoint", "stt_stream_done", "stt_partial")
|
||||||
|
|
||||||
private fun isLocked(): Boolean = try {
|
private fun isLocked(): Boolean = try {
|
||||||
val km = appCtx.getSystemService(Context.KEYGUARD_SERVICE) as android.app.KeyguardManager
|
val km = appCtx.getSystemService(Context.KEYGUARD_SERVICE) as android.app.KeyguardManager
|
||||||
km.isKeyguardLocked
|
km.isKeyguardLocked
|
||||||
@@ -260,6 +290,68 @@ class RvsClient(
|
|||||||
.put("result", JSONObject().put("message", "App '$pkg' gestartet"))
|
.put("result", JSONObject().put("message", "App '$pkg' gestartet"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ARIAs Antwort auf dem Geraet ausgeben: Einblendung (Overlay) und/oder Sprache.
|
||||||
|
* `mode` optional ("overlay" | "speech" | "both") — ohne Angabe gilt die
|
||||||
|
* Einstellung aus der App (config.replyMode). So kann ARIA pro Anfrage
|
||||||
|
* ueberschreiben ("...bitte als Sprache"), ohne die Voreinstellung zu aendern.
|
||||||
|
*/
|
||||||
|
private fun doNotify(p: JSONObject): JSONObject {
|
||||||
|
val text = p.optString("text").ifBlank { p.optString("message") }
|
||||||
|
val title = p.optString("title", "ARIA")
|
||||||
|
if (text.isBlank() && !p.has("image_base64")) return err("notify braucht 'text'.")
|
||||||
|
val mode = p.optString("mode").ifBlank { config.replyMode }
|
||||||
|
val done = mutableListOf<String>()
|
||||||
|
val problems = mutableListOf<String>()
|
||||||
|
|
||||||
|
if (mode == "overlay" || mode == "both") {
|
||||||
|
if (!OverlayView.canDraw(appCtx)) {
|
||||||
|
problems += "Einblendung nicht erlaubt — in der Agent-App auf dem Gerät " +
|
||||||
|
"'Einblendung erlauben' antippen (Berechtigung 'Über anderen Apps anzeigen')"
|
||||||
|
} else {
|
||||||
|
OverlayView.show(appCtx, title, text,
|
||||||
|
p.optInt("seconds", 12),
|
||||||
|
if (p.has("image_base64")) p.optString("image_base64") else null)
|
||||||
|
done += "eingeblendet"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (mode == "speech" || mode == "both") {
|
||||||
|
val r = speakOut(p, text)
|
||||||
|
r.exceptionOrNull()?.let { problems += "Sprachausgabe: ${it.message}" }
|
||||||
|
?: run { done += "gesprochen" }
|
||||||
|
}
|
||||||
|
if (done.isEmpty()) return err(problems.joinToString("; ").ifBlank { "nichts ausgegeben" })
|
||||||
|
val msg = done.joinToString(" + ") + (if (problems.isEmpty()) "" else " (${problems.joinToString("; ")})")
|
||||||
|
return JSONObject().put("ok", true).put("result", JSONObject().put("message", msg))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Nur Sprachausgabe (ARIAs Audio, falls mitgeliefert — sonst Geraete-TTS). */
|
||||||
|
private fun doSpeak(p: JSONObject): JSONObject {
|
||||||
|
val text = p.optString("text")
|
||||||
|
if (text.isBlank() && !p.has("audio_base64")) return err("speak braucht 'text' oder 'audio_base64'.")
|
||||||
|
val r = speakOut(p, text)
|
||||||
|
return r.fold(
|
||||||
|
{ JSONObject().put("ok", true).put("result", JSONObject().put("message", "gesprochen")) },
|
||||||
|
{ err("Sprachausgabe fehlgeschlagen: ${it.message}") },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Audio bevorzugt (ARIAs echte Stimme), sonst Geraete-TTS. */
|
||||||
|
private fun speakOut(p: JSONObject, text: String): Result<Unit> {
|
||||||
|
val audio = p.optString("audio_base64")
|
||||||
|
return if (audio.isNotBlank())
|
||||||
|
Speaker.play(appCtx, audio, p.optString("audio_format", "wav"))
|
||||||
|
else Speaker.speak(appCtx, text)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun doVolume(p: JSONObject): JSONObject {
|
||||||
|
if (!p.has("percent")) return err("volume braucht 'percent' (0-100).")
|
||||||
|
val v = p.optInt("percent")
|
||||||
|
Speaker.setVolume(appCtx, v)
|
||||||
|
return JSONObject().put("ok", true)
|
||||||
|
.put("result", JSONObject().put("message", "Lautstärke auf $v% gesetzt"))
|
||||||
|
}
|
||||||
|
|
||||||
/** Paketname per Label-Teilstring finden (case-insensitive). */
|
/** Paketname per Label-Teilstring finden (case-insensitive). */
|
||||||
private fun resolvePackage(query: String): String? {
|
private fun resolvePackage(query: String): String? {
|
||||||
val pm = appCtx.packageManager
|
val pm = appCtx.packageManager
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
package de.hackersoft.ariaagent
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.media.AudioAttributes
|
||||||
|
import android.media.AudioManager
|
||||||
|
import android.media.MediaPlayer
|
||||||
|
import android.speech.tts.TextToSpeech
|
||||||
|
import android.util.Base64
|
||||||
|
import java.io.File
|
||||||
|
import java.util.Locale
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sprachausgabe auf dem Geraet (v.a. Fire TV: ARIA spricht ueber die TV-Lautsprecher).
|
||||||
|
*
|
||||||
|
* Zwei Wege:
|
||||||
|
* 1) `play(audioBase64)` — ARIAs ECHTE Stimme: fertiges Audio (WAV/MP3) aus dem
|
||||||
|
* bestehenden TTS-Pfad (F5/XTTS) wird einfach abgespielt. Bevorzugt.
|
||||||
|
* 2) `speak(text)` — geraeteeigene TextToSpeech als Rueckfallebene, falls kein
|
||||||
|
* Audio mitgeliefert wurde (klingt nach Standard-Stimme, nicht nach ARIA).
|
||||||
|
*/
|
||||||
|
object Speaker {
|
||||||
|
|
||||||
|
private var tts: TextToSpeech? = null
|
||||||
|
private var ttsReady = false
|
||||||
|
private var player: MediaPlayer? = null
|
||||||
|
|
||||||
|
/** Spielt fertiges Audio (Base64, z.B. WAV aus dem ARIA-TTS) ueber die Lautsprecher. */
|
||||||
|
fun play(ctx: Context, audioB64: String, format: String = "wav"): Result<Unit> = runCatching {
|
||||||
|
stop()
|
||||||
|
val bytes = Base64.decode(audioB64, Base64.DEFAULT)
|
||||||
|
val f = File(ctx.cacheDir, "aria_speak.${format.ifBlank { "wav" }}")
|
||||||
|
f.writeBytes(bytes)
|
||||||
|
player = MediaPlayer().apply {
|
||||||
|
setAudioAttributes(
|
||||||
|
AudioAttributes.Builder()
|
||||||
|
.setUsage(AudioAttributes.USAGE_MEDIA)
|
||||||
|
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
setDataSource(f.absolutePath)
|
||||||
|
setOnCompletionListener { mp -> try { mp.release() } catch (_: Exception) {}; player = null }
|
||||||
|
prepare()
|
||||||
|
start()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rueckfallebene: geraeteeigene Sprachausgabe. */
|
||||||
|
fun speak(ctx: Context, text: String): Result<Unit> = runCatching {
|
||||||
|
if (tts == null) {
|
||||||
|
tts = TextToSpeech(ctx.applicationContext) { status ->
|
||||||
|
ttsReady = status == TextToSpeech.SUCCESS
|
||||||
|
if (ttsReady) tts?.language = Locale.GERMAN
|
||||||
|
}
|
||||||
|
// Kurz auf die Initialisierung warten (TTS meldet sich asynchron).
|
||||||
|
var waited = 0
|
||||||
|
while (!ttsReady && waited < 3000) { Thread.sleep(100); waited += 100 }
|
||||||
|
}
|
||||||
|
if (!ttsReady) throw IllegalStateException("Keine Sprachausgabe verfuegbar (TTS-Engine fehlt)")
|
||||||
|
tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, "aria")
|
||||||
|
Unit
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Lautstaerke 0..100 (Medien-Kanal). */
|
||||||
|
fun setVolume(ctx: Context, percent: Int) {
|
||||||
|
val am = ctx.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||||
|
val max = am.getStreamMaxVolume(AudioManager.STREAM_MUSIC)
|
||||||
|
val v = (max * percent.coerceIn(0, 100) / 100.0).toInt()
|
||||||
|
am.setStreamVolume(AudioManager.STREAM_MUSIC, v, AudioManager.FLAG_SHOW_UI)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun stop() {
|
||||||
|
try { player?.stop(); player?.release() } catch (_: Exception) {}
|
||||||
|
player = null
|
||||||
|
try { tts?.stop() } catch (_: Exception) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
package de.hackersoft.ariaagent
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.media.AudioAttributes
|
||||||
|
import android.media.AudioFormat
|
||||||
|
import android.media.AudioManager
|
||||||
|
import android.media.AudioRecord
|
||||||
|
import android.media.AudioTrack
|
||||||
|
import android.media.MediaRecorder
|
||||||
|
import android.media.audiofx.AcousticEchoCanceler
|
||||||
|
import android.media.audiofx.NoiseSuppressor
|
||||||
|
import android.util.Base64
|
||||||
|
import okhttp3.WebSocket
|
||||||
|
import org.json.JSONObject
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sprachdialog mit ARIA direkt auf dem Geraet (v.a. Fire TV: reden ueber die
|
||||||
|
* Fernbedienung, Antwort ueber die TV-Lautsprecher bzw. als Einblendung).
|
||||||
|
*
|
||||||
|
* Nutzt GENAU das Protokoll der ARIA-App, damit serverseitig nichts angepasst
|
||||||
|
* werden muss:
|
||||||
|
* rein : stt_stream_start -> stt_audio_chunk (Feld 'pcm', 16 kHz mono s16le,
|
||||||
|
* 6400-Byte-Bloecke) -> stt_stream_end
|
||||||
|
* raus : chat (sender=aria, messageId/speak/converse) + audio_pcm (Feld
|
||||||
|
* 'base64', 24 kHz mono s16le, chunkweise bis final=true)
|
||||||
|
*
|
||||||
|
* WICHTIG — Turn-Ownership: Der RVS ist ein Broadcast-Relay ohne Adressierung.
|
||||||
|
* Jeder Client im Raum bekommt JEDE Antwort. Ohne Filter wuerden Handy-App und
|
||||||
|
* Fire TV dieselbe Antwort gleichzeitig vorlesen. Deshalb gibt dieses Geraet nur
|
||||||
|
* aus, was aus SEINER eigenen Frage entstanden ist (audioRequestId -> messageId).
|
||||||
|
*/
|
||||||
|
class VoiceSession(
|
||||||
|
private val appCtx: Context,
|
||||||
|
private val config: AgentConfig,
|
||||||
|
private val send: (type: String, payload: JSONObject) -> Unit,
|
||||||
|
private val onEvent: (String) -> Unit = {},
|
||||||
|
) {
|
||||||
|
companion object {
|
||||||
|
// Aufnahme — muss exakt zur STT-Bridge passen.
|
||||||
|
private const val IN_RATE = 16000
|
||||||
|
private const val CHUNK_BYTES = 6400 // 3200 Samples = 200 ms
|
||||||
|
// Wiedergabe — F5-TTS liefert 24 kHz (sampleRate steht aber im Payload).
|
||||||
|
private const val OUT_RATE_DEFAULT = 24000
|
||||||
|
private const val MAX_RECORD_MS = 60000
|
||||||
|
}
|
||||||
|
|
||||||
|
@Volatile private var recording = false
|
||||||
|
@Volatile private var recordThread: Thread? = null
|
||||||
|
private var rec: AudioRecord? = null
|
||||||
|
private var aec: AcousticEchoCanceler? = null
|
||||||
|
private var ns: NoiseSuppressor? = null
|
||||||
|
|
||||||
|
private var track: AudioTrack? = null
|
||||||
|
@Volatile private var playingMessageId: String? = null
|
||||||
|
|
||||||
|
/** Eigener Turn: gesetzt beim Aufnehmen, wandert ueber stt_endpoint zur messageId. */
|
||||||
|
@Volatile private var myAudioRequestId: String? = null
|
||||||
|
@Volatile private var myMessageId: String? = null
|
||||||
|
@Volatile private var sessionId: String? = null
|
||||||
|
/** Text der letzten ARIA-Antwort auf UNSERE Frage (fuer die Einblendung). */
|
||||||
|
@Volatile private var lastReplyText: String = ""
|
||||||
|
|
||||||
|
val isRecording: Boolean get() = recording
|
||||||
|
|
||||||
|
// ── Aufnehmen + streamen ────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Startet die Aufnahme und streamt live an die STT. */
|
||||||
|
fun startListening(): JSONObject {
|
||||||
|
if (recording) return JSONObject().put("ok", false).put("error", "Nimmt bereits auf.")
|
||||||
|
val stamp = System.currentTimeMillis()
|
||||||
|
val rnd = Random.nextInt(100000, 999999)
|
||||||
|
val sid = "sttstr_${stamp}_$rnd"
|
||||||
|
val aid = "audio_${stamp}_$rnd"
|
||||||
|
sessionId = sid
|
||||||
|
myAudioRequestId = aid
|
||||||
|
myMessageId = null
|
||||||
|
|
||||||
|
val minBuf = AudioRecord.getMinBufferSize(
|
||||||
|
IN_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
||||||
|
if (minBuf <= 0) return JSONObject().put("ok", false)
|
||||||
|
.put("error", "Kein Mikrofon verfuegbar (Fire-TV-Fernbedienung aktiv? USB-Mikro?)")
|
||||||
|
|
||||||
|
val r = try {
|
||||||
|
AudioRecord(MediaRecorder.AudioSource.VOICE_COMMUNICATION, IN_RATE,
|
||||||
|
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT,
|
||||||
|
maxOf(minBuf, CHUNK_BYTES * 4))
|
||||||
|
} catch (e: Throwable) {
|
||||||
|
return JSONObject().put("ok", false)
|
||||||
|
.put("error", "Mikrofon-Zugriff fehlgeschlagen: ${e.message}")
|
||||||
|
}
|
||||||
|
if (r.state != AudioRecord.STATE_INITIALIZED) {
|
||||||
|
try { r.release() } catch (_: Exception) {}
|
||||||
|
return JSONObject().put("ok", false)
|
||||||
|
.put("error", "Mikrofon nicht initialisiert (Berechtigung erteilt? Mikro vorhanden?)")
|
||||||
|
}
|
||||||
|
// Echo/Rauschen daempfen, damit der Stick nicht seine eigene Ausgabe hoert.
|
||||||
|
try { if (AcousticEchoCanceler.isAvailable()) aec = AcousticEchoCanceler.create(r.audioSessionId)?.apply { enabled = true } } catch (_: Exception) {}
|
||||||
|
try { if (NoiseSuppressor.isAvailable()) ns = NoiseSuppressor.create(r.audioSessionId)?.apply { enabled = true } } catch (_: Exception) {}
|
||||||
|
|
||||||
|
rec = r
|
||||||
|
recording = true
|
||||||
|
|
||||||
|
send("stt_stream_start", JSONObject().apply {
|
||||||
|
put("requestId", sid)
|
||||||
|
put("audioRequestId", aid)
|
||||||
|
put("voice", "")
|
||||||
|
put("speed", 1.0)
|
||||||
|
put("interrupted", false)
|
||||||
|
put("endpointMs", 2400)
|
||||||
|
put("hardCapMs", MAX_RECORD_MS)
|
||||||
|
put("sampleRate", IN_RATE)
|
||||||
|
put("projectId", "")
|
||||||
|
put("targetInstance", "")
|
||||||
|
put("language", "de")
|
||||||
|
})
|
||||||
|
onEvent("hoert zu …")
|
||||||
|
|
||||||
|
recordThread = Thread {
|
||||||
|
val buf = ByteArray(CHUNK_BYTES)
|
||||||
|
var seq = 0
|
||||||
|
try {
|
||||||
|
r.startRecording()
|
||||||
|
val until = System.currentTimeMillis() + MAX_RECORD_MS
|
||||||
|
while (recording && System.currentTimeMillis() < until) {
|
||||||
|
val n = r.read(buf, 0, buf.size)
|
||||||
|
if (n <= 0) continue
|
||||||
|
val slice = if (n == buf.size) buf else buf.copyOf(n)
|
||||||
|
send("stt_audio_chunk", JSONObject().apply {
|
||||||
|
put("requestId", sid)
|
||||||
|
put("pcm", Base64.encodeToString(slice, Base64.NO_WRAP))
|
||||||
|
put("seq", seq++)
|
||||||
|
put("targetInstance", "")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
// Aufnahme abgebrochen — stopListening raeumt auf.
|
||||||
|
} finally {
|
||||||
|
stopListening("auto")
|
||||||
|
}
|
||||||
|
}.also { it.isDaemon = true; it.start() }
|
||||||
|
|
||||||
|
return JSONObject().put("ok", true)
|
||||||
|
.put("result", JSONObject().put("message", "Hoere zu …"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Beendet die Aufnahme (die STT antwortet dann per stt_endpoint). */
|
||||||
|
fun stopListening(reason: String = "user"): JSONObject {
|
||||||
|
if (!recording) return JSONObject().put("ok", true)
|
||||||
|
.put("result", JSONObject().put("message", "Nahm nicht auf"))
|
||||||
|
recording = false
|
||||||
|
try { rec?.stop() } catch (_: Exception) {}
|
||||||
|
try { rec?.release() } catch (_: Exception) {}
|
||||||
|
try { aec?.release() } catch (_: Exception) {}
|
||||||
|
try { ns?.release() } catch (_: Exception) {}
|
||||||
|
rec = null; aec = null; ns = null
|
||||||
|
sessionId?.let {
|
||||||
|
send("stt_stream_end", JSONObject().apply {
|
||||||
|
put("requestId", it); put("reason", reason); put("targetInstance", "")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
onEvent("verarbeite …")
|
||||||
|
return JSONObject().put("ok", true)
|
||||||
|
.put("result", JSONObject().put("message", "Aufnahme beendet"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Text direkt an ARIA schicken (ohne Mikrofon) — Antwort landet hier genauso. */
|
||||||
|
fun ask(text: String): JSONObject {
|
||||||
|
if (text.isBlank()) return JSONObject().put("ok", false).put("error", "leerer Text")
|
||||||
|
val stamp = System.currentTimeMillis()
|
||||||
|
myAudioRequestId = "audio_${stamp}_${Random.nextInt(100000, 999999)}"
|
||||||
|
myMessageId = null
|
||||||
|
send("chat", JSONObject().apply {
|
||||||
|
put("text", text)
|
||||||
|
put("voice", ""); put("speed", 1.0); put("projectId", "")
|
||||||
|
put("clientMsgId", "cmsg_${stamp}_${Random.nextInt(100000, 999999)}")
|
||||||
|
})
|
||||||
|
return JSONObject().put("ok", true)
|
||||||
|
.put("result", JSONObject().put("message", "Frage gesendet"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Eingehende Nachrichten ──────────────────────────────────────
|
||||||
|
|
||||||
|
/** Verarbeitet Sprach-/Chat-Nachrichten. true = war fuer uns relevant. */
|
||||||
|
fun handle(type: String, p: JSONObject): Boolean = when (type) {
|
||||||
|
"stt_endpoint", "stt_stream_done" -> {
|
||||||
|
if (p.optString("requestId") == sessionId) {
|
||||||
|
val text = p.optString("text")
|
||||||
|
if (type == "stt_endpoint" && text.isNotBlank()) onEvent("verstanden: $text")
|
||||||
|
if (type == "stt_stream_done") { recording = false; sessionId = null }
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
"chat" -> {
|
||||||
|
if (p.optString("sender") == "aria" && isMyTurn(p)) {
|
||||||
|
myMessageId = p.optString("messageId")
|
||||||
|
lastReplyText = p.optString("text")
|
||||||
|
deliverReply(p)
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
"audio_pcm" -> {
|
||||||
|
// Nur abspielen, wenn es die Antwort auf UNSERE Frage ist und
|
||||||
|
// Sprachausgabe gewuenscht war (sonst redet auch das Handy mit).
|
||||||
|
val mid = p.optString("messageId")
|
||||||
|
if (mid.isNotBlank() && mid == myMessageId && wantSpeech()) feedAudio(p)
|
||||||
|
true
|
||||||
|
}
|
||||||
|
else -> false
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Gehoert diese Antwort zu unserer Frage? */
|
||||||
|
private fun isMyTurn(p: JSONObject): Boolean {
|
||||||
|
val aid = myAudioRequestId ?: return false
|
||||||
|
val pa = p.optString("audioRequestId")
|
||||||
|
// Sprach-Turn: audioRequestId wird durchgereicht. Text-Turn (ask): die
|
||||||
|
// naechste ARIA-Antwort nach unserer Frage gehoert uns.
|
||||||
|
return if (pa.isNotBlank()) pa == aid else myMessageId == null
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun wantSpeech(): Boolean = config.replyMode == "speech"
|
||||||
|
|
||||||
|
/** Antwort ausgeben: Einblendung und/oder Sprache (je nach Einstellung). */
|
||||||
|
private fun deliverReply(p: JSONObject) {
|
||||||
|
val text = p.optString("text")
|
||||||
|
if (text.isBlank()) return
|
||||||
|
if (config.replyMode == "overlay" && OverlayView.canDraw(appCtx)) {
|
||||||
|
OverlayView.show(appCtx, "ARIA", text, 15, null)
|
||||||
|
} else if (config.replyMode == "overlay") {
|
||||||
|
onEvent("Antwort (Einblendung nicht erlaubt): $text")
|
||||||
|
}
|
||||||
|
// Bei "speech" kommt das Audio ueber audio_pcm; falls ARIA nicht spricht
|
||||||
|
// (speak=false), trotzdem einblenden, damit nichts verloren geht.
|
||||||
|
if (wantSpeech() && !p.optBoolean("speak", true) && OverlayView.canDraw(appCtx)) {
|
||||||
|
OverlayView.show(appCtx, "ARIA", text, 15, null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── TTS-Wiedergabe (chunkweises PCM) ────────────────────────────
|
||||||
|
|
||||||
|
private fun feedAudio(p: JSONObject) {
|
||||||
|
val mid = p.optString("messageId")
|
||||||
|
val rate = p.optInt("sampleRate", OUT_RATE_DEFAULT)
|
||||||
|
if (p.optBoolean("final", false)) {
|
||||||
|
try { track?.stop() } catch (_: Exception) {}
|
||||||
|
try { track?.release() } catch (_: Exception) {}
|
||||||
|
track = null; playingMessageId = null
|
||||||
|
onEvent("bereit")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
val b64 = p.optString("base64")
|
||||||
|
if (b64.isBlank()) return
|
||||||
|
val pcm = try { Base64.decode(b64, Base64.DEFAULT) } catch (_: Exception) { return }
|
||||||
|
if (track == null || playingMessageId != mid) {
|
||||||
|
try { track?.release() } catch (_: Exception) {}
|
||||||
|
val min = AudioTrack.getMinBufferSize(rate, AudioFormat.CHANNEL_OUT_MONO,
|
||||||
|
AudioFormat.ENCODING_PCM_16BIT)
|
||||||
|
track = AudioTrack.Builder()
|
||||||
|
.setAudioAttributes(AudioAttributes.Builder()
|
||||||
|
.setUsage(AudioAttributes.USAGE_ASSISTANT)
|
||||||
|
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH).build())
|
||||||
|
.setAudioFormat(AudioFormat.Builder()
|
||||||
|
.setSampleRate(rate)
|
||||||
|
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
|
||||||
|
.setEncoding(AudioFormat.ENCODING_PCM_16BIT).build())
|
||||||
|
.setBufferSizeInBytes(maxOf(min, 16384))
|
||||||
|
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||||
|
.build().also { it.play() }
|
||||||
|
playingMessageId = mid
|
||||||
|
onEvent("spricht …")
|
||||||
|
}
|
||||||
|
try { track?.write(pcm, 0, pcm.size) } catch (_: Exception) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun shutdown() {
|
||||||
|
stopListening("shutdown")
|
||||||
|
try { track?.stop(); track?.release() } catch (_: Exception) {}
|
||||||
|
track = null
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -98,6 +98,14 @@
|
|||||||
android:text="Agent stoppen"
|
android:text="Agent stoppen"
|
||||||
android:paddingTop="8dp" />
|
android:paddingTop="8dp" />
|
||||||
|
|
||||||
|
<Button
|
||||||
|
android:id="@+id/btnTalk"
|
||||||
|
android:layout_width="match_parent"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:focusable="true"
|
||||||
|
android:text="🎤 Mit ARIA sprechen"
|
||||||
|
android:paddingTop="16dp" />
|
||||||
|
|
||||||
<TextView
|
<TextView
|
||||||
android:layout_width="wrap_content"
|
android:layout_width="wrap_content"
|
||||||
android:layout_height="wrap_content"
|
android:layout_height="wrap_content"
|
||||||
@@ -126,5 +134,48 @@
|
|||||||
android:text="Bedienungshilfe öffnen"
|
android:text="Bedienungshilfe öffnen"
|
||||||
android:paddingTop="8dp" />
|
android:paddingTop="8dp" />
|
||||||
|
|
||||||
|
<TextView
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:text="Antworten (Fernseher)"
|
||||||
|
android:textStyle="bold"
|
||||||
|
android:paddingTop="24dp"
|
||||||
|
android:paddingBottom="4dp" />
|
||||||
|
|
||||||
|
<TextView
|
||||||
|
android:layout_width="wrap_content"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:text="Wie ARIA hier standardmäßig antwortet. Sie kann das pro Anfrage überschreiben (z.B. mit dem Zusatz: bitte als Sprache)."
|
||||||
|
android:textSize="13sp"
|
||||||
|
android:paddingBottom="8dp" />
|
||||||
|
|
||||||
|
<RadioGroup
|
||||||
|
android:id="@+id/replyMode"
|
||||||
|
android:layout_width="match_parent"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:orientation="vertical">
|
||||||
|
|
||||||
|
<RadioButton
|
||||||
|
android:id="@+id/modeOverlay"
|
||||||
|
android:layout_width="match_parent"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:focusable="true"
|
||||||
|
android:text="Einblendung über dem Bild (Standard)" />
|
||||||
|
|
||||||
|
<RadioButton
|
||||||
|
android:id="@+id/modeSpeech"
|
||||||
|
android:layout_width="match_parent"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:focusable="true"
|
||||||
|
android:text="Sprachausgabe über die Lautsprecher" />
|
||||||
|
</RadioGroup>
|
||||||
|
|
||||||
|
<Button
|
||||||
|
android:id="@+id/btnOverlayPerm"
|
||||||
|
android:layout_width="match_parent"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
android:text="Einblendung erlauben"
|
||||||
|
android:paddingTop="8dp" />
|
||||||
|
|
||||||
</LinearLayout>
|
</LinearLayout>
|
||||||
</ScrollView>
|
</ScrollView>
|
||||||
|
|||||||
Reference in New Issue
Block a user