Hacker-Software/ARIA-AGENT
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 245 Wiki Activity

feat(android-agent): Meilenstein 3 — steuern (tap/text/swipe/key/app_launch)

Browse Source 
Bedienungshilfe kann jetzt bedienen: canPerformGestures + dispatchGesture (tap/
swipe), ACTION_SET_TEXT (text), performGlobalAction (back/home/recents/notif).
app_launch via getLaunchIntentForPackage (+ Label-Suche, QUERY_ALL_PACKAGES).
caps erweitert. Brain-Tools host_ui_tap/_text/_swipe/_key/host_app_launch ->
generischer _UI_ACTIONS-Dispatch. Tap-Koordinaten = ui_dump-x/y (echte Pixel,
nicht der skalierte Screenshot). Damit voller Ablauf sehen->steuern. Version 0.0.0.6.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
duffyduck
2026-09-24 21:53:02 +02:00
co-authored by Claude Opus 4.8
parent 8b84ee3b9c
commit 2038b676e6
7 changed files with 300 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
host-agent/android/app/src/main/AndroidManifest.xml
+2
View File
@@ -9,6 +9,8 @@
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS" />
<!-- app_launch: Start-Intents/App-Labels sind ab Android 11 sonst unsichtbar. -->
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
<uses-feature android:name="android.hardware.camera" android:required="false" />
<application
host-agent/android/app/src/main/java/de/hackersoft/ariaagent/AriaAccessibilityService.kt
+85
View File
@@ -1,11 +1,17 @@
package de.hackersoft.ariaagent
import android.accessibilityservice.AccessibilityService
import android.accessibilityservice.GestureDescription
import android.graphics.Path
import android.graphics.Rect
import android.os.Bundle
import android.view.accessibility.AccessibilityEvent
import android.view.accessibility.AccessibilityNodeInfo
import org.json.JSONArray
import org.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
/**
* Bedienungshilfe-Dienst (Meilenstein 2 — nur LESEND).
@@ -82,6 +88,85 @@ class AriaAccessibilityService : AccessibilityService() {
}
}
// ─── Steuern (Meilenstein 3) ────────────────────────────────────
/** Tippt auf Bildschirm-Koordinaten (Pixel wie in ui_dump x/y). */
fun tap(x: Int, y: Int): JSONObject {
val path = Path().apply { moveTo(x.toFloat(), y.toFloat()) }
return gesture(path, 0, 60, "Tippen ($x,$y)")
}
/** Wischt von (x1,y1) nach (x2,y2) ueber dauerMs (Scrollen/Swipen). */
fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Int): JSONObject {
val path = Path().apply {
moveTo(x1.toFloat(), y1.toFloat())
lineTo(x2.toFloat(), y2.toFloat())
}
return gesture(path, 0, durationMs.coerceIn(50, 5000).toLong(),
"Wischen ($x1,$y1 -> $x2,$y2)")
}
private fun gesture(path: Path, startMs: Long, durationMs: Long, desc: String): JSONObject {
val g = GestureDescription.Builder()
.addStroke(GestureDescription.StrokeDescription(path, startMs, durationMs))
.build()
val latch = CountDownLatch(1)
val ok = AtomicBoolean(false)
val dispatched = dispatchGesture(g, object : GestureResultCallback() {
override fun onCompleted(d: GestureDescription?) { ok.set(true); latch.countDown() }
override fun onCancelled(d: GestureDescription?) { latch.countDown() }
}, null)
if (!dispatched) return errMsg("Geste konnte nicht ausgeloest werden ($desc)")
try { latch.await(6, TimeUnit.SECONDS) } catch (_: InterruptedException) {}
return if (ok.get()) okMsg("$desc ausgefuehrt") else errMsg("$desc abgebrochen/timeout")
}
/** Schreibt Text in ein Eingabefeld an (x,y) — oder in das fokussierte Feld. */
fun setText(x: Int, y: Int, text: String): JSONObject {
val root = rootInActiveWindow ?: return errMsg("kein aktives Fenster")
val node = editableAt(root, x, y)
?: root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.takeIf { it.isEditable }
?: return errMsg("kein Textfeld an ($x,$y) gefunden — vorher ui_tap aufs Feld?")
node.performAction(AccessibilityNodeInfo.ACTION_FOCUS)
val args = Bundle().apply {
putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text)
}
val done = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)
return if (done) okMsg("Text gesetzt (${text.length} Zeichen)")
else errMsg("Text setzen fehlgeschlagen (Feld nicht editierbar?)")
}
/** Tiefste editierbare Node, deren Rahmen (x,y) enthaelt. */
private fun editableAt(node: AccessibilityNodeInfo?, x: Int, y: Int): AccessibilityNodeInfo? {
if (node == null) return null
var found: AccessibilityNodeInfo? = null
for (i in 0 until node.childCount) {
editableAt(node.getChild(i), x, y)?.let { found = it }
}
if (found != null) return found
val r = Rect(); node.getBoundsInScreen(r)
return if (node.isEditable && r.contains(x, y)) node else null
}
/** Globale Taste: back/home/recents/notifications. */
fun globalKey(name: String): JSONObject {
val action = when (name.trim().lowercase()) {
"back", "zurueck", "zurück" -> GLOBAL_ACTION_BACK
"home", "start", "startseite" -> GLOBAL_ACTION_HOME
"recents", "letzte", "uebersicht", "übersicht" -> GLOBAL_ACTION_RECENTS
"notifications", "benachrichtigungen" -> GLOBAL_ACTION_NOTIFICATIONS
else -> return errMsg("Taste '$name' unbekannt (back/home/recents/notifications)")
}
return if (performGlobalAction(action)) okMsg("Taste '$name' ausgefuehrt")
else errMsg("Taste '$name' fehlgeschlagen")
}
private fun okMsg(m: String): JSONObject =
JSONObject().put("ok", true).put("result", JSONObject().put("message", m))
private fun errMsg(m: String): JSONObject =
JSONObject().put("ok", false).put("error", m)
companion object {
@Volatile
var instance: AriaAccessibilityService? = null
host-agent/android/app/src/main/java/de/hackersoft/ariaagent/RvsClient.kt
+78 -7
View File
@@ -33,7 +33,10 @@ class RvsClient(
@Volatile private var running = false
private var pingThread: Thread? = null
private val caps = listOf("info", "screenshot", "ui_dump") // M3: ui_tap/ui_text/…
private val caps = listOf(
"info", "screenshot", "ui_dump",
"ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch",
)
fun start() {
running = true
@@ -126,6 +129,7 @@ class RvsClient(
&& !target.equals(config.displayName(), true)) return
val action = payload.optString("action")
val params = payload.optJSONObject("params") ?: JSONObject()
// WICHTIG: JEDE Aktion muss ein host_result liefern — auch bei Exception
// ODER OutOfMemoryError (Throwable!). Sonst bekommt ARIA statt einer
// Fehlermeldung nur einen Timeout (kein Result kommt zurueck).
@@ -136,9 +140,13 @@ class RvsClient(
action == "info" -> doInfo()
action == "screenshot" -> doScreenshot()
action == "ui_dump" -> doUiDump()
action in listOf("ui_tap", "ui_text", "ui_swipe", "ui_key",
"app_launch", "app_list", "notify") ->
err("Aktion '$action' kommt in Meilenstein 3 (noch nicht implementiert).")
action == "ui_tap" -> doUiTap(params)
action == "ui_text" -> doUiText(params)
action == "ui_swipe" -> doUiSwipe(params)
action == "ui_key" -> doUiKey(params)
action == "app_launch" -> doAppLaunch(params)
action in listOf("app_list", "notify") ->
err("Aktion '$action' kommt spaeter (noch nicht implementiert).")
else -> err("Aktion '$action' unbekannt.")
}
} catch (t: Throwable) {
@@ -168,12 +176,75 @@ class RvsClient(
/** Sichtbare Bedienelemente als Baum (Bedienungshilfe). */
private fun doUiDump(): JSONObject {
val svc = AriaAccessibilityService.instance
?: return err("Bedienungshilfe nicht aktiv. In der Agent-App 'Bedienungshilfe " +
"öffnen' antippen und 'ARIA Host-Agent' einschalten.")
val svc = a11y() ?: return a11yMissing()
return svc.dump()
}
private fun a11y(): AriaAccessibilityService? = AriaAccessibilityService.instance
private fun a11yMissing(): JSONObject =
err("Bedienungshilfe nicht aktiv. In der Agent-App 'Bedienungshilfe öffnen' " +
"antippen und 'ARIA Host-Agent' einschalten.")
/** Tippen auf Koordinaten (Pixel wie in ui_dump x/y). */
private fun doUiTap(p: JSONObject): JSONObject {
val svc = a11y() ?: return a11yMissing()
if (!p.has("x") || !p.has("y")) return err("ui_tap braucht x und y (aus ui_dump).")
return svc.tap(p.optInt("x"), p.optInt("y"))
}
/** Text in Feld an (x,y) schreiben. */
private fun doUiText(p: JSONObject): JSONObject {
val svc = a11y() ?: return a11yMissing()
val text = p.optString("text")
if (!p.has("x") || !p.has("y")) return err("ui_text braucht x, y und text.")
return svc.setText(p.optInt("x"), p.optInt("y"), text)
}
/** Wischen/Scrollen von (x1,y1) nach (x2,y2). */
private fun doUiSwipe(p: JSONObject): JSONObject {
val svc = a11y() ?: return a11yMissing()
if (!p.has("x1") || !p.has("y1") || !p.has("x2") || !p.has("y2"))
return err("ui_swipe braucht x1,y1,x2,y2 (optional duration_ms).")
return svc.swipe(p.optInt("x1"), p.optInt("y1"), p.optInt("x2"), p.optInt("y2"),
p.optInt("duration_ms", 300))
}
/** Globale Taste: back/home/recents/notifications. */
private fun doUiKey(p: JSONObject): JSONObject {
val svc = a11y() ?: return a11yMissing()
return svc.globalKey(p.optString("key"))
}
/** App starten (per Paketname oder Namens-Suche). */
private fun doAppLaunch(p: JSONObject): JSONObject {
val pm = appCtx.packageManager
var pkg = p.optString("package").trim()
val query = p.optString("query").trim()
if (pkg.isBlank() && query.isNotBlank()) {
pkg = resolvePackage(query) ?: return err("Keine App zu '$query' gefunden.")
}
if (pkg.isBlank()) return err("app_launch braucht 'package' ODER 'query' (App-Name).")
val intent = pm.getLaunchIntentForPackage(pkg)
?: return err("App '$pkg' nicht installiert oder ohne Start-Symbol.")
intent.addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK)
appCtx.startActivity(intent)
return JSONObject().put("ok", true)
.put("result", JSONObject().put("message", "App '$pkg' gestartet"))
}
/** Paketname per Label-Teilstring finden (case-insensitive). */
private fun resolvePackage(query: String): String? {
val pm = appCtx.packageManager
val q = query.lowercase()
val launch = android.content.Intent(android.content.Intent.ACTION_MAIN)
.addCategory(android.content.Intent.CATEGORY_LAUNCHER)
return pm.queryIntentActivities(launch, 0)
.mapNotNull { it.activityInfo }
.firstOrNull { pm.getApplicationLabel(it.applicationInfo).toString().lowercase().contains(q) }
?.packageName
}
private fun doInfo(): JSONObject {
val res = JSONObject()
res.put("host", config.displayName())
host-agent/android/app/src/main/res/xml/accessibility_config.xml
+1
View File
@@ -4,5 +4,6 @@
android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagRetrieveInteractiveWindows|flagReportViewIds"
android:canRetrieveWindowContent="true"
android:canPerformGestures="true"
android:notificationTimeout="100"
android:description="@string/accessibility_desc" />