diff --git a/aria-brain/agent.py b/aria-brain/agent.py index e2a57e6..122fe11 100644 --- a/aria-brain/agent.py +++ b/aria-brain/agent.py @@ -1384,6 +1384,107 @@ META_TOOLS = [ }, }, }, + { + "type": "function", + "function": { + "name": "host_ui_tap", + "description": ( + "Tippt auf einem Host-Agenten (v.a. Android) auf eine Bildschirm-Position. " + "Nutze die x/y-Koordinaten AUS host_ui_dump (Element-Mittelpunkt, echte " + "Bildschirm-Pixel) — NICHT aus dem Screenshot (der ist skaliert). Ablauf: " + "erst host_ui_dump/host_screenshot (sehen), dann host_ui_tap (steuern)." + ), + "parameters": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Host-ID/Name."}, + "x": {"type": "integer", "description": "X (Pixel, aus ui_dump)."}, + "y": {"type": "integer", "description": "Y (Pixel, aus ui_dump)."}, + }, + "required": ["host", "x", "y"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "host_ui_text", + "description": ( + "Schreibt Text in ein Eingabefeld an Position x/y (aus host_ui_dump, " + "editable=true). Tippe ggf. vorher mit host_ui_tap ins Feld, damit es " + "fokussiert ist." + ), + "parameters": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Host-ID/Name."}, + "x": {"type": "integer", "description": "X des Feldes (aus ui_dump)."}, + "y": {"type": "integer", "description": "Y des Feldes (aus ui_dump)."}, + "text": {"type": "string", "description": "Einzugebender Text."}, + }, + "required": ["host", "x", "y", "text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "host_ui_swipe", + "description": ( + "Wischt/scrollt auf einem Host-Agenten von (x1,y1) nach (x2,y2). Zum " + "Scrollen nach unten z.B. von weiter unten nach weiter oben wischen." + ), + "parameters": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Host-ID/Name."}, + "x1": {"type": "integer"}, "y1": {"type": "integer"}, + "x2": {"type": "integer"}, "y2": {"type": "integer"}, + "duration_ms": {"type": "integer", "description": "Dauer in ms (Default 300)."}, + }, + "required": ["host", "x1", "y1", "x2", "y2"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "host_ui_key", + "description": ( + "Drueckt eine globale Taste auf einem Host-Agenten (Android): " + "back, home, recents, notifications." + ), + "parameters": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Host-ID/Name."}, + "key": {"type": "string", + "description": "back | home | recents | notifications"}, + }, + "required": ["host", "key"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "host_app_launch", + "description": ( + "Startet eine App auf einem Host-Agenten (Android) — per Paketname " + "(package, z.B. 'com.google.android.gm') ODER Namens-Suche (query, z.B. " + "'Einstellungen'). Danach mit host_screenshot/host_ui_dump weiterarbeiten." + ), + "parameters": { + "type": "object", + "properties": { + "host": {"type": "string", "description": "Host-ID/Name."}, + "package": {"type": "string", "description": "Paketname (optional)."}, + "query": {"type": "string", "description": "App-Name/Teilstring (optional)."}, + }, + "required": ["host"], + }, + }, + }, { "type": "function", "function": { @@ -2686,6 +2787,24 @@ class Agent: + json.dumps(nodes, ensure_ascii=False, indent=2) ) + # ── Steuern (Meilenstein 3) ────────────────────────────── + _UI_ACTIONS = { + "host_ui_tap": ("ui_tap", ["x", "y"]), + "host_ui_text": ("ui_text", ["x", "y", "text"]), + "host_ui_swipe": ("ui_swipe", ["x1", "y1", "x2", "y2", "duration_ms"]), + "host_ui_key": ("ui_key", ["key"]), + "host_app_launch": ("app_launch", ["package", "query"]), + } + if name in _UI_ACTIONS: + action, keys = _UI_ACTIONS[name] + params = {k: arguments[k] for k in keys if arguments.get(k) is not None} + result = _post("/internal/host", + {"host": host, "action": action, "params": params}, 20) + if not result.get("ok"): + return f"FEHLER: {result.get('error')}" + r = result.get("result") or {} + return f"OK ({host}): {r.get('message', 'ausgefuehrt')}" + return f"FEHLER: unbekanntes Host-Tool {name}" except Exception as exc: return f"FEHLER: Host/Bridge nicht erreichbar: {exc}" @@ -3483,7 +3602,9 @@ class Agent: if name in ("satellite_list", "satellite_devices", "satellite_command"): return self._dispatch_satellite(name, arguments) if name in ("host_list", "host_exec", "host_read", "host_write", - "host_info", "host_screenshot", "host_ui_dump"): + "host_info", "host_screenshot", "host_ui_dump", + "host_ui_tap", "host_ui_text", "host_ui_swipe", + "host_ui_key", "host_app_launch"): return self._dispatch_host(name, arguments) if name == "vm_register": pid = (project_id or "").strip() diff --git a/host-agent/android/README.md b/host-agent/android/README.md index 988ba41..802014f 100644 --- a/host-agent/android/README.md +++ b/host-agent/android/README.md @@ -111,10 +111,12 @@ identisch, nur der Wecker ändert sich. `ui_dump` (`AriaAccessibilityService`, nur lesend → Brain-Tool `host_ui_dump`). Freigabe einmalig in der App: „Bildschirm-Zugriff erlauben" + „Bedienungshilfe öffnen". → ARIA sieht den Schirm und liest die UI-Elemente mit Koordinaten. -3. **Steuern** — `ui_tap`/`ui_text`/`ui_swipe`/`ui_key`/`app_launch` über den - AccessibilityService. → ARIA bedient Apps (E-Mail-Setup). -4. **Feinschliff** — `info`/`app_list`/`notify`, Build-Härtung, `release.sh` - (Version-Param → Gitea-Release-Asset, wie die App). +3. **✅ Steuern** — `ui_tap`/`ui_text`/`ui_swipe`/`ui_key`/`app_launch` über den + AccessibilityService (`canPerformGestures`, `dispatchGesture`, `ACTION_SET_TEXT`, + `performGlobalAction`) → Brain-Tools `host_ui_tap`/`_text`/`_swipe`/`_key`/ + `host_app_launch`. Tap-Koordinaten = die x/y aus `ui_dump` (echte Pixel), NICHT + aus dem (skalierten) Screenshot. → ARIA bedient Apps (E-Mail-Setup). +4. **Feinschliff** — `app_list`/`notify`, Build-Härtung, `release_agent.sh`. ## Bauen @@ -190,5 +192,7 @@ Gitea-Zugang (`GITEA_URL`, `GITEA_REPO`, `GITEA_USER`) kommt aus der Umgebung od einer `.env`; das Kennwort wird interaktiv abgefragt. **Binaries landen unter „Releases", nie im Tree.** -> Status: **M1 + M2 fertig** (verbinden, `info`, `screenshot`, `ui_dump`), -> `release_agent.sh` vorhanden. Als Nächstes Meilenstein 3 (Steuern). +> Status: **M1–M3 fertig** — verbinden, `info`, `screenshot`, `ui_dump` (sehen) +> und `ui_tap`/`ui_text`/`ui_swipe`/`ui_key`/`app_launch` (steuern); +> `release_agent.sh` vorhanden. Damit läuft der volle Ablauf sehen→steuern +> (z.B. E-Mail-Konto einrichten). Nächstes: Feinschliff (M4). diff --git a/host-agent/android/app/build.gradle b/host-agent/android/app/build.gradle index 120fb0a..17a0335 100644 --- a/host-agent/android/app/build.gradle +++ b/host-agent/android/app/build.gradle @@ -11,8 +11,8 @@ android { applicationId 'de.hackersoft.ariaagent' minSdk 26 targetSdk 33 // 33 vermeidet die Foreground-Service-Typ-Pflicht von 34 - versionCode 5 - versionName '0.0.0.5' + versionCode 6 + versionName '0.0.0.6' } // Fester Signaturschlüssel: jeder Build signiert mit DEMSELBEN Key, damit diff --git a/host-agent/android/app/src/main/AndroidManifest.xml b/host-agent/android/app/src/main/AndroidManifest.xml index e03e3c6..8ebd8bb 100644 --- a/host-agent/android/app/src/main/AndroidManifest.xml +++ b/host-agent/android/app/src/main/AndroidManifest.xml @@ -9,6 +9,8 @@ + + $x2,$y2)") + } + + private fun gesture(path: Path, startMs: Long, durationMs: Long, desc: String): JSONObject { + val g = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, startMs, durationMs)) + .build() + val latch = CountDownLatch(1) + val ok = AtomicBoolean(false) + val dispatched = dispatchGesture(g, object : GestureResultCallback() { + override fun onCompleted(d: GestureDescription?) { ok.set(true); latch.countDown() } + override fun onCancelled(d: GestureDescription?) { latch.countDown() } + }, null) + if (!dispatched) return errMsg("Geste konnte nicht ausgeloest werden ($desc)") + try { latch.await(6, TimeUnit.SECONDS) } catch (_: InterruptedException) {} + return if (ok.get()) okMsg("$desc ausgefuehrt") else errMsg("$desc abgebrochen/timeout") + } + + /** Schreibt Text in ein Eingabefeld an (x,y) — oder in das fokussierte Feld. */ + fun setText(x: Int, y: Int, text: String): JSONObject { + val root = rootInActiveWindow ?: return errMsg("kein aktives Fenster") + val node = editableAt(root, x, y) + ?: root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.takeIf { it.isEditable } + ?: return errMsg("kein Textfeld an ($x,$y) gefunden — vorher ui_tap aufs Feld?") + node.performAction(AccessibilityNodeInfo.ACTION_FOCUS) + val args = Bundle().apply { + putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + } + val done = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + return if (done) okMsg("Text gesetzt (${text.length} Zeichen)") + else errMsg("Text setzen fehlgeschlagen (Feld nicht editierbar?)") + } + + /** Tiefste editierbare Node, deren Rahmen (x,y) enthaelt. */ + private fun editableAt(node: AccessibilityNodeInfo?, x: Int, y: Int): AccessibilityNodeInfo? { + if (node == null) return null + var found: AccessibilityNodeInfo? = null + for (i in 0 until node.childCount) { + editableAt(node.getChild(i), x, y)?.let { found = it } + } + if (found != null) return found + val r = Rect(); node.getBoundsInScreen(r) + return if (node.isEditable && r.contains(x, y)) node else null + } + + /** Globale Taste: back/home/recents/notifications. */ + fun globalKey(name: String): JSONObject { + val action = when (name.trim().lowercase()) { + "back", "zurueck", "zurück" -> GLOBAL_ACTION_BACK + "home", "start", "startseite" -> GLOBAL_ACTION_HOME + "recents", "letzte", "uebersicht", "übersicht" -> GLOBAL_ACTION_RECENTS + "notifications", "benachrichtigungen" -> GLOBAL_ACTION_NOTIFICATIONS + else -> return errMsg("Taste '$name' unbekannt (back/home/recents/notifications)") + } + return if (performGlobalAction(action)) okMsg("Taste '$name' ausgefuehrt") + else errMsg("Taste '$name' fehlgeschlagen") + } + + private fun okMsg(m: String): JSONObject = + JSONObject().put("ok", true).put("result", JSONObject().put("message", m)) + + private fun errMsg(m: String): JSONObject = + JSONObject().put("ok", false).put("error", m) + companion object { @Volatile var instance: AriaAccessibilityService? = null diff --git a/host-agent/android/app/src/main/java/de/hackersoft/ariaagent/RvsClient.kt b/host-agent/android/app/src/main/java/de/hackersoft/ariaagent/RvsClient.kt index cec9ccb..fb4ca92 100644 --- a/host-agent/android/app/src/main/java/de/hackersoft/ariaagent/RvsClient.kt +++ b/host-agent/android/app/src/main/java/de/hackersoft/ariaagent/RvsClient.kt @@ -33,7 +33,10 @@ class RvsClient( @Volatile private var running = false private var pingThread: Thread? = null - private val caps = listOf("info", "screenshot", "ui_dump") // M3: ui_tap/ui_text/… + private val caps = listOf( + "info", "screenshot", "ui_dump", + "ui_tap", "ui_text", "ui_swipe", "ui_key", "app_launch", + ) fun start() { running = true @@ -126,6 +129,7 @@ class RvsClient( && !target.equals(config.displayName(), true)) return val action = payload.optString("action") + val params = payload.optJSONObject("params") ?: JSONObject() // WICHTIG: JEDE Aktion muss ein host_result liefern — auch bei Exception // ODER OutOfMemoryError (Throwable!). Sonst bekommt ARIA statt einer // Fehlermeldung nur einen Timeout (kein Result kommt zurueck). @@ -136,9 +140,13 @@ class RvsClient( action == "info" -> doInfo() action == "screenshot" -> doScreenshot() action == "ui_dump" -> doUiDump() - action in listOf("ui_tap", "ui_text", "ui_swipe", "ui_key", - "app_launch", "app_list", "notify") -> - err("Aktion '$action' kommt in Meilenstein 3 (noch nicht implementiert).") + action == "ui_tap" -> doUiTap(params) + action == "ui_text" -> doUiText(params) + action == "ui_swipe" -> doUiSwipe(params) + action == "ui_key" -> doUiKey(params) + action == "app_launch" -> doAppLaunch(params) + action in listOf("app_list", "notify") -> + err("Aktion '$action' kommt spaeter (noch nicht implementiert).") else -> err("Aktion '$action' unbekannt.") } } catch (t: Throwable) { @@ -168,12 +176,75 @@ class RvsClient( /** Sichtbare Bedienelemente als Baum (Bedienungshilfe). */ private fun doUiDump(): JSONObject { - val svc = AriaAccessibilityService.instance - ?: return err("Bedienungshilfe nicht aktiv. In der Agent-App 'Bedienungshilfe " + - "öffnen' antippen und 'ARIA Host-Agent' einschalten.") + val svc = a11y() ?: return a11yMissing() return svc.dump() } + private fun a11y(): AriaAccessibilityService? = AriaAccessibilityService.instance + + private fun a11yMissing(): JSONObject = + err("Bedienungshilfe nicht aktiv. In der Agent-App 'Bedienungshilfe öffnen' " + + "antippen und 'ARIA Host-Agent' einschalten.") + + /** Tippen auf Koordinaten (Pixel wie in ui_dump x/y). */ + private fun doUiTap(p: JSONObject): JSONObject { + val svc = a11y() ?: return a11yMissing() + if (!p.has("x") || !p.has("y")) return err("ui_tap braucht x und y (aus ui_dump).") + return svc.tap(p.optInt("x"), p.optInt("y")) + } + + /** Text in Feld an (x,y) schreiben. */ + private fun doUiText(p: JSONObject): JSONObject { + val svc = a11y() ?: return a11yMissing() + val text = p.optString("text") + if (!p.has("x") || !p.has("y")) return err("ui_text braucht x, y und text.") + return svc.setText(p.optInt("x"), p.optInt("y"), text) + } + + /** Wischen/Scrollen von (x1,y1) nach (x2,y2). */ + private fun doUiSwipe(p: JSONObject): JSONObject { + val svc = a11y() ?: return a11yMissing() + if (!p.has("x1") || !p.has("y1") || !p.has("x2") || !p.has("y2")) + return err("ui_swipe braucht x1,y1,x2,y2 (optional duration_ms).") + return svc.swipe(p.optInt("x1"), p.optInt("y1"), p.optInt("x2"), p.optInt("y2"), + p.optInt("duration_ms", 300)) + } + + /** Globale Taste: back/home/recents/notifications. */ + private fun doUiKey(p: JSONObject): JSONObject { + val svc = a11y() ?: return a11yMissing() + return svc.globalKey(p.optString("key")) + } + + /** App starten (per Paketname oder Namens-Suche). */ + private fun doAppLaunch(p: JSONObject): JSONObject { + val pm = appCtx.packageManager + var pkg = p.optString("package").trim() + val query = p.optString("query").trim() + if (pkg.isBlank() && query.isNotBlank()) { + pkg = resolvePackage(query) ?: return err("Keine App zu '$query' gefunden.") + } + if (pkg.isBlank()) return err("app_launch braucht 'package' ODER 'query' (App-Name).") + val intent = pm.getLaunchIntentForPackage(pkg) + ?: return err("App '$pkg' nicht installiert oder ohne Start-Symbol.") + intent.addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK) + appCtx.startActivity(intent) + return JSONObject().put("ok", true) + .put("result", JSONObject().put("message", "App '$pkg' gestartet")) + } + + /** Paketname per Label-Teilstring finden (case-insensitive). */ + private fun resolvePackage(query: String): String? { + val pm = appCtx.packageManager + val q = query.lowercase() + val launch = android.content.Intent(android.content.Intent.ACTION_MAIN) + .addCategory(android.content.Intent.CATEGORY_LAUNCHER) + return pm.queryIntentActivities(launch, 0) + .mapNotNull { it.activityInfo } + .firstOrNull { pm.getApplicationLabel(it.applicationInfo).toString().lowercase().contains(q) } + ?.packageName + } + private fun doInfo(): JSONObject { val res = JSONObject() res.put("host", config.displayName()) diff --git a/host-agent/android/app/src/main/res/xml/accessibility_config.xml b/host-agent/android/app/src/main/res/xml/accessibility_config.xml index 8b432c6..5b3c810 100644 --- a/host-agent/android/app/src/main/res/xml/accessibility_config.xml +++ b/host-agent/android/app/src/main/res/xml/accessibility_config.xml @@ -4,5 +4,6 @@ android:accessibilityFeedbackType="feedbackGeneric" android:accessibilityFlags="flagRetrieveInteractiveWindows|flagReportViewIds" android:canRetrieveWindowContent="true" + android:canPerformGestures="true" android:notificationTimeout="100" android:description="@string/accessibility_desc" />