From 176997a800b4b05320440c67ad736ff9ce7fce79 Mon Sep 17 00:00:00 2001 From: duffyduck Date: Fri, 25 Sep 2026 14:30:38 +0200 Subject: [PATCH] feat(host-agent): GUI-Steuerung fuer Desktop (capability-gated) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Desktop-Agenten (Win/Linux/Mac) koennen jetzt bedienen, nicht nur sehen — mit DENSELBEN Actions/Brain-Tools wie Android (ui_tap/ui_text/ui_key/ui_swipe/ app_launch), ein Werkzeugset fuer Handy und Rechner. - host_agent.py: _gui_input_method() erkennt X11(xdotool)/Wayland(ydotool)/ macOS(osascript+cliclick)/Windows(PowerShell). CAPS werden dynamisch erweitert -> reiner Terminal-Server (kein DISPLAY) meldet KEINE ui_*-Caps. _do_ui_tap/ text/key/swipe + _do_app_launch je Methode; info liefert 'gui'. - Brain: Tool-Beschreibungen decken Desktop ab (Koordinaten aus dem Screenshot, kein ui_dump; button/double bei tap; command bei app_launch); _UI_ACTIONS reicht die neuen Params durch. - README: Fähigkeiten/Plattform-Tabelle + Helfer je OS + HiDPI-Hinweis. Co-Authored-By: Claude Opus 4.8 --- aria-brain/agent.py | 42 ++++---- host-agent/README.md | 35 +++++-- host-agent/host_agent.py | 213 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 26 deletions(-) diff --git a/aria-brain/agent.py b/aria-brain/agent.py index a1674a3..8e14b2d 100644 --- a/aria-brain/agent.py +++ b/aria-brain/agent.py @@ -1389,17 +1389,20 @@ META_TOOLS = [ "function": { "name": "host_ui_tap", "description": ( - "Tippt auf einem Host-Agenten (v.a. Android) auf eine Bildschirm-Position. " - "Nutze die x/y-Koordinaten AUS host_ui_dump (Element-Mittelpunkt, echte " - "Bildschirm-Pixel) — NICHT aus dem Screenshot (der ist skaliert). Ablauf: " - "erst host_ui_dump/host_screenshot (sehen), dann host_ui_tap (steuern)." + "Klickt/tippt auf einem Host-Agenten auf eine Bildschirm-Position (x,y). " + "ANDROID: nimm die Koordinaten aus host_ui_dump (echte Pixel) — NICHT aus " + "dem Screenshot (der ist skaliert). DESKTOP (Windows/Linux/Mac): es gibt " + "kein ui_dump; nimm die Koordinaten DIREKT aus dem host_screenshot (voller " + "Auflösung = echte Pixel). button/double optional (Rechtsklick/Doppelklick)." ), "parameters": { "type": "object", "properties": { "host": {"type": "string", "description": "Host-ID/Name."}, - "x": {"type": "integer", "description": "X (Pixel, aus ui_dump)."}, - "y": {"type": "integer", "description": "Y (Pixel, aus ui_dump)."}, + "x": {"type": "integer", "description": "X (Pixel)."}, + "y": {"type": "integer", "description": "Y (Pixel)."}, + "button": {"type": "string", "description": "left (Default) | right | middle."}, + "double": {"type": "boolean", "description": "Doppelklick."}, }, "required": ["host", "x", "y"], }, @@ -1410,9 +1413,9 @@ META_TOOLS = [ "function": { "name": "host_ui_text", "description": ( - "Schreibt Text in ein Eingabefeld an Position x/y (aus host_ui_dump, " - "editable=true). Tippe ggf. vorher mit host_ui_tap ins Feld, damit es " - "fokussiert ist." + "Schreibt Text auf einem Host-Agenten. ANDROID: in das Feld an x/y (aus " + "host_ui_dump, editable=true). DESKTOP: tippt in das AKTUELL fokussierte " + "Feld (x/y optional) — ggf. vorher mit host_ui_tap ins Feld klicken." ), "parameters": { "type": "object", @@ -1451,15 +1454,16 @@ META_TOOLS = [ "function": { "name": "host_ui_key", "description": ( - "Drueckt eine globale Taste auf einem Host-Agenten (Android): " - "back, home, recents, notifications." + "Drueckt eine Taste auf einem Host-Agenten. ANDROID: back | home | recents " + "| notifications. DESKTOP (Win/Linux/Mac): Tastennamen/Kombis wie Return, " + "Escape, Tab, Delete, Up/Down, ctrl+c, alt+F4 (Linux/xdotool-Syntax)." ), "parameters": { "type": "object", "properties": { "host": {"type": "string", "description": "Host-ID/Name."}, "key": {"type": "string", - "description": "back | home | recents | notifications"}, + "description": "Android: back/home/recents/notifications. Desktop: Return, Escape, ctrl+c, …"}, }, "required": ["host", "key"], }, @@ -1470,16 +1474,18 @@ META_TOOLS = [ "function": { "name": "host_app_launch", "description": ( - "Startet eine App auf einem Host-Agenten (Android) — per Paketname " - "(package, z.B. 'com.google.android.gm') ODER Namens-Suche (query, z.B. " - "'Einstellungen'). Danach mit host_screenshot/host_ui_dump weiterarbeiten." + "Startet eine App auf einem Host-Agenten. ANDROID: per Paketname (package, " + "z.B. 'com.google.android.gm') ODER Namens-Suche (query). DESKTOP " + "(Win/Linux/Mac): per App-Name (query, z.B. 'firefox'/'Safari') ODER " + "beliebigem Startbefehl (command). Danach mit host_screenshot weiterarbeiten." ), "parameters": { "type": "object", "properties": { "host": {"type": "string", "description": "Host-ID/Name."}, - "package": {"type": "string", "description": "Paketname (optional)."}, + "package": {"type": "string", "description": "Android-Paketname (optional)."}, "query": {"type": "string", "description": "App-Name/Teilstring (optional)."}, + "command": {"type": "string", "description": "Desktop: Startbefehl (optional)."}, }, "required": ["host"], }, @@ -2835,11 +2841,11 @@ class Agent: # ── Steuern (Meilenstein 3) ────────────────────────────── _UI_ACTIONS = { - "host_ui_tap": ("ui_tap", ["x", "y"]), + "host_ui_tap": ("ui_tap", ["x", "y", "button", "double"]), "host_ui_text": ("ui_text", ["x", "y", "text"]), "host_ui_swipe": ("ui_swipe", ["x1", "y1", "x2", "y2", "duration_ms"]), "host_ui_key": ("ui_key", ["key"]), - "host_app_launch": ("app_launch", ["package", "query"]), + "host_app_launch": ("app_launch", ["package", "query", "command"]), } if name in _UI_ACTIONS: action, keys = _UI_ACTIONS[name] diff --git a/host-agent/README.md b/host-agent/README.md index 3c0a1d4..88c43c2 100644 --- a/host-agent/README.md +++ b/host-agent/README.md @@ -15,22 +15,41 @@ Geräte in einem LAN; der Host-Agent steuert *den Rechner, auf dem er läuft*. | `exec` | Shell-Kommando ausführen (optional `sudo`), stdout/stderr/exit | | `read` | Datei lesen (Base64, mit Offset/Limit) | | `write` | Datei schreiben/anhängen (Base64 oder Text) | -| `info` | OS, CPU/RAM/Disk-Auslastung, Uptime, IP | +| `info` | OS, CPU/RAM/Disk-Auslastung, Uptime, IP, `gui`-Methode | | `screenshot` | Bildschirmfoto (X11: scrot/maim · Wayland: grim) | +| `ui_tap` | Mausklick auf x,y (button/double optional) *(nur mit GUI)* | +| `ui_text` | Text ins fokussierte Feld tippen *(nur mit GUI)* | +| `ui_key` | Taste/Kombi (Return, Escape, ctrl+c …) *(nur mit GUI)* | +| `ui_swipe` | Maus ziehen x1,y1 → x2,y2 *(nur mit GUI)* | +| `app_launch` | App/Programm starten (`app`/`query` oder `command`) *(nur mit GUI)* | -ARIA nutzt diese über die Brain-Tools `host_list` / `host_exec` / `host_read` / -`host_write` / `host_info` / `host_screenshot`. +ARIA nutzt diese über die Brain-Tools `host_list`/`host_exec`/`host_read`/ +`host_write`/`host_info`/`host_screenshot` sowie `host_ui_tap`/`host_ui_text`/ +`host_ui_key`/`host_ui_swipe`/`host_app_launch` (dieselben Tools wie beim +Android-Agenten — ein Werkzeugset für Handy **und** Rechner). + +> **GUI-Steuerung ist capability-gated.** Der Agent meldet die `ui_*`/`app_launch`- +> Fähigkeiten in seinen `caps` nur, wenn eine grafische Session da ist — ein reiner +> Terminal-/Server-Linux (kein `DISPLAY`/`WAYLAND_DISPLAY`) bekommt sie **nicht** +> (dort bleibt es bei `exec` & Co.). ARIA arbeitet mit dem **Screenshot** als Augen +> (voller Auflösung = echte Pixel; auf dem Desktop gibt es kein `ui_dump`). +> +> Nötige Helfer je System: **Linux/X11** → `xdotool`; **Linux/Wayland** → `ydotool` +> (Daemon, meist root); **macOS** → Text/Tasten via Bordmittel (`osascript`), Maus +> via `cliclick` (`brew install cliclick`); **Windows** → Bordmittel (PowerShell). +> HiDPI/Retina-Hinweis: Screenshot-Pixel können 2× der Klick-Koordinaten sein — bei +> Fehlklicks Koordinaten halbieren. ## Plattformen Eine Codebasis, läuft auf **Linux, macOS und Windows** (der Agent wählt Shell, Screenshot-Methode und Root/Admin-Check je OS automatisch): -| | exec | Root/Admin | Screenshot | -|---|---|---|---| -| **Linux** | `bash -lc` | sudo (`SUDO_PASSWORD`/`SUDO_NOPASSWD`) / root | grim (Wayland) · scrot/maim (X11) | -| **macOS** | `bash -lc` | sudo (wie Linux) | `screencapture` (Bordmittel) | -| **Windows** | PowerShell | Agent **als Administrator** starten (kein sudo) | PowerShell/System.Drawing (Bordmittel) | +| | exec | Root/Admin | Screenshot | GUI-Steuerung (`ui_*`) | +|---|---|---|---|---| +| **Linux** | `bash -lc` | sudo (`SUDO_PASSWORD`/`SUDO_NOPASSWD`) / root | grim (Wayland) · scrot/maim (X11) | `xdotool` (X11) · `ydotool` (Wayland) | +| **macOS** | `bash -lc` | sudo (wie Linux) | `screencapture` (Bordmittel) | `osascript` (Text/Tasten) · `cliclick` (Maus) | +| **Windows** | PowerShell | Agent **als Administrator** starten (kein sudo) | PowerShell/System.Drawing (Bordmittel) | PowerShell (Bordmittel) | PyInstaller kann **nicht cross-kompilieren** — jede Binary wird auf ihrem OS gebaut. diff --git a/host-agent/host_agent.py b/host-agent/host_agent.py index f496b6a..53b2561 100644 --- a/host-agent/host_agent.py +++ b/host-agent/host_agent.py @@ -165,7 +165,37 @@ FILE_MAX_BYTES = int(os.environ.get("FILE_MAX_BYTES", str(10 * 1024 * 1024)) or AGENT_VERSION = "0.0.0.7" HEARTBEAT_SEC = 25 + + +def _gui_input_method() -> str: + """Welche Methode zur GUI-Steuerung (Maus/Tastatur) ist verfuegbar? + '' = keine (z.B. Linux-Server ohne X/Wayland -> nur Terminal/exec). + windows -> PowerShell (Bordmittel) + macos -> osascript (Text/Tasten) + cliclick (Maus, falls installiert) + xdotool -> Linux/X11 + ydotool -> Linux/Wayland (Daemon noetig)""" + if IS_WINDOWS: + return "windows" + if IS_MAC: + return "macos" + if os.environ.get("DISPLAY") and shutil.which("xdotool"): + return "xdotool" + if os.environ.get("WAYLAND_DISPLAY") and shutil.which("ydotool"): + return "ydotool" + return "" + + +GUI_METHOD = _gui_input_method() +# macOS: Maus braucht cliclick; Text/Tasten/App gehen per osascript (Bordmittel). +_MAC_MOUSE = IS_MAC and shutil.which("cliclick") is not None + CAPS = ["exec", "read", "write", "info", "screenshot"] +if GUI_METHOD: + # Tastatur/Text/App gehen bei jeder GUI-Methode; Maus (tap/swipe) auf dem Mac + # nur mit cliclick. + CAPS += ["ui_text", "ui_key", "app_launch"] + if GUI_METHOD != "macos" or _MAC_MOUSE: + CAPS += ["ui_tap", "ui_swipe"] # ─── Text-/Zahl-Helfer ────────────────────────────────────────────── @@ -305,6 +335,7 @@ def _do_info(params: dict) -> dict: "arch": platform.machine(), "python": platform.python_version(), "user": os.environ.get("USER") or os.environ.get("USERNAME") or "", "is_root": _is_admin(), + "gui": GUI_METHOD or "none", # '' => nur Terminal, keine GUI-Steuerung } try: import psutil @@ -390,9 +421,191 @@ def _do_screenshot(params: dict) -> dict: "Laeuft der Agent in derselben grafischen Session?"} +# ─── GUI-Steuerung (Maus/Tastatur/App) — nur mit grafischer Session ── + +def _gui_unavailable() -> dict: + return {"ok": False, "error": + "Keine grafische Session zum Steuern. Linux: X11 (DISPLAY + xdotool) " + "oder Wayland (WAYLAND_DISPLAY + ydotool-Daemon). macOS: cliclick fuer " + "Maus. Auf einem reinen Terminal-/Server-System gibt es keine GUI."} + + +def _run_gui(argv: list, desc: str) -> dict: + try: + r = subprocess.run(argv, capture_output=True, text=True, timeout=15) + if r.returncode == 0: + return {"ok": True, "result": {"message": f"{desc} ausgefuehrt"}} + return {"ok": False, "error": f"{desc} fehlgeschlagen: " + + (r.stderr or r.stdout or f"exit {r.returncode}").strip()[:200]} + except Exception as exc: + return {"ok": False, "error": f"{desc}: {exc}"} + + +def _ps(script: str, desc: str) -> dict: + return _run_gui(["powershell", "-NoProfile", "-NonInteractive", "-Command", script], desc) + + +def _osa(script: str, desc: str) -> dict: + return _run_gui(["osascript", "-e", script], desc) + + +_WIN_MOUSE_DECL = ( + "Add-Type -Name U -Namespace W -MemberDefinition '" + "[DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int x,int y);" + "[DllImport(\"user32.dll\")] public static extern void mouse_event(" + "uint f,uint x,uint y,uint d,int e);';" +) + + +def _do_ui_tap(params: dict) -> dict: + if not GUI_METHOD: + return _gui_unavailable() + x = _to_int(params.get("x")); y = _to_int(params.get("y")) + if x is None or y is None: + return {"ok": False, "error": "ui_tap braucht x,y (Bildschirm-Pixel aus dem Screenshot)."} + button = str(params.get("button", "left")).lower() + double = bool(params.get("double")) + desc = f"Klick ({x},{y})" + if GUI_METHOD == "xdotool": + btn = {"left": "1", "middle": "2", "right": "3"}.get(button, "1") + argv = ["xdotool", "mousemove", "--sync", str(x), str(y), "click"] + if double: + argv += ["--repeat", "2", "--delay", "120"] + return _run_gui(argv + [btn], desc) + if GUI_METHOD == "ydotool": + code = {"left": "0xC0", "right": "0xC1", "middle": "0xC2"}.get(button, "0xC0") + _run_gui(["ydotool", "mousemove", "-a", str(x), str(y)], "move") + r = _run_gui(["ydotool", "click", code], desc) + if double and r.get("ok"): + _run_gui(["ydotool", "click", code], desc) + return r + if GUI_METHOD == "macos": + if not _MAC_MOUSE: + return {"ok": False, "error": "Maus-Steuerung braucht cliclick (brew install cliclick)."} + cmd = ("dc:" if double else ("rc:" if button == "right" else "c:")) + f"{x},{y}" + return _run_gui(["cliclick", cmd], desc) + if GUI_METHOD == "windows": + down, up = ("0x0002", "0x0004") if button != "right" else ("0x0008", "0x0010") + click = f"[W.U]::mouse_event({down},0,0,0,0);[W.U]::mouse_event({up},0,0,0,0);" + script = _WIN_MOUSE_DECL + f"[W.U]::SetCursorPos({x},{y});" + click + (click if double else "") + return _ps(script, desc) + return _gui_unavailable() + + +def _do_ui_text(params: dict) -> dict: + if not GUI_METHOD: + return _gui_unavailable() + text = str(params.get("text", "")) + if not text: + return {"ok": False, "error": "ui_text braucht 'text'."} + desc = f"Text ({len(text)} Zeichen)" + if GUI_METHOD == "xdotool": + return _run_gui(["xdotool", "type", "--clearmodifiers", "--", text], desc) + if GUI_METHOD == "ydotool": + return _run_gui(["ydotool", "type", "--", text], desc) + if GUI_METHOD == "macos": + esc = text.replace("\\", "\\\\").replace('"', '\\"') + return _osa(f'tell application "System Events" to keystroke "{esc}"', desc) + if GUI_METHOD == "windows": + safe = re.sub(r"([+^%~(){}\[\]])", r"{\1}", text).replace('"', '`"') + return _ps("Add-Type -AssemblyName System.Windows.Forms;" + f"[System.Windows.Forms.SendKeys]::SendWait(\"{safe}\")", desc) + return _gui_unavailable() + + +def _do_ui_key(params: dict) -> dict: + if not GUI_METHOD: + return _gui_unavailable() + key = str(params.get("key", "")).strip() + if not key: + return {"ok": False, "error": "ui_key braucht 'key' (z.B. Return, Escape, ctrl+c)."} + desc = f"Taste '{key}'" + if GUI_METHOD == "xdotool": + return _run_gui(["xdotool", "key", "--clearmodifiers", key], desc) + if GUI_METHOD == "ydotool": + return _run_gui(["ydotool", "key", key], desc) # best effort (Keycodes) + if GUI_METHOD == "macos": + codes = {"return": 36, "enter": 36, "tab": 48, "space": 49, "delete": 51, + "escape": 53, "esc": 53, "left": 123, "right": 124, "down": 125, "up": 126, + "home": 115, "end": 119, "pageup": 116, "pagedown": 121} + k = key.lower() + if k in codes: + return _osa(f'tell application "System Events" to key code {codes[k]}', desc) + return _osa(f'tell application "System Events" to keystroke "{key}"', desc) + if GUI_METHOD == "windows": + m = {"return": "{ENTER}", "enter": "{ENTER}", "escape": "{ESC}", "esc": "{ESC}", + "tab": "{TAB}", "backspace": "{BACKSPACE}", "delete": "{DEL}", + "up": "{UP}", "down": "{DOWN}", "left": "{LEFT}", "right": "{RIGHT}", + "home": "{HOME}", "end": "{END}", "pageup": "{PGUP}", "pagedown": "{PGDN}"} + send = m.get(key.lower(), key) + return _ps("Add-Type -AssemblyName System.Windows.Forms;" + f"[System.Windows.Forms.SendKeys]::SendWait('{send}')", desc) + return _gui_unavailable() + + +def _do_ui_swipe(params: dict) -> dict: + """Auf dem Desktop = Maus ziehen von (x1,y1) nach (x2,y2).""" + if not GUI_METHOD: + return _gui_unavailable() + x1 = _to_int(params.get("x1")); y1 = _to_int(params.get("y1")) + x2 = _to_int(params.get("x2")); y2 = _to_int(params.get("y2")) + if None in (x1, y1, x2, y2): + return {"ok": False, "error": "ui_swipe braucht x1,y1,x2,y2."} + desc = f"Ziehen ({x1},{y1} -> {x2},{y2})" + if GUI_METHOD == "xdotool": + return _run_gui(["xdotool", "mousemove", "--sync", str(x1), str(y1), + "mousedown", "1", "mousemove", "--sync", str(x2), str(y2), + "mouseup", "1"], desc) + if GUI_METHOD == "ydotool": + _run_gui(["ydotool", "mousemove", "-a", str(x1), str(y1)], "move") + _run_gui(["ydotool", "click", "0x40"], "down") + _run_gui(["ydotool", "mousemove", "-a", str(x2), str(y2)], "move") + return _run_gui(["ydotool", "click", "0x80"], desc) + if GUI_METHOD == "macos": + if not _MAC_MOUSE: + return {"ok": False, "error": "Ziehen braucht cliclick."} + return _run_gui(["cliclick", f"dd:{x1},{y1}", f"du:{x2},{y2}"], desc) + if GUI_METHOD == "windows": + script = (_WIN_MOUSE_DECL + + f"[W.U]::SetCursorPos({x1},{y1});[W.U]::mouse_event(0x0002,0,0,0,0);" + f"Start-Sleep -Milliseconds 80;[W.U]::SetCursorPos({x2},{y2});" + "[W.U]::mouse_event(0x0004,0,0,0,0);") + return _ps(script, desc) + return _gui_unavailable() + + +def _do_app_launch(params: dict) -> dict: + """Startet eine App/ein Programm (nicht-blockierend). 'app' = Name/Pfad, + 'command' = beliebiger Startbefehl.""" + app = str(params.get("app") or params.get("query") or "").strip() + command = str(params.get("command") or "").strip() + if not app and not command: + return {"ok": False, "error": "app_launch braucht 'app' (Name) oder 'command'."} + try: + if command: + argv = command if IS_WINDOWS else ["/bin/sh", "-c", command] + subprocess.Popen(argv, shell=bool(IS_WINDOWS), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return {"ok": True, "result": {"message": f"Befehl gestartet: {command}"}} + if IS_MAC: + subprocess.Popen(["open", "-a", app]) + elif IS_WINDOWS: + subprocess.Popen(["cmd", "/c", "start", "", app]) + else: + launcher = shutil.which(app) or app + head = ["setsid"] if shutil.which("setsid") else [] + subprocess.Popen(head + [launcher], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return {"ok": True, "result": {"message": f"App gestartet: {app}"}} + except Exception as exc: + return {"ok": False, "error": f"Start fehlgeschlagen: {exc}"} + + ACTIONS = { "exec": _do_exec, "read": _do_read, "write": _do_write, "info": _do_info, "screenshot": _do_screenshot, + "ui_tap": _do_ui_tap, "ui_text": _do_ui_text, "ui_key": _do_ui_key, + "ui_swipe": _do_ui_swipe, "app_launch": _do_app_launch, }