feat(host-agent): GUI-Steuerung fuer Desktop (capability-gated)

Desktop-Agenten (Win/Linux/Mac) koennen jetzt bedienen, nicht nur sehen — mit DENSELBEN Actions/Brain-Tools wie Android (ui_tap/ui_text/ui_key/ui_swipe/ app_launch), ein Werkzeugset fuer Handy und Rechner. - host_agent.py: _gui_input_method() erkennt X11(xdotool)/Wayland(ydotool)/ macOS(osascript+cliclick)/Windows(PowerShell). CAPS werden dynamisch erweitert -> reiner Terminal-Server (kein DISPLAY) meldet KEINE ui_*-Caps. _do_ui_tap/ text/key/swipe + _do_app_launch je Methode; info liefert 'gui'. - Brain: Tool-Beschreibungen decken Desktop ab (Koordinaten aus dem Screenshot, kein ui_dump; button/double bei tap; command bei app_launch); _UI_ACTIONS reicht die neuen Params durch. - README: Fähigkeiten/Plattform-Tabelle + Helfer je OS + HiDPI-Hinweis. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-09-25 14:30:38 +02:00
parent 5b7e205ede
commit 176997a800
3 changed files with 264 additions and 26 deletions
@@ -165,7 +165,37 @@ FILE_MAX_BYTES = int(os.environ.get("FILE_MAX_BYTES", str(10 * 1024 * 1024)) or
 AGENT_VERSION = "0.0.0.7"

 HEARTBEAT_SEC = 25
+
+
+def _gui_input_method() -> str:
+    """Welche Methode zur GUI-Steuerung (Maus/Tastatur) ist verfuegbar?
+    '' = keine (z.B. Linux-Server ohne X/Wayland -> nur Terminal/exec).
+      windows  -> PowerShell (Bordmittel)
+      macos    -> osascript (Text/Tasten) + cliclick (Maus, falls installiert)
+      xdotool  -> Linux/X11
+      ydotool  -> Linux/Wayland (Daemon noetig)"""
+    if IS_WINDOWS:
+        return "windows"
+    if IS_MAC:
+        return "macos"
+    if os.environ.get("DISPLAY") and shutil.which("xdotool"):
+        return "xdotool"
+    if os.environ.get("WAYLAND_DISPLAY") and shutil.which("ydotool"):
+        return "ydotool"
+    return ""
+
+
+GUI_METHOD = _gui_input_method()
+# macOS: Maus braucht cliclick; Text/Tasten/App gehen per osascript (Bordmittel).
+_MAC_MOUSE = IS_MAC and shutil.which("cliclick") is not None
+
 CAPS = ["exec", "read", "write", "info", "screenshot"]
+if GUI_METHOD:
+    # Tastatur/Text/App gehen bei jeder GUI-Methode; Maus (tap/swipe) auf dem Mac
+    # nur mit cliclick.
+    CAPS += ["ui_text", "ui_key", "app_launch"]
+    if GUI_METHOD != "macos" or _MAC_MOUSE:
+        CAPS += ["ui_tap", "ui_swipe"]


 # ─── Text-/Zahl-Helfer ──────────────────────────────────────────────
@@ -305,6 +335,7 @@ def _do_info(params: dict) -> dict:
        "arch": platform.machine(), "python": platform.python_version(),
        "user": os.environ.get("USER") or os.environ.get("USERNAME") or "",
        "is_root": _is_admin(),
+        "gui": GUI_METHOD or "none",   # '' => nur Terminal, keine GUI-Steuerung
    }
    try:
        import psutil
@@ -390,9 +421,191 @@ def _do_screenshot(params: dict) -> dict:
            "Laeuft der Agent in derselben grafischen Session?"}


+# ─── GUI-Steuerung (Maus/Tastatur/App) — nur mit grafischer Session ──
+
+def _gui_unavailable() -> dict:
+    return {"ok": False, "error":
+            "Keine grafische Session zum Steuern. Linux: X11 (DISPLAY + xdotool) "
+            "oder Wayland (WAYLAND_DISPLAY + ydotool-Daemon). macOS: cliclick fuer "
+            "Maus. Auf einem reinen Terminal-/Server-System gibt es keine GUI."}
+
+
+def _run_gui(argv: list, desc: str) -> dict:
+    try:
+        r = subprocess.run(argv, capture_output=True, text=True, timeout=15)
+        if r.returncode == 0:
+            return {"ok": True, "result": {"message": f"{desc} ausgefuehrt"}}
+        return {"ok": False, "error": f"{desc} fehlgeschlagen: "
+                + (r.stderr or r.stdout or f"exit {r.returncode}").strip()[:200]}
+    except Exception as exc:
+        return {"ok": False, "error": f"{desc}: {exc}"}
+
+
+def _ps(script: str, desc: str) -> dict:
+    return _run_gui(["powershell", "-NoProfile", "-NonInteractive", "-Command", script], desc)
+
+
+def _osa(script: str, desc: str) -> dict:
+    return _run_gui(["osascript", "-e", script], desc)
+
+
+_WIN_MOUSE_DECL = (
+    "Add-Type -Name U -Namespace W -MemberDefinition '"
+    "[DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int x,int y);"
+    "[DllImport(\"user32.dll\")] public static extern void mouse_event("
+    "uint f,uint x,uint y,uint d,int e);';"
+)
+
+
+def _do_ui_tap(params: dict) -> dict:
+    if not GUI_METHOD:
+        return _gui_unavailable()
+    x = _to_int(params.get("x")); y = _to_int(params.get("y"))
+    if x is None or y is None:
+        return {"ok": False, "error": "ui_tap braucht x,y (Bildschirm-Pixel aus dem Screenshot)."}
+    button = str(params.get("button", "left")).lower()
+    double = bool(params.get("double"))
+    desc = f"Klick ({x},{y})"
+    if GUI_METHOD == "xdotool":
+        btn = {"left": "1", "middle": "2", "right": "3"}.get(button, "1")
+        argv = ["xdotool", "mousemove", "--sync", str(x), str(y), "click"]
+        if double:
+            argv += ["--repeat", "2", "--delay", "120"]
+        return _run_gui(argv + [btn], desc)
+    if GUI_METHOD == "ydotool":
+        code = {"left": "0xC0", "right": "0xC1", "middle": "0xC2"}.get(button, "0xC0")
+        _run_gui(["ydotool", "mousemove", "-a", str(x), str(y)], "move")
+        r = _run_gui(["ydotool", "click", code], desc)
+        if double and r.get("ok"):
+            _run_gui(["ydotool", "click", code], desc)
+        return r
+    if GUI_METHOD == "macos":
+        if not _MAC_MOUSE:
+            return {"ok": False, "error": "Maus-Steuerung braucht cliclick (brew install cliclick)."}
+        cmd = ("dc:" if double else ("rc:" if button == "right" else "c:")) + f"{x},{y}"
+        return _run_gui(["cliclick", cmd], desc)
+    if GUI_METHOD == "windows":
+        down, up = ("0x0002", "0x0004") if button != "right" else ("0x0008", "0x0010")
+        click = f"[W.U]::mouse_event({down},0,0,0,0);[W.U]::mouse_event({up},0,0,0,0);"
+        script = _WIN_MOUSE_DECL + f"[W.U]::SetCursorPos({x},{y});" + click + (click if double else "")
+        return _ps(script, desc)
+    return _gui_unavailable()
+
+
+def _do_ui_text(params: dict) -> dict:
+    if not GUI_METHOD:
+        return _gui_unavailable()
+    text = str(params.get("text", ""))
+    if not text:
+        return {"ok": False, "error": "ui_text braucht 'text'."}
+    desc = f"Text ({len(text)} Zeichen)"
+    if GUI_METHOD == "xdotool":
+        return _run_gui(["xdotool", "type", "--clearmodifiers", "--", text], desc)
+    if GUI_METHOD == "ydotool":
+        return _run_gui(["ydotool", "type", "--", text], desc)
+    if GUI_METHOD == "macos":
+        esc = text.replace("\\", "\\\\").replace('"', '\\"')
+        return _osa(f'tell application "System Events" to keystroke "{esc}"', desc)
+    if GUI_METHOD == "windows":
+        safe = re.sub(r"([+^%~(){}\[\]])", r"{\1}", text).replace('"', '`"')
+        return _ps("Add-Type -AssemblyName System.Windows.Forms;"
+                   f"[System.Windows.Forms.SendKeys]::SendWait(\"{safe}\")", desc)
+    return _gui_unavailable()
+
+
+def _do_ui_key(params: dict) -> dict:
+    if not GUI_METHOD:
+        return _gui_unavailable()
+    key = str(params.get("key", "")).strip()
+    if not key:
+        return {"ok": False, "error": "ui_key braucht 'key' (z.B. Return, Escape, ctrl+c)."}
+    desc = f"Taste '{key}'"
+    if GUI_METHOD == "xdotool":
+        return _run_gui(["xdotool", "key", "--clearmodifiers", key], desc)
+    if GUI_METHOD == "ydotool":
+        return _run_gui(["ydotool", "key", key], desc)   # best effort (Keycodes)
+    if GUI_METHOD == "macos":
+        codes = {"return": 36, "enter": 36, "tab": 48, "space": 49, "delete": 51,
+                 "escape": 53, "esc": 53, "left": 123, "right": 124, "down": 125, "up": 126,
+                 "home": 115, "end": 119, "pageup": 116, "pagedown": 121}
+        k = key.lower()
+        if k in codes:
+            return _osa(f'tell application "System Events" to key code {codes[k]}', desc)
+        return _osa(f'tell application "System Events" to keystroke "{key}"', desc)
+    if GUI_METHOD == "windows":
+        m = {"return": "{ENTER}", "enter": "{ENTER}", "escape": "{ESC}", "esc": "{ESC}",
+             "tab": "{TAB}", "backspace": "{BACKSPACE}", "delete": "{DEL}",
+             "up": "{UP}", "down": "{DOWN}", "left": "{LEFT}", "right": "{RIGHT}",
+             "home": "{HOME}", "end": "{END}", "pageup": "{PGUP}", "pagedown": "{PGDN}"}
+        send = m.get(key.lower(), key)
+        return _ps("Add-Type -AssemblyName System.Windows.Forms;"
+                   f"[System.Windows.Forms.SendKeys]::SendWait('{send}')", desc)
+    return _gui_unavailable()
+
+
+def _do_ui_swipe(params: dict) -> dict:
+    """Auf dem Desktop = Maus ziehen von (x1,y1) nach (x2,y2)."""
+    if not GUI_METHOD:
+        return _gui_unavailable()
+    x1 = _to_int(params.get("x1")); y1 = _to_int(params.get("y1"))
+    x2 = _to_int(params.get("x2")); y2 = _to_int(params.get("y2"))
+    if None in (x1, y1, x2, y2):
+        return {"ok": False, "error": "ui_swipe braucht x1,y1,x2,y2."}
+    desc = f"Ziehen ({x1},{y1} -> {x2},{y2})"
+    if GUI_METHOD == "xdotool":
+        return _run_gui(["xdotool", "mousemove", "--sync", str(x1), str(y1),
+                         "mousedown", "1", "mousemove", "--sync", str(x2), str(y2),
+                         "mouseup", "1"], desc)
+    if GUI_METHOD == "ydotool":
+        _run_gui(["ydotool", "mousemove", "-a", str(x1), str(y1)], "move")
+        _run_gui(["ydotool", "click", "0x40"], "down")
+        _run_gui(["ydotool", "mousemove", "-a", str(x2), str(y2)], "move")
+        return _run_gui(["ydotool", "click", "0x80"], desc)
+    if GUI_METHOD == "macos":
+        if not _MAC_MOUSE:
+            return {"ok": False, "error": "Ziehen braucht cliclick."}
+        return _run_gui(["cliclick", f"dd:{x1},{y1}", f"du:{x2},{y2}"], desc)
+    if GUI_METHOD == "windows":
+        script = (_WIN_MOUSE_DECL +
+                  f"[W.U]::SetCursorPos({x1},{y1});[W.U]::mouse_event(0x0002,0,0,0,0);"
+                  f"Start-Sleep -Milliseconds 80;[W.U]::SetCursorPos({x2},{y2});"
+                  "[W.U]::mouse_event(0x0004,0,0,0,0);")
+        return _ps(script, desc)
+    return _gui_unavailable()
+
+
+def _do_app_launch(params: dict) -> dict:
+    """Startet eine App/ein Programm (nicht-blockierend). 'app' = Name/Pfad,
+    'command' = beliebiger Startbefehl."""
+    app = str(params.get("app") or params.get("query") or "").strip()
+    command = str(params.get("command") or "").strip()
+    if not app and not command:
+        return {"ok": False, "error": "app_launch braucht 'app' (Name) oder 'command'."}
+    try:
+        if command:
+            argv = command if IS_WINDOWS else ["/bin/sh", "-c", command]
+            subprocess.Popen(argv, shell=bool(IS_WINDOWS),
+                             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            return {"ok": True, "result": {"message": f"Befehl gestartet: {command}"}}
+        if IS_MAC:
+            subprocess.Popen(["open", "-a", app])
+        elif IS_WINDOWS:
+            subprocess.Popen(["cmd", "/c", "start", "", app])
+        else:
+            launcher = shutil.which(app) or app
+            head = ["setsid"] if shutil.which("setsid") else []
+            subprocess.Popen(head + [launcher],
+                             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        return {"ok": True, "result": {"message": f"App gestartet: {app}"}}
+    except Exception as exc:
+        return {"ok": False, "error": f"Start fehlgeschlagen: {exc}"}
+
+
 ACTIONS = {
    "exec": _do_exec, "read": _do_read, "write": _do_write,
    "info": _do_info, "screenshot": _do_screenshot,
+    "ui_tap": _do_ui_tap, "ui_text": _do_ui_text, "ui_key": _do_ui_key,
+    "ui_swipe": _do_ui_swipe, "app_launch": _do_app_launch,
 }