feat(android-agent): Meilenstein 3 — steuern (tap/text/swipe/key/app_launch)
Bedienungshilfe kann jetzt bedienen: canPerformGestures + dispatchGesture (tap/ swipe), ACTION_SET_TEXT (text), performGlobalAction (back/home/recents/notif). app_launch via getLaunchIntentForPackage (+ Label-Suche, QUERY_ALL_PACKAGES). caps erweitert. Brain-Tools host_ui_tap/_text/_swipe/_key/host_app_launch -> generischer _UI_ACTIONS-Dispatch. Tap-Koordinaten = ui_dump-x/y (echte Pixel, nicht der skalierte Screenshot). Damit voller Ablauf sehen->steuern. Version 0.0.0.6. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+122
-1
@@ -1384,6 +1384,107 @@ META_TOOLS = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_ui_tap",
|
||||
"description": (
|
||||
"Tippt auf einem Host-Agenten (v.a. Android) auf eine Bildschirm-Position. "
|
||||
"Nutze die x/y-Koordinaten AUS host_ui_dump (Element-Mittelpunkt, echte "
|
||||
"Bildschirm-Pixel) — NICHT aus dem Screenshot (der ist skaliert). Ablauf: "
|
||||
"erst host_ui_dump/host_screenshot (sehen), dann host_ui_tap (steuern)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||
"x": {"type": "integer", "description": "X (Pixel, aus ui_dump)."},
|
||||
"y": {"type": "integer", "description": "Y (Pixel, aus ui_dump)."},
|
||||
},
|
||||
"required": ["host", "x", "y"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_ui_text",
|
||||
"description": (
|
||||
"Schreibt Text in ein Eingabefeld an Position x/y (aus host_ui_dump, "
|
||||
"editable=true). Tippe ggf. vorher mit host_ui_tap ins Feld, damit es "
|
||||
"fokussiert ist."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||
"x": {"type": "integer", "description": "X des Feldes (aus ui_dump)."},
|
||||
"y": {"type": "integer", "description": "Y des Feldes (aus ui_dump)."},
|
||||
"text": {"type": "string", "description": "Einzugebender Text."},
|
||||
},
|
||||
"required": ["host", "x", "y", "text"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_ui_swipe",
|
||||
"description": (
|
||||
"Wischt/scrollt auf einem Host-Agenten von (x1,y1) nach (x2,y2). Zum "
|
||||
"Scrollen nach unten z.B. von weiter unten nach weiter oben wischen."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||
"x1": {"type": "integer"}, "y1": {"type": "integer"},
|
||||
"x2": {"type": "integer"}, "y2": {"type": "integer"},
|
||||
"duration_ms": {"type": "integer", "description": "Dauer in ms (Default 300)."},
|
||||
},
|
||||
"required": ["host", "x1", "y1", "x2", "y2"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_ui_key",
|
||||
"description": (
|
||||
"Drueckt eine globale Taste auf einem Host-Agenten (Android): "
|
||||
"back, home, recents, notifications."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||
"key": {"type": "string",
|
||||
"description": "back | home | recents | notifications"},
|
||||
},
|
||||
"required": ["host", "key"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "host_app_launch",
|
||||
"description": (
|
||||
"Startet eine App auf einem Host-Agenten (Android) — per Paketname "
|
||||
"(package, z.B. 'com.google.android.gm') ODER Namens-Suche (query, z.B. "
|
||||
"'Einstellungen'). Danach mit host_screenshot/host_ui_dump weiterarbeiten."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {"type": "string", "description": "Host-ID/Name."},
|
||||
"package": {"type": "string", "description": "Paketname (optional)."},
|
||||
"query": {"type": "string", "description": "App-Name/Teilstring (optional)."},
|
||||
},
|
||||
"required": ["host"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@@ -2686,6 +2787,24 @@ class Agent:
|
||||
+ json.dumps(nodes, ensure_ascii=False, indent=2)
|
||||
)
|
||||
|
||||
# ── Steuern (Meilenstein 3) ──────────────────────────────
|
||||
_UI_ACTIONS = {
|
||||
"host_ui_tap": ("ui_tap", ["x", "y"]),
|
||||
"host_ui_text": ("ui_text", ["x", "y", "text"]),
|
||||
"host_ui_swipe": ("ui_swipe", ["x1", "y1", "x2", "y2", "duration_ms"]),
|
||||
"host_ui_key": ("ui_key", ["key"]),
|
||||
"host_app_launch": ("app_launch", ["package", "query"]),
|
||||
}
|
||||
if name in _UI_ACTIONS:
|
||||
action, keys = _UI_ACTIONS[name]
|
||||
params = {k: arguments[k] for k in keys if arguments.get(k) is not None}
|
||||
result = _post("/internal/host",
|
||||
{"host": host, "action": action, "params": params}, 20)
|
||||
if not result.get("ok"):
|
||||
return f"FEHLER: {result.get('error')}"
|
||||
r = result.get("result") or {}
|
||||
return f"OK ({host}): {r.get('message', 'ausgefuehrt')}"
|
||||
|
||||
return f"FEHLER: unbekanntes Host-Tool {name}"
|
||||
except Exception as exc:
|
||||
return f"FEHLER: Host/Bridge nicht erreichbar: {exc}"
|
||||
@@ -3483,7 +3602,9 @@ class Agent:
|
||||
if name in ("satellite_list", "satellite_devices", "satellite_command"):
|
||||
return self._dispatch_satellite(name, arguments)
|
||||
if name in ("host_list", "host_exec", "host_read", "host_write",
|
||||
"host_info", "host_screenshot", "host_ui_dump"):
|
||||
"host_info", "host_screenshot", "host_ui_dump",
|
||||
"host_ui_tap", "host_ui_text", "host_ui_swipe",
|
||||
"host_ui_key", "host_app_launch"):
|
||||
return self._dispatch_host(name, arguments)
|
||||
if name == "vm_register":
|
||||
pid = (project_id or "").strip()
|
||||
|
||||
Reference in New Issue
Block a user