From 3b19f05c5b3666dc5d0cca3dae65e4ff1d1ee80f Mon Sep 17 00:00:00 2001 From: duffyduck Date: Sun, 10 May 2026 17:56:47 +0200 Subject: [PATCH] feat: ARIA kann Dateien an User zurueckgeben (PDFs, Bilder, Office-Docs, ...) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ARIA setzt im Antworttext einen Marker `[FILE: /shared/uploads/aria_xxx.ext]`. Bridge extrahiert ihn (Marker wird aus dem TTS-Text entfernt) und sendet ein neues file_from_aria-Event ueber RVS an App + Diagnostic. Diagnostic: - Eigene Bubble mit Datei-Icon + Klick-Handler - PDF/Bild → neuer Browser-Tab via /shared/* HTTP-Route - Andere → Download via download-Attribut App: - Neues FileOpenerModule (Kotlin) — Intent.ACTION_VIEW mit FileProvider, Android-Picker waehlt App nach MIME-Type - file_paths.xml erweitert (cache + files + external) - file_response liefert jetzt mimeType mit - Klick auf ARIA-Anhang: lokal vorhanden → direkt oeffnen, sonst file_request mit autoOpen-Flag → bei Empfang persistAttachment + open Stefan muss noch im aria-core/OpenClaw System-Prompt einen Hinweis einbauen: "Wenn du dem User eine Datei erstellt hast (Pfad in /shared/uploads/), haenge am Ende deiner Antwort einmalig [FILE: /shared/uploads/aria_.] an. Der Marker wird aus dem sichtbaren Text entfernt und als Anhang in App und Diagnostic angezeigt." Co-Authored-By: Claude Opus 4.7 (1M context) --- .../com/ariacockpit/ApkInstallerPackage.kt | 2 +- .../java/com/ariacockpit/FileOpenerModule.kt | 54 ++++++++++++++ .../app/src/main/res/xml/file_paths.xml | 4 ++ android/src/screens/ChatScreen.tsx | 71 +++++++++++++++++-- bridge/aria_bridge.py | 54 ++++++++++++++ diagnostic/index.html | 39 ++++++++++ diagnostic/server.js | 5 ++ rvs/server.js | 1 + 8 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 android/android/app/src/main/java/com/ariacockpit/FileOpenerModule.kt diff --git a/android/android/app/src/main/java/com/ariacockpit/ApkInstallerPackage.kt b/android/android/app/src/main/java/com/ariacockpit/ApkInstallerPackage.kt index 2282536..1cc5869 100644 --- a/android/android/app/src/main/java/com/ariacockpit/ApkInstallerPackage.kt +++ b/android/android/app/src/main/java/com/ariacockpit/ApkInstallerPackage.kt @@ -7,7 +7,7 @@ import com.facebook.react.uimanager.ViewManager class ApkInstallerPackage : ReactPackage { override fun createNativeModules(reactContext: ReactApplicationContext): List { - return listOf(ApkInstallerModule(reactContext)) + return listOf(ApkInstallerModule(reactContext), FileOpenerModule(reactContext)) } override fun createViewManagers(reactContext: ReactApplicationContext): List> { diff --git a/android/android/app/src/main/java/com/ariacockpit/FileOpenerModule.kt b/android/android/app/src/main/java/com/ariacockpit/FileOpenerModule.kt new file mode 100644 index 0000000..af7333e --- /dev/null +++ b/android/android/app/src/main/java/com/ariacockpit/FileOpenerModule.kt @@ -0,0 +1,54 @@ +package com.ariacockpit + +import android.content.Intent +import android.net.Uri +import android.os.Build +import androidx.core.content.FileProvider +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReactContextBaseJavaModule +import com.facebook.react.bridge.ReactMethod +import java.io.File + +/** + * Oeffnet eine beliebige Datei (PDF, Bild, Office-Doc, ...) mit der vom User + * gewaehlten App via Android-Intent-Picker. Nutzt FileProvider damit auch + * Android 7+ (content:// statt file://) das URI lesen darf. + * + * MIME-Type wird vom Caller bestimmt — App-Auswahl ist davon abhaengig (PDF + * → PDF-Viewer, image/* → Galerie, etc.). + */ +class FileOpenerModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { + override fun getName() = "FileOpener" + + @ReactMethod + fun open(filePath: String, mimeType: String, promise: Promise) { + try { + val cleanPath = filePath.removePrefix("file://") + val file = File(cleanPath) + if (!file.exists()) { + promise.reject("FILE_NOT_FOUND", "Datei nicht gefunden: $cleanPath") + return + } + val context = reactApplicationContext + val uri: Uri = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + FileProvider.getUriForFile(context, "${context.packageName}.fileprovider", file) + } else { + Uri.fromFile(file) + } + val intent = Intent(Intent.ACTION_VIEW).apply { + setDataAndType(uri, mimeType.ifBlank { "*/*" }) + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + // Chooser zeigt Android-Auswahl falls mehrere Apps das MIME oeffnen koennen. + val chooser = Intent.createChooser(intent, "Oeffnen mit").apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(chooser) + promise.resolve(true) + } catch (e: Exception) { + promise.reject("OPEN_ERROR", e.message, e) + } + } +} diff --git a/android/android/app/src/main/res/xml/file_paths.xml b/android/android/app/src/main/res/xml/file_paths.xml index d2e2c8d..312b4e1 100644 --- a/android/android/app/src/main/res/xml/file_paths.xml +++ b/android/android/app/src/main/res/xml/file_paths.xml @@ -1,4 +1,8 @@ + + + + diff --git a/android/src/screens/ChatScreen.tsx b/android/src/screens/ChatScreen.tsx index d86dbfb..7c2dc27 100644 --- a/android/src/screens/ChatScreen.tsx +++ b/android/src/screens/ChatScreen.tsx @@ -20,6 +20,7 @@ import { Modal, ToastAndroid, AppState, + NativeModules, } from 'react-native'; import AsyncStorage from '@react-native-async-storage/async-storage'; import RNFS from 'react-native-fs'; @@ -80,6 +81,23 @@ const capMessages = (msgs: ChatMessage[]): ChatMessage[] => const DEFAULT_ATTACHMENT_DIR = `${RNFS.DocumentDirectoryPath}/chat_attachments`; const STORAGE_PATH_KEY = 'aria_attachment_storage_path'; +const { FileOpener } = NativeModules as { + FileOpener?: { open: (filePath: string, mimeType: string) => Promise }; +}; + +/** Datei mit Android-Intent-Picker oeffnen (System waehlt App nach MIME). */ +async function openFileWithIntent(filePath: string, mimeType: string): Promise { + if (!FileOpener) { + ToastAndroid.show('FileOpener Native Module fehlt', ToastAndroid.SHORT); + return; + } + try { + await FileOpener.open(filePath, mimeType || 'application/octet-stream'); + } catch (err: any) { + ToastAndroid.show(`Oeffnen fehlgeschlagen: ${err?.message || err}`, ToastAndroid.LONG); + } +} + /** Image-Vorschau in der Chat-Bubble. Misst die echte Bild-Dimension via * Image.getSize + setzt aspectRatio dynamisch — dadurch passt sich die * Bubble ans Bild an (kein "Strich" mehr bei breiten oder hohen Bildern). */ @@ -179,6 +197,10 @@ const ChatScreen: React.FC = () => { const flatListRef = useRef(null); const messageIdCounter = useRef(0); + // ServerPaths fuer die der User auf "oeffnen" geklickt hat — beim + // file_response wird die Datei nach dem Speichern direkt mit dem System- + // Intent geoeffnet (PDF-Viewer, Galerie, etc.). + const autoOpenPaths = useRef>(new Set()); // Eindeutige Message-ID generieren const nextId = (): string => { @@ -349,11 +371,32 @@ const ChatScreen: React.FC = () => { return; } + // file_from_aria: ARIA hat eine Datei rausgegeben → als ARIA-Bubble anzeigen + if (message.type === 'file_from_aria') { + const p = message.payload || {}; + const ariaMsg: ChatMessage = { + id: nextId(), + sender: 'aria', + text: '', + timestamp: Date.now(), + attachments: [{ + type: (typeof p.mimeType === 'string' && p.mimeType.startsWith('image/')) ? 'image' : 'file', + name: (p.name as string) || 'datei', + size: (p.size as number) || 0, + mimeType: (p.mimeType as string) || '', + serverPath: (p.serverPath as string) || '', + }], + }; + setMessages(prev => capMessages([...prev, ariaMsg])); + return; + } + // file_response: Re-Download von Server — lokal speichern if (message.type === 'file_response') { const reqId = (message.payload.requestId as string) || ''; const b64 = (message.payload.base64 as string) || ''; const serverPath = (message.payload.serverPath as string) || ''; + const mimeType = (message.payload.mimeType as string) || ''; if (b64 && reqId) { const fileName = (message.payload.name as string) || 'download'; persistAttachment(b64, reqId, fileName).then(filePath => { @@ -363,6 +406,11 @@ const ChatScreen: React.FC = () => { a.serverPath === serverPath ? { ...a, uri: filePath } : a ), }))); + // Wenn der User dieses File explizit oeffnen wollte → Intent-Picker + if (serverPath && autoOpenPaths.current.has(serverPath)) { + autoOpenPaths.current.delete(serverPath); + openFileWithIntent(filePath.replace(/^file:\/\//, ''), mimeType); + } }).catch(() => {}); } return; @@ -1008,7 +1056,22 @@ const ChatScreen: React.FC = () => { ) : ( - + { + // Lokal vorhanden \u2192 direkt mit System-Intent oeffnen + if (att.uri) { + openFileWithIntent(att.uri.replace(/^file:\/\//, ''), att.mimeType || ''); + return; + } + // Sonst: file_request \u2192 bei file_response wird die Datei + // gespeichert UND geoeffnet (autoOpenPaths-Tracking). + if (att.serverPath) { + autoOpenPaths.current.add(att.serverPath); + rvs.send('file_request' as any, { serverPath: att.serverPath, requestId: item.id }); + } + }} + > {att.mimeType?.includes('pdf') ? '\uD83D\uDCC4' : att.mimeType?.includes('word') || att.mimeType?.includes('document') ? '\uD83D\uDCC3' : @@ -1018,12 +1081,10 @@ const ChatScreen: React.FC = () => { {att.name} {att.size ? {Math.round(att.size / 1024)}KB : null} {!att.uri && att.serverPath && ( - rvs.send('file_request' as any, { serverPath: att.serverPath, requestId: item.id })}> - (laden) - + (tippen zum oeffnen) )} {!att.uri && !att.serverPath && (nicht verfuegbar)} - + )} ))} diff --git a/bridge/aria_bridge.py b/bridge/aria_bridge.py index f2c10ec..699b529 100644 --- a/bridge/aria_bridge.py +++ b/bridge/aria_bridge.py @@ -16,7 +16,9 @@ import asyncio import base64 import json import logging +import mimetypes import os +import re import signal import ssl import sys @@ -882,6 +884,48 @@ class ARIABridge: pass return payload.get("text", "") + # File-Marker-Pattern: `[FILE: /pfad/zur/datei.ext]` (Pfad kann Spaces + # enthalten, Endung beliebig). Mehrfach im Text moeglich. + _FILE_MARKER_RE = re.compile(r"\[FILE:\s*(/shared/uploads/[^\]]+?)\s*\]", re.IGNORECASE) + + def _extract_file_markers(self, text: str) -> tuple[str, list[dict]]: + """Sucht [FILE: /shared/uploads/...]-Marker, gibt (cleaned_text, file_list) zurueck.""" + files: list[dict] = [] + for m in self._FILE_MARKER_RE.finditer(text): + path = m.group(1).strip() + if not path.startswith("/shared/uploads/"): + logger.warning("[core] FILE-Marker mit unerlaubtem Pfad ignoriert: %s", path) + continue + if not os.path.isfile(path): + logger.warning("[core] FILE-Marker zeigt auf nicht existente Datei: %s", path) + continue + name = os.path.basename(path) + mime, _ = mimetypes.guess_type(path) + size = os.path.getsize(path) + files.append({ + "serverPath": path, + "name": name, + "mimeType": mime or "application/octet-stream", + "size": size, + }) + cleaned = self._FILE_MARKER_RE.sub("", text).strip() + # Zwei aufeinanderfolgende Leerzeilen → eine + cleaned = re.sub(r"\n{3,}", "\n\n", cleaned) + return cleaned, files + + async def _broadcast_aria_file(self, file_info: dict) -> None: + """ARIA hat eine Datei fuer den User erstellt — App+Diagnostic informieren.""" + logger.info("[rvs] ARIA-Datei rausgeben: %s (%s, %dKB)", + file_info["name"], file_info["mimeType"], file_info["size"] // 1024) + try: + await self._send_to_rvs({ + "type": "file_from_aria", + "payload": file_info, + "timestamp": int(asyncio.get_event_loop().time() * 1000), + }) + except Exception as e: + logger.warning("[rvs] file_from_aria broadcast fehlgeschlagen: %s", e) + async def _process_core_response(self, text: str, payload: dict) -> None: """Verarbeitet eine fertige Antwort von aria-core. @@ -896,6 +940,14 @@ class ARIABridge: logger.info("[core] NO_REPLY empfangen — Antwort still verworfen") return + # File-Marker `[FILE: /shared/uploads/aria_xyz.pdf]` extrahieren — + # ARIA legt damit Dateien fuer den User bereit (Bilder, PDFs, etc.). + # Der Marker wird aus dem Antworttext entfernt (TTS soll ihn nicht + # vorlesen) und parallel als file_from_aria-Event geschickt. + text, aria_files = self._extract_file_markers(text) + for f in aria_files: + await self._broadcast_aria_file(f) + metadata = payload.get("metadata", {}) is_critical = metadata.get("critical", False) requested_voice = metadata.get("voice") @@ -1545,6 +1597,7 @@ class ARIABridge: return with open(server_path, "rb") as f: file_b64 = base64.b64encode(f.read()).decode("ascii") + mime, _ = mimetypes.guess_type(server_path) logger.info("[rvs] Re-Download: %s (%dKB)", server_path, len(file_b64) // 1365) await self._send_to_rvs({ "type": "file_response", @@ -1553,6 +1606,7 @@ class ARIABridge: "serverPath": server_path, "base64": file_b64, "name": os.path.basename(server_path), + "mimeType": mime or "application/octet-stream", }, "timestamp": int(asyncio.get_event_loop().time() * 1000), }) diff --git a/diagnostic/index.html b/diagnostic/index.html index 3905857..0f900cb 100644 --- a/diagnostic/index.html +++ b/diagnostic/index.html @@ -996,6 +996,11 @@ addChat('received', msg.text, 'chat:final'); return; } + if (msg.type === 'file_from_aria') { + const p = msg.payload || {}; + addAriaFile(p); + return; + } if (msg.type === 'chat_delta') { return; } if (msg.type === 'chat_error') { addChat('error', msg.error, 'chat:error'); @@ -1475,6 +1480,40 @@ } } + /** ARIA hat eine Datei rausgegeben — als eigene Bubble mit Klick-Handler. */ + function addAriaFile(p) { + const name = p.name || 'datei'; + const serverPath = p.serverPath || ''; + const mimeType = p.mimeType || ''; + const sizeKB = p.size ? Math.round(p.size / 1024) : 0; + const isImage = mimeType.startsWith('image/'); + const isPdf = mimeType === 'application/pdf'; + const url = serverPath; // Diagnostic-Server liefert /shared/* aus + const sizeStr = sizeKB > 1024 ? `${(sizeKB/1024).toFixed(1)}MB` : `${sizeKB}KB`; + const icon = isImage ? '🖼️' : isPdf ? '📄' : '📎'; + // PDFs/Bilder: target=_blank → neuer Tab. Andere: download-Attribut. + const linkAttrs = (isImage || isPdf) + ? `href="${url}" target="_blank" rel="noopener"` + : `href="${url}" download="${escapeHtml(name)}"`; + let preview = ''; + if (isImage) { + preview = ``; + } + const html = `
${icon} ARIA hat eine Datei erstellt
` + + `${escapeHtml(name)}` + + ` (${escapeHtml(mimeType)}, ${sizeStr})` + + preview + + `
ARIA-Datei — ${new Date().toLocaleTimeString('de-DE')}
`; + for (const box of [chatBox, document.getElementById('chat-box-fs')]) { + if (!box) continue; + const el = document.createElement('div'); + el.className = 'chat-msg received'; + el.innerHTML = html; + box.appendChild(el); + box.scrollTop = box.scrollHeight; + } + } + let chatFullscreen = false; function toggleChatFullscreen() { const modal = document.getElementById('chat-fullscreen'); diff --git a/diagnostic/server.js b/diagnostic/server.js index 5be17b1..5716904 100644 --- a/diagnostic/server.js +++ b/diagnostic/server.js @@ -620,6 +620,11 @@ function connectRVS(forcePlain) { type: "chat", payload: { text: `Anhang: ${name}\n${serverPath}`, sender: "user" } }}); + } else if (msg.type === "file_from_aria" && msg.payload) { + // ARIA hat eine Datei fuer den User erstellt — im Chat als Anhang anzeigen + const p = msg.payload; + log("info", "rvs", `ARIA-Datei: ${p.name} (${p.mimeType}, ${(p.size||0)/1024|0}KB)`); + broadcast({ type: "file_from_aria", payload: p }); } else if (msg.type === "heartbeat") { // ignorieren } else if (msg.type === "mode") { diff --git a/rvs/server.js b/rvs/server.js index ab63e53..cf75142 100644 --- a/rvs/server.js +++ b/rvs/server.js @@ -18,6 +18,7 @@ const ALLOWED_TYPES = new Set([ "update_check", "update_available", "update_download", "update_data", "agent_activity", "cancel_request", "audio_pcm", + "file_from_aria", "xtts_delete_voice", "voice_preload", "voice_ready", "stt_request", "stt_response",