feat: Workspace-Umbau Schritt 7 — VNC-Live-Desktop durch RVS getunnelt
QEMU-VNC erscheint live in der App-Desktop-Kachel, komplett ueber RVS: - Bridge: TCP↔RVS-VNC-Bruecke. vnc_open oeffnet asyncio-Verbindung zu host.docker.internal:5901, Reader-Loop streamt RFB-Bytes als vnc_data; vnc_input schreibt zurueck; vnc_close raeumt auf. check_desktop probt den Port und meldet desktop_status. Base64-in-JSON, kein websockify/noVNC auf dem Host noetig. - App: novncHtml.ts laedt noVNC (CDN) und ersetzt window.WebSocket durch einen Shim, der RFB-Bytes per postMessage ueber RVS brueckt (server-speaks-first → timing-robust). VncTile mountet die WebView nur im Fokus, oeffnet bei 'ready' den Tunnel (desktop.openVnc), speist Server-Bytes ein und schickt Eingaben als vnc_input; beim Verlassen wird der Tunnel geschlossen (VM laeuft weiter). tsc/py clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,97 @@
|
|||||||
|
/**
|
||||||
|
* novncHtml — noVNC-Client fuer die WebView, dessen WebSocket durch den
|
||||||
|
* RVS-Tunnel gebrueckt wird.
|
||||||
|
*
|
||||||
|
* Trick: window.WebSocket wird VOR dem Laden von noVNC durch einen Shim
|
||||||
|
* ersetzt. noVNC (RFB) glaubt, ein echtes WebSocket zu benutzen; tatsaechlich
|
||||||
|
* gehen die RFB-Bytes als Base64 per postMessage an RN → RVS → Bridge → QEMU
|
||||||
|
* (und zurueck). Da RFB "server-speaks-first" ist, ist die Reihenfolge robust.
|
||||||
|
*
|
||||||
|
* noVNC wird vom CDN geladen (das Telefon hat Internet, da es ohnehin am RVS
|
||||||
|
* haengt). Voll-offline-Bundling waere ein spaeterer Schritt.
|
||||||
|
*
|
||||||
|
* Protokoll:
|
||||||
|
* RN -> WebView window.ariaVnc.onData(b64) RFB-Bytes vom Server
|
||||||
|
* WebView -> RN {event:'ready'} RFB initialisiert → Tunnel oeffnen
|
||||||
|
* {event:'vnc_send', b64} RFB-Bytes an den Server
|
||||||
|
* {event:'vnc_close'} RFB hat geschlossen
|
||||||
|
* {event:'vnc_state', state} connected|disconnected
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const NOVNC_HTML = `<!doctype html><html><head><meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
||||||
|
<style>
|
||||||
|
* { margin:0; padding:0; }
|
||||||
|
html, body { height:100%; background:#000; overflow:hidden; }
|
||||||
|
#screen { width:100%; height:100%; }
|
||||||
|
#msg { position:absolute; top:8px; left:0; right:0; text-align:center;
|
||||||
|
color:#9090B0; font-family:sans-serif; font-size:12px; pointer-events:none; }
|
||||||
|
</style></head><body>
|
||||||
|
<div id="screen"></div>
|
||||||
|
<div id="msg">Verbinde mit Desktop …</div>
|
||||||
|
<script>
|
||||||
|
(function(){
|
||||||
|
function post(o){ if(window.ReactNativeWebView) window.ReactNativeWebView.postMessage(JSON.stringify(o)); }
|
||||||
|
function b64FromBytes(bytes){
|
||||||
|
var CHUNK=0x8000, parts=[];
|
||||||
|
for(var i=0;i<bytes.length;i+=CHUNK){ parts.push(String.fromCharCode.apply(null, bytes.subarray(i,i+CHUNK))); }
|
||||||
|
return btoa(parts.join(''));
|
||||||
|
}
|
||||||
|
function bytesFromB64(b64){
|
||||||
|
var s=atob(b64), a=new Uint8Array(s.length);
|
||||||
|
for(var i=0;i<s.length;i++) a[i]=s.charCodeAt(i);
|
||||||
|
return a;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- WebSocket-Shim ---
|
||||||
|
function BridgeSocket(url, protocols){
|
||||||
|
this.url=url; this.protocol=''; this.readyState=0; this.binaryType='arraybuffer';
|
||||||
|
this.onopen=null; this.onclose=null; this.onerror=null; this.onmessage=null;
|
||||||
|
var self=this; window.__vncSocket=self;
|
||||||
|
setTimeout(function(){ self.readyState=1; if(self.onopen) self.onopen({type:'open'}); }, 0);
|
||||||
|
}
|
||||||
|
BridgeSocket.CONNECTING=0; BridgeSocket.OPEN=1; BridgeSocket.CLOSING=2; BridgeSocket.CLOSED=3;
|
||||||
|
BridgeSocket.prototype.send=function(data){
|
||||||
|
var bytes;
|
||||||
|
if(data instanceof ArrayBuffer) bytes=new Uint8Array(data);
|
||||||
|
else if(ArrayBuffer.isView(data)) bytes=new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
||||||
|
else bytes=new Uint8Array(0);
|
||||||
|
post({event:'vnc_send', b64:b64FromBytes(bytes)});
|
||||||
|
};
|
||||||
|
BridgeSocket.prototype.close=function(){
|
||||||
|
if(this.readyState===3) return;
|
||||||
|
this.readyState=3; if(this.onclose) this.onclose({type:'close'}); post({event:'vnc_close'});
|
||||||
|
};
|
||||||
|
BridgeSocket.prototype.addEventListener=function(t,fn){ this['on'+t]=fn; };
|
||||||
|
BridgeSocket.prototype.removeEventListener=function(t){ this['on'+t]=null; };
|
||||||
|
window.WebSocket = BridgeSocket;
|
||||||
|
|
||||||
|
// Eingehende Server-Bytes → in den Shim einspeisen.
|
||||||
|
window.ariaVnc = {
|
||||||
|
onData:function(b64){
|
||||||
|
var sock=window.__vncSocket;
|
||||||
|
if(!sock || !sock.onmessage) return;
|
||||||
|
sock.onmessage({ type:'message', data: bytesFromB64(b64).buffer });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var msg=document.getElementById('msg');
|
||||||
|
// noVNC per ESM vom CDN laden.
|
||||||
|
import('https://cdn.jsdelivr.net/npm/@novnc/novnc@1.4.0/core/rfb.js').then(function(mod){
|
||||||
|
var RFB = mod.default;
|
||||||
|
var rfb = new RFB(document.getElementById('screen'), 'ws://aria-vnc/', {});
|
||||||
|
rfb.scaleViewport = true;
|
||||||
|
rfb.clipViewport = false;
|
||||||
|
rfb.addEventListener('connect', function(){ msg.style.display='none'; post({event:'vnc_state', state:'connected'}); });
|
||||||
|
rfb.addEventListener('disconnect', function(e){
|
||||||
|
msg.style.display='block'; msg.textContent='Desktop getrennt';
|
||||||
|
post({event:'vnc_state', state:'disconnected'});
|
||||||
|
});
|
||||||
|
window.__rfb = rfb;
|
||||||
|
post({event:'ready'});
|
||||||
|
}).catch(function(err){
|
||||||
|
msg.textContent='noVNC konnte nicht geladen werden (Internet?)';
|
||||||
|
post({event:'vnc_state', state:'error', error:String(err)});
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
</script></body></html>`;
|
||||||
@@ -1,31 +1,105 @@
|
|||||||
/**
|
/**
|
||||||
* VncTile — Live-Desktop der QEMU-VM (noVNC in einer WebView, RFB durch RVS).
|
* VncTile — Live-Desktop der QEMU-VM (noVNC in einer WebView, RFB durch RVS).
|
||||||
* Platzhalter fuer Commit 4; der noVNC-Tunnel folgt in Commit 7.
|
*
|
||||||
|
* Nur im Fokus aktiv: dann wird die WebView gemountet, bei 'ready' der
|
||||||
|
* RVS-VNC-Tunnel geoeffnet (desktop.openVnc). Server-Bytes (vnc_data) werden in
|
||||||
|
* die WebView injiziert, RFB-Bytes der WebView (vnc_send) gehen als vnc_input
|
||||||
|
* zurueck. Verlaesst man die Kachel, wird der Tunnel geschlossen (die VM laeuft
|
||||||
|
* auf dem Host weiter).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import React from 'react';
|
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
import { StyleSheet, Text, View } from 'react-native';
|
import { StyleSheet, Text, View } from 'react-native';
|
||||||
|
import { WebView, WebViewMessageEvent } from 'react-native-webview';
|
||||||
|
import desktop from '../../services/desktop';
|
||||||
|
import { NOVNC_HTML } from '../assets/novncHtml';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
projectId: string;
|
projectId: string;
|
||||||
focused: boolean;
|
focused: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const VncTile: React.FC<Props> = () => {
|
const VncTile: React.FC<Props> = ({ projectId, focused }) => {
|
||||||
|
const webRef = useRef<WebView>(null);
|
||||||
|
const [status, setStatus] = useState<'idle' | 'connecting' | 'connected' | 'disconnected'>('idle');
|
||||||
|
const unsubDataRef = useRef<null | (() => void)>(null);
|
||||||
|
|
||||||
|
// Aufraeumen: Tunnel zu + Daten-Abo weg.
|
||||||
|
const teardown = useCallback(() => {
|
||||||
|
if (unsubDataRef.current) { unsubDataRef.current(); unsubDataRef.current = null; }
|
||||||
|
desktop.closeVnc();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Fokus verloren / Unmount → Tunnel schliessen.
|
||||||
|
useEffect(() => {
|
||||||
|
if (!focused) { teardown(); setStatus('idle'); }
|
||||||
|
return () => teardown();
|
||||||
|
}, [focused, teardown]);
|
||||||
|
|
||||||
|
const onMessage = useCallback((e: WebViewMessageEvent) => {
|
||||||
|
let m: any;
|
||||||
|
try { m = JSON.parse(e.nativeEvent.data); } catch { return; }
|
||||||
|
if (m.event === 'ready') {
|
||||||
|
// WebView + RFB bereit → Tunnel oeffnen und Server-Bytes einspeisen.
|
||||||
|
setStatus('connecting');
|
||||||
|
unsubDataRef.current = desktop.onVncData((b64) => {
|
||||||
|
const js = `window.ariaVnc && window.ariaVnc.onData(${JSON.stringify(b64)}); true;`;
|
||||||
|
webRef.current?.injectJavaScript(js);
|
||||||
|
});
|
||||||
|
desktop.openVnc(projectId);
|
||||||
|
} else if (m.event === 'vnc_send') {
|
||||||
|
desktop.sendInput(m.b64);
|
||||||
|
} else if (m.event === 'vnc_close') {
|
||||||
|
desktop.closeVnc();
|
||||||
|
} else if (m.event === 'vnc_state') {
|
||||||
|
if (m.state === 'connected') setStatus('connected');
|
||||||
|
else if (m.state === 'disconnected') setStatus('disconnected');
|
||||||
|
}
|
||||||
|
}, [projectId]);
|
||||||
|
|
||||||
|
if (!focused) {
|
||||||
|
return (
|
||||||
|
<View style={styles.placeholder}>
|
||||||
|
<Text style={styles.icon}>🖥️</Text>
|
||||||
|
<Text style={styles.text}>Desktop</Text>
|
||||||
|
<Text style={styles.sub}>Antippen zum Verbinden</Text>
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<View style={styles.container}>
|
<View style={styles.container}>
|
||||||
<Text style={styles.icon}>🖥️</Text>
|
<WebView
|
||||||
<Text style={styles.text}>Desktop</Text>
|
ref={webRef}
|
||||||
<Text style={styles.sub}>Kein Desktop verbunden</Text>
|
style={styles.web}
|
||||||
|
originWhitelist={['*']}
|
||||||
|
source={{ html: NOVNC_HTML, baseUrl: 'https://aria-vnc.local/' }}
|
||||||
|
onMessage={onMessage}
|
||||||
|
javaScriptEnabled
|
||||||
|
domStorageEnabled
|
||||||
|
mixedContentMode="always"
|
||||||
|
androidLayerType="hardware"
|
||||||
|
/>
|
||||||
|
{status !== 'connected' && (
|
||||||
|
<View style={styles.overlay} pointerEvents="none">
|
||||||
|
<Text style={styles.overlayText}>
|
||||||
|
{status === 'connecting' ? 'Verbinde …' : status === 'disconnected' ? 'Getrennt' : ''}
|
||||||
|
</Text>
|
||||||
|
</View>
|
||||||
|
)}
|
||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
const styles = StyleSheet.create({
|
const styles = StyleSheet.create({
|
||||||
container: { flex: 1, backgroundColor: '#000000', alignItems: 'center', justifyContent: 'center' },
|
container: { flex: 1, backgroundColor: '#000000' },
|
||||||
|
web: { flex: 1, backgroundColor: '#000000' },
|
||||||
|
placeholder: { flex: 1, backgroundColor: '#000000', alignItems: 'center', justifyContent: 'center' },
|
||||||
icon: { fontSize: 64, marginBottom: 16 },
|
icon: { fontSize: 64, marginBottom: 16 },
|
||||||
text: { color: '#FFFFFF', fontSize: 18, fontWeight: '700' },
|
text: { color: '#FFFFFF', fontSize: 18, fontWeight: '700' },
|
||||||
sub: { color: '#9090B0', fontSize: 14, marginTop: 8 },
|
sub: { color: '#9090B0', fontSize: 14, marginTop: 8 },
|
||||||
|
overlay: { position: 'absolute', top: 10, left: 0, right: 0, alignItems: 'center' },
|
||||||
|
overlayText: { color: '#9090B0', fontSize: 12, backgroundColor: 'rgba(0,0,0,0.6)', paddingHorizontal: 10, paddingVertical: 4, borderRadius: 10, overflow: 'hidden' },
|
||||||
});
|
});
|
||||||
|
|
||||||
export default VncTile;
|
export default VncTile;
|
||||||
|
|||||||
@@ -734,6 +734,11 @@ class ARIABridge:
|
|||||||
# Beeinflusst das Timeout fuer stt_request — bei "loading" warten wir laenger,
|
# Beeinflusst das Timeout fuer stt_request — bei "loading" warten wir laenger,
|
||||||
# weil das Modell beim ersten Request noch ~1-2 Min runtergeladen werden kann.
|
# weil das Modell beim ersten Request noch ~1-2 Min runtergeladen werden kann.
|
||||||
self._remote_stt_ready: bool = False
|
self._remote_stt_ready: bool = False
|
||||||
|
# VNC-Tunnel (Workspace-Desktop): session → {"writer": StreamWriter,
|
||||||
|
# "task": asyncio.Task}. Wir bruecken rohes RFB-TCP (QEMU-VNC auf dem
|
||||||
|
# Host) <-> RVS (vnc_data/vnc_input, Base64-in-JSON).
|
||||||
|
self._vnc_sessions: dict[str, dict] = {}
|
||||||
|
self._vnc_host: str = os.environ.get("ARIA_VNC_HOST", "host.docker.internal")
|
||||||
# FLUX-Render-Requests die aktuell auf Antwort der flux-bridge (Gamebox) warten.
|
# FLUX-Render-Requests die aktuell auf Antwort der flux-bridge (Gamebox) warten.
|
||||||
# requestId → Future mit dem flux_response-Payload (oder None bei Fehler).
|
# requestId → Future mit dem flux_response-Payload (oder None bei Fehler).
|
||||||
self._pending_flux: dict[str, asyncio.Future] = {}
|
self._pending_flux: dict[str, asyncio.Future] = {}
|
||||||
@@ -3364,6 +3369,43 @@ class ARIABridge:
|
|||||||
pid or "main", rel, len(full_text), "ok" if ok else "abgelehnt")
|
pid or "main", rel, len(full_text), "ok" if ok else "abgelehnt")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
elif msg_type == "check_desktop":
|
||||||
|
session = str(payload.get("session") or "main")
|
||||||
|
try:
|
||||||
|
port = int(payload.get("port") or 5901)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
port = 5901
|
||||||
|
asyncio.create_task(self._check_desktop(session, port))
|
||||||
|
return
|
||||||
|
|
||||||
|
elif msg_type == "vnc_open":
|
||||||
|
session = str(payload.get("session") or "main")
|
||||||
|
try:
|
||||||
|
port = int(payload.get("port") or 5901)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
port = 5901
|
||||||
|
asyncio.create_task(self._vnc_open(session, port))
|
||||||
|
return
|
||||||
|
|
||||||
|
elif msg_type == "vnc_close":
|
||||||
|
session = str(payload.get("session") or "main")
|
||||||
|
asyncio.create_task(self._vnc_close(session))
|
||||||
|
return
|
||||||
|
|
||||||
|
elif msg_type == "vnc_input":
|
||||||
|
session = str(payload.get("session") or "main")
|
||||||
|
b64 = payload.get("b64") or ""
|
||||||
|
sess = self._vnc_sessions.get(session)
|
||||||
|
if sess and b64:
|
||||||
|
try:
|
||||||
|
data = base64.b64decode(b64)
|
||||||
|
sess["writer"].write(data)
|
||||||
|
# drain nicht awaiten (wir sind synchron im Dispatcher) —
|
||||||
|
# write puffert, der Kernel-Socket schluckt RFB-Input locker.
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("[vnc] input schreiben (%s) fehlgeschlagen: %s", session, exc)
|
||||||
|
return
|
||||||
|
|
||||||
elif msg_type == "config_request":
|
elif msg_type == "config_request":
|
||||||
# Eine andere Bridge (whisper/f5tts) bittet um die aktuelle Voice-
|
# Eine andere Bridge (whisper/f5tts) bittet um die aktuelle Voice-
|
||||||
# Config — passiert wenn sie sich connected, weil sie sonst die
|
# Config — passiert wenn sie sich connected, weil sie sonst die
|
||||||
@@ -4363,6 +4405,94 @@ class ARIABridge:
|
|||||||
logger.warning("[code_file] Schreiben fehlgeschlagen %s/%s: %s", project_id, rel_path, exc)
|
logger.warning("[code_file] Schreiben fehlgeschlagen %s/%s: %s", project_id, rel_path, exc)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
async def _check_desktop(self, session: str, port: int) -> None:
|
||||||
|
"""Probt ob auf dem Host eine QEMU-VNC laeuft und meldet desktop_status."""
|
||||||
|
available = False
|
||||||
|
try:
|
||||||
|
fut = asyncio.open_connection(self._vnc_host, port)
|
||||||
|
reader, writer = await asyncio.wait_for(fut, timeout=2.0)
|
||||||
|
available = True
|
||||||
|
writer.close()
|
||||||
|
try:
|
||||||
|
await writer.wait_closed()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
available = False
|
||||||
|
await self._send_to_rvs({
|
||||||
|
"type": "desktop_status",
|
||||||
|
"payload": {
|
||||||
|
"available": available,
|
||||||
|
"session": session,
|
||||||
|
"message": (f"Desktop laeuft (VNC {self._vnc_host}:{port})" if available
|
||||||
|
else "Kein Desktop verbunden"),
|
||||||
|
},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
})
|
||||||
|
|
||||||
|
async def _vnc_open(self, session: str, port: int) -> None:
|
||||||
|
"""Oeffnet die TCP-Verbindung zur QEMU-VNC und streamt RFB-Bytes als
|
||||||
|
vnc_data ueber RVS. Bestehende Session wird vorher geschlossen."""
|
||||||
|
await self._vnc_close(session)
|
||||||
|
try:
|
||||||
|
reader, writer = await asyncio.open_connection(self._vnc_host, port)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("[vnc] open %s:%d fehlgeschlagen: %s", self._vnc_host, port, exc)
|
||||||
|
await self._send_to_rvs({
|
||||||
|
"type": "desktop_status",
|
||||||
|
"payload": {"available": False, "session": session,
|
||||||
|
"message": f"VNC-Verbindung fehlgeschlagen: {exc}"},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
task = asyncio.create_task(self._vnc_reader_loop(session, reader))
|
||||||
|
self._vnc_sessions[session] = {"writer": writer, "task": task}
|
||||||
|
logger.info("[vnc] Tunnel offen: session=%s → %s:%d", session, self._vnc_host, port)
|
||||||
|
|
||||||
|
async def _vnc_reader_loop(self, session: str, reader: asyncio.StreamReader) -> None:
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
data = await reader.read(16384)
|
||||||
|
if not data:
|
||||||
|
break
|
||||||
|
await self._send_to_rvs({
|
||||||
|
"type": "vnc_data",
|
||||||
|
"payload": {"session": session, "b64": base64.b64encode(data).decode("ascii")},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
})
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.info("[vnc] reader-loop (%s) beendet: %s", session, exc)
|
||||||
|
finally:
|
||||||
|
# TCP-Ende → App informieren (Desktop weg).
|
||||||
|
if session in self._vnc_sessions:
|
||||||
|
await self._send_to_rvs({
|
||||||
|
"type": "desktop_status",
|
||||||
|
"payload": {"available": False, "session": session, "message": "VNC-Verbindung beendet"},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
})
|
||||||
|
|
||||||
|
async def _vnc_close(self, session: str) -> None:
|
||||||
|
sess = self._vnc_sessions.pop(session, None)
|
||||||
|
if not sess:
|
||||||
|
return
|
||||||
|
task = sess.get("task")
|
||||||
|
if task:
|
||||||
|
task.cancel()
|
||||||
|
try:
|
||||||
|
await task
|
||||||
|
except (asyncio.CancelledError, Exception):
|
||||||
|
pass
|
||||||
|
writer = sess.get("writer")
|
||||||
|
if writer:
|
||||||
|
try:
|
||||||
|
writer.close()
|
||||||
|
await writer.wait_closed()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
logger.info("[vnc] Tunnel geschlossen: session=%s", session)
|
||||||
|
|
||||||
async def _delete_chat_message(self, ts: int) -> dict:
|
async def _delete_chat_message(self, ts: int) -> dict:
|
||||||
"""Entfernt eine Bubble: aus chat_backup.jsonl + Brain conversation,
|
"""Entfernt eine Bubble: aus chat_backup.jsonl + Brain conversation,
|
||||||
broadcastet chat_message_deleted via RVS.
|
broadcastet chat_message_deleted via RVS.
|
||||||
|
|||||||
Reference in New Issue
Block a user