Files
ARIA-AGENT/android/src/workspace/tiles/VncTile.tsx
T
duffyduckandClaude Opus 4.8 a49c022308 fix(vnc): Tastatur-Button folgt echter Tastatur-Sichtbarkeit
Androids Zurueck-Taste versteckt die Tastatur ohne den TextInput zu
blurren → ⌨-Button blieb an, erst der uebernaechste Tap oeffnete wieder.
Jetzt setzt keyboardDidShow/Hide den Button-Zustand; Oeffnen erzwingt
blur→focus, damit ein noch fokussiertes Feld die Tastatur neu aufklappt.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-21 01:19:37 +02:00

299 lines
14 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* VncTile — Live-Desktop der QEMU-VM (noVNC in einer WebView, RFB durch RVS).
*
* Nur aktiv, wenn das Desktop-Panel offen ist (focused): dann WebView mounten,
* bei 'ready' den RVS-VNC-Tunnel oeffnen. Zwei Bedien-Leisten machen die VM auf
* dem Handy voll bedienbar:
* - ctlBar (oben rechts): Fn-Leiste ein/aus, Software-Tastatur, Fit ↔ 1:1.
* - keyBar (oben, Fn): echte Steuertasten, die keine Software-Tastatur
* liefert — Esc, Tab, Pfeile, Pos1/Ende/Bild, Einfg/Entf, Enter, F1F12 und
* Sticky-Modifier Strg/Alt/Shift (fuer Strg+C, Strg+Alt+Entf, …).
*/
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { Keyboard, NativeSyntheticEvent, ScrollView, StyleSheet, Text, TextInput, TextInputChangeEventData, TextInputKeyPressEventData, TouchableOpacity, View } from 'react-native';
import { WebView, WebViewMessageEvent } from 'react-native-webview';
import desktop from '../../services/desktop';
import { NOVNC_HTML } from '../assets/novncHtml';
interface Props {
projectId: string;
focused: boolean;
port?: number; // VNC-Port der zu verbindenden VM (Default 5901 = Display :1)
}
// X11-Keysyms fuer Sondertasten, die kein druckbares Zeichen liefern.
const KEYSYM = { Backspace: 0xff08, Enter: 0xff0d, Tab: 0xff09 };
const MOD = { ctrl: 0xffe3, alt: 0xffe9, shift: 0xffe1 };
const cpToKeysym = (cp: number) => (cp < 0x100 ? cp : 0x01000000 + cp);
// Sondertasten fuer die Fn-Leiste (Label → Keysym).
const NAV_KEYS: { label: string; ks: number }[] = [
{ label: 'Esc', ks: 0xff1b }, { label: 'Tab', ks: 0xff09 },
{ label: '←', ks: 0xff51 }, { label: '↑', ks: 0xff52 }, { label: '↓', ks: 0xff54 }, { label: '→', ks: 0xff53 },
{ label: 'Pos1', ks: 0xff50 }, { label: 'Ende', ks: 0xff57 },
{ label: 'Bild↑', ks: 0xff55 }, { label: 'Bild↓', ks: 0xff56 },
{ label: 'Einfg', ks: 0xff63 }, { label: 'Entf', ks: 0xffff }, { label: '⏎', ks: 0xff0d },
];
const F_KEYS: { label: string; ks: number }[] = Array.from({ length: 12 }, (_, i) => ({ label: 'F' + (i + 1), ks: 0xffbe + i }));
const VncTile: React.FC<Props> = ({ projectId, focused, port = 5901 }) => {
const webRef = useRef<WebView>(null);
const kbdRef = useRef<TextInput>(null);
const bufRef = useRef(''); // Spiegel des TextInput-Textes
const [status, setStatus] = useState<'idle' | 'connecting' | 'connected' | 'disconnected'>('idle');
const [kbdOn, setKbdOn] = useState(false);
const [keyBar, setKeyBar] = useState(false); // Fn-Leiste sichtbar?
const [mods, setMods] = useState({ ctrl: false, alt: false, shift: false });
const modRef = useRef({ ctrl: false, alt: false, shift: false }); // Spiegel fuer Closures
const unsubDataRef = useRef<null | (() => void)>(null);
const teardown = useCallback(() => {
if (unsubDataRef.current) { unsubDataRef.current(); unsubDataRef.current = null; }
desktop.closeVnc();
}, []);
useEffect(() => {
if (!focused) { teardown(); setStatus('idle'); }
return () => teardown();
}, [focused, teardown]);
// Button-Zustand an die ECHTE Tastatur-Sichtbarkeit koppeln: Androids
// Zurueck-Taste blendet die Tastatur aus, ohne den TextInput zu blurren —
// ueber keyboardDidHide setzen wir das ⌨-Symbol trotzdem zurueck.
useEffect(() => {
if (!focused) return;
const show = Keyboard.addListener('keyboardDidShow', () => setKbdOn(true));
const hide = Keyboard.addListener('keyboardDidHide', () => setKbdOn(false));
return () => { show.remove(); hide.remove(); };
}, [focused]);
const ctl = useCallback((fn: string) => {
webRef.current?.injectJavaScript(`window.ariaVncCtl && window.ariaVncCtl.${fn}(); true;`);
}, []);
const sendKeysym = useCallback((ks: number) => {
webRef.current?.injectJavaScript(`window.ariaVncKey && window.ariaVncKey.keysym(${ks}); true;`);
}, []);
const sendCombo = useCallback((modKeysyms: number[], ks: number) => {
webRef.current?.injectJavaScript(`window.ariaVncKey && window.ariaVncKey.combo(${JSON.stringify(modKeysyms)}, ${ks}); true;`);
}, []);
// Aktive Sticky-Modifier als Keysym-Liste; nach dem Anwenden one-shot zuruecksetzen.
const activeMods = useCallback(() => {
const m = modRef.current; const a: number[] = [];
if (m.ctrl) a.push(MOD.ctrl); if (m.alt) a.push(MOD.alt); if (m.shift) a.push(MOD.shift);
return a;
}, []);
const clearMods = useCallback(() => {
if (modRef.current.ctrl || modRef.current.alt || modRef.current.shift) {
modRef.current = { ctrl: false, alt: false, shift: false };
setMods(modRef.current);
}
}, []);
const toggleMod = useCallback((k: 'ctrl' | 'alt' | 'shift') => {
modRef.current = { ...modRef.current, [k]: !modRef.current[k] };
setMods(modRef.current);
}, []);
// Eine Taste (fertiges Keysym) senden — mit ggf. aktiven Modifiern.
const pressKey = useCallback((ks: number) => {
const m = activeMods();
if (m.length) { sendCombo(m, ks); clearMods(); } else sendKeysym(ks);
}, [activeMods, sendCombo, clearMods, sendKeysym]);
// Ein druckbares Zeichen senden — mit ggf. aktiven Modifiern (Strg+C etc.).
const pressChar = useCallback((cp: number) => {
const m = activeMods();
if (m.length) { sendCombo(m, cpToKeysym(cp)); clearMods(); }
else webRef.current?.injectJavaScript(`window.ariaVncKey && window.ariaVncKey.char(${cp}); true;`);
}, [activeMods, sendCombo, clearMods]);
// Tastatur ein-/ausblenden. Oeffnen: blur→focus erzwingt das Aufklappen auch
// dann, wenn der TextInput noch fokussiert ist (Tastatur per Zurueck-Taste
// versteckt). Schliessen: Keyboard.dismiss(); den Button-Zustand setzt der
// keyboardDidShow/Hide-Listener — nicht hier —, damit er nie „haengen" bleibt.
const toggleKbd = useCallback(() => {
if (kbdOn) { Keyboard.dismiss(); }
else { kbdRef.current?.blur(); setTimeout(() => kbdRef.current?.focus(), 30); }
}, [kbdOn]);
// Druckbare Zeichen: Prefix-Diff des (wachsenden) Feldes → nur neu Getipptes an
// die VM. Loeschungen kommen ueber onKeyPress(Backspace), daher hier nur Inserts.
const onKbdChange = useCallback((e: NativeSyntheticEvent<TextInputChangeEventData>) => {
const text = e.nativeEvent.text || '';
const prev = bufRef.current;
let i = 0;
const min = Math.min(prev.length, text.length);
while (i < min && prev.charCodeAt(i) === text.charCodeAt(i)) i++;
for (const ch of text.slice(i)) { const cp = ch.codePointAt(0); if (cp) pressChar(cp); }
bufRef.current = text;
if (text.length > 200) { bufRef.current = ''; kbdRef.current?.setNativeProps({ text: '' }); }
}, [pressChar]);
// Sondertasten der Software-Tastatur: Backspace feuert auf Android zuverlaessig
// als keyPress; die Return-Taste (Haken) kommt als onSubmitEditing (s.u.).
const onKbdKeyPress = useCallback((e: NativeSyntheticEvent<TextInputKeyPressEventData>) => {
const k = e.nativeEvent.key;
if (k === 'Backspace') pressKey(KEYSYM.Backspace);
else if (k === 'Enter') pressKey(KEYSYM.Enter);
}, [pressKey]);
const onMessage = useCallback((e: WebViewMessageEvent) => {
let m: any;
try { m = JSON.parse(e.nativeEvent.data); } catch { return; }
if (m.event === 'ready') {
setStatus('connecting');
unsubDataRef.current = desktop.onVncData((b64) => {
const js = `window.ariaVnc && window.ariaVnc.onData(${JSON.stringify(b64)}); true;`;
webRef.current?.injectJavaScript(js);
});
desktop.openVnc(projectId, port);
} else if (m.event === 'vnc_send') {
desktop.sendInput(m.b64);
} else if (m.event === 'vnc_close') {
desktop.closeVnc();
} else if (m.event === 'vnc_state') {
if (m.state === 'connected') setStatus('connected');
else if (m.state === 'disconnected') setStatus('disconnected');
}
}, [projectId, port]);
if (!focused) {
return (
<View style={styles.placeholder}>
<Text style={styles.icon}>🖥️</Text>
<Text style={styles.text}>Desktop</Text>
<Text style={styles.sub}>Panel öffnen zum Verbinden</Text>
</View>
);
}
const connected = status === 'connected';
return (
<View style={styles.container}>
<WebView
ref={webRef}
style={styles.web}
originWhitelist={['*']}
source={{ html: NOVNC_HTML, baseUrl: 'https://aria-vnc.local/' }}
onMessage={onMessage}
javaScriptEnabled
domStorageEnabled
mixedContentMode="always"
androidLayerType="hardware"
keyboardDisplayRequiresUserAction={false}
/>
{/* Verstecktes Eingabefeld: fokussiert → Android-Tastatur tippt in die VM.
keyboardType=visible-password schaltet Autokorrektur/Vorschlaege ab und
liefert saubere Einzelzeichen. Offscreen, aber fokussierbar. */}
<TextInput
ref={kbdRef}
style={styles.hiddenInput}
onChange={onKbdChange}
onKeyPress={onKbdKeyPress}
onSubmitEditing={() => pressKey(KEYSYM.Enter)}
keyboardType="visible-password"
returnKeyType="send"
autoCapitalize="none"
autoCorrect={false}
spellCheck={false}
blurOnSubmit={false}
caretHidden
contextMenuHidden
multiline={false}
/>
{/* Steuerungs-Leiste — nur wenn verbunden */}
{connected && (
<View style={styles.ctlBar}>
<TouchableOpacity style={[styles.ctlBtn, keyBar && styles.ctlBtnOn]} onPress={() => setKeyBar(v => !v)} activeOpacity={0.7}>
<Text style={styles.ctlText}>Fn</Text>
</TouchableOpacity>
<TouchableOpacity style={[styles.ctlBtn, kbdOn && styles.ctlBtnOn]} onPress={toggleKbd} activeOpacity={0.7}>
<Text style={styles.ctlText}></Text>
</TouchableOpacity>
<TouchableOpacity style={styles.ctlBtn} onPress={() => ctl('toggleFit')} activeOpacity={0.7}>
<Text style={styles.ctlText}></Text>
</TouchableOpacity>
</View>
)}
{/* Fn-Leiste — echte Steuertasten (oben, ueber der Software-Tastatur). */}
{connected && keyBar && (
<View style={styles.keyBar} pointerEvents="box-none">
<ScrollView horizontal showsHorizontalScrollIndicator={false} keyboardShouldPersistTaps="always" contentContainerStyle={styles.keyRow}>
<TouchableOpacity style={[styles.key, mods.ctrl && styles.keyOn]} onPress={() => toggleMod('ctrl')} activeOpacity={0.7}><Text style={styles.keyText}>Strg</Text></TouchableOpacity>
<TouchableOpacity style={[styles.key, mods.alt && styles.keyOn]} onPress={() => toggleMod('alt')} activeOpacity={0.7}><Text style={styles.keyText}>Alt</Text></TouchableOpacity>
<TouchableOpacity style={[styles.key, mods.shift && styles.keyOn]} onPress={() => toggleMod('shift')} activeOpacity={0.7}><Text style={styles.keyText}>Shift</Text></TouchableOpacity>
{NAV_KEYS.map(k => (
<TouchableOpacity key={k.label} style={styles.key} onPress={() => pressKey(k.ks)} activeOpacity={0.7}><Text style={styles.keyText}>{k.label}</Text></TouchableOpacity>
))}
</ScrollView>
<ScrollView horizontal showsHorizontalScrollIndicator={false} keyboardShouldPersistTaps="always" contentContainerStyle={styles.keyRow}>
{F_KEYS.map(k => (
<TouchableOpacity key={k.label} style={styles.key} onPress={() => pressKey(k.ks)} activeOpacity={0.7}><Text style={styles.keyText}>{k.label}</Text></TouchableOpacity>
))}
<TouchableOpacity style={styles.key} onPress={() => ctl('cad')} activeOpacity={0.7}><Text style={styles.keyTextSm}>Strg+Alt+Entf</Text></TouchableOpacity>
</ScrollView>
</View>
)}
{!connected && (
<View style={styles.overlay} pointerEvents="none">
<Text style={styles.overlayText}>
{status === 'connecting' ? 'Verbinde …' : status === 'disconnected' ? 'Getrennt' : ''}
</Text>
</View>
)}
</View>
);
};
const styles = StyleSheet.create({
container: { flex: 1, backgroundColor: '#000000' },
web: { flex: 1, backgroundColor: '#000000' },
placeholder: { flex: 1, backgroundColor: '#000000', alignItems: 'center', justifyContent: 'center' },
icon: { fontSize: 64, marginBottom: 16 },
text: { color: '#FFFFFF', fontSize: 18, fontWeight: '700' },
sub: { color: '#9090B0', fontSize: 14, marginTop: 8 },
ctlBar: {
position: 'absolute',
top: 34,
right: 8,
flexDirection: 'row',
gap: 6,
},
ctlBtn: {
backgroundColor: 'rgba(18,18,42,0.9)',
borderColor: '#2A2A3E',
borderWidth: 1,
borderRadius: 10,
paddingHorizontal: 10,
paddingVertical: 7,
minWidth: 38,
alignItems: 'center',
justifyContent: 'center',
},
ctlBtnOn: { backgroundColor: 'rgba(0,150,255,0.85)', borderColor: '#0096FF' },
ctlText: { color: '#E0E0F0', fontSize: 16, fontWeight: '700' },
ctlTextSmall: { color: '#E0E0F0', fontSize: 11, fontWeight: '700' },
// Fokussierbar (nicht display:none), aber aus dem Sichtfeld geschoben.
hiddenInput: { position: 'absolute', width: 1, height: 1, top: -100, left: -100, opacity: 0, padding: 0 },
keyBar: { position: 'absolute', top: 74, left: 0, right: 0, gap: 5 },
keyRow: { paddingHorizontal: 6, gap: 5, alignItems: 'center' },
key: {
backgroundColor: 'rgba(18,18,42,0.92)', borderColor: '#2A2A3E', borderWidth: 1,
borderRadius: 8, paddingHorizontal: 9, paddingVertical: 7, minWidth: 34,
alignItems: 'center', justifyContent: 'center',
},
keyOn: { backgroundColor: 'rgba(0,150,255,0.85)', borderColor: '#0096FF' },
keyText: { color: '#E0E0F0', fontSize: 13, fontWeight: '700' },
keyTextSm: { color: '#E0E0F0', fontSize: 10, fontWeight: '700' },
overlay: { position: 'absolute', top: 12, left: 0, right: 0, alignItems: 'center' },
overlayText: { color: '#9090B0', fontSize: 12, backgroundColor: 'rgba(0,0,0,0.6)', paddingHorizontal: 10, paddingVertical: 4, borderRadius: 10, overflow: 'hidden' },
});
export default VncTile;