added audio workword, and recording, editied readme

This commit is contained in:
2026-03-29 11:29:15 +02:00
parent b687f790ba
commit dbd97d3cf4
15 changed files with 912 additions and 798 deletions
+80
View File
@@ -20,6 +20,7 @@ import {
import AsyncStorage from '@react-native-async-storage/async-storage';
import rvs, { RVSMessage, ConnectionState } from '../services/rvs';
import audioService from '../services/audio';
import wakeWordService from '../services/wakeword';
import VoiceButton from '../components/VoiceButton';
import FileUpload, { FileData } from '../components/FileUpload';
import CameraUpload, { PhotoData } from '../components/CameraUpload';
@@ -56,6 +57,7 @@ const ChatScreen: React.FC = () => {
const [showFileUpload, setShowFileUpload] = useState(false);
const [showCameraUpload, setShowCameraUpload] = useState(false);
const [gpsEnabled, setGpsEnabled] = useState(false);
const [wakeWordActive, setWakeWordActive] = useState(false);
const flatListRef = useRef<FlatList>(null);
const messageIdCounter = useRef(0);
@@ -134,6 +136,62 @@ const ChatScreen: React.FC = () => {
};
}, []);
// Wake Word: "ARIA" Erkennung → Auto-Aufnahme starten
useEffect(() => {
const unsubWake = wakeWordService.onWakeWord(async () => {
console.log('[Chat] Wake Word erkannt — starte Auto-Aufnahme');
// TTS stoppen damit ARIA sich nicht selbst hoert
audioService.stopPlayback();
// Aufnahme mit Auto-Stop (VAD) starten
const started = await audioService.startRecording(true);
if (!started) {
// Mikrofon nicht verfuegbar, Wake Word wieder aktivieren
wakeWordService.resume();
}
});
// Auto-Stop Callback: wenn Stille erkannt → Aufnahme senden + Wake Word wieder starten
const unsubSilence = audioService.onSilenceDetected(async () => {
const result = await audioService.stopRecording();
if (result && result.durationMs > 500) {
// Sprachnachricht senden (gleiche Logik wie handleVoiceRecording)
const location = await getCurrentLocation();
const userMsg: ChatMessage = {
id: nextId(),
sender: 'user',
text: '[Sprachnachricht]',
timestamp: Date.now(),
attachments: [{ type: 'audio', name: 'Sprachaufnahme' }],
};
setMessages(prev => [...prev, userMsg]);
rvs.send('audio', {
base64: result.base64,
durationMs: result.durationMs,
mimeType: result.mimeType,
...(location && { location }),
});
}
// Wake Word wieder aktivieren
if (wakeWordActive) wakeWordService.resume();
});
return () => {
unsubWake();
unsubSilence();
};
}, [wakeWordActive]);
// Wake Word Toggle Handler
const toggleWakeWord = useCallback(async () => {
if (wakeWordActive) {
wakeWordService.stop();
setWakeWordActive(false);
} else {
const started = await wakeWordService.start();
setWakeWordActive(started);
}
}, [wakeWordActive]);
// Chat-Verlauf in AsyncStorage speichern (letzte N Nachrichten)
useEffect(() => {
if (messages.length === 0) return;
@@ -366,7 +424,14 @@ const ChatScreen: React.FC = () => {
<VoiceButton
onRecordingComplete={handleVoiceRecording}
disabled={connectionState !== 'connected'}
wakeWordActive={wakeWordActive}
/>
<TouchableOpacity
style={[styles.wakeWordBtn, wakeWordActive && styles.wakeWordBtnActive]}
onPress={toggleWakeWord}
>
<Text style={styles.wakeWordIcon}>{wakeWordActive ? '👂' : '🔇'}</Text>
</TouchableOpacity>
)}
</View>
@@ -530,6 +595,21 @@ const styles = StyleSheet.create({
sendIcon: {
fontSize: 18,
},
wakeWordBtn: {
width: 32,
height: 32,
borderRadius: 16,
backgroundColor: 'rgba(255,255,255,0.1)',
alignItems: 'center',
justifyContent: 'center',
marginLeft: 4,
},
wakeWordBtnActive: {
backgroundColor: 'rgba(52, 199, 89, 0.3)',
},
wakeWordIcon: {
fontSize: 16,
},
modalOverlay: {
flex: 1,
backgroundColor: 'rgba(0,0,0,0.6)',