feat(app): manueller Aufnahme-Knopf nutzt jetzt auch Streaming-STT
VoiceButton rewrite — dB/VAD-Pfad endgueltig raus. Knopf ist jetzt nur
noch UI-Trigger:
- onTapStart (ChatScreen baut Bubble + startStreamingRecording)
- onTapStop (ChatScreen ruft stopStreamingRecording)
- audioService.onStateChange treibt die Animation (statt internem
isRecording-Flag)
- onSilenceDetected-Subscription weg
ChatScreen:
- handleVoiceRecording (Legacy) → handleVoiceButtonStart +
handleVoiceButtonStop
- Bubble wird beim Tap SOFORT gebaut (vorher: erst nach Stop), Text
landet via audioRequestId-Match im chat-Handler-Update-Pfad
- noSpeechTimeoutMs=0 (manueller Modus, User kontrolliert via Tap),
hardCapMs=300_000 (5 Minuten Notbremse)
- Wake-Word-conversing + manueller Stop = endConversation (User
will nicht in Multi-Turn-Modus)
- RecordingResult-Import entfaellt (nicht mehr genutzt)
Damit ist die komplette App-seitige Aufnahme auf Streaming + ML-
Endpointer. Der ganze dB/VAD-Apparat (vadEnabled, vadBaselineSamples,
loadVadSilenceDbOverride, vadTimer, noSpeechTimer, etc.) ist jetzt
nur noch Dead-Code — wird in einem Folge-Commit gemeinsam mit dem
zugehoerigen Settings-Slider abgeraeumt.
This commit is contained in:
@@ -1,12 +1,19 @@
|
|||||||
/**
|
/**
|
||||||
* VoiceButton - Push-to-Talk + Auto-Stop Aufnahmeknopf
|
* VoiceButton — Tap-to-Talk-Aufnahmeknopf (Streaming-Variante).
|
||||||
*
|
*
|
||||||
* Zwei Modi:
|
* Push-to-Talk gibt's nicht mehr. Tap startet Streaming-Aufnahme an die
|
||||||
* 1. Push-to-Talk: gedrueckt halten zum Aufnehmen, loslassen zum Senden
|
* Whisper-Bridge. Tap nochmal sendet stt_stream_end → Whisper liefert den
|
||||||
* 2. Tap-to-Talk: einmal tippen startet Aufnahme, VAD stoppt automatisch bei Stille
|
* finalen Text → aria-bridge forwardet direkt an Brain. Keine dB/VAD-
|
||||||
* (auch genutzt fuer Wake-Word-getriggerte Aufnahme)
|
* Stille-Erkennung mehr — Whisper hoert auf semantische Stille (kein
|
||||||
|
* neuer Text mehr).
|
||||||
*
|
*
|
||||||
* Visuelles Feedback durch pulsierende Animation waehrend der Aufnahme.
|
* Diese Komponente ist absichtlich "dumm": sie kapselt nur den
|
||||||
|
* Tap-Lifecycle + die Animation. Recording-Optionen (voice/speed/
|
||||||
|
* location/interrupted) baut ChatScreen, die User-Bubble ebenfalls.
|
||||||
|
*
|
||||||
|
* Visuelles Feedback: pulsierende Animation + Dauer + dB-Pegel via
|
||||||
|
* audioService.onMeterUpdate (das macht audio.ts noch fuer alte Records;
|
||||||
|
* neu kommt der Pegel via NativeEventEmitter (PcmStreamMeter) — folgt).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import React, { useState, useRef, useEffect, useCallback } from 'react';
|
import React, { useState, useRef, useEffect, useCallback } from 'react';
|
||||||
@@ -17,25 +24,28 @@ import {
|
|||||||
StyleSheet,
|
StyleSheet,
|
||||||
Easing,
|
Easing,
|
||||||
TouchableOpacity,
|
TouchableOpacity,
|
||||||
Pressable,
|
|
||||||
} from 'react-native';
|
} from 'react-native';
|
||||||
import audioService, { RecordingResult } from '../services/audio';
|
import audioService, { RecordingState } from '../services/audio';
|
||||||
|
|
||||||
// --- Typen ---
|
// --- Typen ---
|
||||||
|
|
||||||
interface VoiceButtonProps {
|
interface VoiceButtonProps {
|
||||||
/** Wird aufgerufen wenn die Aufnahme fertig ist */
|
/** User hat getippt — ChatScreen soll Bubble bauen + startStreamingRecording.
|
||||||
onRecordingComplete: (result: RecordingResult) => void;
|
* Returns true wenn die Aufnahme tatsaechlich gestartet ist. */
|
||||||
|
onTapStart: () => Promise<boolean>;
|
||||||
|
/** User hat nochmal getippt — ChatScreen soll stopStreamingRecording rufen. */
|
||||||
|
onTapStop: () => Promise<void>;
|
||||||
/** Button deaktivieren */
|
/** Button deaktivieren */
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
/** Wake-Word-Modus aktiv (zeigt Indikator) */
|
/** Wake-Word-Modus aktiv (zeigt gruenen Indikator-Dot) */
|
||||||
wakeWordActive?: boolean;
|
wakeWordActive?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Komponente ---
|
// --- Komponente ---
|
||||||
|
|
||||||
const VoiceButton: React.FC<VoiceButtonProps> = ({
|
const VoiceButton: React.FC<VoiceButtonProps> = ({
|
||||||
onRecordingComplete,
|
onTapStart,
|
||||||
|
onTapStop,
|
||||||
disabled = false,
|
disabled = false,
|
||||||
wakeWordActive = false,
|
wakeWordActive = false,
|
||||||
}) => {
|
}) => {
|
||||||
@@ -45,6 +55,21 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
const pulseAnim = useRef(new Animated.Value(1)).current;
|
const pulseAnim = useRef(new Animated.Value(1)).current;
|
||||||
const durationTimer = useRef<ReturnType<typeof setInterval> | null>(null);
|
const durationTimer = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
|
||||||
|
// State via audioService.onStateChange spiegeln — der Service ist die
|
||||||
|
// Quelle der Wahrheit (Streaming-Session, Wake-Word-Multi-Turn, etc.
|
||||||
|
// koennen den Recording-State von extern aendern). isStreamingRecording
|
||||||
|
// ist auch true wenn die Wake-Word-Konversation gerade aufzeichnet —
|
||||||
|
// dann zeigt der Button "stop"-Symbol, und Tap stoppt die laufende
|
||||||
|
// Aufnahme (egal ob via Wake-Word oder Knopf gestartet).
|
||||||
|
useEffect(() => {
|
||||||
|
const unsub = audioService.onStateChange((next: RecordingState) => {
|
||||||
|
setIsRecording(next === 'recording');
|
||||||
|
});
|
||||||
|
// Initial-State synchronisieren
|
||||||
|
setIsRecording(audioService.getRecordingState() === 'recording');
|
||||||
|
return unsub;
|
||||||
|
}, []);
|
||||||
|
|
||||||
// Puls-Animation starten/stoppen
|
// Puls-Animation starten/stoppen
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
@@ -71,14 +96,13 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
}
|
}
|
||||||
}, [isRecording, pulseAnim]);
|
}, [isRecording, pulseAnim]);
|
||||||
|
|
||||||
// Aufnahmedauer zaehlen + Metering
|
// Aufnahmedauer zaehlen + Metering (Pegel-Bar)
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
setDurationMs(0);
|
setDurationMs(0);
|
||||||
durationTimer.current = setInterval(() => {
|
durationTimer.current = setInterval(() => {
|
||||||
setDurationMs(prev => prev + 100);
|
setDurationMs(prev => prev + 100);
|
||||||
}, 100);
|
}, 100);
|
||||||
|
|
||||||
const unsubMeter = audioService.onMeterUpdate(setMeterDb);
|
const unsubMeter = audioService.onMeterUpdate(setMeterDb);
|
||||||
return () => {
|
return () => {
|
||||||
unsubMeter();
|
unsubMeter();
|
||||||
@@ -89,74 +113,28 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
clearInterval(durationTimer.current);
|
clearInterval(durationTimer.current);
|
||||||
durationTimer.current = null;
|
durationTimer.current = null;
|
||||||
}
|
}
|
||||||
|
setMeterDb(-160);
|
||||||
}
|
}
|
||||||
}, [isRecording]);
|
}, [isRecording]);
|
||||||
|
|
||||||
// VAD Silence Callback — Auto-Stop.
|
// Tap-Handler. Guard gegen Doppel-Tap waehrend asyncer Start/Stop.
|
||||||
// WICHTIG: NICHT auf isRecording prüfen (Closure ist stale) — stattdessen
|
|
||||||
// audioService selber fragen. Empty deps → Listener wird EINMAL registriert.
|
|
||||||
// audioService garantiert jetzt dass der Callback pro Aufnahme nur einmal
|
|
||||||
// feuert (silenceFired-Latch).
|
|
||||||
const onCompleteRef = useRef(onRecordingComplete);
|
|
||||||
useEffect(() => { onCompleteRef.current = onRecordingComplete; }, [onRecordingComplete]);
|
|
||||||
useEffect(() => {
|
|
||||||
const unsubSilence = audioService.onSilenceDetected(async () => {
|
|
||||||
if (audioService.getRecordingState() !== 'recording') return;
|
|
||||||
const result = await audioService.stopRecording();
|
|
||||||
setIsRecording(false);
|
|
||||||
if (result && result.durationMs > 500) {
|
|
||||||
onCompleteRef.current(result);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return unsubSilence;
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Auto-Start fuer Wake Word (extern getriggert)
|
|
||||||
const startAutoRecording = useCallback(async () => {
|
|
||||||
if (disabled || isRecording) return;
|
|
||||||
const started = await audioService.startRecording(true); // autoStop = true
|
|
||||||
if (started) {
|
|
||||||
setIsRecording(true);
|
|
||||||
}
|
|
||||||
}, [disabled, isRecording]);
|
|
||||||
|
|
||||||
// Tap-to-Talk: Einmal tippen startet mit Auto-Stop.
|
|
||||||
// Guard gegen Doppel-Tap während asyncer Start/Stop.
|
|
||||||
const tapBusy = useRef(false);
|
const tapBusy = useRef(false);
|
||||||
const handleTap = async () => {
|
const handleTap = useCallback(async () => {
|
||||||
if (disabled || tapBusy.current) return;
|
if (disabled || tapBusy.current) return;
|
||||||
tapBusy.current = true;
|
tapBusy.current = true;
|
||||||
try {
|
try {
|
||||||
// Fragen WIR den Service, nicht den React-State (Closure kann stale sein)
|
// Service-State fragen statt React-State (Closure koennte stale sein)
|
||||||
const svcState = audioService.getRecordingState();
|
const svcState = audioService.getRecordingState();
|
||||||
if (svcState === 'recording') {
|
if (svcState === 'recording') {
|
||||||
// Aufnahme manuell stoppen
|
await onTapStop();
|
||||||
const result = await audioService.stopRecording();
|
|
||||||
setIsRecording(false);
|
|
||||||
if (result && result.durationMs > 300) {
|
|
||||||
onRecordingComplete(result);
|
|
||||||
}
|
|
||||||
} else if (svcState === 'idle') {
|
} else if (svcState === 'idle') {
|
||||||
// Aufnahme mit Auto-Stop starten
|
await onTapStart();
|
||||||
const started = await audioService.startRecording(true);
|
|
||||||
if (started) {
|
|
||||||
setIsRecording(true);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// svcState === 'processing': Stopp in progress — nichts tun, User
|
// 'processing': Stop laeuft gerade — nichts tun, User muss nochmal tippen
|
||||||
// muss nochmal tippen wenn fertig. Aber wir blockieren mit tapBusy
|
|
||||||
// kurz damit der User's UI-Feedback synchron bleibt.
|
|
||||||
} finally {
|
} finally {
|
||||||
tapBusy.current = false;
|
tapBusy.current = false;
|
||||||
}
|
}
|
||||||
};
|
}, [disabled, onTapStart, onTapStop]);
|
||||||
|
|
||||||
// Expose startAutoRecording via ref fuer Wake Word
|
|
||||||
React.useImperativeHandle(
|
|
||||||
React.createRef(),
|
|
||||||
() => ({ startAutoRecording }),
|
|
||||||
[startAutoRecording],
|
|
||||||
);
|
|
||||||
|
|
||||||
const formatDuration = (ms: number): string => {
|
const formatDuration = (ms: number): string => {
|
||||||
const seconds = Math.floor(ms / 1000);
|
const seconds = Math.floor(ms / 1000);
|
||||||
@@ -164,7 +142,11 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
return `${seconds}.${tenths}s`;
|
return `${seconds}.${tenths}s`;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Meter-Visualisierung (0-1 Skala)
|
// Meter-Visualisierung (-60..0 dB → 0..1). Bei Streaming-Mode liefert
|
||||||
|
// audio.ts (noch) keinen Pegel, also bleibt der Balken leer — wird in
|
||||||
|
// einem Folge-Commit nachgerueckt (PcmStreamRecorder-Module muss dafuer
|
||||||
|
// einen RMS-Wert mit-emitten). Tut der Streaming-Funktion keinen Abbruch,
|
||||||
|
// ist reines UI-Beiwerk.
|
||||||
const meterLevel = Math.max(0, Math.min(1, (meterDb + 60) / 60));
|
const meterLevel = Math.max(0, Math.min(1, (meterDb + 60) / 60));
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -198,9 +180,6 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
// Expose startAutoRecording fuer externe Aufrufe (Wake Word)
|
|
||||||
export type VoiceButtonHandle = { startAutoRecording: () => Promise<void> };
|
|
||||||
|
|
||||||
// --- Styles ---
|
// --- Styles ---
|
||||||
|
|
||||||
const styles = StyleSheet.create({
|
const styles = StyleSheet.create({
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ import VoiceButton from '../components/VoiceButton';
|
|||||||
import FileUpload, { FileData } from '../components/FileUpload';
|
import FileUpload, { FileData } from '../components/FileUpload';
|
||||||
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
||||||
import MessageText from '../components/MessageText';
|
import MessageText from '../components/MessageText';
|
||||||
import { RecordingResult, loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT } from '../services/audio';
|
import { loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT } from '../services/audio';
|
||||||
import Geolocation from '@react-native-community/geolocation';
|
import Geolocation from '@react-native-community/geolocation';
|
||||||
|
|
||||||
// --- Typen ---
|
// --- Typen ---
|
||||||
@@ -1761,49 +1761,59 @@ const ChatScreen: React.FC = () => {
|
|||||||
return true;
|
return true;
|
||||||
}, [agentActivity]);
|
}, [agentActivity]);
|
||||||
|
|
||||||
// Sprachaufnahme abgeschlossen
|
// Manueller Aufnahme-Knopf (VoiceButton) — Start.
|
||||||
const handleVoiceRecording = useCallback(async (result: RecordingResult) => {
|
// Streaming-Variante: PcmStreamRecorder + Whisper-ML-Endpointer ersetzen
|
||||||
// Barge-In: laufende ARIA-Aktivitaet abbrechen falls aktiv.
|
// die alte dB-VAD-Schleife. Knopf-1.-Tap startet, Knopf-2.-Tap stoppt.
|
||||||
|
// Bubble bauen wir SOFORT damit der User sofort Feedback hat — Text wird
|
||||||
|
// ueber audioRequestId-Match nachgereicht wenn whisper das Endpoint feuert.
|
||||||
|
const handleVoiceButtonStart = useCallback(async (): Promise<boolean> => {
|
||||||
|
const audioRequestId = `audio_${Date.now()}_${Math.floor(Math.random() * 100000)}`;
|
||||||
const wasInterrupted = interruptAriaIfBusy();
|
const wasInterrupted = interruptAriaIfBusy();
|
||||||
const location = await getCurrentLocation();
|
const location = await getCurrentLocation();
|
||||||
const audioRequestId = `audio_${Date.now()}_${Math.floor(Math.random() * 100000)}`;
|
|
||||||
|
|
||||||
const cmid = nextClientMsgId();
|
|
||||||
const userMsg: ChatMessage = {
|
const userMsg: ChatMessage = {
|
||||||
id: nextId(),
|
id: nextId(),
|
||||||
sender: 'user',
|
sender: 'user',
|
||||||
text: '🎙 Spracheingabe wird verarbeitet...',
|
text: '🎙 Spracheingabe wird verarbeitet...',
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
|
attachments: [{ type: 'audio', name: 'Sprachaufnahme' }],
|
||||||
audioRequestId,
|
audioRequestId,
|
||||||
clientMsgId: cmid,
|
|
||||||
deliveryStatus: connectionStateRef.current === 'connected' ? 'sending' : 'queued',
|
|
||||||
sendAttempts: 1,
|
|
||||||
};
|
};
|
||||||
setMessages(prev => capMessages([...prev, userMsg]));
|
setMessages(prev => capMessages([...prev, userMsg]));
|
||||||
|
|
||||||
dispatchWithAck(cmid, 'audio', {
|
const { ok } = await audioService.startStreamingRecording({
|
||||||
base64: result.base64,
|
audioRequestId,
|
||||||
durationMs: result.durationMs,
|
|
||||||
mimeType: result.mimeType,
|
|
||||||
voice: localXttsVoiceRef.current,
|
voice: localXttsVoiceRef.current,
|
||||||
speed: ttsSpeedRef.current,
|
speed: ttsSpeedRef.current,
|
||||||
interrupted: wasInterrupted,
|
interrupted: wasInterrupted,
|
||||||
audioRequestId,
|
location: location || null,
|
||||||
...(location && { location }),
|
// Manueller Knopf: kein no-speech-Watchdog (User kontrolliert via Tap-zum-
|
||||||
|
// Stoppen). Hard-Cap 5 Minuten als Notbremse — danach killt Whisper
|
||||||
|
// die Session auch app-seitig haben wir +2s Toleranz.
|
||||||
|
noSpeechTimeoutMs: 0,
|
||||||
|
endpointMs: 1500,
|
||||||
|
hardCapMs: 300000,
|
||||||
});
|
});
|
||||||
scheduleStaleAudioCleanup(audioRequestId, result.durationMs);
|
if (!ok) {
|
||||||
|
// Mikro nicht verfuegbar (Anruf? OpenWakeWord blockiert?) — Bubble weg.
|
||||||
|
setMessages(prev => prev.filter(m => m.audioRequestId !== audioRequestId));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
scheduleStaleAudioCleanup(audioRequestId, 60000);
|
||||||
|
return true;
|
||||||
|
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
||||||
|
|
||||||
// Manueller Mikro-Stop waehrend Wake-Word-Konversation: User hat explizit
|
// Manueller Aufnahme-Knopf — Stop. Sendet stt_stream_end an Whisper, die
|
||||||
// den Knopf gedrueckt → er moechte nicht in den automatischen Multi-Turn-
|
// dann ihrerseits den finalen Text als stt_endpoint emittiert. aria-bridge
|
||||||
// Modus, sondern nach ARIAs Antwort zurueck zu passivem Wake-Word-Lauschen.
|
// forwarded direkt an Brain. Im wake-word-conversing-Fall zusaetzlich
|
||||||
// Bei VAD-Auto-Stop (Wake-Word-Pfad) laeuft das ueber den silence-callback
|
// endConversation: User hat explizit gestoppt → kein Multi-Turn-Resume.
|
||||||
// und endet mit resume() — der manuelle Stop hier ist der "ich bin fertig"-
|
const handleVoiceButtonStop = useCallback(async (): Promise<void> => {
|
||||||
// Knopf.
|
await audioService.stopStreamingRecording('user');
|
||||||
if (wakeWordService.isConversing()) {
|
if (wakeWordService.isConversing()) {
|
||||||
console.log('[Chat] Manueller Stop in Konversation → endConversation, zurueck zu armed');
|
console.log('[Chat] Manueller Stop in Konversation → endConversation, zurueck zu armed');
|
||||||
await wakeWordService.endConversation();
|
await wakeWordService.endConversation();
|
||||||
}
|
}
|
||||||
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
}, []);
|
||||||
|
|
||||||
// Datei auswaehlen → zur Pending-Liste hinzufuegen
|
// Datei auswaehlen → zur Pending-Liste hinzufuegen
|
||||||
const handleFileSelected = useCallback(async (file: FileData) => {
|
const handleFileSelected = useCallback(async (file: FileData) => {
|
||||||
@@ -2572,7 +2582,8 @@ const ChatScreen: React.FC = () => {
|
|||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
<VoiceButton
|
<VoiceButton
|
||||||
onRecordingComplete={handleVoiceRecording}
|
onTapStart={handleVoiceButtonStart}
|
||||||
|
onTapStop={handleVoiceButtonStop}
|
||||||
disabled={connectionState !== 'connected'}
|
disabled={connectionState !== 'connected'}
|
||||||
wakeWordActive={wakeWordActive}
|
wakeWordActive={wakeWordActive}
|
||||||
/>
|
/>
|
||||||
|
|||||||
Reference in New Issue
Block a user