Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b2edee9adb | |||
| bb13477ef9 | |||
| 710e7c88d8 | |||
| b6ee5552f0 | |||
| 570eb031e0 | |||
| e9615d987e | |||
| 5e95eacd11 | |||
| ece08f0f2f | |||
| 31fd0d7f7a | |||
| 263835ad74 | |||
| ab7e9801ee | |||
| 3d001a1d03 | |||
| 91760dd2e1 | |||
| 3c2e537420 | |||
| 97b6ea1b3e | |||
| 94ee0455a2 | |||
| 0bf6d49432 | |||
| 493cba36a2 |
@@ -79,8 +79,8 @@ android {
|
|||||||
applicationId "com.ariacockpit"
|
applicationId "com.ariacockpit"
|
||||||
minSdkVersion rootProject.ext.minSdkVersion
|
minSdkVersion rootProject.ext.minSdkVersion
|
||||||
targetSdkVersion rootProject.ext.targetSdkVersion
|
targetSdkVersion rootProject.ext.targetSdkVersion
|
||||||
versionCode 10800
|
versionCode 10808
|
||||||
versionName "0.1.8.0"
|
versionName "0.1.8.8"
|
||||||
// Fallback fuer Libraries mit Product Flavors
|
// Fallback fuer Libraries mit Product Flavors
|
||||||
missingDimensionStrategy 'react-native-camera', 'general'
|
missingDimensionStrategy 'react-native-camera', 'general'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "aria-cockpit",
|
"name": "aria-cockpit",
|
||||||
"version": "0.1.8.0",
|
"version": "0.1.8.8",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"android": "react-native run-android",
|
"android": "react-native run-android",
|
||||||
|
|||||||
@@ -1,12 +1,19 @@
|
|||||||
/**
|
/**
|
||||||
* VoiceButton - Push-to-Talk + Auto-Stop Aufnahmeknopf
|
* VoiceButton — Tap-to-Talk-Aufnahmeknopf (Streaming-Variante).
|
||||||
*
|
*
|
||||||
* Zwei Modi:
|
* Push-to-Talk gibt's nicht mehr. Tap startet Streaming-Aufnahme an die
|
||||||
* 1. Push-to-Talk: gedrueckt halten zum Aufnehmen, loslassen zum Senden
|
* Whisper-Bridge. Tap nochmal sendet stt_stream_end → Whisper liefert den
|
||||||
* 2. Tap-to-Talk: einmal tippen startet Aufnahme, VAD stoppt automatisch bei Stille
|
* finalen Text → aria-bridge forwardet direkt an Brain. Keine dB/VAD-
|
||||||
* (auch genutzt fuer Wake-Word-getriggerte Aufnahme)
|
* Stille-Erkennung mehr — Whisper hoert auf semantische Stille (kein
|
||||||
|
* neuer Text mehr).
|
||||||
*
|
*
|
||||||
* Visuelles Feedback durch pulsierende Animation waehrend der Aufnahme.
|
* Diese Komponente ist absichtlich "dumm": sie kapselt nur den
|
||||||
|
* Tap-Lifecycle + die Animation. Recording-Optionen (voice/speed/
|
||||||
|
* location/interrupted) baut ChatScreen, die User-Bubble ebenfalls.
|
||||||
|
*
|
||||||
|
* Visuelles Feedback: pulsierende Animation + Dauer + dB-Pegel via
|
||||||
|
* audioService.onMeterUpdate (das macht audio.ts noch fuer alte Records;
|
||||||
|
* neu kommt der Pegel via NativeEventEmitter (PcmStreamMeter) — folgt).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import React, { useState, useRef, useEffect, useCallback } from 'react';
|
import React, { useState, useRef, useEffect, useCallback } from 'react';
|
||||||
@@ -17,25 +24,28 @@ import {
|
|||||||
StyleSheet,
|
StyleSheet,
|
||||||
Easing,
|
Easing,
|
||||||
TouchableOpacity,
|
TouchableOpacity,
|
||||||
Pressable,
|
|
||||||
} from 'react-native';
|
} from 'react-native';
|
||||||
import audioService, { RecordingResult } from '../services/audio';
|
import audioService, { RecordingState } from '../services/audio';
|
||||||
|
|
||||||
// --- Typen ---
|
// --- Typen ---
|
||||||
|
|
||||||
interface VoiceButtonProps {
|
interface VoiceButtonProps {
|
||||||
/** Wird aufgerufen wenn die Aufnahme fertig ist */
|
/** User hat getippt — ChatScreen soll Bubble bauen + startStreamingRecording.
|
||||||
onRecordingComplete: (result: RecordingResult) => void;
|
* Returns true wenn die Aufnahme tatsaechlich gestartet ist. */
|
||||||
|
onTapStart: () => Promise<boolean>;
|
||||||
|
/** User hat nochmal getippt — ChatScreen soll stopStreamingRecording rufen. */
|
||||||
|
onTapStop: () => Promise<void>;
|
||||||
/** Button deaktivieren */
|
/** Button deaktivieren */
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
/** Wake-Word-Modus aktiv (zeigt Indikator) */
|
/** Wake-Word-Modus aktiv (zeigt gruenen Indikator-Dot) */
|
||||||
wakeWordActive?: boolean;
|
wakeWordActive?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Komponente ---
|
// --- Komponente ---
|
||||||
|
|
||||||
const VoiceButton: React.FC<VoiceButtonProps> = ({
|
const VoiceButton: React.FC<VoiceButtonProps> = ({
|
||||||
onRecordingComplete,
|
onTapStart,
|
||||||
|
onTapStop,
|
||||||
disabled = false,
|
disabled = false,
|
||||||
wakeWordActive = false,
|
wakeWordActive = false,
|
||||||
}) => {
|
}) => {
|
||||||
@@ -45,6 +55,21 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
const pulseAnim = useRef(new Animated.Value(1)).current;
|
const pulseAnim = useRef(new Animated.Value(1)).current;
|
||||||
const durationTimer = useRef<ReturnType<typeof setInterval> | null>(null);
|
const durationTimer = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
|
||||||
|
// State via audioService.onStateChange spiegeln — der Service ist die
|
||||||
|
// Quelle der Wahrheit (Streaming-Session, Wake-Word-Multi-Turn, etc.
|
||||||
|
// koennen den Recording-State von extern aendern). isStreamingRecording
|
||||||
|
// ist auch true wenn die Wake-Word-Konversation gerade aufzeichnet —
|
||||||
|
// dann zeigt der Button "stop"-Symbol, und Tap stoppt die laufende
|
||||||
|
// Aufnahme (egal ob via Wake-Word oder Knopf gestartet).
|
||||||
|
useEffect(() => {
|
||||||
|
const unsub = audioService.onStateChange((next: RecordingState) => {
|
||||||
|
setIsRecording(next === 'recording');
|
||||||
|
});
|
||||||
|
// Initial-State synchronisieren
|
||||||
|
setIsRecording(audioService.getRecordingState() === 'recording');
|
||||||
|
return unsub;
|
||||||
|
}, []);
|
||||||
|
|
||||||
// Puls-Animation starten/stoppen
|
// Puls-Animation starten/stoppen
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
@@ -71,14 +96,13 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
}
|
}
|
||||||
}, [isRecording, pulseAnim]);
|
}, [isRecording, pulseAnim]);
|
||||||
|
|
||||||
// Aufnahmedauer zaehlen + Metering
|
// Aufnahmedauer zaehlen + Metering (Pegel-Bar)
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
setDurationMs(0);
|
setDurationMs(0);
|
||||||
durationTimer.current = setInterval(() => {
|
durationTimer.current = setInterval(() => {
|
||||||
setDurationMs(prev => prev + 100);
|
setDurationMs(prev => prev + 100);
|
||||||
}, 100);
|
}, 100);
|
||||||
|
|
||||||
const unsubMeter = audioService.onMeterUpdate(setMeterDb);
|
const unsubMeter = audioService.onMeterUpdate(setMeterDb);
|
||||||
return () => {
|
return () => {
|
||||||
unsubMeter();
|
unsubMeter();
|
||||||
@@ -89,74 +113,28 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
clearInterval(durationTimer.current);
|
clearInterval(durationTimer.current);
|
||||||
durationTimer.current = null;
|
durationTimer.current = null;
|
||||||
}
|
}
|
||||||
|
setMeterDb(-160);
|
||||||
}
|
}
|
||||||
}, [isRecording]);
|
}, [isRecording]);
|
||||||
|
|
||||||
// VAD Silence Callback — Auto-Stop.
|
// Tap-Handler. Guard gegen Doppel-Tap waehrend asyncer Start/Stop.
|
||||||
// WICHTIG: NICHT auf isRecording prüfen (Closure ist stale) — stattdessen
|
|
||||||
// audioService selber fragen. Empty deps → Listener wird EINMAL registriert.
|
|
||||||
// audioService garantiert jetzt dass der Callback pro Aufnahme nur einmal
|
|
||||||
// feuert (silenceFired-Latch).
|
|
||||||
const onCompleteRef = useRef(onRecordingComplete);
|
|
||||||
useEffect(() => { onCompleteRef.current = onRecordingComplete; }, [onRecordingComplete]);
|
|
||||||
useEffect(() => {
|
|
||||||
const unsubSilence = audioService.onSilenceDetected(async () => {
|
|
||||||
if (audioService.getRecordingState() !== 'recording') return;
|
|
||||||
const result = await audioService.stopRecording();
|
|
||||||
setIsRecording(false);
|
|
||||||
if (result && result.durationMs > 500) {
|
|
||||||
onCompleteRef.current(result);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return unsubSilence;
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Auto-Start fuer Wake Word (extern getriggert)
|
|
||||||
const startAutoRecording = useCallback(async () => {
|
|
||||||
if (disabled || isRecording) return;
|
|
||||||
const started = await audioService.startRecording(true); // autoStop = true
|
|
||||||
if (started) {
|
|
||||||
setIsRecording(true);
|
|
||||||
}
|
|
||||||
}, [disabled, isRecording]);
|
|
||||||
|
|
||||||
// Tap-to-Talk: Einmal tippen startet mit Auto-Stop.
|
|
||||||
// Guard gegen Doppel-Tap während asyncer Start/Stop.
|
|
||||||
const tapBusy = useRef(false);
|
const tapBusy = useRef(false);
|
||||||
const handleTap = async () => {
|
const handleTap = useCallback(async () => {
|
||||||
if (disabled || tapBusy.current) return;
|
if (disabled || tapBusy.current) return;
|
||||||
tapBusy.current = true;
|
tapBusy.current = true;
|
||||||
try {
|
try {
|
||||||
// Fragen WIR den Service, nicht den React-State (Closure kann stale sein)
|
// Service-State fragen statt React-State (Closure koennte stale sein)
|
||||||
const svcState = audioService.getRecordingState();
|
const svcState = audioService.getRecordingState();
|
||||||
if (svcState === 'recording') {
|
if (svcState === 'recording') {
|
||||||
// Aufnahme manuell stoppen
|
await onTapStop();
|
||||||
const result = await audioService.stopRecording();
|
|
||||||
setIsRecording(false);
|
|
||||||
if (result && result.durationMs > 300) {
|
|
||||||
onRecordingComplete(result);
|
|
||||||
}
|
|
||||||
} else if (svcState === 'idle') {
|
} else if (svcState === 'idle') {
|
||||||
// Aufnahme mit Auto-Stop starten
|
await onTapStart();
|
||||||
const started = await audioService.startRecording(true);
|
|
||||||
if (started) {
|
|
||||||
setIsRecording(true);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// svcState === 'processing': Stopp in progress — nichts tun, User
|
// 'processing': Stop laeuft gerade — nichts tun, User muss nochmal tippen
|
||||||
// muss nochmal tippen wenn fertig. Aber wir blockieren mit tapBusy
|
|
||||||
// kurz damit der User's UI-Feedback synchron bleibt.
|
|
||||||
} finally {
|
} finally {
|
||||||
tapBusy.current = false;
|
tapBusy.current = false;
|
||||||
}
|
}
|
||||||
};
|
}, [disabled, onTapStart, onTapStop]);
|
||||||
|
|
||||||
// Expose startAutoRecording via ref fuer Wake Word
|
|
||||||
React.useImperativeHandle(
|
|
||||||
React.createRef(),
|
|
||||||
() => ({ startAutoRecording }),
|
|
||||||
[startAutoRecording],
|
|
||||||
);
|
|
||||||
|
|
||||||
const formatDuration = (ms: number): string => {
|
const formatDuration = (ms: number): string => {
|
||||||
const seconds = Math.floor(ms / 1000);
|
const seconds = Math.floor(ms / 1000);
|
||||||
@@ -164,7 +142,11 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
return `${seconds}.${tenths}s`;
|
return `${seconds}.${tenths}s`;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Meter-Visualisierung (0-1 Skala)
|
// Meter-Visualisierung (-60..0 dB → 0..1). Bei Streaming-Mode liefert
|
||||||
|
// audio.ts (noch) keinen Pegel, also bleibt der Balken leer — wird in
|
||||||
|
// einem Folge-Commit nachgerueckt (PcmStreamRecorder-Module muss dafuer
|
||||||
|
// einen RMS-Wert mit-emitten). Tut der Streaming-Funktion keinen Abbruch,
|
||||||
|
// ist reines UI-Beiwerk.
|
||||||
const meterLevel = Math.max(0, Math.min(1, (meterDb + 60) / 60));
|
const meterLevel = Math.max(0, Math.min(1, (meterDb + 60) / 60));
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -198,9 +180,6 @@ const VoiceButton: React.FC<VoiceButtonProps> = ({
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
// Expose startAutoRecording fuer externe Aufrufe (Wake Word)
|
|
||||||
export type VoiceButtonHandle = { startAutoRecording: () => Promise<void> };
|
|
||||||
|
|
||||||
// --- Styles ---
|
// --- Styles ---
|
||||||
|
|
||||||
const styles = StyleSheet.create({
|
const styles = StyleSheet.create({
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ import VoiceButton from '../components/VoiceButton';
|
|||||||
import FileUpload, { FileData } from '../components/FileUpload';
|
import FileUpload, { FileData } from '../components/FileUpload';
|
||||||
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
||||||
import MessageText from '../components/MessageText';
|
import MessageText from '../components/MessageText';
|
||||||
import { RecordingResult, loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT } from '../services/audio';
|
import { loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT } from '../services/audio';
|
||||||
import Geolocation from '@react-native-community/geolocation';
|
import Geolocation from '@react-native-community/geolocation';
|
||||||
|
|
||||||
// --- Typen ---
|
// --- Typen ---
|
||||||
@@ -1263,11 +1263,30 @@ const ChatScreen: React.FC = () => {
|
|||||||
return () => { unsubUpdate(); clearTimeout(timer); };
|
return () => { unsubUpdate(); clearTimeout(timer); };
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Gespraechsmodus: Nach TTS-Wiedergabe automatisch Aufnahme starten
|
// Gespraechsmodus: Nach TTS-Wiedergabe weiter im Multi-Turn (Conversation-
|
||||||
|
// Window) oder zurueck zu armed (Wake-Word lauscht wieder)?
|
||||||
|
//
|
||||||
|
// Foreground → resume() oeffnet das Mikro fuer N Sekunden Follow-Up
|
||||||
|
// (natuerlicher Dialog moeglich ohne erneutes "Computer")
|
||||||
|
// Background → endConversation() — Wake-Word direkt wieder armed.
|
||||||
|
//
|
||||||
|
// Grund: der setTimeout(800ms) in resume() wird im Doze stark verzoegert
|
||||||
|
// (siehe Wake-Detect-Bug von 0.1.7.0). Das hat zwei nervige Folgen:
|
||||||
|
// 1) Wake-Word ist solange "tot" — User kann ARIA nicht mehr triggern
|
||||||
|
// bis er die App vorholt
|
||||||
|
// 2) Wenn er die App dann vorholt, oeffnet der verspaetete Timer das
|
||||||
|
// Mikro — sieht aus wie ein Phantom-Wake-Word-Trigger
|
||||||
|
// Background = User nutzt das Handy anderweitig, das Multi-Turn-Konzept
|
||||||
|
// ist da eh nicht nuetzlich. Direkt re-armen ist robust und erwartungs-
|
||||||
|
// konform.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const unsubPlayback = audioService.onPlaybackFinished(() => {
|
const unsubPlayback = audioService.onPlaybackFinished(() => {
|
||||||
if (wakeWordService.isActive()) {
|
if (!wakeWordService.isActive()) return;
|
||||||
|
if (AppState.currentState === 'active') {
|
||||||
wakeWordService.resume();
|
wakeWordService.resume();
|
||||||
|
} else {
|
||||||
|
console.log('[Chat] TTS fertig im Background → endConversation (kein Multi-Turn)');
|
||||||
|
wakeWordService.endConversation().catch(() => {});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
return () => unsubPlayback();
|
return () => unsubPlayback();
|
||||||
@@ -1761,49 +1780,59 @@ const ChatScreen: React.FC = () => {
|
|||||||
return true;
|
return true;
|
||||||
}, [agentActivity]);
|
}, [agentActivity]);
|
||||||
|
|
||||||
// Sprachaufnahme abgeschlossen
|
// Manueller Aufnahme-Knopf (VoiceButton) — Start.
|
||||||
const handleVoiceRecording = useCallback(async (result: RecordingResult) => {
|
// Streaming-Variante: PcmStreamRecorder + Whisper-ML-Endpointer ersetzen
|
||||||
// Barge-In: laufende ARIA-Aktivitaet abbrechen falls aktiv.
|
// die alte dB-VAD-Schleife. Knopf-1.-Tap startet, Knopf-2.-Tap stoppt.
|
||||||
|
// Bubble bauen wir SOFORT damit der User sofort Feedback hat — Text wird
|
||||||
|
// ueber audioRequestId-Match nachgereicht wenn whisper das Endpoint feuert.
|
||||||
|
const handleVoiceButtonStart = useCallback(async (): Promise<boolean> => {
|
||||||
|
const audioRequestId = `audio_${Date.now()}_${Math.floor(Math.random() * 100000)}`;
|
||||||
const wasInterrupted = interruptAriaIfBusy();
|
const wasInterrupted = interruptAriaIfBusy();
|
||||||
const location = await getCurrentLocation();
|
const location = await getCurrentLocation();
|
||||||
const audioRequestId = `audio_${Date.now()}_${Math.floor(Math.random() * 100000)}`;
|
|
||||||
|
|
||||||
const cmid = nextClientMsgId();
|
|
||||||
const userMsg: ChatMessage = {
|
const userMsg: ChatMessage = {
|
||||||
id: nextId(),
|
id: nextId(),
|
||||||
sender: 'user',
|
sender: 'user',
|
||||||
text: '🎙 Spracheingabe wird verarbeitet...',
|
text: '🎙 Spracheingabe wird verarbeitet...',
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
|
attachments: [{ type: 'audio', name: 'Sprachaufnahme' }],
|
||||||
audioRequestId,
|
audioRequestId,
|
||||||
clientMsgId: cmid,
|
|
||||||
deliveryStatus: connectionStateRef.current === 'connected' ? 'sending' : 'queued',
|
|
||||||
sendAttempts: 1,
|
|
||||||
};
|
};
|
||||||
setMessages(prev => capMessages([...prev, userMsg]));
|
setMessages(prev => capMessages([...prev, userMsg]));
|
||||||
|
|
||||||
dispatchWithAck(cmid, 'audio', {
|
const { ok } = await audioService.startStreamingRecording({
|
||||||
base64: result.base64,
|
audioRequestId,
|
||||||
durationMs: result.durationMs,
|
|
||||||
mimeType: result.mimeType,
|
|
||||||
voice: localXttsVoiceRef.current,
|
voice: localXttsVoiceRef.current,
|
||||||
speed: ttsSpeedRef.current,
|
speed: ttsSpeedRef.current,
|
||||||
interrupted: wasInterrupted,
|
interrupted: wasInterrupted,
|
||||||
audioRequestId,
|
location: location || null,
|
||||||
...(location && { location }),
|
// Manueller Knopf: kein no-speech-Watchdog (User kontrolliert via Tap-zum-
|
||||||
|
// Stoppen). Hard-Cap 5 Minuten als Notbremse — danach killt Whisper
|
||||||
|
// die Session auch app-seitig haben wir +2s Toleranz.
|
||||||
|
noSpeechTimeoutMs: 0,
|
||||||
|
endpointMs: 1500,
|
||||||
|
hardCapMs: 300000,
|
||||||
});
|
});
|
||||||
scheduleStaleAudioCleanup(audioRequestId, result.durationMs);
|
if (!ok) {
|
||||||
|
// Mikro nicht verfuegbar (Anruf? OpenWakeWord blockiert?) — Bubble weg.
|
||||||
|
setMessages(prev => prev.filter(m => m.audioRequestId !== audioRequestId));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
scheduleStaleAudioCleanup(audioRequestId, 60000);
|
||||||
|
return true;
|
||||||
|
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
||||||
|
|
||||||
// Manueller Mikro-Stop waehrend Wake-Word-Konversation: User hat explizit
|
// Manueller Aufnahme-Knopf — Stop. Sendet stt_stream_end an Whisper, die
|
||||||
// den Knopf gedrueckt → er moechte nicht in den automatischen Multi-Turn-
|
// dann ihrerseits den finalen Text als stt_endpoint emittiert. aria-bridge
|
||||||
// Modus, sondern nach ARIAs Antwort zurueck zu passivem Wake-Word-Lauschen.
|
// forwarded direkt an Brain. Im wake-word-conversing-Fall zusaetzlich
|
||||||
// Bei VAD-Auto-Stop (Wake-Word-Pfad) laeuft das ueber den silence-callback
|
// endConversation: User hat explizit gestoppt → kein Multi-Turn-Resume.
|
||||||
// und endet mit resume() — der manuelle Stop hier ist der "ich bin fertig"-
|
const handleVoiceButtonStop = useCallback(async (): Promise<void> => {
|
||||||
// Knopf.
|
await audioService.stopStreamingRecording('user');
|
||||||
if (wakeWordService.isConversing()) {
|
if (wakeWordService.isConversing()) {
|
||||||
console.log('[Chat] Manueller Stop in Konversation → endConversation, zurueck zu armed');
|
console.log('[Chat] Manueller Stop in Konversation → endConversation, zurueck zu armed');
|
||||||
await wakeWordService.endConversation();
|
await wakeWordService.endConversation();
|
||||||
}
|
}
|
||||||
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
}, []);
|
||||||
|
|
||||||
// Datei auswaehlen → zur Pending-Liste hinzufuegen
|
// Datei auswaehlen → zur Pending-Liste hinzufuegen
|
||||||
const handleFileSelected = useCallback(async (file: FileData) => {
|
const handleFileSelected = useCallback(async (file: FileData) => {
|
||||||
@@ -2572,7 +2601,8 @@ const ChatScreen: React.FC = () => {
|
|||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
<VoiceButton
|
<VoiceButton
|
||||||
onRecordingComplete={handleVoiceRecording}
|
onTapStart={handleVoiceButtonStart}
|
||||||
|
onTapStop={handleVoiceButtonStop}
|
||||||
disabled={connectionState !== 'connected'}
|
disabled={connectionState !== 'connected'}
|
||||||
wakeWordActive={wakeWordActive}
|
wakeWordActive={wakeWordActive}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -497,6 +497,49 @@ const SettingsScreen: React.FC = () => {
|
|||||||
})();
|
})();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Datei-Manager: Einzel-Datei-Download. ChatScreen subscribet auch auf
|
||||||
|
// file_response — der versucht aber nur Chat-Bubble-Attachments zu
|
||||||
|
// patchen und macht nix wenn die requestId nicht zu einer Nachricht
|
||||||
|
// passt. Hier behandeln wir die Manager-initiierten Downloads
|
||||||
|
// (requestId-Praefix 'single-' aus bulkDownload). Schreibt nach
|
||||||
|
// ~/Download/ wie der ZIP-Pfad.
|
||||||
|
if (message.type === ('file_response' as any)) {
|
||||||
|
const p: any = message.payload || {};
|
||||||
|
const reqId = (p.requestId as string) || '';
|
||||||
|
if (!reqId.startsWith('single-')) return; // nicht unsere Anfrage
|
||||||
|
if (p.error) {
|
||||||
|
ToastAndroid.show('Download fehlgeschlagen: ' + p.error, ToastAndroid.LONG);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const b64 = (p.base64 as string) || '';
|
||||||
|
if (!b64) return;
|
||||||
|
const fileName = (p.name as string) ||
|
||||||
|
(p.serverPath as string || '').split('/').pop() ||
|
||||||
|
'aria-download';
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const dir = RNFS.DownloadDirectoryPath;
|
||||||
|
const filePath = `${dir}/${fileName}`;
|
||||||
|
// Falls Datei schon existiert: Suffix anhaengen damit nichts
|
||||||
|
// ueberschrieben wird.
|
||||||
|
let target = filePath;
|
||||||
|
let i = 1;
|
||||||
|
while (await RNFS.exists(target)) {
|
||||||
|
const dot = fileName.lastIndexOf('.');
|
||||||
|
const base = dot > 0 ? fileName.slice(0, dot) : fileName;
|
||||||
|
const ext = dot > 0 ? fileName.slice(dot) : '';
|
||||||
|
target = `${dir}/${base} (${i})${ext}`;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
await RNFS.writeFile(target, b64, 'base64');
|
||||||
|
const sizeKb = Math.round(((b64.length * 0.75)) / 1024);
|
||||||
|
ToastAndroid.show(`Gespeichert: ${target.split('/').pop()} (${sizeKb} KB)`, ToastAndroid.LONG);
|
||||||
|
} catch (e: any) {
|
||||||
|
ToastAndroid.show('Speichern fehlgeschlagen: ' + e.message, ToastAndroid.LONG);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
// Voice wurde gespeichert → Liste neu laden + ggf. auswaehlen
|
// Voice wurde gespeichert → Liste neu laden + ggf. auswaehlen
|
||||||
if (message.type === ('xtts_voice_saved' as any)) {
|
if (message.type === ('xtts_voice_saved' as any)) {
|
||||||
const name = (message.payload as any).name as string;
|
const name = (message.payload as any).name as string;
|
||||||
|
|||||||
@@ -312,6 +312,10 @@ class AudioService {
|
|||||||
// lich Chunks einer alten Session in eine neue mischen.
|
// lich Chunks einer alten Session in eine neue mischen.
|
||||||
private streamRequestId: string = '';
|
private streamRequestId: string = '';
|
||||||
private streamAudioRequestId: string = '';
|
private streamAudioRequestId: string = '';
|
||||||
|
// Latch: ist endpointListeners fuer den aktuellen Session-Cycle schon gefeuert
|
||||||
|
// worden? Wird auf false gesetzt beim startStreamingRecording, auf true beim
|
||||||
|
// ersten Endpoint (egal ob via RVS oder Fallback). Verhindert Doppel-Fires.
|
||||||
|
private streamEndpointFired: boolean = false;
|
||||||
// Subscriber-Handles fuer Native-Events + RVS-Listener (cleanup beim stop)
|
// Subscriber-Handles fuer Native-Events + RVS-Listener (cleanup beim stop)
|
||||||
private streamPcmChunkSub: { remove: () => void } | null = null;
|
private streamPcmChunkSub: { remove: () => void } | null = null;
|
||||||
private streamPcmErrorSub: { remove: () => void } | null = null;
|
private streamPcmErrorSub: { remove: () => void } | null = null;
|
||||||
@@ -337,8 +341,21 @@ class AudioService {
|
|||||||
try {
|
try {
|
||||||
const emitter = new NativeEventEmitter(NativeModules.PcmStreamPlayer as any);
|
const emitter = new NativeEventEmitter(NativeModules.PcmStreamPlayer as any);
|
||||||
emitter.addListener('PcmPlaybackFinished', () => {
|
emitter.addListener('PcmPlaybackFinished', () => {
|
||||||
console.log('[Audio] PcmPlaybackFinished — Focus jetzt freigeben');
|
console.log('[Audio] PcmPlaybackFinished — AudioTrack drained');
|
||||||
this._releaseFocusDeferred();
|
this._releaseFocusDeferred();
|
||||||
|
// Erst HIER playbackFinished-Listener feuern — nicht schon beim
|
||||||
|
// Empfang des letzten PCM-Chunks (siehe handlePcmChunk). AudioTrack
|
||||||
|
// braucht nach end() noch 1-2s zum Drainen seines Hardware-Buffers.
|
||||||
|
// Wenn wir die Listener zu frueh feuern, re-armt OpenWakeWord
|
||||||
|
// waehrend ARIA noch hoerbar spricht → ARIAs Stimme verwirrt die
|
||||||
|
// Wake-Word-Detection (kein gemeinsames AEC zwischen AudioTrack-
|
||||||
|
// und AudioRecord-Session). Stefan-Reproduktion: nach jeder ARIA-
|
||||||
|
// Antwort schluckte das Wake-Word den naechsten Trigger.
|
||||||
|
import('./logger').then(m => m.reportAppDebug('audio.playback',
|
||||||
|
'PcmPlaybackFinished native event → fire listeners')).catch(()=>{});
|
||||||
|
this.playbackFinishedListeners.forEach(cb => {
|
||||||
|
try { cb(); } catch (e) { console.warn('[Audio] playbackFinished cb err:', e); }
|
||||||
|
});
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn('[Audio] PcmPlaybackFinished-Subscription fehlgeschlagen:', err);
|
console.warn('[Audio] PcmPlaybackFinished-Subscription fehlgeschlagen:', err);
|
||||||
@@ -389,10 +406,8 @@ class AudioService {
|
|||||||
// Wir stoppen die Aufnahme — whisper hat alles was es braucht.
|
// Wir stoppen die Aufnahme — whisper hat alles was es braucht.
|
||||||
// Kein stt_stream_end senden: das Endpoint kam von der Bridge,
|
// Kein stt_stream_end senden: das Endpoint kam von der Bridge,
|
||||||
// sie hat schon finalisiert.
|
// sie hat schon finalisiert.
|
||||||
|
this._fireEndpoint(ev);
|
||||||
this._cleanupStreamLocal('endpoint');
|
this._cleanupStreamLocal('endpoint');
|
||||||
this.endpointListeners.forEach(cb => {
|
|
||||||
try { cb(ev); } catch (e) { console.warn('[Audio] endpoint listener err:', e); }
|
|
||||||
});
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (t === 'stt_stream_done') {
|
if (t === 'stt_stream_done') {
|
||||||
@@ -979,6 +994,7 @@ class AudioService {
|
|||||||
this.streamRequestId = requestId;
|
this.streamRequestId = requestId;
|
||||||
this.streamAudioRequestId = opts.audioRequestId || '';
|
this.streamAudioRequestId = opts.audioRequestId || '';
|
||||||
this.streamGotPartial = false;
|
this.streamGotPartial = false;
|
||||||
|
this.streamEndpointFired = false;
|
||||||
this.recordingStartTime = Date.now();
|
this.recordingStartTime = Date.now();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -1066,10 +1082,17 @@ class AudioService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Sauberer User-initiated Stop. Sendet stt_stream_end an die Bridge,
|
/** Sauberer User-initiated Stop. Sendet stt_stream_end an die Bridge,
|
||||||
* die noch ihren Final-Transcribe macht. */
|
* die noch ihren Final-Transcribe macht.
|
||||||
|
*
|
||||||
|
* Plus: Fallback-Timer (3s). Wenn die Bridge nicht antwortet (z.B. weil
|
||||||
|
* veraltete Version ohne Streaming-Handler laeuft), feuern wir den
|
||||||
|
* Endpoint-Listener trotzdem mit text='' damit die App-UI nicht in
|
||||||
|
* "wird verarbeitet..." haengt. ChatScreen behandelt das wie den
|
||||||
|
* No-Speech-Fall (Bubble weg + endConversation). */
|
||||||
async stopStreamingRecording(reason: string = 'user'): Promise<void> {
|
async stopStreamingRecording(reason: string = 'user'): Promise<void> {
|
||||||
const reqId = this.streamRequestId;
|
const reqId = this.streamRequestId;
|
||||||
if (!reqId) return;
|
if (!reqId) return;
|
||||||
|
const audioReqId = this.streamAudioRequestId;
|
||||||
try {
|
try {
|
||||||
rvs.send('stt_stream_end' as any, { requestId: reqId, reason });
|
rvs.send('stt_stream_end' as any, { requestId: reqId, reason });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -1078,6 +1101,21 @@ class AudioService {
|
|||||||
// Recorder lokal abschalten — Bridge feuert dann ihrerseits noch
|
// Recorder lokal abschalten — Bridge feuert dann ihrerseits noch
|
||||||
// stt_endpoint + stt_stream_done.
|
// stt_endpoint + stt_stream_done.
|
||||||
this._cleanupStreamLocal(`stop:${reason}`);
|
this._cleanupStreamLocal(`stop:${reason}`);
|
||||||
|
// Fallback-Watchdog: nach 3s noch immer kein Endpoint via RVS angekommen
|
||||||
|
// → _fireEndpoint mit text='' (idempotent via streamEndpointFired-Latch,
|
||||||
|
// d.h. wenn echtes stt_endpoint zwischen jetzt und +3s ankommt feuert
|
||||||
|
// dieser Fallback NICHT).
|
||||||
|
setTimeout(() => {
|
||||||
|
if (this.streamEndpointFired) return;
|
||||||
|
console.log('[Audio] stopStreamingRecording: 3s ohne Bridge-Antwort — fallback fire');
|
||||||
|
this._fireEndpoint({
|
||||||
|
audioRequestId: audioReqId,
|
||||||
|
text: '',
|
||||||
|
reason: `stop:${reason}:no-response`,
|
||||||
|
durationS: 0,
|
||||||
|
sttMs: 0,
|
||||||
|
});
|
||||||
|
}, 3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Abbruch ohne dass Brain den Text verarbeitet — z.B. wenn der User
|
/** Abbruch ohne dass Brain den Text verarbeitet — z.B. wenn der User
|
||||||
@@ -1095,15 +1133,23 @@ class AudioService {
|
|||||||
} catch {}
|
} catch {}
|
||||||
this._cleanupStreamLocal(`cancel:${reason}`);
|
this._cleanupStreamLocal(`cancel:${reason}`);
|
||||||
// Listener feuern damit ChatScreen reagieren kann (endConversation etc.)
|
// Listener feuern damit ChatScreen reagieren kann (endConversation etc.)
|
||||||
const ev: SttEndpointEvent = {
|
this._fireEndpoint({
|
||||||
audioRequestId: audioReqId,
|
audioRequestId: audioReqId,
|
||||||
text: '',
|
text: '',
|
||||||
reason: `cancel:${reason}`,
|
reason: `cancel:${reason}`,
|
||||||
durationS: 0,
|
durationS: 0,
|
||||||
sttMs: 0,
|
sttMs: 0,
|
||||||
};
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Feuert den Endpoint-Listener — aber nur einmal pro Session-Cycle.
|
||||||
|
* Wird sowohl vom RVS-stt_endpoint-Pfad als auch vom Fallback-Watchdog
|
||||||
|
* und cancelStreamingRecording aufgerufen. */
|
||||||
|
private _fireEndpoint(ev: SttEndpointEvent): void {
|
||||||
|
if (this.streamEndpointFired) return;
|
||||||
|
this.streamEndpointFired = true;
|
||||||
this.endpointListeners.forEach(cb => {
|
this.endpointListeners.forEach(cb => {
|
||||||
try { cb(ev); } catch (e) { console.warn('[Audio] endpoint listener (cancel) err:', e); }
|
try { cb(ev); } catch (e) { console.warn('[Audio] endpoint listener err:', e); }
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1335,12 +1381,13 @@ class AudioService {
|
|||||||
// releasen den AudioFocus NICHT hier — der writer braucht u.U. noch
|
// releasen den AudioFocus NICHT hier — der writer braucht u.U. noch
|
||||||
// 30+ Sekunden bis der Buffer wirklich abgespielt ist. Den release
|
// 30+ Sekunden bis der Buffer wirklich abgespielt ist. Den release
|
||||||
// triggert das native Event "PcmPlaybackFinished" wenn AudioTrack
|
// triggert das native Event "PcmPlaybackFinished" wenn AudioTrack
|
||||||
// wirklich am Ende ist (siehe ensurePlaybackFinishedListener).
|
// wirklich am Ende ist (siehe Constructor-PcmPlaybackFinished-Handler).
|
||||||
|
//
|
||||||
|
// playbackFinishedListeners feuern AUCH erst dort — frueher feuerten
|
||||||
|
// sie hier (beim Eintreffen des letzten Chunks), das fuehrte zu
|
||||||
|
// einem Race: OpenWakeWord re-armte waehrend AudioTrack noch hoerbar
|
||||||
|
// ARIAs Stimme abspielte → naechstes Wake-Word ging unter.
|
||||||
try { await PcmStreamPlayer!.end(); } catch {}
|
try { await PcmStreamPlayer!.end(); } catch {}
|
||||||
// playbackFinished-Listener informieren (UI-Logik)
|
|
||||||
this.playbackFinishedListeners.forEach(cb => {
|
|
||||||
try { cb(); } catch (e) { console.warn('[Audio] playbackFinished cb err:', e); }
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
this.pcmStreamActive = false;
|
this.pcmStreamActive = false;
|
||||||
|
|
||||||
|
|||||||
@@ -344,21 +344,51 @@ class WakeWordService {
|
|||||||
/** Konversation beenden — User hat im Window nichts gesagt.
|
/** Konversation beenden — User hat im Window nichts gesagt.
|
||||||
* Mit Wake-Word: zurueck zu 'armed' (Listener wieder an).
|
* Mit Wake-Word: zurueck zu 'armed' (Listener wieder an).
|
||||||
* Ohne: zurueck zu 'off'.
|
* Ohne: zurueck zu 'off'.
|
||||||
|
*
|
||||||
|
* WICHTIG: setzt bargeListening=false BEVOR OpenWakeWord.start() laeuft.
|
||||||
|
* Grund: wenn endConversation aus dem onPlaybackFinished-Handler kommt,
|
||||||
|
* feuert direkt danach ein zweiter Listener (stopBargeListening) — der
|
||||||
|
* wuerde sonst OpenWakeWord.stop() rufen weil bargeListening noch true
|
||||||
|
* ist, und unseren frisch re-armierten Listener killen.
|
||||||
*/
|
*/
|
||||||
async endConversation(): Promise<void> {
|
async endConversation(): Promise<void> {
|
||||||
if (this.state !== 'conversing') return;
|
if (this.state !== 'conversing') {
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.end',
|
||||||
|
`endConversation called but state=${this.state} → noop`)).catch(()=>{});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const wasBarge = this.bargeListening;
|
||||||
|
// Flag NULLEN bevor wir die Listener triggern. Sonst killt der parallele
|
||||||
|
// stopBargeListening-Listener (TTS-end) gleich danach unseren Native-
|
||||||
|
// OpenWakeWord, weil er bargeListening=true sieht und annimmt er muss
|
||||||
|
// den Listener stoppen.
|
||||||
|
this.bargeListening = false;
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.end',
|
||||||
|
`endConversation called, wasBarge=${wasBarge}, nativeReady=${this.nativeReady}`)).catch(()=>{});
|
||||||
if (this.nativeReady && OpenWakeWord) {
|
if (this.nativeReady && OpenWakeWord) {
|
||||||
|
// Wenn wakeword schon laeuft (war Barge-Listener waehrend TTS):
|
||||||
|
// OpenWakeWord.start() ist idempotent (Kotlin checkt running.get()
|
||||||
|
// und resolved sofort). Wir koennen es trotzdem rufen — billiger
|
||||||
|
// als state extra zu fragen, garantiert dass nach diesem Pfad
|
||||||
|
// Native auch wirklich an ist falls es out-of-band gestoppt wurde.
|
||||||
try {
|
try {
|
||||||
await OpenWakeWord.start();
|
await OpenWakeWord.start();
|
||||||
console.log('[WakeWord] Konversation zu Ende — zurueck zu armed');
|
console.log('[WakeWord] Konversation zu Ende — zurueck zu armed (wasBarge=%s)', wasBarge);
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.end',
|
||||||
|
`OpenWakeWord.start() OK → state=armed, wasBarge=${wasBarge}`)).catch(()=>{});
|
||||||
ToastAndroid.show(`Lausche wieder auf "${KEYWORD_LABELS[this.keyword]}"`, ToastAndroid.SHORT);
|
ToastAndroid.show(`Lausche wieder auf "${KEYWORD_LABELS[this.keyword]}"`, ToastAndroid.SHORT);
|
||||||
this.setState('armed');
|
this.setState('armed');
|
||||||
return;
|
return;
|
||||||
} catch (err) {
|
} catch (err: any) {
|
||||||
console.warn('[WakeWord] re-arm fehlgeschlagen:', err);
|
console.warn('[WakeWord] re-arm fehlgeschlagen:', err);
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.end',
|
||||||
|
`OpenWakeWord.start() FAIL: ${err?.message || err} → state=off`,
|
||||||
|
)).catch(()=>{});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
console.log('[WakeWord] Konversation zu Ende — Ohr aus');
|
console.log('[WakeWord] Konversation zu Ende — Ohr aus');
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.end',
|
||||||
|
`fallback: nativeReady=${this.nativeReady} → state=off`)).catch(()=>{});
|
||||||
ToastAndroid.show('Mikro aus', ToastAndroid.SHORT);
|
ToastAndroid.show('Mikro aus', ToastAndroid.SHORT);
|
||||||
this.setState('off');
|
this.setState('off');
|
||||||
}
|
}
|
||||||
@@ -390,15 +420,35 @@ class WakeWordService {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Nach ARIA-Antwort (TTS fertig): naechste Aufnahme im Conversation-Window starten */
|
/** Nach ARIA-Antwort (TTS fertig): naechste Aufnahme im Conversation-Window starten.
|
||||||
|
*
|
||||||
|
* WICHTIG: setTimeout(800ms) kann im Hintergrund (Display aus) verspaetet
|
||||||
|
* feuern — JS-Thread ist geparkt. Wenn der Timer >2s ueberfaellig ist,
|
||||||
|
* hat der User offensichtlich die App verlassen und kommt erst spaeter
|
||||||
|
* wieder — wir oeffnen das Mikro dann NICHT, sondern beenden die
|
||||||
|
* Konversation. Sonst sieht der User nach dem App-Resume "Mikro plus-
|
||||||
|
* aufnahme laeuft" obwohl er gar nichts gesagt hat → wirkt wie Phantom-
|
||||||
|
* Wake-Word. Klassische Doze-Throttling-Falle wie bei wake.detect frueher. */
|
||||||
async resume(): Promise<void> {
|
async resume(): Promise<void> {
|
||||||
if (this.state !== 'conversing') return;
|
if (this.state !== 'conversing') return;
|
||||||
|
const scheduledAt = Date.now();
|
||||||
// Kurze Pause damit TTS-Audio nicht ins Mikrofon geht
|
// Kurze Pause damit TTS-Audio nicht ins Mikrofon geht
|
||||||
await new Promise(resolve => setTimeout(resolve, 800));
|
await new Promise(resolve => setTimeout(resolve, 800));
|
||||||
if (this.state === 'conversing') {
|
if (this.state !== 'conversing') return;
|
||||||
console.log('[WakeWord] TTS fertig — naechste Aufnahme im Conversation-Window');
|
const delay = Date.now() - scheduledAt;
|
||||||
this.wakeCallbacks.forEach(cb => cb());
|
if (delay > 2800) {
|
||||||
|
// Timer war stark verspaetet — JS-Thread war im Hintergrund geparkt.
|
||||||
|
// Conversation als beendet behandeln statt das Mikro zu oeffnen.
|
||||||
|
console.log('[WakeWord] resume(): %dms statt ~800ms — App war im Background. endConversation statt mic-open', delay);
|
||||||
|
import('./logger').then(m => m.reportAppDebug('wake.resume',
|
||||||
|
`delayed ${delay}ms (>2800) — endConversation statt mic-open`)).catch(()=>{});
|
||||||
|
// Asynchroner Aufruf — endConversation ist async, kein await damit wir
|
||||||
|
// hier nicht in einem Promise-Chain haengen.
|
||||||
|
this.endConversation().catch(() => {});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
console.log('[WakeWord] TTS fertig — naechste Aufnahme im Conversation-Window (delay=%dms)', delay);
|
||||||
|
this.wakeCallbacks.forEach(cb => cb());
|
||||||
}
|
}
|
||||||
|
|
||||||
/** True solange das Ohr aktiv ist (armed ODER conversing). */
|
/** True solange das Ohr aktiv ist (armed ODER conversing). */
|
||||||
|
|||||||
@@ -556,6 +556,12 @@ class ARIABridge:
|
|||||||
for k in ("fluxDefaultModel", "fluxKeywordRaw", "fluxKeywordSwitch", "huggingfaceToken"):
|
for k in ("fluxDefaultModel", "fluxKeywordRaw", "fluxKeywordSwitch", "huggingfaceToken"):
|
||||||
if k in vc:
|
if k in vc:
|
||||||
self._flux_config[k] = vc[k]
|
self._flux_config[k] = vc[k]
|
||||||
|
# Debug-Log-Toggles fuer Whisper / F5TTS Bridges (Diagnostic-Toggle).
|
||||||
|
# Default: aus — sonst muellen wir uns volle Disk wenn alles laeuft.
|
||||||
|
self._debug_log_config: dict = {}
|
||||||
|
for k in ("whisperDebugLog", "f5ttsDebugLog"):
|
||||||
|
if k in vc:
|
||||||
|
self._debug_log_config[k] = bool(vc[k])
|
||||||
logger.info("Voice-Config geladen: tts=%s voice=%s f5tts=%s flux=%s",
|
logger.info("Voice-Config geladen: tts=%s voice=%s f5tts=%s flux=%s",
|
||||||
self.tts_enabled, self.xtts_voice or "default",
|
self.tts_enabled, self.xtts_voice or "default",
|
||||||
self._f5tts_config or "defaults",
|
self._f5tts_config or "defaults",
|
||||||
@@ -1304,6 +1310,7 @@ class ARIABridge:
|
|||||||
payload["xttsSpeed"] = self._persistent_xtts_speed
|
payload["xttsSpeed"] = self._persistent_xtts_speed
|
||||||
payload.update(getattr(self, "_f5tts_config", {}) or {})
|
payload.update(getattr(self, "_f5tts_config", {}) or {})
|
||||||
payload.update(getattr(self, "_flux_config", {}) or {})
|
payload.update(getattr(self, "_flux_config", {}) or {})
|
||||||
|
payload.update(getattr(self, "_debug_log_config", {}) or {})
|
||||||
await self._send_to_rvs({
|
await self._send_to_rvs({
|
||||||
"type": "config",
|
"type": "config",
|
||||||
"payload": payload,
|
"payload": payload,
|
||||||
@@ -1978,6 +1985,15 @@ class ARIABridge:
|
|||||||
self._flux_config = {}
|
self._flux_config = {}
|
||||||
self._flux_config[k] = payload[k]
|
self._flux_config[k] = payload[k]
|
||||||
changed = True
|
changed = True
|
||||||
|
# Debug-Log-Toggles fuer Whisper- und F5TTS-Bridge — werden via
|
||||||
|
# naechstem config-Broadcast an die jeweiligen Bridges weitergegeben.
|
||||||
|
# Persistent damit Toggle einen Container-Restart ueberlebt.
|
||||||
|
for k in ("whisperDebugLog", "f5ttsDebugLog"):
|
||||||
|
if k in payload:
|
||||||
|
if not hasattr(self, "_debug_log_config"):
|
||||||
|
self._debug_log_config = {}
|
||||||
|
self._debug_log_config[k] = bool(payload[k])
|
||||||
|
changed = True
|
||||||
# Persistent speichern in Shared Volume
|
# Persistent speichern in Shared Volume
|
||||||
if changed:
|
if changed:
|
||||||
try:
|
try:
|
||||||
@@ -1991,6 +2007,7 @@ class ARIABridge:
|
|||||||
config_data["xttsSpeed"] = self._persistent_xtts_speed
|
config_data["xttsSpeed"] = self._persistent_xtts_speed
|
||||||
config_data.update(getattr(self, "_f5tts_config", {}))
|
config_data.update(getattr(self, "_f5tts_config", {}))
|
||||||
config_data.update(getattr(self, "_flux_config", {}))
|
config_data.update(getattr(self, "_flux_config", {}))
|
||||||
|
config_data.update(getattr(self, "_debug_log_config", {}))
|
||||||
with open("/shared/config/voice_config.json", "w") as f:
|
with open("/shared/config/voice_config.json", "w") as f:
|
||||||
json.dump(config_data, f, indent=2)
|
json.dump(config_data, f, indent=2)
|
||||||
logger.info("[rvs] Voice-Config gespeichert: %s", config_data)
|
logger.info("[rvs] Voice-Config gespeichert: %s", config_data)
|
||||||
|
|||||||
@@ -38,6 +38,10 @@ const ALLOWED_TYPES = new Set([
|
|||||||
"xtts_delete_voice",
|
"xtts_delete_voice",
|
||||||
"voice_preload", "voice_ready",
|
"voice_preload", "voice_ready",
|
||||||
"stt_request", "stt_response",
|
"stt_request", "stt_response",
|
||||||
|
// Streaming-STT (Phase 1+2): App schickt PCM live an whisper-bridge,
|
||||||
|
// die feuert stt_endpoint mit dem finalen Text — kein Audio-Roundtrip.
|
||||||
|
"stt_stream_start", "stt_audio_chunk", "stt_stream_end",
|
||||||
|
"stt_partial", "stt_endpoint", "stt_stream_done",
|
||||||
"service_status",
|
"service_status",
|
||||||
"config_request",
|
"config_request",
|
||||||
"flux_request", "flux_response",
|
"flux_request", "flux_response",
|
||||||
|
|||||||
@@ -375,6 +375,41 @@ async def _send(ws, mtype: str, payload: dict) -> None:
|
|||||||
logger.warning("Send fehlgeschlagen (%s): %s", mtype, e)
|
logger.warning("Send fehlgeschlagen (%s): %s", mtype, e)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
# DEBUG-LOG ueber RVS → /shared/logs/app.log
|
||||||
|
#
|
||||||
|
# Gleiches Pattern wie in whisper-bridge: Stefan's Gamebox ist
|
||||||
|
# Windows (kein SSH), in Zukunft koennten whisper + f5tts auf
|
||||||
|
# unterschiedlichen Hosts laufen. Logs ueber RVS heisst: ein Pfad.
|
||||||
|
#
|
||||||
|
# Toggle via aria-bridge config broadcast: f5ttsDebugLog (bool).
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
_DEBUG_LOG_TO_BRIDGE: bool = False # default OFF — TTS-Renders sind teurer
|
||||||
|
# zu debuggen, normalerweise nicht noetig
|
||||||
|
|
||||||
|
|
||||||
|
async def _debug_log(ws, scope: str, message: str, level: str = "info") -> None:
|
||||||
|
"""Schickt einen app_log via RVS → /shared/logs/app.log mit platform='f5tts'.
|
||||||
|
No-op wenn Toggle aus."""
|
||||||
|
if not _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"type": "app_log",
|
||||||
|
"payload": {
|
||||||
|
"ts": int(time.time() * 1000),
|
||||||
|
"platform": "f5tts",
|
||||||
|
"level": level,
|
||||||
|
"scope": scope,
|
||||||
|
"message": str(message)[:2000],
|
||||||
|
"stack": "",
|
||||||
|
},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
}))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
# ── Interne Transkription via whisper-bridge ────────────────
|
# ── Interne Transkription via whisper-bridge ────────────────
|
||||||
|
|
||||||
_pending_stt: dict[str, asyncio.Future] = {}
|
_pending_stt: dict[str, asyncio.Future] = {}
|
||||||
@@ -867,6 +902,30 @@ async def run_loop(runner: F5Runner) -> None:
|
|||||||
else:
|
else:
|
||||||
fut.set_result(payload.get("text") or "")
|
fut.set_result(payload.get("text") or "")
|
||||||
elif mtype == "config":
|
elif mtype == "config":
|
||||||
|
# Debug-Toggle (gleiche Semantik wie in whisper-bridge)
|
||||||
|
if "f5ttsDebugLog" in payload:
|
||||||
|
global _DEBUG_LOG_TO_BRIDGE
|
||||||
|
old = _DEBUG_LOG_TO_BRIDGE
|
||||||
|
_DEBUG_LOG_TO_BRIDGE = bool(payload.get("f5ttsDebugLog", False))
|
||||||
|
if old != _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
logger.info("Debug-Log-to-Bridge: %s", "ON" if _DEBUG_LOG_TO_BRIDGE else "OFF")
|
||||||
|
# Last gasp wenn ausgeschaltet wird
|
||||||
|
if not _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
try:
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"type": "app_log",
|
||||||
|
"payload": {
|
||||||
|
"ts": int(time.time() * 1000),
|
||||||
|
"platform": "f5tts",
|
||||||
|
"level": "info",
|
||||||
|
"scope": "config",
|
||||||
|
"message": "debug-log OFF (toggle aus)",
|
||||||
|
"stack": "",
|
||||||
|
},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
}))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
# F5-TTS-Settings aktualisieren (Modell, cfg_strength, nfe)
|
# F5-TTS-Settings aktualisieren (Modell, cfg_strength, nfe)
|
||||||
async def _update_with_status(p):
|
async def _update_with_status(p):
|
||||||
# Schaut ob ein Modell-Wechsel ansteht — falls ja:
|
# Schaut ob ein Modell-Wechsel ansteht — falls ja:
|
||||||
|
|||||||
@@ -171,6 +171,43 @@ async def _send(ws, mtype: str, payload: dict) -> None:
|
|||||||
logger.warning("Send fehlgeschlagen (%s): %s", mtype, e)
|
logger.warning("Send fehlgeschlagen (%s): %s", mtype, e)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
# DEBUG-LOG ueber RVS → /shared/logs/app.log
|
||||||
|
#
|
||||||
|
# Stefan's Gamebox ist Windows, kein SSH → wir brauchen Whisper-Bridge-
|
||||||
|
# Logs ueber den gleichen Pfad wie die App: app_log-Messages via RVS,
|
||||||
|
# aria-bridge schreibt sie in /shared/logs/app.log. Diagnostic / App-
|
||||||
|
# Logs-Tab zeigen sie dann mit platform="whisper".
|
||||||
|
#
|
||||||
|
# Toggle via aria-bridge config broadcast: whisperDebugLog (bool).
|
||||||
|
# Default ON solange wir Phase-1/2-Pipeline einfahren — danach
|
||||||
|
# defaultet aria-bridge ihn aus damit kein Spam.
|
||||||
|
# ──────────────────────────────────────────────────────────────
|
||||||
|
_DEBUG_LOG_TO_BRIDGE: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
async def _debug_log(ws, scope: str, message: str, level: str = "info") -> None:
|
||||||
|
"""Schickt einen app_log via RVS → landet in /shared/logs/app.log mit
|
||||||
|
platform='whisper'. Idempotent: wenn Toggle aus → no-op."""
|
||||||
|
if not _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"type": "app_log",
|
||||||
|
"payload": {
|
||||||
|
"ts": int(time.time() * 1000),
|
||||||
|
"platform": "whisper",
|
||||||
|
"level": level,
|
||||||
|
"scope": scope,
|
||||||
|
"message": str(message)[:2000],
|
||||||
|
"stack": "",
|
||||||
|
},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
}))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
# STREAMING-SESSIONS
|
# STREAMING-SESSIONS
|
||||||
# ──────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────
|
||||||
@@ -365,6 +402,8 @@ class SessionManager:
|
|||||||
"audioRequestId": sess.audio_request_id,
|
"audioRequestId": sess.audio_request_id,
|
||||||
"text": text,
|
"text": text,
|
||||||
})
|
})
|
||||||
|
await _debug_log(ws, "stream.partial",
|
||||||
|
f"id={sess.request_id[:12]} text={text[:80]!r}")
|
||||||
else:
|
else:
|
||||||
# Stagnation pruefen — Endpoint-Bedingung
|
# Stagnation pruefen — Endpoint-Bedingung
|
||||||
if sess.last_growth_at == 0.0:
|
if sess.last_growth_at == 0.0:
|
||||||
@@ -410,6 +449,9 @@ class SessionManager:
|
|||||||
|
|
||||||
logger.info("Stream %s: FINAL (reason=%s, %.1fs Audio, %dms): %r",
|
logger.info("Stream %s: FINAL (reason=%s, %.1fs Audio, %dms): %r",
|
||||||
sess.request_id[:8], reason, duration_s, stt_ms, final_text[:120])
|
sess.request_id[:8], reason, duration_s, stt_ms, final_text[:120])
|
||||||
|
await _debug_log(ws, "stream.final",
|
||||||
|
f"id={sess.request_id[:12]} reason={reason} "
|
||||||
|
f"audio={duration_s:.1f}s stt={stt_ms}ms text={final_text[:80]!r}")
|
||||||
|
|
||||||
# stt_endpoint: das ist DAS Event auf das aria-bridge horcht fuer den
|
# stt_endpoint: das ist DAS Event auf das aria-bridge horcht fuer den
|
||||||
# Brain-Shortcut. Enthaelt alle Felder die bisher in 'audio' lagen,
|
# Brain-Shortcut. Enthaelt alle Felder die bisher in 'audio' lagen,
|
||||||
@@ -537,6 +579,11 @@ async def run_loop(runner: WhisperRunner, sessions: SessionManager) -> None:
|
|||||||
await _broadcast_status(ws, "loading", model=init_model)
|
await _broadcast_status(ws, "loading", model=init_model)
|
||||||
logger.info("Initial: sende config_request an aria-bridge")
|
logger.info("Initial: sende config_request an aria-bridge")
|
||||||
await _send(ws, "config_request", {"service": "whisper"})
|
await _send(ws, "config_request", {"service": "whisper"})
|
||||||
|
# Startup-Marker — App-Logs zeigen damit ob Streaming-Code
|
||||||
|
# ueberhaupt aktiv ist (Stefan baut auf Gamebox via PS,
|
||||||
|
# Build/Restart kann unbeabsichtigt alte Version weiterfahren).
|
||||||
|
await _debug_log(ws, "boot",
|
||||||
|
"whisper-bridge online — streaming-mode ENABLED, debug-log ON")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Initial-Handshake crashed: %s", e)
|
logger.exception("Initial-Handshake crashed: %s", e)
|
||||||
asyncio.create_task(_initial_handshake())
|
asyncio.create_task(_initial_handshake())
|
||||||
@@ -557,6 +604,11 @@ async def run_loop(runner: WhisperRunner, sessions: SessionManager) -> None:
|
|||||||
asyncio.create_task(handle_stt_request(ws, payload, runner))
|
asyncio.create_task(handle_stt_request(ws, payload, runner))
|
||||||
|
|
||||||
elif mtype == "stt_stream_start":
|
elif mtype == "stt_stream_start":
|
||||||
|
await _debug_log(ws, "stream.start",
|
||||||
|
f"received id={payload.get('requestId', '?')[:12]} "
|
||||||
|
f"audioReqId={payload.get('audioRequestId', '?')[:16]} "
|
||||||
|
f"endpointMs={payload.get('endpointMs')} "
|
||||||
|
f"hardCapMs={payload.get('hardCapMs')}")
|
||||||
# Ggf. Modell sicherstellen — sonst antwortet der erste
|
# Ggf. Modell sicherstellen — sonst antwortet der erste
|
||||||
# transcribe-Call mit Leerstring weil Model None.
|
# transcribe-Call mit Leerstring weil Model None.
|
||||||
target_model = payload.get("model") or runner.model_size or WHISPER_MODEL
|
target_model = payload.get("model") or runner.model_size or WHISPER_MODEL
|
||||||
@@ -581,14 +633,52 @@ async def run_loop(runner: WhisperRunner, sessions: SessionManager) -> None:
|
|||||||
# Sehr verbose im Schlimmstfall — debug-Level reicht.
|
# Sehr verbose im Schlimmstfall — debug-Level reicht.
|
||||||
logger.debug("stt_audio_chunk: unbekannte/closed session %s",
|
logger.debug("stt_audio_chunk: unbekannte/closed session %s",
|
||||||
payload.get("requestId", "")[:8])
|
payload.get("requestId", "")[:8])
|
||||||
|
await _debug_log(ws, "stream.chunk.reject",
|
||||||
|
f"unknown/closed session id={payload.get('requestId', '?')[:12]}",
|
||||||
|
level="warn")
|
||||||
|
else:
|
||||||
|
# Nur alle 25 Chunks loggen (=5s Audio) — sonst Spam.
|
||||||
|
try:
|
||||||
|
seq = int(payload.get("seq", 0) or 0)
|
||||||
|
if seq % 25 == 0:
|
||||||
|
await _debug_log(ws, "stream.chunk",
|
||||||
|
f"id={payload.get('requestId', '?')[:12]} seq={seq}")
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
elif mtype == "stt_stream_end":
|
elif mtype == "stt_stream_end":
|
||||||
req_id = payload.get("requestId", "")
|
req_id = payload.get("requestId", "")
|
||||||
logger.info("stt_stream_end empfangen: id=%s reason=%s",
|
logger.info("stt_stream_end empfangen: id=%s reason=%s",
|
||||||
req_id[:8], payload.get("reason", ""))
|
req_id[:8], payload.get("reason", ""))
|
||||||
|
await _debug_log(ws, "stream.end",
|
||||||
|
f"received id={req_id[:12]} reason={payload.get('reason', '')}")
|
||||||
sessions.end_session(req_id)
|
sessions.end_session(req_id)
|
||||||
|
|
||||||
elif mtype == "config":
|
elif mtype == "config":
|
||||||
|
# Debug-Toggle: aria-bridge broadcastet jetzt whisperDebugLog
|
||||||
|
# damit Stefan im laufenden Betrieb via Diagnostic-Settings
|
||||||
|
# die Logs an/aus schalten kann.
|
||||||
|
if "whisperDebugLog" in payload:
|
||||||
|
global _DEBUG_LOG_TO_BRIDGE
|
||||||
|
old = _DEBUG_LOG_TO_BRIDGE
|
||||||
|
_DEBUG_LOG_TO_BRIDGE = bool(payload.get("whisperDebugLog", False))
|
||||||
|
if old != _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
logger.info("Debug-Log-to-Bridge: %s", "ON" if _DEBUG_LOG_TO_BRIDGE else "OFF")
|
||||||
|
# Last gasp wenn ausgeschaltet wird damit Stefan im Log sieht
|
||||||
|
# dass der Toggle griff.
|
||||||
|
if not _DEBUG_LOG_TO_BRIDGE:
|
||||||
|
await ws.send(json.dumps({
|
||||||
|
"type": "app_log",
|
||||||
|
"payload": {
|
||||||
|
"ts": int(time.time() * 1000),
|
||||||
|
"platform": "whisper",
|
||||||
|
"level": "info",
|
||||||
|
"scope": "config",
|
||||||
|
"message": "debug-log OFF (toggle aus)",
|
||||||
|
"stack": "",
|
||||||
|
},
|
||||||
|
"timestamp": int(time.time() * 1000),
|
||||||
|
}))
|
||||||
new_model = payload.get("whisperModel") or WHISPER_MODEL
|
new_model = payload.get("whisperModel") or WHISPER_MODEL
|
||||||
needs_load = (runner.model is None) or (new_model != runner.model_size)
|
needs_load = (runner.model is None) or (new_model != runner.model_size)
|
||||||
if needs_load:
|
if needs_load:
|
||||||
|
|||||||
Reference in New Issue
Block a user