Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9aae5af6a9 | ||
|
|
a8ff73f93d | ||
|
|
0e9adeee5c | ||
|
|
6addb2f8fe | ||
|
|
9bdfb7193e | ||
|
|
9e78d75149 | ||
|
|
517c993ac8 | ||
|
|
c1bd13687d | ||
|
|
d9bb7239c6 | ||
|
|
0265aabb5e | ||
|
|
17bc50b847 | ||
|
|
1f2be4299d | ||
|
|
0ca8a82013 | ||
|
|
7bc3f827d0 | ||
|
|
e7da9cf9c4 | ||
|
|
353fd98d3f | ||
|
|
0350dd33c8 | ||
|
|
7b956c6606 | ||
|
|
36f04f83ff | ||
|
|
01df26e6df | ||
|
|
f226c91973 | ||
|
|
3324d39d50 | ||
|
|
fff2e7df34 | ||
|
|
0aac114142 | ||
|
|
ba60f793fb | ||
|
|
a7c2f07361 | ||
|
|
ccf6dd84fb | ||
|
|
75675ed3aa | ||
|
|
73fee27e90 |
@@ -11,6 +11,8 @@
|
|||||||
#
|
#
|
||||||
# Optionen:
|
# Optionen:
|
||||||
# --up am Ende den xtts-Stack starten (Default-Profil, OHNE voxtral)
|
# --up am Ende den xtts-Stack starten (Default-Profil, OHNE voxtral)
|
||||||
|
# --upgrade-driver NVIDIA-Treiber aus trixie-backports (modernes CUDA fuer Voxtral).
|
||||||
|
# Danach REBOOT noetig. Default-Bootstrap laesst 550 unangetastet.
|
||||||
# --token <TOK> RVS_TOKEN in xtts/.env eintragen (alternativ: env RVS_TOKEN=...)
|
# --token <TOK> RVS_TOKEN in xtts/.env eintragen (alternativ: env RVS_TOKEN=...)
|
||||||
# --rvs-host <H> RVS_HOST setzen (Default aus .env.example)
|
# --rvs-host <H> RVS_HOST setzen (Default aus .env.example)
|
||||||
#
|
#
|
||||||
@@ -21,11 +23,13 @@ set -euo pipefail
|
|||||||
|
|
||||||
# ── CLI ──
|
# ── CLI ──
|
||||||
DO_UP=0
|
DO_UP=0
|
||||||
|
DO_UPGRADE_DRIVER=0
|
||||||
RVS_TOKEN_ARG="${RVS_TOKEN:-}"
|
RVS_TOKEN_ARG="${RVS_TOKEN:-}"
|
||||||
RVS_HOST_ARG="${RVS_HOST:-}"
|
RVS_HOST_ARG="${RVS_HOST:-}"
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--up) DO_UP=1; shift ;;
|
--up) DO_UP=1; shift ;;
|
||||||
|
--upgrade-driver) DO_UPGRADE_DRIVER=1; shift ;;
|
||||||
--token) RVS_TOKEN_ARG="${2:-}"; shift 2 ;;
|
--token) RVS_TOKEN_ARG="${2:-}"; shift 2 ;;
|
||||||
--rvs-host) RVS_HOST_ARG="${2:-}"; shift 2 ;;
|
--rvs-host) RVS_HOST_ARG="${2:-}"; shift 2 ;;
|
||||||
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
@@ -90,6 +94,45 @@ fi
|
|||||||
[[ $ENABLED_ANY -eq 0 ]] && ok "non-free schon aktiv"
|
[[ $ENABLED_ANY -eq 0 ]] && ok "non-free schon aktiv"
|
||||||
apt-get update -qq # immer neu einlesen, damit der Kandidat sicher da ist
|
apt-get update -qq # immer neu einlesen, damit der Kandidat sicher da ist
|
||||||
|
|
||||||
|
# ── 2b. (opt-in) Treiber-Upgrade via trixie-backports — fuer Voxtral/modernes CUDA ──
|
||||||
|
# Der Trixie-Standardtreiber (550, CUDA 12.4) ist zu alt fuer den modernen Stack
|
||||||
|
# (torch 2.13, vLLM). Backports bringt einen neueren, apt-verwalteten Treiber.
|
||||||
|
# NUR mit --upgrade-driver, damit ein laufendes Setup nicht ungewollt angefasst wird.
|
||||||
|
if [[ "$DO_UPGRADE_DRIVER" -eq 1 ]]; then
|
||||||
|
step "NVIDIA-Treiber-Upgrade (trixie-backports)"
|
||||||
|
BP_FILE="/etc/apt/sources.list.d/backports.sources"
|
||||||
|
if ! grep -rqs "trixie-backports" /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null; then
|
||||||
|
cat > "$BP_FILE" <<'EOF'
|
||||||
|
Types: deb
|
||||||
|
URIs: http://deb.debian.org/debian
|
||||||
|
Suites: trixie-backports
|
||||||
|
Components: main contrib non-free non-free-firmware
|
||||||
|
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
|
||||||
|
EOF
|
||||||
|
ok "trixie-backports hinzugefuegt"
|
||||||
|
else
|
||||||
|
ok "trixie-backports bereits aktiv"
|
||||||
|
fi
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y linux-headers-amd64 || true
|
||||||
|
apt-get install -y "linux-headers-$(uname -r)" || true
|
||||||
|
if apt-get install -y -t trixie-backports nvidia-driver; then
|
||||||
|
dkms autoinstall >/dev/null 2>&1 || true
|
||||||
|
NEWV="$(dpkg-query -W -f='${Version}' nvidia-driver 2>/dev/null || echo '?')"
|
||||||
|
ok "nvidia-driver aus backports installiert: ${NEWV}"
|
||||||
|
echo
|
||||||
|
echo -e "${c_y}==> REBOOT noetig, damit der neue Treiber laedt:${c_0}"
|
||||||
|
echo -e "${c_y} sudo reboot && danach: cd ai-box && sudo ./bootstrap.sh --up --token <TOKEN>${c_0}"
|
||||||
|
echo -e "${c_y} (nvidia-smi zeigt dann die neue Version + CUDA-Level)${c_0}"
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
warn "backports-Install fehlgeschlagen — 550er bleibt aktiv."
|
||||||
|
warn "Alternative fuer den neuesten Treiber: NVIDIAs CUDA-Repo fuer debian13"
|
||||||
|
warn " (developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64) → Paket 'cuda-drivers'."
|
||||||
|
die "Treiber-Upgrade nicht moeglich — siehe oben."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ── 3. NVIDIA-Treiber ──
|
# ── 3. NVIDIA-Treiber ──
|
||||||
step "NVIDIA-Treiber"
|
step "NVIDIA-Treiber"
|
||||||
if nvidia-smi >/dev/null 2>&1; then
|
if nvidia-smi >/dev/null 2>&1; then
|
||||||
|
|||||||
@@ -79,8 +79,8 @@ android {
|
|||||||
applicationId "com.ariacockpit"
|
applicationId "com.ariacockpit"
|
||||||
minSdkVersion rootProject.ext.minSdkVersion
|
minSdkVersion rootProject.ext.minSdkVersion
|
||||||
targetSdkVersion rootProject.ext.targetSdkVersion
|
targetSdkVersion rootProject.ext.targetSdkVersion
|
||||||
versionCode 20302
|
versionCode 20309
|
||||||
versionName "0.2.3.2"
|
versionName "0.2.3.9"
|
||||||
// Fallback fuer Libraries mit Product Flavors
|
// Fallback fuer Libraries mit Product Flavors
|
||||||
missingDimensionStrategy 'react-native-camera', 'general'
|
missingDimensionStrategy 'react-native-camera', 'general'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ class OpenWakeWordModule(reactContext: ReactApplicationContext) : ReactContextBa
|
|||||||
// Trigger eingestuft werden kann. Folge: App pausiert beim Oeffnen die Musik,
|
// Trigger eingestuft werden kann. Folge: App pausiert beim Oeffnen die Musik,
|
||||||
// weil der False-Positive die AudioFocus-Switch-Logik anwirft (Stefan-Bug 06/2026).
|
// weil der False-Positive die AudioFocus-Switch-Logik anwirft (Stefan-Bug 06/2026).
|
||||||
// Loesung: in dieser Phase keine Detections an JS weiterleiten.
|
// Loesung: in dieser Phase keine Detections an JS weiterleiten.
|
||||||
private const val STARTUP_SUPPRESSION_MS = 1500L
|
private const val STARTUP_SUPPRESSION_MS = 600L
|
||||||
}
|
}
|
||||||
|
|
||||||
private val env: OrtEnvironment = OrtEnvironment.getEnvironment()
|
private val env: OrtEnvironment = OrtEnvironment.getEnvironment()
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "aria-cockpit",
|
"name": "aria-cockpit",
|
||||||
"version": "0.2.3.2",
|
"version": "0.2.3.9",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"android": "react-native run-android",
|
"android": "react-native run-android",
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ import VoiceButton from '../components/VoiceButton';
|
|||||||
import FileUpload, { FileData } from '../components/FileUpload';
|
import FileUpload, { FileData } from '../components/FileUpload';
|
||||||
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
import CameraUpload, { PhotoData } from '../components/CameraUpload';
|
||||||
import MessageText from '../components/MessageText';
|
import MessageText from '../components/MessageText';
|
||||||
import { loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT, loadSttEndpointMs } from '../services/audio';
|
import { loadConvWindowMs, loadTtsSpeed, TTS_SPEED_DEFAULT, loadSttEndpointMs, loadMaxRecordingMs, loadBargeInEnabled } from '../services/audio';
|
||||||
import Geolocation from '@react-native-community/geolocation';
|
import Geolocation from '@react-native-community/geolocation';
|
||||||
|
|
||||||
// --- Typen ---
|
// --- Typen ---
|
||||||
@@ -93,6 +93,9 @@ interface ChatMessage {
|
|||||||
* gespiegelt damit wir die EXAKT richtige Placeholder-Bubble ersetzen,
|
* gespiegelt damit wir die EXAKT richtige Placeholder-Bubble ersetzen,
|
||||||
* auch wenn mehrere Aufnahmen parallel offen sind. */
|
* auch wenn mehrere Aufnahmen parallel offen sind. */
|
||||||
audioRequestId?: string;
|
audioRequestId?: string;
|
||||||
|
/** Laenge der Sprachaufnahme in Sekunden (aus dem stt_endpoint) — fuer die
|
||||||
|
* Dauer-Anzeige an der Voice-Bubble. */
|
||||||
|
durationS?: number;
|
||||||
/** Skill-Created-Bubble: ARIA hat einen neuen Skill angelegt */
|
/** Skill-Created-Bubble: ARIA hat einen neuen Skill angelegt */
|
||||||
skillCreated?: {
|
skillCreated?: {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -184,6 +187,14 @@ function stripSystemHints(text: string): string {
|
|||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
/** Sekunden → "M:SS" fuer die Sprachnachricht-Dauer. */
|
||||||
|
function formatDur(sec: number): string {
|
||||||
|
const s = Math.max(0, Math.round(sec));
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const r = s % 60;
|
||||||
|
return `${m}:${r.toString().padStart(2, '0')}`;
|
||||||
|
}
|
||||||
|
|
||||||
const DEFAULT_ATTACHMENT_DIR = `${RNFS.DocumentDirectoryPath}/chat_attachments`;
|
const DEFAULT_ATTACHMENT_DIR = `${RNFS.DocumentDirectoryPath}/chat_attachments`;
|
||||||
const STORAGE_PATH_KEY = 'aria_attachment_storage_path';
|
const STORAGE_PATH_KEY = 'aria_attachment_storage_path';
|
||||||
|
|
||||||
@@ -373,6 +384,8 @@ const ChatScreen: React.FC = () => {
|
|||||||
// stoppen? Kommt als 'converse' in der Chat-Payload; onPlaybackFinished liest
|
// stoppen? Kommt als 'converse' in der Chat-Payload; onPlaybackFinished liest
|
||||||
// es. Default true (Konversation). false = Einzelaktion/Skill-Antwort.
|
// es. Default true (Konversation). false = Einzelaktion/Skill-Antwort.
|
||||||
const converseRef = useRef<boolean>(true);
|
const converseRef = useRef<boolean>(true);
|
||||||
|
// Barge-in erlaubt? Default false = Halb-Duplex (waehrend TTS kein Mikro).
|
||||||
|
const bargeInEnabledRef = useRef<boolean>(false);
|
||||||
|
|
||||||
const flatListRef = useRef<FlatList>(null);
|
const flatListRef = useRef<FlatList>(null);
|
||||||
const messageIdCounter = useRef(0);
|
const messageIdCounter = useRef(0);
|
||||||
@@ -651,6 +664,7 @@ const ChatScreen: React.FC = () => {
|
|||||||
const voice = await AsyncStorage.getItem('aria_xtts_voice');
|
const voice = await AsyncStorage.getItem('aria_xtts_voice');
|
||||||
localXttsVoiceRef.current = voice || '';
|
localXttsVoiceRef.current = voice || '';
|
||||||
ttsSpeedRef.current = await loadTtsSpeed();
|
ttsSpeedRef.current = await loadTtsSpeed();
|
||||||
|
bargeInEnabledRef.current = await loadBargeInEnabled();
|
||||||
const gps = await AsyncStorage.getItem('aria_gps_enabled');
|
const gps = await AsyncStorage.getItem('aria_gps_enabled');
|
||||||
setGpsEnabled(gps === 'true');
|
setGpsEnabled(gps === 'true');
|
||||||
const hints = await AsyncStorage.getItem('aria_show_hints');
|
const hints = await AsyncStorage.getItem('aria_show_hints');
|
||||||
@@ -1387,13 +1401,30 @@ const ChatScreen: React.FC = () => {
|
|||||||
// Fallback mehr: die Bridge schickt speak zuverlaessig mit.
|
// Fallback mehr: die Bridge schickt speak zuverlaessig mit.
|
||||||
// Merken ob nach dem Vorlesen 30s weiterlauschen (Gespraech) oder direkt
|
// Merken ob nach dem Vorlesen 30s weiterlauschen (Gespraech) oder direkt
|
||||||
// stoppen — onPlaybackFinished liest converseRef. Default true.
|
// stoppen — onPlaybackFinished liest converseRef. Default true.
|
||||||
converseRef.current = (message.payload as any).converse !== false;
|
// Passiv-Lauschen (30s) NUR wenn das Brain explizit converse:true schickt.
|
||||||
|
// Vorher default true → jeder Befehl (auch "Spiele Spotify" mit gesproche-
|
||||||
|
// ner Bestaetigung) landete im 30s-Fenster. Jetzt: einzelne Befehle enden
|
||||||
|
// sofort (zurueck aufs Wake-Word), nur echte Gespraeche lauschen weiter.
|
||||||
|
converseRef.current = (message.payload as any).converse === true;
|
||||||
const _isSilent = (message.payload as any).speak === false;
|
const _isSilent = (message.payload as any).speak === false;
|
||||||
if (_isSilent && wakeWordService.isConversing()) {
|
if (_isSilent) {
|
||||||
// Klarer Steuerbefehl (Liedersteuerung etc.) = KEINE Konversation →
|
// Steuerbefehl (speak=false) ist ausgefuehrt und wird NICHT vorgelesen.
|
||||||
// STOP: direkt zurueck aufs Wake-Word. Kein Gong, keine Aufnahme,
|
// Ohne TTS feuert onPlaybackFinished nie — der Mikro-/Konversations-
|
||||||
// kein 30s-Fenster (skipPassive=true).
|
// Lifecycle muss hier selbst weitergeschaltet werden, sonst haengt das Ohr.
|
||||||
wakeWordService.endConversation(true).catch(() => {});
|
if (converseRef.current) {
|
||||||
|
// Befehlskette laeuft WEITER ([[WEITER]]): Mikro NICHT schliessen,
|
||||||
|
// sondern das passive Lausch-Fenster oeffnen (endConversation(false)),
|
||||||
|
// damit der naechste Kettenbefehl direkt gesprochen werden kann. ARIA
|
||||||
|
// haelt bewusst offen, bis sie [[ENDE]] (converse=false) schickt.
|
||||||
|
if (wakeWordService.isConversing()) {
|
||||||
|
wakeWordService.endConversation(false).catch(() => {});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Einzelbefehl / [[ENDE]] → ARIA "drueckt selbst Stop": jede offene
|
||||||
|
// Aufnahme schliessen + zurueck aufs Wake-Word, egal in welchem Zustand
|
||||||
|
// (conversing, passives Lauschen ODER offene Streaming-Aufnahme).
|
||||||
|
ariaStopRecording('silent-command').catch(() => {});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1668,7 +1699,9 @@ const ChatScreen: React.FC = () => {
|
|||||||
location: location || null,
|
location: location || null,
|
||||||
noSpeechTimeoutMs: windowMs,
|
noSpeechTimeoutMs: windowMs,
|
||||||
endpointMs: await loadSttEndpointMs(),
|
endpointMs: await loadSttEndpointMs(),
|
||||||
hardCapMs: 60000,
|
// Notbremse 5 min (nicht 1 min) — der Stille-Endpoint beendet normale
|
||||||
|
// Turns eh sofort; der Cap darf lange Diktate nicht mitten drin kappen.
|
||||||
|
hardCapMs: await loadMaxRecordingMs(),
|
||||||
projectId: focusedProjectIdRef.current,
|
projectId: focusedProjectIdRef.current,
|
||||||
});
|
});
|
||||||
import('../services/logger').then(m => m.reportAppDebug('wake.cb', `startStreamingRecording returned ok=${ok}`)).catch(()=>{});
|
import('../services/logger').then(m => m.reportAppDebug('wake.cb', `startStreamingRecording returned ok=${ok}`)).catch(()=>{});
|
||||||
@@ -1696,6 +1729,13 @@ const ChatScreen: React.FC = () => {
|
|||||||
if (ev.text && ev.text.trim()) {
|
if (ev.text && ev.text.trim()) {
|
||||||
console.log('[Chat] STT-Endpoint: %r (reason=%s, %dms, %.1fs Audio)',
|
console.log('[Chat] STT-Endpoint: %r (reason=%s, %dms, %.1fs Audio)',
|
||||||
ev.text.slice(0, 80), ev.reason, ev.sttMs, ev.durationS);
|
ev.text.slice(0, 80), ev.reason, ev.sttMs, ev.durationS);
|
||||||
|
// Aufnahme-Dauer an die passende Voice-Bubble haengen (Anzeige). Der
|
||||||
|
// spaetere STT-Text-Update spreadet die Message, die Dauer bleibt.
|
||||||
|
if (ev.audioRequestId && typeof ev.durationS === 'number' && ev.durationS > 0) {
|
||||||
|
const dur = ev.durationS;
|
||||||
|
setMessages(prev => prev.map(m =>
|
||||||
|
m.audioRequestId === ev.audioRequestId ? { ...m, durationS: dur } : m));
|
||||||
|
}
|
||||||
// Wenn passive lauschend: User hat tatsaechlich was gesagt → uebergang
|
// Wenn passive lauschend: User hat tatsaechlich was gesagt → uebergang
|
||||||
// zu 'conversing' damit der normale Flow greift (TTS, resume, etc.)
|
// zu 'conversing' damit der normale Flow greift (TTS, resume, etc.)
|
||||||
if (wakeWordService.getState() === 'listening') {
|
if (wakeWordService.getState() === 'listening') {
|
||||||
@@ -1771,7 +1811,8 @@ const ChatScreen: React.FC = () => {
|
|||||||
location: location || null,
|
location: location || null,
|
||||||
noSpeechTimeoutMs: windowMs,
|
noSpeechTimeoutMs: windowMs,
|
||||||
endpointMs: await loadSttEndpointMs(),
|
endpointMs: await loadSttEndpointMs(),
|
||||||
hardCapMs: 60000,
|
// Notbremse 5 min (s.o.) — lange Diktate nicht bei 1 min abschneiden.
|
||||||
|
hardCapMs: await loadMaxRecordingMs(),
|
||||||
projectId: focusedProjectIdRef.current,
|
projectId: focusedProjectIdRef.current,
|
||||||
});
|
});
|
||||||
if (ok) {
|
if (ok) {
|
||||||
@@ -1789,7 +1830,9 @@ const ChatScreen: React.FC = () => {
|
|||||||
// Prozess nicht killt wenn die App im Hintergrund ist.
|
// Prozess nicht killt wenn die App im Hintergrund ist.
|
||||||
const unsubTtsStart = audioService.onPlaybackStarted(() => {
|
const unsubTtsStart = audioService.onPlaybackStarted(() => {
|
||||||
acquireBackgroundAudio('tts').catch(() => {});
|
acquireBackgroundAudio('tts').catch(() => {});
|
||||||
if (wakeWordService.isConversing() && wakeWordService.hasWakeWord()) {
|
// Barge-Listening (Mikro waehrend TTS) NUR im Barge-in-Modus. Default aus =
|
||||||
|
// Halb-Duplex: ARIA spricht ungestoert zu Ende, dann erst geht das Mikro auf.
|
||||||
|
if (bargeInEnabledRef.current && wakeWordService.isConversing() && wakeWordService.hasWakeWord()) {
|
||||||
wakeWordService.startBargeListening().catch(() => {});
|
wakeWordService.startBargeListening().catch(() => {});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -2210,15 +2253,16 @@ const ChatScreen: React.FC = () => {
|
|||||||
advanceQueue(pid);
|
advanceQueue(pid);
|
||||||
}, [advanceQueue]);
|
}, [advanceQueue]);
|
||||||
|
|
||||||
// Queue-Modus („immer anstellen"): eine neue Sprachnachricht bricht ARIAs
|
// Nimmt der User das Mikro waehrend ARIA SPRICHT, ist das ein echter Interrupt:
|
||||||
// laufende Arbeit NICHT mehr ab. Sie wird — wie Text — angestellt und laeuft
|
// TTS stoppen UND die laufende Brain-Antwort abbrechen (cancel_request). Sonst
|
||||||
// serialisiert (der Brain-Lock pro Projekt reiht /chat-/audio-Turns auf).
|
// produziert das Brain weiter TTS, die ins offene Mikro laeuft → genau der
|
||||||
// Nur das TTS wird akustisch gestoppt, damit das Mikro ARIAs eigene Stimme
|
// "Mischmasch" (ARIA antwortet weiter waehrend ich rede). Fuer bewusstes
|
||||||
// nicht mithoert. Explizites Abbrechen laeuft ueber den Stop-Button
|
// Nicht-Abbrechen gibt es weiterhin den separaten Zwischenruf-Button (📣).
|
||||||
// (cancelRequest). Rueckgabe = false, weil kein Barge-In/Interrupt mehr.
|
|
||||||
const interruptAriaIfBusy = useCallback(() => {
|
const interruptAriaIfBusy = useCallback(() => {
|
||||||
if (audioService.isPlayingAudio()) {
|
if (audioService.isPlayingAudio()) {
|
||||||
audioService.haltAllPlayback('user startet Aufnahme (Queue-Modus, kein Abbruch)');
|
audioService.haltAllPlayback('user startet Aufnahme — Interrupt');
|
||||||
|
rvs.send('cancel_request' as any, { hard: true, source: 'voice-interrupt' });
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}, []);
|
}, []);
|
||||||
@@ -2255,7 +2299,7 @@ const ChatScreen: React.FC = () => {
|
|||||||
// die Session auch app-seitig haben wir +2s Toleranz.
|
// die Session auch app-seitig haben wir +2s Toleranz.
|
||||||
noSpeechTimeoutMs: 0,
|
noSpeechTimeoutMs: 0,
|
||||||
endpointMs: await loadSttEndpointMs(),
|
endpointMs: await loadSttEndpointMs(),
|
||||||
hardCapMs: 300000,
|
hardCapMs: await loadMaxRecordingMs(),
|
||||||
projectId: focusedProjectIdRef.current,
|
projectId: focusedProjectIdRef.current,
|
||||||
});
|
});
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
@@ -2267,11 +2311,50 @@ const ChatScreen: React.FC = () => {
|
|||||||
return true;
|
return true;
|
||||||
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
}, [getCurrentLocation, interruptAriaIfBusy, scheduleStaleAudioCleanup]);
|
||||||
|
|
||||||
|
// ARIA schliesst die Aufnahme SELBST — das programmatische Gegenstueck zum
|
||||||
|
// Stop-Button. Aufgerufen nach einem stillen Steuerbefehl (speak=false): der
|
||||||
|
// Befehl ist ausgefuehrt, ARIA hat die Rueckinfo (Skill-Ergebnis) und antwortet
|
||||||
|
// NICHT vorgelesen. Weil ohne TTS kein onPlaybackFinished kommt, muss der
|
||||||
|
// Aufnahme-/Konversations-Zustand hier aktiv aufgeraeumt werden, sonst bleibt
|
||||||
|
// das Ohr haengen bzw. das Aufnahme-Fenster laeuft leer weiter (Stefans
|
||||||
|
// Reproduktion: "spotify play" und das Mikro wartet trotzdem 30s).
|
||||||
|
// Unterschied zum manuellen Stop: der verwirft NICHT, sondern finalisiert die
|
||||||
|
// Aufnahme (User will seinen Satz verarbeitet haben) — hier ist der Befehl
|
||||||
|
// schon durch, ein evtl. offenes Folge-Fenster wird verworfen.
|
||||||
|
const ariaStopRecording = useCallback(async (reason: string): Promise<void> => {
|
||||||
|
converseRef.current = false;
|
||||||
|
// 1) Passiv-Lauschen: sauber beenden (cancelt den Stream selbst, startet
|
||||||
|
// KEINE neue passive Aufnahme).
|
||||||
|
if (wakeWordService.getState() === 'listening') {
|
||||||
|
await wakeWordService.exitPassiveListening('manual').catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 2) Noch offene Streaming-Aufnahme (aktiv / Barge-In) verwerfen.
|
||||||
|
if (audioService.isStreamingRecording()) {
|
||||||
|
await audioService.cancelStreamingRecording(reason).catch(() => {});
|
||||||
|
}
|
||||||
|
// 3) Konversation beenden → zurueck aufs Wake-Word (skipPassive: kein 30s-Fenster).
|
||||||
|
if (wakeWordService.isConversing()) {
|
||||||
|
await wakeWordService.endConversation(true).catch(() => {});
|
||||||
|
} else if (!wakeWordService.isActive()) {
|
||||||
|
setWakeWordActive(false);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
// Manueller Aufnahme-Knopf — Stop. Sendet stt_stream_end an Whisper, die
|
// Manueller Aufnahme-Knopf — Stop. Sendet stt_stream_end an Whisper, die
|
||||||
// dann ihrerseits den finalen Text als stt_endpoint emittiert. aria-bridge
|
// dann ihrerseits den finalen Text als stt_endpoint emittiert. aria-bridge
|
||||||
// forwarded direkt an Brain. Im wake-word-conversing-Fall zusaetzlich
|
// forwarded direkt an Brain. Im wake-word-conversing-Fall zusaetzlich
|
||||||
// endConversation: User hat explizit gestoppt → kein Multi-Turn-Resume.
|
// endConversation: User hat explizit gestoppt → kein Multi-Turn-Resume.
|
||||||
const handleVoiceButtonStop = useCallback(async (): Promise<void> => {
|
const handleVoiceButtonStop = useCallback(async (): Promise<void> => {
|
||||||
|
// Manueller Stop = endgueltig: auch die NACH der Antwort kommende
|
||||||
|
// onPlaybackFinished darf kein 30s-Passiv-Fenster mehr oeffnen.
|
||||||
|
converseRef.current = false;
|
||||||
|
// Stop = ALLES beenden, vorhersehbar. Spricht ARIA gerade, hart stoppen +
|
||||||
|
// laufende Brain-Antwort abbrechen (sonst "sagt sie ihren letzten Satz").
|
||||||
|
if (audioService.isPlayingAudio()) {
|
||||||
|
audioService.haltAllPlayback('user stop');
|
||||||
|
rvs.send('cancel_request' as any, { hard: true, source: 'voice-stop' });
|
||||||
|
}
|
||||||
// Stop WAEHREND des passiven 30s-Lauschens ('listening'): sauber beenden
|
// Stop WAEHREND des passiven 30s-Lauschens ('listening'): sauber beenden
|
||||||
// (zurueck aufs Wake-Word), NICHT den passiven Stream neu starten.
|
// (zurueck aufs Wake-Word), NICHT den passiven Stream neu starten.
|
||||||
// exitPassiveListening cancelt den Stream selbst (via _freeMic) → es feuert
|
// exitPassiveListening cancelt den Stream selbst (via _freeMic) → es feuert
|
||||||
@@ -2647,6 +2730,16 @@ const ChatScreen: React.FC = () => {
|
|||||||
{att.serverPath ? '(tippen zum Laden)' : '(nicht verfuegbar)'}
|
{att.serverPath ? '(tippen zum Laden)' : '(nicht verfuegbar)'}
|
||||||
</Text>
|
</Text>
|
||||||
</TouchableOpacity>
|
</TouchableOpacity>
|
||||||
|
) : att.type === 'audio' ? (
|
||||||
|
<View style={styles.attachmentFile}>
|
||||||
|
<Text style={styles.attachmentFileIcon}>{'🎙'}</Text>
|
||||||
|
<Text style={styles.attachmentFileName} numberOfLines={1}>
|
||||||
|
{att.name || 'Sprachaufnahme'}
|
||||||
|
</Text>
|
||||||
|
{typeof item.durationS === 'number' && item.durationS > 0 ? (
|
||||||
|
<Text style={styles.attachmentFileSize}>{formatDur(item.durationS)}</Text>
|
||||||
|
) : null}
|
||||||
|
</View>
|
||||||
) : (
|
) : (
|
||||||
<TouchableOpacity
|
<TouchableOpacity
|
||||||
style={styles.attachmentFile}
|
style={styles.attachmentFile}
|
||||||
|
|||||||
@@ -67,10 +67,16 @@ import {
|
|||||||
CONV_WINDOW_MIN_SEC,
|
CONV_WINDOW_MIN_SEC,
|
||||||
CONV_WINDOW_MAX_SEC,
|
CONV_WINDOW_MAX_SEC,
|
||||||
CONV_WINDOW_STORAGE_KEY,
|
CONV_WINDOW_STORAGE_KEY,
|
||||||
|
STT_ENDPOINT_DEFAULT_MS,
|
||||||
|
STT_ENDPOINT_MIN_MS,
|
||||||
|
STT_ENDPOINT_MAX_MS,
|
||||||
|
STT_ENDPOINT_STORAGE_KEY,
|
||||||
MAX_RECORDING_DEFAULT_SEC,
|
MAX_RECORDING_DEFAULT_SEC,
|
||||||
MAX_RECORDING_MIN_SEC,
|
MAX_RECORDING_MIN_SEC,
|
||||||
MAX_RECORDING_MAX_SEC,
|
MAX_RECORDING_MAX_SEC,
|
||||||
MAX_RECORDING_STORAGE_KEY,
|
MAX_RECORDING_STORAGE_KEY,
|
||||||
|
loadBargeInEnabled,
|
||||||
|
saveBargeInEnabled,
|
||||||
VAD_SILENCE_DB_DEFAULT,
|
VAD_SILENCE_DB_DEFAULT,
|
||||||
VAD_SILENCE_DB_MIN,
|
VAD_SILENCE_DB_MIN,
|
||||||
VAD_SILENCE_DB_MAX,
|
VAD_SILENCE_DB_MAX,
|
||||||
@@ -195,8 +201,13 @@ const SettingsScreen: React.FC = () => {
|
|||||||
const [ttsEnabled, setTtsEnabled] = useState(true);
|
const [ttsEnabled, setTtsEnabled] = useState(true);
|
||||||
const [ttsPrerollSec, setTtsPrerollSec] = useState<number>(TTS_PREROLL_DEFAULT_SEC);
|
const [ttsPrerollSec, setTtsPrerollSec] = useState<number>(TTS_PREROLL_DEFAULT_SEC);
|
||||||
const [vadSilenceSec, setVadSilenceSec] = useState<number>(VAD_SILENCE_DEFAULT_SEC);
|
const [vadSilenceSec, setVadSilenceSec] = useState<number>(VAD_SILENCE_DEFAULT_SEC);
|
||||||
|
// Aktive Streaming-Pausen-Toleranz (STT_ENDPOINT) — der "Stille-Toleranz"-Regler
|
||||||
|
// steuert jetzt DIESEN Wert (der alte vadSilenceSec war der tote Legacy-dB-Pfad).
|
||||||
|
const [sttEndpointSec, setSttEndpointSec] = useState<number>(STT_ENDPOINT_DEFAULT_MS / 1000);
|
||||||
const [convWindowSec, setConvWindowSec] = useState<number>(CONV_WINDOW_DEFAULT_SEC);
|
const [convWindowSec, setConvWindowSec] = useState<number>(CONV_WINDOW_DEFAULT_SEC);
|
||||||
const [maxRecordingSec, setMaxRecordingSec] = useState<number>(MAX_RECORDING_DEFAULT_SEC);
|
const [maxRecordingSec, setMaxRecordingSec] = useState<number>(MAX_RECORDING_DEFAULT_SEC);
|
||||||
|
// Barge-in: ARIA waehrend ihrer Antwort unterbrechen duerfen. Default aus (Halb-Duplex).
|
||||||
|
const [bargeIn, setBargeIn] = useState<boolean>(false);
|
||||||
// null = automatisch (adaptive Baseline), sonst manueller dB-Override
|
// null = automatisch (adaptive Baseline), sonst manueller dB-Override
|
||||||
const [vadSilenceDb, setVadSilenceDb] = useState<number | null>(null);
|
const [vadSilenceDb, setVadSilenceDb] = useState<number | null>(null);
|
||||||
const [showVadInfo, setShowVadInfo] = useState(false);
|
const [showVadInfo, setShowVadInfo] = useState(false);
|
||||||
@@ -306,6 +317,14 @@ const SettingsScreen: React.FC = () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
AsyncStorage.getItem(STT_ENDPOINT_STORAGE_KEY).then(saved => {
|
||||||
|
if (saved != null) {
|
||||||
|
const n = parseInt(saved, 10);
|
||||||
|
if (isFinite(n) && n >= STT_ENDPOINT_MIN_MS && n <= STT_ENDPOINT_MAX_MS) {
|
||||||
|
setSttEndpointSec(n / 1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
AsyncStorage.getItem(MAX_RECORDING_STORAGE_KEY).then(saved => {
|
AsyncStorage.getItem(MAX_RECORDING_STORAGE_KEY).then(saved => {
|
||||||
if (saved != null) {
|
if (saved != null) {
|
||||||
const n = parseFloat(saved);
|
const n = parseFloat(saved);
|
||||||
@@ -314,6 +333,7 @@ const SettingsScreen: React.FC = () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
loadBargeInEnabled().then(setBargeIn).catch(() => {});
|
||||||
AsyncStorage.getItem(VAD_SILENCE_DB_OVERRIDE_KEY).then(saved => {
|
AsyncStorage.getItem(VAD_SILENCE_DB_OVERRIDE_KEY).then(saved => {
|
||||||
if (saved != null && saved !== '') {
|
if (saved != null && saved !== '') {
|
||||||
const n = parseFloat(saved);
|
const n = parseFloat(saved);
|
||||||
@@ -1651,34 +1671,51 @@ const SettingsScreen: React.FC = () => {
|
|||||||
{currentSection === 'voice_input' && (<>
|
{currentSection === 'voice_input' && (<>
|
||||||
<Text style={styles.sectionTitle}>Spracheingabe</Text>
|
<Text style={styles.sectionTitle}>Spracheingabe</Text>
|
||||||
<View style={styles.card}>
|
<View style={styles.card}>
|
||||||
<Text style={styles.toggleLabel}>Stille-Toleranz</Text>
|
<View style={styles.toggleRow}>
|
||||||
|
<View style={styles.toggleInfo}>
|
||||||
|
<Text style={styles.toggleLabel}>Barge-in (unterbrechen)</Text>
|
||||||
|
<Text style={styles.toggleHint}>
|
||||||
|
AUS (empfohlen): ARIA spricht ihre Antwort ZU ENDE, dann geht das
|
||||||
|
Mikro auf — sauber, kein Selbst-Echo, du hoerst sie ganz. AN: du
|
||||||
|
kannst sie waehrend des Sprechens per Wake-Wort unterbrechen.
|
||||||
|
</Text>
|
||||||
|
</View>
|
||||||
|
<Switch
|
||||||
|
value={bargeIn}
|
||||||
|
onValueChange={(v) => { setBargeIn(v); saveBargeInEnabled(v).catch(() => {}); }}
|
||||||
|
trackColor={{ false: '#2A2A3E', true: '#0096FF' }}
|
||||||
|
thumbColor={bargeIn ? '#FFFFFF' : '#666680'}
|
||||||
|
/>
|
||||||
|
</View>
|
||||||
|
|
||||||
|
<Text style={[styles.toggleLabel, {marginTop: 20}]}>Stille-Toleranz</Text>
|
||||||
<Text style={styles.toggleHint}>
|
<Text style={styles.toggleHint}>
|
||||||
Wie lange du eine Sprechpause machen darfst, bevor die Aufnahme
|
Wie lange du eine Sprechpause machen darfst, bevor die Aufnahme
|
||||||
automatisch beendet und gesendet wird. Hoeher = mehr Zeit zum
|
automatisch beendet und gesendet wird. Hoeher = mehr Zeit zum
|
||||||
Nachdenken; niedriger = schnelleres Senden.
|
Nachdenken (z.B. im Auto); niedriger = schnelleres Senden.
|
||||||
Default: {VAD_SILENCE_DEFAULT_SEC.toFixed(1)}s.
|
Default: {(STT_ENDPOINT_DEFAULT_MS / 1000).toFixed(1)}s.
|
||||||
</Text>
|
</Text>
|
||||||
<View style={styles.prerollRow}>
|
<View style={styles.prerollRow}>
|
||||||
<TouchableOpacity
|
<TouchableOpacity
|
||||||
style={styles.prerollButton}
|
style={styles.prerollButton}
|
||||||
onPress={() => {
|
onPress={() => {
|
||||||
const next = Math.max(VAD_SILENCE_MIN_SEC, Math.round((vadSilenceSec - 0.5) * 10) / 10);
|
const next = Math.max(STT_ENDPOINT_MIN_MS / 1000, Math.round((sttEndpointSec - 0.5) * 10) / 10);
|
||||||
setVadSilenceSec(next);
|
setSttEndpointSec(next);
|
||||||
AsyncStorage.setItem(VAD_SILENCE_STORAGE_KEY, String(next));
|
AsyncStorage.setItem(STT_ENDPOINT_STORAGE_KEY, String(Math.round(next * 1000)));
|
||||||
}}
|
}}
|
||||||
disabled={vadSilenceSec <= VAD_SILENCE_MIN_SEC}
|
disabled={sttEndpointSec <= STT_ENDPOINT_MIN_MS / 1000}
|
||||||
>
|
>
|
||||||
<Text style={styles.prerollButtonText}>−0.5</Text>
|
<Text style={styles.prerollButtonText}>−0.5</Text>
|
||||||
</TouchableOpacity>
|
</TouchableOpacity>
|
||||||
<Text style={styles.prerollValue}>{vadSilenceSec.toFixed(1)} s</Text>
|
<Text style={styles.prerollValue}>{sttEndpointSec.toFixed(1)} s</Text>
|
||||||
<TouchableOpacity
|
<TouchableOpacity
|
||||||
style={styles.prerollButton}
|
style={styles.prerollButton}
|
||||||
onPress={() => {
|
onPress={() => {
|
||||||
const next = Math.min(VAD_SILENCE_MAX_SEC, Math.round((vadSilenceSec + 0.5) * 10) / 10);
|
const next = Math.min(STT_ENDPOINT_MAX_MS / 1000, Math.round((sttEndpointSec + 0.5) * 10) / 10);
|
||||||
setVadSilenceSec(next);
|
setSttEndpointSec(next);
|
||||||
AsyncStorage.setItem(VAD_SILENCE_STORAGE_KEY, String(next));
|
AsyncStorage.setItem(STT_ENDPOINT_STORAGE_KEY, String(Math.round(next * 1000)));
|
||||||
}}
|
}}
|
||||||
disabled={vadSilenceSec >= VAD_SILENCE_MAX_SEC}
|
disabled={sttEndpointSec >= STT_ENDPOINT_MAX_MS / 1000}
|
||||||
>
|
>
|
||||||
<Text style={styles.prerollButtonText}>+0.5</Text>
|
<Text style={styles.prerollButtonText}>+0.5</Text>
|
||||||
</TouchableOpacity>
|
</TouchableOpacity>
|
||||||
@@ -1749,56 +1786,10 @@ const SettingsScreen: React.FC = () => {
|
|||||||
</TouchableOpacity>
|
</TouchableOpacity>
|
||||||
</View>
|
</View>
|
||||||
|
|
||||||
<View style={{flexDirection: 'row', alignItems: 'center', marginTop: 24, gap: 8}}>
|
{/* "Stille-Pegel (dB)"-Regler entfernt: der aktive Streaming-STT nutzt
|
||||||
<Text style={styles.toggleLabel}>Stille-Pegel (dB)</Text>
|
einen adaptiven Rausch-Boden (automatisch), ein manueller dB-Wert war
|
||||||
<TouchableOpacity onPress={() => setShowVadInfo(true)} style={styles.infoBtn}>
|
wirkungslos. Rauschen-als-Wort verhindert das STT-Modell selbst
|
||||||
<Text style={styles.infoBtnText}>i</Text>
|
(no_speech_prob-Filter), nicht die dB-Schwelle. */}
|
||||||
</TouchableOpacity>
|
|
||||||
</View>
|
|
||||||
<Text style={styles.toggleHint}>
|
|
||||||
Welcher Mikro-Pegel als "Stille" gilt. Standard: automatisch (Baseline aus
|
|
||||||
den ersten 500ms). Manuell setzen wenn Auto nicht zuverlaessig greift.
|
|
||||||
</Text>
|
|
||||||
<View style={styles.prerollRow}>
|
|
||||||
<TouchableOpacity
|
|
||||||
style={styles.prerollButton}
|
|
||||||
onPress={() => {
|
|
||||||
const next = vadSilenceDb == null
|
|
||||||
? VAD_SILENCE_DB_DEFAULT - 1
|
|
||||||
: Math.max(VAD_SILENCE_DB_MIN, vadSilenceDb - 1);
|
|
||||||
setVadSilenceDb(next);
|
|
||||||
AsyncStorage.setItem(VAD_SILENCE_DB_OVERRIDE_KEY, String(next));
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Text style={styles.prerollButtonText}>−1</Text>
|
|
||||||
</TouchableOpacity>
|
|
||||||
<Text style={styles.prerollValue}>
|
|
||||||
{vadSilenceDb == null ? 'auto' : `${vadSilenceDb} dB`}
|
|
||||||
</Text>
|
|
||||||
<TouchableOpacity
|
|
||||||
style={styles.prerollButton}
|
|
||||||
onPress={() => {
|
|
||||||
const next = vadSilenceDb == null
|
|
||||||
? VAD_SILENCE_DB_DEFAULT + 1
|
|
||||||
: Math.min(VAD_SILENCE_DB_MAX, vadSilenceDb + 1);
|
|
||||||
setVadSilenceDb(next);
|
|
||||||
AsyncStorage.setItem(VAD_SILENCE_DB_OVERRIDE_KEY, String(next));
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Text style={styles.prerollButtonText}>+1</Text>
|
|
||||||
</TouchableOpacity>
|
|
||||||
</View>
|
|
||||||
{vadSilenceDb != null && (
|
|
||||||
<TouchableOpacity
|
|
||||||
onPress={() => {
|
|
||||||
setVadSilenceDb(null);
|
|
||||||
AsyncStorage.removeItem(VAD_SILENCE_DB_OVERRIDE_KEY);
|
|
||||||
}}
|
|
||||||
style={{alignSelf: 'center', marginTop: 8, paddingVertical: 6, paddingHorizontal: 12}}
|
|
||||||
>
|
|
||||||
<Text style={{color: '#0096FF', fontSize: 13}}>↻ Auf automatisch zuruecksetzen</Text>
|
|
||||||
</TouchableOpacity>
|
|
||||||
)}
|
|
||||||
</View>
|
</View>
|
||||||
|
|
||||||
<Modal
|
<Modal
|
||||||
|
|||||||
@@ -152,14 +152,33 @@ export const CONV_WINDOW_MAX_SEC = 20.0;
|
|||||||
export const CONV_WINDOW_STORAGE_KEY = 'aria_conv_window_sec';
|
export const CONV_WINDOW_STORAGE_KEY = 'aria_conv_window_sec';
|
||||||
|
|
||||||
// STT-Endpoint (ms Stille bis "fertig gesprochen"). Zu kurz = schneidet mitten
|
// STT-Endpoint (ms Stille bis "fertig gesprochen"). Zu kurz = schneidet mitten
|
||||||
// im Satz ab, besonders im Auto wo man mit Pausen spricht (Reproduktion: die
|
// im Satz ab, besonders im Auto oder wenn man zum Nachdenken pausiert. 1500 war
|
||||||
// 11.8s-Frage wurde bei "…ohne dass ein" gekappt). 1500 war zu aggressiv;
|
// zu aggressiv; 2400 default, bis 8s hoch stellbar (Denkpausen). In den Settings
|
||||||
// 2400 default, im Auto ggf. hoeher. Konfigurierbar in den Settings.
|
// unter "Stille-Toleranz" konfigurierbar.
|
||||||
export const STT_ENDPOINT_DEFAULT_MS = 2400;
|
export const STT_ENDPOINT_DEFAULT_MS = 2400;
|
||||||
export const STT_ENDPOINT_MIN_MS = 1000;
|
export const STT_ENDPOINT_MIN_MS = 1000;
|
||||||
export const STT_ENDPOINT_MAX_MS = 4000;
|
export const STT_ENDPOINT_MAX_MS = 8000; // bis 8s: genug Zeit zum Ueberlegen
|
||||||
export const STT_ENDPOINT_STORAGE_KEY = 'aria_stt_endpoint_ms';
|
export const STT_ENDPOINT_STORAGE_KEY = 'aria_stt_endpoint_ms';
|
||||||
|
|
||||||
|
// Barge-in-Modus: darf man ARIA waehrend ihrer TTS-Antwort unterbrechen (reden)?
|
||||||
|
// Default AUS = sauberes Halb-Duplex (ARIA spricht aus, DANN oeffnet das Mikro —
|
||||||
|
// kein Selbst-Echo, kein Mischmasch). AN = waehrend TTS auf Wake-Wort lauschen.
|
||||||
|
export const BARGE_IN_STORAGE_KEY = 'aria_barge_in_enabled';
|
||||||
|
|
||||||
|
export async function loadBargeInEnabled(): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
return (await AsyncStorage.getItem(BARGE_IN_STORAGE_KEY)) === 'true';
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveBargeInEnabled(enabled: boolean): Promise<void> {
|
||||||
|
try {
|
||||||
|
await AsyncStorage.setItem(BARGE_IN_STORAGE_KEY, String(enabled));
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
|
||||||
export async function loadSttEndpointMs(): Promise<number> {
|
export async function loadSttEndpointMs(): Promise<number> {
|
||||||
try {
|
try {
|
||||||
const raw = await AsyncStorage.getItem(STT_ENDPOINT_STORAGE_KEY);
|
const raw = await AsyncStorage.getItem(STT_ENDPOINT_STORAGE_KEY);
|
||||||
|
|||||||
@@ -54,10 +54,10 @@ export async function savePassiveListenMs(ms: number): Promise<void> {
|
|||||||
export const WAKE_KEYWORD_STORAGE = 'aria_wake_keyword';
|
export const WAKE_KEYWORD_STORAGE = 'aria_wake_keyword';
|
||||||
|
|
||||||
// Wake-Word-Empfindlichkeit (openWakeWord-Threshold). Hoeher = strenger =
|
// Wake-Word-Empfindlichkeit (openWakeWord-Threshold). Hoeher = strenger =
|
||||||
// weniger Fehlauslösung (z.B. durch Musik/Radio ueber die Auto-Lautsprecher,
|
// weniger Fehlauslösung, aber man muss deutlicher/lauter sprechen (fuehlt sich
|
||||||
// die das Mikro mithoert — der App-Echo-Canceler kann nur ARIAs eigenes TTS
|
// "traege" an). Fehlausloeser werden ueber Speaker-ID (E3) ohnehin verworfen,
|
||||||
// rausrechnen, NICHT Spotify). Default 0.6 (war 0.5). 0..1.
|
// deshalb darf der Default empfindlicher sein. 0.45 (war 0.6/0.5). 0..1.
|
||||||
export const WAKE_THRESHOLD_DEFAULT = 0.6;
|
export const WAKE_THRESHOLD_DEFAULT = 0.45;
|
||||||
export const WAKE_THRESHOLD_MIN = 0.3;
|
export const WAKE_THRESHOLD_MIN = 0.3;
|
||||||
export const WAKE_THRESHOLD_MAX = 0.9;
|
export const WAKE_THRESHOLD_MAX = 0.9;
|
||||||
export const WAKE_THRESHOLD_STORAGE_KEY = 'aria_wake_threshold';
|
export const WAKE_THRESHOLD_STORAGE_KEY = 'aria_wake_threshold';
|
||||||
@@ -103,7 +103,9 @@ export const KEYWORD_LABELS: Record<WakeKeyword, string> = {
|
|||||||
// Detection-Tuning. Threshold ist ueber die Settings konfigurierbar
|
// Detection-Tuning. Threshold ist ueber die Settings konfigurierbar
|
||||||
// (loadWakeThreshold) — der Wert hier ist nur der Fallback.
|
// (loadWakeThreshold) — der Wert hier ist nur der Fallback.
|
||||||
const DEFAULT_THRESHOLD = WAKE_THRESHOLD_DEFAULT;
|
const DEFAULT_THRESHOLD = WAKE_THRESHOLD_DEFAULT;
|
||||||
const DEFAULT_PATIENCE = 2;
|
// patience=1 statt 2: nur EIN Frame ueber Threshold noetig → deutlich schneller.
|
||||||
|
// Speaker-ID filtert Fehlausloeser, also ist das vertretbar.
|
||||||
|
const DEFAULT_PATIENCE = 1;
|
||||||
const DEFAULT_DEBOUNCE_MS = 1500;
|
const DEFAULT_DEBOUNCE_MS = 1500;
|
||||||
|
|
||||||
interface OpenWakeWordModule {
|
interface OpenWakeWordModule {
|
||||||
@@ -310,7 +312,7 @@ class WakeWordService {
|
|||||||
/** Cooldown setzen — alle Wake-Word-Detections in den naechsten ms ignorieren.
|
/** Cooldown setzen — alle Wake-Word-Detections in den naechsten ms ignorieren.
|
||||||
* Wird beim App-Resume gerufen weil AppState-Wechsel Audio-Spikes erzeugen
|
* Wird beim App-Resume gerufen weil AppState-Wechsel Audio-Spikes erzeugen
|
||||||
* die openWakeWord faelschlich als Trigger interpretiert. */
|
* die openWakeWord faelschlich als Trigger interpretiert. */
|
||||||
setResumeCooldown(ms: number = 1500): void {
|
setResumeCooldown(ms: number = 500): void {
|
||||||
this.cooldownUntilMs = Date.now() + ms;
|
this.cooldownUntilMs = Date.now() + ms;
|
||||||
console.log('[WakeWord] Cooldown aktiv fuer %dms', ms);
|
console.log('[WakeWord] Cooldown aktiv fuer %dms', ms);
|
||||||
}
|
}
|
||||||
@@ -323,13 +325,14 @@ class WakeWordService {
|
|||||||
console.log('[WakeWord] App im Hintergrund — Detections gesperrt');
|
console.log('[WakeWord] App im Hintergrund — Detections gesperrt');
|
||||||
}
|
}
|
||||||
|
|
||||||
/** App im Vordergrund: Detections wieder freigeben, plus 3s Cooldown
|
/** App im Vordergrund: Detections wieder freigeben, plus kurzer Cooldown
|
||||||
* als Schutz gegen den AudioFocus-/AudioTrack-Spike der direkt nach
|
* als Schutz gegen den AudioFocus-/AudioTrack-Spike direkt nach dem Resume.
|
||||||
* dem Resume kommt. Ersetzt das alte setResumeCooldown(3000)-Pattern. */
|
* 1s statt 3s — 3s hat sich "traege" angefuehlt (Trigger direkt nach dem
|
||||||
|
* App-Oeffnen wurden verschluckt). */
|
||||||
setForeground(): void {
|
setForeground(): void {
|
||||||
this.inBackground = false;
|
this.inBackground = false;
|
||||||
this.cooldownUntilMs = Date.now() + 3000;
|
this.cooldownUntilMs = Date.now() + 1000;
|
||||||
console.log('[WakeWord] App im Vordergrund — Cooldown 3s aktiv');
|
console.log('[WakeWord] App im Vordergrund — Cooldown 1s aktiv');
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Wake-Word getriggert: Native-Modul pausieren, Konversation starten. */
|
/** Wake-Word getriggert: Native-Modul pausieren, Konversation starten. */
|
||||||
|
|||||||
+97
-5
@@ -1347,6 +1347,77 @@ def _extract_await_marker(text: str) -> tuple:
|
|||||||
return text, False
|
return text, False
|
||||||
|
|
||||||
|
|
||||||
|
# ── Sprach-/Gespraechs-Steuermarker (ARIA deklariert die Phase SELBST) ──
|
||||||
|
#
|
||||||
|
# Voice-First: ARIA erkennt aus dem Text, ob Stefan einen BEFEHL gibt (etwas tun)
|
||||||
|
# oder eine FRAGE stellt (etwas wissen), und ob das Gespraech/eine Befehlskette
|
||||||
|
# weiterlaeuft oder endet. Sie haengt dazu Marker ans Ende ihrer Antwort — genau
|
||||||
|
# wie [[AWAIT]], und sie werden ebenso entfernt (nicht angezeigt/vorgelesen/in
|
||||||
|
# History). Der Marker ist AUTORITATIV — er ueberschreibt das Skill-Manifest-Flag,
|
||||||
|
# denn dasselbe Skill (z.B. VM-/GUI-Steuerung) ist mal Befehl, mal Auskunft; nur
|
||||||
|
# ARIA weiss aus dem Kontext, was gerade gemeint ist.
|
||||||
|
#
|
||||||
|
# [[STUMM]] -> reiner Steuerbefehl: NICHT vorlesen (speak=false). Allein =
|
||||||
|
# Einzelbefehl → danach zurueck aufs Wake-Word (converse=false).
|
||||||
|
# [[WEITER]] -> Konversation/Befehlskette laeuft weiter: Mikro offen halten
|
||||||
|
# (converse=true) — kein erneutes "Computer" noetig.
|
||||||
|
# [[ENDE]] -> Konversation/Kette beenden: zurueck aufs Wake-Word (converse=false).
|
||||||
|
_SILENT_MARKER_RE = re.compile(r"\[\[\s*STUMM\s*\]\]", re.IGNORECASE)
|
||||||
|
_CONT_MARKER_RE = re.compile(r"\[\[\s*WEITER\s*\]\]", re.IGNORECASE)
|
||||||
|
_END_MARKER_RE = re.compile(r"\[\[\s*ENDE\s*\]\]", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_flow_markers(text: str) -> tuple:
|
||||||
|
"""Zieht [[STUMM]]/[[WEITER]]/[[ENDE]] aus dem finalen Text.
|
||||||
|
Gibt (clean_text, speak_override, converse_override) zurueck; ein Override ist
|
||||||
|
None, wenn der jeweilige Marker fehlt (dann gilt Default/Skill-Flag).
|
||||||
|
Regeln: [[STUMM]] alleine = Einzelbefehl → auch converse=false (Mikro zu),
|
||||||
|
ausser [[WEITER]] haelt es explizit offen. [[ENDE]] gewinnt gegen [[WEITER]]."""
|
||||||
|
if not text:
|
||||||
|
return text, None, None
|
||||||
|
speak_ov = None
|
||||||
|
conv_ov = None
|
||||||
|
if _SILENT_MARKER_RE.search(text):
|
||||||
|
speak_ov = False
|
||||||
|
text = _SILENT_MARKER_RE.sub("", text)
|
||||||
|
if _END_MARKER_RE.search(text):
|
||||||
|
conv_ov = False
|
||||||
|
text = _END_MARKER_RE.sub("", text)
|
||||||
|
if _CONT_MARKER_RE.search(text):
|
||||||
|
# [[ENDE]] hat Vorrang — widerspruechliche Marker → beenden.
|
||||||
|
if conv_ov is None:
|
||||||
|
conv_ov = True
|
||||||
|
text = _CONT_MARKER_RE.sub("", text)
|
||||||
|
# Stiller Einzelbefehl ohne explizites Weiterlauschen → Mikro zu.
|
||||||
|
if speak_ov is False and conv_ov is None:
|
||||||
|
conv_ov = False
|
||||||
|
return text.strip(), speak_ov, conv_ov
|
||||||
|
|
||||||
|
|
||||||
|
# Explizite "Konversation beenden"-Phrasen vom USER — deterministisch, NICHT auf
|
||||||
|
# ARIAs [[ENDE]]-Marker angewiesen. Stefan will "Konversation Ende" o.ae. als
|
||||||
|
# festen Trigger: danach zurueck aufs Wake-Word, egal was ARIA sonst tut. Eine in
|
||||||
|
# derselben Nachricht enthaltene Frage beantwortet sie normal (wird vorgelesen),
|
||||||
|
# aber converse wird auf false gezwungen. Nomen + Ende-Wort in EINEM Satzteil
|
||||||
|
# ([^.!?]{0,15}) in beliebiger Reihenfolge; "befehls?kette" damit "Lieferkette"
|
||||||
|
# o.ae. nicht faelschlich matcht.
|
||||||
|
_CONV_NOUN = r"(?:konversation|gespr[aä]ch|befehls?kette)"
|
||||||
|
_CONV_END_VERB = r"(?:ende|beenden|beende|aus|stop|stopp|schluss)"
|
||||||
|
_END_CONVERSATION_RE = re.compile(
|
||||||
|
rf"\b{_CONV_NOUN}\b[^.!?]{{0,15}}\b{_CONV_END_VERB}\b"
|
||||||
|
rf"|\b(?:beende|schlie(?:ß|ss)e?)\b[^.!?]{{0,15}}\b{_CONV_NOUN}\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _user_wants_conversation_end(text: str) -> bool:
|
||||||
|
"""True, wenn der User in dieser Nachricht explizit die Konversation/Kette
|
||||||
|
beenden will (deterministisch, unabhaengig vom LLM-Marker)."""
|
||||||
|
if not text:
|
||||||
|
return False
|
||||||
|
return bool(_END_CONVERSATION_RE.search(_strip_leading_hint_blocks(text)))
|
||||||
|
|
||||||
|
|
||||||
def _normalize_for_fast_match(text: str) -> str:
|
def _normalize_for_fast_match(text: str) -> str:
|
||||||
norm = _strip_leading_hint_blocks(text).lower()
|
norm = _strip_leading_hint_blocks(text).lower()
|
||||||
norm = _fold_umlauts(norm)
|
norm = _fold_umlauts(norm)
|
||||||
@@ -1755,6 +1826,11 @@ class Agent:
|
|||||||
if not user_message:
|
if not user_message:
|
||||||
raise ValueError("Leere Nachricht")
|
raise ValueError("Leere Nachricht")
|
||||||
|
|
||||||
|
# Expliziter "Konversation/Kette beenden"-Wunsch (deterministisch). Wird an
|
||||||
|
# JEDEM Return auf converse=false angewendet — unabhaengig davon, ob ARIA
|
||||||
|
# den [[ENDE]]-Marker setzt. Stefans fester Trigger "Konversation Ende".
|
||||||
|
_wants_end = _user_wants_conversation_end(user_message)
|
||||||
|
|
||||||
# Events vom letzten Turn weglassen
|
# Events vom letzten Turn weglassen
|
||||||
self._pending_events = []
|
self._pending_events = []
|
||||||
|
|
||||||
@@ -1782,6 +1858,8 @@ class Agent:
|
|||||||
speak = bool(getattr(self, "_fast_path_speak", False))
|
speak = bool(getattr(self, "_fast_path_speak", False))
|
||||||
# converse folgt dem Skill (Manifest/Output) — nicht mehr generell False.
|
# converse folgt dem Skill (Manifest/Output) — nicht mehr generell False.
|
||||||
converse = bool(getattr(self, "_fast_path_converse", False))
|
converse = bool(getattr(self, "_fast_path_converse", False))
|
||||||
|
if _wants_end:
|
||||||
|
converse = False
|
||||||
# Fast-Path = reiner Steuerbefehl, nie eine Rueckfrage → awaiting=False.
|
# Fast-Path = reiner Steuerbefehl, nie eine Rueckfrage → awaiting=False.
|
||||||
return fast_reply, "fast-path", speak, converse, False
|
return fast_reply, "fast-path", speak, converse, False
|
||||||
|
|
||||||
@@ -1801,6 +1879,8 @@ class Agent:
|
|||||||
# dem Skill (bzw. Default: Info/Gespraech = vorlesen + 30s).
|
# dem Skill (bzw. Default: Info/Gespraech = vorlesen + 30s).
|
||||||
speak = getattr(self, "_local_turn_speak", True)
|
speak = getattr(self, "_local_turn_speak", True)
|
||||||
converse = getattr(self, "_local_turn_converse", True)
|
converse = getattr(self, "_local_turn_converse", True)
|
||||||
|
if _wants_end:
|
||||||
|
converse = False
|
||||||
# Local ist tool-loses Reden; blockierende Rueckfragen macht Claude.
|
# Local ist tool-loses Reden; blockierende Rueckfragen macht Claude.
|
||||||
return local_reply, "local", speak, converse, False
|
return local_reply, "local", speak, converse, False
|
||||||
|
|
||||||
@@ -2033,16 +2113,28 @@ class Agent:
|
|||||||
# Rueckfrage-Marker aus dem finalen Text ziehen (vor History/Return, damit
|
# Rueckfrage-Marker aus dem finalen Text ziehen (vor History/Return, damit
|
||||||
# er nicht angezeigt/vorgelesen wird und nicht die Conversation vergiftet).
|
# er nicht angezeigt/vorgelesen wird und nicht die Conversation vergiftet).
|
||||||
final_reply, awaiting_reply = _extract_await_marker(final_reply)
|
final_reply, awaiting_reply = _extract_await_marker(final_reply)
|
||||||
|
# ARIAs Phasen-Marker ([[STUMM]]/[[WEITER]]/[[ENDE]]) ziehen — VOR History,
|
||||||
|
# damit sie nicht angezeigt/vorgelesen/gespeichert werden.
|
||||||
|
final_reply, _speak_ov, _conv_ov = _extract_flow_markers(final_reply)
|
||||||
|
|
||||||
# 7. Assistant-Turn (final reply) in die Conversation
|
# 7. Assistant-Turn (final reply) in die Conversation
|
||||||
self.conversation.add("assistant", final_reply,
|
self.conversation.add("assistant", final_reply,
|
||||||
project_id=active_project_id)
|
project_id=active_project_id)
|
||||||
# speak/converse folgen dem ausgefuehrten Skill (sonst Default: Gespraech);
|
# speak/converse folgen dem ausgefuehrten Skill (sonst Default: Gespraech).
|
||||||
|
# ARIAs Phasen-Marker sind AUTORITATIV: sie kennt aus dem Text den Unter-
|
||||||
|
# schied Befehl/Frage und Kette/Ende, den das Skill-Manifest nicht kennt.
|
||||||
|
speak = bool(getattr(self, "_claude_turn_speak", True))
|
||||||
|
converse = bool(getattr(self, "_claude_turn_converse", True))
|
||||||
|
if _speak_ov is not None:
|
||||||
|
speak = _speak_ov
|
||||||
|
if _conv_ov is not None:
|
||||||
|
converse = _conv_ov
|
||||||
|
# Expliziter User-Wunsch "Konversation beenden" gewinnt IMMER — Frage wird
|
||||||
|
# noch beantwortet (speak bleibt), aber danach zurueck aufs Wake-Word.
|
||||||
|
if _wants_end:
|
||||||
|
converse = False
|
||||||
# awaiting_reply = ARIA stellt eine blockierende Rueckfrage (Queue pausiert).
|
# awaiting_reply = ARIA stellt eine blockierende Rueckfrage (Queue pausiert).
|
||||||
return (final_reply, "claude",
|
return (final_reply, "claude", speak, converse, awaiting_reply)
|
||||||
bool(getattr(self, "_claude_turn_speak", True)),
|
|
||||||
bool(getattr(self, "_claude_turn_converse", True)),
|
|
||||||
awaiting_reply)
|
|
||||||
|
|
||||||
# ── Tool-Dispatcher ───────────────────────────────────────
|
# ── Tool-Dispatcher ───────────────────────────────────────
|
||||||
|
|
||||||
|
|||||||
+42
-1
@@ -162,6 +162,46 @@ def build_time_section() -> str:
|
|||||||
]
|
]
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
def build_voice_flow_section() -> str:
|
||||||
|
"""Sprach-/Gespraechssteuerung: ARIA erkennt AUS DEM TEXT die Phase (Befehl vs.
|
||||||
|
Frage, Kette vs. Ende) und deklariert sie per Marker — wie [[AWAIT]]. Die
|
||||||
|
Marker werden im Brain entfernt (nie angezeigt/vorgelesen)."""
|
||||||
|
return "\n".join([
|
||||||
|
"## Sprach- & Gespraechssteuerung (Voice-First — du entscheidest die Phase)",
|
||||||
|
"Stefan spricht meist mit dir. DU erkennst aus dem Text, was gerade Phase "
|
||||||
|
"ist — niemand raet das fuer dich. Dazu haengst du EINEN Marker (bei Bedarf "
|
||||||
|
"zwei) ganz ans ENDE deiner Antwort. Sie werden entfernt: nicht angezeigt, "
|
||||||
|
"nicht vorgelesen, nicht gespeichert — genau wie `[[AWAIT]]`.",
|
||||||
|
"",
|
||||||
|
"- `[[STUMM]]` → Deine Antwort ist ein reiner **Steuerbefehl** (du hast etwas "
|
||||||
|
"GETAN: Musik, VNC oeffnen, einen Menuepunkt klicken, Licht …). Sie wird "
|
||||||
|
"NICHT vorgelesen; der kurze Bestaetigungstext steht nur in der Bubble. "
|
||||||
|
"Setz das IMMER, wenn Stefan dir einen Befehl gibt statt eine Frage stellt — "
|
||||||
|
"AUCH wenn du den Befehl ueber ein Skill/Tool ausfuehrst (nicht nur beim "
|
||||||
|
"Fast-Path). `[[STUMM]]` ALLEIN = Einzelbefehl → danach direkt zurueck aufs "
|
||||||
|
"Wake-Word.",
|
||||||
|
"- `[[WEITER]]` → Das Gespraech bzw. eine **Befehlskette** laeuft weiter: das "
|
||||||
|
"Mikro bleibt offen, du wartest auf die naechste Eingabe (kein erneutes "
|
||||||
|
"\"Computer\" noetig). Setz das, wenn Stefan eine Kette ankuendigt ('ich geb "
|
||||||
|
"dir gleich mehrere Befehle', 'wir machen das jetzt Schritt fuer Schritt') "
|
||||||
|
"oder das Gespraech klar weitergeht.",
|
||||||
|
"- `[[ENDE]]` → Konversation/Kette ist zu Ende: zurueck aufs Wake-Word. Setz "
|
||||||
|
"das, wenn Stefan schliesst ('das war's', 'Konversation Ende', 'Befehlskette "
|
||||||
|
"Ende', 'danke, fertig'). Stellt er in DERSELBEN Nachricht noch eine Frage, "
|
||||||
|
"beantworte sie normal (OHNE `[[STUMM]]`, wird also vorgelesen) UND haeng "
|
||||||
|
"`[[ENDE]]` an.",
|
||||||
|
"",
|
||||||
|
"Regeln:",
|
||||||
|
"- Befehl (etwas TUN) → `[[STUMM]]`. Frage (etwas WISSEN / plaudern) → normal, "
|
||||||
|
"ohne Marker (wird vorgelesen).",
|
||||||
|
"- Befehlskette: JEDER Schritt `[[STUMM]] [[WEITER]]` (stumm arbeiten, Mikro "
|
||||||
|
"offen), bis Stefan die Kette beendet → letzter Turn `[[ENDE]]`.",
|
||||||
|
"- Ohne Marker = normales Gespraech: du wirst vorgelesen und ich lausche "
|
||||||
|
"danach kurz weiter (Stefan kann einfach antworten, ohne 'Computer').",
|
||||||
|
"- Nie widerspruechlich: `[[ENDE]]` schlaegt `[[WEITER]]`.",
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
TYPE_HEADINGS = {
|
TYPE_HEADINGS = {
|
||||||
"identity": "## Wer du bist",
|
"identity": "## Wer du bist",
|
||||||
"rule": "## Sicherheitsregeln & Prinzipien",
|
"rule": "## Sicherheitsregeln & Prinzipien",
|
||||||
@@ -463,7 +503,8 @@ def build_system_prompt(
|
|||||||
"""Kompletter System-Prompt: Hot + Cold + Skills + Triggers + FLUX + OAuth."""
|
"""Kompletter System-Prompt: Hot + Cold + Skills + Triggers + FLUX + OAuth."""
|
||||||
# Identitaets-Anker IMMER zuerst — vor allen Memories/Sektionen, damit die
|
# Identitaets-Anker IMMER zuerst — vor allen Memories/Sektionen, damit die
|
||||||
# ARIA-Rolle auch in Projekten mit injection-artigem Inhalt (Pentest) haelt.
|
# ARIA-Rolle auch in Projekten mit injection-artigem Inhalt (Pentest) haelt.
|
||||||
parts = [IDENTITY_ANCHOR, "", build_hot_memory_section(pinned), "", build_time_section()]
|
parts = [IDENTITY_ANCHOR, "", build_hot_memory_section(pinned), "", build_time_section(),
|
||||||
|
"", build_voice_flow_section()]
|
||||||
if skills:
|
if skills:
|
||||||
parts.append("")
|
parts.append("")
|
||||||
parts.append(build_skills_section(skills))
|
parts.append(build_skills_section(skills))
|
||||||
|
|||||||
+19
-1
@@ -788,6 +788,18 @@
|
|||||||
<div id="voice-id-status" style="font-size:13px;color:#E0E0F0;margin-bottom:10px;">
|
<div id="voice-id-status" style="font-size:13px;color:#E0E0F0;margin-bottom:10px;">
|
||||||
Status wird geladen...
|
Status wird geladen...
|
||||||
</div>
|
</div>
|
||||||
|
<div style="display:flex;align-items:center;gap:12px;margin-bottom:8px;">
|
||||||
|
<label style="color:#8888AA;font-size:12px;min-width:130px;">Nur meine Stimme:</label>
|
||||||
|
<label style="display:flex;align-items:center;gap:8px;cursor:pointer;flex:1;">
|
||||||
|
<input type="checkbox" id="diag-voice-id-enabled" onchange="sendVoiceConfig()">
|
||||||
|
<span style="color:#E0E0F0;font-size:12px;">Speaker-ID-Prüfung aktiv</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div style="font-size:10px;color:#555570;margin-bottom:12px;">
|
||||||
|
AUS (Default) = alle Stimmen kommen durch (fail-open). AN = nur der enrollte
|
||||||
|
Sprecher wird ans Brain geleitet, fremde Stimmen werden verworfen. Erst
|
||||||
|
einschalten wenn ein Fingerprint eingelernt ist — sonst hört ARIA niemanden.
|
||||||
|
</div>
|
||||||
<div style="display:flex;align-items:center;gap:12px;margin-bottom:8px;">
|
<div style="display:flex;align-items:center;gap:12px;margin-bottom:8px;">
|
||||||
<label style="color:#8888AA;font-size:12px;min-width:130px;">Match-Threshold:</label>
|
<label style="color:#8888AA;font-size:12px;min-width:130px;">Match-Threshold:</label>
|
||||||
<input type="range" id="diag-voice-id-threshold" min="0.30" max="0.70" step="0.05" value="0.50"
|
<input type="range" id="diag-voice-id-threshold" min="0.30" max="0.70" step="0.05" value="0.50"
|
||||||
@@ -1899,6 +1911,11 @@
|
|||||||
if (slider) slider.value = msg.voiceIdThreshold;
|
if (slider) slider.value = msg.voiceIdThreshold;
|
||||||
if (display) display.textContent = Number(msg.voiceIdThreshold).toFixed(2);
|
if (display) display.textContent = Number(msg.voiceIdThreshold).toFixed(2);
|
||||||
}
|
}
|
||||||
|
// Speaker-ID Gating-Schalter wiederherstellen (Default aus)
|
||||||
|
{
|
||||||
|
const cb = document.getElementById('diag-voice-id-enabled');
|
||||||
|
if (cb) cb.checked = !!msg.voiceIdEnabled;
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3600,13 +3617,14 @@
|
|||||||
const huggingfaceToken = document.getElementById('diag-flux-hf-token')?.value;
|
const huggingfaceToken = document.getElementById('diag-flux-hf-token')?.value;
|
||||||
const voiceIdThresholdRaw = document.getElementById('diag-voice-id-threshold')?.value;
|
const voiceIdThresholdRaw = document.getElementById('diag-voice-id-threshold')?.value;
|
||||||
const voiceIdThreshold = voiceIdThresholdRaw ? parseFloat(voiceIdThresholdRaw) : undefined;
|
const voiceIdThreshold = voiceIdThresholdRaw ? parseFloat(voiceIdThresholdRaw) : undefined;
|
||||||
|
const voiceIdEnabled = document.getElementById('diag-voice-id-enabled')?.checked;
|
||||||
send({
|
send({
|
||||||
action: 'send_voice_config',
|
action: 'send_voice_config',
|
||||||
ttsEnabled, xttsVoice, whisperModel,
|
ttsEnabled, xttsVoice, whisperModel,
|
||||||
f5ttsModel, f5ttsCkptFile, f5ttsVocabFile,
|
f5ttsModel, f5ttsCkptFile, f5ttsVocabFile,
|
||||||
f5ttsCfgStrength, f5ttsNfeStep,
|
f5ttsCfgStrength, f5ttsNfeStep,
|
||||||
fluxDefaultModel, fluxKeywordRaw, fluxKeywordSwitch, huggingfaceToken,
|
fluxDefaultModel, fluxKeywordRaw, fluxKeywordSwitch, huggingfaceToken,
|
||||||
voiceIdThreshold,
|
voiceIdThreshold, voiceIdEnabled,
|
||||||
});
|
});
|
||||||
const statusEl = document.getElementById('voice-status');
|
const statusEl = document.getElementById('voice-status');
|
||||||
if (statusEl && xttsVoice) {
|
if (statusEl && xttsVoice) {
|
||||||
|
|||||||
@@ -2681,6 +2681,12 @@ wss.on("connection", (ws) => {
|
|||||||
const t = parseFloat(msg.voiceIdThreshold);
|
const t = parseFloat(msg.voiceIdThreshold);
|
||||||
if (t >= 0.0 && t <= 1.0) voiceConfig.voiceIdThreshold = t;
|
if (t >= 0.0 && t <= 1.0) voiceConfig.voiceIdThreshold = t;
|
||||||
}
|
}
|
||||||
|
// Speaker-ID Gating an/aus ("nur meine Stimme"). Default aus (fail-open) —
|
||||||
|
// bewusster Schalter. voxtral/whisper-bridge lesen voiceIdEnabled aus dem
|
||||||
|
// config-Broadcast; aus = gar keine Pruefung.
|
||||||
|
if (msg.voiceIdEnabled !== undefined) {
|
||||||
|
voiceConfig.voiceIdEnabled = !!msg.voiceIdEnabled;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
fs.mkdirSync("/shared/config", { recursive: true });
|
fs.mkdirSync("/shared/config", { recursive: true });
|
||||||
fs.writeFileSync("/shared/config/voice_config.json", JSON.stringify(voiceConfig, null, 2));
|
fs.writeFileSync("/shared/config/voice_config.json", JSON.stringify(voiceConfig, null, 2));
|
||||||
|
|||||||
+15
-36
@@ -63,6 +63,7 @@ services:
|
|||||||
whisper-bridge:
|
whisper-bridge:
|
||||||
build: ./whisper
|
build: ./whisper
|
||||||
container_name: aria-whisper-bridge
|
container_name: aria-whisper-bridge
|
||||||
|
profiles: ["whisper"] # Fallback-STT — startet nur mit --profile whisper
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
reservations:
|
reservations:
|
||||||
@@ -138,55 +139,33 @@ services:
|
|||||||
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600}
|
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
# ─── Voxtral STT (GPU, Realtime) — PROFIL "voxtral" ───────────
|
# ─── Voxtral STT-3B (Transformers, GPU) — DEFAULT-STT ─────────
|
||||||
# Ersetzt whisper als STT sobald die 24-GB-Karte da ist. Startet NUR mit
|
# Ersetzt whisper als STT. Laeuft auf Treiber 550/CUDA 12.4 (torch cu124, KEIN
|
||||||
# docker compose --profile voxtral up -d
|
# Treiber-Upgrade noetig). Modell Voxtral-Mini-3B-2507 (~9 GB bf16) → GPU 1.
|
||||||
# (sonst kollidiert es mit whisper — beide wuerden stt_* beantworten).
|
# Startet bei jedem `docker compose up -d`. Whisper ist der opt-in Fallback
|
||||||
#
|
# (Profil "whisper") — beide zusammen wuerden stt_* doppelt beantworten, also
|
||||||
# ⚠️ VRAM: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB (BF16, laut vLLM-
|
# immer nur EINEN STT laufen lassen.
|
||||||
# Rezept keine Quant). Laeuft NICHT auf der 3060 (12 GB) — erst 24-GB-Karte.
|
voxtral-bridge:
|
||||||
# ⚠️ vLLM: Version >=0.20.0 noetig. Entrypoint/Serve-Form beim ersten Lauf
|
build: ./voxtral
|
||||||
# gegen das offizielle Rezept pruefen (siehe voxtral/README.md).
|
container_name: aria-voxtral-bridge
|
||||||
voxtral-vllm:
|
|
||||||
image: vllm/vllm-openai:latest
|
|
||||||
container_name: aria-voxtral-vllm
|
|
||||||
profiles: ["voxtral"]
|
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
reservations:
|
reservations:
|
||||||
devices:
|
devices:
|
||||||
- driver: nvidia
|
- driver: nvidia
|
||||||
count: 1
|
device_ids: ["1"] # 12-GB-Karte (STT-3B ~9 GB); GPU 0 (8 GB) bleibt fuer F5/LLM
|
||||||
capabilities: [gpu]
|
capabilities: [gpu]
|
||||||
volumes:
|
volumes:
|
||||||
- ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5
|
- ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5
|
||||||
environment:
|
- ./voice-id:/voice-id # Speaker-Fingerprint (wie whisper)
|
||||||
- VLLM_DISABLE_COMPILE_CACHE=1
|
|
||||||
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
|
||||||
# Serve-Command aus dem vLLM-Rezept (Voxtral-Mini-4B-Realtime-2602).
|
|
||||||
command:
|
|
||||||
- --model
|
|
||||||
- mistralai/Voxtral-Mini-4B-Realtime-2602
|
|
||||||
- --tokenizer-mode
|
|
||||||
- mistral
|
|
||||||
- --compilation_config
|
|
||||||
- '{"cudagraph_mode":"PIECEWISE"}'
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
# ─── Voxtral-Bridge — RVS <-> vLLM-Realtime-WS (CPU-Glue) ─────
|
|
||||||
voxtral-bridge:
|
|
||||||
build: ./voxtral
|
|
||||||
container_name: aria-voxtral-bridge
|
|
||||||
profiles: ["voxtral"]
|
|
||||||
depends_on:
|
|
||||||
- voxtral-vllm
|
|
||||||
environment:
|
environment:
|
||||||
- RVS_HOST=${RVS_HOST}
|
- RVS_HOST=${RVS_HOST}
|
||||||
- RVS_PORT=${RVS_PORT:-443}
|
- RVS_PORT=${RVS_PORT:-443}
|
||||||
- RVS_TLS=${RVS_TLS:-true}
|
- RVS_TLS=${RVS_TLS:-true}
|
||||||
- RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true}
|
- RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true}
|
||||||
- RVS_TOKEN=${RVS_TOKEN}
|
- RVS_TOKEN=${RVS_TOKEN}
|
||||||
- VOXTRAL_VLLM_URL=ws://voxtral-vllm:8000/v1/realtime
|
- VOXTRAL_MODEL=mistralai/Voxtral-Mini-3B-2507
|
||||||
- VOXTRAL_MODEL=mistralai/Voxtral-Mini-4B-Realtime-2602
|
|
||||||
- VOXTRAL_LANGUAGE=${WHISPER_LANGUAGE:-de}
|
- VOXTRAL_LANGUAGE=${WHISPER_LANGUAGE:-de}
|
||||||
|
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} # falls das Modell HF-gated ist
|
||||||
|
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True # weniger VRAM-Fragmentierung
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
+19
-7
@@ -1,14 +1,26 @@
|
|||||||
# Voxtral-BRIDGE (nicht das Modell!) — leichte CPU-Glue zwischen RVS und dem
|
# Voxtral-STT-3B Bridge (Transformers). Laeuft auf Treiber 550/CUDA 12.4 via
|
||||||
# vLLM-Realtime-Server. Das eigentliche Voxtral-Modell laeuft im Container
|
# torch cu124 — KEIN Treiber-Upgrade noetig (gleicher Trick wie f5tts).
|
||||||
# `voxtral-vllm` (GPU, vllm/vllm-openai). Deshalb hier kein CUDA-Base noetig.
|
# Modell: Voxtral-Mini-3B-2507 (~9 GB in bf16) → passt auf die 12-GB-Karte (GPU 1).
|
||||||
FROM python:3.11-slim
|
FROM nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
ENV PYTHONUNBUFFERED=1
|
ENV PYTHONUNBUFFERED=1
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY requirements.txt .
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
python3 python3-pip ffmpeg git \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY bridge.py ./
|
# torch FEST auf cu124 (Treiber 550 = CUDA 12.4). 2.6.0 ist der neueste cu124-Build;
|
||||||
|
# torch 2.7+ gibt es nur fuer cu126+ und braeuchte einen neueren Treiber.
|
||||||
|
RUN pip3 install --no-cache-dir torch==2.6.0 torchaudio==2.6.0 \
|
||||||
|
--index-url https://download.pytorch.org/whl/cu124
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
# Constraint haelt transformers/mistral-common davon ab, torch wieder hochzuziehen.
|
||||||
|
RUN printf 'torch==2.6.0\ntorchaudio==2.6.0\n' > /tmp/torch-constraint.txt && \
|
||||||
|
pip3 install --no-cache-dir -c /tmp/torch-constraint.txt -r requirements.txt
|
||||||
|
|
||||||
|
COPY bridge.py speaker_id.py ./
|
||||||
|
|
||||||
CMD ["python3", "bridge.py"]
|
CMD ["python3", "bridge.py"]
|
||||||
|
|||||||
+31
-54
@@ -1,70 +1,47 @@
|
|||||||
# Voxtral-STT-Satellit (M0.3)
|
# Voxtral-STT-3B-Satellit (Transformers)
|
||||||
|
|
||||||
Streaming-STT via **Voxtral-Mini-4B-Realtime-2602** (Mistral, Apache 2.0) auf
|
Streaming-STT via **Voxtral-Mini-3B-2507** (Mistral, Apache 2.0) über
|
||||||
vLLM. Ersetzt whisper als STT — genauer (~5,9 % WER vs 7,4 % FLEURS) und mit
|
🤗 Transformers. Ersetzt whisper als STT — genauer, und **ohne Treiber-Upgrade**:
|
||||||
echtem Realtime-Streaming. Deutsch ist in den 13 Sprachen abgedeckt.
|
läuft auf dem Trixie-Standardtreiber (550/CUDA 12.4) via **torch 2.6.0+cu124**
|
||||||
|
(derselbe Trick wie bei f5tts).
|
||||||
|
|
||||||
Zwei Container:
|
- Modell ~9 GB (bf16) → **GPU 1** (die 12-GB-Karte; per Compose gepinnt).
|
||||||
- **`voxtral-vllm`** — das Modell auf vLLM (GPU). Exponiert die Realtime-WS-API.
|
- **F5-TTS + LLM** bleiben auf GPU 0 (8 GB).
|
||||||
- **`voxtral-bridge`** — CPU-Glue: RVS ⇄ vLLM-Realtime-WS. Macht das Endpointing
|
- Arbeitsweise = chunked wie whisper: live PCM → alle ~1 s transkribieren
|
||||||
selbst (adaptiver Rausch-Boden-VAD, identisch zur whisper-Bridge / M0.1).
|
(Partials) → **adaptiver Endpointer** (Rausch-Boden-VAD + semantische
|
||||||
|
Stagnation, aus M0.1) feuert `stt_endpoint`. RVS-Protokoll identisch → drop-in.
|
||||||
|
|
||||||
## ⚠️ Hardware-Realität — läuft NICHT auf der 3060
|
## Starten (Profil `voxtral`)
|
||||||
|
|
||||||
Das Realtime-Modell braucht laut [vLLM-Rezept](https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602)
|
|
||||||
**≥ 16 GB VRAM (BF16, keine Quant)**. Die RTX 3060 hat 12 GB → passt nicht.
|
|
||||||
|
|
||||||
- **Interim-gpubox (nur 3060):** whisper (mit M0.1-Fix) + F5-TTS bleiben aktiv.
|
|
||||||
Voxtral NICHT starten.
|
|
||||||
- **Ab der 24-GB-Karte:** Voxtral-Profil hochziehen, whisper wird Fallback.
|
|
||||||
|
|
||||||
Deshalb liegen beide Services hinter dem Compose-**Profil `voxtral`** und starten
|
|
||||||
NUR explizit — sonst würden whisper *und* voxtral dieselben `stt_*`-Messages
|
|
||||||
beantworten (Kollision).
|
|
||||||
|
|
||||||
## Starten (erst wenn die 24-GB-Karte drin ist)
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd xtts
|
cd xtts
|
||||||
|
docker compose stop whisper-bridge # sonst beantworten beide stt_*
|
||||||
docker compose --profile voxtral up -d --build
|
docker compose --profile voxtral up -d --build
|
||||||
docker logs -f aria-voxtral-vllm # laedt Modell (mehrere GB, dauert)
|
docker logs -f aria-voxtral-bridge # Modell laedt (mehrere GB), dann "RVS verbunden"
|
||||||
docker logs -f aria-voxtral-bridge # "RVS verbunden" + service_status ready
|
|
||||||
```
|
|
||||||
Whisper vorher stoppen, damit nur eine STT-Engine antwortet:
|
|
||||||
```bash
|
|
||||||
docker compose stop whisper-bridge
|
|
||||||
```
|
```
|
||||||
|
Zurück zu whisper: `docker compose --profile voxtral down && docker compose up -d whisper-bridge`.
|
||||||
|
|
||||||
## ⚠️ Auf echter Hardware verifizieren (blind gebaut, kein Test hier)
|
## ⚠️ Auf echter Hardware verifizieren (blind gebaut)
|
||||||
|
|
||||||
1. **vLLM-Version ≥ 0.20.0** und die **Serve-Form**. Das Rezept nutzt
|
1. **Transformers-API.** Die exakte Voxtral-Transkriptions-API ist in `bridge.py`
|
||||||
`vllm serve <model> …`. Falls das `vllm/vllm-openai`-Image einen anderen
|
in **einer** Methode gekapselt (`VoxtralRunner._transcribe_blocking`), modelliert
|
||||||
Entrypoint hat, das `command:` in `docker-compose.yml` anpassen
|
nach der HF-Modelcard (`apply_transcription_request` → `generate` → `batch_decode`).
|
||||||
(Rezept-Command steht dort als Kommentar).
|
Beim ersten Lauf gegen die Modelcard prüfen und dort anpassen.
|
||||||
2. **Realtime-WS-Frames.** Die exakten Event-Namen sind in `bridge.py` ganz oben
|
2. **HF-Gating.** Ist `Voxtral-Mini-3B-2507` gated, `HF_TOKEN` in `xtts/.env` setzen
|
||||||
als Konstanten gebündelt (`VLLM_SEND_APPEND`, `VLLM_DELTA_SUFFIXES`, …),
|
(wird als `HUGGING_FACE_HUB_TOKEN` durchgereicht).
|
||||||
modelliert nach dem OpenAI-Realtime-Schema. Gegen das offizielle
|
3. **VRAM/Tempo.** 3B in bf16 ~9 GB auf der 12-GB-Karte — Rest fürs KV-Cache. Ist die
|
||||||
**vLLM-Realtime-Client-Beispiel** prüfen und dort anpassen — nur an dieser
|
Partial-Transkription (alle 1 s) zu schwer, `STREAM_TRANSCRIBE_INTERVAL_MS` hochsetzen.
|
||||||
einen Stelle. Das Response-Handling ist bereits defensiv (mehrere Feldnamen).
|
4. **torch-Konflikt.** Falls `transformers`/`mistral-common` beim Build torch>2.6
|
||||||
3. **Endpoint-URL/Port.** Default `ws://voxtral-vllm:8000/v1/realtime` — prüfen ob
|
erzwingen, meldet der Constraint einen Konflikt → dann brauchen wir doch das
|
||||||
vLLM auf 8000 lauscht und `/v1/realtime` registriert (Log-Zeile
|
Treiber-Upgrade (`bootstrap.sh --upgrade-driver` via NVIDIA-CUDA-Repo) + cu126-torch.
|
||||||
`Route: /v1/realtime`).
|
|
||||||
4. **Endpointing.** Voxtral liefert keine eigene VAD → unser adaptiver Endpointer
|
|
||||||
entscheidet (akustisch + semantisch am Delta-Wachstum). `endpointMs` kommt wie
|
|
||||||
bei whisper aus der App; `voiceFactor` per Session tunebar.
|
|
||||||
|
|
||||||
## Protokoll (RVS, identisch zu whisper — drop-in)
|
## Protokoll (RVS, identisch zu whisper — drop-in)
|
||||||
|
|
||||||
Rein: `stt_stream_start`, `stt_audio_chunk` (16 kHz mono s16le, base64), `stt_stream_end`.
|
Rein: `stt_stream_start`, `stt_audio_chunk` (16 kHz mono s16le, base64), `stt_stream_end`.
|
||||||
Raus: `stt_partial`, `stt_endpoint` (das Event, auf das aria-bridge horcht), `stt_stream_done`.
|
Raus: `stt_partial`, `stt_endpoint`, `stt_stream_done`.
|
||||||
|
|
||||||
## TTS-Hinweis
|
## TTS
|
||||||
|
Bleibt **F5-TTS** (klingt gut, passt auf GPU 0). Voxtral-TTS bräuchte ~24 GB — separates Thema.
|
||||||
Voxtral-**TTS** (Voice-Cloning) ist hier NICHT enthalten — braucht ebenfalls
|
|
||||||
16 GB VRAM und ist ein eigener Bau. Bis zur 24-GB-Karte bleibt **F5-TTS** aktiv.
|
|
||||||
Danach: eigener `voxtral-tts`-Satellit (separates Ticket).
|
|
||||||
|
|
||||||
## Quellen
|
## Quellen
|
||||||
- Rezept: https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602
|
- Modell: https://huggingface.co/mistralai/Voxtral-Mini-3B-2507
|
||||||
- vLLM Speech-to-Text: https://docs.vllm.ai/en/latest/serving/online_serving/speech_to_text/
|
- Transformers-Nutzung: HF-Modelcard (Voxtral) + `mistral-common`
|
||||||
- Modell: https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602
|
|
||||||
|
|||||||
+324
-233
@@ -1,52 +1,45 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
ARIA Voxtral Bridge — Streaming-STT via Voxtral-Mini-4B-Realtime-2602 (vLLM).
|
ARIA Voxtral-STT-3B Bridge (Transformers) — Ersatz fuer whisper.
|
||||||
|
|
||||||
Zwilling der whisper-Bridge, aber das Transkribieren macht NICHT faster-whisper
|
Laeuft auf Treiber 550/CUDA 12.4 via torch cu124 (kein Treiber-Upgrade noetig).
|
||||||
im selben Prozess, sondern der separate vLLM-Realtime-Server (Container
|
Modell: Voxtral-Mini-3B-2507 (bf16, ~9 GB) → GPU 1 (12 GB, per Compose gepinnt).
|
||||||
`voxtral-vllm`) ueber dessen WebSocket-API `/v1/realtime`. Diese Bridge ist
|
|
||||||
reine Glue:
|
|
||||||
|
|
||||||
App ──(RVS: stt_stream_start / stt_audio_chunk / stt_stream_end)──▶ diese Bridge
|
Arbeitsweise = Zwilling der whisper-Bridge: App schickt live PCM-Chunks; wir
|
||||||
diese Bridge ──(WS /v1/realtime: PCM16-b64 append)──▶ voxtral-vllm
|
transkribieren alle ~STREAM_TRANSCRIBE_INTERVAL_MS auf dem Ringbuffer (Partials)
|
||||||
voxtral-vllm ──(transcription.delta / transcription.done)──▶ diese Bridge
|
und feuern stt_endpoint, sobald der ADAPTIVE Endpointer (Rausch-Boden-VAD +
|
||||||
diese Bridge ──(RVS: stt_partial / stt_endpoint / stt_stream_done)──▶ App/aria-bridge
|
semantische Stagnation, aus M0.1) "fertig" sagt. RVS-Wire-Protokoll identisch zu
|
||||||
|
whisper → drop-in (die App merkt nur bessere Genauigkeit).
|
||||||
|
|
||||||
Das RVS-Wire-Protokoll ist IDENTISCH zur whisper-Bridge (drop-in). Das
|
⚠️ VERIFY-ON-FIRST-RUN: Die exakte Transformers-Transkriptions-API von Voxtral
|
||||||
Endpointing (wann hat der User aufgehoert zu sprechen) macht diese Bridge
|
(apply_transcription_request / generate / decode) ist unten in EINER Methode
|
||||||
selbst — mit demselben ADAPTIVEN Rausch-Boden-Endpointer wie whisper (Voxtral
|
(VoxtralRunner._transcribe_blocking) gekapselt und nach dem HF-Modelcard-Muster
|
||||||
Realtime liefert laut vLLM-Doku keine eigene VAD/„speaker done"-Semantik, nur
|
modelliert. Beim ersten echten Lauf gegen die Voxtral-Modelcard pruefen und dort
|
||||||
transcription.delta/.done). Die akustische Energie messen wir auf unserer
|
anpassen. Alles andere (RVS, Endpointer) ist bewaehrt.
|
||||||
eigenen PCM-Kopie, die semantische Stagnation am Delta-Textwachstum.
|
|
||||||
|
|
||||||
⚠️ HARDWARE: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB VRAM (BF16). Auf der
|
|
||||||
RTX 3060 (12 GB) laeuft es NICHT — erst auf der 24-GB-Karte. Bis dahin
|
|
||||||
bleibt die whisper-Bridge aktiv (Profil-gesteuert im docker-compose).
|
|
||||||
|
|
||||||
⚠️ VERIFY-ON-FIRST-RUN: Die exakten vLLM-Realtime-FRAME-Namen (Audio-Append,
|
|
||||||
Delta/Done-Event-Typen) sind unten als Konstanten gebuendelt und nach dem
|
|
||||||
OpenAI-Realtime-Schema modelliert. Gegen das offizielle vLLM-Realtime-
|
|
||||||
Client-Beispiel pruefen und ggf. anpassen — sie stehen bewusst an EINER
|
|
||||||
Stelle. Response-Handling ist defensiv (mehrere moegliche Feldnamen).
|
|
||||||
|
|
||||||
Env:
|
Env:
|
||||||
RVS_HOST, RVS_PORT, RVS_TLS, RVS_TLS_FALLBACK, RVS_TOKEN
|
RVS_HOST, RVS_PORT, RVS_TLS, RVS_TLS_FALLBACK, RVS_TOKEN
|
||||||
VOXTRAL_VLLM_URL Default: ws://voxtral-vllm:8000/v1/realtime
|
VOXTRAL_MODEL Default: mistralai/Voxtral-Mini-3B-2507
|
||||||
VOXTRAL_MODEL Default: mistralai/Voxtral-Mini-4B-Realtime-2602
|
VOXTRAL_LANGUAGE Default: de
|
||||||
VOXTRAL_LANGUAGE Default: de
|
VOXTRAL_DEVICE Default: cuda
|
||||||
|
STREAM_TRANSCRIBE_INTERVAL_MS Default 1000 (3B ist schwerer als whisper-small)
|
||||||
"""
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
import soundfile as sf
|
||||||
import websockets
|
import websockets
|
||||||
|
|
||||||
|
import speaker_id # Speaker-ID (nur Stefans Stimme) — portiert aus der whisper-Bridge
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||||
@@ -60,52 +53,112 @@ RVS_TLS = os.getenv("RVS_TLS", "true").lower() == "true"
|
|||||||
RVS_TLS_FALLBACK = os.getenv("RVS_TLS_FALLBACK", "true").lower() == "true"
|
RVS_TLS_FALLBACK = os.getenv("RVS_TLS_FALLBACK", "true").lower() == "true"
|
||||||
RVS_TOKEN = os.getenv("RVS_TOKEN", "").strip()
|
RVS_TOKEN = os.getenv("RVS_TOKEN", "").strip()
|
||||||
|
|
||||||
VOXTRAL_VLLM_URL = os.getenv("VOXTRAL_VLLM_URL", "ws://voxtral-vllm:8000/v1/realtime")
|
VOXTRAL_MODEL = os.getenv("VOXTRAL_MODEL", "mistralai/Voxtral-Mini-3B-2507")
|
||||||
VOXTRAL_MODEL = os.getenv("VOXTRAL_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602")
|
|
||||||
VOXTRAL_LANGUAGE = os.getenv("VOXTRAL_LANGUAGE", "de")
|
VOXTRAL_LANGUAGE = os.getenv("VOXTRAL_LANGUAGE", "de")
|
||||||
|
VOXTRAL_DEVICE = os.getenv("VOXTRAL_DEVICE", "cuda")
|
||||||
|
|
||||||
# ── vLLM-Realtime-Frames — HIER anpassen falls das Client-Beispiel abweicht ──
|
STREAM_TRANSCRIBE_INTERVAL_MS = int(os.getenv("STREAM_TRANSCRIBE_INTERVAL_MS", "1000"))
|
||||||
# Senderichtung (wir → vLLM): PCM16-16kHz-mono base64 anhaengen + committen.
|
|
||||||
VLLM_SEND_APPEND = "input_audio_buffer.append" # {"type":..., "audio": "<b64>"}
|
|
||||||
VLLM_SEND_COMMIT = "input_audio_buffer.commit" # Buffer abschliessen
|
|
||||||
VLLM_AUDIO_FIELD = "audio"
|
|
||||||
# Empfangsrichtung (vLLM → wir): inkrementeller Text + final. Defensiv geprueft.
|
|
||||||
VLLM_DELTA_SUFFIXES = ("transcription.delta",) # msg["type"] endet hierauf
|
|
||||||
VLLM_DONE_SUFFIXES = ("transcription.done", "transcription.completed")
|
|
||||||
VLLM_DELTA_FIELDS = ("delta", "text", "transcription") # eins davon traegt den Text
|
|
||||||
|
|
||||||
# ── Streaming-/Endpointing-Parameter (analog whisper-Bridge) ──
|
|
||||||
STREAM_DEFAULT_ENDPOINT_MS = 2400
|
STREAM_DEFAULT_ENDPOINT_MS = 2400
|
||||||
STREAM_DEFAULT_HARD_CAP_MS = 60000
|
STREAM_DEFAULT_HARD_CAP_MS = 300000
|
||||||
STREAM_MIN_AUDIO_MS = 600
|
STREAM_MIN_AUDIO_MS = 600
|
||||||
|
STREAM_SPEAKER_CHECK_MS = 1500 # ab so viel Audio einmalig Speaker-ID pruefen
|
||||||
STREAM_SESSION_TTL_S = 120
|
STREAM_SESSION_TTL_S = 120
|
||||||
STREAM_ENERGY_WINDOW_MS = 300
|
STREAM_ENERGY_WINDOW_MS = 300
|
||||||
STREAM_SEMANTIC_BACKUP_FACTOR = 2.0
|
STREAM_SEMANTIC_BACKUP_FACTOR = 2.0
|
||||||
# Adaptiver Voice-Schwellwert (siehe whisper-Bridge M0.1): Grenze relativ zum
|
# Adaptiver Voice-Schwellwert (M0.1): relativ zum gemessenen Rausch-Boden.
|
||||||
# gemessenen Rausch-Boden statt fix — schneidet leises Sprechen nicht ab.
|
|
||||||
STREAM_VOICE_FACTOR = 2.5
|
STREAM_VOICE_FACTOR = 2.5
|
||||||
STREAM_VOICE_RMS_MIN = 0.005
|
STREAM_VOICE_RMS_MIN = 0.005
|
||||||
STREAM_VOICE_RMS_MAX = 0.020
|
STREAM_VOICE_RMS_MAX = 0.020
|
||||||
|
# Mindest-Stimme (in ~200ms-Endpointer-Frames), ab der eine Aufnahme ueberhaupt
|
||||||
|
# als Sprache gilt. Darunter = Stille / kurzer Geraeusch-Blip → KEIN Transkript
|
||||||
|
# (Voxtral halluziniert aus Fast-Nichts sonst einen Fuellsatz). 2 ≈ 400ms.
|
||||||
|
STREAM_MIN_VOICED_FRAMES = int(os.getenv("STREAM_MIN_VOICED_FRAMES", "2"))
|
||||||
|
|
||||||
|
# Speaker-ID Gating global an/aus. DEFAULT AUS (fail-open) — die "nur meine Stimme"-
|
||||||
|
# Pruefung ist ein BEWUSSTER Schalter, kein Automatismus: ein einziger schlechter
|
||||||
|
# Enroll darf nie die ganze STT lahmlegen (genau das ist passiert). Wird per config-
|
||||||
|
# Broadcast (voiceIdEnabled, aus dem Diagnostic) zur Laufzeit gesetzt. Kann per ENV
|
||||||
|
# vorbelegt werden.
|
||||||
|
SPEAKER_ID_ENABLED = os.getenv("VOICE_ID_ENABLED", "false").lower() in ("1", "true", "yes")
|
||||||
|
|
||||||
|
|
||||||
|
def _set_speaker_id_enabled(val: bool) -> None:
|
||||||
|
global SPEAKER_ID_ENABLED
|
||||||
|
SPEAKER_ID_ENABLED = bool(val)
|
||||||
|
|
||||||
|
|
||||||
def pcm_s16le_to_float32(data: bytes) -> np.ndarray:
|
def pcm_s16le_to_float32(data: bytes) -> np.ndarray:
|
||||||
if not data:
|
if not data:
|
||||||
return np.zeros(0, dtype=np.float32)
|
return np.zeros(0, dtype=np.float32)
|
||||||
arr = np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0
|
return np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0
|
||||||
return arr
|
|
||||||
|
|
||||||
|
|
||||||
async def _send(ws, mtype: str, payload: dict) -> None:
|
async def _send(ws, mtype: str, payload: dict) -> None:
|
||||||
try:
|
try:
|
||||||
await ws.send(json.dumps({
|
await ws.send(json.dumps({
|
||||||
"type": mtype,
|
"type": mtype, "payload": payload, "timestamp": int(time.time() * 1000),
|
||||||
"payload": payload,
|
|
||||||
"timestamp": int(time.time() * 1000),
|
|
||||||
}))
|
}))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("RVS-Send fehlgeschlagen (%s): %s", mtype, e)
|
logger.warning("RVS-Send fehlgeschlagen (%s): %s", mtype, e)
|
||||||
|
|
||||||
|
|
||||||
|
class VoxtralRunner:
|
||||||
|
"""Haelt das Voxtral-Modell (Transformers). transcribe() blockiert → aus dem
|
||||||
|
Event-Loop via run_in_executor aufrufen. Ein Lock serialisiert GPU-Zugriffe."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.model = None
|
||||||
|
self.processor = None
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
|
def load(self) -> None:
|
||||||
|
import torch
|
||||||
|
from transformers import AutoProcessor, VoxtralForConditionalGeneration
|
||||||
|
t0 = time.time()
|
||||||
|
logger.info("Lade Voxtral '%s' (device=%s, bf16)…", VOXTRAL_MODEL, VOXTRAL_DEVICE)
|
||||||
|
self.processor = AutoProcessor.from_pretrained(VOXTRAL_MODEL)
|
||||||
|
self.model = VoxtralForConditionalGeneration.from_pretrained(
|
||||||
|
VOXTRAL_MODEL, torch_dtype=torch.bfloat16, device_map=VOXTRAL_DEVICE,
|
||||||
|
)
|
||||||
|
logger.info("Voxtral geladen in %.1fs", time.time() - t0)
|
||||||
|
|
||||||
|
def _transcribe_blocking(self, audio_f32: np.ndarray, language: str) -> str:
|
||||||
|
import torch
|
||||||
|
proc, model = self.processor, self.model
|
||||||
|
if proc is None or model is None or audio_f32.size == 0:
|
||||||
|
return ""
|
||||||
|
# VoxtralProcessor verlangt bei rohen Arrays ein 'format'. Robuster:
|
||||||
|
# in ein temp-WAV schreiben und den PFAD uebergeben — der Processor liest
|
||||||
|
# Format + Samplerate selbst, kein 'format'-Argument noetig.
|
||||||
|
wav_path = None
|
||||||
|
try:
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tf:
|
||||||
|
wav_path = tf.name
|
||||||
|
sf.write(wav_path, audio_f32, 16000, subtype="PCM_16")
|
||||||
|
inputs = proc.apply_transcription_request(
|
||||||
|
language=language, audio=wav_path, model_id=VOXTRAL_MODEL,
|
||||||
|
)
|
||||||
|
inputs = inputs.to(VOXTRAL_DEVICE, dtype=torch.bfloat16)
|
||||||
|
with torch.no_grad():
|
||||||
|
# hoch genug fuer lange Diktate (stoppt eh am EOS); 512 hat
|
||||||
|
# mehrminutige Aufnahmen abgeschnitten.
|
||||||
|
outputs = model.generate(**inputs, max_new_tokens=4096)
|
||||||
|
trimmed = outputs[:, inputs.input_ids.shape[1]:]
|
||||||
|
text = proc.batch_decode(trimmed, skip_special_tokens=True)
|
||||||
|
return (text[0] if text else "").strip()
|
||||||
|
finally:
|
||||||
|
if wav_path:
|
||||||
|
try:
|
||||||
|
os.unlink(wav_path)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def transcribe(self, audio_f32: np.ndarray, language: str) -> str:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
async with self._lock:
|
||||||
|
return await loop.run_in_executor(None, self._transcribe_blocking, audio_f32, language)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class StreamSession:
|
class StreamSession:
|
||||||
request_id: str
|
request_id: str
|
||||||
@@ -126,28 +179,38 @@ class StreamSession:
|
|||||||
last_chunk_at: float = field(default_factory=time.time)
|
last_chunk_at: float = field(default_factory=time.time)
|
||||||
last_partial: str = ""
|
last_partial: str = ""
|
||||||
last_growth_at: float = 0.0
|
last_growth_at: float = 0.0
|
||||||
|
last_transcribe_at: float = 0.0
|
||||||
last_voice_at: float = 0.0
|
last_voice_at: float = 0.0
|
||||||
noise_floor: float = 0.0
|
noise_floor: float = 0.0
|
||||||
closed: bool = False
|
closed: bool = False
|
||||||
endpoint_sent: bool = False
|
endpoint_sent: bool = False
|
||||||
# vLLM-Realtime-Session
|
# Einmaliges "Sprache erkannt"-Signal an die App gesendet? Voxtral schickt
|
||||||
vllm_ws: object = None
|
# keine Live-Partials, aber der App-No-Speech-Watchdog wartet auf ein
|
||||||
vllm_reader: object = None
|
# stt_partial, um "der User redet" zu erkennen — sonst cancelt er mitten im
|
||||||
|
# Satz. Wir feuern EIN leeres stt_partial beim ersten Voice-Frame.
|
||||||
|
speech_signaled: bool = False
|
||||||
|
# Anzahl Endpointer-Frames (~200ms) mit echter Stimme. Gate gegen Halluzination
|
||||||
|
# aus Stille/Blips: unter STREAM_MIN_VOICED_FRAMES wird nicht transkribiert.
|
||||||
|
voiced_frames: int = 0
|
||||||
|
# Speaker-ID Gating (einmalig auf die ersten ~1.5s der Aufnahme)
|
||||||
|
speaker_checked: bool = False
|
||||||
|
speaker_match: Optional[bool] = None
|
||||||
|
speaker_similarity: float = 0.0
|
||||||
|
|
||||||
|
|
||||||
class SessionManager:
|
class SessionManager:
|
||||||
def __init__(self) -> None:
|
def __init__(self, runner: VoxtralRunner) -> None:
|
||||||
|
self.runner = runner
|
||||||
self._sessions: dict[str, StreamSession] = {}
|
self._sessions: dict[str, StreamSession] = {}
|
||||||
self._ws = None # RVS
|
self._ws = None
|
||||||
|
|
||||||
def attach_ws(self, ws) -> None:
|
def attach_ws(self, ws) -> None:
|
||||||
self._ws = ws
|
self._ws = ws
|
||||||
|
|
||||||
async def start_session(self, payload: dict) -> Optional[StreamSession]:
|
def start_session(self, payload: dict) -> None:
|
||||||
request_id = (payload.get("requestId") or "").strip()
|
rid = (payload.get("requestId") or "").strip()
|
||||||
if not request_id:
|
if not rid:
|
||||||
logger.warning("stt_stream_start ohne requestId — ignoriert")
|
return
|
||||||
return None
|
|
||||||
try:
|
try:
|
||||||
endpoint_ms = int(payload.get("endpointMs") or STREAM_DEFAULT_ENDPOINT_MS)
|
endpoint_ms = int(payload.get("endpointMs") or STREAM_DEFAULT_ENDPOINT_MS)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
@@ -160,8 +223,8 @@ class SessionManager:
|
|||||||
voice_factor = float(payload.get("voiceFactor") or STREAM_VOICE_FACTOR)
|
voice_factor = float(payload.get("voiceFactor") or STREAM_VOICE_FACTOR)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
voice_factor = STREAM_VOICE_FACTOR
|
voice_factor = STREAM_VOICE_FACTOR
|
||||||
sess = StreamSession(
|
self._sessions[rid] = StreamSession(
|
||||||
request_id=request_id,
|
request_id=rid,
|
||||||
audio_request_id=payload.get("audioRequestId", "") or "",
|
audio_request_id=payload.get("audioRequestId", "") or "",
|
||||||
language=payload.get("language") or VOXTRAL_LANGUAGE,
|
language=payload.get("language") or VOXTRAL_LANGUAGE,
|
||||||
endpoint_ms=endpoint_ms,
|
endpoint_ms=endpoint_ms,
|
||||||
@@ -173,159 +236,44 @@ class SessionManager:
|
|||||||
location=payload.get("location") or None,
|
location=payload.get("location") or None,
|
||||||
sample_rate=int(payload.get("sampleRate") or 16000),
|
sample_rate=int(payload.get("sampleRate") or 16000),
|
||||||
)
|
)
|
||||||
# vLLM-Realtime-Session oeffnen + Reader starten.
|
|
||||||
try:
|
|
||||||
sess.vllm_ws = await websockets.connect(VOXTRAL_VLLM_URL, max_size=8 * 1024 * 1024)
|
|
||||||
await self._vllm_configure(sess)
|
|
||||||
sess.vllm_reader = asyncio.create_task(self._vllm_read_loop(sess))
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception("Stream %s: vLLM-Realtime-Connect fehlgeschlagen: %s",
|
|
||||||
request_id[:8], e)
|
|
||||||
# ohne Backend keine Transkription → sofort leeres Endpoint melden
|
|
||||||
self._sessions[request_id] = sess
|
|
||||||
await self._finalize(sess, reason="vllm_unavailable")
|
|
||||||
return None
|
|
||||||
self._sessions[request_id] = sess
|
|
||||||
logger.info("Voxtral-Session offen: id=%s lang=%s endpointMs=%d",
|
logger.info("Voxtral-Session offen: id=%s lang=%s endpointMs=%d",
|
||||||
request_id[:8], sess.language, sess.endpoint_ms)
|
rid[:8], self._sessions[rid].language, endpoint_ms)
|
||||||
return sess
|
|
||||||
|
|
||||||
async def _vllm_configure(self, sess: StreamSession) -> None:
|
|
||||||
"""Optionale Session-Konfig an vLLM (Modell/Sprache/temperature=0).
|
|
||||||
VERIFY: exaktes session.update-Schema gegen vLLM-Realtime-Beispiel.
|
|
||||||
Best-effort — Fehler hier sind nicht fatal."""
|
|
||||||
try:
|
|
||||||
await sess.vllm_ws.send(json.dumps({
|
|
||||||
"type": "session.update",
|
|
||||||
"session": {
|
|
||||||
"model": VOXTRAL_MODEL,
|
|
||||||
"language": sess.language,
|
|
||||||
"temperature": 0.0,
|
|
||||||
"input_audio_format": "pcm16",
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
async def _vllm_read_loop(self, sess: StreamSession) -> None:
|
|
||||||
"""Liest transcription.delta/.done vom vLLM-Realtime-Server."""
|
|
||||||
ws = sess.vllm_ws
|
|
||||||
try:
|
|
||||||
async for raw in ws:
|
|
||||||
try:
|
|
||||||
msg = json.loads(raw)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
mtype = str(msg.get("type", ""))
|
|
||||||
if any(mtype.endswith(s) for s in VLLM_DELTA_SUFFIXES):
|
|
||||||
text = self._extract_text(msg)
|
|
||||||
if text:
|
|
||||||
await self._on_delta(sess, text)
|
|
||||||
elif any(mtype.endswith(s) for s in VLLM_DONE_SUFFIXES):
|
|
||||||
text = self._extract_text(msg)
|
|
||||||
if text:
|
|
||||||
await self._on_delta(sess, text, final=True)
|
|
||||||
except Exception:
|
|
||||||
logger.debug("Stream %s: vLLM-Reader beendet", sess.request_id[:8])
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extract_text(msg: dict) -> str:
|
|
||||||
for f in VLLM_DELTA_FIELDS:
|
|
||||||
v = msg.get(f)
|
|
||||||
if isinstance(v, str) and v:
|
|
||||||
return v
|
|
||||||
return ""
|
|
||||||
|
|
||||||
async def _on_delta(self, sess: StreamSession, text: str, final: bool = False) -> None:
|
|
||||||
"""Neuer/finaler Transkript-Text vom vLLM. delta = inkrementell; wir
|
|
||||||
haengen an, wenn er den bisherigen Partial verlaengert, sonst ersetzen
|
|
||||||
wir (Voxtral kann korrigieren)."""
|
|
||||||
if final or text.startswith(sess.last_partial):
|
|
||||||
new_full = text if final else text
|
|
||||||
else:
|
|
||||||
new_full = (sess.last_partial + text).strip()
|
|
||||||
new_full = new_full.strip()
|
|
||||||
if new_full and new_full != sess.last_partial:
|
|
||||||
sess.last_partial = new_full
|
|
||||||
sess.last_growth_at = time.time()
|
|
||||||
if self._ws is not None:
|
|
||||||
await _send(self._ws, "stt_partial", {
|
|
||||||
"requestId": sess.request_id,
|
|
||||||
"audioRequestId": sess.audio_request_id,
|
|
||||||
"text": new_full,
|
|
||||||
})
|
|
||||||
|
|
||||||
def feed_chunk(self, payload: dict) -> bool:
|
def feed_chunk(self, payload: dict) -> bool:
|
||||||
request_id = payload.get("requestId", "")
|
sess = self._sessions.get(payload.get("requestId", ""))
|
||||||
sess = self._sessions.get(request_id)
|
|
||||||
if sess is None or sess.closed:
|
if sess is None or sess.closed:
|
||||||
return False
|
return False
|
||||||
pcm_b64 = payload.get("pcm", "")
|
pcm_b64 = payload.get("pcm", "")
|
||||||
if not pcm_b64:
|
if pcm_b64:
|
||||||
return True
|
try:
|
||||||
try:
|
sess.pcm_buffer.extend(base64.b64decode(pcm_b64))
|
||||||
pcm = base64.b64decode(pcm_b64)
|
except Exception:
|
||||||
except Exception:
|
pass
|
||||||
return True
|
|
||||||
sess.pcm_buffer.extend(pcm)
|
|
||||||
sess.last_chunk_at = time.time()
|
sess.last_chunk_at = time.time()
|
||||||
# An vLLM weiterreichen (fire-and-forget).
|
|
||||||
if sess.vllm_ws is not None:
|
|
||||||
asyncio.create_task(self._vllm_append(sess, pcm_b64))
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def _vllm_append(self, sess: StreamSession, pcm_b64: str) -> None:
|
|
||||||
try:
|
|
||||||
await sess.vllm_ws.send(json.dumps({
|
|
||||||
"type": VLLM_SEND_APPEND,
|
|
||||||
VLLM_AUDIO_FIELD: pcm_b64,
|
|
||||||
}))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def end_session(self, request_id: str) -> None:
|
def end_session(self, request_id: str) -> None:
|
||||||
sess = self._sessions.get(request_id)
|
sess = self._sessions.get(request_id)
|
||||||
if sess is not None:
|
if sess is not None:
|
||||||
sess.closed = True
|
sess.closed = True
|
||||||
|
|
||||||
def drop(self, request_id: str) -> None:
|
def drop(self, request_id: str) -> None:
|
||||||
sess = self._sessions.pop(request_id, None)
|
self._sessions.pop(request_id, None)
|
||||||
if sess is not None:
|
|
||||||
self._teardown_vllm(sess)
|
|
||||||
|
|
||||||
def _teardown_vllm(self, sess: StreamSession) -> None:
|
# ── Endpointer (adaptiv, M0.1) ──
|
||||||
try:
|
def _buffer_ms(self, sess: StreamSession) -> float:
|
||||||
if sess.vllm_reader is not None:
|
|
||||||
sess.vllm_reader.cancel()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if sess.vllm_ws is not None:
|
|
||||||
asyncio.create_task(self._close_ws(sess.vllm_ws))
|
|
||||||
sess.vllm_ws = None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
async def _close_ws(ws) -> None:
|
|
||||||
try:
|
|
||||||
await ws.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# ── Endpointer (adaptiv, wie whisper-Bridge M0.1) ──
|
|
||||||
def _buffer_duration_ms(self, sess: StreamSession) -> float:
|
|
||||||
samples = len(sess.pcm_buffer) // 2
|
samples = len(sess.pcm_buffer) // 2
|
||||||
return (samples / sess.sample_rate) * 1000.0 if samples else 0.0
|
return (samples / sess.sample_rate) * 1000.0 if samples else 0.0
|
||||||
|
|
||||||
def _tail_rms(self, sess: StreamSession) -> float:
|
def _tail_rms(self, sess: StreamSession) -> float:
|
||||||
win_bytes = int(sess.sample_rate * STREAM_ENERGY_WINDOW_MS / 1000) * 2
|
win = int(sess.sample_rate * STREAM_ENERGY_WINDOW_MS / 1000) * 2
|
||||||
if win_bytes <= 0:
|
if win <= 0:
|
||||||
return 0.0
|
return 0.0
|
||||||
tail = sess.pcm_buffer[-win_bytes:]
|
tail = sess.pcm_buffer[-win:]
|
||||||
if len(tail) < 2:
|
if len(tail) < 2:
|
||||||
return 0.0
|
return 0.0
|
||||||
arr = pcm_s16le_to_float32(bytes(tail))
|
arr = pcm_s16le_to_float32(bytes(tail))
|
||||||
if arr.size == 0:
|
return float(np.sqrt(np.mean(arr * arr))) if arr.size else 0.0
|
||||||
return 0.0
|
|
||||||
return float(np.sqrt(np.mean(arr * arr)))
|
|
||||||
|
|
||||||
def _voice_threshold(self, sess: StreamSession) -> float:
|
def _voice_threshold(self, sess: StreamSession) -> float:
|
||||||
nf = sess.noise_floor
|
nf = sess.noise_floor
|
||||||
@@ -342,8 +290,66 @@ class SessionManager:
|
|||||||
else:
|
else:
|
||||||
sess.noise_floor = 0.98 * nf + 0.02 * rms
|
sess.noise_floor = 0.98 * nf + 0.02 * rms
|
||||||
|
|
||||||
|
async def _check_speaker(self, sess: StreamSession) -> None:
|
||||||
|
"""Einmalig: erste ~1.5s → Embedding → Vergleich mit Fingerprint.
|
||||||
|
Ohne Fingerprint fail-open (match=True). Bei Mismatch: Session beenden."""
|
||||||
|
sess.speaker_checked = True
|
||||||
|
# Schalter aus (Default) → gar keine Pruefung, alles durchlassen.
|
||||||
|
if not SPEAKER_ID_ENABLED:
|
||||||
|
sess.speaker_match = True
|
||||||
|
return
|
||||||
|
head = bytes(sess.pcm_buffer[: STREAM_SPEAKER_CHECK_MS * 32])
|
||||||
|
if len(head) < speaker_id.MIN_SAMPLE_BYTES:
|
||||||
|
sess.speaker_match = True
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
is_match, sim = await loop.run_in_executor(None, speaker_id.verify, head)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Stream %s: speaker-check crashed (%s) — fail-open",
|
||||||
|
sess.request_id[:8], exc)
|
||||||
|
sess.speaker_match = True
|
||||||
|
return
|
||||||
|
sess.speaker_match = is_match
|
||||||
|
sess.speaker_similarity = sim
|
||||||
|
logger.info("Stream %s: speaker-check sim=%.2f → %s (thr=%.2f)",
|
||||||
|
sess.request_id[:8], sim, "MATCH" if is_match else "REJECT",
|
||||||
|
speaker_id.DEFAULT_THRESHOLD)
|
||||||
|
if not is_match:
|
||||||
|
await self._finalize_speaker_mismatch(sess, sim)
|
||||||
|
|
||||||
|
async def _finalize_speaker_mismatch(self, sess: StreamSession, similarity: float) -> None:
|
||||||
|
"""Fremde Stimme: synthetisches leeres stt_endpoint (reason=speaker_mismatch),
|
||||||
|
Session droppen — kein Voxtral-Transcribe, kein Brain-Call."""
|
||||||
|
if sess.endpoint_sent:
|
||||||
|
return
|
||||||
|
sess.endpoint_sent = True
|
||||||
|
duration_s = self._buffer_ms(sess) / 1000.0
|
||||||
|
logger.info("Stream %s: speaker-mismatch (sim=%.2f) — DROP nach %.1fs",
|
||||||
|
sess.request_id[:8], similarity, duration_s)
|
||||||
|
if self._ws is not None:
|
||||||
|
payload = {
|
||||||
|
"requestId": sess.request_id,
|
||||||
|
"audioRequestId": sess.audio_request_id,
|
||||||
|
"text": "", "reason": "speaker_mismatch",
|
||||||
|
"durationS": duration_s, "sttMs": 0,
|
||||||
|
"voice": sess.voice, "speed": sess.speed,
|
||||||
|
"interrupted": sess.interrupted,
|
||||||
|
"speakerSimilarity": float(similarity),
|
||||||
|
}
|
||||||
|
if sess.location:
|
||||||
|
payload["location"] = sess.location
|
||||||
|
await _send(self._ws, "stt_endpoint", payload)
|
||||||
|
await _send(self._ws, "stt_stream_done", {
|
||||||
|
"requestId": sess.request_id,
|
||||||
|
"audioRequestId": sess.audio_request_id,
|
||||||
|
"text": "", "reason": "speaker_mismatch",
|
||||||
|
})
|
||||||
|
self.drop(sess.request_id)
|
||||||
|
|
||||||
async def run_endpointer(self) -> None:
|
async def run_endpointer(self) -> None:
|
||||||
logger.info("Voxtral-Endpointer gestartet (adaptiver VAD)")
|
logger.info("Voxtral-Endpointer gestartet (adaptiver VAD, interval=%dms)",
|
||||||
|
STREAM_TRANSCRIBE_INTERVAL_MS)
|
||||||
while True:
|
while True:
|
||||||
await asyncio.sleep(0.2)
|
await asyncio.sleep(0.2)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
@@ -351,7 +357,7 @@ class SessionManager:
|
|||||||
try:
|
try:
|
||||||
await self._tick(sess, now)
|
await self._tick(sess, now)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Endpointer-Tick crashed (session=%s)", sid[:8])
|
logger.exception("Tick crashed (session=%s)", sid[:8])
|
||||||
for sid, sess in list(self._sessions.items()):
|
for sid, sess in list(self._sessions.items()):
|
||||||
if now - sess.last_chunk_at > STREAM_SESSION_TTL_S:
|
if now - sess.last_chunk_at > STREAM_SESSION_TTL_S:
|
||||||
logger.info("Stream %s: TTL — drop", sid[:8])
|
logger.info("Stream %s: TTL — drop", sid[:8])
|
||||||
@@ -360,60 +366,104 @@ class SessionManager:
|
|||||||
async def _tick(self, sess: StreamSession, now: float) -> None:
|
async def _tick(self, sess: StreamSession, now: float) -> None:
|
||||||
if sess.endpoint_sent:
|
if sess.endpoint_sent:
|
||||||
return
|
return
|
||||||
elapsed_ms = (now - sess.started_at) * 1000.0
|
if (now - sess.started_at) * 1000.0 > sess.hard_cap_ms and not sess.closed:
|
||||||
if elapsed_ms > sess.hard_cap_ms and not sess.closed:
|
await self._finalize(sess, "hardcap")
|
||||||
await self._finalize(sess, reason="hardcap")
|
|
||||||
return
|
return
|
||||||
if sess.closed:
|
if sess.closed:
|
||||||
await self._finalize(sess, reason="stream_end")
|
await self._finalize(sess, "stream_end")
|
||||||
return
|
return
|
||||||
if self._buffer_duration_ms(sess) < STREAM_MIN_AUDIO_MS:
|
if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS:
|
||||||
return
|
return
|
||||||
# adaptive akustische Sprach-Aktivitaet
|
# Speaker-ID einmalig: ist es Stefans Stimme? Fremde → Session verwerfen
|
||||||
|
# (kein Transcribe, kein Brain-Call). Ohne Enrollment fail-open.
|
||||||
|
if not sess.speaker_checked and self._buffer_ms(sess) >= STREAM_SPEAKER_CHECK_MS:
|
||||||
|
await self._check_speaker(sess)
|
||||||
|
if sess.speaker_match is False:
|
||||||
|
return
|
||||||
|
# Adaptive akustische Sprach-Aktivitaet (M0.1). KEINE Live-Partials mehr:
|
||||||
|
# Voxtral-3B transkribiert den ganzen WACHSENDEN Buffer und braucht dafuer
|
||||||
|
# bei langen Aufnahmen 5-6 s — zu langsam fuer Live-Text, UND diese Latenz
|
||||||
|
# hat den semantischen Endpoint faelschlich ausgeloest (Partial-Latenz >
|
||||||
|
# Timeout → willkuerliche Abbrueche nach 20-40 s). Deshalb: Turn-Ende rein
|
||||||
|
# AKUSTISCH, transkribiert wird nur EINMAL im _finalize.
|
||||||
rms = self._tail_rms(sess)
|
rms = self._tail_rms(sess)
|
||||||
if rms >= self._voice_threshold(sess):
|
if rms >= self._voice_threshold(sess):
|
||||||
sess.last_voice_at = now
|
sess.last_voice_at = now
|
||||||
|
sess.voiced_frames += 1
|
||||||
|
# Einmalig der App melden, dass Sprache begonnen hat — aber ERST ab genug
|
||||||
|
# echter Stimme (>= STREAM_MIN_VOICED_FRAMES). Ein einzelner Geraeusch-
|
||||||
|
# Blip darf den No-Speech-Watchdog NICHT loeschen, sonst transkribiert
|
||||||
|
# Voxtral das Fast-Nichts und HALLUZINIERT einen Phantom-Satz. Ohne Live-
|
||||||
|
# Partials wuerde der Watchdog die Aufnahme sonst am Konversationsfenster
|
||||||
|
# canceln, obwohl der User redet ("beendet nach ~4s"-Repro). Leeres
|
||||||
|
# stt_partial: App setzt streamGotPartial=true + loescht den Watchdog.
|
||||||
|
# Nach der Speaker-ID-Pruefung (oben) → fremde Stimmen signalisieren NICHT.
|
||||||
|
if (not sess.speech_signaled and self._ws is not None
|
||||||
|
and sess.voiced_frames >= STREAM_MIN_VOICED_FRAMES):
|
||||||
|
sess.speech_signaled = True
|
||||||
|
await _send(self._ws, "stt_partial", {
|
||||||
|
"requestId": sess.request_id,
|
||||||
|
"audioRequestId": sess.audio_request_id,
|
||||||
|
"text": "",
|
||||||
|
})
|
||||||
else:
|
else:
|
||||||
self._update_noise_floor(sess, rms)
|
self._update_noise_floor(sess, rms)
|
||||||
# Endpoint: akustisch (Primaer) oder semantisch (Backstop), sobald Text da
|
# Endpoint: hat der User schon gesprochen UND ist es seit endpoint_ms still?
|
||||||
if sess.last_growth_at > 0.0:
|
if sess.last_voice_at > 0 and (now - sess.last_voice_at) * 1000.0 >= sess.endpoint_ms:
|
||||||
acoustic_silence_ms = (now - sess.last_voice_at) * 1000.0 if sess.last_voice_at > 0 else 0.0
|
await self._finalize(sess, "endpoint")
|
||||||
semantic_silence_ms = (now - sess.last_growth_at) * 1000.0
|
|
||||||
acoustic_done = sess.last_voice_at > 0 and acoustic_silence_ms >= sess.endpoint_ms
|
|
||||||
semantic_done = semantic_silence_ms >= sess.endpoint_ms * STREAM_SEMANTIC_BACKUP_FACTOR
|
|
||||||
if acoustic_done or semantic_done:
|
|
||||||
await self._finalize(sess, reason="endpoint" if acoustic_done else "endpoint_semantic")
|
|
||||||
|
|
||||||
async def _finalize(self, sess: StreamSession, reason: str) -> None:
|
async def _finalize(self, sess: StreamSession, reason: str) -> None:
|
||||||
if sess.endpoint_sent:
|
if sess.endpoint_sent:
|
||||||
return
|
return
|
||||||
sess.endpoint_sent = True
|
sess.endpoint_sent = True
|
||||||
# vLLM ggf. committen, damit ein letztes transcription.done kommt.
|
# Halluzinations-Guard: zu wenig echte Stimme (Stille / kurzer Blip im
|
||||||
if sess.vllm_ws is not None:
|
# Passiv-/Wake-Fenster) → NICHT transkribieren. Voxtral (wie Whisper) baut
|
||||||
try:
|
# aus Fast-Nichts gern einen Fuellsatz ("keine Ahnung" o.ae.), der dann als
|
||||||
await sess.vllm_ws.send(json.dumps({"type": VLLM_SEND_COMMIT}))
|
# PHANTOM-Nachricht ans Brain geht und das Gespraech entgleisen laesst
|
||||||
await asyncio.sleep(0.15) # kurz auf finalen Delta warten
|
# (Stefans Repro: "kam Nachricht von mir, obwohl ich nichts sagte"). Leeres
|
||||||
except Exception:
|
# Endpoint = no-speech → App re-armt still. Der manuelle Stop (stream_end)
|
||||||
pass
|
# ist ausgenommen: dort hat der User bewusst gesprochen (kurze Woerter ok).
|
||||||
final_text = sess.last_partial.strip()
|
if reason != "stream_end" and sess.voiced_frames < STREAM_MIN_VOICED_FRAMES:
|
||||||
duration_s = self._buffer_duration_ms(sess) / 1000.0
|
logger.info("Stream %s: no-speech (voiced_frames=%d<%d, reason=%s) — leeres Endpoint",
|
||||||
logger.info("Stream %s: FINAL (reason=%s, %.1fs): %r",
|
sess.request_id[:8], sess.voiced_frames, STREAM_MIN_VOICED_FRAMES, reason)
|
||||||
sess.request_id[:8], reason, duration_s, final_text[:120])
|
if self._ws is not None:
|
||||||
|
nospeech = {"requestId": sess.request_id,
|
||||||
|
"audioRequestId": sess.audio_request_id,
|
||||||
|
"text": "", "reason": f"no_speech:{reason}",
|
||||||
|
"durationS": 0.0, "sttMs": 0}
|
||||||
|
await _send(self._ws, "stt_endpoint", nospeech)
|
||||||
|
await _send(self._ws, "stt_stream_done", {
|
||||||
|
"requestId": sess.request_id,
|
||||||
|
"audioRequestId": sess.audio_request_id,
|
||||||
|
"text": "", "reason": f"no_speech:{reason}"})
|
||||||
|
self.drop(sess.request_id)
|
||||||
|
return
|
||||||
|
audio = pcm_s16le_to_float32(bytes(sess.pcm_buffer))
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
final_text = (await self.runner.transcribe(audio, sess.language)).strip()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Stream %s: Final-Transcribe crashed", sess.request_id[:8])
|
||||||
|
final_text = sess.last_partial
|
||||||
|
stt_ms = int((time.time() - t0) * 1000)
|
||||||
|
duration_s = audio.size / 16000.0
|
||||||
|
logger.info("Stream %s: FINAL (reason=%s, %.1fs, %dms): %r",
|
||||||
|
sess.request_id[:8], reason, duration_s, stt_ms, final_text[:120])
|
||||||
if self._ws is not None:
|
if self._ws is not None:
|
||||||
endpoint_payload = {
|
payload = {
|
||||||
"requestId": sess.request_id,
|
"requestId": sess.request_id,
|
||||||
"audioRequestId": sess.audio_request_id,
|
"audioRequestId": sess.audio_request_id,
|
||||||
"text": final_text,
|
"text": final_text,
|
||||||
"reason": reason,
|
"reason": reason,
|
||||||
"durationS": duration_s,
|
"durationS": duration_s,
|
||||||
"sttMs": 0,
|
"sttMs": stt_ms,
|
||||||
"voice": sess.voice,
|
"voice": sess.voice,
|
||||||
"speed": sess.speed,
|
"speed": sess.speed,
|
||||||
"interrupted": sess.interrupted,
|
"interrupted": sess.interrupted,
|
||||||
}
|
}
|
||||||
if sess.location:
|
if sess.location:
|
||||||
endpoint_payload["location"] = sess.location
|
payload["location"] = sess.location
|
||||||
await _send(self._ws, "stt_endpoint", endpoint_payload)
|
await _send(self._ws, "stt_endpoint", payload)
|
||||||
await _send(self._ws, "stt_stream_done", {
|
await _send(self._ws, "stt_stream_done", {
|
||||||
"requestId": sess.request_id,
|
"requestId": sess.request_id,
|
||||||
"audioRequestId": sess.audio_request_id,
|
"audioRequestId": sess.audio_request_id,
|
||||||
@@ -447,7 +497,6 @@ async def run_loop(sessions: SessionManager) -> None:
|
|||||||
sessions.attach_ws(ws)
|
sessions.attach_ws(ws)
|
||||||
await _broadcast_status(ws, "ready", model=VOXTRAL_MODEL)
|
await _broadcast_status(ws, "ready", model=VOXTRAL_MODEL)
|
||||||
await _send(ws, "config_request", {"service": "voxtral"})
|
await _send(ws, "config_request", {"service": "voxtral"})
|
||||||
|
|
||||||
async for raw in ws:
|
async for raw in ws:
|
||||||
try:
|
try:
|
||||||
msg = json.loads(raw)
|
msg = json.loads(raw)
|
||||||
@@ -456,35 +505,77 @@ async def run_loop(sessions: SessionManager) -> None:
|
|||||||
mtype = msg.get("type", "")
|
mtype = msg.get("type", "")
|
||||||
payload = msg.get("payload", {}) or {}
|
payload = msg.get("payload", {}) or {}
|
||||||
if mtype == "stt_stream_start":
|
if mtype == "stt_stream_start":
|
||||||
asyncio.create_task(sessions.start_session(payload))
|
sessions.start_session(payload)
|
||||||
elif mtype == "stt_audio_chunk":
|
elif mtype == "stt_audio_chunk":
|
||||||
sessions.feed_chunk(payload)
|
sessions.feed_chunk(payload)
|
||||||
elif mtype == "stt_stream_end":
|
elif mtype == "stt_stream_end":
|
||||||
sessions.end_session(payload.get("requestId", ""))
|
sessions.end_session(payload.get("requestId", ""))
|
||||||
# stt_request (Legacy One-Shot) macht Voxtral hier NICHT —
|
elif mtype == "voice_id_status_request":
|
||||||
# dafuer bleibt die whisper-Bridge (Fallback).
|
req_id = payload.get("requestId", "")
|
||||||
|
try:
|
||||||
|
status = speaker_id.status()
|
||||||
|
await _send(ws, "voice_id_status_response",
|
||||||
|
{"requestId": req_id, "ok": True, **status})
|
||||||
|
except Exception as exc:
|
||||||
|
await _send(ws, "voice_id_status_response",
|
||||||
|
{"requestId": req_id, "ok": False, "error": str(exc)[:200]})
|
||||||
|
elif mtype == "voice_id_enroll_request":
|
||||||
|
req_id = payload.get("requestId", "")
|
||||||
|
samples = payload.get("samples") or []
|
||||||
|
logger.info("voice_id_enroll_request: %d Samples (id=%s)", len(samples), req_id[:8])
|
||||||
|
try:
|
||||||
|
result = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None, speaker_id.enroll_from_samples, samples)
|
||||||
|
await _send(ws, "voice_id_enroll_response", {
|
||||||
|
"requestId": req_id, "ok": True,
|
||||||
|
"sample_count": result.get("sample_count", 0),
|
||||||
|
"rejected": result.get("rejected", []),
|
||||||
|
"updated_at": result.get("updated_at"),
|
||||||
|
"embedding_dim": result.get("embedding_dim"),
|
||||||
|
})
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("voice_id_enroll failed: %s", exc)
|
||||||
|
await _send(ws, "voice_id_enroll_response",
|
||||||
|
{"requestId": req_id, "ok": False, "error": str(exc)[:300]})
|
||||||
|
elif mtype == "voice_id_delete_request":
|
||||||
|
req_id = payload.get("requestId", "")
|
||||||
|
removed = speaker_id.delete_fingerprint()
|
||||||
|
await _send(ws, "voice_id_delete_response",
|
||||||
|
{"requestId": req_id, "ok": True, "removed": removed})
|
||||||
|
elif mtype == "config":
|
||||||
|
if "voiceIdThreshold" in payload:
|
||||||
|
try:
|
||||||
|
t = float(payload.get("voiceIdThreshold", 0.5))
|
||||||
|
if 0.0 <= t <= 1.0:
|
||||||
|
speaker_id.DEFAULT_THRESHOLD = t
|
||||||
|
logger.info("[speaker-id] threshold gesetzt: %.2f", t)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
if "voiceIdEnabled" in payload:
|
||||||
|
_set_speaker_id_enabled(payload.get("voiceIdEnabled"))
|
||||||
|
logger.info("[speaker-id] Gating %s (voiceIdEnabled)",
|
||||||
|
"AN" if SPEAKER_ID_ENABLED else "AUS")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s)
|
logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s)
|
||||||
if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried:
|
if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried:
|
||||||
use_tls = False
|
use_tls = False
|
||||||
tls_fallback_tried = True
|
tls_fallback_tried = True
|
||||||
logger.info("TLS-Fallback: versuche ws:// (kein TLS)")
|
|
||||||
continue
|
continue
|
||||||
await asyncio.sleep(retry_s)
|
await asyncio.sleep(retry_s)
|
||||||
retry_s = min(retry_s * 2, 30)
|
retry_s = min(retry_s * 2, 30)
|
||||||
use_tls = RVS_TLS # fuer den naechsten Zyklus zuruecksetzen
|
use_tls = RVS_TLS
|
||||||
|
|
||||||
|
|
||||||
async def main() -> None:
|
async def main() -> None:
|
||||||
if not RVS_HOST or not RVS_TOKEN:
|
if not RVS_HOST or not RVS_TOKEN:
|
||||||
logger.error("RVS_HOST/RVS_TOKEN fehlen — .env pruefen. Abbruch.")
|
logger.error("RVS_HOST/RVS_TOKEN fehlen — .env pruefen. Abbruch.")
|
||||||
return
|
return
|
||||||
sessions = SessionManager()
|
runner = VoxtralRunner()
|
||||||
logger.info("Voxtral-Bridge startet — vLLM=%s Modell=%s", VOXTRAL_VLLM_URL, VOXTRAL_MODEL)
|
loop = asyncio.get_running_loop()
|
||||||
await asyncio.gather(
|
await loop.run_in_executor(None, runner.load) # Modell laden (blockierend)
|
||||||
run_loop(sessions),
|
sessions = SessionManager(runner)
|
||||||
sessions.run_endpointer(),
|
logger.info("Voxtral-Bridge startet — Modell=%s", VOXTRAL_MODEL)
|
||||||
)
|
await asyncio.gather(run_loop(sessions), sessions.run_endpointer())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Voxtral-Bridge ist reine CPU-Glue (RVS-WS <-> vLLM-Realtime-WS). Das Modell
|
# Voxtral-3B via Transformers. torch/torchaudio kommen cu124-gepinnt aus dem
|
||||||
# selbst laeuft im separaten voxtral-vllm-Container (GPU). Deshalb hier KEIN
|
# Dockerfile (nicht hier, sonst zieht pip das Default-CUDA-Wheel).
|
||||||
# torch/vllm — nur der WebSocket-Client + numpy fuer die RMS-Energiemessung.
|
transformers>=4.54
|
||||||
websockets>=12.0
|
mistral-common[audio]>=1.8.1
|
||||||
|
accelerate>=0.30
|
||||||
|
speechbrain>=1.0 # Speaker-ID (ECAPA-TDNN) — nur Stefans Stimme
|
||||||
|
soundfile>=0.12
|
||||||
|
librosa>=0.10 # VoxtralProcessor.load_audio_as nutzt librosa zum WAV-Laden
|
||||||
numpy>=1.24
|
numpy>=1.24
|
||||||
|
websockets>=12.0
|
||||||
|
|||||||
@@ -0,0 +1,272 @@
|
|||||||
|
"""
|
||||||
|
Speaker-ID Backend fuer ARIAs Stimmen-Erkennung.
|
||||||
|
|
||||||
|
Nutzt SpeechBrain ECAPA-TDNN (192-dim Embeddings, auf VoxCeleb-1+2 trainiert).
|
||||||
|
Fingerprint = gemittelter, L2-normalisierter Embedding-Vektor aus N
|
||||||
|
Enrollment-Samples. Verify: cosine_similarity(neue_aufnahme, fingerprint).
|
||||||
|
|
||||||
|
Persistenz: /voice-id/fingerprint.json (Float-Liste + Metadaten).
|
||||||
|
Modell-Cache: /root/.cache/huggingface/ (Bind-Mount mit f5tts geteilt).
|
||||||
|
|
||||||
|
Verhalten OHNE Enrollment (kein Fingerprint vorhanden):
|
||||||
|
verify() → (True, 0.0) — Fail-open, damit Speaker-ID-Gating den
|
||||||
|
ungeenrollten Brain-Pfad nicht versehentlich blockiert.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
VOICE_ID_DIR = Path(os.environ.get("VOICE_ID_DIR", "/voice-id"))
|
||||||
|
FINGERPRINT_FILE = VOICE_ID_DIR / "fingerprint.json"
|
||||||
|
|
||||||
|
# Cosine-Threshold: 0.5 ist konservativ (wenig false-positives), 0.3 ist
|
||||||
|
# locker (mehr Treffer auch bei Nebengeraeuschen). Stefan kann's per
|
||||||
|
# Diagnostic-Setting feintunen.
|
||||||
|
DEFAULT_THRESHOLD = 0.5
|
||||||
|
|
||||||
|
# Minimal-Sample-Laenge fuer ein verlaessliches Embedding (~1s @ 16kHz int16 = 32000 bytes)
|
||||||
|
MIN_SAMPLE_BYTES = 32000
|
||||||
|
|
||||||
|
_model = None
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_loaded():
|
||||||
|
"""Lazy-Load des ECAPA-TDNN. Holt das Modell beim ersten Aufruf von HF;
|
||||||
|
danach cached im HF-Cache-Volume. Erste Init: ~30s download + load,
|
||||||
|
danach <1s warm. Wirft bei Fehler — Caller muss catchen + fail-open."""
|
||||||
|
global _model
|
||||||
|
if _model is not None:
|
||||||
|
return _model
|
||||||
|
import torch
|
||||||
|
from speechbrain.inference.speaker import EncoderClassifier
|
||||||
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||||
|
logger.info("[speaker-id] loading ECAPA-TDNN on %s ...", device)
|
||||||
|
_model = EncoderClassifier.from_hparams(
|
||||||
|
source="speechbrain/spkrec-ecapa-voxceleb",
|
||||||
|
savedir="/root/.cache/huggingface/speechbrain-ecapa",
|
||||||
|
run_opts={"device": device},
|
||||||
|
)
|
||||||
|
logger.info("[speaker-id] model ready (device=%s)", device)
|
||||||
|
return _model
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_compressed_to_pcm(audio_bytes: bytes) -> bytes:
|
||||||
|
"""Dekodiert komprimiertes Audio (MP4/M4A/AAC vom Android-Recorder) via ffmpeg
|
||||||
|
(im Container vorhanden) auf rohes 16kHz mono int16 LE PCM. Input geht ueber
|
||||||
|
eine Temp-Datei (nicht Pipe): Androids MediaRecorder legt das moov-Atom ans
|
||||||
|
ENDE, das braucht seekbaren Input, sonst 'moov atom not found'."""
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
tmp = None
|
||||||
|
try:
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tf:
|
||||||
|
tf.write(audio_bytes)
|
||||||
|
tmp = tf.name
|
||||||
|
proc = subprocess.run(
|
||||||
|
["ffmpeg", "-hide_banner", "-loglevel", "error", "-i", tmp,
|
||||||
|
"-f", "s16le", "-ac", "1", "-ar", "16000", "pipe:1"],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
if proc.returncode != 0 or not proc.stdout:
|
||||||
|
raise ValueError(
|
||||||
|
f"ffmpeg decode failed: {proc.stderr.decode('utf-8', 'ignore')[:200]}")
|
||||||
|
return proc.stdout
|
||||||
|
finally:
|
||||||
|
if tmp:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_audio_bytes(audio_bytes: bytes) -> bytes:
|
||||||
|
"""Akzeptiert rohes 16kHz int16 LE PCM, eine WAV-Datei (RIFF/WAVE) ODER einen
|
||||||
|
komprimierten MP4/M4A/AAC-Container (Android-Recorder). WAV → Header strippen +
|
||||||
|
Format validieren; MP4/AAC → via ffmpeg dekodieren. Ergebnis: rohes PCM."""
|
||||||
|
if (len(audio_bytes) >= 44
|
||||||
|
and audio_bytes[:4] == b"RIFF"
|
||||||
|
and audio_bytes[8:12] == b"WAVE"):
|
||||||
|
import io
|
||||||
|
import wave
|
||||||
|
with wave.open(io.BytesIO(audio_bytes), "rb") as wav:
|
||||||
|
sr = wav.getframerate()
|
||||||
|
ch = wav.getnchannels()
|
||||||
|
sw = wav.getsampwidth()
|
||||||
|
if sr != 16000:
|
||||||
|
raise ValueError(f"WAV-Samplerate {sr} != 16000")
|
||||||
|
if ch != 1:
|
||||||
|
raise ValueError(f"WAV-Kanalzahl {ch} != 1 (mono erwartet)")
|
||||||
|
if sw != 2:
|
||||||
|
raise ValueError(f"WAV-Sampleweite {sw} != 2 (int16 erwartet)")
|
||||||
|
return wav.readframes(wav.getnframes())
|
||||||
|
# MP4/M4A/AAC-Container: Android-AAC-Recorder legt 'ftyp' bei Offset 4 an.
|
||||||
|
if len(audio_bytes) >= 12 and audio_bytes[4:8] == b"ftyp":
|
||||||
|
return _decode_compressed_to_pcm(audio_bytes)
|
||||||
|
return audio_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def _audio_bytes_to_tensor(audio_bytes: bytes):
|
||||||
|
"""int16 LE PCM (16kHz mono) → Torch-Tensor (1, N), normalisiert auf [-1, 1].
|
||||||
|
WAV wird vorher auf rohes PCM reduziert (Header strippen)."""
|
||||||
|
import torch
|
||||||
|
raw = _normalize_audio_bytes(audio_bytes)
|
||||||
|
arr = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
|
|
||||||
|
|
||||||
|
def embed(audio_bytes: bytes) -> np.ndarray:
|
||||||
|
"""Berechnet das Speaker-Embedding fuer einen Audio-Chunk.
|
||||||
|
Erwartet 16kHz int16 LE PCM Mono. Returns 192-dim numpy float32."""
|
||||||
|
import torch
|
||||||
|
model = _ensure_loaded()
|
||||||
|
wav = _audio_bytes_to_tensor(audio_bytes)
|
||||||
|
with torch.no_grad():
|
||||||
|
emb = model.encode_batch(wav)
|
||||||
|
return emb.squeeze().cpu().numpy().astype(np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
|
||||||
|
"""Kosinus-Aehnlichkeit zwischen zwei 1D-Vektoren, Range [-1, 1].
|
||||||
|
Hoeher = aehnlicher. Bei normalisierten Vektoren ist das gleich dem Skalarprodukt."""
|
||||||
|
na = np.linalg.norm(a)
|
||||||
|
nb = np.linalg.norm(b)
|
||||||
|
if na < 1e-9 or nb < 1e-9:
|
||||||
|
return 0.0
|
||||||
|
return float(np.dot(a, b) / (na * nb))
|
||||||
|
|
||||||
|
|
||||||
|
def save_fingerprint(embeddings: list[np.ndarray], sample_durations_s: list[float]) -> dict:
|
||||||
|
"""Mittelt + L2-normalisiert die Embeddings und schreibt sie nach
|
||||||
|
FINGERPRINT_FILE. Returns das gespeicherte Dict."""
|
||||||
|
if not embeddings:
|
||||||
|
raise ValueError("Keine Embeddings zum Speichern")
|
||||||
|
VOICE_ID_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
stacked = np.stack(embeddings)
|
||||||
|
mean = stacked.mean(axis=0)
|
||||||
|
mean = mean / max(np.linalg.norm(mean), 1e-9)
|
||||||
|
data = {
|
||||||
|
"version": 1,
|
||||||
|
"embedding": mean.tolist(),
|
||||||
|
"embedding_dim": int(mean.shape[0]),
|
||||||
|
"sample_count": len(embeddings),
|
||||||
|
"sample_durations_s": [float(s) for s in sample_durations_s],
|
||||||
|
"updated_at": int(time.time()),
|
||||||
|
}
|
||||||
|
FINGERPRINT_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||||
|
logger.info("[speaker-id] fingerprint gespeichert: %d Samples, dim=%d, total_s=%.1f",
|
||||||
|
len(embeddings), mean.shape[0], sum(sample_durations_s))
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def load_fingerprint() -> Optional[dict]:
|
||||||
|
"""Returns das Fingerprint-Dict oder None wenn noch nicht enrolled."""
|
||||||
|
if not FINGERPRINT_FILE.exists():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return json.loads(FINGERPRINT_FILE.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("[speaker-id] fingerprint laden fehlgeschlagen: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def delete_fingerprint() -> bool:
|
||||||
|
"""Loescht den Fingerprint (z.B. fuer Re-Enrollment). True wenn was weg ist."""
|
||||||
|
if FINGERPRINT_FILE.exists():
|
||||||
|
FINGERPRINT_FILE.unlink()
|
||||||
|
logger.info("[speaker-id] fingerprint geloescht")
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def verify(audio_bytes: bytes, threshold: Optional[float] = None) -> tuple[bool, float]:
|
||||||
|
"""Returns (is_match, similarity).
|
||||||
|
|
||||||
|
Wenn threshold=None: nutzt den Modul-Default (DEFAULT_THRESHOLD) — der wird
|
||||||
|
vom config-Broadcast zur Laufzeit auf den Diagnostic-Slider-Wert gesetzt.
|
||||||
|
Default-Arg-Bindung waere zur Def-Zeit, also bewusst None statt direkt.
|
||||||
|
|
||||||
|
Fail-open: wenn kein Fingerprint vorhanden ist oder das Embedding-Modell
|
||||||
|
crasht, returnt (True, 0.0) — kein Filtering. Sonst wuerde ein kaputter
|
||||||
|
Speaker-ID-Service die ganze Aufnahme blockieren."""
|
||||||
|
if threshold is None:
|
||||||
|
threshold = DEFAULT_THRESHOLD
|
||||||
|
fp = load_fingerprint()
|
||||||
|
if fp is None:
|
||||||
|
return True, 0.0
|
||||||
|
if len(audio_bytes) < MIN_SAMPLE_BYTES:
|
||||||
|
# Zu wenig Audio fuer ein verlaessliches Embedding → durchlassen
|
||||||
|
return True, 0.0
|
||||||
|
try:
|
||||||
|
saved_emb = np.array(fp["embedding"], dtype=np.float32)
|
||||||
|
new_emb = embed(audio_bytes)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("[speaker-id] verify embed failed: %s — fail-open", exc)
|
||||||
|
return True, 0.0
|
||||||
|
sim = cosine_similarity(new_emb, saved_emb)
|
||||||
|
return sim >= threshold, sim
|
||||||
|
|
||||||
|
|
||||||
|
def status() -> dict:
|
||||||
|
"""Status-Snapshot fuer die App / Diagnostic."""
|
||||||
|
fp = load_fingerprint()
|
||||||
|
return {
|
||||||
|
"enrolled": fp is not None,
|
||||||
|
"sample_count": fp.get("sample_count", 0) if fp else 0,
|
||||||
|
"sample_durations_s": fp.get("sample_durations_s", []) if fp else [],
|
||||||
|
"updated_at": fp.get("updated_at") if fp else None,
|
||||||
|
"embedding_dim": fp.get("embedding_dim") if fp else None,
|
||||||
|
"default_threshold": DEFAULT_THRESHOLD,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def enroll_from_samples(samples_b64: list[str]) -> dict:
|
||||||
|
"""Verarbeitet base64-Samples (16kHz int16 LE PCM Mono) zu einem neuen
|
||||||
|
Fingerprint. Returns Status-Dict. Wirft ValueError wenn nichts brauchbar ist."""
|
||||||
|
if not samples_b64:
|
||||||
|
raise ValueError("Keine Samples uebergeben")
|
||||||
|
embeddings: list[np.ndarray] = []
|
||||||
|
durations: list[float] = []
|
||||||
|
rejected: list[dict] = []
|
||||||
|
for idx, s in enumerate(samples_b64):
|
||||||
|
try:
|
||||||
|
raw = base64.b64decode(s)
|
||||||
|
except Exception as exc:
|
||||||
|
rejected.append({"index": idx, "reason": f"base64: {exc}"})
|
||||||
|
continue
|
||||||
|
# Erst dekodieren (WAV/MP4/AAC → rohes PCM), DANN Laenge pruefen: der
|
||||||
|
# Android-Recorder liefert komprimiertes MP4, dessen Byte-Laenge nichts
|
||||||
|
# ueber die Dauer sagt (4s AAC < 32KB → faelschlich "zu kurz").
|
||||||
|
try:
|
||||||
|
pcm = _normalize_audio_bytes(raw)
|
||||||
|
except Exception as exc:
|
||||||
|
rejected.append({"index": idx, "reason": f"decode: {exc}"})
|
||||||
|
continue
|
||||||
|
if len(pcm) < MIN_SAMPLE_BYTES:
|
||||||
|
rejected.append({"index": idx, "reason": f"zu kurz ({len(pcm)} bytes PCM)"})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
emb = embed(pcm)
|
||||||
|
embeddings.append(emb)
|
||||||
|
durations.append(len(pcm) / 2 / 16000.0)
|
||||||
|
except Exception as exc:
|
||||||
|
rejected.append({"index": idx, "reason": f"embed: {exc}"})
|
||||||
|
if not embeddings:
|
||||||
|
raise ValueError(
|
||||||
|
f"Keine Samples konnten verarbeitet werden ({len(rejected)} rejected). "
|
||||||
|
f"Details: {rejected[:3]}"
|
||||||
|
)
|
||||||
|
fingerprint = save_fingerprint(embeddings, durations)
|
||||||
|
fingerprint["rejected"] = rejected
|
||||||
|
return fingerprint
|
||||||
@@ -86,6 +86,16 @@ STREAM_VOICE_FACTOR = 2.5 # Sprache = noise_floor * Faktor
|
|||||||
STREAM_VOICE_RMS_MIN = 0.005 # Untergrenze (stiller Raum: nicht auf 0 kollabieren)
|
STREAM_VOICE_RMS_MIN = 0.005 # Untergrenze (stiller Raum: nicht auf 0 kollabieren)
|
||||||
STREAM_VOICE_RMS_MAX = 0.020 # Obergrenze (lautes Auto: Sprache nie ganz aussperren)
|
STREAM_VOICE_RMS_MAX = 0.020 # Obergrenze (lautes Auto: Sprache nie ganz aussperren)
|
||||||
STREAM_VOICE_RMS_THRESHOLD = 0.012 # Legacy-Konstante (nicht mehr im Cut-Pfad genutzt)
|
STREAM_VOICE_RMS_THRESHOLD = 0.012 # Legacy-Konstante (nicht mehr im Cut-Pfad genutzt)
|
||||||
|
|
||||||
|
# Speaker-ID Gating global an/aus. DEFAULT AUS (fail-open) — bewusster Schalter
|
||||||
|
# ("nur meine Stimme"), kein Automatismus: ein schlechter Enroll darf nie die STT
|
||||||
|
# lahmlegen. Wird per config-Broadcast (voiceIdEnabled) zur Laufzeit gesetzt.
|
||||||
|
SPEAKER_ID_ENABLED = os.getenv("VOICE_ID_ENABLED", "false").lower() in ("1", "true", "yes")
|
||||||
|
|
||||||
|
|
||||||
|
def _set_speaker_id_enabled(val: bool) -> None:
|
||||||
|
global SPEAKER_ID_ENABLED
|
||||||
|
SPEAKER_ID_ENABLED = bool(val)
|
||||||
# Rein-semantischer Backstop: wenn die Energie NIE faellt (laute Umgebung,
|
# Rein-semantischer Backstop: wenn die Energie NIE faellt (laute Umgebung,
|
||||||
# z.B. Auto), endpointen wir trotzdem — aber erst nach diesem Faktor x
|
# z.B. Auto), endpointen wir trotzdem — aber erst nach diesem Faktor x
|
||||||
# endpoint_ms, damit normales Sprechen mit Pausen nicht abgeschnitten wird.
|
# endpoint_ms, damit normales Sprechen mit Pausen nicht abgeschnitten wird.
|
||||||
@@ -467,6 +477,11 @@ class SessionManager:
|
|||||||
Ohne Fingerprint → fail-open (match=True). Bei mismatch wird die
|
Ohne Fingerprint → fail-open (match=True). Bei mismatch wird die
|
||||||
Session sofort beendet mit synthetischem stt_endpoint."""
|
Session sofort beendet mit synthetischem stt_endpoint."""
|
||||||
sess.speaker_checked = True
|
sess.speaker_checked = True
|
||||||
|
# Schalter aus (Default) → gar keine Pruefung, alles durchlassen.
|
||||||
|
if not SPEAKER_ID_ENABLED:
|
||||||
|
sess.speaker_match = True
|
||||||
|
sess.speaker_similarity = 0.0
|
||||||
|
return
|
||||||
# Erste ~1.5s aus dem Buffer entnehmen (16kHz * 2 byte/sample = 32 bytes/ms)
|
# Erste ~1.5s aus dem Buffer entnehmen (16kHz * 2 byte/sample = 32 bytes/ms)
|
||||||
head_bytes = bytes(sess.pcm_buffer[: STREAM_SPEAKER_CHECK_MS * 32])
|
head_bytes = bytes(sess.pcm_buffer[: STREAM_SPEAKER_CHECK_MS * 32])
|
||||||
if len(head_bytes) < speaker_id.MIN_SAMPLE_BYTES:
|
if len(head_bytes) < speaker_id.MIN_SAMPLE_BYTES:
|
||||||
@@ -975,6 +990,10 @@ async def run_loop(runner: WhisperRunner, sessions: SessionManager) -> None:
|
|||||||
logger.info("[speaker-id] threshold gesetzt: %.2f", t)
|
logger.info("[speaker-id] threshold gesetzt: %.2f", t)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
pass
|
pass
|
||||||
|
if "voiceIdEnabled" in payload:
|
||||||
|
_set_speaker_id_enabled(payload.get("voiceIdEnabled"))
|
||||||
|
logger.info("[speaker-id] Gating %s (voiceIdEnabled)",
|
||||||
|
"AN" if SPEAKER_ID_ENABLED else "AUS")
|
||||||
if "whisperDebugLog" in payload:
|
if "whisperDebugLog" in payload:
|
||||||
global _DEBUG_LOG_TO_BRIDGE
|
global _DEBUG_LOG_TO_BRIDGE
|
||||||
old = _DEBUG_LOG_TO_BRIDGE
|
old = _DEBUG_LOG_TO_BRIDGE
|
||||||
|
|||||||
@@ -61,10 +61,40 @@ def _ensure_loaded():
|
|||||||
return _model
|
return _model
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_compressed_to_pcm(audio_bytes: bytes) -> bytes:
|
||||||
|
"""Dekodiert komprimiertes Audio (MP4/M4A/AAC vom Android-Recorder) via ffmpeg
|
||||||
|
(im Container vorhanden) auf rohes 16kHz mono int16 LE PCM. Input geht ueber
|
||||||
|
eine Temp-Datei (nicht Pipe): Androids MediaRecorder legt das moov-Atom ans
|
||||||
|
ENDE, das braucht seekbaren Input, sonst 'moov atom not found'."""
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
tmp = None
|
||||||
|
try:
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tf:
|
||||||
|
tf.write(audio_bytes)
|
||||||
|
tmp = tf.name
|
||||||
|
proc = subprocess.run(
|
||||||
|
["ffmpeg", "-hide_banner", "-loglevel", "error", "-i", tmp,
|
||||||
|
"-f", "s16le", "-ac", "1", "-ar", "16000", "pipe:1"],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
if proc.returncode != 0 or not proc.stdout:
|
||||||
|
raise ValueError(
|
||||||
|
f"ffmpeg decode failed: {proc.stderr.decode('utf-8', 'ignore')[:200]}")
|
||||||
|
return proc.stdout
|
||||||
|
finally:
|
||||||
|
if tmp:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _normalize_audio_bytes(audio_bytes: bytes) -> bytes:
|
def _normalize_audio_bytes(audio_bytes: bytes) -> bytes:
|
||||||
"""Akzeptiert entweder rohes 16kHz int16 LE PCM ODER eine WAV-Datei (RIFF/WAVE).
|
"""Akzeptiert rohes 16kHz int16 LE PCM, eine WAV-Datei (RIFF/WAVE) ODER einen
|
||||||
Bei WAV wird der Header gestrippt + Format validiert (16kHz / mono / int16).
|
komprimierten MP4/M4A/AAC-Container (Android-Recorder). WAV → Header strippen +
|
||||||
Ergebnis: rohes PCM."""
|
Format validieren; MP4/AAC → via ffmpeg dekodieren. Ergebnis: rohes PCM."""
|
||||||
if (len(audio_bytes) >= 44
|
if (len(audio_bytes) >= 44
|
||||||
and audio_bytes[:4] == b"RIFF"
|
and audio_bytes[:4] == b"RIFF"
|
||||||
and audio_bytes[8:12] == b"WAVE"):
|
and audio_bytes[8:12] == b"WAVE"):
|
||||||
@@ -81,6 +111,9 @@ def _normalize_audio_bytes(audio_bytes: bytes) -> bytes:
|
|||||||
if sw != 2:
|
if sw != 2:
|
||||||
raise ValueError(f"WAV-Sampleweite {sw} != 2 (int16 erwartet)")
|
raise ValueError(f"WAV-Sampleweite {sw} != 2 (int16 erwartet)")
|
||||||
return wav.readframes(wav.getnframes())
|
return wav.readframes(wav.getnframes())
|
||||||
|
# MP4/M4A/AAC-Container: Android-AAC-Recorder legt 'ftyp' bei Offset 4 an.
|
||||||
|
if len(audio_bytes) >= 12 and audio_bytes[4:8] == b"ftyp":
|
||||||
|
return _decode_compressed_to_pcm(audio_bytes)
|
||||||
return audio_bytes
|
return audio_bytes
|
||||||
|
|
||||||
|
|
||||||
@@ -212,13 +245,21 @@ def enroll_from_samples(samples_b64: list[str]) -> dict:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
rejected.append({"index": idx, "reason": f"base64: {exc}"})
|
rejected.append({"index": idx, "reason": f"base64: {exc}"})
|
||||||
continue
|
continue
|
||||||
if len(raw) < MIN_SAMPLE_BYTES:
|
# Erst dekodieren (WAV/MP4/AAC → rohes PCM), DANN Laenge pruefen: der
|
||||||
rejected.append({"index": idx, "reason": f"zu kurz ({len(raw)} bytes)"})
|
# Android-Recorder liefert komprimiertes MP4, dessen Byte-Laenge nichts
|
||||||
|
# ueber die Dauer sagt (4s AAC < 32KB → faelschlich "zu kurz").
|
||||||
|
try:
|
||||||
|
pcm = _normalize_audio_bytes(raw)
|
||||||
|
except Exception as exc:
|
||||||
|
rejected.append({"index": idx, "reason": f"decode: {exc}"})
|
||||||
|
continue
|
||||||
|
if len(pcm) < MIN_SAMPLE_BYTES:
|
||||||
|
rejected.append({"index": idx, "reason": f"zu kurz ({len(pcm)} bytes PCM)"})
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
emb = embed(raw)
|
emb = embed(pcm)
|
||||||
embeddings.append(emb)
|
embeddings.append(emb)
|
||||||
durations.append(len(raw) / 2 / 16000.0)
|
durations.append(len(pcm) / 2 / 16000.0)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
rejected.append({"index": idx, "reason": f"embed: {exc}"})
|
rejected.append({"index": idx, "reason": f"embed: {exc}"})
|
||||||
if not embeddings:
|
if not embeddings:
|
||||||
|
|||||||
Reference in New Issue
Block a user