From 7bc3f827d0ecc734c5ad45db91bfa414be1196d5 Mon Sep 17 00:00:00 2001 From: duffyduck Date: Sat, 15 Aug 2026 13:00:32 +0200 Subject: [PATCH] =?UTF-8?q?feat(voxtral):=20Speaker-ID=20portiert=20(nur?= =?UTF-8?q?=20Stefans=20Stimme)=20=E2=80=94=20E3a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voxtral hatte 0 Speaker-Filter (mit Voxtral reagierte ARIA auf JEDE Stimme). Jetzt portiert aus der whisper-Bridge: speaker_id.py (ECAPA/speechbrain) kopiert, Einmal-Check auf die ersten 1.5s (fremde Stimme → leeres stt_endpoint reason=speaker_mismatch, kein Transcribe/Brain), voice_id_enroll/status/delete-RVS-Handler + voiceIdThreshold-config. voice-id-Volume gemountet, speechbrain in requirements. Ohne Enrollment fail-open. Co-Authored-By: Claude Opus 4.8 --- xtts/docker-compose.yml | 1 + xtts/voxtral/Dockerfile | 2 +- xtts/voxtral/bridge.py | 107 ++++++++++++++++ xtts/voxtral/requirements.txt | 1 + xtts/voxtral/speaker_id.py | 231 ++++++++++++++++++++++++++++++++++ 5 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 xtts/voxtral/speaker_id.py diff --git a/xtts/docker-compose.yml b/xtts/docker-compose.yml index d59c71b..6106aa8 100644 --- a/xtts/docker-compose.yml +++ b/xtts/docker-compose.yml @@ -157,6 +157,7 @@ services: capabilities: [gpu] volumes: - ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5 + - ./voice-id:/voice-id # Speaker-Fingerprint (wie whisper) environment: - RVS_HOST=${RVS_HOST} - RVS_PORT=${RVS_PORT:-443} diff --git a/xtts/voxtral/Dockerfile b/xtts/voxtral/Dockerfile index 7894947..976507e 100644 --- a/xtts/voxtral/Dockerfile +++ b/xtts/voxtral/Dockerfile @@ -21,6 +21,6 @@ COPY requirements.txt . RUN printf 'torch==2.6.0\ntorchaudio==2.6.0\n' > /tmp/torch-constraint.txt && \ pip3 install --no-cache-dir -c /tmp/torch-constraint.txt -r requirements.txt -COPY bridge.py . +COPY bridge.py speaker_id.py ./ CMD ["python3", "bridge.py"] diff --git a/xtts/voxtral/bridge.py b/xtts/voxtral/bridge.py index c6d5195..b051148 100644 --- a/xtts/voxtral/bridge.py +++ b/xtts/voxtral/bridge.py @@ -38,6 +38,8 @@ import numpy as np import soundfile as sf import websockets +import speaker_id # Speaker-ID (nur Stefans Stimme) — portiert aus der whisper-Bridge + logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", @@ -59,6 +61,7 @@ STREAM_TRANSCRIBE_INTERVAL_MS = int(os.getenv("STREAM_TRANSCRIBE_INTERVAL_MS", " STREAM_DEFAULT_ENDPOINT_MS = 2400 STREAM_DEFAULT_HARD_CAP_MS = 300000 STREAM_MIN_AUDIO_MS = 600 +STREAM_SPEAKER_CHECK_MS = 1500 # ab so viel Audio einmalig Speaker-ID pruefen STREAM_SESSION_TTL_S = 120 STREAM_ENERGY_WINDOW_MS = 300 STREAM_SEMANTIC_BACKUP_FACTOR = 2.0 @@ -165,6 +168,10 @@ class StreamSession: noise_floor: float = 0.0 closed: bool = False endpoint_sent: bool = False + # Speaker-ID Gating (einmalig auf die ersten ~1.5s der Aufnahme) + speaker_checked: bool = False + speaker_match: Optional[bool] = None + speaker_similarity: float = 0.0 class SessionManager: @@ -259,6 +266,59 @@ class SessionManager: else: sess.noise_floor = 0.98 * nf + 0.02 * rms + async def _check_speaker(self, sess: StreamSession) -> None: + """Einmalig: erste ~1.5s → Embedding → Vergleich mit Fingerprint. + Ohne Fingerprint fail-open (match=True). Bei Mismatch: Session beenden.""" + sess.speaker_checked = True + head = bytes(sess.pcm_buffer[: STREAM_SPEAKER_CHECK_MS * 32]) + if len(head) < speaker_id.MIN_SAMPLE_BYTES: + sess.speaker_match = True + return + try: + loop = asyncio.get_running_loop() + is_match, sim = await loop.run_in_executor(None, speaker_id.verify, head) + except Exception as exc: + logger.warning("Stream %s: speaker-check crashed (%s) — fail-open", + sess.request_id[:8], exc) + sess.speaker_match = True + return + sess.speaker_match = is_match + sess.speaker_similarity = sim + logger.info("Stream %s: speaker-check sim=%.2f → %s (thr=%.2f)", + sess.request_id[:8], sim, "MATCH" if is_match else "REJECT", + speaker_id.DEFAULT_THRESHOLD) + if not is_match: + await self._finalize_speaker_mismatch(sess, sim) + + async def _finalize_speaker_mismatch(self, sess: StreamSession, similarity: float) -> None: + """Fremde Stimme: synthetisches leeres stt_endpoint (reason=speaker_mismatch), + Session droppen — kein Voxtral-Transcribe, kein Brain-Call.""" + if sess.endpoint_sent: + return + sess.endpoint_sent = True + duration_s = self._buffer_ms(sess) / 1000.0 + logger.info("Stream %s: speaker-mismatch (sim=%.2f) — DROP nach %.1fs", + sess.request_id[:8], similarity, duration_s) + if self._ws is not None: + payload = { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": "", "reason": "speaker_mismatch", + "durationS": duration_s, "sttMs": 0, + "voice": sess.voice, "speed": sess.speed, + "interrupted": sess.interrupted, + "speakerSimilarity": float(similarity), + } + if sess.location: + payload["location"] = sess.location + await _send(self._ws, "stt_endpoint", payload) + await _send(self._ws, "stt_stream_done", { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": "", "reason": "speaker_mismatch", + }) + self.drop(sess.request_id) + async def run_endpointer(self) -> None: logger.info("Voxtral-Endpointer gestartet (adaptiver VAD, interval=%dms)", STREAM_TRANSCRIBE_INTERVAL_MS) @@ -286,6 +346,12 @@ class SessionManager: return if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS: return + # Speaker-ID einmalig: ist es Stefans Stimme? Fremde → Session verwerfen + # (kein Transcribe, kein Brain-Call). Ohne Enrollment fail-open. + if not sess.speaker_checked and self._buffer_ms(sess) >= STREAM_SPEAKER_CHECK_MS: + await self._check_speaker(sess) + if sess.speaker_match is False: + return # Adaptive akustische Sprach-Aktivitaet (M0.1). KEINE Live-Partials mehr: # Voxtral-3B transkribiert den ganzen WACHSENDEN Buffer und braucht dafuer # bei langen Aufnahmen 5-6 s — zu langsam fuer Live-Text, UND diese Latenz @@ -377,6 +443,47 @@ async def run_loop(sessions: SessionManager) -> None: sessions.feed_chunk(payload) elif mtype == "stt_stream_end": sessions.end_session(payload.get("requestId", "")) + elif mtype == "voice_id_status_request": + req_id = payload.get("requestId", "") + try: + status = speaker_id.status() + await _send(ws, "voice_id_status_response", + {"requestId": req_id, "ok": True, **status}) + except Exception as exc: + await _send(ws, "voice_id_status_response", + {"requestId": req_id, "ok": False, "error": str(exc)[:200]}) + elif mtype == "voice_id_enroll_request": + req_id = payload.get("requestId", "") + samples = payload.get("samples") or [] + logger.info("voice_id_enroll_request: %d Samples (id=%s)", len(samples), req_id[:8]) + try: + result = await asyncio.get_running_loop().run_in_executor( + None, speaker_id.enroll_from_samples, samples) + await _send(ws, "voice_id_enroll_response", { + "requestId": req_id, "ok": True, + "sample_count": result.get("sample_count", 0), + "rejected": result.get("rejected", []), + "updated_at": result.get("updated_at"), + "embedding_dim": result.get("embedding_dim"), + }) + except Exception as exc: + logger.warning("voice_id_enroll failed: %s", exc) + await _send(ws, "voice_id_enroll_response", + {"requestId": req_id, "ok": False, "error": str(exc)[:300]}) + elif mtype == "voice_id_delete_request": + req_id = payload.get("requestId", "") + removed = speaker_id.delete_fingerprint() + await _send(ws, "voice_id_delete_response", + {"requestId": req_id, "ok": True, "removed": removed}) + elif mtype == "config": + if "voiceIdThreshold" in payload: + try: + t = float(payload.get("voiceIdThreshold", 0.5)) + if 0.0 <= t <= 1.0: + speaker_id.DEFAULT_THRESHOLD = t + logger.info("[speaker-id] threshold gesetzt: %.2f", t) + except (TypeError, ValueError): + pass except Exception as e: logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s) if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried: diff --git a/xtts/voxtral/requirements.txt b/xtts/voxtral/requirements.txt index a9f14be..7006b2f 100644 --- a/xtts/voxtral/requirements.txt +++ b/xtts/voxtral/requirements.txt @@ -3,6 +3,7 @@ transformers>=4.54 mistral-common[audio]>=1.8.1 accelerate>=0.30 +speechbrain>=1.0 # Speaker-ID (ECAPA-TDNN) — nur Stefans Stimme soundfile>=0.12 librosa>=0.10 # VoxtralProcessor.load_audio_as nutzt librosa zum WAV-Laden numpy>=1.24 diff --git a/xtts/voxtral/speaker_id.py b/xtts/voxtral/speaker_id.py new file mode 100644 index 0000000..7ed82f2 --- /dev/null +++ b/xtts/voxtral/speaker_id.py @@ -0,0 +1,231 @@ +""" +Speaker-ID Backend fuer ARIAs Stimmen-Erkennung. + +Nutzt SpeechBrain ECAPA-TDNN (192-dim Embeddings, auf VoxCeleb-1+2 trainiert). +Fingerprint = gemittelter, L2-normalisierter Embedding-Vektor aus N +Enrollment-Samples. Verify: cosine_similarity(neue_aufnahme, fingerprint). + +Persistenz: /voice-id/fingerprint.json (Float-Liste + Metadaten). +Modell-Cache: /root/.cache/huggingface/ (Bind-Mount mit f5tts geteilt). + +Verhalten OHNE Enrollment (kein Fingerprint vorhanden): + verify() → (True, 0.0) — Fail-open, damit Speaker-ID-Gating den + ungeenrollten Brain-Pfad nicht versehentlich blockiert. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import os +import time +from pathlib import Path +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + +VOICE_ID_DIR = Path(os.environ.get("VOICE_ID_DIR", "/voice-id")) +FINGERPRINT_FILE = VOICE_ID_DIR / "fingerprint.json" + +# Cosine-Threshold: 0.5 ist konservativ (wenig false-positives), 0.3 ist +# locker (mehr Treffer auch bei Nebengeraeuschen). Stefan kann's per +# Diagnostic-Setting feintunen. +DEFAULT_THRESHOLD = 0.5 + +# Minimal-Sample-Laenge fuer ein verlaessliches Embedding (~1s @ 16kHz int16 = 32000 bytes) +MIN_SAMPLE_BYTES = 32000 + +_model = None + + +def _ensure_loaded(): + """Lazy-Load des ECAPA-TDNN. Holt das Modell beim ersten Aufruf von HF; + danach cached im HF-Cache-Volume. Erste Init: ~30s download + load, + danach <1s warm. Wirft bei Fehler — Caller muss catchen + fail-open.""" + global _model + if _model is not None: + return _model + import torch + from speechbrain.inference.speaker import EncoderClassifier + device = "cuda" if torch.cuda.is_available() else "cpu" + logger.info("[speaker-id] loading ECAPA-TDNN on %s ...", device) + _model = EncoderClassifier.from_hparams( + source="speechbrain/spkrec-ecapa-voxceleb", + savedir="/root/.cache/huggingface/speechbrain-ecapa", + run_opts={"device": device}, + ) + logger.info("[speaker-id] model ready (device=%s)", device) + return _model + + +def _normalize_audio_bytes(audio_bytes: bytes) -> bytes: + """Akzeptiert entweder rohes 16kHz int16 LE PCM ODER eine WAV-Datei (RIFF/WAVE). + Bei WAV wird der Header gestrippt + Format validiert (16kHz / mono / int16). + Ergebnis: rohes PCM.""" + if (len(audio_bytes) >= 44 + and audio_bytes[:4] == b"RIFF" + and audio_bytes[8:12] == b"WAVE"): + import io + import wave + with wave.open(io.BytesIO(audio_bytes), "rb") as wav: + sr = wav.getframerate() + ch = wav.getnchannels() + sw = wav.getsampwidth() + if sr != 16000: + raise ValueError(f"WAV-Samplerate {sr} != 16000") + if ch != 1: + raise ValueError(f"WAV-Kanalzahl {ch} != 1 (mono erwartet)") + if sw != 2: + raise ValueError(f"WAV-Sampleweite {sw} != 2 (int16 erwartet)") + return wav.readframes(wav.getnframes()) + return audio_bytes + + +def _audio_bytes_to_tensor(audio_bytes: bytes): + """int16 LE PCM (16kHz mono) → Torch-Tensor (1, N), normalisiert auf [-1, 1]. + WAV wird vorher auf rohes PCM reduziert (Header strippen).""" + import torch + raw = _normalize_audio_bytes(audio_bytes) + arr = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def embed(audio_bytes: bytes) -> np.ndarray: + """Berechnet das Speaker-Embedding fuer einen Audio-Chunk. + Erwartet 16kHz int16 LE PCM Mono. Returns 192-dim numpy float32.""" + import torch + model = _ensure_loaded() + wav = _audio_bytes_to_tensor(audio_bytes) + with torch.no_grad(): + emb = model.encode_batch(wav) + return emb.squeeze().cpu().numpy().astype(np.float32) + + +def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: + """Kosinus-Aehnlichkeit zwischen zwei 1D-Vektoren, Range [-1, 1]. + Hoeher = aehnlicher. Bei normalisierten Vektoren ist das gleich dem Skalarprodukt.""" + na = np.linalg.norm(a) + nb = np.linalg.norm(b) + if na < 1e-9 or nb < 1e-9: + return 0.0 + return float(np.dot(a, b) / (na * nb)) + + +def save_fingerprint(embeddings: list[np.ndarray], sample_durations_s: list[float]) -> dict: + """Mittelt + L2-normalisiert die Embeddings und schreibt sie nach + FINGERPRINT_FILE. Returns das gespeicherte Dict.""" + if not embeddings: + raise ValueError("Keine Embeddings zum Speichern") + VOICE_ID_DIR.mkdir(parents=True, exist_ok=True) + stacked = np.stack(embeddings) + mean = stacked.mean(axis=0) + mean = mean / max(np.linalg.norm(mean), 1e-9) + data = { + "version": 1, + "embedding": mean.tolist(), + "embedding_dim": int(mean.shape[0]), + "sample_count": len(embeddings), + "sample_durations_s": [float(s) for s in sample_durations_s], + "updated_at": int(time.time()), + } + FINGERPRINT_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8") + logger.info("[speaker-id] fingerprint gespeichert: %d Samples, dim=%d, total_s=%.1f", + len(embeddings), mean.shape[0], sum(sample_durations_s)) + return data + + +def load_fingerprint() -> Optional[dict]: + """Returns das Fingerprint-Dict oder None wenn noch nicht enrolled.""" + if not FINGERPRINT_FILE.exists(): + return None + try: + return json.loads(FINGERPRINT_FILE.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("[speaker-id] fingerprint laden fehlgeschlagen: %s", exc) + return None + + +def delete_fingerprint() -> bool: + """Loescht den Fingerprint (z.B. fuer Re-Enrollment). True wenn was weg ist.""" + if FINGERPRINT_FILE.exists(): + FINGERPRINT_FILE.unlink() + logger.info("[speaker-id] fingerprint geloescht") + return True + return False + + +def verify(audio_bytes: bytes, threshold: Optional[float] = None) -> tuple[bool, float]: + """Returns (is_match, similarity). + + Wenn threshold=None: nutzt den Modul-Default (DEFAULT_THRESHOLD) — der wird + vom config-Broadcast zur Laufzeit auf den Diagnostic-Slider-Wert gesetzt. + Default-Arg-Bindung waere zur Def-Zeit, also bewusst None statt direkt. + + Fail-open: wenn kein Fingerprint vorhanden ist oder das Embedding-Modell + crasht, returnt (True, 0.0) — kein Filtering. Sonst wuerde ein kaputter + Speaker-ID-Service die ganze Aufnahme blockieren.""" + if threshold is None: + threshold = DEFAULT_THRESHOLD + fp = load_fingerprint() + if fp is None: + return True, 0.0 + if len(audio_bytes) < MIN_SAMPLE_BYTES: + # Zu wenig Audio fuer ein verlaessliches Embedding → durchlassen + return True, 0.0 + try: + saved_emb = np.array(fp["embedding"], dtype=np.float32) + new_emb = embed(audio_bytes) + except Exception as exc: + logger.warning("[speaker-id] verify embed failed: %s — fail-open", exc) + return True, 0.0 + sim = cosine_similarity(new_emb, saved_emb) + return sim >= threshold, sim + + +def status() -> dict: + """Status-Snapshot fuer die App / Diagnostic.""" + fp = load_fingerprint() + return { + "enrolled": fp is not None, + "sample_count": fp.get("sample_count", 0) if fp else 0, + "sample_durations_s": fp.get("sample_durations_s", []) if fp else [], + "updated_at": fp.get("updated_at") if fp else None, + "embedding_dim": fp.get("embedding_dim") if fp else None, + "default_threshold": DEFAULT_THRESHOLD, + } + + +def enroll_from_samples(samples_b64: list[str]) -> dict: + """Verarbeitet base64-Samples (16kHz int16 LE PCM Mono) zu einem neuen + Fingerprint. Returns Status-Dict. Wirft ValueError wenn nichts brauchbar ist.""" + if not samples_b64: + raise ValueError("Keine Samples uebergeben") + embeddings: list[np.ndarray] = [] + durations: list[float] = [] + rejected: list[dict] = [] + for idx, s in enumerate(samples_b64): + try: + raw = base64.b64decode(s) + except Exception as exc: + rejected.append({"index": idx, "reason": f"base64: {exc}"}) + continue + if len(raw) < MIN_SAMPLE_BYTES: + rejected.append({"index": idx, "reason": f"zu kurz ({len(raw)} bytes)"}) + continue + try: + emb = embed(raw) + embeddings.append(emb) + durations.append(len(raw) / 2 / 16000.0) + except Exception as exc: + rejected.append({"index": idx, "reason": f"embed: {exc}"}) + if not embeddings: + raise ValueError( + f"Keine Samples konnten verarbeitet werden ({len(rejected)} rejected). " + f"Details: {rejected[:3]}" + ) + fingerprint = save_fingerprint(embeddings, durations) + fingerprint["rejected"] = rejected + return fingerprint