feat(voxtral): Speaker-ID portiert (nur Stefans Stimme) — E3a
Voxtral hatte 0 Speaker-Filter (mit Voxtral reagierte ARIA auf JEDE Stimme). Jetzt portiert aus der whisper-Bridge: speaker_id.py (ECAPA/speechbrain) kopiert, Einmal-Check auf die ersten 1.5s (fremde Stimme → leeres stt_endpoint reason=speaker_mismatch, kein Transcribe/Brain), voice_id_enroll/status/delete-RVS-Handler + voiceIdThreshold-config. voice-id-Volume gemountet, speechbrain in requirements. Ohne Enrollment fail-open. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -157,6 +157,7 @@ services:
|
||||
capabilities: [gpu]
|
||||
volumes:
|
||||
- ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5
|
||||
- ./voice-id:/voice-id # Speaker-Fingerprint (wie whisper)
|
||||
environment:
|
||||
- RVS_HOST=${RVS_HOST}
|
||||
- RVS_PORT=${RVS_PORT:-443}
|
||||
|
||||
@@ -21,6 +21,6 @@ COPY requirements.txt .
|
||||
RUN printf 'torch==2.6.0\ntorchaudio==2.6.0\n' > /tmp/torch-constraint.txt && \
|
||||
pip3 install --no-cache-dir -c /tmp/torch-constraint.txt -r requirements.txt
|
||||
|
||||
COPY bridge.py .
|
||||
COPY bridge.py speaker_id.py ./
|
||||
|
||||
CMD ["python3", "bridge.py"]
|
||||
|
||||
@@ -38,6 +38,8 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
import websockets
|
||||
|
||||
import speaker_id # Speaker-ID (nur Stefans Stimme) — portiert aus der whisper-Bridge
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
@@ -59,6 +61,7 @@ STREAM_TRANSCRIBE_INTERVAL_MS = int(os.getenv("STREAM_TRANSCRIBE_INTERVAL_MS", "
|
||||
STREAM_DEFAULT_ENDPOINT_MS = 2400
|
||||
STREAM_DEFAULT_HARD_CAP_MS = 300000
|
||||
STREAM_MIN_AUDIO_MS = 600
|
||||
STREAM_SPEAKER_CHECK_MS = 1500 # ab so viel Audio einmalig Speaker-ID pruefen
|
||||
STREAM_SESSION_TTL_S = 120
|
||||
STREAM_ENERGY_WINDOW_MS = 300
|
||||
STREAM_SEMANTIC_BACKUP_FACTOR = 2.0
|
||||
@@ -165,6 +168,10 @@ class StreamSession:
|
||||
noise_floor: float = 0.0
|
||||
closed: bool = False
|
||||
endpoint_sent: bool = False
|
||||
# Speaker-ID Gating (einmalig auf die ersten ~1.5s der Aufnahme)
|
||||
speaker_checked: bool = False
|
||||
speaker_match: Optional[bool] = None
|
||||
speaker_similarity: float = 0.0
|
||||
|
||||
|
||||
class SessionManager:
|
||||
@@ -259,6 +266,59 @@ class SessionManager:
|
||||
else:
|
||||
sess.noise_floor = 0.98 * nf + 0.02 * rms
|
||||
|
||||
async def _check_speaker(self, sess: StreamSession) -> None:
|
||||
"""Einmalig: erste ~1.5s → Embedding → Vergleich mit Fingerprint.
|
||||
Ohne Fingerprint fail-open (match=True). Bei Mismatch: Session beenden."""
|
||||
sess.speaker_checked = True
|
||||
head = bytes(sess.pcm_buffer[: STREAM_SPEAKER_CHECK_MS * 32])
|
||||
if len(head) < speaker_id.MIN_SAMPLE_BYTES:
|
||||
sess.speaker_match = True
|
||||
return
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
is_match, sim = await loop.run_in_executor(None, speaker_id.verify, head)
|
||||
except Exception as exc:
|
||||
logger.warning("Stream %s: speaker-check crashed (%s) — fail-open",
|
||||
sess.request_id[:8], exc)
|
||||
sess.speaker_match = True
|
||||
return
|
||||
sess.speaker_match = is_match
|
||||
sess.speaker_similarity = sim
|
||||
logger.info("Stream %s: speaker-check sim=%.2f → %s (thr=%.2f)",
|
||||
sess.request_id[:8], sim, "MATCH" if is_match else "REJECT",
|
||||
speaker_id.DEFAULT_THRESHOLD)
|
||||
if not is_match:
|
||||
await self._finalize_speaker_mismatch(sess, sim)
|
||||
|
||||
async def _finalize_speaker_mismatch(self, sess: StreamSession, similarity: float) -> None:
|
||||
"""Fremde Stimme: synthetisches leeres stt_endpoint (reason=speaker_mismatch),
|
||||
Session droppen — kein Voxtral-Transcribe, kein Brain-Call."""
|
||||
if sess.endpoint_sent:
|
||||
return
|
||||
sess.endpoint_sent = True
|
||||
duration_s = self._buffer_ms(sess) / 1000.0
|
||||
logger.info("Stream %s: speaker-mismatch (sim=%.2f) — DROP nach %.1fs",
|
||||
sess.request_id[:8], similarity, duration_s)
|
||||
if self._ws is not None:
|
||||
payload = {
|
||||
"requestId": sess.request_id,
|
||||
"audioRequestId": sess.audio_request_id,
|
||||
"text": "", "reason": "speaker_mismatch",
|
||||
"durationS": duration_s, "sttMs": 0,
|
||||
"voice": sess.voice, "speed": sess.speed,
|
||||
"interrupted": sess.interrupted,
|
||||
"speakerSimilarity": float(similarity),
|
||||
}
|
||||
if sess.location:
|
||||
payload["location"] = sess.location
|
||||
await _send(self._ws, "stt_endpoint", payload)
|
||||
await _send(self._ws, "stt_stream_done", {
|
||||
"requestId": sess.request_id,
|
||||
"audioRequestId": sess.audio_request_id,
|
||||
"text": "", "reason": "speaker_mismatch",
|
||||
})
|
||||
self.drop(sess.request_id)
|
||||
|
||||
async def run_endpointer(self) -> None:
|
||||
logger.info("Voxtral-Endpointer gestartet (adaptiver VAD, interval=%dms)",
|
||||
STREAM_TRANSCRIBE_INTERVAL_MS)
|
||||
@@ -286,6 +346,12 @@ class SessionManager:
|
||||
return
|
||||
if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS:
|
||||
return
|
||||
# Speaker-ID einmalig: ist es Stefans Stimme? Fremde → Session verwerfen
|
||||
# (kein Transcribe, kein Brain-Call). Ohne Enrollment fail-open.
|
||||
if not sess.speaker_checked and self._buffer_ms(sess) >= STREAM_SPEAKER_CHECK_MS:
|
||||
await self._check_speaker(sess)
|
||||
if sess.speaker_match is False:
|
||||
return
|
||||
# Adaptive akustische Sprach-Aktivitaet (M0.1). KEINE Live-Partials mehr:
|
||||
# Voxtral-3B transkribiert den ganzen WACHSENDEN Buffer und braucht dafuer
|
||||
# bei langen Aufnahmen 5-6 s — zu langsam fuer Live-Text, UND diese Latenz
|
||||
@@ -377,6 +443,47 @@ async def run_loop(sessions: SessionManager) -> None:
|
||||
sessions.feed_chunk(payload)
|
||||
elif mtype == "stt_stream_end":
|
||||
sessions.end_session(payload.get("requestId", ""))
|
||||
elif mtype == "voice_id_status_request":
|
||||
req_id = payload.get("requestId", "")
|
||||
try:
|
||||
status = speaker_id.status()
|
||||
await _send(ws, "voice_id_status_response",
|
||||
{"requestId": req_id, "ok": True, **status})
|
||||
except Exception as exc:
|
||||
await _send(ws, "voice_id_status_response",
|
||||
{"requestId": req_id, "ok": False, "error": str(exc)[:200]})
|
||||
elif mtype == "voice_id_enroll_request":
|
||||
req_id = payload.get("requestId", "")
|
||||
samples = payload.get("samples") or []
|
||||
logger.info("voice_id_enroll_request: %d Samples (id=%s)", len(samples), req_id[:8])
|
||||
try:
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
None, speaker_id.enroll_from_samples, samples)
|
||||
await _send(ws, "voice_id_enroll_response", {
|
||||
"requestId": req_id, "ok": True,
|
||||
"sample_count": result.get("sample_count", 0),
|
||||
"rejected": result.get("rejected", []),
|
||||
"updated_at": result.get("updated_at"),
|
||||
"embedding_dim": result.get("embedding_dim"),
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.warning("voice_id_enroll failed: %s", exc)
|
||||
await _send(ws, "voice_id_enroll_response",
|
||||
{"requestId": req_id, "ok": False, "error": str(exc)[:300]})
|
||||
elif mtype == "voice_id_delete_request":
|
||||
req_id = payload.get("requestId", "")
|
||||
removed = speaker_id.delete_fingerprint()
|
||||
await _send(ws, "voice_id_delete_response",
|
||||
{"requestId": req_id, "ok": True, "removed": removed})
|
||||
elif mtype == "config":
|
||||
if "voiceIdThreshold" in payload:
|
||||
try:
|
||||
t = float(payload.get("voiceIdThreshold", 0.5))
|
||||
if 0.0 <= t <= 1.0:
|
||||
speaker_id.DEFAULT_THRESHOLD = t
|
||||
logger.info("[speaker-id] threshold gesetzt: %.2f", t)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s)
|
||||
if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried:
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
transformers>=4.54
|
||||
mistral-common[audio]>=1.8.1
|
||||
accelerate>=0.30
|
||||
speechbrain>=1.0 # Speaker-ID (ECAPA-TDNN) — nur Stefans Stimme
|
||||
soundfile>=0.12
|
||||
librosa>=0.10 # VoxtralProcessor.load_audio_as nutzt librosa zum WAV-Laden
|
||||
numpy>=1.24
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""
|
||||
Speaker-ID Backend fuer ARIAs Stimmen-Erkennung.
|
||||
|
||||
Nutzt SpeechBrain ECAPA-TDNN (192-dim Embeddings, auf VoxCeleb-1+2 trainiert).
|
||||
Fingerprint = gemittelter, L2-normalisierter Embedding-Vektor aus N
|
||||
Enrollment-Samples. Verify: cosine_similarity(neue_aufnahme, fingerprint).
|
||||
|
||||
Persistenz: /voice-id/fingerprint.json (Float-Liste + Metadaten).
|
||||
Modell-Cache: /root/.cache/huggingface/ (Bind-Mount mit f5tts geteilt).
|
||||
|
||||
Verhalten OHNE Enrollment (kein Fingerprint vorhanden):
|
||||
verify() → (True, 0.0) — Fail-open, damit Speaker-ID-Gating den
|
||||
ungeenrollten Brain-Pfad nicht versehentlich blockiert.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VOICE_ID_DIR = Path(os.environ.get("VOICE_ID_DIR", "/voice-id"))
|
||||
FINGERPRINT_FILE = VOICE_ID_DIR / "fingerprint.json"
|
||||
|
||||
# Cosine-Threshold: 0.5 ist konservativ (wenig false-positives), 0.3 ist
|
||||
# locker (mehr Treffer auch bei Nebengeraeuschen). Stefan kann's per
|
||||
# Diagnostic-Setting feintunen.
|
||||
DEFAULT_THRESHOLD = 0.5
|
||||
|
||||
# Minimal-Sample-Laenge fuer ein verlaessliches Embedding (~1s @ 16kHz int16 = 32000 bytes)
|
||||
MIN_SAMPLE_BYTES = 32000
|
||||
|
||||
_model = None
|
||||
|
||||
|
||||
def _ensure_loaded():
|
||||
"""Lazy-Load des ECAPA-TDNN. Holt das Modell beim ersten Aufruf von HF;
|
||||
danach cached im HF-Cache-Volume. Erste Init: ~30s download + load,
|
||||
danach <1s warm. Wirft bei Fehler — Caller muss catchen + fail-open."""
|
||||
global _model
|
||||
if _model is not None:
|
||||
return _model
|
||||
import torch
|
||||
from speechbrain.inference.speaker import EncoderClassifier
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
logger.info("[speaker-id] loading ECAPA-TDNN on %s ...", device)
|
||||
_model = EncoderClassifier.from_hparams(
|
||||
source="speechbrain/spkrec-ecapa-voxceleb",
|
||||
savedir="/root/.cache/huggingface/speechbrain-ecapa",
|
||||
run_opts={"device": device},
|
||||
)
|
||||
logger.info("[speaker-id] model ready (device=%s)", device)
|
||||
return _model
|
||||
|
||||
|
||||
def _normalize_audio_bytes(audio_bytes: bytes) -> bytes:
|
||||
"""Akzeptiert entweder rohes 16kHz int16 LE PCM ODER eine WAV-Datei (RIFF/WAVE).
|
||||
Bei WAV wird der Header gestrippt + Format validiert (16kHz / mono / int16).
|
||||
Ergebnis: rohes PCM."""
|
||||
if (len(audio_bytes) >= 44
|
||||
and audio_bytes[:4] == b"RIFF"
|
||||
and audio_bytes[8:12] == b"WAVE"):
|
||||
import io
|
||||
import wave
|
||||
with wave.open(io.BytesIO(audio_bytes), "rb") as wav:
|
||||
sr = wav.getframerate()
|
||||
ch = wav.getnchannels()
|
||||
sw = wav.getsampwidth()
|
||||
if sr != 16000:
|
||||
raise ValueError(f"WAV-Samplerate {sr} != 16000")
|
||||
if ch != 1:
|
||||
raise ValueError(f"WAV-Kanalzahl {ch} != 1 (mono erwartet)")
|
||||
if sw != 2:
|
||||
raise ValueError(f"WAV-Sampleweite {sw} != 2 (int16 erwartet)")
|
||||
return wav.readframes(wav.getnframes())
|
||||
return audio_bytes
|
||||
|
||||
|
||||
def _audio_bytes_to_tensor(audio_bytes: bytes):
|
||||
"""int16 LE PCM (16kHz mono) → Torch-Tensor (1, N), normalisiert auf [-1, 1].
|
||||
WAV wird vorher auf rohes PCM reduziert (Header strippen)."""
|
||||
import torch
|
||||
raw = _normalize_audio_bytes(audio_bytes)
|
||||
arr = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
return torch.from_numpy(arr).unsqueeze(0)
|
||||
|
||||
|
||||
def embed(audio_bytes: bytes) -> np.ndarray:
|
||||
"""Berechnet das Speaker-Embedding fuer einen Audio-Chunk.
|
||||
Erwartet 16kHz int16 LE PCM Mono. Returns 192-dim numpy float32."""
|
||||
import torch
|
||||
model = _ensure_loaded()
|
||||
wav = _audio_bytes_to_tensor(audio_bytes)
|
||||
with torch.no_grad():
|
||||
emb = model.encode_batch(wav)
|
||||
return emb.squeeze().cpu().numpy().astype(np.float32)
|
||||
|
||||
|
||||
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
|
||||
"""Kosinus-Aehnlichkeit zwischen zwei 1D-Vektoren, Range [-1, 1].
|
||||
Hoeher = aehnlicher. Bei normalisierten Vektoren ist das gleich dem Skalarprodukt."""
|
||||
na = np.linalg.norm(a)
|
||||
nb = np.linalg.norm(b)
|
||||
if na < 1e-9 or nb < 1e-9:
|
||||
return 0.0
|
||||
return float(np.dot(a, b) / (na * nb))
|
||||
|
||||
|
||||
def save_fingerprint(embeddings: list[np.ndarray], sample_durations_s: list[float]) -> dict:
|
||||
"""Mittelt + L2-normalisiert die Embeddings und schreibt sie nach
|
||||
FINGERPRINT_FILE. Returns das gespeicherte Dict."""
|
||||
if not embeddings:
|
||||
raise ValueError("Keine Embeddings zum Speichern")
|
||||
VOICE_ID_DIR.mkdir(parents=True, exist_ok=True)
|
||||
stacked = np.stack(embeddings)
|
||||
mean = stacked.mean(axis=0)
|
||||
mean = mean / max(np.linalg.norm(mean), 1e-9)
|
||||
data = {
|
||||
"version": 1,
|
||||
"embedding": mean.tolist(),
|
||||
"embedding_dim": int(mean.shape[0]),
|
||||
"sample_count": len(embeddings),
|
||||
"sample_durations_s": [float(s) for s in sample_durations_s],
|
||||
"updated_at": int(time.time()),
|
||||
}
|
||||
FINGERPRINT_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
logger.info("[speaker-id] fingerprint gespeichert: %d Samples, dim=%d, total_s=%.1f",
|
||||
len(embeddings), mean.shape[0], sum(sample_durations_s))
|
||||
return data
|
||||
|
||||
|
||||
def load_fingerprint() -> Optional[dict]:
|
||||
"""Returns das Fingerprint-Dict oder None wenn noch nicht enrolled."""
|
||||
if not FINGERPRINT_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(FINGERPRINT_FILE.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
logger.warning("[speaker-id] fingerprint laden fehlgeschlagen: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def delete_fingerprint() -> bool:
|
||||
"""Loescht den Fingerprint (z.B. fuer Re-Enrollment). True wenn was weg ist."""
|
||||
if FINGERPRINT_FILE.exists():
|
||||
FINGERPRINT_FILE.unlink()
|
||||
logger.info("[speaker-id] fingerprint geloescht")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def verify(audio_bytes: bytes, threshold: Optional[float] = None) -> tuple[bool, float]:
|
||||
"""Returns (is_match, similarity).
|
||||
|
||||
Wenn threshold=None: nutzt den Modul-Default (DEFAULT_THRESHOLD) — der wird
|
||||
vom config-Broadcast zur Laufzeit auf den Diagnostic-Slider-Wert gesetzt.
|
||||
Default-Arg-Bindung waere zur Def-Zeit, also bewusst None statt direkt.
|
||||
|
||||
Fail-open: wenn kein Fingerprint vorhanden ist oder das Embedding-Modell
|
||||
crasht, returnt (True, 0.0) — kein Filtering. Sonst wuerde ein kaputter
|
||||
Speaker-ID-Service die ganze Aufnahme blockieren."""
|
||||
if threshold is None:
|
||||
threshold = DEFAULT_THRESHOLD
|
||||
fp = load_fingerprint()
|
||||
if fp is None:
|
||||
return True, 0.0
|
||||
if len(audio_bytes) < MIN_SAMPLE_BYTES:
|
||||
# Zu wenig Audio fuer ein verlaessliches Embedding → durchlassen
|
||||
return True, 0.0
|
||||
try:
|
||||
saved_emb = np.array(fp["embedding"], dtype=np.float32)
|
||||
new_emb = embed(audio_bytes)
|
||||
except Exception as exc:
|
||||
logger.warning("[speaker-id] verify embed failed: %s — fail-open", exc)
|
||||
return True, 0.0
|
||||
sim = cosine_similarity(new_emb, saved_emb)
|
||||
return sim >= threshold, sim
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
"""Status-Snapshot fuer die App / Diagnostic."""
|
||||
fp = load_fingerprint()
|
||||
return {
|
||||
"enrolled": fp is not None,
|
||||
"sample_count": fp.get("sample_count", 0) if fp else 0,
|
||||
"sample_durations_s": fp.get("sample_durations_s", []) if fp else [],
|
||||
"updated_at": fp.get("updated_at") if fp else None,
|
||||
"embedding_dim": fp.get("embedding_dim") if fp else None,
|
||||
"default_threshold": DEFAULT_THRESHOLD,
|
||||
}
|
||||
|
||||
|
||||
def enroll_from_samples(samples_b64: list[str]) -> dict:
|
||||
"""Verarbeitet base64-Samples (16kHz int16 LE PCM Mono) zu einem neuen
|
||||
Fingerprint. Returns Status-Dict. Wirft ValueError wenn nichts brauchbar ist."""
|
||||
if not samples_b64:
|
||||
raise ValueError("Keine Samples uebergeben")
|
||||
embeddings: list[np.ndarray] = []
|
||||
durations: list[float] = []
|
||||
rejected: list[dict] = []
|
||||
for idx, s in enumerate(samples_b64):
|
||||
try:
|
||||
raw = base64.b64decode(s)
|
||||
except Exception as exc:
|
||||
rejected.append({"index": idx, "reason": f"base64: {exc}"})
|
||||
continue
|
||||
if len(raw) < MIN_SAMPLE_BYTES:
|
||||
rejected.append({"index": idx, "reason": f"zu kurz ({len(raw)} bytes)"})
|
||||
continue
|
||||
try:
|
||||
emb = embed(raw)
|
||||
embeddings.append(emb)
|
||||
durations.append(len(raw) / 2 / 16000.0)
|
||||
except Exception as exc:
|
||||
rejected.append({"index": idx, "reason": f"embed: {exc}"})
|
||||
if not embeddings:
|
||||
raise ValueError(
|
||||
f"Keine Samples konnten verarbeitet werden ({len(rejected)} rejected). "
|
||||
f"Details: {rejected[:3]}"
|
||||
)
|
||||
fingerprint = save_fingerprint(embeddings, durations)
|
||||
fingerprint["rejected"] = rejected
|
||||
return fingerprint
|
||||
Reference in New Issue
Block a user