From 174d6d643d6af1edc21b4da30e29323dbc882219 Mon Sep 17 00:00:00 2001 From: duffyduck Date: Sat, 15 Aug 2026 01:21:43 +0200 Subject: [PATCH] feat(voxtral): STT-Satellit (Profil) + 2-Karten-GPU-Pinning Neuer xtts/voxtral-Container (RVS<->vLLM-Realtime-WS, adaptiver Endpointer), hinter Compose-Profil 'voxtral' (braucht >=16GB VRAM, startet nicht im Default). Bestehende Satelliten auf die zwei 3060 gepinnt: whisper->GPU0, f5tts->GPU1, llama-swap->GPU0. Co-Authored-By: Claude Opus 4.8 --- xtts/docker-compose.yml | 59 +++- xtts/voxtral/Dockerfile | 14 + xtts/voxtral/README.md | 70 +++++ xtts/voxtral/bridge.py | 494 ++++++++++++++++++++++++++++++++++ xtts/voxtral/requirements.txt | 5 + 5 files changed, 639 insertions(+), 3 deletions(-) create mode 100644 xtts/voxtral/Dockerfile create mode 100644 xtts/voxtral/README.md create mode 100644 xtts/voxtral/bridge.py create mode 100644 xtts/voxtral/requirements.txt diff --git a/xtts/docker-compose.yml b/xtts/docker-compose.yml index c339b91..0605327 100644 --- a/xtts/docker-compose.yml +++ b/xtts/docker-compose.yml @@ -30,7 +30,7 @@ services: reservations: devices: - driver: nvidia - count: 1 + device_ids: ["1"] # TTS-Rolle → GPU 1 (spaeter: Voxtral-TTS) capabilities: [gpu] volumes: - ./voices:/voices # WAV + TXT Referenz @@ -68,7 +68,7 @@ services: reservations: devices: - driver: nvidia - count: 1 + device_ids: ["0"] # STT-Rolle → GPU 0 (spaeter: Voxtral-STT-3B) capabilities: [gpu] environment: - RVS_HOST=${RVS_HOST} @@ -108,7 +108,7 @@ services: reservations: devices: - driver: nvidia - count: 1 + device_ids: ["0"] # LLM → GPU 0 (teilt sich mit dem kleinen STT) capabilities: [gpu] volumes: - ./models:/models # HF-Download-Cache (persistent) @@ -137,3 +137,56 @@ services: # Erster Load eines Modells kann ein GGUF ziehen (mehrere GB) — grosszuegig. - LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600} restart: unless-stopped + + # ─── Voxtral STT (GPU, Realtime) — PROFIL "voxtral" ─────────── + # Ersetzt whisper als STT sobald die 24-GB-Karte da ist. Startet NUR mit + # docker compose --profile voxtral up -d + # (sonst kollidiert es mit whisper — beide wuerden stt_* beantworten). + # + # ⚠️ VRAM: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB (BF16, laut vLLM- + # Rezept keine Quant). Laeuft NICHT auf der 3060 (12 GB) — erst 24-GB-Karte. + # ⚠️ vLLM: Version >=0.20.0 noetig. Entrypoint/Serve-Form beim ersten Lauf + # gegen das offizielle Rezept pruefen (siehe voxtral/README.md). + voxtral-vllm: + image: vllm/vllm-openai:latest + container_name: aria-voxtral-vllm + profiles: ["voxtral"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + volumes: + - ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5 + environment: + - VLLM_DISABLE_COMPILE_CACHE=1 + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + # Serve-Command aus dem vLLM-Rezept (Voxtral-Mini-4B-Realtime-2602). + command: + - --model + - mistralai/Voxtral-Mini-4B-Realtime-2602 + - --tokenizer-mode + - mistral + - --compilation_config + - '{"cudagraph_mode":"PIECEWISE"}' + restart: unless-stopped + + # ─── Voxtral-Bridge — RVS <-> vLLM-Realtime-WS (CPU-Glue) ───── + voxtral-bridge: + build: ./voxtral + container_name: aria-voxtral-bridge + profiles: ["voxtral"] + depends_on: + - voxtral-vllm + environment: + - RVS_HOST=${RVS_HOST} + - RVS_PORT=${RVS_PORT:-443} + - RVS_TLS=${RVS_TLS:-true} + - RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true} + - RVS_TOKEN=${RVS_TOKEN} + - VOXTRAL_VLLM_URL=ws://voxtral-vllm:8000/v1/realtime + - VOXTRAL_MODEL=mistralai/Voxtral-Mini-4B-Realtime-2602 + - VOXTRAL_LANGUAGE=${WHISPER_LANGUAGE:-de} + restart: unless-stopped diff --git a/xtts/voxtral/Dockerfile b/xtts/voxtral/Dockerfile new file mode 100644 index 0000000..948ca09 --- /dev/null +++ b/xtts/voxtral/Dockerfile @@ -0,0 +1,14 @@ +# Voxtral-BRIDGE (nicht das Modell!) — leichte CPU-Glue zwischen RVS und dem +# vLLM-Realtime-Server. Das eigentliche Voxtral-Modell laeuft im Container +# `voxtral-vllm` (GPU, vllm/vllm-openai). Deshalb hier kein CUDA-Base noetig. +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY bridge.py ./ + +CMD ["python3", "bridge.py"] diff --git a/xtts/voxtral/README.md b/xtts/voxtral/README.md new file mode 100644 index 0000000..ba9d66f --- /dev/null +++ b/xtts/voxtral/README.md @@ -0,0 +1,70 @@ +# Voxtral-STT-Satellit (M0.3) + +Streaming-STT via **Voxtral-Mini-4B-Realtime-2602** (Mistral, Apache 2.0) auf +vLLM. Ersetzt whisper als STT — genauer (~5,9 % WER vs 7,4 % FLEURS) und mit +echtem Realtime-Streaming. Deutsch ist in den 13 Sprachen abgedeckt. + +Zwei Container: +- **`voxtral-vllm`** — das Modell auf vLLM (GPU). Exponiert die Realtime-WS-API. +- **`voxtral-bridge`** — CPU-Glue: RVS ⇄ vLLM-Realtime-WS. Macht das Endpointing + selbst (adaptiver Rausch-Boden-VAD, identisch zur whisper-Bridge / M0.1). + +## ⚠️ Hardware-Realität — läuft NICHT auf der 3060 + +Das Realtime-Modell braucht laut [vLLM-Rezept](https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602) +**≥ 16 GB VRAM (BF16, keine Quant)**. Die RTX 3060 hat 12 GB → passt nicht. + +- **Interim-gpubox (nur 3060):** whisper (mit M0.1-Fix) + F5-TTS bleiben aktiv. + Voxtral NICHT starten. +- **Ab der 24-GB-Karte:** Voxtral-Profil hochziehen, whisper wird Fallback. + +Deshalb liegen beide Services hinter dem Compose-**Profil `voxtral`** und starten +NUR explizit — sonst würden whisper *und* voxtral dieselben `stt_*`-Messages +beantworten (Kollision). + +## Starten (erst wenn die 24-GB-Karte drin ist) + +```bash +cd xtts +docker compose --profile voxtral up -d --build +docker logs -f aria-voxtral-vllm # laedt Modell (mehrere GB, dauert) +docker logs -f aria-voxtral-bridge # "RVS verbunden" + service_status ready +``` +Whisper vorher stoppen, damit nur eine STT-Engine antwortet: +```bash +docker compose stop whisper-bridge +``` + +## ⚠️ Auf echter Hardware verifizieren (blind gebaut, kein Test hier) + +1. **vLLM-Version ≥ 0.20.0** und die **Serve-Form**. Das Rezept nutzt + `vllm serve …`. Falls das `vllm/vllm-openai`-Image einen anderen + Entrypoint hat, das `command:` in `docker-compose.yml` anpassen + (Rezept-Command steht dort als Kommentar). +2. **Realtime-WS-Frames.** Die exakten Event-Namen sind in `bridge.py` ganz oben + als Konstanten gebündelt (`VLLM_SEND_APPEND`, `VLLM_DELTA_SUFFIXES`, …), + modelliert nach dem OpenAI-Realtime-Schema. Gegen das offizielle + **vLLM-Realtime-Client-Beispiel** prüfen und dort anpassen — nur an dieser + einen Stelle. Das Response-Handling ist bereits defensiv (mehrere Feldnamen). +3. **Endpoint-URL/Port.** Default `ws://voxtral-vllm:8000/v1/realtime` — prüfen ob + vLLM auf 8000 lauscht und `/v1/realtime` registriert (Log-Zeile + `Route: /v1/realtime`). +4. **Endpointing.** Voxtral liefert keine eigene VAD → unser adaptiver Endpointer + entscheidet (akustisch + semantisch am Delta-Wachstum). `endpointMs` kommt wie + bei whisper aus der App; `voiceFactor` per Session tunebar. + +## Protokoll (RVS, identisch zu whisper — drop-in) + +Rein: `stt_stream_start`, `stt_audio_chunk` (16 kHz mono s16le, base64), `stt_stream_end`. +Raus: `stt_partial`, `stt_endpoint` (das Event, auf das aria-bridge horcht), `stt_stream_done`. + +## TTS-Hinweis + +Voxtral-**TTS** (Voice-Cloning) ist hier NICHT enthalten — braucht ebenfalls +16 GB VRAM und ist ein eigener Bau. Bis zur 24-GB-Karte bleibt **F5-TTS** aktiv. +Danach: eigener `voxtral-tts`-Satellit (separates Ticket). + +## Quellen +- Rezept: https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602 +- vLLM Speech-to-Text: https://docs.vllm.ai/en/latest/serving/online_serving/speech_to_text/ +- Modell: https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602 diff --git a/xtts/voxtral/bridge.py b/xtts/voxtral/bridge.py new file mode 100644 index 0000000..3bd17bc --- /dev/null +++ b/xtts/voxtral/bridge.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +""" +ARIA Voxtral Bridge — Streaming-STT via Voxtral-Mini-4B-Realtime-2602 (vLLM). + +Zwilling der whisper-Bridge, aber das Transkribieren macht NICHT faster-whisper +im selben Prozess, sondern der separate vLLM-Realtime-Server (Container +`voxtral-vllm`) ueber dessen WebSocket-API `/v1/realtime`. Diese Bridge ist +reine Glue: + + App ──(RVS: stt_stream_start / stt_audio_chunk / stt_stream_end)──▶ diese Bridge + diese Bridge ──(WS /v1/realtime: PCM16-b64 append)──▶ voxtral-vllm + voxtral-vllm ──(transcription.delta / transcription.done)──▶ diese Bridge + diese Bridge ──(RVS: stt_partial / stt_endpoint / stt_stream_done)──▶ App/aria-bridge + +Das RVS-Wire-Protokoll ist IDENTISCH zur whisper-Bridge (drop-in). Das +Endpointing (wann hat der User aufgehoert zu sprechen) macht diese Bridge +selbst — mit demselben ADAPTIVEN Rausch-Boden-Endpointer wie whisper (Voxtral +Realtime liefert laut vLLM-Doku keine eigene VAD/„speaker done"-Semantik, nur +transcription.delta/.done). Die akustische Energie messen wir auf unserer +eigenen PCM-Kopie, die semantische Stagnation am Delta-Textwachstum. + +⚠️ HARDWARE: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB VRAM (BF16). Auf der + RTX 3060 (12 GB) laeuft es NICHT — erst auf der 24-GB-Karte. Bis dahin + bleibt die whisper-Bridge aktiv (Profil-gesteuert im docker-compose). + +⚠️ VERIFY-ON-FIRST-RUN: Die exakten vLLM-Realtime-FRAME-Namen (Audio-Append, + Delta/Done-Event-Typen) sind unten als Konstanten gebuendelt und nach dem + OpenAI-Realtime-Schema modelliert. Gegen das offizielle vLLM-Realtime- + Client-Beispiel pruefen und ggf. anpassen — sie stehen bewusst an EINER + Stelle. Response-Handling ist defensiv (mehrere moegliche Feldnamen). + +Env: + RVS_HOST, RVS_PORT, RVS_TLS, RVS_TLS_FALLBACK, RVS_TOKEN + VOXTRAL_VLLM_URL Default: ws://voxtral-vllm:8000/v1/realtime + VOXTRAL_MODEL Default: mistralai/Voxtral-Mini-4B-Realtime-2602 + VOXTRAL_LANGUAGE Default: de +""" +import asyncio +import base64 +import json +import logging +import os +import time +from dataclasses import dataclass, field +from typing import Optional + +import numpy as np +import websockets + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger("voxtral-bridge") + +RVS_HOST = os.getenv("RVS_HOST", "").strip() +RVS_PORT = int(os.getenv("RVS_PORT", "443")) +RVS_TLS = os.getenv("RVS_TLS", "true").lower() == "true" +RVS_TLS_FALLBACK = os.getenv("RVS_TLS_FALLBACK", "true").lower() == "true" +RVS_TOKEN = os.getenv("RVS_TOKEN", "").strip() + +VOXTRAL_VLLM_URL = os.getenv("VOXTRAL_VLLM_URL", "ws://voxtral-vllm:8000/v1/realtime") +VOXTRAL_MODEL = os.getenv("VOXTRAL_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602") +VOXTRAL_LANGUAGE = os.getenv("VOXTRAL_LANGUAGE", "de") + +# ── vLLM-Realtime-Frames — HIER anpassen falls das Client-Beispiel abweicht ── +# Senderichtung (wir → vLLM): PCM16-16kHz-mono base64 anhaengen + committen. +VLLM_SEND_APPEND = "input_audio_buffer.append" # {"type":..., "audio": ""} +VLLM_SEND_COMMIT = "input_audio_buffer.commit" # Buffer abschliessen +VLLM_AUDIO_FIELD = "audio" +# Empfangsrichtung (vLLM → wir): inkrementeller Text + final. Defensiv geprueft. +VLLM_DELTA_SUFFIXES = ("transcription.delta",) # msg["type"] endet hierauf +VLLM_DONE_SUFFIXES = ("transcription.done", "transcription.completed") +VLLM_DELTA_FIELDS = ("delta", "text", "transcription") # eins davon traegt den Text + +# ── Streaming-/Endpointing-Parameter (analog whisper-Bridge) ── +STREAM_DEFAULT_ENDPOINT_MS = 2400 +STREAM_DEFAULT_HARD_CAP_MS = 60000 +STREAM_MIN_AUDIO_MS = 600 +STREAM_SESSION_TTL_S = 120 +STREAM_ENERGY_WINDOW_MS = 300 +STREAM_SEMANTIC_BACKUP_FACTOR = 2.0 +# Adaptiver Voice-Schwellwert (siehe whisper-Bridge M0.1): Grenze relativ zum +# gemessenen Rausch-Boden statt fix — schneidet leises Sprechen nicht ab. +STREAM_VOICE_FACTOR = 2.5 +STREAM_VOICE_RMS_MIN = 0.005 +STREAM_VOICE_RMS_MAX = 0.020 + + +def pcm_s16le_to_float32(data: bytes) -> np.ndarray: + if not data: + return np.zeros(0, dtype=np.float32) + arr = np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0 + return arr + + +async def _send(ws, mtype: str, payload: dict) -> None: + try: + await ws.send(json.dumps({ + "type": mtype, + "payload": payload, + "timestamp": int(time.time() * 1000), + })) + except Exception as e: + logger.warning("RVS-Send fehlgeschlagen (%s): %s", mtype, e) + + +@dataclass +class StreamSession: + request_id: str + audio_request_id: str + language: str + endpoint_ms: int + hard_cap_ms: int + voice: str = "" + speed: float = 1.0 + interrupted: bool = False + location: Optional[dict] = None + sample_rate: int = 16000 + voice_factor: float = STREAM_VOICE_FACTOR + voice_rms_min: float = STREAM_VOICE_RMS_MIN + voice_rms_max: float = STREAM_VOICE_RMS_MAX + pcm_buffer: bytearray = field(default_factory=bytearray) + started_at: float = field(default_factory=time.time) + last_chunk_at: float = field(default_factory=time.time) + last_partial: str = "" + last_growth_at: float = 0.0 + last_voice_at: float = 0.0 + noise_floor: float = 0.0 + closed: bool = False + endpoint_sent: bool = False + # vLLM-Realtime-Session + vllm_ws: object = None + vllm_reader: object = None + + +class SessionManager: + def __init__(self) -> None: + self._sessions: dict[str, StreamSession] = {} + self._ws = None # RVS + + def attach_ws(self, ws) -> None: + self._ws = ws + + async def start_session(self, payload: dict) -> Optional[StreamSession]: + request_id = (payload.get("requestId") or "").strip() + if not request_id: + logger.warning("stt_stream_start ohne requestId — ignoriert") + return None + try: + endpoint_ms = int(payload.get("endpointMs") or STREAM_DEFAULT_ENDPOINT_MS) + except (TypeError, ValueError): + endpoint_ms = STREAM_DEFAULT_ENDPOINT_MS + try: + hard_cap_ms = int(payload.get("hardCapMs") or STREAM_DEFAULT_HARD_CAP_MS) + except (TypeError, ValueError): + hard_cap_ms = STREAM_DEFAULT_HARD_CAP_MS + try: + voice_factor = float(payload.get("voiceFactor") or STREAM_VOICE_FACTOR) + except (TypeError, ValueError): + voice_factor = STREAM_VOICE_FACTOR + sess = StreamSession( + request_id=request_id, + audio_request_id=payload.get("audioRequestId", "") or "", + language=payload.get("language") or VOXTRAL_LANGUAGE, + endpoint_ms=endpoint_ms, + hard_cap_ms=hard_cap_ms, + voice=payload.get("voice", "") or "", + speed=float(payload.get("speed") or 1.0), + voice_factor=voice_factor, + interrupted=bool(payload.get("interrupted", False)), + location=payload.get("location") or None, + sample_rate=int(payload.get("sampleRate") or 16000), + ) + # vLLM-Realtime-Session oeffnen + Reader starten. + try: + sess.vllm_ws = await websockets.connect(VOXTRAL_VLLM_URL, max_size=8 * 1024 * 1024) + await self._vllm_configure(sess) + sess.vllm_reader = asyncio.create_task(self._vllm_read_loop(sess)) + except Exception as e: + logger.exception("Stream %s: vLLM-Realtime-Connect fehlgeschlagen: %s", + request_id[:8], e) + # ohne Backend keine Transkription → sofort leeres Endpoint melden + self._sessions[request_id] = sess + await self._finalize(sess, reason="vllm_unavailable") + return None + self._sessions[request_id] = sess + logger.info("Voxtral-Session offen: id=%s lang=%s endpointMs=%d", + request_id[:8], sess.language, sess.endpoint_ms) + return sess + + async def _vllm_configure(self, sess: StreamSession) -> None: + """Optionale Session-Konfig an vLLM (Modell/Sprache/temperature=0). + VERIFY: exaktes session.update-Schema gegen vLLM-Realtime-Beispiel. + Best-effort — Fehler hier sind nicht fatal.""" + try: + await sess.vllm_ws.send(json.dumps({ + "type": "session.update", + "session": { + "model": VOXTRAL_MODEL, + "language": sess.language, + "temperature": 0.0, + "input_audio_format": "pcm16", + }, + })) + except Exception: + pass + + async def _vllm_read_loop(self, sess: StreamSession) -> None: + """Liest transcription.delta/.done vom vLLM-Realtime-Server.""" + ws = sess.vllm_ws + try: + async for raw in ws: + try: + msg = json.loads(raw) + except Exception: + continue + mtype = str(msg.get("type", "")) + if any(mtype.endswith(s) for s in VLLM_DELTA_SUFFIXES): + text = self._extract_text(msg) + if text: + await self._on_delta(sess, text) + elif any(mtype.endswith(s) for s in VLLM_DONE_SUFFIXES): + text = self._extract_text(msg) + if text: + await self._on_delta(sess, text, final=True) + except Exception: + logger.debug("Stream %s: vLLM-Reader beendet", sess.request_id[:8]) + + @staticmethod + def _extract_text(msg: dict) -> str: + for f in VLLM_DELTA_FIELDS: + v = msg.get(f) + if isinstance(v, str) and v: + return v + return "" + + async def _on_delta(self, sess: StreamSession, text: str, final: bool = False) -> None: + """Neuer/finaler Transkript-Text vom vLLM. delta = inkrementell; wir + haengen an, wenn er den bisherigen Partial verlaengert, sonst ersetzen + wir (Voxtral kann korrigieren).""" + if final or text.startswith(sess.last_partial): + new_full = text if final else text + else: + new_full = (sess.last_partial + text).strip() + new_full = new_full.strip() + if new_full and new_full != sess.last_partial: + sess.last_partial = new_full + sess.last_growth_at = time.time() + if self._ws is not None: + await _send(self._ws, "stt_partial", { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": new_full, + }) + + def feed_chunk(self, payload: dict) -> bool: + request_id = payload.get("requestId", "") + sess = self._sessions.get(request_id) + if sess is None or sess.closed: + return False + pcm_b64 = payload.get("pcm", "") + if not pcm_b64: + return True + try: + pcm = base64.b64decode(pcm_b64) + except Exception: + return True + sess.pcm_buffer.extend(pcm) + sess.last_chunk_at = time.time() + # An vLLM weiterreichen (fire-and-forget). + if sess.vllm_ws is not None: + asyncio.create_task(self._vllm_append(sess, pcm_b64)) + return True + + async def _vllm_append(self, sess: StreamSession, pcm_b64: str) -> None: + try: + await sess.vllm_ws.send(json.dumps({ + "type": VLLM_SEND_APPEND, + VLLM_AUDIO_FIELD: pcm_b64, + })) + except Exception: + pass + + def end_session(self, request_id: str) -> None: + sess = self._sessions.get(request_id) + if sess is not None: + sess.closed = True + + def drop(self, request_id: str) -> None: + sess = self._sessions.pop(request_id, None) + if sess is not None: + self._teardown_vllm(sess) + + def _teardown_vllm(self, sess: StreamSession) -> None: + try: + if sess.vllm_reader is not None: + sess.vllm_reader.cancel() + except Exception: + pass + if sess.vllm_ws is not None: + asyncio.create_task(self._close_ws(sess.vllm_ws)) + sess.vllm_ws = None + + @staticmethod + async def _close_ws(ws) -> None: + try: + await ws.close() + except Exception: + pass + + # ── Endpointer (adaptiv, wie whisper-Bridge M0.1) ── + def _buffer_duration_ms(self, sess: StreamSession) -> float: + samples = len(sess.pcm_buffer) // 2 + return (samples / sess.sample_rate) * 1000.0 if samples else 0.0 + + def _tail_rms(self, sess: StreamSession) -> float: + win_bytes = int(sess.sample_rate * STREAM_ENERGY_WINDOW_MS / 1000) * 2 + if win_bytes <= 0: + return 0.0 + tail = sess.pcm_buffer[-win_bytes:] + if len(tail) < 2: + return 0.0 + arr = pcm_s16le_to_float32(bytes(tail)) + if arr.size == 0: + return 0.0 + return float(np.sqrt(np.mean(arr * arr))) + + def _voice_threshold(self, sess: StreamSession) -> float: + nf = sess.noise_floor + if nf <= 0.0: + return sess.voice_rms_min + return min(max(nf * sess.voice_factor, sess.voice_rms_min), sess.voice_rms_max) + + def _update_noise_floor(self, sess: StreamSession, rms: float) -> None: + nf = sess.noise_floor + if nf <= 0.0: + sess.noise_floor = rms + elif rms < nf: + sess.noise_floor = 0.90 * nf + 0.10 * rms + else: + sess.noise_floor = 0.98 * nf + 0.02 * rms + + async def run_endpointer(self) -> None: + logger.info("Voxtral-Endpointer gestartet (adaptiver VAD)") + while True: + await asyncio.sleep(0.2) + now = time.time() + for sid, sess in list(self._sessions.items()): + try: + await self._tick(sess, now) + except Exception: + logger.exception("Endpointer-Tick crashed (session=%s)", sid[:8]) + for sid, sess in list(self._sessions.items()): + if now - sess.last_chunk_at > STREAM_SESSION_TTL_S: + logger.info("Stream %s: TTL — drop", sid[:8]) + self.drop(sid) + + async def _tick(self, sess: StreamSession, now: float) -> None: + if sess.endpoint_sent: + return + elapsed_ms = (now - sess.started_at) * 1000.0 + if elapsed_ms > sess.hard_cap_ms and not sess.closed: + await self._finalize(sess, reason="hardcap") + return + if sess.closed: + await self._finalize(sess, reason="stream_end") + return + if self._buffer_duration_ms(sess) < STREAM_MIN_AUDIO_MS: + return + # adaptive akustische Sprach-Aktivitaet + rms = self._tail_rms(sess) + if rms >= self._voice_threshold(sess): + sess.last_voice_at = now + else: + self._update_noise_floor(sess, rms) + # Endpoint: akustisch (Primaer) oder semantisch (Backstop), sobald Text da + if sess.last_growth_at > 0.0: + acoustic_silence_ms = (now - sess.last_voice_at) * 1000.0 if sess.last_voice_at > 0 else 0.0 + semantic_silence_ms = (now - sess.last_growth_at) * 1000.0 + acoustic_done = sess.last_voice_at > 0 and acoustic_silence_ms >= sess.endpoint_ms + semantic_done = semantic_silence_ms >= sess.endpoint_ms * STREAM_SEMANTIC_BACKUP_FACTOR + if acoustic_done or semantic_done: + await self._finalize(sess, reason="endpoint" if acoustic_done else "endpoint_semantic") + + async def _finalize(self, sess: StreamSession, reason: str) -> None: + if sess.endpoint_sent: + return + sess.endpoint_sent = True + # vLLM ggf. committen, damit ein letztes transcription.done kommt. + if sess.vllm_ws is not None: + try: + await sess.vllm_ws.send(json.dumps({"type": VLLM_SEND_COMMIT})) + await asyncio.sleep(0.15) # kurz auf finalen Delta warten + except Exception: + pass + final_text = sess.last_partial.strip() + duration_s = self._buffer_duration_ms(sess) / 1000.0 + logger.info("Stream %s: FINAL (reason=%s, %.1fs): %r", + sess.request_id[:8], reason, duration_s, final_text[:120]) + if self._ws is not None: + endpoint_payload = { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": final_text, + "reason": reason, + "durationS": duration_s, + "sttMs": 0, + "voice": sess.voice, + "speed": sess.speed, + "interrupted": sess.interrupted, + } + if sess.location: + endpoint_payload["location"] = sess.location + await _send(self._ws, "stt_endpoint", endpoint_payload) + await _send(self._ws, "stt_stream_done", { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": final_text, + "reason": reason, + }) + self.drop(sess.request_id) + + +async def _broadcast_status(ws, state: str, **extra) -> None: + payload = {"service": "voxtral", "state": state} + payload.update(extra) + await _send(ws, "service_status", payload) + + +async def run_loop(sessions: SessionManager) -> None: + use_tls = RVS_TLS + retry_s = 2 + tls_fallback_tried = False + while True: + scheme = "wss" if use_tls else "ws" + url = f"{scheme}://{RVS_HOST}:{RVS_PORT}/ws?token={RVS_TOKEN}" + masked = url.replace(RVS_TOKEN, "***") if RVS_TOKEN else url + try: + logger.info("Verbinde zu RVS: %s", masked) + async with websockets.connect(url, ping_interval=20, ping_timeout=10, + max_size=50 * 1024 * 1024) as ws: + logger.info("RVS verbunden") + retry_s = 2 + tls_fallback_tried = False + sessions.attach_ws(ws) + await _broadcast_status(ws, "ready", model=VOXTRAL_MODEL) + await _send(ws, "config_request", {"service": "voxtral"}) + + async for raw in ws: + try: + msg = json.loads(raw) + except Exception: + continue + mtype = msg.get("type", "") + payload = msg.get("payload", {}) or {} + if mtype == "stt_stream_start": + asyncio.create_task(sessions.start_session(payload)) + elif mtype == "stt_audio_chunk": + sessions.feed_chunk(payload) + elif mtype == "stt_stream_end": + sessions.end_session(payload.get("requestId", "")) + # stt_request (Legacy One-Shot) macht Voxtral hier NICHT — + # dafuer bleibt die whisper-Bridge (Fallback). + except Exception as e: + logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s) + if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried: + use_tls = False + tls_fallback_tried = True + logger.info("TLS-Fallback: versuche ws:// (kein TLS)") + continue + await asyncio.sleep(retry_s) + retry_s = min(retry_s * 2, 30) + use_tls = RVS_TLS # fuer den naechsten Zyklus zuruecksetzen + + +async def main() -> None: + if not RVS_HOST or not RVS_TOKEN: + logger.error("RVS_HOST/RVS_TOKEN fehlen — .env pruefen. Abbruch.") + return + sessions = SessionManager() + logger.info("Voxtral-Bridge startet — vLLM=%s Modell=%s", VOXTRAL_VLLM_URL, VOXTRAL_MODEL) + await asyncio.gather( + run_loop(sessions), + sessions.run_endpointer(), + ) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass diff --git a/xtts/voxtral/requirements.txt b/xtts/voxtral/requirements.txt new file mode 100644 index 0000000..31f911d --- /dev/null +++ b/xtts/voxtral/requirements.txt @@ -0,0 +1,5 @@ +# Voxtral-Bridge ist reine CPU-Glue (RVS-WS <-> vLLM-Realtime-WS). Das Modell +# selbst laeuft im separaten voxtral-vllm-Container (GPU). Deshalb hier KEIN +# torch/vllm — nur der WebSocket-Client + numpy fuer die RMS-Energiemessung. +websockets>=12.0 +numpy>=1.24