diff --git a/xtts/docker-compose.yml b/xtts/docker-compose.yml index 710ca14..db4950f 100644 --- a/xtts/docker-compose.yml +++ b/xtts/docker-compose.yml @@ -138,55 +138,32 @@ services: - LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600} restart: unless-stopped - # ─── Voxtral STT (GPU, Realtime) — PROFIL "voxtral" ─────────── - # Ersetzt whisper als STT sobald die 24-GB-Karte da ist. Startet NUR mit - # docker compose --profile voxtral up -d - # (sonst kollidiert es mit whisper — beide wuerden stt_* beantworten). - # - # ⚠️ VRAM: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB (BF16, laut vLLM- - # Rezept keine Quant). Laeuft NICHT auf der 3060 (12 GB) — erst 24-GB-Karte. - # ⚠️ vLLM: Version >=0.20.0 noetig. Entrypoint/Serve-Form beim ersten Lauf - # gegen das offizielle Rezept pruefen (siehe voxtral/README.md). - voxtral-vllm: - image: vllm/vllm-openai:latest - container_name: aria-voxtral-vllm + # ─── Voxtral STT-3B (Transformers, GPU) — PROFIL "voxtral" ───── + # Ersetzt whisper als STT. Laeuft auf Treiber 550/CUDA 12.4 (torch cu124, KEIN + # Treiber-Upgrade noetig). Modell Voxtral-Mini-3B-2507 (~9 GB bf16) → GPU 1. + # Startet NUR mit: docker compose --profile voxtral up -d --build + # Vorher whisper stoppen, sonst beantworten beide stt_* (Kollision): + # docker compose stop whisper-bridge + voxtral-bridge: + build: ./voxtral + container_name: aria-voxtral-bridge profiles: ["voxtral"] deploy: resources: reservations: devices: - driver: nvidia - count: 1 + device_ids: ["1"] # 12-GB-Karte (STT-3B ~9 GB); GPU 0 (8 GB) bleibt fuer F5/LLM capabilities: [gpu] volumes: - ./hf-cache:/root/.cache/huggingface # gleicher Modell-Cache wie whisper/f5 - environment: - - VLLM_DISABLE_COMPILE_CACHE=1 - - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} - # Serve-Command aus dem vLLM-Rezept (Voxtral-Mini-4B-Realtime-2602). - command: - - --model - - mistralai/Voxtral-Mini-4B-Realtime-2602 - - --tokenizer-mode - - mistral - - --compilation_config - - '{"cudagraph_mode":"PIECEWISE"}' - restart: unless-stopped - - # ─── Voxtral-Bridge — RVS <-> vLLM-Realtime-WS (CPU-Glue) ───── - voxtral-bridge: - build: ./voxtral - container_name: aria-voxtral-bridge - profiles: ["voxtral"] - depends_on: - - voxtral-vllm environment: - RVS_HOST=${RVS_HOST} - RVS_PORT=${RVS_PORT:-443} - RVS_TLS=${RVS_TLS:-true} - RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true} - RVS_TOKEN=${RVS_TOKEN} - - VOXTRAL_VLLM_URL=ws://voxtral-vllm:8000/v1/realtime - - VOXTRAL_MODEL=mistralai/Voxtral-Mini-4B-Realtime-2602 + - VOXTRAL_MODEL=mistralai/Voxtral-Mini-3B-2507 - VOXTRAL_LANGUAGE=${WHISPER_LANGUAGE:-de} + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} # falls das Modell HF-gated ist restart: unless-stopped diff --git a/xtts/voxtral/Dockerfile b/xtts/voxtral/Dockerfile index 948ca09..7894947 100644 --- a/xtts/voxtral/Dockerfile +++ b/xtts/voxtral/Dockerfile @@ -1,14 +1,26 @@ -# Voxtral-BRIDGE (nicht das Modell!) — leichte CPU-Glue zwischen RVS und dem -# vLLM-Realtime-Server. Das eigentliche Voxtral-Modell laeuft im Container -# `voxtral-vllm` (GPU, vllm/vllm-openai). Deshalb hier kein CUDA-Base noetig. -FROM python:3.11-slim +# Voxtral-STT-3B Bridge (Transformers). Laeuft auf Treiber 550/CUDA 12.4 via +# torch cu124 — KEIN Treiber-Upgrade noetig (gleicher Trick wie f5tts). +# Modell: Voxtral-Mini-3B-2507 (~9 GB in bf16) → passt auf die 12-GB-Karte (GPU 1). +FROM nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04 +ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip ffmpeg git \ + && rm -rf /var/lib/apt/lists/* -COPY bridge.py ./ +# torch FEST auf cu124 (Treiber 550 = CUDA 12.4). 2.6.0 ist der neueste cu124-Build; +# torch 2.7+ gibt es nur fuer cu126+ und braeuchte einen neueren Treiber. +RUN pip3 install --no-cache-dir torch==2.6.0 torchaudio==2.6.0 \ + --index-url https://download.pytorch.org/whl/cu124 + +COPY requirements.txt . +# Constraint haelt transformers/mistral-common davon ab, torch wieder hochzuziehen. +RUN printf 'torch==2.6.0\ntorchaudio==2.6.0\n' > /tmp/torch-constraint.txt && \ + pip3 install --no-cache-dir -c /tmp/torch-constraint.txt -r requirements.txt + +COPY bridge.py . CMD ["python3", "bridge.py"] diff --git a/xtts/voxtral/README.md b/xtts/voxtral/README.md index ba9d66f..4cededd 100644 --- a/xtts/voxtral/README.md +++ b/xtts/voxtral/README.md @@ -1,70 +1,47 @@ -# Voxtral-STT-Satellit (M0.3) +# Voxtral-STT-3B-Satellit (Transformers) -Streaming-STT via **Voxtral-Mini-4B-Realtime-2602** (Mistral, Apache 2.0) auf -vLLM. Ersetzt whisper als STT — genauer (~5,9 % WER vs 7,4 % FLEURS) und mit -echtem Realtime-Streaming. Deutsch ist in den 13 Sprachen abgedeckt. +Streaming-STT via **Voxtral-Mini-3B-2507** (Mistral, Apache 2.0) über +🤗 Transformers. Ersetzt whisper als STT — genauer, und **ohne Treiber-Upgrade**: +läuft auf dem Trixie-Standardtreiber (550/CUDA 12.4) via **torch 2.6.0+cu124** +(derselbe Trick wie bei f5tts). -Zwei Container: -- **`voxtral-vllm`** — das Modell auf vLLM (GPU). Exponiert die Realtime-WS-API. -- **`voxtral-bridge`** — CPU-Glue: RVS ⇄ vLLM-Realtime-WS. Macht das Endpointing - selbst (adaptiver Rausch-Boden-VAD, identisch zur whisper-Bridge / M0.1). +- Modell ~9 GB (bf16) → **GPU 1** (die 12-GB-Karte; per Compose gepinnt). +- **F5-TTS + LLM** bleiben auf GPU 0 (8 GB). +- Arbeitsweise = chunked wie whisper: live PCM → alle ~1 s transkribieren + (Partials) → **adaptiver Endpointer** (Rausch-Boden-VAD + semantische + Stagnation, aus M0.1) feuert `stt_endpoint`. RVS-Protokoll identisch → drop-in. -## ⚠️ Hardware-Realität — läuft NICHT auf der 3060 - -Das Realtime-Modell braucht laut [vLLM-Rezept](https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602) -**≥ 16 GB VRAM (BF16, keine Quant)**. Die RTX 3060 hat 12 GB → passt nicht. - -- **Interim-gpubox (nur 3060):** whisper (mit M0.1-Fix) + F5-TTS bleiben aktiv. - Voxtral NICHT starten. -- **Ab der 24-GB-Karte:** Voxtral-Profil hochziehen, whisper wird Fallback. - -Deshalb liegen beide Services hinter dem Compose-**Profil `voxtral`** und starten -NUR explizit — sonst würden whisper *und* voxtral dieselben `stt_*`-Messages -beantworten (Kollision). - -## Starten (erst wenn die 24-GB-Karte drin ist) +## Starten (Profil `voxtral`) ```bash cd xtts +docker compose stop whisper-bridge # sonst beantworten beide stt_* docker compose --profile voxtral up -d --build -docker logs -f aria-voxtral-vllm # laedt Modell (mehrere GB, dauert) -docker logs -f aria-voxtral-bridge # "RVS verbunden" + service_status ready -``` -Whisper vorher stoppen, damit nur eine STT-Engine antwortet: -```bash -docker compose stop whisper-bridge +docker logs -f aria-voxtral-bridge # Modell laedt (mehrere GB), dann "RVS verbunden" ``` +Zurück zu whisper: `docker compose --profile voxtral down && docker compose up -d whisper-bridge`. -## ⚠️ Auf echter Hardware verifizieren (blind gebaut, kein Test hier) +## ⚠️ Auf echter Hardware verifizieren (blind gebaut) -1. **vLLM-Version ≥ 0.20.0** und die **Serve-Form**. Das Rezept nutzt - `vllm serve …`. Falls das `vllm/vllm-openai`-Image einen anderen - Entrypoint hat, das `command:` in `docker-compose.yml` anpassen - (Rezept-Command steht dort als Kommentar). -2. **Realtime-WS-Frames.** Die exakten Event-Namen sind in `bridge.py` ganz oben - als Konstanten gebündelt (`VLLM_SEND_APPEND`, `VLLM_DELTA_SUFFIXES`, …), - modelliert nach dem OpenAI-Realtime-Schema. Gegen das offizielle - **vLLM-Realtime-Client-Beispiel** prüfen und dort anpassen — nur an dieser - einen Stelle. Das Response-Handling ist bereits defensiv (mehrere Feldnamen). -3. **Endpoint-URL/Port.** Default `ws://voxtral-vllm:8000/v1/realtime` — prüfen ob - vLLM auf 8000 lauscht und `/v1/realtime` registriert (Log-Zeile - `Route: /v1/realtime`). -4. **Endpointing.** Voxtral liefert keine eigene VAD → unser adaptiver Endpointer - entscheidet (akustisch + semantisch am Delta-Wachstum). `endpointMs` kommt wie - bei whisper aus der App; `voiceFactor` per Session tunebar. +1. **Transformers-API.** Die exakte Voxtral-Transkriptions-API ist in `bridge.py` + in **einer** Methode gekapselt (`VoxtralRunner._transcribe_blocking`), modelliert + nach der HF-Modelcard (`apply_transcription_request` → `generate` → `batch_decode`). + Beim ersten Lauf gegen die Modelcard prüfen und dort anpassen. +2. **HF-Gating.** Ist `Voxtral-Mini-3B-2507` gated, `HF_TOKEN` in `xtts/.env` setzen + (wird als `HUGGING_FACE_HUB_TOKEN` durchgereicht). +3. **VRAM/Tempo.** 3B in bf16 ~9 GB auf der 12-GB-Karte — Rest fürs KV-Cache. Ist die + Partial-Transkription (alle 1 s) zu schwer, `STREAM_TRANSCRIBE_INTERVAL_MS` hochsetzen. +4. **torch-Konflikt.** Falls `transformers`/`mistral-common` beim Build torch>2.6 + erzwingen, meldet der Constraint einen Konflikt → dann brauchen wir doch das + Treiber-Upgrade (`bootstrap.sh --upgrade-driver` via NVIDIA-CUDA-Repo) + cu126-torch. ## Protokoll (RVS, identisch zu whisper — drop-in) - Rein: `stt_stream_start`, `stt_audio_chunk` (16 kHz mono s16le, base64), `stt_stream_end`. -Raus: `stt_partial`, `stt_endpoint` (das Event, auf das aria-bridge horcht), `stt_stream_done`. +Raus: `stt_partial`, `stt_endpoint`, `stt_stream_done`. -## TTS-Hinweis - -Voxtral-**TTS** (Voice-Cloning) ist hier NICHT enthalten — braucht ebenfalls -16 GB VRAM und ist ein eigener Bau. Bis zur 24-GB-Karte bleibt **F5-TTS** aktiv. -Danach: eigener `voxtral-tts`-Satellit (separates Ticket). +## TTS +Bleibt **F5-TTS** (klingt gut, passt auf GPU 0). Voxtral-TTS bräuchte ~24 GB — separates Thema. ## Quellen -- Rezept: https://recipes.vllm.ai/mistralai/Voxtral-Mini-4B-Realtime-2602 -- vLLM Speech-to-Text: https://docs.vllm.ai/en/latest/serving/online_serving/speech_to_text/ -- Modell: https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602 +- Modell: https://huggingface.co/mistralai/Voxtral-Mini-3B-2507 +- Transformers-Nutzung: HF-Modelcard (Voxtral) + `mistral-common` diff --git a/xtts/voxtral/bridge.py b/xtts/voxtral/bridge.py index 3bd17bc..0341970 100644 --- a/xtts/voxtral/bridge.py +++ b/xtts/voxtral/bridge.py @@ -1,39 +1,28 @@ #!/usr/bin/env python3 """ -ARIA Voxtral Bridge — Streaming-STT via Voxtral-Mini-4B-Realtime-2602 (vLLM). +ARIA Voxtral-STT-3B Bridge (Transformers) — Ersatz fuer whisper. -Zwilling der whisper-Bridge, aber das Transkribieren macht NICHT faster-whisper -im selben Prozess, sondern der separate vLLM-Realtime-Server (Container -`voxtral-vllm`) ueber dessen WebSocket-API `/v1/realtime`. Diese Bridge ist -reine Glue: +Laeuft auf Treiber 550/CUDA 12.4 via torch cu124 (kein Treiber-Upgrade noetig). +Modell: Voxtral-Mini-3B-2507 (bf16, ~9 GB) → GPU 1 (12 GB, per Compose gepinnt). - App ──(RVS: stt_stream_start / stt_audio_chunk / stt_stream_end)──▶ diese Bridge - diese Bridge ──(WS /v1/realtime: PCM16-b64 append)──▶ voxtral-vllm - voxtral-vllm ──(transcription.delta / transcription.done)──▶ diese Bridge - diese Bridge ──(RVS: stt_partial / stt_endpoint / stt_stream_done)──▶ App/aria-bridge +Arbeitsweise = Zwilling der whisper-Bridge: App schickt live PCM-Chunks; wir +transkribieren alle ~STREAM_TRANSCRIBE_INTERVAL_MS auf dem Ringbuffer (Partials) +und feuern stt_endpoint, sobald der ADAPTIVE Endpointer (Rausch-Boden-VAD + +semantische Stagnation, aus M0.1) "fertig" sagt. RVS-Wire-Protokoll identisch zu +whisper → drop-in (die App merkt nur bessere Genauigkeit). -Das RVS-Wire-Protokoll ist IDENTISCH zur whisper-Bridge (drop-in). Das -Endpointing (wann hat der User aufgehoert zu sprechen) macht diese Bridge -selbst — mit demselben ADAPTIVEN Rausch-Boden-Endpointer wie whisper (Voxtral -Realtime liefert laut vLLM-Doku keine eigene VAD/„speaker done"-Semantik, nur -transcription.delta/.done). Die akustische Energie messen wir auf unserer -eigenen PCM-Kopie, die semantische Stagnation am Delta-Textwachstum. - -⚠️ HARDWARE: Voxtral-Mini-4B-Realtime-2602 braucht >=16 GB VRAM (BF16). Auf der - RTX 3060 (12 GB) laeuft es NICHT — erst auf der 24-GB-Karte. Bis dahin - bleibt die whisper-Bridge aktiv (Profil-gesteuert im docker-compose). - -⚠️ VERIFY-ON-FIRST-RUN: Die exakten vLLM-Realtime-FRAME-Namen (Audio-Append, - Delta/Done-Event-Typen) sind unten als Konstanten gebuendelt und nach dem - OpenAI-Realtime-Schema modelliert. Gegen das offizielle vLLM-Realtime- - Client-Beispiel pruefen und ggf. anpassen — sie stehen bewusst an EINER - Stelle. Response-Handling ist defensiv (mehrere moegliche Feldnamen). +⚠️ VERIFY-ON-FIRST-RUN: Die exakte Transformers-Transkriptions-API von Voxtral + (apply_transcription_request / generate / decode) ist unten in EINER Methode + (VoxtralRunner._transcribe_blocking) gekapselt und nach dem HF-Modelcard-Muster + modelliert. Beim ersten echten Lauf gegen die Voxtral-Modelcard pruefen und dort + anpassen. Alles andere (RVS, Endpointer) ist bewaehrt. Env: RVS_HOST, RVS_PORT, RVS_TLS, RVS_TLS_FALLBACK, RVS_TOKEN - VOXTRAL_VLLM_URL Default: ws://voxtral-vllm:8000/v1/realtime - VOXTRAL_MODEL Default: mistralai/Voxtral-Mini-4B-Realtime-2602 - VOXTRAL_LANGUAGE Default: de + VOXTRAL_MODEL Default: mistralai/Voxtral-Mini-3B-2507 + VOXTRAL_LANGUAGE Default: de + VOXTRAL_DEVICE Default: cuda + STREAM_TRANSCRIBE_INTERVAL_MS Default 1000 (3B ist schwerer als whisper-small) """ import asyncio import base64 @@ -60,29 +49,18 @@ RVS_TLS = os.getenv("RVS_TLS", "true").lower() == "true" RVS_TLS_FALLBACK = os.getenv("RVS_TLS_FALLBACK", "true").lower() == "true" RVS_TOKEN = os.getenv("RVS_TOKEN", "").strip() -VOXTRAL_VLLM_URL = os.getenv("VOXTRAL_VLLM_URL", "ws://voxtral-vllm:8000/v1/realtime") -VOXTRAL_MODEL = os.getenv("VOXTRAL_MODEL", "mistralai/Voxtral-Mini-4B-Realtime-2602") +VOXTRAL_MODEL = os.getenv("VOXTRAL_MODEL", "mistralai/Voxtral-Mini-3B-2507") VOXTRAL_LANGUAGE = os.getenv("VOXTRAL_LANGUAGE", "de") +VOXTRAL_DEVICE = os.getenv("VOXTRAL_DEVICE", "cuda") -# ── vLLM-Realtime-Frames — HIER anpassen falls das Client-Beispiel abweicht ── -# Senderichtung (wir → vLLM): PCM16-16kHz-mono base64 anhaengen + committen. -VLLM_SEND_APPEND = "input_audio_buffer.append" # {"type":..., "audio": ""} -VLLM_SEND_COMMIT = "input_audio_buffer.commit" # Buffer abschliessen -VLLM_AUDIO_FIELD = "audio" -# Empfangsrichtung (vLLM → wir): inkrementeller Text + final. Defensiv geprueft. -VLLM_DELTA_SUFFIXES = ("transcription.delta",) # msg["type"] endet hierauf -VLLM_DONE_SUFFIXES = ("transcription.done", "transcription.completed") -VLLM_DELTA_FIELDS = ("delta", "text", "transcription") # eins davon traegt den Text - -# ── Streaming-/Endpointing-Parameter (analog whisper-Bridge) ── +STREAM_TRANSCRIBE_INTERVAL_MS = int(os.getenv("STREAM_TRANSCRIBE_INTERVAL_MS", "1000")) STREAM_DEFAULT_ENDPOINT_MS = 2400 -STREAM_DEFAULT_HARD_CAP_MS = 60000 +STREAM_DEFAULT_HARD_CAP_MS = 300000 STREAM_MIN_AUDIO_MS = 600 STREAM_SESSION_TTL_S = 120 STREAM_ENERGY_WINDOW_MS = 300 STREAM_SEMANTIC_BACKUP_FACTOR = 2.0 -# Adaptiver Voice-Schwellwert (siehe whisper-Bridge M0.1): Grenze relativ zum -# gemessenen Rausch-Boden statt fix — schneidet leises Sprechen nicht ab. +# Adaptiver Voice-Schwellwert (M0.1): relativ zum gemessenen Rausch-Boden. STREAM_VOICE_FACTOR = 2.5 STREAM_VOICE_RMS_MIN = 0.005 STREAM_VOICE_RMS_MAX = 0.020 @@ -91,21 +69,60 @@ STREAM_VOICE_RMS_MAX = 0.020 def pcm_s16le_to_float32(data: bytes) -> np.ndarray: if not data: return np.zeros(0, dtype=np.float32) - arr = np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0 - return arr + return np.frombuffer(data, dtype=np.int16).astype(np.float32) / 32768.0 async def _send(ws, mtype: str, payload: dict) -> None: try: await ws.send(json.dumps({ - "type": mtype, - "payload": payload, - "timestamp": int(time.time() * 1000), + "type": mtype, "payload": payload, "timestamp": int(time.time() * 1000), })) except Exception as e: logger.warning("RVS-Send fehlgeschlagen (%s): %s", mtype, e) +class VoxtralRunner: + """Haelt das Voxtral-Modell (Transformers). transcribe() blockiert → aus dem + Event-Loop via run_in_executor aufrufen. Ein Lock serialisiert GPU-Zugriffe.""" + + def __init__(self) -> None: + self.model = None + self.processor = None + self._lock = asyncio.Lock() + + def load(self) -> None: + import torch + from transformers import AutoProcessor, VoxtralForConditionalGeneration + t0 = time.time() + logger.info("Lade Voxtral '%s' (device=%s, bf16)…", VOXTRAL_MODEL, VOXTRAL_DEVICE) + self.processor = AutoProcessor.from_pretrained(VOXTRAL_MODEL) + self.model = VoxtralForConditionalGeneration.from_pretrained( + VOXTRAL_MODEL, torch_dtype=torch.bfloat16, device_map=VOXTRAL_DEVICE, + ) + logger.info("Voxtral geladen in %.1fs", time.time() - t0) + + def _transcribe_blocking(self, audio_f32: np.ndarray, language: str) -> str: + # ⚠️ VERIFY: exakte Voxtral-Transformers-API gegen die HF-Modelcard. + import torch + proc, model = self.processor, self.model + if proc is None or model is None or audio_f32.size == 0: + return "" + inputs = proc.apply_transcription_request( + language=language, audio=audio_f32, model_id=VOXTRAL_MODEL, sampling_rate=16000, + ) + inputs = inputs.to(VOXTRAL_DEVICE, dtype=torch.bfloat16) + with torch.no_grad(): + outputs = model.generate(**inputs, max_new_tokens=512) + trimmed = outputs[:, inputs.input_ids.shape[1]:] + text = proc.batch_decode(trimmed, skip_special_tokens=True) + return (text[0] if text else "").strip() + + async def transcribe(self, audio_f32: np.ndarray, language: str) -> str: + loop = asyncio.get_running_loop() + async with self._lock: + return await loop.run_in_executor(None, self._transcribe_blocking, audio_f32, language) + + @dataclass class StreamSession: request_id: str @@ -126,28 +143,26 @@ class StreamSession: last_chunk_at: float = field(default_factory=time.time) last_partial: str = "" last_growth_at: float = 0.0 + last_transcribe_at: float = 0.0 last_voice_at: float = 0.0 noise_floor: float = 0.0 closed: bool = False endpoint_sent: bool = False - # vLLM-Realtime-Session - vllm_ws: object = None - vllm_reader: object = None class SessionManager: - def __init__(self) -> None: + def __init__(self, runner: VoxtralRunner) -> None: + self.runner = runner self._sessions: dict[str, StreamSession] = {} - self._ws = None # RVS + self._ws = None def attach_ws(self, ws) -> None: self._ws = ws - async def start_session(self, payload: dict) -> Optional[StreamSession]: - request_id = (payload.get("requestId") or "").strip() - if not request_id: - logger.warning("stt_stream_start ohne requestId — ignoriert") - return None + def start_session(self, payload: dict) -> None: + rid = (payload.get("requestId") or "").strip() + if not rid: + return try: endpoint_ms = int(payload.get("endpointMs") or STREAM_DEFAULT_ENDPOINT_MS) except (TypeError, ValueError): @@ -160,8 +175,8 @@ class SessionManager: voice_factor = float(payload.get("voiceFactor") or STREAM_VOICE_FACTOR) except (TypeError, ValueError): voice_factor = STREAM_VOICE_FACTOR - sess = StreamSession( - request_id=request_id, + self._sessions[rid] = StreamSession( + request_id=rid, audio_request_id=payload.get("audioRequestId", "") or "", language=payload.get("language") or VOXTRAL_LANGUAGE, endpoint_ms=endpoint_ms, @@ -173,159 +188,44 @@ class SessionManager: location=payload.get("location") or None, sample_rate=int(payload.get("sampleRate") or 16000), ) - # vLLM-Realtime-Session oeffnen + Reader starten. - try: - sess.vllm_ws = await websockets.connect(VOXTRAL_VLLM_URL, max_size=8 * 1024 * 1024) - await self._vllm_configure(sess) - sess.vllm_reader = asyncio.create_task(self._vllm_read_loop(sess)) - except Exception as e: - logger.exception("Stream %s: vLLM-Realtime-Connect fehlgeschlagen: %s", - request_id[:8], e) - # ohne Backend keine Transkription → sofort leeres Endpoint melden - self._sessions[request_id] = sess - await self._finalize(sess, reason="vllm_unavailable") - return None - self._sessions[request_id] = sess logger.info("Voxtral-Session offen: id=%s lang=%s endpointMs=%d", - request_id[:8], sess.language, sess.endpoint_ms) - return sess - - async def _vllm_configure(self, sess: StreamSession) -> None: - """Optionale Session-Konfig an vLLM (Modell/Sprache/temperature=0). - VERIFY: exaktes session.update-Schema gegen vLLM-Realtime-Beispiel. - Best-effort — Fehler hier sind nicht fatal.""" - try: - await sess.vllm_ws.send(json.dumps({ - "type": "session.update", - "session": { - "model": VOXTRAL_MODEL, - "language": sess.language, - "temperature": 0.0, - "input_audio_format": "pcm16", - }, - })) - except Exception: - pass - - async def _vllm_read_loop(self, sess: StreamSession) -> None: - """Liest transcription.delta/.done vom vLLM-Realtime-Server.""" - ws = sess.vllm_ws - try: - async for raw in ws: - try: - msg = json.loads(raw) - except Exception: - continue - mtype = str(msg.get("type", "")) - if any(mtype.endswith(s) for s in VLLM_DELTA_SUFFIXES): - text = self._extract_text(msg) - if text: - await self._on_delta(sess, text) - elif any(mtype.endswith(s) for s in VLLM_DONE_SUFFIXES): - text = self._extract_text(msg) - if text: - await self._on_delta(sess, text, final=True) - except Exception: - logger.debug("Stream %s: vLLM-Reader beendet", sess.request_id[:8]) - - @staticmethod - def _extract_text(msg: dict) -> str: - for f in VLLM_DELTA_FIELDS: - v = msg.get(f) - if isinstance(v, str) and v: - return v - return "" - - async def _on_delta(self, sess: StreamSession, text: str, final: bool = False) -> None: - """Neuer/finaler Transkript-Text vom vLLM. delta = inkrementell; wir - haengen an, wenn er den bisherigen Partial verlaengert, sonst ersetzen - wir (Voxtral kann korrigieren).""" - if final or text.startswith(sess.last_partial): - new_full = text if final else text - else: - new_full = (sess.last_partial + text).strip() - new_full = new_full.strip() - if new_full and new_full != sess.last_partial: - sess.last_partial = new_full - sess.last_growth_at = time.time() - if self._ws is not None: - await _send(self._ws, "stt_partial", { - "requestId": sess.request_id, - "audioRequestId": sess.audio_request_id, - "text": new_full, - }) + rid[:8], self._sessions[rid].language, endpoint_ms) def feed_chunk(self, payload: dict) -> bool: - request_id = payload.get("requestId", "") - sess = self._sessions.get(request_id) + sess = self._sessions.get(payload.get("requestId", "")) if sess is None or sess.closed: return False pcm_b64 = payload.get("pcm", "") - if not pcm_b64: - return True - try: - pcm = base64.b64decode(pcm_b64) - except Exception: - return True - sess.pcm_buffer.extend(pcm) + if pcm_b64: + try: + sess.pcm_buffer.extend(base64.b64decode(pcm_b64)) + except Exception: + pass sess.last_chunk_at = time.time() - # An vLLM weiterreichen (fire-and-forget). - if sess.vllm_ws is not None: - asyncio.create_task(self._vllm_append(sess, pcm_b64)) return True - async def _vllm_append(self, sess: StreamSession, pcm_b64: str) -> None: - try: - await sess.vllm_ws.send(json.dumps({ - "type": VLLM_SEND_APPEND, - VLLM_AUDIO_FIELD: pcm_b64, - })) - except Exception: - pass - def end_session(self, request_id: str) -> None: sess = self._sessions.get(request_id) if sess is not None: sess.closed = True def drop(self, request_id: str) -> None: - sess = self._sessions.pop(request_id, None) - if sess is not None: - self._teardown_vllm(sess) + self._sessions.pop(request_id, None) - def _teardown_vllm(self, sess: StreamSession) -> None: - try: - if sess.vllm_reader is not None: - sess.vllm_reader.cancel() - except Exception: - pass - if sess.vllm_ws is not None: - asyncio.create_task(self._close_ws(sess.vllm_ws)) - sess.vllm_ws = None - - @staticmethod - async def _close_ws(ws) -> None: - try: - await ws.close() - except Exception: - pass - - # ── Endpointer (adaptiv, wie whisper-Bridge M0.1) ── - def _buffer_duration_ms(self, sess: StreamSession) -> float: + # ── Endpointer (adaptiv, M0.1) ── + def _buffer_ms(self, sess: StreamSession) -> float: samples = len(sess.pcm_buffer) // 2 return (samples / sess.sample_rate) * 1000.0 if samples else 0.0 def _tail_rms(self, sess: StreamSession) -> float: - win_bytes = int(sess.sample_rate * STREAM_ENERGY_WINDOW_MS / 1000) * 2 - if win_bytes <= 0: + win = int(sess.sample_rate * STREAM_ENERGY_WINDOW_MS / 1000) * 2 + if win <= 0: return 0.0 - tail = sess.pcm_buffer[-win_bytes:] + tail = sess.pcm_buffer[-win:] if len(tail) < 2: return 0.0 arr = pcm_s16le_to_float32(bytes(tail)) - if arr.size == 0: - return 0.0 - return float(np.sqrt(np.mean(arr * arr))) + return float(np.sqrt(np.mean(arr * arr))) if arr.size else 0.0 def _voice_threshold(self, sess: StreamSession) -> float: nf = sess.noise_floor @@ -343,7 +243,8 @@ class SessionManager: sess.noise_floor = 0.98 * nf + 0.02 * rms async def run_endpointer(self) -> None: - logger.info("Voxtral-Endpointer gestartet (adaptiver VAD)") + logger.info("Voxtral-Endpointer gestartet (adaptiver VAD, interval=%dms)", + STREAM_TRANSCRIBE_INTERVAL_MS) while True: await asyncio.sleep(0.2) now = time.time() @@ -351,7 +252,7 @@ class SessionManager: try: await self._tick(sess, now) except Exception: - logger.exception("Endpointer-Tick crashed (session=%s)", sid[:8]) + logger.exception("Tick crashed (session=%s)", sid[:8]) for sid, sess in list(self._sessions.items()): if now - sess.last_chunk_at > STREAM_SESSION_TTL_S: logger.info("Stream %s: TTL — drop", sid[:8]) @@ -360,14 +261,13 @@ class SessionManager: async def _tick(self, sess: StreamSession, now: float) -> None: if sess.endpoint_sent: return - elapsed_ms = (now - sess.started_at) * 1000.0 - if elapsed_ms > sess.hard_cap_ms and not sess.closed: - await self._finalize(sess, reason="hardcap") + if (now - sess.started_at) * 1000.0 > sess.hard_cap_ms and not sess.closed: + await self._finalize(sess, "hardcap") return if sess.closed: - await self._finalize(sess, reason="stream_end") + await self._finalize(sess, "stream_end") return - if self._buffer_duration_ms(sess) < STREAM_MIN_AUDIO_MS: + if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS: return # adaptive akustische Sprach-Aktivitaet rms = self._tail_rms(sess) @@ -375,45 +275,65 @@ class SessionManager: sess.last_voice_at = now else: self._update_noise_floor(sess, rms) - # Endpoint: akustisch (Primaer) oder semantisch (Backstop), sobald Text da + # Endpoint-Entscheidung, sobald Text erkannt wurde if sess.last_growth_at > 0.0: - acoustic_silence_ms = (now - sess.last_voice_at) * 1000.0 if sess.last_voice_at > 0 else 0.0 - semantic_silence_ms = (now - sess.last_growth_at) * 1000.0 - acoustic_done = sess.last_voice_at > 0 and acoustic_silence_ms >= sess.endpoint_ms - semantic_done = semantic_silence_ms >= sess.endpoint_ms * STREAM_SEMANTIC_BACKUP_FACTOR - if acoustic_done or semantic_done: - await self._finalize(sess, reason="endpoint" if acoustic_done else "endpoint_semantic") + ac_sil = (now - sess.last_voice_at) * 1000.0 if sess.last_voice_at > 0 else 0.0 + se_sil = (now - sess.last_growth_at) * 1000.0 + ac_done = sess.last_voice_at > 0 and ac_sil >= sess.endpoint_ms + se_done = se_sil >= sess.endpoint_ms * STREAM_SEMANTIC_BACKUP_FACTOR + if ac_done or se_done: + await self._finalize(sess, "endpoint" if ac_done else "endpoint_semantic") + return + # Partial-Transkription (throttled) + if (now - sess.last_transcribe_at) * 1000.0 < STREAM_TRANSCRIBE_INTERVAL_MS: + return + sess.last_transcribe_at = now + audio = pcm_s16le_to_float32(bytes(sess.pcm_buffer)) + try: + text = (await self.runner.transcribe(audio, sess.language)).strip() + except Exception: + logger.exception("Stream %s: Partial-Transcribe crashed", sess.request_id[:8]) + return + if text and text != sess.last_partial: + sess.last_partial = text + sess.last_growth_at = now + if self._ws is not None: + await _send(self._ws, "stt_partial", { + "requestId": sess.request_id, + "audioRequestId": sess.audio_request_id, + "text": text, + }) async def _finalize(self, sess: StreamSession, reason: str) -> None: if sess.endpoint_sent: return sess.endpoint_sent = True - # vLLM ggf. committen, damit ein letztes transcription.done kommt. - if sess.vllm_ws is not None: - try: - await sess.vllm_ws.send(json.dumps({"type": VLLM_SEND_COMMIT})) - await asyncio.sleep(0.15) # kurz auf finalen Delta warten - except Exception: - pass - final_text = sess.last_partial.strip() - duration_s = self._buffer_duration_ms(sess) / 1000.0 - logger.info("Stream %s: FINAL (reason=%s, %.1fs): %r", - sess.request_id[:8], reason, duration_s, final_text[:120]) + audio = pcm_s16le_to_float32(bytes(sess.pcm_buffer)) + t0 = time.time() + try: + final_text = (await self.runner.transcribe(audio, sess.language)).strip() + except Exception: + logger.exception("Stream %s: Final-Transcribe crashed", sess.request_id[:8]) + final_text = sess.last_partial + stt_ms = int((time.time() - t0) * 1000) + duration_s = audio.size / 16000.0 + logger.info("Stream %s: FINAL (reason=%s, %.1fs, %dms): %r", + sess.request_id[:8], reason, duration_s, stt_ms, final_text[:120]) if self._ws is not None: - endpoint_payload = { + payload = { "requestId": sess.request_id, "audioRequestId": sess.audio_request_id, "text": final_text, "reason": reason, "durationS": duration_s, - "sttMs": 0, + "sttMs": stt_ms, "voice": sess.voice, "speed": sess.speed, "interrupted": sess.interrupted, } if sess.location: - endpoint_payload["location"] = sess.location - await _send(self._ws, "stt_endpoint", endpoint_payload) + payload["location"] = sess.location + await _send(self._ws, "stt_endpoint", payload) await _send(self._ws, "stt_stream_done", { "requestId": sess.request_id, "audioRequestId": sess.audio_request_id, @@ -447,7 +367,6 @@ async def run_loop(sessions: SessionManager) -> None: sessions.attach_ws(ws) await _broadcast_status(ws, "ready", model=VOXTRAL_MODEL) await _send(ws, "config_request", {"service": "voxtral"}) - async for raw in ws: try: msg = json.loads(raw) @@ -456,35 +375,32 @@ async def run_loop(sessions: SessionManager) -> None: mtype = msg.get("type", "") payload = msg.get("payload", {}) or {} if mtype == "stt_stream_start": - asyncio.create_task(sessions.start_session(payload)) + sessions.start_session(payload) elif mtype == "stt_audio_chunk": sessions.feed_chunk(payload) elif mtype == "stt_stream_end": sessions.end_session(payload.get("requestId", "")) - # stt_request (Legacy One-Shot) macht Voxtral hier NICHT — - # dafuer bleibt die whisper-Bridge (Fallback). except Exception as e: logger.warning("RVS-Verbindung verloren: %s — retry in %ds", e, retry_s) if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried: use_tls = False tls_fallback_tried = True - logger.info("TLS-Fallback: versuche ws:// (kein TLS)") continue await asyncio.sleep(retry_s) retry_s = min(retry_s * 2, 30) - use_tls = RVS_TLS # fuer den naechsten Zyklus zuruecksetzen + use_tls = RVS_TLS async def main() -> None: if not RVS_HOST or not RVS_TOKEN: logger.error("RVS_HOST/RVS_TOKEN fehlen — .env pruefen. Abbruch.") return - sessions = SessionManager() - logger.info("Voxtral-Bridge startet — vLLM=%s Modell=%s", VOXTRAL_VLLM_URL, VOXTRAL_MODEL) - await asyncio.gather( - run_loop(sessions), - sessions.run_endpointer(), - ) + runner = VoxtralRunner() + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, runner.load) # Modell laden (blockierend) + sessions = SessionManager(runner) + logger.info("Voxtral-Bridge startet — Modell=%s", VOXTRAL_MODEL) + await asyncio.gather(run_loop(sessions), sessions.run_endpointer()) if __name__ == "__main__": diff --git a/xtts/voxtral/requirements.txt b/xtts/voxtral/requirements.txt index 31f911d..f55406b 100644 --- a/xtts/voxtral/requirements.txt +++ b/xtts/voxtral/requirements.txt @@ -1,5 +1,8 @@ -# Voxtral-Bridge ist reine CPU-Glue (RVS-WS <-> vLLM-Realtime-WS). Das Modell -# selbst laeuft im separaten voxtral-vllm-Container (GPU). Deshalb hier KEIN -# torch/vllm — nur der WebSocket-Client + numpy fuer die RMS-Energiemessung. -websockets>=12.0 +# Voxtral-3B via Transformers. torch/torchaudio kommen cu124-gepinnt aus dem +# Dockerfile (nicht hier, sonst zieht pip das Default-CUDA-Wheel). +transformers>=4.54 +mistral-common[audio]>=1.8.1 +accelerate>=0.30 +soundfile>=0.12 numpy>=1.24 +websockets>=12.0