Hacker-Software/ARIA-AGENT
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 236 Wiki Activity

fix(voxtral): willkuerliche Abbrueche — semantischen Endpoint + Live-Partials raus

Browse Source 
Ursache: Voxtral-3B transkribiert den ganzen wachsenden Buffer (~5-6s bei langen Aufnahmen). Diese Partial-Latenz war groesser als der semantische Endpoint-Timeout (4.8s) → 'Text waechst nicht mehr' feuerte faelschlich → Abbruch nach 20-40s. Fix: keine Live-Partials mehr, kein semantischer Endpoint — Turn-Ende rein akustisch (Stille-VAD), transkribiert wird nur EINMAL im _finalize. max_new_tokens 512->4096 (512 schnitt lange Diktate ab).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
duffyduck
2026-08-15 12:02:44 +02:00
co-authored by Claude Opus 4.8
parent 01df26e6df
commit 36f04f83ff
1 changed files with 12 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xtts/voxtral/bridge.py
+12 -30
View File
@@ -121,7 +121,9 @@ class VoxtralRunner:
) )
inputs = inputs.to(VOXTRAL_DEVICE, dtype=torch.bfloat16) inputs = inputs.to(VOXTRAL_DEVICE, dtype=torch.bfloat16)
with torch.no_grad(): with torch.no_grad():
outputs = model.generate(**inputs, max_new_tokens=512) # hoch genug fuer lange Diktate (stoppt eh am EOS); 512 hat
# mehrminutige Aufnahmen abgeschnitten.
outputs = model.generate(**inputs, max_new_tokens=4096)
trimmed = outputs[:, inputs.input_ids.shape[1]:] trimmed = outputs[:, inputs.input_ids.shape[1]:]
text = proc.batch_decode(trimmed, skip_special_tokens=True) text = proc.batch_decode(trimmed, skip_special_tokens=True)
return (text[0] if text else "").strip() return (text[0] if text else "").strip()
@@ -284,40 +286,20 @@ class SessionManager:
return return
if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS: if self._buffer_ms(sess) < STREAM_MIN_AUDIO_MS:
return return
# adaptive akustische Sprach-Aktivitaet # Adaptive akustische Sprach-Aktivitaet (M0.1). KEINE Live-Partials mehr:
# Voxtral-3B transkribiert den ganzen WACHSENDEN Buffer und braucht dafuer
# bei langen Aufnahmen 5-6 s — zu langsam fuer Live-Text, UND diese Latenz
# hat den semantischen Endpoint faelschlich ausgeloest (Partial-Latenz >
# Timeout → willkuerliche Abbrueche nach 20-40 s). Deshalb: Turn-Ende rein
# AKUSTISCH, transkribiert wird nur EINMAL im _finalize.
rms = self._tail_rms(sess) rms = self._tail_rms(sess)
if rms >= self._voice_threshold(sess): if rms >= self._voice_threshold(sess):
sess.last_voice_at = now sess.last_voice_at = now
else: else:
self._update_noise_floor(sess, rms) self._update_noise_floor(sess, rms)
# Endpoint-Entscheidung, sobald Text erkannt wurde # Endpoint: hat der User schon gesprochen UND ist es seit endpoint_ms still?
if sess.last_growth_at > 0.0: if sess.last_voice_at > 0 and (now - sess.last_voice_at) * 1000.0 >= sess.endpoint_ms:
ac_sil = (now - sess.last_voice_at) * 1000.0 if sess.last_voice_at > 0 else 0.0 await self._finalize(sess, "endpoint")
se_sil = (now - sess.last_growth_at) * 1000.0
ac_done = sess.last_voice_at > 0 and ac_sil >= sess.endpoint_ms
se_done = se_sil >= sess.endpoint_ms * STREAM_SEMANTIC_BACKUP_FACTOR
if ac_done or se_done:
await self._finalize(sess, "endpoint" if ac_done else "endpoint_semantic")
return
# Partial-Transkription (throttled)
if (now - sess.last_transcribe_at) * 1000.0 < STREAM_TRANSCRIBE_INTERVAL_MS:
return
sess.last_transcribe_at = now
audio = pcm_s16le_to_float32(bytes(sess.pcm_buffer))
try:
text = (await self.runner.transcribe(audio, sess.language)).strip()
except Exception:
logger.exception("Stream %s: Partial-Transcribe crashed", sess.request_id[:8])
return
if text and text != sess.last_partial:
sess.last_partial = text
sess.last_growth_at = now
if self._ws is not None:
await _send(self._ws, "stt_partial", {
"requestId": sess.request_id,
"audioRequestId": sess.audio_request_id,
"text": text,
})
async def _finalize(self, sess: StreamSession, reason: str) -> None: async def _finalize(self, sess: StreamSession, reason: str) -> None:
if sess.endpoint_sent: if sess.endpoint_sent: