feat: Streaming TTS — PCM-Stream statt WAV-Chunks (Weg A)
Pipeline: XTTS-Server → xtts-bridge → aria-bridge → RVS → App AudioTrack XTTS-Bridge (Gaming-PC): - streamXTTSAsPCM(): liest /tts_to_audio/ Response inkrementell, parst WAV-Header (samplerate/channels), teilt PCM in 8KB-Chunks (~170ms bei 24kHz s16 mono) und sendet jeden als audio_pcm. - Finaler Chunk mit final=true nach letztem Text-Chunk aria-bridge: - audio_pcm Handler leitet payload 1:1 weiter, filled messageId aus requestId → messageId Map falls XTTS-Bridge messageId nicht hatte - Alter xtts_response Pfad bleibt als Legacy-Fallback (WAV) RVS: audio_pcm in ALLOWED_TYPES Android Native: - PcmStreamPlayerModule (Kotlin): AudioTrack MODE_STREAM mit Writer-Thread und BlockingQueue. start(rate, ch) / writeChunk(b64) / end() / stop() - 8x MinBufferSize grosszuegig dimensioniert, glatt auch bei Netz-Aussetzern - Registered im MainApplication via PcmStreamPlayerPackage App JS: - audioService.handlePcmChunk(): erkennt neue Session (messageId-Wechsel), started nativen Stream, cached PCM-Bytes pro Message. Bei final=true Stream sauber schliessen + _savePcmBufferAsWav → WAV-File im tts_cache/<messageId>.wav - _savePcmBufferAsWav: baut 44-byte WAV-Header (PCM s16le, korrekte samplerate/channels), haengt alle gesammelten base64-PCM-Chunks an - stopPlayback beendet auch aktiven PCM-Stream - ChatScreen routet type=audio_pcm an handlePcmChunk, bei final setzt audioPath in der Message Play-Button: falls messageId einen audioPath hat → WAV aus Cache (Sound-basiert), egal ob Original-TTS Piper oder XTTS war. Audio-Focus: - requestDuck() beim Stream-Start, release() bei Stream-Ende - Andere Apps (Spotify etc.) werden leiser waehrend ARIA spricht Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+124
-44
@@ -94,34 +94,33 @@ function connectRVS(forcePlain) {
|
||||
// ── TTS Request Handler ─────────────────────────────
|
||||
|
||||
async function handleTTSRequest(payload) {
|
||||
const { text, voice, requestId, language } = payload;
|
||||
const { text, voice, requestId, language, messageId } = payload;
|
||||
if (!text) return;
|
||||
|
||||
// Markdown + Sonderzeichen entfernen fuer natuerliche Sprache
|
||||
// Markdown-Cleanup (Bridge macht jetzt auch Cleanup, aber safety net)
|
||||
let cleanText = text
|
||||
.replace(/\*\*([^*]+)\*\*/g, "$1") // **fett** → fett
|
||||
.replace(/\*([^*]+)\*/g, "$1") // *kursiv* → kursiv
|
||||
.replace(/`([^`]+)`/g, "$1") // `code` → code
|
||||
.replace(/```[\s\S]*?```/g, "") // Code-Bloecke entfernen
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // [text](url) → text
|
||||
.replace(/#{1,6}\s*/g, "") // ### Ueberschriften → entfernen
|
||||
.replace(/>\s*/g, "") // > Zitate → entfernen
|
||||
.replace(/[-*]\s+/g, "") // - Listen → entfernen
|
||||
.replace(/\n{2,}/g, ". ") // Mehrere Newlines → Punkt
|
||||
.replace(/\n/g, ", ") // Einzelne Newlines → Komma
|
||||
.replace(/\s{2,}/g, " ") // Mehrfach-Leerzeichen
|
||||
.replace(/["""„]/g, "") // Anfuehrungszeichen entfernen
|
||||
.replace(/\(\)/g, "") // Leere Klammern
|
||||
.replace(/\*\*([^*]+)\*\*/g, "$1")
|
||||
.replace(/\*([^*]+)\*/g, "$1")
|
||||
.replace(/`([^`]+)`/g, "$1")
|
||||
.replace(/```[\s\S]*?```/g, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/#{1,6}\s*/g, "")
|
||||
.replace(/>\s*/g, "")
|
||||
.replace(/[-*]\s+/g, "")
|
||||
.replace(/\n{2,}/g, ". ")
|
||||
.replace(/\n/g, ", ")
|
||||
.replace(/\s{2,}/g, " ")
|
||||
.replace(/["""„]/g, "")
|
||||
.replace(/\(\)/g, "")
|
||||
.trim();
|
||||
|
||||
// Text in Saetze aufteilen, dann zu Chunks von 2-3 Saetzen zusammenfassen
|
||||
// (mehr Kontext = konsistentere Stimme/Lautstaerke, aber nicht zu lang fuer WebSocket)
|
||||
// Satzweise Chunks (XTTS Modell laedt Context pro Call — Saetze gruppieren)
|
||||
const sentences = cleanText.split(/(?<=[.!?])\s+/)
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0)
|
||||
.map(s => s.replace(/[.]+$/, '')); // Punkt am Ende entfernen
|
||||
.map(s => s.replace(/[.]+$/, ''));
|
||||
|
||||
const MAX_CHUNK_CHARS = 150; // Max ~150 Zeichen pro Chunk (schnelles Rendering, Preloading reicht)
|
||||
const MAX_CHUNK_CHARS = 150;
|
||||
const chunks = [];
|
||||
let currentChunk = '';
|
||||
for (const sentence of sentences) {
|
||||
@@ -135,45 +134,70 @@ async function handleTTSRequest(payload) {
|
||||
if (currentChunk) chunks.push(currentChunk);
|
||||
if (chunks.length === 0) return;
|
||||
|
||||
log(`TTS-Request: "${cleanText.slice(0, 60)}..." (${sentences.length} Saetze → ${chunks.length} Chunks, voice: ${voice || "default"}, lang: ${language || "de"})`);
|
||||
log(`TTS-Request (streaming): "${cleanText.slice(0, 60)}..." (${chunks.length} Chunks, voice: ${voice || "default"})`);
|
||||
|
||||
try {
|
||||
const voiceSample = voice ? path.join(VOICES_DIR, `${voice}.wav`) : null;
|
||||
const hasCustomVoice = voiceSample && fs.existsSync(voiceSample);
|
||||
|
||||
// Streaming: Chunk rendern → sofort senden → naechster Chunk
|
||||
// App spielt mit Preloading-Queue nahtlos ab
|
||||
let sentCount = 0;
|
||||
let chunkIndex = 0;
|
||||
// Audio-Format (aus WAV-Header extrahiert, einmal pro Request)
|
||||
let pcmMeta = null;
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const isLastChunk = i === chunks.length - 1;
|
||||
try {
|
||||
const audioBuffer = await callXTTSAPI(chunk, language || "de", hasCustomVoice ? voiceSample : null);
|
||||
|
||||
if (audioBuffer && audioBuffer.length > 100) {
|
||||
log(`TTS [${i + 1}/${chunks.length}]: ${(audioBuffer.length / 1024).toFixed(0)}KB — "${chunk.slice(0, 50)}"`);
|
||||
// Streaming: PCM-Frames werden nacheinander an RVS gepusht,
|
||||
// sobald sie vom XTTS-Server reinkommen
|
||||
await streamXTTSAsPCM(
|
||||
chunk,
|
||||
language || "de",
|
||||
hasCustomVoice ? voiceSample : null,
|
||||
(pcmBase64, meta) => {
|
||||
if (!pcmMeta) pcmMeta = meta;
|
||||
sendToRVS({
|
||||
type: "audio_pcm",
|
||||
payload: {
|
||||
requestId: requestId || "",
|
||||
messageId: messageId || "",
|
||||
base64: pcmBase64,
|
||||
format: "pcm_s16le",
|
||||
sampleRate: meta.sampleRate,
|
||||
channels: meta.channels,
|
||||
voice: voice || "default",
|
||||
chunk: chunkIndex++,
|
||||
final: false,
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// Nach letztem Text-Chunk: final-Flag senden damit App weiss "fertig"
|
||||
if (isLastChunk && pcmMeta) {
|
||||
sendToRVS({
|
||||
type: "xtts_response",
|
||||
type: "audio_pcm",
|
||||
payload: {
|
||||
requestId: `${requestId || ""}_${i}`,
|
||||
base64: audioBuffer.toString("base64"),
|
||||
mimeType: "audio/wav",
|
||||
requestId: requestId || "",
|
||||
messageId: messageId || "",
|
||||
base64: "",
|
||||
format: "pcm_s16le",
|
||||
sampleRate: pcmMeta.sampleRate,
|
||||
channels: pcmMeta.channels,
|
||||
voice: voice || "default",
|
||||
engine: "xtts",
|
||||
part: i + 1,
|
||||
totalParts: chunks.length,
|
||||
chunk: chunkIndex++,
|
||||
final: true,
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
sentCount++;
|
||||
}
|
||||
} catch (chunkErr) {
|
||||
log(`TTS [${i + 1}/${chunks.length}] Fehler: ${chunkErr.message} — ueberspringe`);
|
||||
}
|
||||
}
|
||||
|
||||
log(`TTS komplett: ${sentCount}/${chunks.length} Chunks gestreamt`);
|
||||
log(`TTS komplett: ${chunkIndex} PCM-Frames gestreamt (${cleanText.length} chars)`);
|
||||
} catch (err) {
|
||||
log(`TTS Fehler: ${err.message}`);
|
||||
sendToRVS({
|
||||
@@ -184,7 +208,19 @@ async function handleTTSRequest(payload) {
|
||||
}
|
||||
}
|
||||
|
||||
function callXTTSAPI(text, language, speakerWav) {
|
||||
/**
|
||||
* Ruft /tts_to_audio/ auf und streamt das resultierende WAV bereits waehrend
|
||||
* des Empfangs in PCM-Frames an den Callback. Der WAV-Header wird einmal
|
||||
* geparst, danach werden nur noch raw PCM-Samples weitergeleitet.
|
||||
*
|
||||
* Warum nicht echtes /tts_stream/? daswer123 hat den Endpoint, aber die
|
||||
* Audio-Quality ist dort niedriger und er produziert beim ersten Chunk
|
||||
* oft Artefakte. Pragmatischer Weg: /tts_to_audio/ + Response-Stream
|
||||
* chunkweise auslesen. Das ist zwar kein echtes Server-Streaming, aber
|
||||
* gibt uns deutlich kleinere Netzwerk-Haeppchen und die App kann via
|
||||
* AudioTrack MODE_STREAM sofort nahtlos abspielen.
|
||||
*/
|
||||
function streamXTTSAsPCM(text, language, speakerWav, onPcmChunk) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const body = JSON.stringify({
|
||||
text,
|
||||
@@ -206,15 +242,59 @@ function callXTTSAPI(text, language, speakerWav) {
|
||||
};
|
||||
|
||||
const req = http.request(options, (res) => {
|
||||
const chunks = [];
|
||||
res.on("data", (chunk) => chunks.push(chunk));
|
||||
res.on("end", () => {
|
||||
if (res.statusCode === 200) {
|
||||
resolve(Buffer.concat(chunks));
|
||||
} else {
|
||||
reject(new Error(`XTTS API HTTP ${res.statusCode}: ${Buffer.concat(chunks).toString().slice(0, 200)}`));
|
||||
if (res.statusCode !== 200) {
|
||||
let body = "";
|
||||
res.on("data", (d) => { body += d.toString(); });
|
||||
res.on("end", () => reject(new Error(`XTTS HTTP ${res.statusCode}: ${body.slice(0, 200)}`)));
|
||||
return;
|
||||
}
|
||||
|
||||
let headerParsed = false;
|
||||
let sampleRate = 24000;
|
||||
let channels = 1;
|
||||
let leftover = Buffer.alloc(0); // ungerade Byte-Reste fuer das naechste Chunk
|
||||
const HEADER_BYTES = 44;
|
||||
let headerBuf = Buffer.alloc(0);
|
||||
const PCM_CHUNK_BYTES = 8192; // ~170ms bei 24kHz s16 mono
|
||||
|
||||
res.on("data", (chunk) => {
|
||||
let data = chunk;
|
||||
|
||||
// WAV-Header konsumieren (44 Bytes)
|
||||
if (!headerParsed) {
|
||||
headerBuf = Buffer.concat([headerBuf, data]);
|
||||
if (headerBuf.length < HEADER_BYTES) return;
|
||||
// Header lesen
|
||||
const header = headerBuf.slice(0, HEADER_BYTES);
|
||||
try {
|
||||
channels = header.readUInt16LE(22);
|
||||
sampleRate = header.readUInt32LE(24);
|
||||
} catch (_) {}
|
||||
headerParsed = true;
|
||||
data = headerBuf.slice(HEADER_BYTES);
|
||||
}
|
||||
|
||||
// leftover aus vorherigem Chunk + neuer data
|
||||
let combined = Buffer.concat([leftover, data]);
|
||||
|
||||
// In PCM_CHUNK_BYTES-Happen zerlegen (Vielfache von 2 damit keine Sample-Splits)
|
||||
while (combined.length >= PCM_CHUNK_BYTES) {
|
||||
const slice = combined.slice(0, PCM_CHUNK_BYTES);
|
||||
combined = combined.slice(PCM_CHUNK_BYTES);
|
||||
onPcmChunk(slice.toString("base64"), { sampleRate, channels });
|
||||
}
|
||||
leftover = combined;
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
// Rest-Daten senden
|
||||
if (leftover.length > 0) {
|
||||
onPcmChunk(leftover.toString("base64"), { sampleRate, channels });
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
res.on("error", reject);
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
|
||||
Reference in New Issue
Block a user