feat(local-llm): B0 — Gamebox llama.cpp + RVS-Adapter (Provider-Seite)
Plan B, Phase B0 (Provider): lokales Qwen3-8B auf der Gamebox, angebunden per RVS wie f5tts/whisper (kein IP-Pflegen, nur URL+Token). - xtts/llm-adapter/: RVS-Client (spiegelt whisper-bridge: TLS+ws-Fallback, Reconnect-Backoff), nimmt llm_request, ruft llama.cpp /v1/chat/completions lokal, antwortet llm_response (korreliert per requestId). Nicht-streamend in B0; llm_partial fuer B2 reserviert. - xtts/docker-compose.yml: neue Services `llama` (llama.cpp server-cuda, GGUF via ./models, OpenAI-API auf :8081) + `llm-adapter`. - rvs/server.js: ALLOWED_TYPES += llm_request/llm_response/llm_partial. - GGUF (mehrere GB) via .gitignore aus dem Repo; xtts/models/ mit .gitkeep. Topologie-Hinweis: Gamebox@home, ARIA@RZ -> Bounce ueber Internet ist unvermeidbar (Voice macht's schon so); Router faellt bei Nichterreichbarkeit per Escalation auf Claude zurueck. Consumer-Seite (Bridge-Relay + Brain- Client + Router) kommt als naechstes. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+4
-1
@@ -78,4 +78,7 @@ __pycache__/
|
||||
.vscode/settings.json
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*.swo
|
||||
|
||||
# Lokale LLM-Modelle (Plan B) — GGUF sind mehrere GB, nicht ins Repo
|
||||
xtts/models/*.gguf
|
||||
|
||||
@@ -62,6 +62,10 @@ const ALLOWED_TYPES = new Set([
|
||||
"flux_request", "flux_response",
|
||||
"agent_stream",
|
||||
"oauth_callback",
|
||||
// Lokales LLM (Plan B) — Router im Brain schickt einfache Turns an das
|
||||
// Qwen3 auf der Gamebox (via Bridge → RVS → llm-adapter → llama.cpp).
|
||||
// llm_partial ist fuer B2 (Token-Streaming) reserviert, noch ungenutzt.
|
||||
"llm_request", "llm_response", "llm_partial",
|
||||
]);
|
||||
|
||||
// Token-Raum: token -> { clients: Set<ws> }
|
||||
|
||||
@@ -89,3 +89,47 @@ services:
|
||||
# Stimm-Embedding) persistent zwischen
|
||||
# Container-Restarts.
|
||||
restart: unless-stopped
|
||||
|
||||
# ─── Lokales LLM (Plan B, B0) — llama.cpp-Server (GPU) ────────
|
||||
# Serviert Qwen3-8B (GGUF Q4_K_M) OpenAI-kompatibel auf :8081, NUR im
|
||||
# Compose-Netz (kein RVS direkt) — die Bruecke macht der llm-adapter.
|
||||
# Modell-Datei nach ./models/ legen: siehe llm-adapter/README.md.
|
||||
# VRAM auf der RTX 3060 (12 GB): whisper-small (~1-2) + f5tts (~1-2) +
|
||||
# qwen3-8b-q4 (~6) ~= 9-10 GB. Passt, aber knapp — bei OOM: LLM_CTX kleiner
|
||||
# oder Modell auf Q4_K_S/IQ4 wechseln.
|
||||
llama:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
||||
container_name: aria-llama
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
volumes:
|
||||
- ./models:/models
|
||||
command: >
|
||||
-m /models/${LLM_GGUF:-qwen3-8b-q4_k_m.gguf}
|
||||
--host 0.0.0.0 --port 8081
|
||||
-ngl 99 -c ${LLM_CTX:-8192} --jinja
|
||||
restart: unless-stopped
|
||||
|
||||
# ─── Local-LLM-Adapter — RVS <-> llama.cpp (Plan B, B0) ──────
|
||||
# Verbindet sich per Token an den RVS (wie f5tts/whisper), nimmt
|
||||
# llm_request entgegen, ruft llama.cpp lokal, antwortet llm_response.
|
||||
llm-adapter:
|
||||
build: ./llm-adapter
|
||||
container_name: aria-llm-adapter
|
||||
depends_on:
|
||||
- llama
|
||||
environment:
|
||||
- RVS_HOST=${RVS_HOST}
|
||||
- RVS_PORT=${RVS_PORT:-443}
|
||||
- RVS_TLS=${RVS_TLS:-true}
|
||||
- RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true}
|
||||
- RVS_TOKEN=${RVS_TOKEN}
|
||||
- LLAMA_URL=http://llama:8081
|
||||
- LLM_MODEL=${LLM_MODEL:-qwen3-8b}
|
||||
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-60}
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY adapter.py .
|
||||
|
||||
CMD ["python", "-u", "adapter.py"]
|
||||
@@ -0,0 +1,64 @@
|
||||
# Local-LLM-Adapter (Gamebox) — Plan B, Phase B0
|
||||
|
||||
Bringt ein lokales, schnelles LLM (Qwen3 8B) auf die Gamebox und haengt es
|
||||
per RVS an ARIA — fuer die einfachen ~80 % der Turns (<1 s), waehrend Claude
|
||||
das Tiefen-Hirn bleibt. Siehe `docs/plan-local-llm-router.md` im Repo-Root.
|
||||
|
||||
## Zwei Container (in `xtts/docker-compose.yml`)
|
||||
|
||||
- **`llama`** — `llama.cpp`-Server (CUDA), serviert das GGUF OpenAI-kompatibel
|
||||
auf `:8081`, nur im Compose-Netz.
|
||||
- **`llm-adapter`** — verbindet sich per Token an den RVS (wie f5tts/whisper),
|
||||
nimmt `llm_request` entgegen, ruft `llama` lokal, antwortet `llm_response`.
|
||||
|
||||
## Modell besorgen (einmalig)
|
||||
|
||||
GGUF nach `xtts/models/` legen. Empfohlen: **Qwen3 8B, Q4_K_M**.
|
||||
|
||||
```bash
|
||||
mkdir -p xtts/models
|
||||
# z.B. von Hugging Face (huggingface-cli oder Browser-Download):
|
||||
# Suche: "Qwen3-8B-GGUF" -> Datei qwen3-8b-q4_k_m.gguf
|
||||
# Datei ablegen als:
|
||||
# xtts/models/qwen3-8b-q4_k_m.gguf
|
||||
```
|
||||
|
||||
Anderer Dateiname? In der `.env` der Gamebox setzen:
|
||||
|
||||
```
|
||||
LLM_GGUF=dein-modell.gguf
|
||||
LLM_CTX=8192 # Kontextfenster (kleiner = weniger VRAM)
|
||||
```
|
||||
|
||||
Mistral statt Qwen testen (A/B): einfach ein Mistral-Small-3-GGUF ablegen und
|
||||
`LLM_GGUF` umstellen — Ein-Datei-Wechsel, kein Code.
|
||||
|
||||
## Start (auf der Gamebox)
|
||||
|
||||
```bash
|
||||
cd xtts
|
||||
docker compose up -d --build llama llm-adapter
|
||||
docker logs -f aria-llm-adapter # "RVS verbunden — llm-adapter online"
|
||||
```
|
||||
|
||||
## Standalone-Test (ohne ARIA), direkt gegen llama.cpp
|
||||
|
||||
```bash
|
||||
curl http://localhost:8081/v1/chat/completions -H "Content-Type: application/json" -d '{
|
||||
"messages":[{"role":"system","content":"Du bist ARIA."},
|
||||
{"role":"user","content":"sag kurz hallo"}],
|
||||
"max_tokens":64
|
||||
}'
|
||||
```
|
||||
|
||||
## VRAM-Hinweis (RTX 3060, 12 GB)
|
||||
|
||||
whisper-small (~1–2) + f5tts (~1–2) + qwen3-8b-q4 (~6) ≈ 9–10 GB. Passt, aber
|
||||
knapp. Bei OOM: `LLM_CTX` reduzieren, `-ngl` senken (weniger Layer auf GPU),
|
||||
oder kleineres Quant (Q4_K_S / IQ4_XS).
|
||||
|
||||
## Nachrichten-Kontrakt (RVS)
|
||||
|
||||
- `llm_request` → `{ requestId, messages:[{role,content}], max_tokens?, temperature?, stop? }`
|
||||
- `llm_response` ← `{ requestId, ok, content, error?, model, elapsedMs }`
|
||||
- `llm_partial` — reserviert fuer B2 (Token-Streaming), noch ungenutzt.
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
ARIA Local-LLM-Adapter (Gamebox) — Plan B, Phase B0.
|
||||
|
||||
Bruecke zwischen RVS und dem lokalen llama.cpp-Server. Spiegelt das Muster der
|
||||
whisper-bridge: verbindet sich per WebSocket mit dem RVS (Token-Room, TLS mit
|
||||
ws-Fallback, Reconnect-Backoff), lauscht auf `llm_request` und ruft den lokalen
|
||||
llama.cpp-`/v1/chat/completions`-Endpoint (OpenAI-kompatibel), antwortet mit
|
||||
`llm_response` (korreliert per requestId).
|
||||
|
||||
Topologie: Gamebox steht zuhause, ARIA im RZ — die Kommunikation laeuft ueber
|
||||
den RVS (wie TTS/STT), keine IPs zu pflegen. Nur URL + Token.
|
||||
|
||||
Env:
|
||||
RVS_HOST, RVS_PORT, RVS_TLS, RVS_TLS_FALLBACK, RVS_TOKEN (wie f5tts/whisper)
|
||||
LLAMA_URL Default http://llama:8081 (llama.cpp im selben Compose-Netz)
|
||||
LLM_MODEL optionaler Modell-Name fuer llama (llama.cpp ignoriert ihn
|
||||
meist, dient nur der Transparenz im Log)
|
||||
LLM_TIMEOUT_SEC Default 60
|
||||
|
||||
Bewusst NICHT-streamend in B0 (volle llm_response). Token-Streaming (llm_partial)
|
||||
kommt in B2 zusammen mit TTS-on-first-sentence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import websockets
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("llm-adapter")
|
||||
|
||||
RVS_HOST = os.getenv("RVS_HOST", "").strip()
|
||||
RVS_PORT = os.getenv("RVS_PORT", "443").strip()
|
||||
RVS_TLS = os.getenv("RVS_TLS", "true").lower() == "true"
|
||||
RVS_TLS_FALLBACK = os.getenv("RVS_TLS_FALLBACK", "true").lower() == "true"
|
||||
RVS_TOKEN = os.getenv("RVS_TOKEN", "").strip()
|
||||
|
||||
LLAMA_URL = os.getenv("LLAMA_URL", "http://llama:8081").rstrip("/")
|
||||
LLM_MODEL = os.getenv("LLM_MODEL", "qwen3-8b")
|
||||
LLM_TIMEOUT_SEC = float(os.getenv("LLM_TIMEOUT_SEC", "60"))
|
||||
|
||||
|
||||
async def _send(ws, mtype: str, payload: dict) -> None:
|
||||
try:
|
||||
await ws.send(json.dumps({
|
||||
"type": mtype,
|
||||
"payload": payload,
|
||||
"timestamp": int(time.time() * 1000),
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.warning("Send fehlgeschlagen (%s): %s", mtype, e)
|
||||
|
||||
|
||||
async def _call_llama(messages: list, *, max_tokens: int, temperature: float,
|
||||
stop) -> dict:
|
||||
"""Ruft llama.cpp /v1/chat/completions (OpenAI-Format). Gibt
|
||||
{ok, content, error} zurueck — wirft nie."""
|
||||
body = {
|
||||
"model": LLM_MODEL,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"stream": False,
|
||||
}
|
||||
if stop:
|
||||
body["stop"] = stop
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=LLM_TIMEOUT_SEC) as client:
|
||||
r = await client.post(f"{LLAMA_URL}/v1/chat/completions", json=body)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
content = (data.get("choices") or [{}])[0].get("message", {}).get("content", "")
|
||||
return {"ok": True, "content": content or "", "usage": data.get("usage")}
|
||||
except Exception as e:
|
||||
logger.warning("llama.cpp-Call fehlgeschlagen: %s", e)
|
||||
return {"ok": False, "content": "", "error": str(e)[:300]}
|
||||
|
||||
|
||||
async def _handle_llm_request(ws, payload: dict) -> None:
|
||||
req_id = payload.get("requestId", "")
|
||||
messages = payload.get("messages") or []
|
||||
if not isinstance(messages, list) or not messages:
|
||||
await _send(ws, "llm_response", {
|
||||
"requestId": req_id, "ok": False, "error": "leere/ungueltige messages",
|
||||
})
|
||||
return
|
||||
max_tokens = int(payload.get("max_tokens", 512) or 512)
|
||||
temperature = float(payload.get("temperature", 0.7) or 0.7)
|
||||
stop = payload.get("stop")
|
||||
t0 = time.time()
|
||||
res = await _call_llama(messages, max_tokens=max_tokens,
|
||||
temperature=temperature, stop=stop)
|
||||
dt = time.time() - t0
|
||||
logger.info("llm_request id=%s -> ok=%s %.2fs content_len=%d",
|
||||
(req_id[:8] if req_id else "?"), res.get("ok"), dt,
|
||||
len(res.get("content") or ""))
|
||||
await _send(ws, "llm_response", {
|
||||
"requestId": req_id,
|
||||
"ok": res.get("ok", False),
|
||||
"content": res.get("content", ""),
|
||||
"error": res.get("error"),
|
||||
"model": LLM_MODEL,
|
||||
"elapsedMs": int(dt * 1000),
|
||||
})
|
||||
|
||||
|
||||
async def _run() -> None:
|
||||
if not RVS_HOST:
|
||||
logger.error("RVS_HOST nicht gesetzt — Abbruch")
|
||||
return
|
||||
if not RVS_TOKEN:
|
||||
logger.error("RVS_TOKEN nicht gesetzt — Abbruch")
|
||||
return
|
||||
|
||||
use_tls = RVS_TLS
|
||||
retry_s = 2
|
||||
tls_fallback_tried = False
|
||||
|
||||
while True:
|
||||
scheme = "wss" if use_tls else "ws"
|
||||
url = f"{scheme}://{RVS_HOST}:{RVS_PORT}/ws?token={RVS_TOKEN}"
|
||||
masked = url.replace(RVS_TOKEN, "***") if RVS_TOKEN else url
|
||||
try:
|
||||
logger.info("Verbinde zu RVS: %s (llama=%s)", masked, LLAMA_URL)
|
||||
async with websockets.connect(
|
||||
url, ping_interval=20, ping_timeout=10, max_size=16 * 1024 * 1024
|
||||
) as ws:
|
||||
logger.info("RVS verbunden — llm-adapter online")
|
||||
retry_s = 2
|
||||
tls_fallback_tried = False
|
||||
async for raw in ws:
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if msg.get("type") != "llm_request":
|
||||
continue
|
||||
payload = msg.get("payload", {}) or {}
|
||||
# Jede Anfrage nebenlaeufig — llama.cpp serialisiert intern,
|
||||
# aber wir blockieren so nicht den Empfang weiterer Messages.
|
||||
asyncio.create_task(_handle_llm_request(ws, payload))
|
||||
except Exception as e:
|
||||
logger.warning("RVS-Verbindung verloren/fehlgeschlagen: %s", e)
|
||||
if use_tls and RVS_TLS_FALLBACK and not tls_fallback_tried:
|
||||
tls_fallback_tried = True
|
||||
use_tls = False
|
||||
logger.info("TLS fehlgeschlagen — Fallback auf ws://")
|
||||
continue
|
||||
await asyncio.sleep(min(retry_s, 30))
|
||||
retry_s = min(retry_s * 2, 30)
|
||||
use_tls = RVS_TLS
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_run())
|
||||
@@ -0,0 +1,2 @@
|
||||
websockets>=12.0
|
||||
httpx>=0.27.0
|
||||
Reference in New Issue
Block a user