From 8ae20a9bd8950b456f9632bc0dcf368da16d664a Mon Sep 17 00:00:00 2001 From: duffyduck Date: Sat, 11 Jul 2026 14:11:59 +0200 Subject: [PATCH] =?UTF-8?q?feat(local-llm):=20B0.5=20=E2=80=94=20llama-swa?= =?UTF-8?q?p=20+=20lokale=20Modellauswahl=20in=20Diagnostic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mehrere lokale Modelle, on-demand geladen/geswappt, in Diagnostic waehlbar. Design: das Brain schickt den Modellnamen (aus local_llm.json) im llm_request mit -> Adapter -> llama-swap laedt/swappt. Keine separate Gamebox-Config noetig. - xtts: `llama`-Container -> `llama-swap` (unified-cuda), config.yaml mit qwen3-8b (Standard) + qwen3-4b; Auto-Download via -hf, Cache /models geteilt (qwen3-8b schon da). Adapter -> llama-swap:8080, Timeout 600s (Erst-Download). - adapter: `model` aus dem Request an llama-swap durchreichen (Fallback env). - brain: router.load_config liest localLlmModel; local_llm_chat(model=...); agent gibt cfg-Modell mit; bridge reicht model durch (_local_llm + Route). - diagnostic: /api/local-models-list (aus /shared/config/local_models.json, seeded), local-llm-config um localLlmModel erweitert; Dropdown "Lokales Modell" im Settings-Block + Erst-Download-Hinweis. BLIND gebaut (Gamebox nicht testbar hier): llama-swap CLI/Config-Pfad beim ersten Start via `docker logs aria-llama-swap` pruefen. Live-Lade-Status (Adapter->Diagnostic) ist B0.5-2 (Folgeschritt). Co-Authored-By: Claude Opus 4.8 --- aria-brain/agent.py | 4 ++- aria-brain/local_llm.py | 11 +++++--- aria-brain/router.py | 6 ++++- bridge/aria_bridge.py | 12 ++++++--- diagnostic/index.html | 51 +++++++++++++++++++++++++++++++++++-- diagnostic/server.js | 28 +++++++++++++++++++- xtts/docker-compose.yml | 44 +++++++++++++++----------------- xtts/llama-swap/config.yaml | 42 ++++++++++++++++++++++++++++++ xtts/llm-adapter/adapter.py | 22 +++++++++------- 9 files changed, 174 insertions(+), 46 deletions(-) create mode 100644 xtts/llama-swap/config.yaml diff --git a/aria-brain/agent.py b/aria-brain/agent.py index ca4e95b..9ab1293 100644 --- a/aria-brain/agent.py +++ b/aria-brain/agent.py @@ -1171,6 +1171,7 @@ class Agent: if not router_mod.should_try_local(user_message, cfg): return None local_only = bool(cfg.get("localOnly")) + local_model = cfg.get("localLlmModel") or "qwen3-8b" # B0.5: llama-swap-Key tools = self._build_local_tools() # B1b: kuratierte Tools sys_prompt = router_mod.build_local_system_prompt(IDENTITY_ANCHOR, @@ -1185,7 +1186,8 @@ class Agent: # Spotify aufrufen. Ergebnisse zurueck, bis es final (ohne tool_calls) antwortet. final = "" for _ in range(self._LOCAL_TOOL_ITERATIONS): - res = local_llm_chat(messages, max_tokens=500, temperature=0.5, tools=tools) + res = local_llm_chat(messages, max_tokens=500, temperature=0.5, + tools=tools, model=local_model) if not res.get("ok"): logger.info("[router] lokal fehlgeschlagen (%s) — %s", res.get("error"), "kein Fallback (localOnly)" if local_only else "→ Claude") diff --git a/aria-brain/local_llm.py b/aria-brain/local_llm.py index 205a514..b5df701 100644 --- a/aria-brain/local_llm.py +++ b/aria-brain/local_llm.py @@ -29,11 +29,12 @@ LOCAL_LLM_HTTP_TIMEOUT_SEC = float(os.environ.get("LOCAL_LLM_HTTP_TIMEOUT_SEC", def local_llm_chat(messages: list, *, max_tokens: int = 512, - temperature: float = 0.7, stop=None, tools=None) -> dict: + temperature: float = 0.7, stop=None, tools=None, + model=None) -> dict: """Ein Chat-Call ans lokale LLM. messages = [{role, content}, ...]. - tools (B1b): optionale OpenAI-Tool-Defs; das Ergebnis kann dann - result['tool_calls'] enthalten. Blockierend (urllib) — chat() laeuft - ohnehin im Executor-Thread.""" + model (B0.5): welches Modell llama-swap laden soll. tools (B1b): optionale + OpenAI-Tool-Defs; das Ergebnis kann dann result['tool_calls'] enthalten. + Blockierend (urllib) — chat() laeuft ohnehin im Executor-Thread.""" if not isinstance(messages, list) or not messages: return {"ok": False, "error": "messages leer/ungueltig"} req = {"messages": messages, "max_tokens": max_tokens, "temperature": temperature} @@ -41,6 +42,8 @@ def local_llm_chat(messages: list, *, max_tokens: int = 512, req["stop"] = stop if tools: req["tools"] = tools + if model: + req["model"] = model try: body = json.dumps(req).encode("utf-8") http_req = urllib.request.Request( diff --git a/aria-brain/router.py b/aria-brain/router.py index c77dca4..4085f2f 100644 --- a/aria-brain/router.py +++ b/aria-brain/router.py @@ -30,7 +30,8 @@ CONFIG_PATH = os.environ.get("LOCAL_LLM_CONFIG", "/shared/config/local_llm.json" ESCALATE_MARKER = "<>" -DEFAULT_CONFIG = {"enabled": False, "localOnly": False, "toolVariant": "slim"} +DEFAULT_CONFIG = {"enabled": False, "localOnly": False, + "toolVariant": "slim", "localLlmModel": "qwen3-8b"} def load_config() -> dict: @@ -42,6 +43,9 @@ def load_config() -> dict: "enabled": bool(data.get("enabled", False)), "localOnly": bool(data.get("localOnly", False)), "toolVariant": data.get("toolVariant", "slim") or "slim", + # Welches lokale Modell llama-swap laden soll (B0.5). Muss zu einem + # Key in xtts/llama-swap/config.yaml passen. + "localLlmModel": (data.get("localLlmModel") or "qwen3-8b").strip(), } except (FileNotFoundError, json.JSONDecodeError): return dict(DEFAULT_CONFIG) diff --git a/bridge/aria_bridge.py b/bridge/aria_bridge.py index a0559e9..778efb2 100644 --- a/bridge/aria_bridge.py +++ b/bridge/aria_bridge.py @@ -3375,7 +3375,8 @@ class ARIABridge: _LLM_TIMEOUT_S = 30.0 async def _local_llm(self, messages: list, max_tokens: int = 512, - temperature: float = 0.7, stop=None, tools=None) -> dict: + temperature: float = 0.7, stop=None, tools=None, + model=None) -> dict: """Schickt einen llm_request an den llm-adapter (Gamebox), wartet auf llm_response. tools (B1b) werden durchgereicht; tool_calls kommen zurueck. Rueckgabe: {ok, content, tool_calls, model, elapsedMs} oder {ok:False, error}.""" @@ -3399,8 +3400,10 @@ class ARIABridge: req_payload["stop"] = stop if tools: req_payload["tools"] = tools - logger.info("[rvs] llm_request → llm-adapter (id=%s, msgs=%d, max_tokens=%d, tools=%d)", - request_id[:8], len(messages), max_tokens, len(tools) if tools else 0) + if model: + req_payload["model"] = model + logger.info("[rvs] llm_request → llm-adapter (id=%s, msgs=%d, max_tokens=%d, tools=%d, model=%s)", + request_id[:8], len(messages), max_tokens, len(tools) if tools else 0, model or "-") ok = await self._send_to_rvs({ "type": "llm_request", "payload": req_payload, @@ -3898,10 +3901,11 @@ class ARIABridge: except (TypeError, ValueError): temperature = 0.7 _tools = data.get("tools") if isinstance(data.get("tools"), list) else None + _model = data.get("model") if isinstance(data.get("model"), str) else None result = await self._local_llm( messages=messages, max_tokens=max_tokens, temperature=temperature, stop=data.get("stop"), - tools=_tools, + tools=_tools, model=_model, ) status = 200 if result.get("ok") else 502 await _send_response(writer, status, result) diff --git a/diagnostic/index.html b/diagnostic/index.html index 310ff62..d7a576c 100644 --- a/diagnostic/index.html +++ b/diagnostic/index.html @@ -956,6 +956,21 @@
Aktueller Stand (B1a): das lokale Modell plaudert nur — Werkzeuge macht noch Claude. Lokale Tools kommen mit B1b. + +
+ Lokales Modell: + + +
+
+
+ Beim ersten Wechsel zu einem Modell lädt die Gamebox das GGUF (mehrere GB) — + die erste Antwort dauert dann länger, danach ist es gecacht. + Liste kommt aus /shared/config/local_models.json (Keys = xtts/llama-swap/config.yaml). +
+
@@ -1571,8 +1586,8 @@ try { loadBrainStatus(); } catch {} // Sprachmodell-Dropdown befuellen (kuratierte Tier-Liste vom Proxy) try { loadModelList(); } catch {} - // Lokales-LLM-Schalter aus /shared/config/local_llm.json laden - try { loadLocalLlmConfig(); } catch {} + // Lokales-LLM: erst Modell-Liste (Dropdown), dann Config (Auswahl setzen) + try { loadLocalModelList().then(() => loadLocalLlmConfig()); } catch {} }; // Brain-Status periodisch refreshen damit die Card live bleibt @@ -6254,6 +6269,29 @@ el.style.color = '#4ADE80'; } } + let _localModelsCache = []; + let _currentLocalModel = 'qwen3-8b'; + function updateLocalModelDesc() { + const el = document.getElementById('local-llm-model-desc'); + const sel = document.getElementById('local-llm-model'); + if (!el || !sel) return; + const m = _localModelsCache.find(x => x.id === sel.value); + el.textContent = m && m.description ? m.description : ''; + } + async function loadLocalModelList() { + try { + const r = await fetch('/api/local-models-list'); + const j = await r.json(); + _localModelsCache = (j && j.models) || []; + } catch (e) { _localModelsCache = []; } + const sel = document.getElementById('local-llm-model'); + if (sel) { + sel.innerHTML = _localModelsCache.map(m => + ``).join('') || ''; + if (_localModelsCache.some(m => m.id === _currentLocalModel)) sel.value = _currentLocalModel; + updateLocalModelDesc(); + } + } async function loadLocalLlmConfig() { try { const r = await fetch('/api/local-llm-config'); @@ -6264,15 +6302,24 @@ if (en) en.checked = !!c.enabled; if (ol) ol.checked = !!c.localOnly; if (tv) tv.value = (c.toolVariant === 'full') ? 'full' : 'slim'; + _currentLocalModel = c.localLlmModel || 'qwen3-8b'; + const sel = document.getElementById('local-llm-model'); + if (sel && _localModelsCache.some(m => m.id === _currentLocalModel)) { + sel.value = _currentLocalModel; + updateLocalModelDesc(); + } setLocalLlmStatus(c); } catch (e) { /* still */ } } async function saveLocalLlmConfig() { + const modelSel = document.getElementById('local-llm-model'); const body = { enabled: document.getElementById('local-llm-enabled').checked, localOnly: document.getElementById('local-llm-onlylocal').checked, toolVariant: document.getElementById('local-llm-toolvariant').value, + localLlmModel: (modelSel && modelSel.value) || '', }; + updateLocalModelDesc(); try { const r = await fetch('/api/local-llm-config', { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/diagnostic/server.js b/diagnostic/server.js index 4208270..7122741 100644 --- a/diagnostic/server.js +++ b/diagnostic/server.js @@ -309,9 +309,10 @@ function readLocalLlmConfig() { enabled: !!p.enabled, localOnly: !!p.localOnly, toolVariant: p.toolVariant === "full" ? "full" : "slim", + localLlmModel: (typeof p.localLlmModel === "string" && p.localLlmModel) ? p.localLlmModel : "qwen3-8b", }; } catch { - return { enabled: false, localOnly: false, toolVariant: "slim" }; + return { enabled: false, localOnly: false, toolVariant: "slim", localLlmModel: "qwen3-8b" }; } } function writeLocalLlmConfig(patch) { @@ -319,6 +320,7 @@ function writeLocalLlmConfig(patch) { if (typeof patch.enabled === "boolean") cur.enabled = patch.enabled; if (typeof patch.localOnly === "boolean") cur.localOnly = patch.localOnly; if (patch.toolVariant === "slim" || patch.toolVariant === "full") cur.toolVariant = patch.toolVariant; + if (typeof patch.localLlmModel === "string" && patch.localLlmModel.trim()) cur.localLlmModel = patch.localLlmModel.trim(); fs.mkdirSync("/shared/config", { recursive: true }); const tmp = LOCAL_LLM_CONFIG_FILE + ".tmp"; fs.writeFileSync(tmp, JSON.stringify(cur, null, 2)); @@ -326,6 +328,27 @@ function writeLocalLlmConfig(patch) { return cur; } +// ── Lokale Modell-Liste (Diagnostic-Dropdown) ──────────────── +// /shared/config/local_models.json — kuratierte Liste; muss zu den KEYS in +// xtts/llama-swap/config.yaml passen. Wird bei Bedarf mit Defaults seeded. +const LOCAL_MODELS_FILE = "/shared/config/local_models.json"; +const DEFAULT_LOCAL_MODELS = [ + { id: "qwen3-8b", display_name: "Qwen3 8B (Standard)", description: "Bestes Tool-Calling, ~6 GB. Passt auf 12 GB." }, + { id: "qwen3-4b", display_name: "Qwen3 4B (schneller)", description: "Kleiner + flotter, ~3 GB. Etwas schwaecher." }, +]; +function loadLocalModels() { + try { + const arr = JSON.parse(fs.readFileSync(LOCAL_MODELS_FILE, "utf-8")); + if (Array.isArray(arr) && arr.length && arr.every(m => m && typeof m.id === "string")) return arr; + } catch {} + // Seed defaults + try { + fs.mkdirSync("/shared/config", { recursive: true }); + fs.writeFileSync(LOCAL_MODELS_FILE, JSON.stringify(DEFAULT_LOCAL_MODELS, null, 2)); + } catch {} + return DEFAULT_LOCAL_MODELS; +} + // ── File-Project-Manifest ─────────────────────────────────────────── // Jeder Eintrag map[absoluter_pfad] = project_id (leer = Hauptchat). // Wird vom files-list-Endpoint + files-set-project gepflegt. @@ -1602,6 +1625,9 @@ const server = http.createServer((req, res) => { } }); return; + } else if (req.url === "/api/local-models-list" && req.method === "GET") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, models: loadLocalModels() })); } else if (req.url === "/api/local-llm-config" && req.method === "GET") { res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(readLocalLlmConfig())); diff --git a/xtts/docker-compose.yml b/xtts/docker-compose.yml index 64bcf34..c339b91 100644 --- a/xtts/docker-compose.yml +++ b/xtts/docker-compose.yml @@ -90,22 +90,19 @@ services: # Container-Restarts. restart: unless-stopped - # ─── Lokales LLM (Plan B, B0) — llama.cpp-Server (GPU) ──────── - # Serviert Qwen3-8B (GGUF Q4_K_M) OpenAI-kompatibel auf :8081, NUR im - # Compose-Netz (kein RVS direkt) — die Bruecke macht der llm-adapter. + # ─── Lokales LLM (Plan B, B0.5) — llama-swap (GPU) ──────────── + # llama-swap laedt/swappt mehrere Modelle on-demand (nur eins passt gleich- + # zeitig in die 12 GB). Welches geladen wird, bestimmt das `model`-Feld im + # Request — das Brain schickt es aus local_llm.json mit. Erster Load eines + # Modells zieht das GGUF via -hf von HF (Cache unter /models, persistent). + # OpenAI-kompatibel auf :8080, nur im Compose-Netz; die Bruecke macht der + # llm-adapter. Modell-Liste: ./llama-swap/config.yaml. # - # AUTO-DOWNLOAD: llama.cpp zieht das GGUF beim ersten Start selbst von - # Hugging Face (-hf :) und cached es unter /models (persistent - # via Bind-Mount -> kein Re-Download bei Restart). Kein manuelles Ablegen - # noetig. Modell wechseln = LLM_HF_REPO/LLM_HF_QUANT in der .env aendern + - # Container neu. (Alternativ lokale Datei: command auf -m /models/x.gguf.) - # - # VRAM auf der RTX 3060 (12 GB): whisper-small (~1-2) + f5tts (~1-2) + - # qwen3-8b-q4 (~6) ~= 9-10 GB. Passt, aber knapp — bei OOM: LLM_CTX kleiner - # oder Quant auf Q4_K_S/IQ4_XS wechseln. - llama: - image: ghcr.io/ggml-org/llama.cpp:server-cuda - container_name: aria-llama + # BLIND GEBAUT (kein Gamebox-Test hier): beim ersten Start + # `docker logs -f aria-llama-swap` pruefen. Image bundelt llama-server. + llama-swap: + image: ghcr.io/mostlygeek/llama-swap:unified-cuda + container_name: aria-llama-swap deploy: resources: reservations: @@ -114,13 +111,11 @@ services: count: 1 capabilities: [gpu] volumes: - - ./models:/models # HF-Download-Cache (persistent) + - ./models:/models # HF-Download-Cache (persistent) + - ./llama-swap/config.yaml:/app/config.yaml:ro # Modell-Liste environment: - - LLAMA_CACHE=/models # llama.cpp legt -hf-Downloads hier ab - command: > - -hf ${LLM_HF_REPO:-Qwen/Qwen3-8B-GGUF}:${LLM_HF_QUANT:-Q4_K_M} - --host 0.0.0.0 --port 8081 - -ngl 99 -c ${LLM_CTX:-8192} --jinja + - LLAMA_CACHE=/models # llama-server legt -hf-Downloads hier ab + command: ["--config", "/app/config.yaml", "--listen", "0.0.0.0:8080"] restart: unless-stopped # ─── Local-LLM-Adapter — RVS <-> llama.cpp (Plan B, B0) ────── @@ -130,14 +125,15 @@ services: build: ./llm-adapter container_name: aria-llm-adapter depends_on: - - llama + - llama-swap environment: - RVS_HOST=${RVS_HOST} - RVS_PORT=${RVS_PORT:-443} - RVS_TLS=${RVS_TLS:-true} - RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true} - RVS_TOKEN=${RVS_TOKEN} - - LLAMA_URL=http://llama:8081 + - LLAMA_URL=http://llama-swap:8080 - LLM_MODEL=${LLM_MODEL:-qwen3-8b} - - LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-60} + # Erster Load eines Modells kann ein GGUF ziehen (mehrere GB) — grosszuegig. + - LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600} restart: unless-stopped diff --git a/xtts/llama-swap/config.yaml b/xtts/llama-swap/config.yaml new file mode 100644 index 0000000..0d4bd21 --- /dev/null +++ b/xtts/llama-swap/config.yaml @@ -0,0 +1,42 @@ +# llama-swap Modell-Liste fuer ARIA (Plan B, B0.5). +# Welches Modell geladen wird, bestimmt das `model`-Feld im Request (das Brain +# schickt es aus /shared/config/local_llm.json mit). llama-swap laedt es +# on-demand, swappt bei Bedarf (nur eins passt gleichzeitig in die 12 GB). +# Erster Load zieht das GGUF via -hf von Hugging Face (Cache unter /models). +# +# Die Modell-KEYS hier muessen zu local_models.json (Diagnostic-Dropdown) passen. +# +# healthCheckTimeout: Sekunden, die llama-swap auf "Modell bereit" wartet. +# GROSSZUEGIG, weil der erste Load ein GGUF (mehrere GB) herunterlaedt. Wenn der +# erste Download laenger dauert und abbricht: hier hochsetzen. +healthCheckTimeout: 1800 + +models: + # Standard — Qwen3 8B (~6 GB Q4). Bestes Tool-Calling, passt auf 12 GB. + "qwen3-8b": + cmd: | + llama-server --port ${PORT} --host 127.0.0.1 + -hf Qwen/Qwen3-8B-GGUF:Q4_K_M + -ngl 99 -c 8192 --jinja + ttl: 3600 # nach 1h Idle entladen (VRAM freigeben) + + # Kleiner + schneller — Qwen3 4B (~3 GB). Fuer noch flottere Antworten, + # etwas schwaecher. Guter A/B-Vergleich gegen 8B. + "qwen3-4b": + cmd: | + llama-server --port ${PORT} --host 127.0.0.1 + -hf Qwen/Qwen3-4B-GGUF:Q4_K_M + -ngl 99 -c 8192 --jinja + ttl: 3600 + + # ── Vorlagen fuer spaeter (auskommentiert; brauchen mehr VRAM / 2. Karte) ── + # "qwen3-14b": + # cmd: | + # llama-server --port ${PORT} --host 127.0.0.1 + # -hf Qwen/Qwen3-14B-GGUF:Q4_K_M -ngl 99 -c 8192 --jinja + # ttl: 3600 + # "mistral-small-3": + # cmd: | + # llama-server --port ${PORT} --host 127.0.0.1 + # -hf :Q4_K_M -ngl 99 -c 8192 --jinja + # ttl: 3600 diff --git a/xtts/llm-adapter/adapter.py b/xtts/llm-adapter/adapter.py index e69f5e6..1da8fc7 100644 --- a/xtts/llm-adapter/adapter.py +++ b/xtts/llm-adapter/adapter.py @@ -69,14 +69,16 @@ async def _send(ws, mtype: str, payload: dict) -> None: async def _call_llama(messages: list, *, max_tokens: int, temperature: float, - stop, tools=None) -> dict: - """Ruft llama.cpp /v1/chat/completions (OpenAI-Format). Gibt + stop, tools=None, model=None) -> dict: + """Ruft llama.cpp/llama-swap /v1/chat/completions (OpenAI-Format). Gibt {ok, content, tool_calls, error} zurueck — wirft nie. - tools: optionale OpenAI-Tool-Definitionen (B1b). llama.cpp (--jinja) mit - Qwen3 kann natives Tool-Calling und liefert dann message.tool_calls.""" + model: welches Modell llama-swap laden soll (B0.5). Kommt aus dem Request + (Brain -> local_llm.json). Faellt auf LLM_MODEL (env) zurueck. + tools: optionale OpenAI-Tool-Definitionen (B1b). Qwen3 (--jinja) kann + natives Tool-Calling und liefert dann message.tool_calls.""" body = { - "model": LLM_MODEL, + "model": model or LLM_MODEL, "messages": messages, "max_tokens": max_tokens, "temperature": temperature, @@ -120,13 +122,15 @@ async def _handle_llm_request(ws, payload: dict) -> None: temperature = float(payload.get("temperature", 0.7) or 0.7) stop = payload.get("stop") tools = payload.get("tools") or None + model = (payload.get("model") or "").strip() or None t0 = time.time() res = await _call_llama(messages, max_tokens=max_tokens, - temperature=temperature, stop=stop, tools=tools) + temperature=temperature, stop=stop, tools=tools, + model=model) dt = time.time() - t0 tc = res.get("tool_calls") - logger.info("llm_request id=%s -> ok=%s %.2fs content_len=%d tool_calls=%d", - (req_id[:8] if req_id else "?"), res.get("ok"), dt, + logger.info("llm_request id=%s model=%s -> ok=%s %.2fs content_len=%d tool_calls=%d", + (req_id[:8] if req_id else "?"), model or LLM_MODEL, res.get("ok"), dt, len(res.get("content") or ""), len(tc) if tc else 0) await _send(ws, "llm_response", { "requestId": req_id, @@ -134,7 +138,7 @@ async def _handle_llm_request(ws, payload: dict) -> None: "content": res.get("content", ""), "tool_calls": tc, "error": res.get("error"), - "model": LLM_MODEL, + "model": model or LLM_MODEL, "elapsedMs": int(dt * 1000), })