diff --git a/diagnostic/index.html b/diagnostic/index.html index 61d269c..3fd420f 100644 --- a/diagnostic/index.html +++ b/diagnostic/index.html @@ -609,9 +609,37 @@
+ +
+
Test-Chat (aktuelles Modell, direkt ans lokale LLM):
+
+ + +
+
+
+
+ + +
+
+

Modell-Katalog

+ +
+
+

+ Gaengige lokale GGUF-Modelle. „Laden" schickt das Modell an die gewaehlte + LLM-Box — llama-swap zieht das GGUF beim ersten Mal (mehrere GB) und meldet + den Fortschritt. Danach ist es im Modell-Dropdown oben waehlbar. + Achte auf den VRAM der Box (Groesse je Modell). +

+
(lade Katalog…)
+
+
+

Externe Anbieter (OpenRouter & Co)

@@ -2010,7 +2038,27 @@ } if (msg.type === 'sat_update') { satellites = msg.satellites || []; renderSatellites(); return; } - if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); if (typeof refreshLocalLlmModelChoices === 'function') refreshLocalLlmModelChoices(); return; } + if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); if (typeof refreshLocalLlmModelChoices === 'function') refreshLocalLlmModelChoices(); if (typeof renderLlmCatalog === 'function') renderLlmCatalog(); return; } + if (msg.type === 'llm_response') { + const p = msg.payload || {}; + const out = document.getElementById('llm-test-result'); + if (out) { + const ms = window._llmTestStart ? (Date.now() - window._llmTestStart) : (p.elapsedMs || 0); + if (p.ok) { out.textContent = (p.content || '(leer)') + '\n— ' + (p.model || '') + ' · ' + ms + ' ms'; out.style.color = '#E0E0F0'; } + else { out.textContent = '✗ ' + (p.error || 'Fehler'); out.style.color = '#FF6B6B'; } + } + return; + } + if (msg.type === 'llm_provision_result') { + const p = msg.payload || {}; + const st = document.getElementById('llm-catalog-status'); + if (st) { + if (p.removed) { st.textContent = `✓ '${p.key}' entfernt`; st.style.color = '#8888AA'; } + else if (p.ok) { st.textContent = `✓ '${p.key}' geladen/aktiv auf ${p.instanceId || '?'}`; st.style.color = '#3FFF3F'; } + else { st.textContent = `✗ '${p.key}': ${p.error || 'Fehler'}`; st.style.color = '#FF6B6B'; } + } + return; + } if (msg.type === 'sat_devices') { if (msg.satellite) { satDevices[msg.satellite] = { devices: msg.devices || [], location: msg.location, ts: Date.now() }; } satScanning = null; @@ -2026,6 +2074,7 @@ alert('Es war kein Fingerprint vorhanden.'); } refreshVoiceIdStatus(); + loadLlmCatalog(); switchSettingsTab(localStorage.getItem('diag_settings_subtab') || 'models'); return; } @@ -6928,6 +6977,75 @@ } catch (e) { /* still */ } } + // ── Modell-Katalog (Stage D) ──────────────────────────── + let _llmCatalog = []; + async function loadLlmCatalog() { + try { + const r = await fetch('/api/llm-catalog'); + const j = await r.json(); + _llmCatalog = (j && j.models) || []; + } catch (e) { _llmCatalog = []; } + renderLlmCatalog(); + } + async function refreshLlmCatalog() { + const st = document.getElementById('llm-catalog-status'); + if (st) { st.textContent = 'Aktualisiere von HuggingFace…'; st.style.color = '#8888AA'; } + try { + const r = await fetch('/api/llm-catalog/refresh', { method: 'POST' }); + const j = await r.json(); + _llmCatalog = (j && j.models) || _llmCatalog; + renderLlmCatalog(); + if (st) { st.textContent = j.ok ? `✓ ${j.added || 0} neue Modelle von HuggingFace` : `✗ ${j.error || 'Fehler'}`; st.style.color = j.ok ? '#3FFF3F' : '#FF6B6B'; } + } catch (e) { if (st) { st.textContent = '✗ ' + e.message; st.style.color = '#FF6B6B'; } } + } + // online LLM-Boxen (fuer die Ziel-Auswahl + Verfuegbarkeit) + function onlineLlmBoxes() { return workers.filter(w => w.service === 'llm' && w.online); } + function boxesServing(id) { return onlineLlmBoxes().filter(w => (w.models || []).includes(id) || w.model === id); } + function renderLlmCatalog() { + const box = document.getElementById('llm-catalog-list'); + if (!box) return; + if (!_llmCatalog.length) { box.innerHTML = 'Katalog leer.'; return; } + const boxes = onlineLlmBoxes(); + box.innerHTML = _llmCatalog.map((m, i) => { + const have = boxesServing(m.id); + const haveTxt = have.length ? `✓ auf ${have.map(b => escapeHtml(b.node)).join(', ')}` : 'nicht geladen'; + const sizeTxt = m.sizeGB ? ` · ~${m.sizeGB} GB` : ''; + const opts = boxes.length + ? boxes.map(b => ``).join('') + : ''; + return '
' + + '' + escapeHtml(m.id) + '' + sizeTxt + '' + + '' + escapeHtml(m.description || m.hfRepo || '') + '' + + haveTxt + + '' + + '' + + '
'; + }).join(''); + } + function provisionModel(i) { + const m = _llmCatalog[i]; + if (!m) return; + const sel = document.getElementById('llm-cat-box-' + i); + const target = sel && sel.value; + if (!target) return; + const st = document.getElementById('llm-catalog-status'); + if (st) { st.textContent = `Lade '${m.id}' auf ${target}… (GGUF-Download kann dauern)`; st.style.color = '#FFD60A'; } + send({ action: 'llm_provision_model', targetInstance: target, key: m.id, hfRepo: m.hfRepo, quant: m.quant, ctx: m.ctx }); + } + function runLlmTest() { + const inp = document.getElementById('llm-test-input'); + const out = document.getElementById('llm-test-result'); + const text = (inp && inp.value || '').trim(); + if (!text) return; + const sel = document.getElementById('local-llm-model'); + const model = (sel && sel.value) || _currentLocalModel || ''; + const serving = boxesServing(model); + const target = serving.length ? serving[0].instanceId : ''; + if (out) { out.textContent = '… sende an ' + (target || '(Broadcast)') + ' (' + model + ')'; out.style.color = '#8888AA'; } + window._llmTestStart = Date.now(); + send({ action: 'llm_test', text, model, targetInstance: target }); + } + // ── Einstellungen: OpenClaw Config ────────────────────── // loadOpenClawConfig entfernt — aria-core ist raus. diff --git a/diagnostic/server.js b/diagnostic/server.js index 8e05e3d..6a8be8c 100644 --- a/diagnostic/server.js +++ b/diagnostic/server.js @@ -349,6 +349,66 @@ function loadLocalModels() { return DEFAULT_LOCAL_MODELS; } +// ── LLM-Modell-Katalog (Stage D): herunterladbare GGUF-Modelle ─────── +// /shared/config/llm_catalog.json — kuratierte Liste guter GGUF-Modelle plus +// per HuggingFace-Refresh nachgeladene. Der llm-adapter zieht ein Modell via +// -hf beim ersten Load. { id(key), hfRepo, quant, sizeGB, description, source }. +const LLM_CATALOG_FILE = "/shared/config/llm_catalog.json"; +const DEFAULT_LLM_CATALOG = [ + { id: "qwen3-8b", hfRepo: "Qwen/Qwen3-8B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 6, description: "Bestes Tool-Calling, passt auf 12 GB.", source: "curated" }, + { id: "qwen3-4b", hfRepo: "Qwen/Qwen3-4B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 3, description: "Kleiner + flotter, etwas schwaecher.", source: "curated" }, + { id: "qwen3-14b", hfRepo: "Qwen/Qwen3-14B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 10, description: "Staerker, braucht mehr VRAM (~16 GB).", source: "curated" }, + { id: "llama-3.1-8b", hfRepo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 5, description: "Llama 3.1 8B Instruct.", source: "curated" }, + { id: "mistral-small-3", hfRepo: "bartowski/Mistral-Small-24B-Instruct-2501-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 14, description: "Mistral Small 24B — stark, viel VRAM.", source: "curated" }, + { id: "gemma-2-9b", hfRepo: "bartowski/gemma-2-9b-it-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 6, description: "Google Gemma 2 9B Instruct.", source: "curated" }, +]; +function loadLlmCatalog() { + try { + const arr = JSON.parse(fs.readFileSync(LLM_CATALOG_FILE, "utf-8")); + if (Array.isArray(arr) && arr.length && arr.every(m => m && typeof m.id === "string")) return arr; + } catch {} + try { + fs.mkdirSync("/shared/config", { recursive: true }); + fs.writeFileSync(LLM_CATALOG_FILE, JSON.stringify(DEFAULT_LLM_CATALOG, null, 2)); + } catch {} + return DEFAULT_LLM_CATALOG; +} +function saveLlmCatalog(arr) { + try { + fs.mkdirSync("/shared/config", { recursive: true }); + const tmp = LLM_CATALOG_FILE + ".tmp"; + fs.writeFileSync(tmp, JSON.stringify(arr, null, 2)); + fs.renameSync(tmp, LLM_CATALOG_FILE); + return true; + } catch (e) { log("warn", "llm", `Katalog speichern fehlgeschlagen: ${e.message}`); return false; } +} +function slugModelId(repo) { + return String(repo).toLowerCase().replace(/^.*\//, "").replace(/-gguf$/,"").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "model"; +} +// Holt populaere GGUF-Modelle von der HuggingFace-API und merged sie in den +// Katalog (kuratierte Eintraege + Beschreibungen bleiben erhalten). +async function refreshLlmCatalogFromHF() { + const url = "https://huggingface.co/api/models?search=GGUF&sort=downloads&direction=-1&limit=40"; + const r = await fetch(url, { headers: { "User-Agent": "aria-diagnostic" } }); + if (!r.ok) throw new Error(`HF API ${r.status}`); + const list = await r.json(); + const existing = loadLlmCatalog(); + const byId = new Map(existing.map(m => [m.id, m])); + let added = 0; + for (const m of (Array.isArray(list) ? list : [])) { + const repo = m.id || m.modelId; + if (!repo || !/gguf/i.test(repo)) continue; + const id = slugModelId(repo); + if (byId.has(id)) continue; // kuratierte/vorhandene nicht ueberschreiben + const entry = { id, hfRepo: repo, quant: "Q4_K_M", ctx: 8192, sizeGB: 0, + description: `HuggingFace · ${(m.downloads || 0).toLocaleString("de")} Downloads`, source: "hf" }; + byId.set(id, entry); added++; + } + const merged = Array.from(byId.values()); + saveLlmCatalog(merged); + return { models: merged, added }; +} + // ── File-Project-Manifest ─────────────────────────────────────────── // Jeder Eintrag map[absoluter_pfad] = project_id (leer = Hauptchat). // Wird vom files-list-Endpoint + files-set-project gepflegt. @@ -1125,6 +1185,9 @@ function connectRVS(forcePlain) { log("info", "rvs", `service_status ${svc} ${state}${model ? ` (${model})` : ""}`); } broadcast({ type: "service_status", payload: msg.payload }); + } else if (msg.type === "llm_provision_result") { + // Ergebnis eines Modell-Downloads/Aktivierens → an Browser (Katalog-Status). + broadcast({ type: "llm_provision_result", payload: msg.payload || {} }); } else if (msg.type === "audio_pcm" && msg.payload && _previewPending.size > 0) { // PCM-Chunks einer laufenden Voice-Preview — sammeln + WAV bauen _handlePreviewChunk(msg.payload); @@ -1867,6 +1930,20 @@ const server = http.createServer((req, res) => { } else if (req.url === "/api/local-models-list" && req.method === "GET") { res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ ok: true, models: loadLocalModels() })); + } else if (req.url === "/api/llm-catalog" && req.method === "GET") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, models: loadLlmCatalog() })); + } else if (req.url === "/api/llm-catalog/refresh" && req.method === "POST") { + refreshLlmCatalogFromHF() + .then(r => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, models: r.models, added: r.added })); + log("info", "llm", `LLM-Katalog von HuggingFace aktualisiert: +${r.added} Modelle`); + }) + .catch(err => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: err.message, models: loadLlmCatalog() })); + }); } else if (req.url === "/api/local-llm-config" && req.method === "GET") { res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(readLocalLlmConfig())); @@ -2838,6 +2915,27 @@ wss.on("connection", (ws) => { // Sessions- und Brain-File-Viewer entfernt — Sessions sind raus, Memory // laeuft jetzt komplett ueber die Vector-DB im aria-brain (siehe Gehirn-Tab). // restart_session kommt weiter rein, weil der Watchdog ihn manchmal triggert. + } else if (msg.action === "llm_provision_model") { + // Modell auf eine bestimmte LLM-Box laden/aktivieren (Stage D). + sendToRVS_raw({ type: "llm_provision_model", payload: { + targetInstance: msg.targetInstance || "", + key: msg.key, hfRepo: msg.hfRepo, quant: msg.quant, ctx: msg.ctx, ngl: msg.ngl, + }, timestamp: Date.now() }); + log("info", "llm", `provision '${msg.key}' (${msg.hfRepo}) → ${msg.targetInstance || "?"}`); + } else if (msg.action === "llm_remove_model") { + sendToRVS_raw({ type: "llm_remove_model", payload: { + targetInstance: msg.targetInstance || "", key: msg.key }, timestamp: Date.now() }); + log("info", "llm", `remove '${msg.key}' → ${msg.targetInstance || "?"}`); + } else if (msg.action === "llm_test") { + // Test-Chat: kurze Nachricht direkt ans lokale LLM (llm_request/llm_response). + const reqId = "diagtest_" + Date.now(); + sendToRVS_withResponse("llm_request", { + requestId: reqId, + messages: [{ role: "user", content: String(msg.text || "Sag kurz Hallo.") }], + max_tokens: 256, temperature: 0.5, + model: msg.model || "", targetInstance: msg.targetInstance || "", + }, "llm_response", ws); + log("info", "llm", `Test-Chat → ${msg.model || "?"} @ ${msg.targetInstance || "(broadcast)"}`); } else if (msg.action === "restart_session") { handleRestartSession(ws); // ── Einstellungen ── diff --git a/xtts/docker-compose.yml b/xtts/docker-compose.yml index 2d0020a..106e172 100644 --- a/xtts/docker-compose.yml +++ b/xtts/docker-compose.yml @@ -123,31 +123,37 @@ services: # bestimmt das `model`-Feld im Request (Brain schickt es aus local_llm.json). # Erster Load zieht das GGUF via -hf von HF (Cache unter /models, persistent). # OpenAI-kompatibel auf :8080, nur im Compose-Netz; die Bruecke macht der - # llm-adapter. Modell-Liste: ./llama-swap/config.yaml. + # llm-adapter. Die Modell-Liste erzeugt der llm-adapter dynamisch aus + # ./llama-swap/config.yaml (Basis) + Registry → /models/llama-swap.config.yaml. llama-swap: image: ghcr.io/mostlygeek/llama-swap:unified-cuda container_name: aria-llama-swap profiles: ["llm"] # startet nur mit COMPOSE_PROFILES=…llm… runtime: nvidia volumes: - - ./models:/models # HF-Download-Cache (persistent) - - ./llama-swap/config.yaml:/app/config.yaml:ro # Modell-Liste + - ./models:/models # HF-Cache + generierte Config environment: - NVIDIA_VISIBLE_DEVICES=${LLM_GPU:-0} - NVIDIA_DRIVER_CAPABILITIES=compute,utility - LLAMA_CACHE=/models # llama-server legt -hf-Downloads hier ab - command: ["--config", "/app/config.yaml", "--listen", "0.0.0.0:8080"] + # Liest die vom llm-adapter generierte Config. Beim allerersten Boot faengt + # restart: unless-stopped die Reihenfolge ab, bis der Adapter sie geschrieben hat. + command: ["--config", "/models/llama-swap.config.yaml", "--listen", "0.0.0.0:8080"] restart: unless-stopped # ─── Local-LLM-Adapter — RVS <-> llama.cpp ──── # Verbindet sich per Token an den RVS (wie f5tts/whisper), nimmt llm_request - # entgegen, ruft llama.cpp lokal, antwortet llm_response. + # entgegen, ruft llama.cpp lokal, antwortet llm_response. Verwaltet ausserdem + # llama-swaps Config (Modelle hinzufuegen/entfernen via llm_provision_model). llm-adapter: build: ./llm-adapter container_name: aria-llm-adapter profiles: ["llm"] depends_on: - llama-swap + volumes: + - ./models:/models # generierte Config + Registry + Cache + - ./llama-swap:/llamaswap:ro # Basis-Template (config.yaml) environment: - NODE_NAME=${NODE_NAME:-node} - RVS_HOST=${RVS_HOST} @@ -157,6 +163,9 @@ services: - RVS_TOKEN=${RVS_TOKEN} - LLAMA_URL=http://llama-swap:8080 - LLM_MODEL=${LLM_MODEL:-qwen3-8b} + - LLAMA_BASE_CONFIG=/llamaswap/config.yaml + - LLAMA_GEN_CONFIG=/models/llama-swap.config.yaml + - LLM_REGISTRY=/models/aria_models.json # Erster Load eines Modells kann ein GGUF ziehen (mehrere GB) — grosszuegig. - LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600} restart: unless-stopped diff --git a/xtts/llm-adapter/adapter.py b/xtts/llm-adapter/adapter.py index 3728a79..caaaf76 100644 --- a/xtts/llm-adapter/adapter.py +++ b/xtts/llm-adapter/adapter.py @@ -65,6 +65,87 @@ _inflight = 0 # laufende llm_requests (busy-Report im ping) # empfindlich reagiert: LLM_DISABLE_THINKING=false setzen. LLM_DISABLE_THINKING = os.getenv("LLM_DISABLE_THINKING", "true").lower() == "true" +# ── Modell-Verwaltung (Stage D): Adapter besitzt llama-swaps Config ── +# llama-swap liest die GENERIERTE Config (beschreibbar, im /models-Bind). Wir +# erzeugen sie aus dem Basis-Template (kuratierte Defaults) + der persistenten +# Box-Registry (per Diagnostic hinzugefuegte Modelle). So werden neue Modelle +# ohne Image-Rebuild waehlbar. +import yaml # pyyaml +BASE_CONFIG_PATH = os.getenv("LLAMA_BASE_CONFIG", "/llamaswap/config.yaml") +GEN_CONFIG_PATH = os.getenv("LLAMA_GEN_CONFIG", "/models/llama-swap.config.yaml") +REGISTRY_PATH = os.getenv("LLM_REGISTRY", "/models/aria_models.json") + + +def _load_registry() -> list: + try: + with open(REGISTRY_PATH) as f: + data = json.load(f) + return data if isinstance(data, list) else [] + except Exception: + return [] + + +def _save_registry(reg: list) -> None: + try: + tmp = REGISTRY_PATH + ".tmp" + with open(tmp, "w") as f: + json.dump(reg, f, indent=2) + os.replace(tmp, REGISTRY_PATH) + except Exception as e: + logger.warning("Registry speichern fehlgeschlagen: %s", e) + + +def _generate_config() -> int: + """Schreibt die llama-swap-Config aus Basis-Template + Registry. Gibt die + Anzahl Modelle zurueck. Idempotent, bei jeder Aenderung + beim Start.""" + base = {} + try: + with open(BASE_CONFIG_PATH) as f: + base = yaml.safe_load(f) or {} + except Exception as e: + logger.warning("Basis-Template %s nicht lesbar (%s)", BASE_CONFIG_PATH, e) + models = dict(base.get("models") or {}) + for e in _load_registry(): + key = (e.get("key") or "").strip() + repo = (e.get("hfRepo") or "").strip() + if not key or not repo: + continue + quant = (e.get("quant") or "Q4_K_M").strip() + ctx = int(e.get("ctx") or 8192) + ngl = int(e.get("ngl") or 99) + models[key] = { + "cmd": (f"llama-server --port ${{PORT}} --host 127.0.0.1\n" + f"-hf {repo}:{quant}\n-ngl {ngl} -c {ctx} --jinja"), + "ttl": 3600, + } + out = dict(base) + out["models"] = models + try: + os.makedirs(os.path.dirname(GEN_CONFIG_PATH), exist_ok=True) + tmp = GEN_CONFIG_PATH + ".tmp" + with open(tmp, "w") as f: + yaml.safe_dump(out, f, sort_keys=False, default_flow_style=False) + os.replace(tmp, GEN_CONFIG_PATH) + logger.info("llama-swap-Config generiert: %d Modelle → %s", len(models), GEN_CONFIG_PATH) + except Exception as e: + logger.error("Config schreiben fehlgeschlagen: %s", e) + return len(models) + + +async def _reload_llama() -> None: + """Stoesst llama-swap-Reload an. Viele Builds watchen die Config-Datei ohnehin; + zusaetzlich versuchen wir bekannte Reload-Endpunkte (Fehler ignoriert).""" + for path in ("/api/config/reload", "/reload"): + try: + async with httpx.AsyncClient(timeout=10) as c: + r = await c.post(f"{LLAMA_URL}{path}") + if r.status_code < 400: + logger.info("llama-swap reload via %s", path) + return + except Exception: + pass + logger.info("llama-swap reload: kein Endpoint — verlasse mich auf File-Watch") + async def _send(ws, mtype: str, payload: dict) -> None: try: @@ -149,17 +230,23 @@ async def _fetch_available_models() -> list: return [LLM_MODEL] +async def _announce(ws) -> None: + """Sendet ein frisches worker_hello mit der aktuellen Modell-Liste (nach + Provision/Remove aufrufen, damit Bridge+Diagnostic das neue Modell lernen).""" + models = await _fetch_available_models() + await _send(ws, "worker_hello", { + "instanceId": INSTANCE_ID, "service": WORKER_SERVICE, + "node": NODE_NAME, "gpus": GPU_IDS, "model": LLM_MODEL, + "models": models, # welche Modelle diese Box fahren kann (llama-swap-Keys) + }) + logger.info("worker_hello: models=%s", models) + + async def _worker_register(ws) -> None: """Meldet diesen Worker bei der aria-bridge an (worker_hello) und haelt die Flotten-Registry per periodischem worker_ping (mit busy-Status) frisch.""" try: - models = await _fetch_available_models() - await _send(ws, "worker_hello", { - "instanceId": INSTANCE_ID, "service": WORKER_SERVICE, - "node": NODE_NAME, "gpus": GPU_IDS, "model": LLM_MODEL, - "models": models, # welche Modelle diese Box fahren kann (llama-swap-Keys) - }) - logger.info("worker_hello: models=%s", models) + await _announce(ws) while True: await asyncio.sleep(WORKER_PING_INTERVAL_S) await _send(ws, "worker_ping", @@ -232,6 +319,61 @@ async def _do_llm_request(ws, payload: dict) -> None: }) +async def _handle_provision(ws, payload: dict) -> None: + """Fuegt ein Modell hinzu: Registry+Config schreiben, reload, dann Warmup + (zieht das GGUF via -hf beim ersten Load). Meldet die neue Modell-Liste.""" + key = (payload.get("key") or "").strip() + repo = (payload.get("hfRepo") or "").strip() + if not key or not repo: + await _send(ws, "llm_provision_result", + {"instanceId": INSTANCE_ID, "key": key, "ok": False, "error": "key/hfRepo fehlt"}) + return + entry = { + "key": key, "hfRepo": repo, + "quant": (payload.get("quant") or "Q4_K_M").strip(), + "ctx": int(payload.get("ctx") or 8192), + "ngl": int(payload.get("ngl") or 99), + } + reg = [e for e in _load_registry() if e.get("key") != key] + reg.append(entry) + _save_registry(reg) + _generate_config() + await _reload_llama() + await _announce(ws) # Bridge/Diagnostic lernen das neue Modell + # Warmup: Mini-Request → llama-swap laedt/zieht das Modell (Fortschritt via + # service_status loading→ready, freshlyDownloaded). + await _emit_llm_status(ws, "loading", key) + t0 = time.time() + res = await _call_llama([{"role": "user", "content": "hi"}], + max_tokens=1, temperature=0.0, stop=None, model=key) + dt = time.time() - t0 + if res.get("ok"): + _ready_models.add(key) + await _emit_llm_status(ws, "ready", key, loadSeconds=round(dt, 1), + freshlyDownloaded=dt > 25) + else: + await _emit_llm_status(ws, "error", key, error=(res.get("error") or "")[:160]) + await _send(ws, "llm_provision_result", + {"instanceId": INSTANCE_ID, "key": key, "ok": res.get("ok", False), + "error": res.get("error"), "elapsedMs": int(dt * 1000)}) + logger.info("provision %s (%s) → ok=%s %.1fs", key, repo, res.get("ok"), dt) + + +async def _handle_remove(ws, payload: dict) -> None: + """Entfernt ein Modell aus Registry+Config (GGUF bleibt im Cache).""" + key = (payload.get("key") or "").strip() + if not key: + return + reg = [e for e in _load_registry() if e.get("key") != key] + _save_registry(reg) + _generate_config() + await _reload_llama() + await _announce(ws) + await _send(ws, "llm_provision_result", + {"instanceId": INSTANCE_ID, "key": key, "ok": True, "removed": True}) + logger.info("removed model %s", key) + + async def _run() -> None: if not RVS_HOST: logger.error("RVS_HOST nicht gesetzt — Abbruch") @@ -240,6 +382,11 @@ async def _run() -> None: logger.error("RVS_TOKEN nicht gesetzt — Abbruch") return + # llama-swap-Config aus Basis-Template + Registry erzeugen, BEVOR llama-swap + # sie braucht (llama-swap restart: unless-stopped faengt die Erst-Boot- + # Reihenfolge ab, falls es kurz vor uns startet). + _generate_config() + use_tls = RVS_TLS retry_s = 2 tls_fallback_tried = False @@ -262,7 +409,8 @@ async def _run() -> None: msg = json.loads(raw) except Exception: continue - if msg.get("type") != "llm_request": + mtype = msg.get("type") + if mtype not in ("llm_request", "llm_provision_model", "llm_remove_model"): continue payload = msg.get("payload", {}) or {} # Redundanz-Routing: gezielt an eine andere Instanz adressiert @@ -270,9 +418,14 @@ async def _run() -> None: tgt = payload.get("targetInstance") if tgt and tgt != INSTANCE_ID: continue - # Jede Anfrage nebenlaeufig — llama.cpp serialisiert intern, - # aber wir blockieren so nicht den Empfang weiterer Messages. - asyncio.create_task(_handle_llm_request(ws, payload)) + if mtype == "llm_provision_model": + asyncio.create_task(_handle_provision(ws, payload)) + elif mtype == "llm_remove_model": + asyncio.create_task(_handle_remove(ws, payload)) + else: + # Jede Anfrage nebenlaeufig — llama.cpp serialisiert intern, + # aber wir blockieren so nicht den Empfang weiterer Messages. + asyncio.create_task(_handle_llm_request(ws, payload)) except Exception as e: logger.warning("RVS-Verbindung verloren/fehlgeschlagen: %s", e) try: diff --git a/xtts/llm-adapter/requirements.txt b/xtts/llm-adapter/requirements.txt index fd45be9..f0b41c9 100644 --- a/xtts/llm-adapter/requirements.txt +++ b/xtts/llm-adapter/requirements.txt @@ -1,2 +1,3 @@ websockets>=12.0 httpx>=0.27.0 +pyyaml>=6.0