feat(llm): Modell-Download + HuggingFace-Katalog (Stage D)
Neue lokale GGUF-Modelle per Knopf auf eine Box laden — ohne Image-Rebuild. - llm-adapter besitzt jetzt llama-swaps Config: generiert /models/llama-swap.config.yaml aus Basis-Template (xtts/llama-swap/config.yaml) + persistenter Registry /models/aria_models.json. Neue RVS-Handler llm_provision_model / llm_remove_model (targetInstance-gefiltert): Registry+ Config schreiben, llama-swap-Reload anstossen, neu announcen, Warmup (zieht das GGUF via -hf, Fortschritt via service_status loading→ready). pyyaml ergaenzt. - compose: llama-swap liest --config /models/llama-swap.config.yaml; llm-adapter mountet ./models (rw) + ./llama-swap (ro Template). - diagnostic/server.js: /shared/config/llm_catalog.json (kuratierte GGUF-Liste) + GET /api/llm-catalog + POST /api/llm-catalog/refresh (HuggingFace-API-Merge); Actions llm_provision_model / llm_remove_model / llm_test; llm_provision_result an Browser durchgereicht. - diagnostic/index.html: "Modell-Katalog"-Card (HF-Refresh, Ziel-Box waehlen, Laden, Verfuegbarkeit) + Test-Chat-Zeile ans lokale LLM (Antwort + Latenz). Download nutzt llama-swaps vorhandenen -hf-Pfad (kein neuer Download-Code). Reload ist der einzige Deploy-Verify-Punkt (llama-swap-Image); Fallback Box-up. Deploy: diagnostic neu bauen (VM) + llm-Boxen neu bauen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+119
-1
@@ -609,9 +609,37 @@
|
||||
<!-- LLM-Flotte: welche Box faehrt welche Modelle (live via RVS) -->
|
||||
<div id="local-llm-fleet" style="font-size:11px;color:#8888AA;margin:6px 0 0 0;"></div>
|
||||
|
||||
<!-- Test-Chat: kurze Nachricht direkt ans lokale LLM -->
|
||||
<div style="margin:12px 0 0 0;padding-top:10px;border-top:1px solid #2a2a3a;">
|
||||
<div style="font-size:12px;color:#E0E0F0;margin-bottom:4px;"><strong>Test-Chat</strong> (aktuelles Modell, direkt ans lokale LLM):</div>
|
||||
<div style="display:flex;gap:6px;">
|
||||
<input id="llm-test-input" type="text" placeholder="z. B. Sag kurz Hallo." style="flex:1;background:#1E1E2E;border:1px solid #333;border-radius:4px;padding:6px 8px;color:#E0E0F0;font-family:inherit;font-size:12px;" onkeydown="if(event.key==='Enter')runLlmTest();">
|
||||
<button class="btn secondary" onclick="runLlmTest()" style="padding:4px 12px;font-size:11px;">Senden</button>
|
||||
</div>
|
||||
<div id="llm-test-result" style="font-size:11px;color:#8888AA;margin-top:6px;white-space:pre-wrap;"></div>
|
||||
</div>
|
||||
|
||||
<div id="local-llm-status" style="font-size:11px;color:#6a6a88;margin-top:8px;padding-top:8px;border-top:1px solid #2a2a3a;min-height:14px;"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Modell-Katalog: GGUF-Modelle herunterladen/aktivieren (Stage D) -->
|
||||
<div class="settings-section">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;">
|
||||
<h2 style="margin:0;">Modell-Katalog</h2>
|
||||
<button class="btn secondary" onclick="refreshLlmCatalog()" style="padding:4px 10px;font-size:11px;">Von HuggingFace aktualisieren</button>
|
||||
</div>
|
||||
<div class="card" style="max-width:720px;">
|
||||
<p style="color:#8888AA;font-size:12px;margin:0 0 8px;">
|
||||
Gaengige lokale GGUF-Modelle. „Laden" schickt das Modell an die gewaehlte
|
||||
LLM-Box — llama-swap zieht das GGUF beim ersten Mal (mehrere GB) und meldet
|
||||
den Fortschritt. Danach ist es im Modell-Dropdown oben waehlbar.
|
||||
<span style="color:#FFD60A;">Achte auf den VRAM der Box (Groesse je Modell).</span>
|
||||
</p>
|
||||
<div id="llm-catalog-list" style="font-size:12px;color:#8888AA;">(lade Katalog…)</div>
|
||||
<div id="llm-catalog-status" style="font-size:11px;color:#6a6a88;margin-top:6px;min-height:14px;"></div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Externe KI-Anbieter (Platzhalter) -->
|
||||
<div class="settings-section">
|
||||
<h2>Externe Anbieter (OpenRouter & Co)</h2>
|
||||
@@ -2010,7 +2038,27 @@
|
||||
}
|
||||
|
||||
if (msg.type === 'sat_update') { satellites = msg.satellites || []; renderSatellites(); return; }
|
||||
if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); if (typeof refreshLocalLlmModelChoices === 'function') refreshLocalLlmModelChoices(); return; }
|
||||
if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); if (typeof refreshLocalLlmModelChoices === 'function') refreshLocalLlmModelChoices(); if (typeof renderLlmCatalog === 'function') renderLlmCatalog(); return; }
|
||||
if (msg.type === 'llm_response') {
|
||||
const p = msg.payload || {};
|
||||
const out = document.getElementById('llm-test-result');
|
||||
if (out) {
|
||||
const ms = window._llmTestStart ? (Date.now() - window._llmTestStart) : (p.elapsedMs || 0);
|
||||
if (p.ok) { out.textContent = (p.content || '(leer)') + '\n— ' + (p.model || '') + ' · ' + ms + ' ms'; out.style.color = '#E0E0F0'; }
|
||||
else { out.textContent = '✗ ' + (p.error || 'Fehler'); out.style.color = '#FF6B6B'; }
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'llm_provision_result') {
|
||||
const p = msg.payload || {};
|
||||
const st = document.getElementById('llm-catalog-status');
|
||||
if (st) {
|
||||
if (p.removed) { st.textContent = `✓ '${p.key}' entfernt`; st.style.color = '#8888AA'; }
|
||||
else if (p.ok) { st.textContent = `✓ '${p.key}' geladen/aktiv auf ${p.instanceId || '?'}`; st.style.color = '#3FFF3F'; }
|
||||
else { st.textContent = `✗ '${p.key}': ${p.error || 'Fehler'}`; st.style.color = '#FF6B6B'; }
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'sat_devices') {
|
||||
if (msg.satellite) { satDevices[msg.satellite] = { devices: msg.devices || [], location: msg.location, ts: Date.now() }; }
|
||||
satScanning = null;
|
||||
@@ -2026,6 +2074,7 @@
|
||||
alert('Es war kein Fingerprint vorhanden.');
|
||||
}
|
||||
refreshVoiceIdStatus();
|
||||
loadLlmCatalog();
|
||||
switchSettingsTab(localStorage.getItem('diag_settings_subtab') || 'models');
|
||||
return;
|
||||
}
|
||||
@@ -6928,6 +6977,75 @@
|
||||
} catch (e) { /* still */ }
|
||||
}
|
||||
|
||||
// ── Modell-Katalog (Stage D) ────────────────────────────
|
||||
let _llmCatalog = [];
|
||||
async function loadLlmCatalog() {
|
||||
try {
|
||||
const r = await fetch('/api/llm-catalog');
|
||||
const j = await r.json();
|
||||
_llmCatalog = (j && j.models) || [];
|
||||
} catch (e) { _llmCatalog = []; }
|
||||
renderLlmCatalog();
|
||||
}
|
||||
async function refreshLlmCatalog() {
|
||||
const st = document.getElementById('llm-catalog-status');
|
||||
if (st) { st.textContent = 'Aktualisiere von HuggingFace…'; st.style.color = '#8888AA'; }
|
||||
try {
|
||||
const r = await fetch('/api/llm-catalog/refresh', { method: 'POST' });
|
||||
const j = await r.json();
|
||||
_llmCatalog = (j && j.models) || _llmCatalog;
|
||||
renderLlmCatalog();
|
||||
if (st) { st.textContent = j.ok ? `✓ ${j.added || 0} neue Modelle von HuggingFace` : `✗ ${j.error || 'Fehler'}`; st.style.color = j.ok ? '#3FFF3F' : '#FF6B6B'; }
|
||||
} catch (e) { if (st) { st.textContent = '✗ ' + e.message; st.style.color = '#FF6B6B'; } }
|
||||
}
|
||||
// online LLM-Boxen (fuer die Ziel-Auswahl + Verfuegbarkeit)
|
||||
function onlineLlmBoxes() { return workers.filter(w => w.service === 'llm' && w.online); }
|
||||
function boxesServing(id) { return onlineLlmBoxes().filter(w => (w.models || []).includes(id) || w.model === id); }
|
||||
function renderLlmCatalog() {
|
||||
const box = document.getElementById('llm-catalog-list');
|
||||
if (!box) return;
|
||||
if (!_llmCatalog.length) { box.innerHTML = '<span style="color:#6a6a88;">Katalog leer.</span>'; return; }
|
||||
const boxes = onlineLlmBoxes();
|
||||
box.innerHTML = _llmCatalog.map((m, i) => {
|
||||
const have = boxesServing(m.id);
|
||||
const haveTxt = have.length ? `<span style="color:#3FFF3F;">✓ auf ${have.map(b => escapeHtml(b.node)).join(', ')}</span>` : '<span style="color:#6a6a88;">nicht geladen</span>';
|
||||
const sizeTxt = m.sizeGB ? ` · ~${m.sizeGB} GB` : '';
|
||||
const opts = boxes.length
|
||||
? boxes.map(b => `<option value="${escapeHtml(b.instanceId)}">${escapeHtml(b.node)}</option>`).join('')
|
||||
: '<option value="">(keine Box online)</option>';
|
||||
return '<div style="display:flex;align-items:center;gap:8px;padding:5px 0;border-bottom:1px solid #1E1E2E;flex-wrap:wrap;">' +
|
||||
'<span style="min-width:150px;"><b>' + escapeHtml(m.id) + '</b>' + sizeTxt + '</span>' +
|
||||
'<span style="color:#8888AA;flex:1;min-width:160px;">' + escapeHtml(m.description || m.hfRepo || '') + '</span>' +
|
||||
haveTxt +
|
||||
'<select id="llm-cat-box-' + i + '" style="background:#1E1E2E;border:1px solid #333;border-radius:4px;padding:3px 6px;color:#E0E0F0;font-size:11px;">' + opts + '</select>' +
|
||||
'<button class="btn secondary" ' + (boxes.length ? '' : 'disabled') + ' onclick="provisionModel(' + i + ')" style="padding:3px 10px;font-size:11px;">Laden</button>' +
|
||||
'</div>';
|
||||
}).join('');
|
||||
}
|
||||
function provisionModel(i) {
|
||||
const m = _llmCatalog[i];
|
||||
if (!m) return;
|
||||
const sel = document.getElementById('llm-cat-box-' + i);
|
||||
const target = sel && sel.value;
|
||||
if (!target) return;
|
||||
const st = document.getElementById('llm-catalog-status');
|
||||
if (st) { st.textContent = `Lade '${m.id}' auf ${target}… (GGUF-Download kann dauern)`; st.style.color = '#FFD60A'; }
|
||||
send({ action: 'llm_provision_model', targetInstance: target, key: m.id, hfRepo: m.hfRepo, quant: m.quant, ctx: m.ctx });
|
||||
}
|
||||
function runLlmTest() {
|
||||
const inp = document.getElementById('llm-test-input');
|
||||
const out = document.getElementById('llm-test-result');
|
||||
const text = (inp && inp.value || '').trim();
|
||||
if (!text) return;
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
const model = (sel && sel.value) || _currentLocalModel || '';
|
||||
const serving = boxesServing(model);
|
||||
const target = serving.length ? serving[0].instanceId : '';
|
||||
if (out) { out.textContent = '… sende an ' + (target || '(Broadcast)') + ' (' + model + ')'; out.style.color = '#8888AA'; }
|
||||
window._llmTestStart = Date.now();
|
||||
send({ action: 'llm_test', text, model, targetInstance: target });
|
||||
}
|
||||
|
||||
// ── Einstellungen: OpenClaw Config ──────────────────────
|
||||
|
||||
// loadOpenClawConfig entfernt — aria-core ist raus.
|
||||
|
||||
@@ -349,6 +349,66 @@ function loadLocalModels() {
|
||||
return DEFAULT_LOCAL_MODELS;
|
||||
}
|
||||
|
||||
// ── LLM-Modell-Katalog (Stage D): herunterladbare GGUF-Modelle ───────
|
||||
// /shared/config/llm_catalog.json — kuratierte Liste guter GGUF-Modelle plus
|
||||
// per HuggingFace-Refresh nachgeladene. Der llm-adapter zieht ein Modell via
|
||||
// -hf beim ersten Load. { id(key), hfRepo, quant, sizeGB, description, source }.
|
||||
const LLM_CATALOG_FILE = "/shared/config/llm_catalog.json";
|
||||
const DEFAULT_LLM_CATALOG = [
|
||||
{ id: "qwen3-8b", hfRepo: "Qwen/Qwen3-8B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 6, description: "Bestes Tool-Calling, passt auf 12 GB.", source: "curated" },
|
||||
{ id: "qwen3-4b", hfRepo: "Qwen/Qwen3-4B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 3, description: "Kleiner + flotter, etwas schwaecher.", source: "curated" },
|
||||
{ id: "qwen3-14b", hfRepo: "Qwen/Qwen3-14B-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 10, description: "Staerker, braucht mehr VRAM (~16 GB).", source: "curated" },
|
||||
{ id: "llama-3.1-8b", hfRepo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 5, description: "Llama 3.1 8B Instruct.", source: "curated" },
|
||||
{ id: "mistral-small-3", hfRepo: "bartowski/Mistral-Small-24B-Instruct-2501-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 14, description: "Mistral Small 24B — stark, viel VRAM.", source: "curated" },
|
||||
{ id: "gemma-2-9b", hfRepo: "bartowski/gemma-2-9b-it-GGUF", quant: "Q4_K_M", ctx: 8192, sizeGB: 6, description: "Google Gemma 2 9B Instruct.", source: "curated" },
|
||||
];
|
||||
function loadLlmCatalog() {
|
||||
try {
|
||||
const arr = JSON.parse(fs.readFileSync(LLM_CATALOG_FILE, "utf-8"));
|
||||
if (Array.isArray(arr) && arr.length && arr.every(m => m && typeof m.id === "string")) return arr;
|
||||
} catch {}
|
||||
try {
|
||||
fs.mkdirSync("/shared/config", { recursive: true });
|
||||
fs.writeFileSync(LLM_CATALOG_FILE, JSON.stringify(DEFAULT_LLM_CATALOG, null, 2));
|
||||
} catch {}
|
||||
return DEFAULT_LLM_CATALOG;
|
||||
}
|
||||
function saveLlmCatalog(arr) {
|
||||
try {
|
||||
fs.mkdirSync("/shared/config", { recursive: true });
|
||||
const tmp = LLM_CATALOG_FILE + ".tmp";
|
||||
fs.writeFileSync(tmp, JSON.stringify(arr, null, 2));
|
||||
fs.renameSync(tmp, LLM_CATALOG_FILE);
|
||||
return true;
|
||||
} catch (e) { log("warn", "llm", `Katalog speichern fehlgeschlagen: ${e.message}`); return false; }
|
||||
}
|
||||
function slugModelId(repo) {
|
||||
return String(repo).toLowerCase().replace(/^.*\//, "").replace(/-gguf$/,"").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "model";
|
||||
}
|
||||
// Holt populaere GGUF-Modelle von der HuggingFace-API und merged sie in den
|
||||
// Katalog (kuratierte Eintraege + Beschreibungen bleiben erhalten).
|
||||
async function refreshLlmCatalogFromHF() {
|
||||
const url = "https://huggingface.co/api/models?search=GGUF&sort=downloads&direction=-1&limit=40";
|
||||
const r = await fetch(url, { headers: { "User-Agent": "aria-diagnostic" } });
|
||||
if (!r.ok) throw new Error(`HF API ${r.status}`);
|
||||
const list = await r.json();
|
||||
const existing = loadLlmCatalog();
|
||||
const byId = new Map(existing.map(m => [m.id, m]));
|
||||
let added = 0;
|
||||
for (const m of (Array.isArray(list) ? list : [])) {
|
||||
const repo = m.id || m.modelId;
|
||||
if (!repo || !/gguf/i.test(repo)) continue;
|
||||
const id = slugModelId(repo);
|
||||
if (byId.has(id)) continue; // kuratierte/vorhandene nicht ueberschreiben
|
||||
const entry = { id, hfRepo: repo, quant: "Q4_K_M", ctx: 8192, sizeGB: 0,
|
||||
description: `HuggingFace · ${(m.downloads || 0).toLocaleString("de")} Downloads`, source: "hf" };
|
||||
byId.set(id, entry); added++;
|
||||
}
|
||||
const merged = Array.from(byId.values());
|
||||
saveLlmCatalog(merged);
|
||||
return { models: merged, added };
|
||||
}
|
||||
|
||||
// ── File-Project-Manifest ───────────────────────────────────────────
|
||||
// Jeder Eintrag map[absoluter_pfad] = project_id (leer = Hauptchat).
|
||||
// Wird vom files-list-Endpoint + files-set-project gepflegt.
|
||||
@@ -1125,6 +1185,9 @@ function connectRVS(forcePlain) {
|
||||
log("info", "rvs", `service_status ${svc} ${state}${model ? ` (${model})` : ""}`);
|
||||
}
|
||||
broadcast({ type: "service_status", payload: msg.payload });
|
||||
} else if (msg.type === "llm_provision_result") {
|
||||
// Ergebnis eines Modell-Downloads/Aktivierens → an Browser (Katalog-Status).
|
||||
broadcast({ type: "llm_provision_result", payload: msg.payload || {} });
|
||||
} else if (msg.type === "audio_pcm" && msg.payload && _previewPending.size > 0) {
|
||||
// PCM-Chunks einer laufenden Voice-Preview — sammeln + WAV bauen
|
||||
_handlePreviewChunk(msg.payload);
|
||||
@@ -1867,6 +1930,20 @@ const server = http.createServer((req, res) => {
|
||||
} else if (req.url === "/api/local-models-list" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true, models: loadLocalModels() }));
|
||||
} else if (req.url === "/api/llm-catalog" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true, models: loadLlmCatalog() }));
|
||||
} else if (req.url === "/api/llm-catalog/refresh" && req.method === "POST") {
|
||||
refreshLlmCatalogFromHF()
|
||||
.then(r => {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true, models: r.models, added: r.added }));
|
||||
log("info", "llm", `LLM-Katalog von HuggingFace aktualisiert: +${r.added} Modelle`);
|
||||
})
|
||||
.catch(err => {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: false, error: err.message, models: loadLlmCatalog() }));
|
||||
});
|
||||
} else if (req.url === "/api/local-llm-config" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(readLocalLlmConfig()));
|
||||
@@ -2838,6 +2915,27 @@ wss.on("connection", (ws) => {
|
||||
// Sessions- und Brain-File-Viewer entfernt — Sessions sind raus, Memory
|
||||
// laeuft jetzt komplett ueber die Vector-DB im aria-brain (siehe Gehirn-Tab).
|
||||
// restart_session kommt weiter rein, weil der Watchdog ihn manchmal triggert.
|
||||
} else if (msg.action === "llm_provision_model") {
|
||||
// Modell auf eine bestimmte LLM-Box laden/aktivieren (Stage D).
|
||||
sendToRVS_raw({ type: "llm_provision_model", payload: {
|
||||
targetInstance: msg.targetInstance || "",
|
||||
key: msg.key, hfRepo: msg.hfRepo, quant: msg.quant, ctx: msg.ctx, ngl: msg.ngl,
|
||||
}, timestamp: Date.now() });
|
||||
log("info", "llm", `provision '${msg.key}' (${msg.hfRepo}) → ${msg.targetInstance || "?"}`);
|
||||
} else if (msg.action === "llm_remove_model") {
|
||||
sendToRVS_raw({ type: "llm_remove_model", payload: {
|
||||
targetInstance: msg.targetInstance || "", key: msg.key }, timestamp: Date.now() });
|
||||
log("info", "llm", `remove '${msg.key}' → ${msg.targetInstance || "?"}`);
|
||||
} else if (msg.action === "llm_test") {
|
||||
// Test-Chat: kurze Nachricht direkt ans lokale LLM (llm_request/llm_response).
|
||||
const reqId = "diagtest_" + Date.now();
|
||||
sendToRVS_withResponse("llm_request", {
|
||||
requestId: reqId,
|
||||
messages: [{ role: "user", content: String(msg.text || "Sag kurz Hallo.") }],
|
||||
max_tokens: 256, temperature: 0.5,
|
||||
model: msg.model || "", targetInstance: msg.targetInstance || "",
|
||||
}, "llm_response", ws);
|
||||
log("info", "llm", `Test-Chat → ${msg.model || "?"} @ ${msg.targetInstance || "(broadcast)"}`);
|
||||
} else if (msg.action === "restart_session") {
|
||||
handleRestartSession(ws);
|
||||
// ── Einstellungen ──
|
||||
|
||||
+14
-5
@@ -123,31 +123,37 @@ services:
|
||||
# bestimmt das `model`-Feld im Request (Brain schickt es aus local_llm.json).
|
||||
# Erster Load zieht das GGUF via -hf von HF (Cache unter /models, persistent).
|
||||
# OpenAI-kompatibel auf :8080, nur im Compose-Netz; die Bruecke macht der
|
||||
# llm-adapter. Modell-Liste: ./llama-swap/config.yaml.
|
||||
# llm-adapter. Die Modell-Liste erzeugt der llm-adapter dynamisch aus
|
||||
# ./llama-swap/config.yaml (Basis) + Registry → /models/llama-swap.config.yaml.
|
||||
llama-swap:
|
||||
image: ghcr.io/mostlygeek/llama-swap:unified-cuda
|
||||
container_name: aria-llama-swap
|
||||
profiles: ["llm"] # startet nur mit COMPOSE_PROFILES=…llm…
|
||||
runtime: nvidia
|
||||
volumes:
|
||||
- ./models:/models # HF-Download-Cache (persistent)
|
||||
- ./llama-swap/config.yaml:/app/config.yaml:ro # Modell-Liste
|
||||
- ./models:/models # HF-Cache + generierte Config
|
||||
environment:
|
||||
- NVIDIA_VISIBLE_DEVICES=${LLM_GPU:-0}
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- LLAMA_CACHE=/models # llama-server legt -hf-Downloads hier ab
|
||||
command: ["--config", "/app/config.yaml", "--listen", "0.0.0.0:8080"]
|
||||
# Liest die vom llm-adapter generierte Config. Beim allerersten Boot faengt
|
||||
# restart: unless-stopped die Reihenfolge ab, bis der Adapter sie geschrieben hat.
|
||||
command: ["--config", "/models/llama-swap.config.yaml", "--listen", "0.0.0.0:8080"]
|
||||
restart: unless-stopped
|
||||
|
||||
# ─── Local-LLM-Adapter — RVS <-> llama.cpp ────
|
||||
# Verbindet sich per Token an den RVS (wie f5tts/whisper), nimmt llm_request
|
||||
# entgegen, ruft llama.cpp lokal, antwortet llm_response.
|
||||
# entgegen, ruft llama.cpp lokal, antwortet llm_response. Verwaltet ausserdem
|
||||
# llama-swaps Config (Modelle hinzufuegen/entfernen via llm_provision_model).
|
||||
llm-adapter:
|
||||
build: ./llm-adapter
|
||||
container_name: aria-llm-adapter
|
||||
profiles: ["llm"]
|
||||
depends_on:
|
||||
- llama-swap
|
||||
volumes:
|
||||
- ./models:/models # generierte Config + Registry + Cache
|
||||
- ./llama-swap:/llamaswap:ro # Basis-Template (config.yaml)
|
||||
environment:
|
||||
- NODE_NAME=${NODE_NAME:-node}
|
||||
- RVS_HOST=${RVS_HOST}
|
||||
@@ -157,6 +163,9 @@ services:
|
||||
- RVS_TOKEN=${RVS_TOKEN}
|
||||
- LLAMA_URL=http://llama-swap:8080
|
||||
- LLM_MODEL=${LLM_MODEL:-qwen3-8b}
|
||||
- LLAMA_BASE_CONFIG=/llamaswap/config.yaml
|
||||
- LLAMA_GEN_CONFIG=/models/llama-swap.config.yaml
|
||||
- LLM_REGISTRY=/models/aria_models.json
|
||||
# Erster Load eines Modells kann ein GGUF ziehen (mehrere GB) — grosszuegig.
|
||||
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600}
|
||||
restart: unless-stopped
|
||||
|
||||
+164
-11
@@ -65,6 +65,87 @@ _inflight = 0 # laufende llm_requests (busy-Report im ping)
|
||||
# empfindlich reagiert: LLM_DISABLE_THINKING=false setzen.
|
||||
LLM_DISABLE_THINKING = os.getenv("LLM_DISABLE_THINKING", "true").lower() == "true"
|
||||
|
||||
# ── Modell-Verwaltung (Stage D): Adapter besitzt llama-swaps Config ──
|
||||
# llama-swap liest die GENERIERTE Config (beschreibbar, im /models-Bind). Wir
|
||||
# erzeugen sie aus dem Basis-Template (kuratierte Defaults) + der persistenten
|
||||
# Box-Registry (per Diagnostic hinzugefuegte Modelle). So werden neue Modelle
|
||||
# ohne Image-Rebuild waehlbar.
|
||||
import yaml # pyyaml
|
||||
BASE_CONFIG_PATH = os.getenv("LLAMA_BASE_CONFIG", "/llamaswap/config.yaml")
|
||||
GEN_CONFIG_PATH = os.getenv("LLAMA_GEN_CONFIG", "/models/llama-swap.config.yaml")
|
||||
REGISTRY_PATH = os.getenv("LLM_REGISTRY", "/models/aria_models.json")
|
||||
|
||||
|
||||
def _load_registry() -> list:
|
||||
try:
|
||||
with open(REGISTRY_PATH) as f:
|
||||
data = json.load(f)
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _save_registry(reg: list) -> None:
|
||||
try:
|
||||
tmp = REGISTRY_PATH + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(reg, f, indent=2)
|
||||
os.replace(tmp, REGISTRY_PATH)
|
||||
except Exception as e:
|
||||
logger.warning("Registry speichern fehlgeschlagen: %s", e)
|
||||
|
||||
|
||||
def _generate_config() -> int:
|
||||
"""Schreibt die llama-swap-Config aus Basis-Template + Registry. Gibt die
|
||||
Anzahl Modelle zurueck. Idempotent, bei jeder Aenderung + beim Start."""
|
||||
base = {}
|
||||
try:
|
||||
with open(BASE_CONFIG_PATH) as f:
|
||||
base = yaml.safe_load(f) or {}
|
||||
except Exception as e:
|
||||
logger.warning("Basis-Template %s nicht lesbar (%s)", BASE_CONFIG_PATH, e)
|
||||
models = dict(base.get("models") or {})
|
||||
for e in _load_registry():
|
||||
key = (e.get("key") or "").strip()
|
||||
repo = (e.get("hfRepo") or "").strip()
|
||||
if not key or not repo:
|
||||
continue
|
||||
quant = (e.get("quant") or "Q4_K_M").strip()
|
||||
ctx = int(e.get("ctx") or 8192)
|
||||
ngl = int(e.get("ngl") or 99)
|
||||
models[key] = {
|
||||
"cmd": (f"llama-server --port ${{PORT}} --host 127.0.0.1\n"
|
||||
f"-hf {repo}:{quant}\n-ngl {ngl} -c {ctx} --jinja"),
|
||||
"ttl": 3600,
|
||||
}
|
||||
out = dict(base)
|
||||
out["models"] = models
|
||||
try:
|
||||
os.makedirs(os.path.dirname(GEN_CONFIG_PATH), exist_ok=True)
|
||||
tmp = GEN_CONFIG_PATH + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
yaml.safe_dump(out, f, sort_keys=False, default_flow_style=False)
|
||||
os.replace(tmp, GEN_CONFIG_PATH)
|
||||
logger.info("llama-swap-Config generiert: %d Modelle → %s", len(models), GEN_CONFIG_PATH)
|
||||
except Exception as e:
|
||||
logger.error("Config schreiben fehlgeschlagen: %s", e)
|
||||
return len(models)
|
||||
|
||||
|
||||
async def _reload_llama() -> None:
|
||||
"""Stoesst llama-swap-Reload an. Viele Builds watchen die Config-Datei ohnehin;
|
||||
zusaetzlich versuchen wir bekannte Reload-Endpunkte (Fehler ignoriert)."""
|
||||
for path in ("/api/config/reload", "/reload"):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as c:
|
||||
r = await c.post(f"{LLAMA_URL}{path}")
|
||||
if r.status_code < 400:
|
||||
logger.info("llama-swap reload via %s", path)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
logger.info("llama-swap reload: kein Endpoint — verlasse mich auf File-Watch")
|
||||
|
||||
|
||||
async def _send(ws, mtype: str, payload: dict) -> None:
|
||||
try:
|
||||
@@ -149,17 +230,23 @@ async def _fetch_available_models() -> list:
|
||||
return [LLM_MODEL]
|
||||
|
||||
|
||||
async def _announce(ws) -> None:
|
||||
"""Sendet ein frisches worker_hello mit der aktuellen Modell-Liste (nach
|
||||
Provision/Remove aufrufen, damit Bridge+Diagnostic das neue Modell lernen)."""
|
||||
models = await _fetch_available_models()
|
||||
await _send(ws, "worker_hello", {
|
||||
"instanceId": INSTANCE_ID, "service": WORKER_SERVICE,
|
||||
"node": NODE_NAME, "gpus": GPU_IDS, "model": LLM_MODEL,
|
||||
"models": models, # welche Modelle diese Box fahren kann (llama-swap-Keys)
|
||||
})
|
||||
logger.info("worker_hello: models=%s", models)
|
||||
|
||||
|
||||
async def _worker_register(ws) -> None:
|
||||
"""Meldet diesen Worker bei der aria-bridge an (worker_hello) und haelt die
|
||||
Flotten-Registry per periodischem worker_ping (mit busy-Status) frisch."""
|
||||
try:
|
||||
models = await _fetch_available_models()
|
||||
await _send(ws, "worker_hello", {
|
||||
"instanceId": INSTANCE_ID, "service": WORKER_SERVICE,
|
||||
"node": NODE_NAME, "gpus": GPU_IDS, "model": LLM_MODEL,
|
||||
"models": models, # welche Modelle diese Box fahren kann (llama-swap-Keys)
|
||||
})
|
||||
logger.info("worker_hello: models=%s", models)
|
||||
await _announce(ws)
|
||||
while True:
|
||||
await asyncio.sleep(WORKER_PING_INTERVAL_S)
|
||||
await _send(ws, "worker_ping",
|
||||
@@ -232,6 +319,61 @@ async def _do_llm_request(ws, payload: dict) -> None:
|
||||
})
|
||||
|
||||
|
||||
async def _handle_provision(ws, payload: dict) -> None:
|
||||
"""Fuegt ein Modell hinzu: Registry+Config schreiben, reload, dann Warmup
|
||||
(zieht das GGUF via -hf beim ersten Load). Meldet die neue Modell-Liste."""
|
||||
key = (payload.get("key") or "").strip()
|
||||
repo = (payload.get("hfRepo") or "").strip()
|
||||
if not key or not repo:
|
||||
await _send(ws, "llm_provision_result",
|
||||
{"instanceId": INSTANCE_ID, "key": key, "ok": False, "error": "key/hfRepo fehlt"})
|
||||
return
|
||||
entry = {
|
||||
"key": key, "hfRepo": repo,
|
||||
"quant": (payload.get("quant") or "Q4_K_M").strip(),
|
||||
"ctx": int(payload.get("ctx") or 8192),
|
||||
"ngl": int(payload.get("ngl") or 99),
|
||||
}
|
||||
reg = [e for e in _load_registry() if e.get("key") != key]
|
||||
reg.append(entry)
|
||||
_save_registry(reg)
|
||||
_generate_config()
|
||||
await _reload_llama()
|
||||
await _announce(ws) # Bridge/Diagnostic lernen das neue Modell
|
||||
# Warmup: Mini-Request → llama-swap laedt/zieht das Modell (Fortschritt via
|
||||
# service_status loading→ready, freshlyDownloaded).
|
||||
await _emit_llm_status(ws, "loading", key)
|
||||
t0 = time.time()
|
||||
res = await _call_llama([{"role": "user", "content": "hi"}],
|
||||
max_tokens=1, temperature=0.0, stop=None, model=key)
|
||||
dt = time.time() - t0
|
||||
if res.get("ok"):
|
||||
_ready_models.add(key)
|
||||
await _emit_llm_status(ws, "ready", key, loadSeconds=round(dt, 1),
|
||||
freshlyDownloaded=dt > 25)
|
||||
else:
|
||||
await _emit_llm_status(ws, "error", key, error=(res.get("error") or "")[:160])
|
||||
await _send(ws, "llm_provision_result",
|
||||
{"instanceId": INSTANCE_ID, "key": key, "ok": res.get("ok", False),
|
||||
"error": res.get("error"), "elapsedMs": int(dt * 1000)})
|
||||
logger.info("provision %s (%s) → ok=%s %.1fs", key, repo, res.get("ok"), dt)
|
||||
|
||||
|
||||
async def _handle_remove(ws, payload: dict) -> None:
|
||||
"""Entfernt ein Modell aus Registry+Config (GGUF bleibt im Cache)."""
|
||||
key = (payload.get("key") or "").strip()
|
||||
if not key:
|
||||
return
|
||||
reg = [e for e in _load_registry() if e.get("key") != key]
|
||||
_save_registry(reg)
|
||||
_generate_config()
|
||||
await _reload_llama()
|
||||
await _announce(ws)
|
||||
await _send(ws, "llm_provision_result",
|
||||
{"instanceId": INSTANCE_ID, "key": key, "ok": True, "removed": True})
|
||||
logger.info("removed model %s", key)
|
||||
|
||||
|
||||
async def _run() -> None:
|
||||
if not RVS_HOST:
|
||||
logger.error("RVS_HOST nicht gesetzt — Abbruch")
|
||||
@@ -240,6 +382,11 @@ async def _run() -> None:
|
||||
logger.error("RVS_TOKEN nicht gesetzt — Abbruch")
|
||||
return
|
||||
|
||||
# llama-swap-Config aus Basis-Template + Registry erzeugen, BEVOR llama-swap
|
||||
# sie braucht (llama-swap restart: unless-stopped faengt die Erst-Boot-
|
||||
# Reihenfolge ab, falls es kurz vor uns startet).
|
||||
_generate_config()
|
||||
|
||||
use_tls = RVS_TLS
|
||||
retry_s = 2
|
||||
tls_fallback_tried = False
|
||||
@@ -262,7 +409,8 @@ async def _run() -> None:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if msg.get("type") != "llm_request":
|
||||
mtype = msg.get("type")
|
||||
if mtype not in ("llm_request", "llm_provision_model", "llm_remove_model"):
|
||||
continue
|
||||
payload = msg.get("payload", {}) or {}
|
||||
# Redundanz-Routing: gezielt an eine andere Instanz adressiert
|
||||
@@ -270,9 +418,14 @@ async def _run() -> None:
|
||||
tgt = payload.get("targetInstance")
|
||||
if tgt and tgt != INSTANCE_ID:
|
||||
continue
|
||||
# Jede Anfrage nebenlaeufig — llama.cpp serialisiert intern,
|
||||
# aber wir blockieren so nicht den Empfang weiterer Messages.
|
||||
asyncio.create_task(_handle_llm_request(ws, payload))
|
||||
if mtype == "llm_provision_model":
|
||||
asyncio.create_task(_handle_provision(ws, payload))
|
||||
elif mtype == "llm_remove_model":
|
||||
asyncio.create_task(_handle_remove(ws, payload))
|
||||
else:
|
||||
# Jede Anfrage nebenlaeufig — llama.cpp serialisiert intern,
|
||||
# aber wir blockieren so nicht den Empfang weiterer Messages.
|
||||
asyncio.create_task(_handle_llm_request(ws, payload))
|
||||
except Exception as e:
|
||||
logger.warning("RVS-Verbindung verloren/fehlgeschlagen: %s", e)
|
||||
try:
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
websockets>=12.0
|
||||
httpx>=0.27.0
|
||||
pyyaml>=6.0
|
||||
|
||||
Reference in New Issue
Block a user