feat(llm): Flotten-Katalog + modell-bewusstes LLM-Routing (Stage C)
ARIA nutzt lokale LLMs auf mehreren ai-boxen; jede Box (llama-swap) kann mehrere Modelle fahren. Auswahl nach MODELL, Box wird automatisch gewaehlt. - xtts/llm-adapter/adapter.py: fragt beim Connect llama-swap GET /v1/models ab und meldet die Modell-Liste in worker_hello (models:[...]), Fallback [LLM_MODEL]. - bridge/aria_bridge.py: Worker-Registry speichert models[]; _pick_worker(service, model=) beruecksichtigt nur Boxen, die das Modell fahren koennen (nachsichtig: keine → None → Broadcast/Claude-Fallback); Round-Robin je service+model verteilt mehrere Projekte auf mehrere Boxen; _local_llm reicht das Modell durch; _worker_list traegt models[]. - diagnostic/server.js: workers-Map + workerList um models[] erweitert. - diagnostic/index.html: Compute-Flotte zeigt bei llm die Modell-Liste; das "Lokales Modell"-Dropdown wird LIVE aus den angemeldeten Boxen gebaut (Vereinigung + kuratierte Namen aus local_models.json, "· N Box(en)"/"offline"), darunter eine kompakte LLM-Box-Liste (Node→Modelle→Health). Speisung aus dem vorhandenen worker_update-Broadcast. Kein Brain-/App-Eingriff. Routing greift nur bei aktivem Lokal-Schalter. Deploy: diagnostic + bridge + llm-Boxen neu bauen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+26
-10
@@ -3544,12 +3544,17 @@ class ARIABridge:
|
||||
iid = (payload.get("instanceId") or "").strip()
|
||||
if iid:
|
||||
prev = self._workers.get(iid, {})
|
||||
_models = payload.get("models")
|
||||
self._workers[iid] = {
|
||||
"instanceId": iid,
|
||||
"service": payload.get("service") or "",
|
||||
"node": payload.get("node") or "",
|
||||
"gpus": payload.get("gpus") or "",
|
||||
"model": payload.get("model") or "",
|
||||
# models: welche Modelle die Box fahren kann (llm/llama-swap).
|
||||
# Fallback auf [model] fuer alte Adapter ohne models-Feld.
|
||||
"models": [m for m in _models if m] if isinstance(_models, list)
|
||||
else ([payload.get("model")] if payload.get("model") else []),
|
||||
"busy": bool(prev.get("busy", False)),
|
||||
"last_seen": time.time(),
|
||||
}
|
||||
@@ -4006,9 +4011,10 @@ class ARIABridge:
|
||||
req_payload["tools"] = tools
|
||||
if model:
|
||||
req_payload["model"] = model
|
||||
# Redundanz/Multitasking: freie llm-Instanz gezielt adressieren; None
|
||||
# → Broadcast wie bisher. Mehrere Instanzen → parallele Turns.
|
||||
llm_target = self._pick_worker("llm")
|
||||
# Redundanz/Multitasking: freie llm-Instanz gezielt adressieren, die
|
||||
# das gewaehlte Modell fahren kann; None → Broadcast wie bisher.
|
||||
# Mehrere Boxen mit demselben Modell → Round-Robin (pro Projekt verteilt).
|
||||
llm_target = self._pick_worker("llm", model=model or None)
|
||||
if llm_target:
|
||||
req_payload["targetInstance"] = llm_target
|
||||
logger.info("[rvs] llm_request → llm-adapter (id=%s, msgs=%d, max_tokens=%d, tools=%d, model=%s, target=%s)",
|
||||
@@ -4767,29 +4773,39 @@ class ARIABridge:
|
||||
out.append({
|
||||
"instanceId": w["instanceId"], "service": w.get("service") or "",
|
||||
"node": w.get("node") or "", "gpus": w.get("gpus") or "",
|
||||
"model": w.get("model") or "", "busy": bool(w.get("busy")),
|
||||
"model": w.get("model") or "", "models": w.get("models") or [],
|
||||
"busy": bool(w.get("busy")),
|
||||
"online": (now - w.get("last_seen", 0)) < self.WORKER_OFFLINE_S,
|
||||
})
|
||||
return out
|
||||
|
||||
def _pick_worker(self, service: str) -> Optional[str]:
|
||||
def _pick_worker(self, service: str, model: Optional[str] = None) -> Optional[str]:
|
||||
"""Waehlt eine online, moeglichst freie Instanz des Diensts (Round-Robin
|
||||
ueber die freien). Gibt die instanceId oder None (keine online). Fuer
|
||||
Stage-3-Routing (targetInstance)."""
|
||||
ueber die freien). Gibt die instanceId oder None. Fuer Stage-3-Routing
|
||||
(targetInstance).
|
||||
|
||||
model: wenn gesetzt (nur llm sinnvoll), kommen nur Boxen in Frage, die das
|
||||
Modell fahren koennen (models-Liste oder legacy model-Feld). Meldet KEINE
|
||||
Box das Modell → None (nachsichtig: Aufrufer faellt auf Broadcast zurueck)."""
|
||||
now = time.time()
|
||||
online = [w for w in self._workers.values()
|
||||
if w.get("service") == service
|
||||
and (now - w.get("last_seen", 0)) < self.WORKER_OFFLINE_S]
|
||||
if model:
|
||||
online = [w for w in online
|
||||
if model in (w.get("models") or [])
|
||||
or w.get("model") == model]
|
||||
if not online:
|
||||
return None
|
||||
free = [w for w in online if not w.get("busy")]
|
||||
pool = free or online # alle busy → trotzdem eine nehmen (least-bad)
|
||||
# Round-Robin: rotierender Zeiger pro Dienst.
|
||||
# Round-Robin: rotierender Zeiger pro Dienst(+Modell).
|
||||
rr_key = f"{service}:{model}" if model else service
|
||||
rr = getattr(self, "_worker_rr", None)
|
||||
if rr is None:
|
||||
rr = self._worker_rr = {}
|
||||
idx = rr.get(service, 0) % len(pool)
|
||||
rr[service] = idx + 1
|
||||
idx = rr.get(rr_key, 0) % len(pool)
|
||||
rr[rr_key] = idx + 1
|
||||
chosen = pool[idx]
|
||||
chosen["busy"] = True # optimistisch, bis der naechste Ping korrigiert
|
||||
return chosen["instanceId"]
|
||||
|
||||
+66
-14
@@ -602,9 +602,13 @@
|
||||
<div style="font-size:10px;color:#FFD60A;margin:0 0 4px 0;line-height:1.5;">
|
||||
Beim ersten Wechsel zu einem Modell lädt die AI-Box das GGUF (mehrere GB) —
|
||||
die <strong>erste Antwort dauert dann länger</strong>, danach ist es gecacht.
|
||||
Liste kommt aus <code>/shared/config/local_models.json</code> (Keys = xtts/llama-swap/config.yaml).
|
||||
Verfügbare Modelle kommen <strong>live</strong> von den angemeldeten LLM-Boxen;
|
||||
Namen aus <code>/shared/config/local_models.json</code>.
|
||||
</div>
|
||||
|
||||
<!-- LLM-Flotte: welche Box faehrt welche Modelle (live via RVS) -->
|
||||
<div id="local-llm-fleet" style="font-size:11px;color:#8888AA;margin:6px 0 0 0;"></div>
|
||||
|
||||
<div id="local-llm-status" style="font-size:11px;color:#6a6a88;margin-top:8px;padding-top:8px;border-top:1px solid #2a2a3a;min-height:14px;"></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -2006,7 +2010,7 @@
|
||||
}
|
||||
|
||||
if (msg.type === 'sat_update') { satellites = msg.satellites || []; renderSatellites(); return; }
|
||||
if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); return; }
|
||||
if (msg.type === 'worker_update') { workers = msg.workers || []; renderWorkers(); if (typeof refreshLocalLlmModelChoices === 'function') refreshLocalLlmModelChoices(); return; }
|
||||
if (msg.type === 'sat_devices') {
|
||||
if (msg.satellite) { satDevices[msg.satellite] = { devices: msg.devices || [], location: msg.location, ts: Date.now() }; }
|
||||
satScanning = null;
|
||||
@@ -4265,10 +4269,13 @@
|
||||
const meta = WORKER_SVC_META[w.service] || { icon: '⚙️', label: w.service };
|
||||
const dot = !w.online ? '#666' : (w.busy ? '#FFB020' : '#3FFF3F');
|
||||
const stat = !w.online ? 'offline' : (w.busy ? 'beschaeftigt' : 'frei');
|
||||
// llm-Boxen koennen mehrere Modelle fahren (llama-swap) → Liste zeigen.
|
||||
const modelText = (w.service === 'llm' && Array.isArray(w.models) && w.models.length)
|
||||
? w.models.join(', ') : (w.model || '');
|
||||
return '<div style="display:flex;align-items:center;gap:8px;padding:4px 0;">' +
|
||||
'<span style="width:8px;height:8px;border-radius:50%;background:' + dot + ';display:inline-block;"></span>' +
|
||||
'<span>' + meta.icon + ' <b>' + escapeHtml(meta.label) + '</b></span>' +
|
||||
'<span style="color:#8888AA;">' + escapeHtml(w.model || '') + '</span>' +
|
||||
'<span style="color:#8888AA;">' + escapeHtml(modelText) + '</span>' +
|
||||
(w.gpus ? '<span style="color:#8888AA;">GPU ' + escapeHtml(w.gpus) + '</span>' : '') +
|
||||
'<span style="margin-left:auto;color:' + dot + ';">' + stat + '</span>' +
|
||||
'</div>';
|
||||
@@ -6825,18 +6832,66 @@
|
||||
el.textContent = m && m.description ? m.description : '';
|
||||
}
|
||||
async function loadLocalModelList() {
|
||||
// Kuratierte Namen/Beschreibungen laden; die tatsaechliche Verfuegbarkeit
|
||||
// kommt live aus der Flotte (refreshLocalLlmModelChoices).
|
||||
try {
|
||||
const r = await fetch('/api/local-models-list');
|
||||
const j = await r.json();
|
||||
_localModelsCache = (j && j.models) || [];
|
||||
} catch (e) { _localModelsCache = []; }
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (sel) {
|
||||
sel.innerHTML = _localModelsCache.map(m =>
|
||||
`<option value="${m.id}">${m.display_name || m.id}</option>`).join('') || '<option value="">(keine)</option>';
|
||||
if (_localModelsCache.some(m => m.id === _currentLocalModel)) sel.value = _currentLocalModel;
|
||||
updateLocalModelDesc();
|
||||
refreshLocalLlmModelChoices();
|
||||
}
|
||||
// Zaehlt pro Modell die online LLM-Boxen, die es fahren koennen.
|
||||
function fleetLlmModelCounts() {
|
||||
const counts = {};
|
||||
for (const w of workers) {
|
||||
if (w.service !== 'llm' || !w.online) continue;
|
||||
const list = (Array.isArray(w.models) && w.models.length) ? w.models : (w.model ? [w.model] : []);
|
||||
for (const id of list) counts[id] = (counts[id] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
// Baut das Modell-Dropdown aus der Vereinigung von live-Flotte + kuratierter
|
||||
// Liste; zeigt pro Modell "· N Box(en)" bzw. "· offline". Auswahl bleibt
|
||||
// erhalten (auch wenn das gewaehlte Modell gerade keine online Box hat).
|
||||
function refreshLocalLlmModelChoices() {
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (!sel) return;
|
||||
const counts = fleetLlmModelCounts();
|
||||
const nameOf = id => { const m = _localModelsCache.find(x => x.id === id); return (m && m.display_name) || id; };
|
||||
const ids = new Set();
|
||||
Object.keys(counts).forEach(id => ids.add(id));
|
||||
_localModelsCache.forEach(m => ids.add(m.id));
|
||||
if (_currentLocalModel) ids.add(_currentLocalModel);
|
||||
const order = Array.from(ids).sort();
|
||||
sel.innerHTML = order.map(id => {
|
||||
const n = counts[id] || 0;
|
||||
const avail = n > 0 ? ` · ${n} Box${n > 1 ? 'en' : ''}` : ' · offline';
|
||||
return `<option value="${id}">${escapeHtml(nameOf(id))}${avail}</option>`;
|
||||
}).join('') || '<option value="">(keine)</option>';
|
||||
if (order.includes(_currentLocalModel)) sel.value = _currentLocalModel;
|
||||
updateLocalModelDesc();
|
||||
renderLocalLlmFleet();
|
||||
}
|
||||
// Kompakte Box-Liste unter dem Dropdown: Node → Modelle → Health.
|
||||
function renderLocalLlmFleet() {
|
||||
const box = document.getElementById('local-llm-fleet');
|
||||
if (!box) return;
|
||||
const llm = workers.filter(w => w.service === 'llm');
|
||||
if (!llm.length) { box.innerHTML = '<span style="color:#6a6a88;">Keine LLM-Box angemeldet. Starte eine Box mit <code>COMPOSE_PROFILES=llm</code>.</span>'; return; }
|
||||
const byNode = {};
|
||||
for (const w of llm) { (byNode[w.node || '?'] = byNode[w.node || '?'] || []).push(w); }
|
||||
box.innerHTML = '<div style="color:#AAB;margin-bottom:2px;">LLM-Boxen:</div>' + Object.keys(byNode).sort().map(node => {
|
||||
return byNode[node].map(w => {
|
||||
const dot = !w.online ? '#666' : (w.busy ? '#FFB020' : '#3FFF3F');
|
||||
const models = (Array.isArray(w.models) && w.models.length) ? w.models.join(', ') : (w.model || '—');
|
||||
return '<div style="display:flex;align-items:center;gap:6px;padding:2px 0;">' +
|
||||
'<span style="width:7px;height:7px;border-radius:50%;background:' + dot + ';display:inline-block;"></span>' +
|
||||
'<span>🖥️ ' + escapeHtml(node) + '</span>' +
|
||||
'<span style="color:#8888AA;">' + escapeHtml(models) + '</span>' +
|
||||
'</div>';
|
||||
}).join('');
|
||||
}).join('');
|
||||
}
|
||||
async function loadLocalLlmConfig() {
|
||||
try {
|
||||
@@ -6849,11 +6904,8 @@
|
||||
if (ol) ol.checked = !!c.localOnly;
|
||||
if (tv) tv.value = (c.toolVariant === 'full') ? 'full' : 'slim';
|
||||
_currentLocalModel = c.localLlmModel || 'qwen3-8b';
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (sel && _localModelsCache.some(m => m.id === _currentLocalModel)) {
|
||||
sel.value = _currentLocalModel;
|
||||
updateLocalModelDesc();
|
||||
}
|
||||
// Dropdown neu aufbauen (Flotte+kuratiert) und die gespeicherte Auswahl setzen.
|
||||
refreshLocalLlmModelChoices();
|
||||
setLocalLlmStatus(c);
|
||||
} catch (e) { /* still */ }
|
||||
}
|
||||
|
||||
@@ -499,7 +499,7 @@ function workerList() {
|
||||
const now = Date.now();
|
||||
return Array.from(workers.values()).map(w => ({
|
||||
instanceId: w.instanceId, service: w.service, node: w.node,
|
||||
gpus: w.gpus, model: w.model, busy: !!w.busy,
|
||||
gpus: w.gpus, model: w.model, models: w.models || [], busy: !!w.busy,
|
||||
online: (now - (w.last_seen || 0)) < WORKER_OFFLINE_MS,
|
||||
}));
|
||||
}
|
||||
@@ -1019,6 +1019,7 @@ function connectRVS(forcePlain) {
|
||||
workers.set(p.instanceId, {
|
||||
instanceId: p.instanceId, service: p.service || "",
|
||||
node: p.node || "", gpus: p.gpus || "", model: p.model || "",
|
||||
models: Array.isArray(p.models) ? p.models : (p.model ? [p.model] : []),
|
||||
busy: !!prev.busy, last_seen: Date.now(),
|
||||
});
|
||||
broadcastWorkers();
|
||||
@@ -1032,7 +1033,7 @@ function connectRVS(forcePlain) {
|
||||
let w = workers.get(p.instanceId);
|
||||
if (!w) {
|
||||
const svc = String(p.instanceId).split("@")[0];
|
||||
w = { instanceId: p.instanceId, service: svc, node: "", gpus: "", model: "", busy: false, last_seen: 0 };
|
||||
w = { instanceId: p.instanceId, service: svc, node: "", gpus: "", model: "", models: [], busy: false, last_seen: 0 };
|
||||
workers.set(p.instanceId, w);
|
||||
}
|
||||
w.busy = !!p.busy;
|
||||
|
||||
@@ -133,14 +133,33 @@ async def _emit_llm_status(ws, state: str, model: str, **extra) -> None:
|
||||
{"service": "llm", "state": state, "model": model, **extra})
|
||||
|
||||
|
||||
async def _fetch_available_models() -> list:
|
||||
"""Fragt llama-swap ab, welche Modelle diese Box fahren kann (GET /v1/models,
|
||||
OpenAI-kompatibel → {data:[{id},...]}). Das sind die config.yaml-Keys.
|
||||
Defensiv: bei Fehler Fallback auf [LLM_MODEL]."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.get(f"{LLAMA_URL}/v1/models")
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
|
||||
return ids or [LLM_MODEL]
|
||||
except Exception as e:
|
||||
logger.warning("llama-swap /v1/models nicht abfragbar (%s) — Fallback [%s]", e, LLM_MODEL)
|
||||
return [LLM_MODEL]
|
||||
|
||||
|
||||
async def _worker_register(ws) -> None:
|
||||
"""Meldet diesen Worker bei der aria-bridge an (worker_hello) und haelt die
|
||||
Flotten-Registry per periodischem worker_ping (mit busy-Status) frisch."""
|
||||
try:
|
||||
models = await _fetch_available_models()
|
||||
await _send(ws, "worker_hello", {
|
||||
"instanceId": INSTANCE_ID, "service": WORKER_SERVICE,
|
||||
"node": NODE_NAME, "gpus": GPU_IDS, "model": LLM_MODEL,
|
||||
"models": models, # welche Modelle diese Box fahren kann (llama-swap-Keys)
|
||||
})
|
||||
logger.info("worker_hello: models=%s", models)
|
||||
while True:
|
||||
await asyncio.sleep(WORKER_PING_INTERVAL_S)
|
||||
await _send(ws, "worker_ping",
|
||||
|
||||
Reference in New Issue
Block a user