feat(diagnostic): Auslastung aller Grafikkarten aller KI-Boxen auf einen Blick
LLM-Karten → "📊 Auslastung": Fenster mit jeder Box und jeder Karte, live jede Sekunde — Rechenlast, Speicher (belegt/gesamt), Temperatur, Strom, Verlauf der letzten 2 min und was auf der Karte laeuft (LLM-Modell mit Kontext, Whisper, F5-TTS, Voxtral). node_stats (alle GPU-Dienste, identische Kopie) meldet im Live-Stream jetzt zusaetzlich die Werte PRO Karte (mit uuid). Pro Box streamt bevorzugt der LLM-Adapter (sieht alle Karten mit echter Nummer); Boxen ohne ihn: jeder GPU-Dienst seine Karte, Nummer aus worker.gpus. Streams werden erneuert, solange das Fenster offen ist, und beim Schliessen beendet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -197,6 +197,18 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Kartenauslastung: alle KI-Boxen, alle Karten, live -->
|
||||
<div id="card-load-modal" style="display:none;position:fixed;inset:0;z-index:1001;background:rgba(0,0,0,0.8);align-items:center;justify-content:center;" onclick="if(event.target===this)closeCardLoad()">
|
||||
<div style="background:#0D0D1A;border:1px solid #2A2A3E;border-radius:10px;padding:16px;max-width:1000px;width:94%;max-height:92vh;overflow:auto;display:flex;flex-direction:column;gap:10px;">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;">
|
||||
<h3 style="margin:0;color:#fff;">Auslastung der Grafikkarten <span style="font-size:11px;color:#8888AA;font-weight:normal;">· live, jede Sekunde</span></h3>
|
||||
<button onclick="closeCardLoad()" style="background:none;border:none;color:#8888AA;font-size:22px;cursor:pointer;">×</button>
|
||||
</div>
|
||||
<div id="card-load-body" style="display:flex;flex-direction:column;gap:14px;"></div>
|
||||
<div style="font-size:11px;color:#555570;">Ausführlich mit <code>nvidia-smi</code> und Verlauf der letzten Stunden: Compute-Flotte → „Auslastung" bei der Box.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Geraete-Credentials Modal -->
|
||||
<div id="dev-creds-modal" style="display:none;position:fixed;inset:0;z-index:1002;background:rgba(0,0,0,0.8);align-items:center;justify-content:center;">
|
||||
<div style="background:#0D0D1A;border:1px solid #2A2A3E;border-radius:10px;padding:18px;max-width:520px;width:94%;max-height:92vh;overflow:auto;display:flex;flex-direction:column;gap:12px;">
|
||||
@@ -844,6 +856,7 @@
|
||||
<div style="display:flex;align-items:center;gap:14px;margin-bottom:8px;">
|
||||
<h2 style="margin:0;">LLM-Karten</h2>
|
||||
<button class="btn secondary" onclick="openLlmDownloads()" style="padding:4px 10px;font-size:11px;">⬇ Downloads</button>
|
||||
<button class="btn secondary" onclick="openCardLoad()" title="Live-Auslastung aller Grafikkarten auf allen KI-Boxen" style="padding:4px 10px;font-size:11px;">📊 Auslastung</button>
|
||||
</div>
|
||||
<div class="card" style="max-width:1100px;">
|
||||
<p style="color:#8888AA;font-size:12px;margin:0 0 10px;line-height:1.5;">
|
||||
@@ -2654,6 +2667,7 @@
|
||||
}
|
||||
if (msg.type === 'node_stats') {
|
||||
const p = msg.payload || {};
|
||||
if (_cardLoadStreams.has(p.instanceId)) onCardLoadStats(p);
|
||||
if (_nodeStatsTarget && p.instanceId === _nodeStatsTarget) {
|
||||
const el = document.getElementById('node-stats-smi');
|
||||
if (el) el.textContent = p.nvidiaSmi || '(leer)';
|
||||
@@ -5309,6 +5323,110 @@
|
||||
}, 60);
|
||||
}
|
||||
|
||||
// ── Kartenauslastung (alle Boxen, alle Karten) ─────────
|
||||
// Pro Box EIN Dienst, der streamt: bevorzugt der LLM-Adapter (sieht alle
|
||||
// Karten mit der echten Nummer). Hat eine Box keinen, streamt jeder GPU-
|
||||
// Dienst seine eigene Karte (dort ist die Nummer im Container immer 0 —
|
||||
// die echte steht im worker.gpus).
|
||||
const _cardLoadStreams = new Map(); // instanceId → {node, gpus, single}
|
||||
const _cardLoad = new Map(); // node|uuid → {node, index, name, util, ..., hist: []}
|
||||
let _cardLoadTimer = null;
|
||||
function openCardLoad() {
|
||||
document.getElementById('card-load-modal').style.display = 'flex';
|
||||
_cardLoad.clear();
|
||||
_cardLoadStreams.clear();
|
||||
const byNode = {};
|
||||
for (const w of workers) {
|
||||
if (!w.online || !['llm', 'whisper', 'f5tts', 'voxtral'].includes(w.service)) continue;
|
||||
(byNode[w.node] = byNode[w.node] || []).push(w);
|
||||
}
|
||||
for (const [node, list] of Object.entries(byNode)) {
|
||||
const llm = list.find(w => w.service === 'llm');
|
||||
const streamers = llm ? [llm] : list;
|
||||
for (const w of streamers) {
|
||||
_cardLoadStreams.set(w.instanceId, { node, gpus: String(w.gpus || ''), single: !llm });
|
||||
send({ action: 'node_stats_stream_start', targetInstance: w.instanceId });
|
||||
}
|
||||
}
|
||||
renderCardLoad();
|
||||
// Streams laufen auf der Box max. 5 min — solange das Fenster offen ist, erneuern
|
||||
clearInterval(_cardLoadTimer);
|
||||
_cardLoadTimer = setInterval(() => {
|
||||
for (const id of _cardLoadStreams.keys()) send({ action: 'node_stats_stream_start', targetInstance: id });
|
||||
}, 240000);
|
||||
}
|
||||
function closeCardLoad() {
|
||||
document.getElementById('card-load-modal').style.display = 'none';
|
||||
clearInterval(_cardLoadTimer); _cardLoadTimer = null;
|
||||
for (const id of _cardLoadStreams.keys()) send({ action: 'node_stats_stream_stop', targetInstance: id });
|
||||
_cardLoadStreams.clear();
|
||||
}
|
||||
function onCardLoadStats(p) {
|
||||
const src = _cardLoadStreams.get(p.instanceId);
|
||||
if (!src || !Array.isArray(p.cards)) return;
|
||||
for (const c of p.cards) {
|
||||
// Einzel-Dienst: Container-Nummer (0) durch die echte Karte ersetzen
|
||||
const index = src.single && src.gpus && !src.gpus.includes(',') ? src.gpus : c.index;
|
||||
const key = src.node + '|' + (c.uuid || index);
|
||||
const cur = _cardLoad.get(key) || { hist: [] };
|
||||
cur.hist.push(c.util || 0);
|
||||
if (cur.hist.length > 120) cur.hist.shift();
|
||||
_cardLoad.set(key, { ...cur, ...c, index, node: src.node, ts: Date.now() });
|
||||
}
|
||||
renderCardLoad();
|
||||
}
|
||||
/** Was laeuft auf Karte <index> von <node>: LLM-Modell + Dienste (whisper/f5tts …) */
|
||||
function _cardJobs(node, index) {
|
||||
const out = [];
|
||||
for (const w of workers) {
|
||||
if (w.node !== node || !w.online) continue;
|
||||
if (w.service === 'llm') {
|
||||
for (const sl of (w.slots || [])) if (String(sl.gpu) === String(index) && sl.model) out.push('🧠 ' + sl.model + (sl.ctx ? ' · ' + Math.round(sl.ctx / 1024) + 'k' : ''));
|
||||
} else if (String(w.gpus || '').split(',').map(x => x.trim()).includes(String(index))) {
|
||||
out.push({ whisper: '🎙️ Whisper (Sprache→Text)', f5tts: '🔊 F5-TTS (Sprachausgabe)', voxtral: '🎙️ Voxtral' }[w.service] || w.service);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
function _bar(pct, color) {
|
||||
const v = Math.max(0, Math.min(100, pct || 0));
|
||||
return '<div style="height:8px;background:#1E1E2E;border-radius:4px;overflow:hidden;"><div style="height:8px;width:' + v + '%;background:' + color + ';"></div></div>';
|
||||
}
|
||||
function renderCardLoad() {
|
||||
const body = document.getElementById('card-load-body');
|
||||
if (!body) return;
|
||||
if (!_cardLoadStreams.size) { body.innerHTML = '<div style="color:#8888AA;">Keine KI-Box mit Grafikkarte online.</div>'; return; }
|
||||
const nodes = [...new Set([..._cardLoadStreams.values()].map(v => v.node))].sort();
|
||||
body.innerHTML = nodes.map(node => {
|
||||
const cards = [..._cardLoad.values()].filter(c => c.node === node)
|
||||
.sort((a, b) => Number(a.index) - Number(b.index));
|
||||
const stale = cards.length && cards.every(c => Date.now() - c.ts > 5000);
|
||||
const inner = cards.length ? cards.map(c => {
|
||||
const memPct = c.memTotal ? c.memUsed / c.memTotal * 100 : 0;
|
||||
const utilColor = c.util >= 85 ? '#FF6B6B' : c.util >= 50 ? '#FFB020' : '#3FFF9F';
|
||||
const memColor = memPct >= 92 ? '#FF6B6B' : memPct >= 75 ? '#FFB020' : '#0096FF';
|
||||
const jobs = _cardJobs(node, c.index);
|
||||
return '<div style="border:1px solid #1E1E2E;border-radius:8px;padding:10px 12px;background:#12122A;">' +
|
||||
'<div style="display:flex;justify-content:space-between;gap:10px;flex-wrap:wrap;">' +
|
||||
'<div><b style="color:#E0E0F0;">Karte ' + escapeHtml(String(c.index)) + '</b> <span style="color:#8888AA;font-size:12px;">' + escapeHtml(c.name || '') + '</span></div>' +
|
||||
'<div style="color:#8888AA;font-size:12px;">' + (c.temp != null ? '🌡 ' + Math.round(c.temp) + ' °C' : '') +
|
||||
(c.power != null ? ' · ⚡ ' + Math.round(c.power) + (c.powerLimit ? ' / ' + Math.round(c.powerLimit) : '') + ' W' : '') + '</div>' +
|
||||
'</div>' +
|
||||
'<div style="display:grid;grid-template-columns:110px 1fr 120px;gap:6px 10px;align-items:center;margin-top:8px;font-size:12px;">' +
|
||||
'<span style="color:#8888AA;">Rechenlast</span>' + _bar(c.util, utilColor) + '<span style="color:#E0E0F0;">' + Math.round(c.util || 0) + ' %</span>' +
|
||||
'<span style="color:#8888AA;">Speicher</span>' + _bar(memPct, memColor) +
|
||||
'<span style="color:#E0E0F0;">' + ((c.memUsed || 0) / 1024).toFixed(1) + ' / ' + ((c.memTotal || 0) / 1024).toFixed(0) + ' GB</span>' +
|
||||
'</div>' +
|
||||
'<div style="margin-top:6px;">' + sparkline(c.hist, { max: 100, color: utilColor, fmt: v => v + ' %' }).replace('height:90px', 'height:40px') + '</div>' +
|
||||
'<div style="margin-top:6px;font-size:12px;color:#C8C8D8;">' + (jobs.length ? jobs.map(escapeHtml).join(' · ') : '<span style="color:#555570;">nichts zugewiesen</span>') + '</div>' +
|
||||
'</div>';
|
||||
}).join('') : '<div style="color:#8888AA;font-size:12px;">warte auf Werte … (Box mit altem Stand? Dann liefert sie noch keine Werte pro Karte.)</div>';
|
||||
return '<div><div style="color:#E0E0F0;font-weight:700;margin-bottom:6px;">🖥 ' + escapeHtml(node) +
|
||||
(stale ? ' <span style="color:#FF9F0A;font-size:11px;font-weight:normal;">· keine neuen Werte</span>' : '') + '</div>' +
|
||||
'<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:10px;">' + inner + '</div></div>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ── Auslastungs-Monitor (Stage E) ───────────────────────
|
||||
let _nodeStatsTarget = '';
|
||||
function openNodeStats(instanceId, node) {
|
||||
|
||||
Reference in New Issue
Block a user