feat(fleet): Auslastungs-Monitor pro Box — live nvidia-smi + Graphen (Stage E)
Pro Box ein "Auslastung"-Button in der Compute-Flotte → Modal mit live nvidia-smi (1s), Graphen (GPU-Auslastung + Tokens/Intervall) und Besen-Reset. Historie liegt auf der Box, Diagnostic holt sie via RVS. - node_stats.py (identisch in allen 4 Worker-Build-Contexts): Sampler alle 15s (nvidia-smi + Token-Delta → Ringpuffer ~500 Punkte, persistent als JSON auf der Box), Live-Stream (node_stats, 1s, Auto-Stop 300s), History-Request, Reset. nvidia-smi via async subprocess, fail-safe ohne GPU. - Worker-Wiring (f5tts/whisper/voxtral/llm-adapter): Import, Sampler-Task, _stats.handle() nach dem targetInstance-Filter. llm-adapter zaehlt Tokens (usage.total_tokens) → Token-Graph nur bei LLM-Boxen. Dockerfiles kopieren node_stats.py. - compose: llm-adapter bekommt runtime:nvidia + NVIDIA_VISIBLE_DEVICES=all + DRIVER_CAPABILITIES=utility (nur nvidia-smi, KEIN VRAM/Compute). - diagnostic/server.js: relay node_stats_* (Browser→Box) + forward (Box→Browser). - diagnostic/index.html: Auslastung-Button pro Node (Ziel bevorzugt llm-Instanz), Modal mit live nvidia-smi + Inline-SVG-Sparklines, Besen-Reset. Reporter-Wahl bevorzugt die llm-Instanz (sieht alle GPUs + Tokens); GPU-Worker sehen ihre gepinnte Karte. Gitignored Historie stoert git-Baum der Box nicht. Deploy: diagnostic + GPU-Boxen neu bauen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+106
-1
@@ -171,6 +171,31 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Auslastungs-Monitor Modal (Stage E) -->
|
||||
<div id="node-stats-modal" style="display:none;position:fixed;inset:0;z-index:1001;background:rgba(0,0,0,0.8);align-items:center;justify-content:center;">
|
||||
<div style="background:#0D0D1A;border:1px solid #2A2A3E;border-radius:10px;padding:16px;max-width:900px;width:94%;max-height:92vh;overflow:auto;display:flex;flex-direction:column;gap:10px;">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;">
|
||||
<h3 style="margin:0;color:#fff;">Auslastung: <span id="node-stats-node">—</span></h3>
|
||||
<div style="display:flex;gap:10px;align-items:center;">
|
||||
<button onclick="resetNodeStats()" title="Historie zuruecksetzen" style="background:none;border:none;color:#FFD60A;font-size:18px;cursor:pointer;">🧹</button>
|
||||
<button onclick="closeNodeStats()" style="background:none;border:none;color:#8888AA;font-size:22px;cursor:pointer;">×</button>
|
||||
</div>
|
||||
</div>
|
||||
<div style="display:flex;gap:14px;flex-wrap:wrap;">
|
||||
<div style="flex:1;min-width:280px;">
|
||||
<div style="font-size:11px;color:#8888AA;margin-bottom:2px;">GPU-Auslastung (%)</div>
|
||||
<div id="node-stats-graph-gpu"></div>
|
||||
</div>
|
||||
<div id="node-stats-tokens-wrap" style="flex:1;min-width:280px;">
|
||||
<div style="font-size:11px;color:#8888AA;margin-bottom:2px;">Tokens / Intervall</div>
|
||||
<div id="node-stats-graph-tokens"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="font-size:11px;color:#8888AA;margin-top:4px;">live <code>nvidia-smi</code>:</div>
|
||||
<pre id="node-stats-smi" style="background:#000;border:1px solid #1E1E2E;border-radius:6px;padding:8px;color:#3FFF9F;font-size:11px;line-height:1.25;overflow:auto;max-height:340px;margin:0;">(warte auf Box…)</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Voice-Preview Modal -->
|
||||
<div id="voice-preview-modal" style="display:none;position:fixed;inset:0;z-index:1000;background:rgba(0,0,0,0.7);align-items:center;justify-content:center;">
|
||||
<div style="background:#1A1A2E;border:1px solid #2A2A3E;border-radius:10px;padding:20px;max-width:560px;width:90%;display:flex;flex-direction:column;gap:12px;">
|
||||
@@ -2059,6 +2084,27 @@
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'node_stats') {
|
||||
const p = msg.payload || {};
|
||||
if (_nodeStatsTarget && p.instanceId === _nodeStatsTarget) {
|
||||
const el = document.getElementById('node-stats-smi');
|
||||
if (el) el.textContent = p.nvidiaSmi || '(leer)';
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'node_stats_history') {
|
||||
const p = msg.payload || {};
|
||||
if (_nodeStatsTarget && p.instanceId === _nodeStatsTarget) renderNodeStatsHistory(p);
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'node_stats_reset_done') {
|
||||
const p = msg.payload || {};
|
||||
if (_nodeStatsTarget && p.instanceId === _nodeStatsTarget) {
|
||||
renderNodeStatsHistory({ samples: [], tokenCapable: false });
|
||||
send({ action: 'node_stats_history_request', targetInstance: _nodeStatsTarget });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'sat_devices') {
|
||||
if (msg.satellite) { satDevices[msg.satellite] = { devices: msg.devices || [], location: msg.location, ts: Date.now() }; }
|
||||
satScanning = null;
|
||||
@@ -4329,12 +4375,71 @@
|
||||
'<span style="margin-left:auto;color:' + dot + ';">' + stat + '</span>' +
|
||||
'</div>';
|
||||
}).join('');
|
||||
// Auslastungs-Button pro Node — Ziel bevorzugt die llm-Instanz (sieht alle
|
||||
// GPUs + Tokens), sonst irgendein online Worker des Node.
|
||||
const group = byNode[node];
|
||||
const rep = group.find(w => w.service === 'llm' && w.online) || group.find(w => w.online) || group[0];
|
||||
const btn = rep ? '<button class="btn secondary" onclick="openNodeStats(\'' + escapeHtml(rep.instanceId) + '\',\'' + escapeHtml(node) + '\')" style="padding:2px 8px;font-size:10px;margin-left:8px;">Auslastung</button>' : '';
|
||||
return '<div style="margin-bottom:10px;">' +
|
||||
'<div style="font-weight:600;color:#AAB;margin-bottom:2px;">🖥️ ' + escapeHtml(node) + '</div>' +
|
||||
'<div style="font-weight:600;color:#AAB;margin-bottom:2px;">🖥️ ' + escapeHtml(node) + btn + '</div>' +
|
||||
rows + '</div>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ── Auslastungs-Monitor (Stage E) ───────────────────────
|
||||
let _nodeStatsTarget = '';
|
||||
function openNodeStats(instanceId, node) {
|
||||
_nodeStatsTarget = instanceId;
|
||||
document.getElementById('node-stats-node').textContent = node + ' (' + instanceId + ')';
|
||||
document.getElementById('node-stats-smi').textContent = '(warte auf Box…)';
|
||||
document.getElementById('node-stats-graph-gpu').innerHTML = '';
|
||||
document.getElementById('node-stats-graph-tokens').innerHTML = '';
|
||||
document.getElementById('node-stats-modal').style.display = 'flex';
|
||||
send({ action: 'node_stats_history_request', targetInstance: instanceId });
|
||||
send({ action: 'node_stats_stream_start', targetInstance: instanceId });
|
||||
}
|
||||
function closeNodeStats() {
|
||||
if (_nodeStatsTarget) send({ action: 'node_stats_stream_stop', targetInstance: _nodeStatsTarget });
|
||||
_nodeStatsTarget = '';
|
||||
document.getElementById('node-stats-modal').style.display = 'none';
|
||||
}
|
||||
function resetNodeStats() {
|
||||
if (!_nodeStatsTarget) return;
|
||||
send({ action: 'node_stats_reset', targetInstance: _nodeStatsTarget });
|
||||
}
|
||||
// Simple Inline-SVG-Sparkline (kein Fremd-Lib, CSP-sicher).
|
||||
function sparkline(values, opts) {
|
||||
opts = opts || {};
|
||||
const w = 320, h = 90, pad = 4, color = opts.color || '#0096FF';
|
||||
if (!values.length) return '<div style="color:#555;font-size:11px;">(keine Daten)</div>';
|
||||
const max = Math.max(opts.max || 0, ...values, 1);
|
||||
const n = values.length;
|
||||
const pts = values.map((v, i) => {
|
||||
const x = pad + (n === 1 ? 0 : (i / (n - 1)) * (w - 2 * pad));
|
||||
const y = h - pad - (Math.max(0, v) / max) * (h - 2 * pad);
|
||||
return x.toFixed(1) + ',' + y.toFixed(1);
|
||||
}).join(' ');
|
||||
return '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:100%;height:90px;background:#0A0A16;border:1px solid #1E1E2E;border-radius:6px;">' +
|
||||
'<polyline points="' + pts + '" fill="none" stroke="' + color + '" stroke-width="1.5"/>' +
|
||||
'<text x="' + (w - pad) + '" y="12" text-anchor="end" fill="#888" font-size="10">max ' + (opts.fmt ? opts.fmt(max) : Math.round(max)) + '</text>' +
|
||||
'</svg>';
|
||||
}
|
||||
function renderNodeStatsHistory(p) {
|
||||
const samples = p.samples || [];
|
||||
const gpu = samples.map(s => Number(s.gpu) || 0);
|
||||
document.getElementById('node-stats-graph-gpu').innerHTML =
|
||||
sparkline(gpu, { color: '#3FFF9F', max: 100, fmt: v => Math.round(v) + '%' });
|
||||
const wrap = document.getElementById('node-stats-tokens-wrap');
|
||||
if (p.tokenCapable) {
|
||||
wrap.style.display = '';
|
||||
const tok = samples.map(s => Number(s.tokens) || 0);
|
||||
document.getElementById('node-stats-graph-tokens').innerHTML =
|
||||
sparkline(tok, { color: '#FFD60A' });
|
||||
} else {
|
||||
wrap.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Satelliten-Ansicht ─────────────────────────────────
|
||||
let satellites = []; // [{id, location, caps, control, online}]
|
||||
let satDevices = {}; // id → {devices, location, ts}
|
||||
|
||||
@@ -1188,6 +1188,9 @@ function connectRVS(forcePlain) {
|
||||
} else if (msg.type === "llm_provision_result") {
|
||||
// Ergebnis eines Modell-Downloads/Aktivierens → an Browser (Katalog-Status).
|
||||
broadcast({ type: "llm_provision_result", payload: msg.payload || {} });
|
||||
} else if (msg.type === "node_stats" || msg.type === "node_stats_history" || msg.type === "node_stats_reset_done") {
|
||||
// Auslastungs-Monitor (Stage E): Box-Antworten an die Browser durchreichen.
|
||||
broadcast({ type: msg.type, payload: msg.payload || {} });
|
||||
} else if (msg.type === "audio_pcm" && msg.payload && _previewPending.size > 0) {
|
||||
// PCM-Chunks einer laufenden Voice-Preview — sammeln + WAV bauen
|
||||
_handlePreviewChunk(msg.payload);
|
||||
@@ -2936,6 +2939,10 @@ wss.on("connection", (ws) => {
|
||||
model: msg.model || "", targetInstance: msg.targetInstance || "",
|
||||
}, "llm_response", ws);
|
||||
log("info", "llm", `Test-Chat → ${msg.model || "?"} @ ${msg.targetInstance || "(broadcast)"}`);
|
||||
} else if (msg.action === "node_stats_stream_start" || msg.action === "node_stats_stream_stop"
|
||||
|| msg.action === "node_stats_history_request" || msg.action === "node_stats_reset") {
|
||||
// Auslastungs-Monitor (Stage E): an die Box (targetInstance) durchreichen.
|
||||
sendToRVS_raw({ type: msg.action, payload: { targetInstance: msg.targetInstance || "" }, timestamp: Date.now() });
|
||||
} else if (msg.action === "restart_session") {
|
||||
handleRestartSession(ws);
|
||||
// ── Einstellungen ──
|
||||
|
||||
Reference in New Issue
Block a user