From 294917fe36126c7d023030d7ecb932adc6981cc7 Mon Sep 17 00:00:00 2001
From: duffyduck <info@hacker-net.de>
Date: Sun, 25 Oct 2026 14:36:02 +0100
Subject: [PATCH] feat(diagnostic): Auslastung aller Grafikkarten aller
 KI-Boxen auf einen Blick
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

LLM-Karten → "📊 Auslastung": Fenster mit jeder Box und jeder Karte, live
jede Sekunde — Rechenlast, Speicher (belegt/gesamt), Temperatur, Strom,
Verlauf der letzten 2 min und was auf der Karte laeuft (LLM-Modell mit
Kontext, Whisper, F5-TTS, Voxtral).

node_stats (alle GPU-Dienste, identische Kopie) meldet im Live-Stream jetzt
zusaetzlich die Werte PRO Karte (mit uuid). Pro Box streamt bevorzugt der
LLM-Adapter (sieht alle Karten mit echter Nummer); Boxen ohne ihn: jeder
GPU-Dienst seine Karte, Nummer aus worker.gpus. Streams werden erneuert,
solange das Fenster offen ist, und beim Schliessen beendet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 diagnostic/index.html          | 118 +++++++++++++++++++++++++++++++++
 xtts/f5tts/node_stats.py       |  28 +++++++-
 xtts/llm-adapter/node_stats.py |  28 +++++++-
 xtts/voxtral/node_stats.py     |  28 +++++++-
 xtts/whisper/node_stats.py     |  28 +++++++-
 5 files changed, 226 insertions(+), 4 deletions(-)

diff --git a/diagnostic/index.html b/diagnostic/index.html
index 4feaa66..f350702 100644
--- a/diagnostic/index.html
+++ b/diagnostic/index.html
@@ -197,6 +197,18 @@
     </div>
   </div>
 
+  <!-- Kartenauslastung: alle KI-Boxen, alle Karten, live -->
+  <div id="card-load-modal" style="display:none;position:fixed;inset:0;z-index:1001;background:rgba(0,0,0,0.8);align-items:center;justify-content:center;" onclick="if(event.target===this)closeCardLoad()">
+    <div style="background:#0D0D1A;border:1px solid #2A2A3E;border-radius:10px;padding:16px;max-width:1000px;width:94%;max-height:92vh;overflow:auto;display:flex;flex-direction:column;gap:10px;">
+      <div style="display:flex;align-items:center;justify-content:space-between;">
+        <h3 style="margin:0;color:#fff;">Auslastung der Grafikkarten <span style="font-size:11px;color:#8888AA;font-weight:normal;">· live, jede Sekunde</span></h3>
+        <button onclick="closeCardLoad()" style="background:none;border:none;color:#8888AA;font-size:22px;cursor:pointer;">&times;</button>
+      </div>
+      <div id="card-load-body" style="display:flex;flex-direction:column;gap:14px;"></div>
+      <div style="font-size:11px;color:#555570;">Ausführlich mit <code>nvidia-smi</code> und Verlauf der letzten Stunden: Compute-Flotte → „Auslastung" bei der Box.</div>
+    </div>
+  </div>
+
   <!-- Geraete-Credentials Modal -->
   <div id="dev-creds-modal" style="display:none;position:fixed;inset:0;z-index:1002;background:rgba(0,0,0,0.8);align-items:center;justify-content:center;">
     <div style="background:#0D0D1A;border:1px solid #2A2A3E;border-radius:10px;padding:18px;max-width:520px;width:94%;max-height:92vh;overflow:auto;display:flex;flex-direction:column;gap:12px;">
@@ -844,6 +856,7 @@
       <div style="display:flex;align-items:center;gap:14px;margin-bottom:8px;">
         <h2 style="margin:0;">LLM-Karten</h2>
         <button class="btn secondary" onclick="openLlmDownloads()" style="padding:4px 10px;font-size:11px;">⬇ Downloads</button>
+        <button class="btn secondary" onclick="openCardLoad()" title="Live-Auslastung aller Grafikkarten auf allen KI-Boxen" style="padding:4px 10px;font-size:11px;">📊 Auslastung</button>
       </div>
       <div class="card" style="max-width:1100px;">
         <p style="color:#8888AA;font-size:12px;margin:0 0 10px;line-height:1.5;">
@@ -2654,6 +2667,7 @@
         }
         if (msg.type === 'node_stats') {
           const p = msg.payload || {};
+          if (_cardLoadStreams.has(p.instanceId)) onCardLoadStats(p);
           if (_nodeStatsTarget && p.instanceId === _nodeStatsTarget) {
             const el = document.getElementById('node-stats-smi');
             if (el) el.textContent = p.nvidiaSmi || '(leer)';
@@ -5309,6 +5323,110 @@
       }, 60);
     }
 
+    // ── Kartenauslastung (alle Boxen, alle Karten) ─────────
+    // Pro Box EIN Dienst, der streamt: bevorzugt der LLM-Adapter (sieht alle
+    // Karten mit der echten Nummer). Hat eine Box keinen, streamt jeder GPU-
+    // Dienst seine eigene Karte (dort ist die Nummer im Container immer 0 —
+    // die echte steht im worker.gpus).
+    const _cardLoadStreams = new Map();   // instanceId → {node, gpus, single}
+    const _cardLoad = new Map();          // node|uuid → {node, index, name, util, ..., hist: []}
+    let _cardLoadTimer = null;
+    function openCardLoad() {
+      document.getElementById('card-load-modal').style.display = 'flex';
+      _cardLoad.clear();
+      _cardLoadStreams.clear();
+      const byNode = {};
+      for (const w of workers) {
+        if (!w.online || !['llm', 'whisper', 'f5tts', 'voxtral'].includes(w.service)) continue;
+        (byNode[w.node] = byNode[w.node] || []).push(w);
+      }
+      for (const [node, list] of Object.entries(byNode)) {
+        const llm = list.find(w => w.service === 'llm');
+        const streamers = llm ? [llm] : list;
+        for (const w of streamers) {
+          _cardLoadStreams.set(w.instanceId, { node, gpus: String(w.gpus || ''), single: !llm });
+          send({ action: 'node_stats_stream_start', targetInstance: w.instanceId });
+        }
+      }
+      renderCardLoad();
+      // Streams laufen auf der Box max. 5 min — solange das Fenster offen ist, erneuern
+      clearInterval(_cardLoadTimer);
+      _cardLoadTimer = setInterval(() => {
+        for (const id of _cardLoadStreams.keys()) send({ action: 'node_stats_stream_start', targetInstance: id });
+      }, 240000);
+    }
+    function closeCardLoad() {
+      document.getElementById('card-load-modal').style.display = 'none';
+      clearInterval(_cardLoadTimer); _cardLoadTimer = null;
+      for (const id of _cardLoadStreams.keys()) send({ action: 'node_stats_stream_stop', targetInstance: id });
+      _cardLoadStreams.clear();
+    }
+    function onCardLoadStats(p) {
+      const src = _cardLoadStreams.get(p.instanceId);
+      if (!src || !Array.isArray(p.cards)) return;
+      for (const c of p.cards) {
+        // Einzel-Dienst: Container-Nummer (0) durch die echte Karte ersetzen
+        const index = src.single && src.gpus && !src.gpus.includes(',') ? src.gpus : c.index;
+        const key = src.node + '|' + (c.uuid || index);
+        const cur = _cardLoad.get(key) || { hist: [] };
+        cur.hist.push(c.util || 0);
+        if (cur.hist.length > 120) cur.hist.shift();
+        _cardLoad.set(key, { ...cur, ...c, index, node: src.node, ts: Date.now() });
+      }
+      renderCardLoad();
+    }
+    /** Was laeuft auf Karte <index> von <node>: LLM-Modell + Dienste (whisper/f5tts …) */
+    function _cardJobs(node, index) {
+      const out = [];
+      for (const w of workers) {
+        if (w.node !== node || !w.online) continue;
+        if (w.service === 'llm') {
+          for (const sl of (w.slots || [])) if (String(sl.gpu) === String(index) && sl.model) out.push('🧠 ' + sl.model + (sl.ctx ? ' · ' + Math.round(sl.ctx / 1024) + 'k' : ''));
+        } else if (String(w.gpus || '').split(',').map(x => x.trim()).includes(String(index))) {
+          out.push({ whisper: '🎙️ Whisper (Sprache→Text)', f5tts: '🔊 F5-TTS (Sprachausgabe)', voxtral: '🎙️ Voxtral' }[w.service] || w.service);
+        }
+      }
+      return out;
+    }
+    function _bar(pct, color) {
+      const v = Math.max(0, Math.min(100, pct || 0));
+      return '<div style="height:8px;background:#1E1E2E;border-radius:4px;overflow:hidden;"><div style="height:8px;width:' + v + '%;background:' + color + ';"></div></div>';
+    }
+    function renderCardLoad() {
+      const body = document.getElementById('card-load-body');
+      if (!body) return;
+      if (!_cardLoadStreams.size) { body.innerHTML = '<div style="color:#8888AA;">Keine KI-Box mit Grafikkarte online.</div>'; return; }
+      const nodes = [...new Set([..._cardLoadStreams.values()].map(v => v.node))].sort();
+      body.innerHTML = nodes.map(node => {
+        const cards = [..._cardLoad.values()].filter(c => c.node === node)
+          .sort((a, b) => Number(a.index) - Number(b.index));
+        const stale = cards.length && cards.every(c => Date.now() - c.ts > 5000);
+        const inner = cards.length ? cards.map(c => {
+          const memPct = c.memTotal ? c.memUsed / c.memTotal * 100 : 0;
+          const utilColor = c.util >= 85 ? '#FF6B6B' : c.util >= 50 ? '#FFB020' : '#3FFF9F';
+          const memColor = memPct >= 92 ? '#FF6B6B' : memPct >= 75 ? '#FFB020' : '#0096FF';
+          const jobs = _cardJobs(node, c.index);
+          return '<div style="border:1px solid #1E1E2E;border-radius:8px;padding:10px 12px;background:#12122A;">' +
+            '<div style="display:flex;justify-content:space-between;gap:10px;flex-wrap:wrap;">' +
+              '<div><b style="color:#E0E0F0;">Karte ' + escapeHtml(String(c.index)) + '</b> <span style="color:#8888AA;font-size:12px;">' + escapeHtml(c.name || '') + '</span></div>' +
+              '<div style="color:#8888AA;font-size:12px;">' + (c.temp != null ? '🌡 ' + Math.round(c.temp) + ' °C' : '') +
+                (c.power != null ? ' · ⚡ ' + Math.round(c.power) + (c.powerLimit ? ' / ' + Math.round(c.powerLimit) : '') + ' W' : '') + '</div>' +
+            '</div>' +
+            '<div style="display:grid;grid-template-columns:110px 1fr 120px;gap:6px 10px;align-items:center;margin-top:8px;font-size:12px;">' +
+              '<span style="color:#8888AA;">Rechenlast</span>' + _bar(c.util, utilColor) + '<span style="color:#E0E0F0;">' + Math.round(c.util || 0) + ' %</span>' +
+              '<span style="color:#8888AA;">Speicher</span>' + _bar(memPct, memColor) +
+                '<span style="color:#E0E0F0;">' + ((c.memUsed || 0) / 1024).toFixed(1) + ' / ' + ((c.memTotal || 0) / 1024).toFixed(0) + ' GB</span>' +
+            '</div>' +
+            '<div style="margin-top:6px;">' + sparkline(c.hist, { max: 100, color: utilColor, fmt: v => v + ' %' }).replace('height:90px', 'height:40px') + '</div>' +
+            '<div style="margin-top:6px;font-size:12px;color:#C8C8D8;">' + (jobs.length ? jobs.map(escapeHtml).join(' · ') : '<span style="color:#555570;">nichts zugewiesen</span>') + '</div>' +
+          '</div>';
+        }).join('') : '<div style="color:#8888AA;font-size:12px;">warte auf Werte … (Box mit altem Stand? Dann liefert sie noch keine Werte pro Karte.)</div>';
+        return '<div><div style="color:#E0E0F0;font-weight:700;margin-bottom:6px;">🖥 ' + escapeHtml(node) +
+          (stale ? ' <span style="color:#FF9F0A;font-size:11px;font-weight:normal;">· keine neuen Werte</span>' : '') + '</div>' +
+          '<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:10px;">' + inner + '</div></div>';
+      }).join('');
+    }
+
     // ── Auslastungs-Monitor (Stage E) ───────────────────────
     let _nodeStatsTarget = '';
     function openNodeStats(instanceId, node) {
diff --git a/xtts/f5tts/node_stats.py b/xtts/f5tts/node_stats.py
index f5a7de3..3e0074a 100644
--- a/xtts/f5tts/node_stats.py
+++ b/xtts/f5tts/node_stats.py
@@ -96,6 +96,31 @@ class NodeStats:
         gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
         return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
 
+    async def _query_cards(self) -> list:
+        """Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
+        uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
+        Dienste einer Box sie melden."""
+        out = await _run_cmd(
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
+            "temperature.gpu,power.draw,power.limit",
+            "--format=csv,noheader,nounits")
+        cards = []
+        for line in out.strip().splitlines():
+            p = [x.strip() for x in line.split(",")]
+            if len(p) < 9:
+                continue
+
+            def num(v):
+                try:
+                    return float(v)
+                except ValueError:
+                    return None
+            cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
+                          "memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
+                          "power": num(p[7]), "powerLimit": num(p[8])})
+        return cards
+
     async def _nvidia_smi_text(self) -> str:
         txt = await _run_cmd("nvidia-smi")
         return txt or "nvidia-smi nicht verfuegbar"
@@ -128,9 +153,10 @@ class NodeStats:
             while time.time() - t0 < STREAM_MAX_SEC:
                 g = await self._query_gpu()
                 smi = await self._nvidia_smi_text()
+                cards = await self._query_cards()
                 await send(ws, "node_stats", {
                     "instanceId": self.instance_id, "node": self.node_name,
-                    "nvidiaSmi": smi, **g,
+                    "nvidiaSmi": smi, "cards": cards, **g,
                 })
                 await asyncio.sleep(1)
         except asyncio.CancelledError:
diff --git a/xtts/llm-adapter/node_stats.py b/xtts/llm-adapter/node_stats.py
index f5a7de3..3e0074a 100644
--- a/xtts/llm-adapter/node_stats.py
+++ b/xtts/llm-adapter/node_stats.py
@@ -96,6 +96,31 @@ class NodeStats:
         gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
         return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
 
+    async def _query_cards(self) -> list:
+        """Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
+        uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
+        Dienste einer Box sie melden."""
+        out = await _run_cmd(
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
+            "temperature.gpu,power.draw,power.limit",
+            "--format=csv,noheader,nounits")
+        cards = []
+        for line in out.strip().splitlines():
+            p = [x.strip() for x in line.split(",")]
+            if len(p) < 9:
+                continue
+
+            def num(v):
+                try:
+                    return float(v)
+                except ValueError:
+                    return None
+            cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
+                          "memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
+                          "power": num(p[7]), "powerLimit": num(p[8])})
+        return cards
+
     async def _nvidia_smi_text(self) -> str:
         txt = await _run_cmd("nvidia-smi")
         return txt or "nvidia-smi nicht verfuegbar"
@@ -128,9 +153,10 @@ class NodeStats:
             while time.time() - t0 < STREAM_MAX_SEC:
                 g = await self._query_gpu()
                 smi = await self._nvidia_smi_text()
+                cards = await self._query_cards()
                 await send(ws, "node_stats", {
                     "instanceId": self.instance_id, "node": self.node_name,
-                    "nvidiaSmi": smi, **g,
+                    "nvidiaSmi": smi, "cards": cards, **g,
                 })
                 await asyncio.sleep(1)
         except asyncio.CancelledError:
diff --git a/xtts/voxtral/node_stats.py b/xtts/voxtral/node_stats.py
index f5a7de3..3e0074a 100644
--- a/xtts/voxtral/node_stats.py
+++ b/xtts/voxtral/node_stats.py
@@ -96,6 +96,31 @@ class NodeStats:
         gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
         return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
 
+    async def _query_cards(self) -> list:
+        """Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
+        uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
+        Dienste einer Box sie melden."""
+        out = await _run_cmd(
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
+            "temperature.gpu,power.draw,power.limit",
+            "--format=csv,noheader,nounits")
+        cards = []
+        for line in out.strip().splitlines():
+            p = [x.strip() for x in line.split(",")]
+            if len(p) < 9:
+                continue
+
+            def num(v):
+                try:
+                    return float(v)
+                except ValueError:
+                    return None
+            cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
+                          "memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
+                          "power": num(p[7]), "powerLimit": num(p[8])})
+        return cards
+
     async def _nvidia_smi_text(self) -> str:
         txt = await _run_cmd("nvidia-smi")
         return txt or "nvidia-smi nicht verfuegbar"
@@ -128,9 +153,10 @@ class NodeStats:
             while time.time() - t0 < STREAM_MAX_SEC:
                 g = await self._query_gpu()
                 smi = await self._nvidia_smi_text()
+                cards = await self._query_cards()
                 await send(ws, "node_stats", {
                     "instanceId": self.instance_id, "node": self.node_name,
-                    "nvidiaSmi": smi, **g,
+                    "nvidiaSmi": smi, "cards": cards, **g,
                 })
                 await asyncio.sleep(1)
         except asyncio.CancelledError:
diff --git a/xtts/whisper/node_stats.py b/xtts/whisper/node_stats.py
index f5a7de3..3e0074a 100644
--- a/xtts/whisper/node_stats.py
+++ b/xtts/whisper/node_stats.py
@@ -96,6 +96,31 @@ class NodeStats:
         gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
         return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
 
+    async def _query_cards(self) -> list:
+        """Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
+        uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
+        Dienste einer Box sie melden."""
+        out = await _run_cmd(
+            "nvidia-smi",
+            "--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
+            "temperature.gpu,power.draw,power.limit",
+            "--format=csv,noheader,nounits")
+        cards = []
+        for line in out.strip().splitlines():
+            p = [x.strip() for x in line.split(",")]
+            if len(p) < 9:
+                continue
+
+            def num(v):
+                try:
+                    return float(v)
+                except ValueError:
+                    return None
+            cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
+                          "memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
+                          "power": num(p[7]), "powerLimit": num(p[8])})
+        return cards
+
     async def _nvidia_smi_text(self) -> str:
         txt = await _run_cmd("nvidia-smi")
         return txt or "nvidia-smi nicht verfuegbar"
@@ -128,9 +153,10 @@ class NodeStats:
             while time.time() - t0 < STREAM_MAX_SEC:
                 g = await self._query_gpu()
                 smi = await self._nvidia_smi_text()
+                cards = await self._query_cards()
                 await send(ws, "node_stats", {
                     "instanceId": self.instance_id, "node": self.node_name,
-                    "nvidiaSmi": smi, **g,
+                    "nvidiaSmi": smi, "cards": cards, **g,
                 })
                 await asyncio.sleep(1)
         except asyncio.CancelledError: