feat(diagnostic): Auslastung aller Grafikkarten aller KI-Boxen auf einen Blick
LLM-Karten → "📊 Auslastung": Fenster mit jeder Box und jeder Karte, live jede Sekunde — Rechenlast, Speicher (belegt/gesamt), Temperatur, Strom, Verlauf der letzten 2 min und was auf der Karte laeuft (LLM-Modell mit Kontext, Whisper, F5-TTS, Voxtral). node_stats (alle GPU-Dienste, identische Kopie) meldet im Live-Stream jetzt zusaetzlich die Werte PRO Karte (mit uuid). Pro Box streamt bevorzugt der LLM-Adapter (sieht alle Karten mit echter Nummer); Boxen ohne ihn: jeder GPU-Dienst seine Karte, Nummer aus worker.gpus. Streams werden erneuert, solange das Fenster offen ist, und beim Schliessen beendet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -96,6 +96,31 @@ class NodeStats:
|
||||
gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
|
||||
return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
|
||||
|
||||
async def _query_cards(self) -> list:
|
||||
"""Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
|
||||
uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
|
||||
Dienste einer Box sie melden."""
|
||||
out = await _run_cmd(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw,power.limit",
|
||||
"--format=csv,noheader,nounits")
|
||||
cards = []
|
||||
for line in out.strip().splitlines():
|
||||
p = [x.strip() for x in line.split(",")]
|
||||
if len(p) < 9:
|
||||
continue
|
||||
|
||||
def num(v):
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
return None
|
||||
cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
|
||||
"memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
|
||||
"power": num(p[7]), "powerLimit": num(p[8])})
|
||||
return cards
|
||||
|
||||
async def _nvidia_smi_text(self) -> str:
|
||||
txt = await _run_cmd("nvidia-smi")
|
||||
return txt or "nvidia-smi nicht verfuegbar"
|
||||
@@ -128,9 +153,10 @@ class NodeStats:
|
||||
while time.time() - t0 < STREAM_MAX_SEC:
|
||||
g = await self._query_gpu()
|
||||
smi = await self._nvidia_smi_text()
|
||||
cards = await self._query_cards()
|
||||
await send(ws, "node_stats", {
|
||||
"instanceId": self.instance_id, "node": self.node_name,
|
||||
"nvidiaSmi": smi, **g,
|
||||
"nvidiaSmi": smi, "cards": cards, **g,
|
||||
})
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -96,6 +96,31 @@ class NodeStats:
|
||||
gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
|
||||
return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
|
||||
|
||||
async def _query_cards(self) -> list:
|
||||
"""Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
|
||||
uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
|
||||
Dienste einer Box sie melden."""
|
||||
out = await _run_cmd(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw,power.limit",
|
||||
"--format=csv,noheader,nounits")
|
||||
cards = []
|
||||
for line in out.strip().splitlines():
|
||||
p = [x.strip() for x in line.split(",")]
|
||||
if len(p) < 9:
|
||||
continue
|
||||
|
||||
def num(v):
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
return None
|
||||
cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
|
||||
"memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
|
||||
"power": num(p[7]), "powerLimit": num(p[8])})
|
||||
return cards
|
||||
|
||||
async def _nvidia_smi_text(self) -> str:
|
||||
txt = await _run_cmd("nvidia-smi")
|
||||
return txt or "nvidia-smi nicht verfuegbar"
|
||||
@@ -128,9 +153,10 @@ class NodeStats:
|
||||
while time.time() - t0 < STREAM_MAX_SEC:
|
||||
g = await self._query_gpu()
|
||||
smi = await self._nvidia_smi_text()
|
||||
cards = await self._query_cards()
|
||||
await send(ws, "node_stats", {
|
||||
"instanceId": self.instance_id, "node": self.node_name,
|
||||
"nvidiaSmi": smi, **g,
|
||||
"nvidiaSmi": smi, "cards": cards, **g,
|
||||
})
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -96,6 +96,31 @@ class NodeStats:
|
||||
gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
|
||||
return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
|
||||
|
||||
async def _query_cards(self) -> list:
|
||||
"""Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
|
||||
uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
|
||||
Dienste einer Box sie melden."""
|
||||
out = await _run_cmd(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw,power.limit",
|
||||
"--format=csv,noheader,nounits")
|
||||
cards = []
|
||||
for line in out.strip().splitlines():
|
||||
p = [x.strip() for x in line.split(",")]
|
||||
if len(p) < 9:
|
||||
continue
|
||||
|
||||
def num(v):
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
return None
|
||||
cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
|
||||
"memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
|
||||
"power": num(p[7]), "powerLimit": num(p[8])})
|
||||
return cards
|
||||
|
||||
async def _nvidia_smi_text(self) -> str:
|
||||
txt = await _run_cmd("nvidia-smi")
|
||||
return txt or "nvidia-smi nicht verfuegbar"
|
||||
@@ -128,9 +153,10 @@ class NodeStats:
|
||||
while time.time() - t0 < STREAM_MAX_SEC:
|
||||
g = await self._query_gpu()
|
||||
smi = await self._nvidia_smi_text()
|
||||
cards = await self._query_cards()
|
||||
await send(ws, "node_stats", {
|
||||
"instanceId": self.instance_id, "node": self.node_name,
|
||||
"nvidiaSmi": smi, **g,
|
||||
"nvidiaSmi": smi, "cards": cards, **g,
|
||||
})
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -96,6 +96,31 @@ class NodeStats:
|
||||
gpu = round(sum(utils) / len(utils), 1) if utils else 0.0
|
||||
return {"gpu": gpu, "memUsed": int(used), "memTotal": int(total)}
|
||||
|
||||
async def _query_cards(self) -> list:
|
||||
"""Pro Karte: Index, Name, Last, Speicher, Temperatur, Strom.
|
||||
uuid, damit die Diagnostic dieselbe Karte erkennt, wenn mehrere
|
||||
Dienste einer Box sie melden."""
|
||||
out = await _run_cmd(
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw,power.limit",
|
||||
"--format=csv,noheader,nounits")
|
||||
cards = []
|
||||
for line in out.strip().splitlines():
|
||||
p = [x.strip() for x in line.split(",")]
|
||||
if len(p) < 9:
|
||||
continue
|
||||
|
||||
def num(v):
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
return None
|
||||
cards.append({"index": p[0], "uuid": p[1], "name": p[2], "util": num(p[3]),
|
||||
"memUsed": num(p[4]), "memTotal": num(p[5]), "temp": num(p[6]),
|
||||
"power": num(p[7]), "powerLimit": num(p[8])})
|
||||
return cards
|
||||
|
||||
async def _nvidia_smi_text(self) -> str:
|
||||
txt = await _run_cmd("nvidia-smi")
|
||||
return txt or "nvidia-smi nicht verfuegbar"
|
||||
@@ -128,9 +153,10 @@ class NodeStats:
|
||||
while time.time() - t0 < STREAM_MAX_SEC:
|
||||
g = await self._query_gpu()
|
||||
smi = await self._nvidia_smi_text()
|
||||
cards = await self._query_cards()
|
||||
await send(ws, "node_stats", {
|
||||
"instanceId": self.instance_id, "node": self.node_name,
|
||||
"nvidiaSmi": smi, **g,
|
||||
"nvidiaSmi": smi, "cards": cards, **g,
|
||||
})
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
Reference in New Issue
Block a user