feat(metrics): lokaler LLM-Verbrauch + Claude-Ersparnis in Diagnostic

metrics.jsonl-Eintraege tragen jetzt 'source' (claude|local|fast-path).
- log_local_call(): echte usage-Tokens vom Adapter (prompt/completion),
  sonst chars/4-Schaetzung. Geloggt pro Tool-Runde im lokalen Fast-Lane.
- log_fast_path(): reiner Skill, 0 Prompt-Tokens — gesparter Claude-Call.
- aggregate() liefert zusaetzlich by_source (calls/tokens_in/tokens_out).
  Alt-Eintraege ohne source zaehlen als claude (rueckwaerts-kompatibel).

Diagnostic Gehirn-Tab: neue Card "Lokales LLM & Claude-Ersparnis" — pro
Fenster (1h/5h/24h/30d) gesparte Claude-Calls (local + fast-path) und
lokale Token-Last (eigene HW, kein Quota) + Info-Block.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-11 16:39:02 +02:00
co-authored by Claude Opus 4.8
parent 8bac7c56bc
commit dc043ceb4d
3 changed files with 111 additions and 10 deletions
+13
View File
@@ -30,6 +30,7 @@ from memory import Embedder, VectorStore, MemoryPoint
from prompts import build_system_prompt, IDENTITY_SEED, IDENTITY_ANCHOR, looks_like_identity_break from prompts import build_system_prompt, IDENTITY_SEED, IDENTITY_ANCHOR, looks_like_identity_break
from proxy_client import ProxyClient, Message as ProxyMessage from proxy_client import ProxyClient, Message as ProxyMessage
import router as router_mod import router as router_mod
import metrics
from local_llm import local_llm_chat from local_llm import local_llm_chat
import skills as skills_mod import skills as skills_mod
import triggers as triggers_mod import triggers as triggers_mod
@@ -1220,6 +1221,13 @@ class Agent:
if local_only: if local_only:
return f"[Lokales LLM nicht erreichbar: {res.get('error', 'unbekannt')}]" return f"[Lokales LLM nicht erreichbar: {res.get('error', 'unbekannt')}]"
return None return None
# Metric: dieser lokale Call (echte usage-Tokens wenn der Adapter sie
# liefert). Erfasst pro Tool-Runde — mehrere Runden = mehrere Calls.
try:
metrics.log_local_call(res.get("model") or local_model, messages,
res.get("content") or "", res.get("usage"))
except Exception:
pass
tcs = res.get("tool_calls") tcs = res.get("tool_calls")
if tcs: if tcs:
messages.append({"role": "assistant", messages.append({"role": "assistant",
@@ -1325,6 +1333,11 @@ class Agent:
self.conversation.add("assistant", fast_reply, project_id=active_project_id) self.conversation.add("assistant", fast_reply, project_id=active_project_id)
if active_project_id: if active_project_id:
projects_mod.touch_project(active_project_id) projects_mod.touch_project(active_project_id)
# Metric: Fast-Path spart einen ganzen Claude-Call zum Nulltarif.
try:
metrics.log_fast_path(fast_reply)
except Exception:
pass
# Fast-Path = reiner Steuerbefehl → NICHT vorlesen (speak=False). # Fast-Path = reiner Steuerbefehl → NICHT vorlesen (speak=False).
# System-Flag statt <voice>-Tag: robust, unabhaengig vom Skill-Inhalt. # System-Flag statt <voice>-Tag: robust, unabhaengig vom Skill-Inhalt.
return fast_reply, "fast-path", False return fast_reply, "fast-path", False
+54 -10
View File
@@ -52,25 +52,55 @@ def _messages_tokens(messages: list) -> int:
return total return total
def log_call(model: str, messages_in: list, reply_text: str = "") -> None: def _append(model: str, tokens_in: int, tokens_out: int, source: str) -> None:
"""Eine Call-Metric anhaengen. Robust gegen Fehler (silent fail).""" """Ein Metric-Entry auf Disk anhaengen. Robust (silent fail)."""
try: try:
tokens_in = _messages_tokens(messages_in)
tokens_out = _estimate_tokens(reply_text)
line = json.dumps({ line = json.dumps({
"ts": int(time.time() * 1000), "ts": int(time.time() * 1000),
"model": model, "model": model,
"in": tokens_in, "in": int(tokens_in),
"out": tokens_out, "out": int(tokens_out),
"source": source, # "claude" | "local" | "fast-path"
}) })
METRICS_FILE.parent.mkdir(parents=True, exist_ok=True) METRICS_FILE.parent.mkdir(parents=True, exist_ok=True)
with METRICS_FILE.open("a", encoding="utf-8") as f: with METRICS_FILE.open("a", encoding="utf-8") as f:
f.write(line + "\n") f.write(line + "\n")
# Sanftes Rotate ohne hohe IO-Kosten — nur alle 1000 Calls checken
if (tokens_in + tokens_out) % 1000 < 4: if (tokens_in + tokens_out) % 1000 < 4:
_maybe_rotate() _maybe_rotate()
except Exception as exc: except Exception as exc:
logger.warning("metrics.log_call: %s", exc) logger.warning("metrics._append: %s", exc)
def log_call(model: str, messages_in: list, reply_text: str = "",
source: str = "claude") -> None:
"""Claude-Call-Metric anhaengen (Tokens per chars/4-Schaetzung)."""
_append(model, _messages_tokens(messages_in), _estimate_tokens(reply_text), source)
def log_local_call(model: str, messages_in: list, reply_text: str = "",
usage: dict | None = None) -> None:
"""Lokaler-LLM-Call-Metric. Nutzt echte usage-Tokens (prompt/completion)
wenn der Adapter sie liefert, sonst chars/4-Schaetzung wie bei Claude.
Quelle = 'local' — damit die Ersparnis-Rechnung local von claude trennt."""
tokens_in = tokens_out = None
if isinstance(usage, dict):
pt = usage.get("prompt_tokens")
ct = usage.get("completion_tokens")
if isinstance(pt, (int, float)):
tokens_in = int(pt)
if isinstance(ct, (int, float)):
tokens_out = int(ct)
if tokens_in is None:
tokens_in = _messages_tokens(messages_in)
if tokens_out is None:
tokens_out = _estimate_tokens(reply_text)
_append(model or "local", tokens_in, tokens_out, "local")
def log_fast_path(reply_text: str = "") -> None:
"""Fast-Path (reiner Skill, KEIN LLM) — spart einen ganzen Claude-Call zum
Nulltarif. tokens_in=0 (kein Prompt ans LLM), out = winzige Quittung."""
_append("fast-path", 0, _estimate_tokens(reply_text), "fast-path")
def _maybe_rotate() -> None: def _maybe_rotate() -> None:
@@ -95,6 +125,11 @@ def aggregate(window_seconds: int) -> dict:
tokens_in = 0 tokens_in = 0
tokens_out = 0 tokens_out = 0
by_model: dict[str, int] = {} by_model: dict[str, int] = {}
# Aufschluesselung nach Quelle (claude / local / fast-path) fuer die
# Ersparnis-Anzeige im Diagnostic.
def _src_bucket() -> dict:
return {"calls": 0, "tokens_in": 0, "tokens_out": 0}
by_source: dict[str, dict] = {}
if METRICS_FILE.exists(): if METRICS_FILE.exists():
try: try:
for raw in METRICS_FILE.read_text(encoding="utf-8").splitlines(): for raw in METRICS_FILE.read_text(encoding="utf-8").splitlines():
@@ -107,11 +142,19 @@ def aggregate(window_seconds: int) -> dict:
continue continue
if obj.get("ts", 0) < cutoff_ms: if obj.get("ts", 0) < cutoff_ms:
continue continue
ti = int(obj.get("in") or 0)
to = int(obj.get("out") or 0)
calls += 1 calls += 1
tokens_in += int(obj.get("in") or 0) tokens_in += ti
tokens_out += int(obj.get("out") or 0) tokens_out += to
m = obj.get("model", "?") m = obj.get("model", "?")
by_model[m] = by_model.get(m, 0) + 1 by_model[m] = by_model.get(m, 0) + 1
# Alt-Eintraege ohne 'source' zaehlen als claude (Rueckwaerts-Kompat).
src = obj.get("source") or "claude"
b = by_source.setdefault(src, _src_bucket())
b["calls"] += 1
b["tokens_in"] += ti
b["tokens_out"] += to
except Exception as exc: except Exception as exc:
logger.warning("metrics aggregate: %s", exc) logger.warning("metrics aggregate: %s", exc)
return { return {
@@ -120,6 +163,7 @@ def aggregate(window_seconds: int) -> dict:
"tokens_in": tokens_in, "tokens_in": tokens_in,
"tokens_out": tokens_out, "tokens_out": tokens_out,
"by_model": by_model, "by_model": by_model,
"by_source": by_source,
} }
+44
View File
@@ -1028,6 +1028,18 @@
</div> </div>
</div> </div>
<div class="settings-section">
<h2>Lokales LLM & Claude-Ersparnis <button class="info-btn" onclick="showInfo('local-savings')" title="Wie wird die Ersparnis gerechnet?"></button></h2>
<div class="card">
<div style="font-size:11px;color:#8888AA;margin-bottom:10px;">
Turns, die das <strong style="color:#B392F0;">lokale LLM</strong> (Qwen) oder ein reiner <strong style="color:#F0B85E;">Skill-Fast-Path</strong> uebernommen hat — jeder davon ist ein Claude-Call, der NICHT passiert ist. Die lokalen Tokens laufen auf deiner eigenen Hardware (kein Subscription-Quota).
</div>
<div id="savings-grid" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:8px;font-size:12px;">
<div class="metric-cell"><div class="metric-label"></div><div class="metric-value"></div></div>
</div>
</div>
</div>
<div class="settings-section"> <div class="settings-section">
<h2>Bootstrap & Migration <button class="info-btn" onclick="showInfo('bootstrap')" title="Was sind die drei Wege?"></button></h2> <h2>Bootstrap & Migration <button class="info-btn" onclick="showInfo('bootstrap')" title="Was sind die drei Wege?"></button></h2>
<div class="card" style="line-height:1.6;"> <div class="card" style="line-height:1.6;">
@@ -5897,6 +5909,29 @@
setCell('metrics-h24', d.h24); setCell('metrics-h24', d.h24);
setCell('metrics-d30', d.d30); setCell('metrics-d30', d.d30);
// Lokales LLM & Claude-Ersparnis — pro Fenster: wie viele Claude-Calls
// durch local/fast-path vermieden wurden + lokale Token-Last.
const savingsGrid = document.getElementById('savings-grid');
if (savingsGrid) {
const wins = [['letzte 1h', d.h1], ['letzte 5h', d.h5],
['letzte 24h', d.h24], ['letzte 30 Tage', d.d30]];
const z = { calls: 0, tokens_in: 0, tokens_out: 0 };
savingsGrid.innerHTML = wins.map(([label, w]) => {
const bs = (w && w.by_source) || {};
const loc = bs.local || z;
const fp = bs['fast-path'] || z;
const saved = (loc.calls || 0) + (fp.calls || 0);
const locTok = (loc.tokens_in || 0) + (loc.tokens_out || 0);
const color = saved > 0 ? '#3FB950' : '#555570';
return `<div class="metric-cell">
<div class="metric-label">${label}</div>
<div class="metric-value" style="color:${color};">${saved} Claude-Calls gespart</div>
<div class="metric-sub">lokal ${loc.calls || 0} · fast-path ${fp.calls || 0}</div>
<div class="metric-sub">${fmtTokens(locTok)} lokale Tokens (eigene HW)</div>
</div>`;
}).join('');
}
// 5h-Fenster gegen Plan-Limit: Warn-Klassen // 5h-Fenster gegen Plan-Limit: Warn-Klassen
const plan = getActivePlanLimit(); const plan = getActivePlanLimit();
const limit = plan.h5; const limit = plan.h5;
@@ -5939,6 +5974,15 @@
// Vor-definierte Info-Blocks // Vor-definierte Info-Blocks
const INFO_TEXTS = { const INFO_TEXTS = {
'local-savings': {
title: 'Lokales LLM & Claude-Ersparnis',
html: `
<p>Jeder Turn, den das <strong>lokale LLM</strong> oder ein <strong>Fast-Path</strong> (reiner Skill, ganz ohne LLM) beantwortet, ist ein Claude-Call, der <strong>nicht</strong> gegen dein Subscription-Quota laeuft.</p>
<p><strong>„Claude-Calls gespart"</strong> = Anzahl der local- + fast-path-Antworten im Zeitfenster. Konservativ gezaehlt: 1 Antwort = mindestens 1 gesparter Claude-Call (bei Tool-Use waeren es real oft mehr).</p>
<p><strong>„lokale Tokens"</strong> = Prompt+Antwort-Tokens, die auf deiner eigenen Gamebox-GPU verarbeitet wurden (echte <code>usage</code>-Zahlen vom Modell, sonst chars/4-Schaetzung). Die kosten dich nichts ausser Strom.</p>
<p>Fast-Path-Antworten (z.B. „nächstes Lied") haben ~0 Tokens — reiner Skill-Aufruf, kein Modell.</p>
`,
},
'local-llm': { 'local-llm': {
title: 'Lokales LLM — schnelle Antworten', title: 'Lokales LLM — schnelle Antworten',
html: ` html: `