feat(metrics): lokaler LLM-Verbrauch + Claude-Ersparnis in Diagnostic
metrics.jsonl-Eintraege tragen jetzt 'source' (claude|local|fast-path). - log_local_call(): echte usage-Tokens vom Adapter (prompt/completion), sonst chars/4-Schaetzung. Geloggt pro Tool-Runde im lokalen Fast-Lane. - log_fast_path(): reiner Skill, 0 Prompt-Tokens — gesparter Claude-Call. - aggregate() liefert zusaetzlich by_source (calls/tokens_in/tokens_out). Alt-Eintraege ohne source zaehlen als claude (rueckwaerts-kompatibel). Diagnostic Gehirn-Tab: neue Card "Lokales LLM & Claude-Ersparnis" — pro Fenster (1h/5h/24h/30d) gesparte Claude-Calls (local + fast-path) und lokale Token-Last (eigene HW, kein Quota) + Info-Block. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -30,6 +30,7 @@ from memory import Embedder, VectorStore, MemoryPoint
|
||||
from prompts import build_system_prompt, IDENTITY_SEED, IDENTITY_ANCHOR, looks_like_identity_break
|
||||
from proxy_client import ProxyClient, Message as ProxyMessage
|
||||
import router as router_mod
|
||||
import metrics
|
||||
from local_llm import local_llm_chat
|
||||
import skills as skills_mod
|
||||
import triggers as triggers_mod
|
||||
@@ -1220,6 +1221,13 @@ class Agent:
|
||||
if local_only:
|
||||
return f"[Lokales LLM nicht erreichbar: {res.get('error', 'unbekannt')}]"
|
||||
return None
|
||||
# Metric: dieser lokale Call (echte usage-Tokens wenn der Adapter sie
|
||||
# liefert). Erfasst pro Tool-Runde — mehrere Runden = mehrere Calls.
|
||||
try:
|
||||
metrics.log_local_call(res.get("model") or local_model, messages,
|
||||
res.get("content") or "", res.get("usage"))
|
||||
except Exception:
|
||||
pass
|
||||
tcs = res.get("tool_calls")
|
||||
if tcs:
|
||||
messages.append({"role": "assistant",
|
||||
@@ -1325,6 +1333,11 @@ class Agent:
|
||||
self.conversation.add("assistant", fast_reply, project_id=active_project_id)
|
||||
if active_project_id:
|
||||
projects_mod.touch_project(active_project_id)
|
||||
# Metric: Fast-Path spart einen ganzen Claude-Call zum Nulltarif.
|
||||
try:
|
||||
metrics.log_fast_path(fast_reply)
|
||||
except Exception:
|
||||
pass
|
||||
# Fast-Path = reiner Steuerbefehl → NICHT vorlesen (speak=False).
|
||||
# System-Flag statt <voice>-Tag: robust, unabhaengig vom Skill-Inhalt.
|
||||
return fast_reply, "fast-path", False
|
||||
|
||||
+54
-10
@@ -52,25 +52,55 @@ def _messages_tokens(messages: list) -> int:
|
||||
return total
|
||||
|
||||
|
||||
def log_call(model: str, messages_in: list, reply_text: str = "") -> None:
|
||||
"""Eine Call-Metric anhaengen. Robust gegen Fehler (silent fail)."""
|
||||
def _append(model: str, tokens_in: int, tokens_out: int, source: str) -> None:
|
||||
"""Ein Metric-Entry auf Disk anhaengen. Robust (silent fail)."""
|
||||
try:
|
||||
tokens_in = _messages_tokens(messages_in)
|
||||
tokens_out = _estimate_tokens(reply_text)
|
||||
line = json.dumps({
|
||||
"ts": int(time.time() * 1000),
|
||||
"model": model,
|
||||
"in": tokens_in,
|
||||
"out": tokens_out,
|
||||
"in": int(tokens_in),
|
||||
"out": int(tokens_out),
|
||||
"source": source, # "claude" | "local" | "fast-path"
|
||||
})
|
||||
METRICS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with METRICS_FILE.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
# Sanftes Rotate ohne hohe IO-Kosten — nur alle 1000 Calls checken
|
||||
if (tokens_in + tokens_out) % 1000 < 4:
|
||||
_maybe_rotate()
|
||||
except Exception as exc:
|
||||
logger.warning("metrics.log_call: %s", exc)
|
||||
logger.warning("metrics._append: %s", exc)
|
||||
|
||||
|
||||
def log_call(model: str, messages_in: list, reply_text: str = "",
|
||||
source: str = "claude") -> None:
|
||||
"""Claude-Call-Metric anhaengen (Tokens per chars/4-Schaetzung)."""
|
||||
_append(model, _messages_tokens(messages_in), _estimate_tokens(reply_text), source)
|
||||
|
||||
|
||||
def log_local_call(model: str, messages_in: list, reply_text: str = "",
|
||||
usage: dict | None = None) -> None:
|
||||
"""Lokaler-LLM-Call-Metric. Nutzt echte usage-Tokens (prompt/completion)
|
||||
wenn der Adapter sie liefert, sonst chars/4-Schaetzung wie bei Claude.
|
||||
Quelle = 'local' — damit die Ersparnis-Rechnung local von claude trennt."""
|
||||
tokens_in = tokens_out = None
|
||||
if isinstance(usage, dict):
|
||||
pt = usage.get("prompt_tokens")
|
||||
ct = usage.get("completion_tokens")
|
||||
if isinstance(pt, (int, float)):
|
||||
tokens_in = int(pt)
|
||||
if isinstance(ct, (int, float)):
|
||||
tokens_out = int(ct)
|
||||
if tokens_in is None:
|
||||
tokens_in = _messages_tokens(messages_in)
|
||||
if tokens_out is None:
|
||||
tokens_out = _estimate_tokens(reply_text)
|
||||
_append(model or "local", tokens_in, tokens_out, "local")
|
||||
|
||||
|
||||
def log_fast_path(reply_text: str = "") -> None:
|
||||
"""Fast-Path (reiner Skill, KEIN LLM) — spart einen ganzen Claude-Call zum
|
||||
Nulltarif. tokens_in=0 (kein Prompt ans LLM), out = winzige Quittung."""
|
||||
_append("fast-path", 0, _estimate_tokens(reply_text), "fast-path")
|
||||
|
||||
|
||||
def _maybe_rotate() -> None:
|
||||
@@ -95,6 +125,11 @@ def aggregate(window_seconds: int) -> dict:
|
||||
tokens_in = 0
|
||||
tokens_out = 0
|
||||
by_model: dict[str, int] = {}
|
||||
# Aufschluesselung nach Quelle (claude / local / fast-path) fuer die
|
||||
# Ersparnis-Anzeige im Diagnostic.
|
||||
def _src_bucket() -> dict:
|
||||
return {"calls": 0, "tokens_in": 0, "tokens_out": 0}
|
||||
by_source: dict[str, dict] = {}
|
||||
if METRICS_FILE.exists():
|
||||
try:
|
||||
for raw in METRICS_FILE.read_text(encoding="utf-8").splitlines():
|
||||
@@ -107,11 +142,19 @@ def aggregate(window_seconds: int) -> dict:
|
||||
continue
|
||||
if obj.get("ts", 0) < cutoff_ms:
|
||||
continue
|
||||
ti = int(obj.get("in") or 0)
|
||||
to = int(obj.get("out") or 0)
|
||||
calls += 1
|
||||
tokens_in += int(obj.get("in") or 0)
|
||||
tokens_out += int(obj.get("out") or 0)
|
||||
tokens_in += ti
|
||||
tokens_out += to
|
||||
m = obj.get("model", "?")
|
||||
by_model[m] = by_model.get(m, 0) + 1
|
||||
# Alt-Eintraege ohne 'source' zaehlen als claude (Rueckwaerts-Kompat).
|
||||
src = obj.get("source") or "claude"
|
||||
b = by_source.setdefault(src, _src_bucket())
|
||||
b["calls"] += 1
|
||||
b["tokens_in"] += ti
|
||||
b["tokens_out"] += to
|
||||
except Exception as exc:
|
||||
logger.warning("metrics aggregate: %s", exc)
|
||||
return {
|
||||
@@ -120,6 +163,7 @@ def aggregate(window_seconds: int) -> dict:
|
||||
"tokens_in": tokens_in,
|
||||
"tokens_out": tokens_out,
|
||||
"by_model": by_model,
|
||||
"by_source": by_source,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1028,6 +1028,18 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="settings-section">
|
||||
<h2>Lokales LLM & Claude-Ersparnis <button class="info-btn" onclick="showInfo('local-savings')" title="Wie wird die Ersparnis gerechnet?">ℹ</button></h2>
|
||||
<div class="card">
|
||||
<div style="font-size:11px;color:#8888AA;margin-bottom:10px;">
|
||||
Turns, die das <strong style="color:#B392F0;">lokale LLM</strong> (Qwen) oder ein reiner <strong style="color:#F0B85E;">Skill-Fast-Path</strong> uebernommen hat — jeder davon ist ein Claude-Call, der NICHT passiert ist. Die lokalen Tokens laufen auf deiner eigenen Hardware (kein Subscription-Quota).
|
||||
</div>
|
||||
<div id="savings-grid" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:8px;font-size:12px;">
|
||||
<div class="metric-cell"><div class="metric-label">–</div><div class="metric-value">–</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="settings-section">
|
||||
<h2>Bootstrap & Migration <button class="info-btn" onclick="showInfo('bootstrap')" title="Was sind die drei Wege?">ℹ</button></h2>
|
||||
<div class="card" style="line-height:1.6;">
|
||||
@@ -5897,6 +5909,29 @@
|
||||
setCell('metrics-h24', d.h24);
|
||||
setCell('metrics-d30', d.d30);
|
||||
|
||||
// Lokales LLM & Claude-Ersparnis — pro Fenster: wie viele Claude-Calls
|
||||
// durch local/fast-path vermieden wurden + lokale Token-Last.
|
||||
const savingsGrid = document.getElementById('savings-grid');
|
||||
if (savingsGrid) {
|
||||
const wins = [['letzte 1h', d.h1], ['letzte 5h', d.h5],
|
||||
['letzte 24h', d.h24], ['letzte 30 Tage', d.d30]];
|
||||
const z = { calls: 0, tokens_in: 0, tokens_out: 0 };
|
||||
savingsGrid.innerHTML = wins.map(([label, w]) => {
|
||||
const bs = (w && w.by_source) || {};
|
||||
const loc = bs.local || z;
|
||||
const fp = bs['fast-path'] || z;
|
||||
const saved = (loc.calls || 0) + (fp.calls || 0);
|
||||
const locTok = (loc.tokens_in || 0) + (loc.tokens_out || 0);
|
||||
const color = saved > 0 ? '#3FB950' : '#555570';
|
||||
return `<div class="metric-cell">
|
||||
<div class="metric-label">${label}</div>
|
||||
<div class="metric-value" style="color:${color};">${saved} Claude-Calls gespart</div>
|
||||
<div class="metric-sub">lokal ${loc.calls || 0} · fast-path ${fp.calls || 0}</div>
|
||||
<div class="metric-sub">${fmtTokens(locTok)} lokale Tokens (eigene HW)</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// 5h-Fenster gegen Plan-Limit: Warn-Klassen
|
||||
const plan = getActivePlanLimit();
|
||||
const limit = plan.h5;
|
||||
@@ -5939,6 +5974,15 @@
|
||||
|
||||
// Vor-definierte Info-Blocks
|
||||
const INFO_TEXTS = {
|
||||
'local-savings': {
|
||||
title: 'Lokales LLM & Claude-Ersparnis',
|
||||
html: `
|
||||
<p>Jeder Turn, den das <strong>lokale LLM</strong> oder ein <strong>Fast-Path</strong> (reiner Skill, ganz ohne LLM) beantwortet, ist ein Claude-Call, der <strong>nicht</strong> gegen dein Subscription-Quota laeuft.</p>
|
||||
<p><strong>„Claude-Calls gespart"</strong> = Anzahl der local- + fast-path-Antworten im Zeitfenster. Konservativ gezaehlt: 1 Antwort = mindestens 1 gesparter Claude-Call (bei Tool-Use waeren es real oft mehr).</p>
|
||||
<p><strong>„lokale Tokens"</strong> = Prompt+Antwort-Tokens, die auf deiner eigenen Gamebox-GPU verarbeitet wurden (echte <code>usage</code>-Zahlen vom Modell, sonst chars/4-Schaetzung). Die kosten dich nichts ausser Strom.</p>
|
||||
<p>Fast-Path-Antworten (z.B. „nächstes Lied") haben ~0 Tokens — reiner Skill-Aufruf, kein Modell.</p>
|
||||
`,
|
||||
},
|
||||
'local-llm': {
|
||||
title: 'Lokales LLM — schnelle Antworten',
|
||||
html: `
|
||||
|
||||
Reference in New Issue
Block a user