feat(llm): Kontext-Auswahl zeigt den Grafikspeicher und geht bis zum Modell-Maximum
Der Adapter liest den Kopf der GGUF-Datei (nur Metadaten): trainiertes Maximum (context_length) und KV-Cache pro Token — bei Sliding-Window-Modellen (Gemma) waechst nur der globale Teil mit dem Kontext. Daraus pro Stufe eine Schaetzung Modell + mmproj + KV (f16) + 1,4 GB Aufschlag (gemessen: Gemma 4 12B @32k geschaetzt 8,8 GB, belegt 8,77 GB). Diagnostic: Liste kommt von der Box (statt fest 4k..128k), bis zum Maximum des Modells (auch krumme wie 40960), mit "ca. X GB" und "passt nicht", wenn es die Karte sprengt. Alte Box ohne Angaben: feste Liste wie bisher. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+140
-1
@@ -517,6 +517,144 @@ def _model_ctx(key: str) -> int:
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
|
||||
# ── Kontextfenster: was das Modell kann + was es an Grafikspeicher kostet ──
|
||||
#
|
||||
# Aus dem Kopf der GGUF-Datei (nur Metadaten, die Gewichte werden nicht
|
||||
# gelesen): trainiertes Maximum (context_length) und die Groesse des KV-Caches
|
||||
# pro Token. Modelle mit Sliding-Window-Attention (Gemma) halten in den
|
||||
# "lokalen" Schichten nur ein festes Fenster — nur die globalen Schichten
|
||||
# wachsen mit dem Kontext. Annahme: KV-Cache in f16 (llama.cpp-Standard).
|
||||
|
||||
_GGUF_T = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
|
||||
_gguf_cache: dict = {} # path → (mtime, meta)
|
||||
CTX_STEPS = [4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576]
|
||||
# Fester Aufschlag (CUDA-Kontext + Rechenpuffer von llama.cpp) — gemessen auf
|
||||
# einer RTX 3060 mit Gemma 4 12B bei 32k: ~1,4 GB ueber Modell + KV-Cache.
|
||||
VRAM_OVERHEAD_MB = 1400
|
||||
UBATCH = 512
|
||||
|
||||
|
||||
def _gguf_meta(path: str) -> dict:
|
||||
"""Metadaten-Block einer GGUF-Datei (gecacht). Lange String-Listen (der
|
||||
Tokenizer) werden uebersprungen, nicht gespeichert."""
|
||||
try:
|
||||
mt = os.path.getmtime(path)
|
||||
except OSError:
|
||||
return {}
|
||||
hit = _gguf_cache.get(path)
|
||||
if hit and hit[0] == mt:
|
||||
return hit[1]
|
||||
import struct
|
||||
meta: dict = {}
|
||||
|
||||
def rd(f, fmt):
|
||||
return struct.unpack("<" + fmt, f.read(struct.calcsize("<" + fmt)))[0]
|
||||
|
||||
def rstr(f):
|
||||
return f.read(rd(f, "Q")).decode("utf-8", "replace")
|
||||
|
||||
def rval(f, t):
|
||||
if t == 8:
|
||||
return rstr(f)
|
||||
if t == 9:
|
||||
et, n = rd(f, "I"), rd(f, "Q")
|
||||
if et == 8:
|
||||
for _ in range(n):
|
||||
f.seek(rd(f, "Q"), 1)
|
||||
return None
|
||||
if et == 9:
|
||||
raise ValueError("verschachtelte Arrays")
|
||||
fmt = _GGUF_T[et]
|
||||
if n > 4096:
|
||||
f.seek(struct.calcsize("<" + fmt) * n, 1)
|
||||
return None
|
||||
return [rd(f, fmt) for _ in range(n)]
|
||||
return rd(f, _GGUF_T[t])
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
if f.read(4) != b"GGUF":
|
||||
return {}
|
||||
rd(f, "I"); rd(f, "Q")
|
||||
for _ in range(rd(f, "Q")):
|
||||
k = rstr(f)
|
||||
v = rval(f, rd(f, "I"))
|
||||
if v is not None:
|
||||
meta[k] = v
|
||||
except Exception as e:
|
||||
logger.debug("GGUF %s nicht lesbar: %s", path, e)
|
||||
meta = {}
|
||||
_gguf_cache[path] = (mt, meta)
|
||||
return meta
|
||||
|
||||
|
||||
def _model_files(key: str) -> tuple:
|
||||
"""(Gewichte.gguf, mmproj.gguf|None) eines Modells im HF-Cache unter /models
|
||||
— aus "-hf org/repo:QUANT" bzw. "-m pfad" im llama-server-Aufruf."""
|
||||
import glob
|
||||
cmd = _model_cmd(key) or ""
|
||||
m = re.search(r"(?:^|\s)-m\s+(\S+)", cmd)
|
||||
if m and os.path.isfile(m.group(1)):
|
||||
return m.group(1), None
|
||||
m = re.search(r"(?:^|\s)-hf\s+([^\s:]+)(?::(\S+))?", cmd)
|
||||
if not m:
|
||||
return None, None
|
||||
repo, quant = m.group(1), (m.group(2) or "").lower()
|
||||
base = os.path.join(MODELS_DIR, "models--" + repo.replace("/", "--"), "snapshots")
|
||||
files = [p for p in glob.glob(os.path.join(base, "*", "*.gguf")) if os.path.isfile(p)]
|
||||
weights = [p for p in files if "mmproj" not in os.path.basename(p).lower()]
|
||||
if quant:
|
||||
weights = [p for p in weights if quant in os.path.basename(p).lower()] or weights
|
||||
mmproj = [p for p in files if "mmproj" in os.path.basename(p).lower()]
|
||||
return (weights[0] if weights else None), (mmproj[0] if mmproj else None)
|
||||
|
||||
|
||||
def _ctx_profile(key: str) -> dict:
|
||||
"""{ctxMax, options: [{ctx, estMB}]} fuer die Kontext-Auswahl der Diagnostic.
|
||||
Leer, wenn die Modelldatei (noch) nicht da oder nicht lesbar ist."""
|
||||
weights, mmproj = _model_files(key)
|
||||
if not weights:
|
||||
return {}
|
||||
meta = _gguf_meta(weights)
|
||||
arch = meta.get("general.architecture")
|
||||
if not arch:
|
||||
return {}
|
||||
g = lambda k, d=None: meta.get(f"{arch}.{k}", d)
|
||||
n_layer = int(g("block_count", 0) or 0)
|
||||
if not n_layer:
|
||||
return {}
|
||||
|
||||
def per_layer(v, d=0):
|
||||
return list(v) if isinstance(v, list) else [v if v is not None else d] * n_layer
|
||||
|
||||
kv_heads = per_layer(g("attention.head_count_kv", g("attention.head_count", 0)))
|
||||
k_len = int(g("attention.key_length", 0) or 0)
|
||||
v_len = int(g("attention.value_length", 0) or 0)
|
||||
if not (k_len and v_len):
|
||||
emb, heads = int(g("embedding_length", 0) or 0), int(g("attention.head_count", 0) or 0)
|
||||
k_len = v_len = emb // heads if heads else 0
|
||||
k_swa = int(g("attention.key_length_swa", k_len) or k_len)
|
||||
v_swa = int(g("attention.value_length_swa", v_len) or v_len)
|
||||
window = int(g("attention.sliding_window", 0) or 0)
|
||||
pattern = g("attention.sliding_window_pattern")
|
||||
swa = per_layer(pattern, False) if isinstance(pattern, list) else [False] * n_layer
|
||||
if not window:
|
||||
swa = [False] * n_layer
|
||||
|
||||
per_tok = sum(int(kv_heads[i]) * (k_len + v_len) * 2 for i in range(n_layer) if not swa[i])
|
||||
swa_tok = sum(int(kv_heads[i]) * (k_swa + v_swa) * 2 for i in range(n_layer) if swa[i])
|
||||
files_mb = (os.path.getsize(weights) + (os.path.getsize(mmproj) if mmproj else 0)) / 1048576
|
||||
ctx_max = int(g("context_length", 0) or 0)
|
||||
steps = [c for c in CTX_STEPS if not ctx_max or c <= ctx_max]
|
||||
if ctx_max and ctx_max not in steps:
|
||||
steps.append(ctx_max) # krummes Maximum (z.B. 40960) mit anbieten
|
||||
opts = []
|
||||
for c in steps:
|
||||
kv_mb = (per_tok * c + swa_tok * min(c, window + UBATCH)) / 1048576
|
||||
opts.append({"ctx": c, "estMB": int(files_mb + kv_mb + VRAM_OVERHEAD_MB)})
|
||||
return {"ctxMax": ctx_max, "options": opts}
|
||||
|
||||
|
||||
async def _vram_used() -> dict:
|
||||
"""{index: belegte MB} — frisch bei jedem Aufruf (aendert sich laufend)."""
|
||||
try:
|
||||
@@ -555,7 +693,8 @@ async def _slots_info() -> list:
|
||||
on_cpu = bool(m and state == "ready" and u is not None and u < ON_CPU_BELOW_MB)
|
||||
out.append({"gpu": card, "gpuName": g.get("name", ""), "vramMB": g.get("vramMB", 0),
|
||||
"vramUsedMB": u if u is not None else -1, "onCpu": on_cpu,
|
||||
"model": m, "state": state, "ctx": _model_ctx(m) if m else 0})
|
||||
"model": m, "state": state, "ctx": _model_ctx(m) if m else 0,
|
||||
**(_ctx_profile(m) if m else {})})
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user