feat(local-llm): B0.5 — llama-swap + lokale Modellauswahl in Diagnostic
Mehrere lokale Modelle, on-demand geladen/geswappt, in Diagnostic waehlbar. Design: das Brain schickt den Modellnamen (aus local_llm.json) im llm_request mit -> Adapter -> llama-swap laedt/swappt. Keine separate Gamebox-Config noetig. - xtts: `llama`-Container -> `llama-swap` (unified-cuda), config.yaml mit qwen3-8b (Standard) + qwen3-4b; Auto-Download via -hf, Cache /models geteilt (qwen3-8b schon da). Adapter -> llama-swap:8080, Timeout 600s (Erst-Download). - adapter: `model` aus dem Request an llama-swap durchreichen (Fallback env). - brain: router.load_config liest localLlmModel; local_llm_chat(model=...); agent gibt cfg-Modell mit; bridge reicht model durch (_local_llm + Route). - diagnostic: /api/local-models-list (aus /shared/config/local_models.json, seeded), local-llm-config um localLlmModel erweitert; Dropdown "Lokales Modell" im Settings-Block + Erst-Download-Hinweis. BLIND gebaut (Gamebox nicht testbar hier): llama-swap CLI/Config-Pfad beim ersten Start via `docker logs aria-llama-swap` pruefen. Live-Lade-Status (Adapter->Diagnostic) ist B0.5-2 (Folgeschritt). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+3
-1
@@ -1171,6 +1171,7 @@ class Agent:
|
||||
if not router_mod.should_try_local(user_message, cfg):
|
||||
return None
|
||||
local_only = bool(cfg.get("localOnly"))
|
||||
local_model = cfg.get("localLlmModel") or "qwen3-8b" # B0.5: llama-swap-Key
|
||||
tools = self._build_local_tools() # B1b: kuratierte Tools
|
||||
|
||||
sys_prompt = router_mod.build_local_system_prompt(IDENTITY_ANCHOR,
|
||||
@@ -1185,7 +1186,8 @@ class Agent:
|
||||
# Spotify aufrufen. Ergebnisse zurueck, bis es final (ohne tool_calls) antwortet.
|
||||
final = ""
|
||||
for _ in range(self._LOCAL_TOOL_ITERATIONS):
|
||||
res = local_llm_chat(messages, max_tokens=500, temperature=0.5, tools=tools)
|
||||
res = local_llm_chat(messages, max_tokens=500, temperature=0.5,
|
||||
tools=tools, model=local_model)
|
||||
if not res.get("ok"):
|
||||
logger.info("[router] lokal fehlgeschlagen (%s) — %s", res.get("error"),
|
||||
"kein Fallback (localOnly)" if local_only else "→ Claude")
|
||||
|
||||
@@ -29,11 +29,12 @@ LOCAL_LLM_HTTP_TIMEOUT_SEC = float(os.environ.get("LOCAL_LLM_HTTP_TIMEOUT_SEC",
|
||||
|
||||
|
||||
def local_llm_chat(messages: list, *, max_tokens: int = 512,
|
||||
temperature: float = 0.7, stop=None, tools=None) -> dict:
|
||||
temperature: float = 0.7, stop=None, tools=None,
|
||||
model=None) -> dict:
|
||||
"""Ein Chat-Call ans lokale LLM. messages = [{role, content}, ...].
|
||||
tools (B1b): optionale OpenAI-Tool-Defs; das Ergebnis kann dann
|
||||
result['tool_calls'] enthalten. Blockierend (urllib) — chat() laeuft
|
||||
ohnehin im Executor-Thread."""
|
||||
model (B0.5): welches Modell llama-swap laden soll. tools (B1b): optionale
|
||||
OpenAI-Tool-Defs; das Ergebnis kann dann result['tool_calls'] enthalten.
|
||||
Blockierend (urllib) — chat() laeuft ohnehin im Executor-Thread."""
|
||||
if not isinstance(messages, list) or not messages:
|
||||
return {"ok": False, "error": "messages leer/ungueltig"}
|
||||
req = {"messages": messages, "max_tokens": max_tokens, "temperature": temperature}
|
||||
@@ -41,6 +42,8 @@ def local_llm_chat(messages: list, *, max_tokens: int = 512,
|
||||
req["stop"] = stop
|
||||
if tools:
|
||||
req["tools"] = tools
|
||||
if model:
|
||||
req["model"] = model
|
||||
try:
|
||||
body = json.dumps(req).encode("utf-8")
|
||||
http_req = urllib.request.Request(
|
||||
|
||||
@@ -30,7 +30,8 @@ CONFIG_PATH = os.environ.get("LOCAL_LLM_CONFIG", "/shared/config/local_llm.json"
|
||||
|
||||
ESCALATE_MARKER = "<<ESCALATE>>"
|
||||
|
||||
DEFAULT_CONFIG = {"enabled": False, "localOnly": False, "toolVariant": "slim"}
|
||||
DEFAULT_CONFIG = {"enabled": False, "localOnly": False,
|
||||
"toolVariant": "slim", "localLlmModel": "qwen3-8b"}
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
@@ -42,6 +43,9 @@ def load_config() -> dict:
|
||||
"enabled": bool(data.get("enabled", False)),
|
||||
"localOnly": bool(data.get("localOnly", False)),
|
||||
"toolVariant": data.get("toolVariant", "slim") or "slim",
|
||||
# Welches lokale Modell llama-swap laden soll (B0.5). Muss zu einem
|
||||
# Key in xtts/llama-swap/config.yaml passen.
|
||||
"localLlmModel": (data.get("localLlmModel") or "qwen3-8b").strip(),
|
||||
}
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return dict(DEFAULT_CONFIG)
|
||||
|
||||
@@ -3375,7 +3375,8 @@ class ARIABridge:
|
||||
_LLM_TIMEOUT_S = 30.0
|
||||
|
||||
async def _local_llm(self, messages: list, max_tokens: int = 512,
|
||||
temperature: float = 0.7, stop=None, tools=None) -> dict:
|
||||
temperature: float = 0.7, stop=None, tools=None,
|
||||
model=None) -> dict:
|
||||
"""Schickt einen llm_request an den llm-adapter (Gamebox), wartet auf
|
||||
llm_response. tools (B1b) werden durchgereicht; tool_calls kommen zurueck.
|
||||
Rueckgabe: {ok, content, tool_calls, model, elapsedMs} oder {ok:False, error}."""
|
||||
@@ -3399,8 +3400,10 @@ class ARIABridge:
|
||||
req_payload["stop"] = stop
|
||||
if tools:
|
||||
req_payload["tools"] = tools
|
||||
logger.info("[rvs] llm_request → llm-adapter (id=%s, msgs=%d, max_tokens=%d, tools=%d)",
|
||||
request_id[:8], len(messages), max_tokens, len(tools) if tools else 0)
|
||||
if model:
|
||||
req_payload["model"] = model
|
||||
logger.info("[rvs] llm_request → llm-adapter (id=%s, msgs=%d, max_tokens=%d, tools=%d, model=%s)",
|
||||
request_id[:8], len(messages), max_tokens, len(tools) if tools else 0, model or "-")
|
||||
ok = await self._send_to_rvs({
|
||||
"type": "llm_request",
|
||||
"payload": req_payload,
|
||||
@@ -3898,10 +3901,11 @@ class ARIABridge:
|
||||
except (TypeError, ValueError):
|
||||
temperature = 0.7
|
||||
_tools = data.get("tools") if isinstance(data.get("tools"), list) else None
|
||||
_model = data.get("model") if isinstance(data.get("model"), str) else None
|
||||
result = await self._local_llm(
|
||||
messages=messages, max_tokens=max_tokens,
|
||||
temperature=temperature, stop=data.get("stop"),
|
||||
tools=_tools,
|
||||
tools=_tools, model=_model,
|
||||
)
|
||||
status = 200 if result.get("ok") else 502
|
||||
await _send_response(writer, status, result)
|
||||
|
||||
+49
-2
@@ -956,6 +956,21 @@
|
||||
<br><span style="color:#FFD60A;">Aktueller Stand (B1a): das lokale Modell <strong>plaudert nur</strong> — Werkzeuge macht noch Claude. Lokale Tools kommen mit B1b.</span>
|
||||
</div>
|
||||
|
||||
<!-- Lokales Modell (llama-swap, B0.5) -->
|
||||
<div style="display:flex;align-items:center;gap:8px;margin:12px 0 4px 0;padding-top:10px;border-top:1px solid #2a2a3a;">
|
||||
<span style="font-size:13px;color:#E0E0F0;"><strong>Lokales Modell:</strong></span>
|
||||
<select id="local-llm-model" onchange="saveLocalLlmConfig()" style="flex:1;background:#1E1E2E;border:1px solid #333;border-radius:4px;padding:6px 8px;color:#E0E0F0;font-family:inherit;font-size:12px;">
|
||||
<option value="">(lade Liste…)</option>
|
||||
</select>
|
||||
<button class="btn secondary" onclick="loadLocalModelList()" title="Liste neu laden" style="padding:4px 8px;font-size:10px;">↻</button>
|
||||
</div>
|
||||
<div id="local-llm-model-desc" style="font-size:10px;color:#8888AA;margin:0 0 2px 0;line-height:1.5;"></div>
|
||||
<div style="font-size:10px;color:#FFD60A;margin:0 0 4px 0;line-height:1.5;">
|
||||
Beim ersten Wechsel zu einem Modell lädt die Gamebox das GGUF (mehrere GB) —
|
||||
die <strong>erste Antwort dauert dann länger</strong>, danach ist es gecacht.
|
||||
Liste kommt aus <code>/shared/config/local_models.json</code> (Keys = xtts/llama-swap/config.yaml).
|
||||
</div>
|
||||
|
||||
<div id="local-llm-status" style="font-size:11px;color:#6a6a88;margin-top:8px;padding-top:8px;border-top:1px solid #2a2a3a;min-height:14px;"></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1571,8 +1586,8 @@
|
||||
try { loadBrainStatus(); } catch {}
|
||||
// Sprachmodell-Dropdown befuellen (kuratierte Tier-Liste vom Proxy)
|
||||
try { loadModelList(); } catch {}
|
||||
// Lokales-LLM-Schalter aus /shared/config/local_llm.json laden
|
||||
try { loadLocalLlmConfig(); } catch {}
|
||||
// Lokales-LLM: erst Modell-Liste (Dropdown), dann Config (Auswahl setzen)
|
||||
try { loadLocalModelList().then(() => loadLocalLlmConfig()); } catch {}
|
||||
};
|
||||
|
||||
// Brain-Status periodisch refreshen damit die Card live bleibt
|
||||
@@ -6254,6 +6269,29 @@
|
||||
el.style.color = '#4ADE80';
|
||||
}
|
||||
}
|
||||
let _localModelsCache = [];
|
||||
let _currentLocalModel = 'qwen3-8b';
|
||||
function updateLocalModelDesc() {
|
||||
const el = document.getElementById('local-llm-model-desc');
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (!el || !sel) return;
|
||||
const m = _localModelsCache.find(x => x.id === sel.value);
|
||||
el.textContent = m && m.description ? m.description : '';
|
||||
}
|
||||
async function loadLocalModelList() {
|
||||
try {
|
||||
const r = await fetch('/api/local-models-list');
|
||||
const j = await r.json();
|
||||
_localModelsCache = (j && j.models) || [];
|
||||
} catch (e) { _localModelsCache = []; }
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (sel) {
|
||||
sel.innerHTML = _localModelsCache.map(m =>
|
||||
`<option value="${m.id}">${m.display_name || m.id}</option>`).join('') || '<option value="">(keine)</option>';
|
||||
if (_localModelsCache.some(m => m.id === _currentLocalModel)) sel.value = _currentLocalModel;
|
||||
updateLocalModelDesc();
|
||||
}
|
||||
}
|
||||
async function loadLocalLlmConfig() {
|
||||
try {
|
||||
const r = await fetch('/api/local-llm-config');
|
||||
@@ -6264,15 +6302,24 @@
|
||||
if (en) en.checked = !!c.enabled;
|
||||
if (ol) ol.checked = !!c.localOnly;
|
||||
if (tv) tv.value = (c.toolVariant === 'full') ? 'full' : 'slim';
|
||||
_currentLocalModel = c.localLlmModel || 'qwen3-8b';
|
||||
const sel = document.getElementById('local-llm-model');
|
||||
if (sel && _localModelsCache.some(m => m.id === _currentLocalModel)) {
|
||||
sel.value = _currentLocalModel;
|
||||
updateLocalModelDesc();
|
||||
}
|
||||
setLocalLlmStatus(c);
|
||||
} catch (e) { /* still */ }
|
||||
}
|
||||
async function saveLocalLlmConfig() {
|
||||
const modelSel = document.getElementById('local-llm-model');
|
||||
const body = {
|
||||
enabled: document.getElementById('local-llm-enabled').checked,
|
||||
localOnly: document.getElementById('local-llm-onlylocal').checked,
|
||||
toolVariant: document.getElementById('local-llm-toolvariant').value,
|
||||
localLlmModel: (modelSel && modelSel.value) || '',
|
||||
};
|
||||
updateLocalModelDesc();
|
||||
try {
|
||||
const r = await fetch('/api/local-llm-config', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
|
||||
+27
-1
@@ -309,9 +309,10 @@ function readLocalLlmConfig() {
|
||||
enabled: !!p.enabled,
|
||||
localOnly: !!p.localOnly,
|
||||
toolVariant: p.toolVariant === "full" ? "full" : "slim",
|
||||
localLlmModel: (typeof p.localLlmModel === "string" && p.localLlmModel) ? p.localLlmModel : "qwen3-8b",
|
||||
};
|
||||
} catch {
|
||||
return { enabled: false, localOnly: false, toolVariant: "slim" };
|
||||
return { enabled: false, localOnly: false, toolVariant: "slim", localLlmModel: "qwen3-8b" };
|
||||
}
|
||||
}
|
||||
function writeLocalLlmConfig(patch) {
|
||||
@@ -319,6 +320,7 @@ function writeLocalLlmConfig(patch) {
|
||||
if (typeof patch.enabled === "boolean") cur.enabled = patch.enabled;
|
||||
if (typeof patch.localOnly === "boolean") cur.localOnly = patch.localOnly;
|
||||
if (patch.toolVariant === "slim" || patch.toolVariant === "full") cur.toolVariant = patch.toolVariant;
|
||||
if (typeof patch.localLlmModel === "string" && patch.localLlmModel.trim()) cur.localLlmModel = patch.localLlmModel.trim();
|
||||
fs.mkdirSync("/shared/config", { recursive: true });
|
||||
const tmp = LOCAL_LLM_CONFIG_FILE + ".tmp";
|
||||
fs.writeFileSync(tmp, JSON.stringify(cur, null, 2));
|
||||
@@ -326,6 +328,27 @@ function writeLocalLlmConfig(patch) {
|
||||
return cur;
|
||||
}
|
||||
|
||||
// ── Lokale Modell-Liste (Diagnostic-Dropdown) ────────────────
|
||||
// /shared/config/local_models.json — kuratierte Liste; muss zu den KEYS in
|
||||
// xtts/llama-swap/config.yaml passen. Wird bei Bedarf mit Defaults seeded.
|
||||
const LOCAL_MODELS_FILE = "/shared/config/local_models.json";
|
||||
const DEFAULT_LOCAL_MODELS = [
|
||||
{ id: "qwen3-8b", display_name: "Qwen3 8B (Standard)", description: "Bestes Tool-Calling, ~6 GB. Passt auf 12 GB." },
|
||||
{ id: "qwen3-4b", display_name: "Qwen3 4B (schneller)", description: "Kleiner + flotter, ~3 GB. Etwas schwaecher." },
|
||||
];
|
||||
function loadLocalModels() {
|
||||
try {
|
||||
const arr = JSON.parse(fs.readFileSync(LOCAL_MODELS_FILE, "utf-8"));
|
||||
if (Array.isArray(arr) && arr.length && arr.every(m => m && typeof m.id === "string")) return arr;
|
||||
} catch {}
|
||||
// Seed defaults
|
||||
try {
|
||||
fs.mkdirSync("/shared/config", { recursive: true });
|
||||
fs.writeFileSync(LOCAL_MODELS_FILE, JSON.stringify(DEFAULT_LOCAL_MODELS, null, 2));
|
||||
} catch {}
|
||||
return DEFAULT_LOCAL_MODELS;
|
||||
}
|
||||
|
||||
// ── File-Project-Manifest ───────────────────────────────────────────
|
||||
// Jeder Eintrag map[absoluter_pfad] = project_id (leer = Hauptchat).
|
||||
// Wird vom files-list-Endpoint + files-set-project gepflegt.
|
||||
@@ -1602,6 +1625,9 @@ const server = http.createServer((req, res) => {
|
||||
}
|
||||
});
|
||||
return;
|
||||
} else if (req.url === "/api/local-models-list" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: true, models: loadLocalModels() }));
|
||||
} else if (req.url === "/api/local-llm-config" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(readLocalLlmConfig()));
|
||||
|
||||
+19
-23
@@ -90,22 +90,19 @@ services:
|
||||
# Container-Restarts.
|
||||
restart: unless-stopped
|
||||
|
||||
# ─── Lokales LLM (Plan B, B0) — llama.cpp-Server (GPU) ────────
|
||||
# Serviert Qwen3-8B (GGUF Q4_K_M) OpenAI-kompatibel auf :8081, NUR im
|
||||
# Compose-Netz (kein RVS direkt) — die Bruecke macht der llm-adapter.
|
||||
# ─── Lokales LLM (Plan B, B0.5) — llama-swap (GPU) ────────────
|
||||
# llama-swap laedt/swappt mehrere Modelle on-demand (nur eins passt gleich-
|
||||
# zeitig in die 12 GB). Welches geladen wird, bestimmt das `model`-Feld im
|
||||
# Request — das Brain schickt es aus local_llm.json mit. Erster Load eines
|
||||
# Modells zieht das GGUF via -hf von HF (Cache unter /models, persistent).
|
||||
# OpenAI-kompatibel auf :8080, nur im Compose-Netz; die Bruecke macht der
|
||||
# llm-adapter. Modell-Liste: ./llama-swap/config.yaml.
|
||||
#
|
||||
# AUTO-DOWNLOAD: llama.cpp zieht das GGUF beim ersten Start selbst von
|
||||
# Hugging Face (-hf <repo>:<quant>) und cached es unter /models (persistent
|
||||
# via Bind-Mount -> kein Re-Download bei Restart). Kein manuelles Ablegen
|
||||
# noetig. Modell wechseln = LLM_HF_REPO/LLM_HF_QUANT in der .env aendern +
|
||||
# Container neu. (Alternativ lokale Datei: command auf -m /models/x.gguf.)
|
||||
#
|
||||
# VRAM auf der RTX 3060 (12 GB): whisper-small (~1-2) + f5tts (~1-2) +
|
||||
# qwen3-8b-q4 (~6) ~= 9-10 GB. Passt, aber knapp — bei OOM: LLM_CTX kleiner
|
||||
# oder Quant auf Q4_K_S/IQ4_XS wechseln.
|
||||
llama:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
||||
container_name: aria-llama
|
||||
# BLIND GEBAUT (kein Gamebox-Test hier): beim ersten Start
|
||||
# `docker logs -f aria-llama-swap` pruefen. Image bundelt llama-server.
|
||||
llama-swap:
|
||||
image: ghcr.io/mostlygeek/llama-swap:unified-cuda
|
||||
container_name: aria-llama-swap
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
@@ -115,12 +112,10 @@ services:
|
||||
capabilities: [gpu]
|
||||
volumes:
|
||||
- ./models:/models # HF-Download-Cache (persistent)
|
||||
- ./llama-swap/config.yaml:/app/config.yaml:ro # Modell-Liste
|
||||
environment:
|
||||
- LLAMA_CACHE=/models # llama.cpp legt -hf-Downloads hier ab
|
||||
command: >
|
||||
-hf ${LLM_HF_REPO:-Qwen/Qwen3-8B-GGUF}:${LLM_HF_QUANT:-Q4_K_M}
|
||||
--host 0.0.0.0 --port 8081
|
||||
-ngl 99 -c ${LLM_CTX:-8192} --jinja
|
||||
- LLAMA_CACHE=/models # llama-server legt -hf-Downloads hier ab
|
||||
command: ["--config", "/app/config.yaml", "--listen", "0.0.0.0:8080"]
|
||||
restart: unless-stopped
|
||||
|
||||
# ─── Local-LLM-Adapter — RVS <-> llama.cpp (Plan B, B0) ──────
|
||||
@@ -130,14 +125,15 @@ services:
|
||||
build: ./llm-adapter
|
||||
container_name: aria-llm-adapter
|
||||
depends_on:
|
||||
- llama
|
||||
- llama-swap
|
||||
environment:
|
||||
- RVS_HOST=${RVS_HOST}
|
||||
- RVS_PORT=${RVS_PORT:-443}
|
||||
- RVS_TLS=${RVS_TLS:-true}
|
||||
- RVS_TLS_FALLBACK=${RVS_TLS_FALLBACK:-true}
|
||||
- RVS_TOKEN=${RVS_TOKEN}
|
||||
- LLAMA_URL=http://llama:8081
|
||||
- LLAMA_URL=http://llama-swap:8080
|
||||
- LLM_MODEL=${LLM_MODEL:-qwen3-8b}
|
||||
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-60}
|
||||
# Erster Load eines Modells kann ein GGUF ziehen (mehrere GB) — grosszuegig.
|
||||
- LLM_TIMEOUT_SEC=${LLM_TIMEOUT_SEC:-600}
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# llama-swap Modell-Liste fuer ARIA (Plan B, B0.5).
|
||||
# Welches Modell geladen wird, bestimmt das `model`-Feld im Request (das Brain
|
||||
# schickt es aus /shared/config/local_llm.json mit). llama-swap laedt es
|
||||
# on-demand, swappt bei Bedarf (nur eins passt gleichzeitig in die 12 GB).
|
||||
# Erster Load zieht das GGUF via -hf von Hugging Face (Cache unter /models).
|
||||
#
|
||||
# Die Modell-KEYS hier muessen zu local_models.json (Diagnostic-Dropdown) passen.
|
||||
#
|
||||
# healthCheckTimeout: Sekunden, die llama-swap auf "Modell bereit" wartet.
|
||||
# GROSSZUEGIG, weil der erste Load ein GGUF (mehrere GB) herunterlaedt. Wenn der
|
||||
# erste Download laenger dauert und abbricht: hier hochsetzen.
|
||||
healthCheckTimeout: 1800
|
||||
|
||||
models:
|
||||
# Standard — Qwen3 8B (~6 GB Q4). Bestes Tool-Calling, passt auf 12 GB.
|
||||
"qwen3-8b":
|
||||
cmd: |
|
||||
llama-server --port ${PORT} --host 127.0.0.1
|
||||
-hf Qwen/Qwen3-8B-GGUF:Q4_K_M
|
||||
-ngl 99 -c 8192 --jinja
|
||||
ttl: 3600 # nach 1h Idle entladen (VRAM freigeben)
|
||||
|
||||
# Kleiner + schneller — Qwen3 4B (~3 GB). Fuer noch flottere Antworten,
|
||||
# etwas schwaecher. Guter A/B-Vergleich gegen 8B.
|
||||
"qwen3-4b":
|
||||
cmd: |
|
||||
llama-server --port ${PORT} --host 127.0.0.1
|
||||
-hf Qwen/Qwen3-4B-GGUF:Q4_K_M
|
||||
-ngl 99 -c 8192 --jinja
|
||||
ttl: 3600
|
||||
|
||||
# ── Vorlagen fuer spaeter (auskommentiert; brauchen mehr VRAM / 2. Karte) ──
|
||||
# "qwen3-14b":
|
||||
# cmd: |
|
||||
# llama-server --port ${PORT} --host 127.0.0.1
|
||||
# -hf Qwen/Qwen3-14B-GGUF:Q4_K_M -ngl 99 -c 8192 --jinja
|
||||
# ttl: 3600
|
||||
# "mistral-small-3":
|
||||
# cmd: |
|
||||
# llama-server --port ${PORT} --host 127.0.0.1
|
||||
# -hf <mistral-small-3-gguf-repo>:Q4_K_M -ngl 99 -c 8192 --jinja
|
||||
# ttl: 3600
|
||||
@@ -69,14 +69,16 @@ async def _send(ws, mtype: str, payload: dict) -> None:
|
||||
|
||||
|
||||
async def _call_llama(messages: list, *, max_tokens: int, temperature: float,
|
||||
stop, tools=None) -> dict:
|
||||
"""Ruft llama.cpp /v1/chat/completions (OpenAI-Format). Gibt
|
||||
stop, tools=None, model=None) -> dict:
|
||||
"""Ruft llama.cpp/llama-swap /v1/chat/completions (OpenAI-Format). Gibt
|
||||
{ok, content, tool_calls, error} zurueck — wirft nie.
|
||||
|
||||
tools: optionale OpenAI-Tool-Definitionen (B1b). llama.cpp (--jinja) mit
|
||||
Qwen3 kann natives Tool-Calling und liefert dann message.tool_calls."""
|
||||
model: welches Modell llama-swap laden soll (B0.5). Kommt aus dem Request
|
||||
(Brain -> local_llm.json). Faellt auf LLM_MODEL (env) zurueck.
|
||||
tools: optionale OpenAI-Tool-Definitionen (B1b). Qwen3 (--jinja) kann
|
||||
natives Tool-Calling und liefert dann message.tool_calls."""
|
||||
body = {
|
||||
"model": LLM_MODEL,
|
||||
"model": model or LLM_MODEL,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
@@ -120,13 +122,15 @@ async def _handle_llm_request(ws, payload: dict) -> None:
|
||||
temperature = float(payload.get("temperature", 0.7) or 0.7)
|
||||
stop = payload.get("stop")
|
||||
tools = payload.get("tools") or None
|
||||
model = (payload.get("model") or "").strip() or None
|
||||
t0 = time.time()
|
||||
res = await _call_llama(messages, max_tokens=max_tokens,
|
||||
temperature=temperature, stop=stop, tools=tools)
|
||||
temperature=temperature, stop=stop, tools=tools,
|
||||
model=model)
|
||||
dt = time.time() - t0
|
||||
tc = res.get("tool_calls")
|
||||
logger.info("llm_request id=%s -> ok=%s %.2fs content_len=%d tool_calls=%d",
|
||||
(req_id[:8] if req_id else "?"), res.get("ok"), dt,
|
||||
logger.info("llm_request id=%s model=%s -> ok=%s %.2fs content_len=%d tool_calls=%d",
|
||||
(req_id[:8] if req_id else "?"), model or LLM_MODEL, res.get("ok"), dt,
|
||||
len(res.get("content") or ""), len(tc) if tc else 0)
|
||||
await _send(ws, "llm_response", {
|
||||
"requestId": req_id,
|
||||
@@ -134,7 +138,7 @@ async def _handle_llm_request(ws, payload: dict) -> None:
|
||||
"content": res.get("content", ""),
|
||||
"tool_calls": tc,
|
||||
"error": res.get("error"),
|
||||
"model": LLM_MODEL,
|
||||
"model": model or LLM_MODEL,
|
||||
"elapsedMs": int(dt * 1000),
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user