fast, readable log: stop routine noise, keyset paging, token search

The log had 6.45M rows, of which 434 were actual actions: every mail x
every non-matching rule wrote "Keine Übereinstimmung" (5.3M rows, ~1 GB),
plus "keine neuen Mails" per rule every 30s and a "Poll gestartet" row.
Free-text searches took 50-70s.

- Routine entries are only written with LOG_VERBOSE=true; the poll
  summary is logged only when there were new mails, actions or errors.
  "Why didn't my rule match" is now answered by the rule tester.
- The scheduler commits every 25 evaluated mails and after each rule,
  since the dropped log calls used to be the implicit commit points.
- Migration rebuilds filter_logs without the routine rows (live:
  6.45M -> ~38k) via copy instead of DELETE.
- /api/logs: keyset pagination (before_id) instead of OFFSET, count
  capped at 1000, since/until range, multi-level filter, per-field
  filters; ordner: also matches the move target stored in details.
- New log page: one search field with shortcuts (von:, betreff:,
  regel:, ordner:, text:), live search with debounce and abort, active
  filters as removable chips, click on rule/sender/folder in a row to
  filter, time range (default 7 days), level quick filter, expandable
  row details, "Ältere laden", filters kept in the URL.
- Disable Pico's colour transition on segmented buttons (they briefly
  flashed in the primary colour on load).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 10:21:32 +02:00
co-authored by Claude Opus 5.5
parent d189424ab3
commit 9bd2507cb1
6 changed files with 617 additions and 307 deletions
@@ -0,0 +1,62 @@
"""purge routine entries from filter_logs
Revision ID: 9a3f5c1d7e42
Revises: 7d1e4a2b9c30
Create Date: 2026-09-29 00:00:00.000000
"""
import re
from typing import Sequence, Union
from alembic import op
from sqlalchemy import text
revision: str = "9a3f5c1d7e42"
down_revision: Union[str, Sequence[str], None] = "7d1e4a2b9c30"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
# Routine-Einträge, die ab jetzt nur noch mit LOG_VERBOSE geschrieben werden.
# Sie machten >99% der Tabelle aus (Millionen Zeilen) und machten Suchen langsam.
NOISE = """
message = 'Keine Übereinstimmung'
OR message LIKE 'Regel %keine neuen Mails%'
OR message LIKE 'Regel %Mail(s) in %prüfen%'
OR message LIKE 'Poll gestartet%'
OR (message LIKE 'Poll abgeschlossen%' AND message LIKE '%, 0 neu, 0 Treffer, 0 Aktion(en)')
"""
INDEXES = {
"ix_filter_logs_created_at": "(created_at)",
"ix_filter_logs_level": "(level)",
"ix_filter_logs_account_created": "(account_id, created_at)",
}
def upgrade() -> None:
"""Tabelle neu aufbauen statt DELETE: nur die relevanten Zeilen kopieren
ist bei Millionen Einträgen um ein Vielfaches schneller und erzeugt kein
riesiges WAL."""
bind = op.get_bind()
ddl = bind.execute(
text("SELECT sql FROM sqlite_master WHERE type='table' AND name='filter_logs'")
).scalar()
if not ddl:
return
new_ddl = re.sub(r'^CREATE TABLE\s+"?filter_logs"?', 'CREATE TABLE filter_logs_new', ddl, count=1)
op.execute("DROP TABLE IF EXISTS filter_logs_new")
op.execute(new_ddl)
op.execute(f"INSERT INTO filter_logs_new SELECT * FROM filter_logs WHERE NOT ({NOISE})")
for name in INDEXES:
op.execute(f"DROP INDEX IF EXISTS {name}")
op.execute("DROP TABLE filter_logs")
op.execute("ALTER TABLE filter_logs_new RENAME TO filter_logs")
for name, cols in INDEXES.items():
op.execute(f"CREATE INDEX {name} ON filter_logs {cols}")
def downgrade() -> None:
# Gelöschte Routine-Einträge lassen sich nicht wiederherstellen.
pass
+3
View File
@@ -9,6 +9,9 @@ class Settings(BaseSettings):
encryption_key: str = ""
yaml_sync_on_startup: bool = True
log_level: str = "INFO"
# Ausführliches Verarbeitungslog (jede Nicht-Übereinstimmung, jede Regel pro
# Poll). Erzeugt Millionen Einträge — nur zur Fehlersuche einschalten.
log_verbose: bool = False
model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
+94 -41
View File
@@ -1,65 +1,118 @@
import logging
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
from sqlalchemy import or_, text
from sqlalchemy.orm import Session
from app.database import engine, get_db
from app.models.db_models import FilterLog
from app.models.db_models import FilterLog, LogLevel
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/logs", tags=["logs"])
# Obergrenze fürs Zählen: exakte Zahlen über Millionen Zeilen kosten Sekunden,
# für die Anzeige reicht "mehr als 1.000".
COUNT_CAP = 1000
def _like(value: str) -> str:
escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
return f"%{escaped}%"
def _parse_ts(value: str | None) -> datetime | None:
if not value:
return None
try:
# Browser schickt ISO mit 'Z' (UTC) — DB speichert naive UTC
return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
except ValueError:
raise HTTPException(400, f"Ungültiges Datum: {value}")
@router.get("/")
def get_logs(
account_id: int | None = None,
level: str | None = None,
search: str | None = None,
search_subject: str | None = None,
search_from: str | None = None,
search_rule: str | None = None,
search_message: str | None = None,
search_details: str | None = None,
search_folder: str | None = None,
limit: int = Query(default=100, le=500),
offset: int = 0,
level: str | None = Query(default=None, description="Kommagetrennt: success,info,warning,error"),
q: str | None = Query(default=None, description="Freitext über alle Felder"),
sender: str | None = None,
subject: str | None = None,
rule: str | None = None,
folder: str | None = None,
message: str | None = None,
since: str | None = None,
until: str | None = None,
before_id: int | None = Query(default=None, description="Für 'Mehr laden': nur Einträge älter als diese ID"),
limit: int = Query(default=100, ge=1, le=500),
db: Session = Depends(get_db),
):
query = db.query(FilterLog).order_by(FilterLog.created_at.desc())
"""Neueste zuerst, geblättert per before_id (Keyset) statt OFFSET: jede Seite
ist gleich schnell, egal wie weit man zurückblättert."""
query = db.query(FilterLog)
if account_id:
query = query.filter(FilterLog.account_id == account_id)
if level:
query = query.filter(FilterLog.level == level)
# Einfache Suche (ODER über alle Felder)
if search:
term = f"%{search}%"
query = query.filter(
FilterLog.message.ilike(term)
| FilterLog.mail_subject.ilike(term)
| FilterLog.mail_from.ilike(term)
| FilterLog.rule_name.ilike(term)
| FilterLog.details.ilike(term)
| FilterLog.folder.ilike(term)
)
# Erweiterte Suche (UND pro Feld)
if search_subject:
query = query.filter(FilterLog.mail_subject.ilike(f"%{search_subject}%"))
if search_from:
query = query.filter(FilterLog.mail_from.ilike(f"%{search_from}%"))
if search_rule:
query = query.filter(FilterLog.rule_name.ilike(f"%{search_rule}%"))
if search_message:
query = query.filter(FilterLog.message.ilike(f"%{search_message}%"))
if search_details:
query = query.filter(FilterLog.details.ilike(f"%{search_details}%"))
if search_folder:
query = query.filter(FilterLog.folder.ilike(f"%{search_folder}%"))
total = query.count()
logs = query.offset(offset).limit(limit).all()
levels = []
for part in level.split(","):
try:
levels.append(LogLevel(part.strip()))
except ValueError:
raise HTTPException(400, f"Unbekanntes Level: {part}")
query = query.filter(FilterLog.level.in_(levels))
ts_since, ts_until = _parse_ts(since), _parse_ts(until)
if ts_since:
query = query.filter(FilterLog.created_at >= ts_since)
if ts_until:
query = query.filter(FilterLog.created_at < ts_until)
# Freitext: jedes Wort muss in irgendeinem Feld vorkommen
if q:
for word in q.split():
pat = _like(word)
query = query.filter(or_(
FilterLog.message.ilike(pat, escape="\\"),
FilterLog.mail_subject.ilike(pat, escape="\\"),
FilterLog.mail_from.ilike(pat, escape="\\"),
FilterLog.rule_name.ilike(pat, escape="\\"),
FilterLog.details.ilike(pat, escape="\\"),
FilterLog.folder.ilike(pat, escape="\\"),
))
for column, value in (
(FilterLog.mail_from, sender),
(FilterLog.mail_subject, subject),
(FilterLog.rule_name, rule),
(FilterLog.message, message),
):
if value:
query = query.filter(column.ilike(_like(value), escape="\\"))
if folder:
# Quellordner steht in folder, das Ziel einer Verschiebe-Aktion in details
pat = _like(folder)
query = query.filter(or_(
FilterLog.folder.ilike(pat, escape="\\"),
(FilterLog.action_type == "move") & FilterLog.details.ilike(pat, escape="\\"),
))
count_query = query
if before_id:
query = query.filter(FilterLog.id < before_id)
rows = query.order_by(FilterLog.id.desc()).limit(limit + 1).all()
has_more = len(rows) > limit
rows = rows[:limit]
# Treffermenge nur bis COUNT_CAP zählen (und nur für die erste Seite)
total = None
if before_id is None:
total = count_query.with_entities(FilterLog.id).limit(COUNT_CAP + 1).count()
return {
"total": total,
"total_capped": total is not None and total > COUNT_CAP,
"has_more": has_more,
"next_before_id": rows[-1].id if rows and has_more else None,
"logs": [
{
"id": log.id,
@@ -76,7 +129,7 @@ def get_logs(
"details": log.details,
"created_at": log.created_at.isoformat() if log.created_at else None,
}
for log in logs
for log in rows
],
}
+27 -7
View File
@@ -5,6 +5,7 @@ from datetime import datetime
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy.orm import Session
from app.config import settings
from app.database import SessionLocal
from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
from app.services.encryption import decrypt
@@ -16,6 +17,17 @@ logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler()
# Nach so vielen geprüften Mails wird zwischengespeichert, damit der Poll die
# SQLite-Schreibsperre nicht minutenlang hält (UI-Aktionen würden sonst warten).
COMMIT_EVERY = 25
def _verbose_log(**kwargs) -> None:
"""Routine-Einträge (Nicht-Übereinstimmung, 'keine neuen Mails', Poll-Start)
erzeugten Millionen Logzeilen ohne Nutzen — nur bei LOG_VERBOSE schreiben."""
if settings.log_verbose:
write_log(**kwargs)
def _build_smtp_config(account: Account) -> dict | None:
if account.smtp_host and account.smtp_username and account.smtp_password:
@@ -42,7 +54,7 @@ def _poll_account_sync(account_id: int) -> None:
.all()
)
write_log(
_verbose_log(
message=f"Poll gestartet ({len(rules)} aktive Regel(n))",
level=LogLevel.INFO,
account_id=account.id,
@@ -72,6 +84,7 @@ def _poll_account_sync(account_id: int) -> None:
total_matched = 0
total_actions = 0
total_errors = 0
evaluated_count = 0
# Mail-Cache: einmal geladene Mails pro Ordner/UID wiederverwenden
mail_cache: dict[tuple[str, str], "MailMessage"] = {}
@@ -126,7 +139,7 @@ def _poll_account_sync(account_id: int) -> None:
new_uids.reverse()
if not new_uids:
write_log(
_verbose_log(
message=f"Regel '{rule.name}': keine neuen Mails in '{folder}' ({len(all_uids)} gesamt, alle bereits geprüft)",
level=LogLevel.INFO,
account_id=account.id,
@@ -146,7 +159,7 @@ def _poll_account_sync(account_id: int) -> None:
msg = f"Regel '{rule.name}': {len(batch_uids)} Mail(s) in '{folder}' prüfen ({len(processed_uids)} bereits geprüft)"
if remaining > 0:
msg += f" — {remaining} weitere beim nächsten Poll"
write_log(
_verbose_log(
message=msg,
level=LogLevel.INFO,
account_id=account.id,
@@ -247,8 +260,8 @@ def _poll_account_sync(account_id: int) -> None:
db=db,
)
else:
write_log(
message=f"Keine Übereinstimmung",
_verbose_log(
message="Keine Übereinstimmung",
level=LogLevel.INFO,
account_id=account.id,
account_name=account.name,
@@ -270,7 +283,11 @@ def _poll_account_sync(account_id: int) -> None:
mail_subject=mail.subject[:500] if mail.subject else None,
mail_from=mail.from_addr[:255] if mail.from_addr else None,
))
db.flush()
evaluated_count += 1
if evaluated_count % COMMIT_EVERY == 0:
db.commit()
db.commit()
# Poll-Zusammenfassung
summary_parts = [
@@ -282,7 +299,10 @@ def _poll_account_sync(account_id: int) -> None:
if total_errors > 0:
summary_parts.append(f"{total_errors} Fehler")
write_log(
# Nur loggen, wenn etwas passiert ist — ein Eintrag alle 30 s ohne
# Inhalt verdeckt die relevanten Zeilen. Letzter Poll steht im Dashboard.
summary_log = write_log if (total_new or total_actions or total_errors) else _verbose_log
summary_log(
message=f"Poll abgeschlossen: {', '.join(summary_parts)}",
level=LogLevel.ERROR if total_errors > 0 else LogLevel.SUCCESS if total_actions > 0 else LogLevel.INFO,
account_id=account.id,
+1
View File
@@ -127,6 +127,7 @@
.toolbar { display:flex; gap:0.5rem; align-items:center; flex-wrap:wrap; margin-bottom:0.6rem; }
.toolbar input[type=search] { flex:1 1 280px; margin:0; padding-top:0.3rem; padding-bottom:0.3rem; font-size:0.9rem; }
.seg { display:inline-flex; flex-wrap:wrap; border:1px solid var(--app-border, #e3e6ea); border-radius:var(--pico-border-radius); overflow:hidden; background:#fff; }
.toolbar .seg button { transition:none; }
.toolbar .seg button { --pico-background-color:transparent; --pico-border-color:transparent; --pico-color:#374151;
margin:0; border-radius:0; padding:0.3rem 0.7rem; font-size:0.8rem; box-shadow:none; }
.toolbar .seg button + button { border-left:1px solid var(--app-border, #e3e6ea); }
+430 -259
View File
@@ -1,305 +1,476 @@
{% extends "base.html" %}
{% block title %}Verarbeitungslog — IMAP Mail Filter{% endblock %}
{% block title %}Log — IMAP Mail Filter{% endblock %}
{% block content %}
<h1>Verarbeitungslog</h1>
<div style="display:flex; gap:1rem; align-items:end; flex-wrap:wrap; margin-bottom:0.5rem;">
<label style="margin-bottom:0;">
Konto
<select id="log-account" onchange="loadLogs()" style="margin-bottom:0;">
<option value="">Alle Konten</option>
{% for acc in accounts %}
<option value="{{ acc.id }}">{{ acc.name }}</option>
{% endfor %}
</select>
</label>
<label style="margin-bottom:0;">
Level
<select id="log-level" onchange="loadLogs()" style="margin-bottom:0;">
<option value="">Alle</option>
<option value="success">Erfolg</option>
<option value="info">Info</option>
<option value="warning">Warnung</option>
<option value="error">Fehler</option>
</select>
</label>
<label style="margin-bottom:0; flex:1; min-width:200px;">
Schnellsuche
<input type="search" id="log-search" placeholder="überall suchen..." style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<div role="group" style="margin-bottom:0;">
<button class="outline small" onclick="loadLogs()">Suchen</button>
<button class="outline small" onclick="retryFailed()" title="Löscht den 'verarbeitet'-Marker für Mails, die einen Fetch-Fehler hatten, damit sie beim nächsten Poll neu versucht werden">Fehlgeschlagene neu einlesen</button>
<button class="outline small contrast" onclick="clearLogs()">Log leeren</button>
</div>
<label style="margin-bottom:0;">
<input type="checkbox" id="auto-refresh" onchange="toggleAutoRefresh()" style="margin-bottom:0;">
Auto-Refresh (5s)
<div class="page-head">
<h1>Log</h1>
<select id="log-account" class="head-select" onchange="reload()" title="Konto">
<option value="">Alle Konten</option>
{% for acc in accounts %}
<option value="{{ acc.id }}">{{ acc.name }}</option>
{% endfor %}
</select>
<select id="log-range" class="head-select" onchange="reload()" title="Zeitraum">
<option value="1h">Letzte Stunde</option>
<option value="today">Heute</option>
<option value="7d" selected>Letzte 7 Tage</option>
<option value="30d">Letzte 30 Tage</option>
<option value="all">Gesamter Zeitraum</option>
</select>
<span id="log-stats" class="muted small-text"></span>
<label class="auto-refresh" title="Erste Seite alle 10 Sekunden neu laden">
<input type="checkbox" role="switch" id="auto-refresh" onchange="toggleAutoRefresh()"> Live
</label>
<details class="dropdown head-menu">
<summary>⋯</summary>
<ul dir="rtl">
<li><a href="#" onclick="retryFailed(); return false;" title="Löscht den 'verarbeitet'-Marker für Mails mit Abruf-Fehler, damit sie beim nächsten Poll neu versucht werden">Fehlgeschlagene neu einlesen</a></li>
<li><a href="#" onclick="clearLogs(); return false;">Log leeren</a></li>
</ul>
</details>
</div>
<details style="margin-bottom:1rem;">
<summary>Erweiterte Suche (UND-verknüpft)</summary>
<div class="grid" style="margin-top:0.5rem;">
<label style="margin-bottom:0;">
Betreff
<input type="search" id="search-subject" placeholder="z.B. Rechnung" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<label style="margin-bottom:0;">
Absender
<input type="search" id="search-from" placeholder="z.B. shop@example.com" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<label style="margin-bottom:0;">
Regel
<input type="search" id="search-rule" placeholder="z.B. Newsletter" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<div class="toolbar">
<div class="search-wrap">
<input type="search" id="log-search" autocomplete="off" spellcheck="false"
placeholder="Suchen … z.B. hetzner oder von:hetzner betreff:rechnung regel:cps"
oninput="onSearchInput()" onkeydown="if(event.key==='Enter'){ reload(); }">
<button type="button" class="help-btn" onclick="toggleHelp()" title="Suchsyntax">?</button>
</div>
<div class="grid" style="margin-top:0.5rem;">
<label style="margin-bottom:0;">
Nachricht
<input type="search" id="search-message" placeholder="z.B. Aktion ausgeführt" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<label style="margin-bottom:0;">
Details
<input type="search" id="search-details" placeholder="z.B. TREFFER" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<label style="margin-bottom:0;">
Ordner
<input type="search" id="search-folder" placeholder="z.B. INBOX" style="margin-bottom:0;" onkeydown="if(event.key==='Enter') loadLogs()">
</label>
<div class="seg" id="level-filter">
<button data-l="" class="active" onclick="setLevel('')">Alle</button>
<button data-l="success" onclick="setLevel('success')">✅ Aktionen</button>
<button data-l="error,warning" onclick="setLevel('error,warning')">❌ Fehler</button>
<button data-l="info" onclick="setLevel('info')">ℹ Info</button>
</div>
<div style="margin-top:0.5rem;">
<button class="outline small" onclick="loadLogs()">Erweitert suchen</button>
<button class="outline small contrast" onclick="clearAdvancedSearch()">Felder leeren</button>
</div>
</details>
<div id="log-stats" style="margin-bottom:1rem;"></div>
<div id="log-container">
<p aria-busy="true">Logs werden geladen...</p>
</div>
<div id="log-paging" style="display:flex; justify-content:center; gap:1rem; margin-top:1rem;"></div>
<div id="search-help" class="search-help" hidden>
<strong>Suchsyntax</strong> — Begriffe werden UND-verknüpft, Groß/Klein egal. Mehrere Wörter in Anführungszeichen: <code>betreff:"ihre rechnung"</code>
<div class="help-grid">
<code>hetzner</code><span>Freitext in allen Feldern</span>
<code>von:hetzner.com</code><span>Absender</span>
<code>betreff:rechnung</code><span>Betreff</span>
<code>regel:cps</code><span>Name der Regel</span>
<code>ordner:Rechnungen</code><span>Ordner / Ziel</span>
<code>text:verschoben</code><span>Log-Nachricht</span>
</div>
Tipp: Klick auf Regel, Absender oder Ordner in einer Zeile fügt ihn als Filter hinzu.
</div>
<div id="active-filters" class="active-filters"></div>
<div id="log-container"><p aria-busy="true">Log wird geladen…</p></div>
<div id="log-more"></div>
{% endblock %}
{% block scripts %}
<style>
.log-table {
width: 100%;
font-size: 0.9rem;
}
.log-table th, .log-table td {
padding: 0.4rem 0.6rem;
vertical-align: top;
}
.log-row { border-left: 3px solid transparent; }
.log-row[data-level="success"] { border-left-color: #28a745; }
.log-row[data-level="info"] { border-left-color: #17a2b8; }
.log-row[data-level="warning"] { border-left-color: #ffc107; }
.log-row[data-level="error"] { border-left-color: #dc3545; }
.muted { color:var(--pico-muted-color); }
.small-text { font-size:0.85rem; }
.page-head { display:flex; align-items:center; gap:0.6rem; flex-wrap:wrap; margin-bottom:0.75rem; }
.page-head h1 { margin:0 0.4rem 0 0; }
.head-select { width:auto; margin:0; padding-top:0.3rem; padding-bottom:0.3rem; font-size:0.9rem; }
.page-head #log-stats { flex:1; }
.auto-refresh { margin:0; font-size:0.85rem; white-space:nowrap; }
.head-menu { margin:0; }
.head-menu summary { padding:0.2rem 0.7rem; height:auto; font-size:1.1rem; line-height:1.2; }
.head-menu summary::after { display:none; }
.head-menu ul { min-width:230px; font-size:0.9rem; }
.log-level {
display: inline-block;
padding: 0.1rem 0.4rem;
border-radius: 3px;
font-size: 0.75rem;
font-weight: bold;
text-transform: uppercase;
}
.log-level-success { background: #d4edda; color: #155724; }
.log-level-info { background: #d1ecf1; color: #0c5460; }
.log-level-warning { background: #fff3cd; color: #856404; }
.log-level-error { background: #f8d7da; color: #721c24; }
.toolbar { display:flex; gap:0.5rem; align-items:center; flex-wrap:wrap; margin-bottom:0.4rem; }
.search-wrap { position:relative; flex:1 1 360px; }
.search-wrap input { margin:0; padding-top:0.35rem; padding-bottom:0.35rem; padding-right:2.4rem; font-size:0.9rem; }
.search-wrap .help-btn { position:absolute; right:0.35rem; top:50%; transform:translateY(-50%); margin:0; padding:0 0.5rem;
font-size:0.8rem; line-height:1.6; border-radius:999px;
--pico-background-color:#eef0f3; --pico-border-color:transparent; --pico-color:#555; }
.seg { display:inline-flex; border:1px solid var(--app-border, #e3e6ea); border-radius:var(--pico-border-radius); overflow:hidden; background:#fff; }
.toolbar .seg button, .search-wrap .help-btn { transition:none; }
.toolbar .seg button { --pico-background-color:transparent; --pico-border-color:transparent; --pico-color:#374151;
margin:0; border-radius:0; padding:0.3rem 0.7rem; font-size:0.8rem; box-shadow:none; }
.toolbar .seg button + button { border-left:1px solid var(--app-border, #e3e6ea); }
.toolbar .seg button:hover { --pico-background-color:#f1f3f5; }
.toolbar .seg button.active { --pico-background-color:var(--pico-primary); --pico-color:var(--pico-primary-inverse); }
.log-detail { font-size: 0.8rem; opacity: 0.7; }
.log-mail-info { font-size: 0.8rem; color: var(--pico-muted-color); }
.search-help { background:#fff; border:1px solid var(--app-border, #e3e6ea); border-radius:var(--pico-border-radius);
padding:0.6rem 0.8rem; font-size:0.83rem; margin-bottom:0.5rem; }
.help-grid { display:grid; grid-template-columns:max-content 1fr; gap:0.15rem 1rem; margin:0.4rem 0; }
.help-grid code { font-size:0.8rem; padding:0 0.3rem; }
.active-filters { display:flex; flex-wrap:wrap; gap:0.3rem; margin-bottom:0.5rem; min-height:0; }
.fchip { display:inline-flex; align-items:center; gap:0.25rem; font-size:0.78rem; padding:0.05rem 0.2rem 0.05rem 0.5rem;
border-radius:999px; background:#e8eefc; color:#274b9f; border:1px solid #cdd9f5; }
.fchip button { --pico-background-color:transparent; --pico-border-color:transparent; --pico-color:#274b9f;
margin:0; padding:0 0.3rem; font-size:0.8rem; line-height:1.3; box-shadow:none; }
.fchip.clear-all { background:transparent; border-color:transparent; }
.logs { background:#fff; border:1px solid var(--app-border, #e3e6ea); border-radius:var(--pico-border-radius); overflow:hidden; font-size:0.84rem; }
.logs .l { display:grid; grid-template-columns:98px 22px minmax(110px, 170px) minmax(0, 1fr) minmax(90px, 190px);
gap:0.6rem; align-items:baseline; padding:0.3rem 0.75rem; border-bottom:1px solid #eef0f3; cursor:pointer; }
.logs .l:hover { background:#f8f9fb; }
.logs .l.open { background:#f4f7fd; }
.logs .l.head { background:#f6f7f9; cursor:default; font-size:0.72rem; font-weight:600; text-transform:uppercase; letter-spacing:0.03em; color:var(--pico-muted-color); }
.logs .l.lv-error { box-shadow:inset 3px 0 0 #dc3545; }
.logs .l.lv-warning { box-shadow:inset 3px 0 0 #ffc107; }
.logs .l.lv-success { box-shadow:inset 3px 0 0 #28a745; }
.l-time { color:var(--pico-muted-color); font-variant-numeric:tabular-nums; white-space:nowrap; font-size:0.78rem; }
.l-day { display:block; font-size:0.68rem; opacity:0.8; }
.l-rule, .l-folder { overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.l-main { min-width:0; }
.l-msg { overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.l-mail { font-size:0.78rem; color:var(--pico-muted-color); overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.l-mail b { color:var(--pico-color); font-weight:500; }
.flink { color:inherit; text-decoration:none; border-bottom:1px dotted #aab; cursor:pointer; }
.flink:hover { color:var(--pico-primary); border-bottom-color:var(--pico-primary); }
.l-folder { color:#1f6f43; }
.l-detail { grid-column:1 / -1; font-size:0.78rem; background:#fff; border:1px solid #e6e9ed; border-radius:4px;
padding:0.4rem 0.6rem; margin:0.2rem 0 0.2rem; cursor:text; white-space:pre-wrap; word-break:break-word; }
.l-detail dl { display:grid; grid-template-columns:max-content 1fr; gap:0.1rem 0.8rem; margin:0; }
.l-detail dt { color:var(--pico-muted-color); }
.l-detail dd { margin:0; }
#log-more { text-align:center; margin-top:0.6rem; }
#log-more button { padding:0.3rem 1rem; font-size:0.85rem; }
@media (max-width: 800px) {
.logs .l { grid-template-columns:80px 22px minmax(0, 1fr); }
.logs .l .l-rule, .logs .l .l-folder, .logs .l.head { display:none; }
}
</style>
<script>
let currentOffset = 0;
const PAGE_SIZE = 50;
const PAGE_SIZE = 100;
const LEVEL_ICON = {success: '✅', info: 'ℹ️', warning: '⚠️', error: '❌'};
// Suchkürzel → API-Parameter
const KEYS = {
von: 'sender', from: 'sender', absender: 'sender',
betreff: 'subject', subject: 'subject',
regel: 'rule', rule: 'rule', filter: 'rule',
ordner: 'folder', folder: 'folder', ziel: 'folder',
text: 'message', nachricht: 'message', msg: 'message',
};
const KEY_LABEL = {sender: 'Von', subject: 'Betreff', rule: 'Regel', folder: 'Ordner', message: 'Text', q: 'Text'};
let levelFilter = '';
let logs = [];
let nextBeforeId = null;
let hasMore = false;
let inflight = null;
let searchTimer = null;
let refreshTimer = null;
async function loadLogs(offset = 0) {
currentOffset = offset;
const accountId = document.getElementById('log-account').value;
const level = document.getElementById('log-level').value;
const container = document.getElementById('log-container');
function esc(s) {
return String(s ?? '').replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
}
function parseUtc(iso) {
// Server liefert naive UTC-Zeitstempel — ohne 'Z' würde der Browser sie als Ortszeit lesen
return new Date(/[zZ]|[+-]\d{2}:?\d{2}$/.test(iso) ? iso : iso + 'Z');
}
const search = document.getElementById('log-search').value.trim();
const searchSubject = document.getElementById('search-subject').value.trim();
const searchFrom = document.getElementById('search-from').value.trim();
const searchRule = document.getElementById('search-rule').value.trim();
const searchMessage = document.getElementById('search-message').value.trim();
const searchDetails = document.getElementById('search-details').value.trim();
const searchFolder = document.getElementById('search-folder').value.trim();
// ---------- Suchsyntax ----------
let url = `/api/logs/?limit=${PAGE_SIZE}&offset=${offset}`;
if (accountId) url += `&account_id=${accountId}`;
if (level) url += `&level=${level}`;
if (search) url += `&search=${encodeURIComponent(search)}`;
if (searchSubject) url += `&search_subject=${encodeURIComponent(searchSubject)}`;
if (searchFrom) url += `&search_from=${encodeURIComponent(searchFrom)}`;
if (searchRule) url += `&search_rule=${encodeURIComponent(searchRule)}`;
if (searchMessage) url += `&search_message=${encodeURIComponent(searchMessage)}`;
if (searchDetails) url += `&search_details=${encodeURIComponent(searchDetails)}`;
if (searchFolder) url += `&search_folder=${encodeURIComponent(searchFolder)}`;
function tokenize(input) {
// Liefert [{key, value, raw}] — key null = Freitext
const re = /(\w+):"([^"]*)"|(\w+):(\S+)|"([^"]*)"|(\S+)/g;
const out = [];
let m;
while ((m = re.exec(input)) !== null) {
let key = m[1] || m[3] || null;
let value = m[2] ?? m[4] ?? m[5] ?? m[6] ?? '';
if (key && !KEYS[key.toLowerCase()]) { value = m[0]; key = null; } // unbekanntes Kürzel = Freitext
if (value === '') continue;
out.push({key: key ? KEYS[key.toLowerCase()] : null, value, raw: m[0]});
}
return out;
}
try {
const resp = await fetch(url);
const data = await resp.json();
renderStats(data.total);
renderLogs(data.logs);
renderPaging(data.total, offset);
} catch(e) {
container.innerHTML = `<p>Fehler beim Laden: ${e.message}</p>`;
function serializeTokens(tokens) {
return tokens.map(t => {
if (!t.key) return /\s/.test(t.value) ? `"${t.value}"` : t.value;
const short = {sender: 'von', subject: 'betreff', rule: 'regel', folder: 'ordner', message: 'text'}[t.key];
return `${short}:${/\s/.test(t.value) ? `"${t.value}"` : t.value}`;
}).join(' ');
}
function addFilter(key, value) {
const input = document.getElementById('log-search');
const tokens = tokenize(input.value).filter(t => !(t.key === key));
tokens.push({key, value});
input.value = serializeTokens(tokens);
reload();
}
function removeToken(index) {
const input = document.getElementById('log-search');
const tokens = tokenize(input.value);
tokens.splice(index, 1);
input.value = serializeTokens(tokens);
reload();
}
function clearFilters() {
document.getElementById('log-search').value = '';
setLevel('');
}
function renderActiveFilters(tokens) {
const box = document.getElementById('active-filters');
const chips = tokens.map((t, i) =>
`<span class="fchip">${esc(KEY_LABEL[t.key || 'q'])}: <b>${esc(t.value)}</b><button title="Filter entfernen" onclick="removeToken(${i})">✕</button></span>`);
if (levelFilter) {
const label = document.querySelector(`#level-filter button[data-l="${levelFilter}"]`)?.textContent || levelFilter;
chips.push(`<span class="fchip">${esc(label)}<button title="Filter entfernen" onclick="setLevel('')">✕</button></span>`);
}
if (chips.length > 1) chips.push(`<span class="fchip clear-all"><button onclick="clearFilters()">alle entfernen</button></span>`);
box.innerHTML = chips.join('');
}
// ---------- Laden ----------
function sinceFor(range) {
const now = new Date();
switch (range) {
case '1h': return new Date(now - 3600e3);
case 'today': { const d = new Date(now); d.setHours(0, 0, 0, 0); return d; }
case '7d': return new Date(now - 7 * 86400e3);
case '30d': return new Date(now - 30 * 86400e3);
default: return null;
}
}
function renderStats(total) {
document.getElementById('log-stats').innerHTML = `<small>${total} Einträge gesamt</small>`;
function buildParams(beforeId) {
const p = new URLSearchParams({limit: PAGE_SIZE});
const acc = document.getElementById('log-account').value;
if (acc) p.set('account_id', acc);
if (levelFilter) p.set('level', levelFilter);
const since = sinceFor(document.getElementById('log-range').value);
if (since) p.set('since', since.toISOString());
const tokens = tokenize(document.getElementById('log-search').value);
const free = [];
for (const t of tokens) {
if (t.key) p.set(t.key, t.value);
else free.push(t.value);
}
if (free.length) p.set('q', free.join(' '));
if (beforeId) p.set('before_id', beforeId);
renderActiveFilters(tokens);
syncUrl();
return p;
}
function renderLogs(logs) {
const container = document.getElementById('log-container');
if (!logs.length) {
container.innerHTML = '<article><p>Keine Log-Einträge vorhanden.</p></article>';
return;
}
let html = `<table class="log-table">
<thead><tr>
<th style="width:140px;">Zeit</th>
<th style="width:60px;">Level</th>
<th style="width:120px;">Konto</th>
<th>Nachricht</th>
<th style="width:120px;">Regel</th>
</tr></thead><tbody>`;
for (const log of logs) {
const time = log.created_at ? new Date(log.created_at).toLocaleString('de-DE', {
day: '2-digit', month: '2-digit', year: '2-digit',
hour: '2-digit', minute: '2-digit', second: '2-digit'
}) : '';
const levelClass = `log-level-${log.level}`;
const levelLabel = {success: 'OK', info: 'Info', warning: 'Warn', error: 'Fehler'}[log.level] || log.level;
let message = escapeHtml(log.message);
if (log.mail_subject || log.mail_from) {
message += `<div class="log-mail-info">`;
if (log.mail_from) message += `Von: ${escapeHtml(log.mail_from)}<br>`;
if (log.mail_subject) message += `Betreff: ${escapeHtml(log.mail_subject)}`;
message += `</div>`;
}
if (log.details) {
message += `<div class="log-detail">${escapeHtml(log.details)}</div>`;
}
html += `<tr class="log-row" data-level="${log.level}">
<td><small>${time}</small></td>
<td><span class="log-level ${levelClass}">${levelLabel}</span></td>
<td><small>${escapeHtml(log.account_name || '')}</small></td>
<td>${message}</td>
<td><small>${escapeHtml(log.rule_name || '')}</small></td>
</tr>`;
}
html += '</tbody></table>';
container.innerHTML = html;
}
function renderPaging(total, offset) {
const paging = document.getElementById('log-paging');
if (total <= PAGE_SIZE) {
paging.innerHTML = '';
return;
}
const page = Math.floor(offset / PAGE_SIZE) + 1;
const totalPages = Math.ceil(total / PAGE_SIZE);
let html = '';
if (offset > 0) {
html += `<button class="outline small" onclick="loadLogs(${offset - PAGE_SIZE})">Neuere</button>`;
}
html += `<small>Seite ${page} / ${totalPages}</small>`;
if (offset + PAGE_SIZE < total) {
html += `<button class="outline small" onclick="loadLogs(${offset + PAGE_SIZE})">Ältere</button>`;
}
paging.innerHTML = html;
}
async function parseResponse(resp) {
// Liest die Response robust: JSON wenn möglich, sonst als Text.
async function fetchLogs(beforeId = null) {
if (inflight) inflight.abort();
inflight = new AbortController();
const params = buildParams(beforeId);
const resp = await fetch(`/api/logs/?${params}`, {signal: inflight.signal});
const text = await resp.text();
try { return {ok: resp.ok, status: resp.status, data: JSON.parse(text)}; }
catch { return {ok: resp.ok, status: resp.status, data: null, text}; }
let data;
try { data = JSON.parse(text); } catch { throw new Error(text || `HTTP ${resp.status}`); }
if (!resp.ok) throw new Error(data.detail || `HTTP ${resp.status}`);
return data;
}
async function reload() {
const container = document.getElementById('log-container');
if (!logs.length) container.innerHTML = '<p aria-busy="true">Log wird geladen…</p>';
else container.style.opacity = '0.55';
try {
const data = await fetchLogs();
logs = data.logs;
hasMore = data.has_more;
nextBeforeId = data.next_before_id;
renderStats(data);
renderLogs();
} catch (e) {
if (e.name === 'AbortError') return;
container.innerHTML = `<p style="color:#b42318">Fehler beim Laden: ${esc(e.message)}</p>`;
} finally {
container.style.opacity = '';
}
}
async function loadMore() {
const btn = document.querySelector('#log-more button');
if (btn) btn.setAttribute('aria-busy', 'true');
try {
const data = await fetchLogs(nextBeforeId);
logs = logs.concat(data.logs);
hasMore = data.has_more;
nextBeforeId = data.next_before_id;
renderLogs();
} catch (e) {
if (e.name !== 'AbortError') alert('Fehler: ' + e.message);
}
}
function onSearchInput() {
clearTimeout(searchTimer);
searchTimer = setTimeout(reload, 350);
}
function setLevel(l) {
levelFilter = l;
document.querySelectorAll('#level-filter button').forEach(b => b.classList.toggle('active', b.dataset.l === l));
reload();
}
function toggleHelp() {
const h = document.getElementById('search-help');
h.hidden = !h.hidden;
}
// ---------- Darstellung ----------
function renderStats(data) {
const el = document.getElementById('log-stats');
if (data.total == null) return;
el.textContent = data.total_capped ? 'mehr als 1.000 Einträge' : `${data.total.toLocaleString('de-DE')} Einträge`;
}
function emailOf(from) {
const m = String(from || '').match(/<([^>]+)>/);
return m ? m[1] : String(from || '').trim();
}
function targetOf(log) {
// Bei Aktionen steht der Zielordner in details, sonst Quellordner
if (log.action_type === 'move' && log.details) return {text: '📁 ' + log.details.replace(/^INBOX[./]/, ''), filter: log.details};
if (log.action_type === 'forward' && log.details) return {text: '↪ ' + log.details, filter: null};
if (log.action_type === 'delete') return {text: '🗑 Papierkorb', filter: null};
if (log.folder) return {text: log.folder, filter: log.folder};
return {text: '', filter: null};
}
function shortMessage(log) {
const m = log.message || '';
if (log.action_type && m.startsWith('Aktion ausgeführt')) {
return {move: 'Verschoben', forward: 'Weitergeleitet', delete: 'Gelöscht', mark_read: 'Als gelesen markiert'}[log.action_type] || m;
}
return m;
}
function jsArg(s) {
return esc(JSON.stringify(String(s)));
}
function renderLogs() {
const container = document.getElementById('log-container');
const more = document.getElementById('log-more');
if (!logs.length) {
const hasFilter = document.getElementById('log-search').value.trim() || levelFilter;
container.innerHTML = `<article><p class="muted">Keine Einträge${hasFilter ? ' für diese Suche' : ''} im gewählten Zeitraum.</p>
${document.getElementById('log-range').value !== 'all' ? '<button class="outline small" onclick="document.getElementById(\'log-range\').value=\'all\'; reload();">Gesamten Zeitraum durchsuchen</button>' : ''}</article>`;
more.innerHTML = '';
return;
}
const today = new Date().toDateString();
let html = `<div class="logs"><div class="l head"><span>Zeit</span><span></span><span>Regel</span><span>Ereignis / Mail</span><span>Ordner / Ziel</span></div>`;
for (const log of logs) {
const d = log.created_at ? parseUtc(log.created_at) : null;
const time = d ? d.toLocaleTimeString('de-DE') : '';
const day = d && d.toDateString() !== today ? d.toLocaleDateString('de-DE', {day: '2-digit', month: '2-digit', year: '2-digit'}) : '';
const target = targetOf(log);
const mail = (log.mail_subject || log.mail_from)
? `<div class="l-mail"><b>${esc(log.mail_subject || '(kein Betreff)')}</b>${log.mail_from ? ` · <a class="flink" onclick="event.stopPropagation(); addFilter('sender', ${jsArg(emailOf(log.mail_from))})" title="Nach diesem Absender filtern">${esc(log.mail_from)}</a>` : ''}</div>`
: '';
html += `
<div class="l lv-${esc(log.level)}" onclick="toggleDetail(this, ${log.id})">
<span class="l-time">${day ? `<span class="l-day">${day}</span>` : ''}${time}</span>
<span title="${esc(log.level)}">${LEVEL_ICON[log.level] || ''}</span>
<span class="l-rule">${log.rule_name ? `<a class="flink" onclick="event.stopPropagation(); addFilter('rule', ${jsArg(log.rule_name)})" title="Nach dieser Regel filtern">${esc(log.rule_name)}</a>` : '<span class="muted">—</span>'}</span>
<span class="l-main"><div class="l-msg">${esc(shortMessage(log))}</div>${mail}</span>
<span class="l-folder">${target.filter ? `<a class="flink" onclick="event.stopPropagation(); addFilter('folder', ${jsArg(target.filter)})" title="Nach diesem Ordner filtern">${esc(target.text)}</a>` : esc(target.text)}</span>
</div>`;
}
html += '</div>';
container.innerHTML = html;
more.innerHTML = hasMore
? `<button class="outline secondary" onclick="loadMore()">Ältere laden</button> <span class="muted small-text">${logs.length} angezeigt</span>`
: `<span class="muted small-text">${logs.length} angezeigt · Ende des Zeitraums</span>`;
}
function toggleDetail(row, id) {
const open = row.querySelector('.l-detail');
if (open) { open.remove(); row.classList.remove('open'); return; }
const log = logs.find(l => l.id === id);
if (!log) return;
const rows = [
['Zeit', log.created_at ? parseUtc(log.created_at).toLocaleString('de-DE') : ''],
['Konto', log.account_name],
['Nachricht', log.message],
['Regel', log.rule_name],
['Von', log.mail_from],
['Betreff', log.mail_subject],
['Ordner', log.folder],
['Mail-UID', log.mail_uid],
['Details', log.details],
].filter(([, v]) => v);
const div = document.createElement('div');
div.className = 'l-detail';
div.onclick = e => e.stopPropagation();
div.innerHTML = `<dl>${rows.map(([k, v]) => `<dt>${esc(k)}</dt><dd>${esc(v)}</dd>`).join('')}</dl>`;
row.appendChild(div);
row.classList.add('open');
}
// ---------- Aktionen ----------
async function readJson(resp) {
const text = await resp.text();
try { return JSON.parse(text); } catch { return {detail: text}; }
}
async function clearLogs() {
if (!confirm('Log wirklich leeren?')) return;
const accountId = document.getElementById('log-account').value;
let url = '/api/logs/';
if (accountId) url += `?account_id=${accountId}`;
try {
const resp = await fetch(url, {method: 'DELETE'});
const r = await parseResponse(resp);
if (!r.ok) {
const msg = r.data?.detail || r.text || `HTTP ${r.status}`;
alert('Fehler beim Leeren: ' + msg);
return;
}
loadLogs();
} catch(e) {
alert('Netzwerk-Fehler: ' + e.message);
}
document.querySelector('.head-menu').open = false;
const acc = document.getElementById('log-account');
const scope = acc.value ? `für „${acc.options[acc.selectedIndex].text}“` : 'für alle Konten';
if (!confirm(`Log ${scope} wirklich leeren?`)) return;
const resp = await fetch(`/api/logs/${acc.value ? `?account_id=${acc.value}` : ''}`, {method: 'DELETE'});
const d = await readJson(resp);
if (!resp.ok) { alert('Fehler beim Leeren: ' + (d.detail || resp.status)); return; }
logs = [];
reload();
}
async function retryFailed() {
document.querySelector('.head-menu').open = false;
if (!confirm('Alle Mails mit "Fehler beim Abrufen" für den nächsten Poll neu vormerken?')) return;
const accountId = document.getElementById('log-account').value;
let url = '/api/filters/retry-failed';
if (accountId) url += `?account_id=${accountId}`;
try {
const resp = await fetch(url, {method: 'POST'});
const r = await parseResponse(resp);
if (!r.ok) {
const msg = r.data?.detail || r.text || `HTTP ${r.status}`;
alert('Fehler bei retry-failed: ' + msg);
return;
}
const d = r.data || {};
alert(`${d.unique_uids || 0} UID(s) neu vorgemerkt (${d.reset || 0} ProcessedMail-Einträge gelöscht). Werden beim nächsten Poll neu versucht.`);
} catch(e) {
alert('Netzwerk-Fehler: ' + e.message);
}
const acc = document.getElementById('log-account').value;
const resp = await fetch(`/api/filters/retry-failed${acc ? `?account_id=${acc}` : ''}`, {method: 'POST'});
const d = await readJson(resp);
if (!resp.ok) { alert('Fehler: ' + (d.detail || resp.status)); return; }
alert(`${d.unique_uids || 0} Mail(s) neu vorgemerkt. Werden beim nächsten Poll erneut geprüft.`);
}
function toggleAutoRefresh() {
clearInterval(refreshTimer);
refreshTimer = null;
if (document.getElementById('auto-refresh').checked) {
refreshTimer = setInterval(() => loadLogs(currentOffset), 5000);
} else {
clearInterval(refreshTimer);
refreshTimer = null;
// Nur neu laden, wenn niemand gerade eine Zeile aufgeklappt hat
refreshTimer = setInterval(() => { if (!document.querySelector('.l.open')) reload(); }, 10000);
reload();
}
}
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
// Suche, Level, Zeitraum und Konto in der URL halten: Reload behält die
// Filter, und eine Suche lässt sich als Lesezeichen speichern.
function syncUrl() {
const u = new URLSearchParams();
const q = document.getElementById('log-search').value.trim();
if (q) u.set('q', q);
if (levelFilter) u.set('level', levelFilter);
const range = document.getElementById('log-range').value;
if (range !== '7d') u.set('range', range);
const acc = document.getElementById('log-account').value;
if (acc) u.set('account', acc);
history.replaceState(null, '', u.toString() ? `?${u}` : location.pathname);
}
function clearAdvancedSearch() {
document.getElementById('search-subject').value = '';
document.getElementById('search-from').value = '';
document.getElementById('search-rule').value = '';
document.getElementById('search-message').value = '';
document.getElementById('search-details').value = '';
document.getElementById('search-folder').value = '';
loadLogs();
}
// Initial load
loadLogs();
(function init() {
const u = new URLSearchParams(location.search);
if (u.get('q')) document.getElementById('log-search').value = u.get('q');
if (u.get('range')) document.getElementById('log-range').value = u.get('range');
if (u.get('account')) document.getElementById('log-account').value = u.get('account');
levelFilter = u.get('level') || '';
document.querySelectorAll('#level-filter button').forEach(b => b.classList.toggle('active', b.dataset.l === levelFilter));
reload();
})();
</script>
{% endblock %}