Files
imap-mail-filter-service/app/routers/logs.py
T
duffyduckandClaude Opus 5.5 9bd2507cb1 fast, readable log: stop routine noise, keyset paging, token search
The log had 6.45M rows, of which 434 were actual actions: every mail x
every non-matching rule wrote "Keine Übereinstimmung" (5.3M rows, ~1 GB),
plus "keine neuen Mails" per rule every 30s and a "Poll gestartet" row.
Free-text searches took 50-70s.

- Routine entries are only written with LOG_VERBOSE=true; the poll
  summary is logged only when there were new mails, actions or errors.
  "Why didn't my rule match" is now answered by the rule tester.
- The scheduler commits every 25 evaluated mails and after each rule,
  since the dropped log calls used to be the implicit commit points.
- Migration rebuilds filter_logs without the routine rows (live:
  6.45M -> ~38k) via copy instead of DELETE.
- /api/logs: keyset pagination (before_id) instead of OFFSET, count
  capped at 1000, since/until range, multi-level filter, per-field
  filters; ordner: also matches the move target stored in details.
- New log page: one search field with shortcuts (von:, betreff:,
  regel:, ordner:, text:), live search with debounce and abort, active
  filters as removable chips, click on rule/sender/folder in a row to
  filter, time range (default 7 days), level quick filter, expandable
  row details, "Ältere laden", filters kept in the URL.
- Disable Pico's colour transition on segmented buttons (they briefly
  flashed in the primary colour on load).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 10:21:32 +02:00

163 lines
6.3 KiB
Python

import logging
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import or_, text
from sqlalchemy.orm import Session
from app.database import engine, get_db
from app.models.db_models import FilterLog, LogLevel
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/logs", tags=["logs"])
# Obergrenze fürs Zählen: exakte Zahlen über Millionen Zeilen kosten Sekunden,
# für die Anzeige reicht "mehr als 1.000".
COUNT_CAP = 1000
def _like(value: str) -> str:
escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
return f"%{escaped}%"
def _parse_ts(value: str | None) -> datetime | None:
if not value:
return None
try:
# Browser schickt ISO mit 'Z' (UTC) — DB speichert naive UTC
return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
except ValueError:
raise HTTPException(400, f"Ungültiges Datum: {value}")
@router.get("/")
def get_logs(
account_id: int | None = None,
level: str | None = Query(default=None, description="Kommagetrennt: success,info,warning,error"),
q: str | None = Query(default=None, description="Freitext über alle Felder"),
sender: str | None = None,
subject: str | None = None,
rule: str | None = None,
folder: str | None = None,
message: str | None = None,
since: str | None = None,
until: str | None = None,
before_id: int | None = Query(default=None, description="Für 'Mehr laden': nur Einträge älter als diese ID"),
limit: int = Query(default=100, ge=1, le=500),
db: Session = Depends(get_db),
):
"""Neueste zuerst, geblättert per before_id (Keyset) statt OFFSET: jede Seite
ist gleich schnell, egal wie weit man zurückblättert."""
query = db.query(FilterLog)
if account_id:
query = query.filter(FilterLog.account_id == account_id)
if level:
levels = []
for part in level.split(","):
try:
levels.append(LogLevel(part.strip()))
except ValueError:
raise HTTPException(400, f"Unbekanntes Level: {part}")
query = query.filter(FilterLog.level.in_(levels))
ts_since, ts_until = _parse_ts(since), _parse_ts(until)
if ts_since:
query = query.filter(FilterLog.created_at >= ts_since)
if ts_until:
query = query.filter(FilterLog.created_at < ts_until)
# Freitext: jedes Wort muss in irgendeinem Feld vorkommen
if q:
for word in q.split():
pat = _like(word)
query = query.filter(or_(
FilterLog.message.ilike(pat, escape="\\"),
FilterLog.mail_subject.ilike(pat, escape="\\"),
FilterLog.mail_from.ilike(pat, escape="\\"),
FilterLog.rule_name.ilike(pat, escape="\\"),
FilterLog.details.ilike(pat, escape="\\"),
FilterLog.folder.ilike(pat, escape="\\"),
))
for column, value in (
(FilterLog.mail_from, sender),
(FilterLog.mail_subject, subject),
(FilterLog.rule_name, rule),
(FilterLog.message, message),
):
if value:
query = query.filter(column.ilike(_like(value), escape="\\"))
if folder:
# Quellordner steht in folder, das Ziel einer Verschiebe-Aktion in details
pat = _like(folder)
query = query.filter(or_(
FilterLog.folder.ilike(pat, escape="\\"),
(FilterLog.action_type == "move") & FilterLog.details.ilike(pat, escape="\\"),
))
count_query = query
if before_id:
query = query.filter(FilterLog.id < before_id)
rows = query.order_by(FilterLog.id.desc()).limit(limit + 1).all()
has_more = len(rows) > limit
rows = rows[:limit]
# Treffermenge nur bis COUNT_CAP zählen (und nur für die erste Seite)
total = None
if before_id is None:
total = count_query.with_entities(FilterLog.id).limit(COUNT_CAP + 1).count()
return {
"total": total,
"total_capped": total is not None and total > COUNT_CAP,
"has_more": has_more,
"next_before_id": rows[-1].id if rows and has_more else None,
"logs": [
{
"id": log.id,
"account_id": log.account_id,
"account_name": log.account_name,
"level": log.level.value if log.level else "info",
"message": log.message,
"rule_name": log.rule_name,
"action_type": log.action_type,
"mail_uid": log.mail_uid,
"mail_subject": log.mail_subject,
"mail_from": log.mail_from,
"folder": log.folder,
"details": log.details,
"created_at": log.created_at.isoformat() if log.created_at else None,
}
for log in rows
],
}
@router.delete("/")
def clear_logs(account_id: int | None = None, db: Session = Depends(get_db)):
"""Löscht Logs. Ohne account_id: schnelles Drop+Recreate der Tabelle
(sekundenschnell auch bei Millionen Zeilen). Mit account_id: normales DELETE."""
try:
if account_id is None:
# Vollständiges Leeren: DROP+CREATE statt DELETE.
# DELETE FROM filter_logs würde jede Zeile einzeln ins WAL schreiben,
# bei Millionen Einträgen dauert das ewig und blockiert die DB.
db.close() # Sitzung freigeben, sonst holds sie noch Locks
row_count = 0
with engine.begin() as conn:
row_count = conn.execute(text("SELECT COUNT(*) FROM filter_logs")).scalar() or 0
FilterLog.__table__.drop(conn, checkfirst=True)
FilterLog.__table__.create(conn, checkfirst=True)
# PRAGMA wal_checkpoint(TRUNCATE) räumt die WAL-Datei auf
conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
logger.info("Log vollständig geleert (%d Einträge, Tabelle neu angelegt)", row_count)
return {"deleted": row_count}
count = db.query(FilterLog).filter(FilterLog.account_id == account_id).delete()
db.commit()
return {"deleted": count}
except Exception as e:
logger.exception("clear_logs fehlgeschlagen")
raise HTTPException(500, f"Fehler beim Leeren des Logs: {e}")