fast, readable log: stop routine noise, keyset paging, token search

The log had 6.45M rows, of which 434 were actual actions: every mail x
every non-matching rule wrote "Keine Übereinstimmung" (5.3M rows, ~1 GB),
plus "keine neuen Mails" per rule every 30s and a "Poll gestartet" row.
Free-text searches took 50-70s.

- Routine entries are only written with LOG_VERBOSE=true; the poll
  summary is logged only when there were new mails, actions or errors.
  "Why didn't my rule match" is now answered by the rule tester.
- The scheduler commits every 25 evaluated mails and after each rule,
  since the dropped log calls used to be the implicit commit points.
- Migration rebuilds filter_logs without the routine rows (live:
  6.45M -> ~38k) via copy instead of DELETE.
- /api/logs: keyset pagination (before_id) instead of OFFSET, count
  capped at 1000, since/until range, multi-level filter, per-field
  filters; ordner: also matches the move target stored in details.
- New log page: one search field with shortcuts (von:, betreff:,
  regel:, ordner:, text:), live search with debounce and abort, active
  filters as removable chips, click on rule/sender/folder in a row to
  filter, time range (default 7 days), level quick filter, expandable
  row details, "Ältere laden", filters kept in the URL.
- Disable Pico's colour transition on segmented buttons (they briefly
  flashed in the primary colour on load).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 10:21:32 +02:00
co-authored by Claude Opus 5.5
parent d189424ab3
commit 9bd2507cb1
6 changed files with 617 additions and 307 deletions
+27 -7
View File
@@ -5,6 +5,7 @@ from datetime import datetime
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy.orm import Session
from app.config import settings
from app.database import SessionLocal
from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
from app.services.encryption import decrypt
@@ -16,6 +17,17 @@ logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler()
# Nach so vielen geprüften Mails wird zwischengespeichert, damit der Poll die
# SQLite-Schreibsperre nicht minutenlang hält (UI-Aktionen würden sonst warten).
COMMIT_EVERY = 25
def _verbose_log(**kwargs) -> None:
"""Routine-Einträge (Nicht-Übereinstimmung, 'keine neuen Mails', Poll-Start)
erzeugten Millionen Logzeilen ohne Nutzen — nur bei LOG_VERBOSE schreiben."""
if settings.log_verbose:
write_log(**kwargs)
def _build_smtp_config(account: Account) -> dict | None:
if account.smtp_host and account.smtp_username and account.smtp_password:
@@ -42,7 +54,7 @@ def _poll_account_sync(account_id: int) -> None:
.all()
)
write_log(
_verbose_log(
message=f"Poll gestartet ({len(rules)} aktive Regel(n))",
level=LogLevel.INFO,
account_id=account.id,
@@ -72,6 +84,7 @@ def _poll_account_sync(account_id: int) -> None:
total_matched = 0
total_actions = 0
total_errors = 0
evaluated_count = 0
# Mail-Cache: einmal geladene Mails pro Ordner/UID wiederverwenden
mail_cache: dict[tuple[str, str], "MailMessage"] = {}
@@ -126,7 +139,7 @@ def _poll_account_sync(account_id: int) -> None:
new_uids.reverse()
if not new_uids:
write_log(
_verbose_log(
message=f"Regel '{rule.name}': keine neuen Mails in '{folder}' ({len(all_uids)} gesamt, alle bereits geprüft)",
level=LogLevel.INFO,
account_id=account.id,
@@ -146,7 +159,7 @@ def _poll_account_sync(account_id: int) -> None:
msg = f"Regel '{rule.name}': {len(batch_uids)} Mail(s) in '{folder}' prüfen ({len(processed_uids)} bereits geprüft)"
if remaining > 0:
msg += f" — {remaining} weitere beim nächsten Poll"
write_log(
_verbose_log(
message=msg,
level=LogLevel.INFO,
account_id=account.id,
@@ -247,8 +260,8 @@ def _poll_account_sync(account_id: int) -> None:
db=db,
)
else:
write_log(
message=f"Keine Übereinstimmung",
_verbose_log(
message="Keine Übereinstimmung",
level=LogLevel.INFO,
account_id=account.id,
account_name=account.name,
@@ -270,7 +283,11 @@ def _poll_account_sync(account_id: int) -> None:
mail_subject=mail.subject[:500] if mail.subject else None,
mail_from=mail.from_addr[:255] if mail.from_addr else None,
))
db.flush()
evaluated_count += 1
if evaluated_count % COMMIT_EVERY == 0:
db.commit()
db.commit()
# Poll-Zusammenfassung
summary_parts = [
@@ -282,7 +299,10 @@ def _poll_account_sync(account_id: int) -> None:
if total_errors > 0:
summary_parts.append(f"{total_errors} Fehler")
write_log(
# Nur loggen, wenn etwas passiert ist — ein Eintrag alle 30 s ohne
# Inhalt verdeckt die relevanten Zeilen. Letzter Poll steht im Dashboard.
summary_log = write_log if (total_new or total_actions or total_errors) else _verbose_log
summary_log(
message=f"Poll abgeschlossen: {', '.join(summary_parts)}",
level=LogLevel.ERROR if total_errors > 0 else LogLevel.SUCCESS if total_actions > 0 else LogLevel.INFO,
account_id=account.id,