fast, readable log: stop routine noise, keyset paging, token search
The log had 6.45M rows, of which 434 were actual actions: every mail x every non-matching rule wrote "Keine Übereinstimmung" (5.3M rows, ~1 GB), plus "keine neuen Mails" per rule every 30s and a "Poll gestartet" row. Free-text searches took 50-70s. - Routine entries are only written with LOG_VERBOSE=true; the poll summary is logged only when there were new mails, actions or errors. "Why didn't my rule match" is now answered by the rule tester. - The scheduler commits every 25 evaluated mails and after each rule, since the dropped log calls used to be the implicit commit points. - Migration rebuilds filter_logs without the routine rows (live: 6.45M -> ~38k) via copy instead of DELETE. - /api/logs: keyset pagination (before_id) instead of OFFSET, count capped at 1000, since/until range, multi-level filter, per-field filters; ordner: also matches the move target stored in details. - New log page: one search field with shortcuts (von:, betreff:, regel:, ordner:, text:), live search with debounce and abort, active filters as removable chips, click on rule/sender/folder in a row to filter, time range (default 7 days), level quick filter, expandable row details, "Ältere laden", filters kept in the URL. - Disable Pico's colour transition on segmented buttons (they briefly flashed in the primary colour on load). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.database import SessionLocal
|
||||
from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
|
||||
from app.services.encryption import decrypt
|
||||
@@ -16,6 +17,17 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
# Nach so vielen geprüften Mails wird zwischengespeichert, damit der Poll die
|
||||
# SQLite-Schreibsperre nicht minutenlang hält (UI-Aktionen würden sonst warten).
|
||||
COMMIT_EVERY = 25
|
||||
|
||||
|
||||
def _verbose_log(**kwargs) -> None:
|
||||
"""Routine-Einträge (Nicht-Übereinstimmung, 'keine neuen Mails', Poll-Start)
|
||||
erzeugten Millionen Logzeilen ohne Nutzen — nur bei LOG_VERBOSE schreiben."""
|
||||
if settings.log_verbose:
|
||||
write_log(**kwargs)
|
||||
|
||||
|
||||
def _build_smtp_config(account: Account) -> dict | None:
|
||||
if account.smtp_host and account.smtp_username and account.smtp_password:
|
||||
@@ -42,7 +54,7 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
.all()
|
||||
)
|
||||
|
||||
write_log(
|
||||
_verbose_log(
|
||||
message=f"Poll gestartet ({len(rules)} aktive Regel(n))",
|
||||
level=LogLevel.INFO,
|
||||
account_id=account.id,
|
||||
@@ -72,6 +84,7 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
total_matched = 0
|
||||
total_actions = 0
|
||||
total_errors = 0
|
||||
evaluated_count = 0
|
||||
|
||||
# Mail-Cache: einmal geladene Mails pro Ordner/UID wiederverwenden
|
||||
mail_cache: dict[tuple[str, str], "MailMessage"] = {}
|
||||
@@ -126,7 +139,7 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
new_uids.reverse()
|
||||
|
||||
if not new_uids:
|
||||
write_log(
|
||||
_verbose_log(
|
||||
message=f"Regel '{rule.name}': keine neuen Mails in '{folder}' ({len(all_uids)} gesamt, alle bereits geprüft)",
|
||||
level=LogLevel.INFO,
|
||||
account_id=account.id,
|
||||
@@ -146,7 +159,7 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
msg = f"Regel '{rule.name}': {len(batch_uids)} Mail(s) in '{folder}' prüfen ({len(processed_uids)} bereits geprüft)"
|
||||
if remaining > 0:
|
||||
msg += f" — {remaining} weitere beim nächsten Poll"
|
||||
write_log(
|
||||
_verbose_log(
|
||||
message=msg,
|
||||
level=LogLevel.INFO,
|
||||
account_id=account.id,
|
||||
@@ -247,8 +260,8 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
db=db,
|
||||
)
|
||||
else:
|
||||
write_log(
|
||||
message=f"Keine Übereinstimmung",
|
||||
_verbose_log(
|
||||
message="Keine Übereinstimmung",
|
||||
level=LogLevel.INFO,
|
||||
account_id=account.id,
|
||||
account_name=account.name,
|
||||
@@ -270,7 +283,11 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
mail_subject=mail.subject[:500] if mail.subject else None,
|
||||
mail_from=mail.from_addr[:255] if mail.from_addr else None,
|
||||
))
|
||||
db.flush()
|
||||
evaluated_count += 1
|
||||
if evaluated_count % COMMIT_EVERY == 0:
|
||||
db.commit()
|
||||
|
||||
db.commit()
|
||||
|
||||
# Poll-Zusammenfassung
|
||||
summary_parts = [
|
||||
@@ -282,7 +299,10 @@ def _poll_account_sync(account_id: int) -> None:
|
||||
if total_errors > 0:
|
||||
summary_parts.append(f"{total_errors} Fehler")
|
||||
|
||||
write_log(
|
||||
# Nur loggen, wenn etwas passiert ist — ein Eintrag alle 30 s ohne
|
||||
# Inhalt verdeckt die relevanten Zeilen. Letzter Poll steht im Dashboard.
|
||||
summary_log = write_log if (total_new or total_actions or total_errors) else _verbose_log
|
||||
summary_log(
|
||||
message=f"Poll abgeschlossen: {', '.join(summary_parts)}",
|
||||
level=LogLevel.ERROR if total_errors > 0 else LogLevel.SUCCESS if total_actions > 0 else LogLevel.INFO,
|
||||
account_id=account.id,
|
||||
|
||||
Reference in New Issue
Block a user