fast, readable log: stop routine noise, keyset paging, token search
The log had 6.45M rows, of which 434 were actual actions: every mail x every non-matching rule wrote "Keine Übereinstimmung" (5.3M rows, ~1 GB), plus "keine neuen Mails" per rule every 30s and a "Poll gestartet" row. Free-text searches took 50-70s. - Routine entries are only written with LOG_VERBOSE=true; the poll summary is logged only when there were new mails, actions or errors. "Why didn't my rule match" is now answered by the rule tester. - The scheduler commits every 25 evaluated mails and after each rule, since the dropped log calls used to be the implicit commit points. - Migration rebuilds filter_logs without the routine rows (live: 6.45M -> ~38k) via copy instead of DELETE. - /api/logs: keyset pagination (before_id) instead of OFFSET, count capped at 1000, since/until range, multi-level filter, per-field filters; ordner: also matches the move target stored in details. - New log page: one search field with shortcuts (von:, betreff:, regel:, ordner:, text:), live search with debounce and abort, active filters as removable chips, click on rule/sender/folder in a row to filter, time range (default 7 days), level quick filter, expandable row details, "Ältere laden", filters kept in the URL. - Disable Pico's colour transition on segmented buttons (they briefly flashed in the primary colour on load). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
"""purge routine entries from filter_logs
|
||||
|
||||
Revision ID: 9a3f5c1d7e42
|
||||
Revises: 7d1e4a2b9c30
|
||||
Create Date: 2026-09-29 00:00:00.000000
|
||||
|
||||
"""
|
||||
import re
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
revision: str = "9a3f5c1d7e42"
|
||||
down_revision: Union[str, Sequence[str], None] = "7d1e4a2b9c30"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# Routine-Einträge, die ab jetzt nur noch mit LOG_VERBOSE geschrieben werden.
|
||||
# Sie machten >99% der Tabelle aus (Millionen Zeilen) und machten Suchen langsam.
|
||||
NOISE = """
|
||||
message = 'Keine Übereinstimmung'
|
||||
OR message LIKE 'Regel %keine neuen Mails%'
|
||||
OR message LIKE 'Regel %Mail(s) in %prüfen%'
|
||||
OR message LIKE 'Poll gestartet%'
|
||||
OR (message LIKE 'Poll abgeschlossen%' AND message LIKE '%, 0 neu, 0 Treffer, 0 Aktion(en)')
|
||||
"""
|
||||
|
||||
INDEXES = {
|
||||
"ix_filter_logs_created_at": "(created_at)",
|
||||
"ix_filter_logs_level": "(level)",
|
||||
"ix_filter_logs_account_created": "(account_id, created_at)",
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Tabelle neu aufbauen statt DELETE: nur die relevanten Zeilen kopieren
|
||||
ist bei Millionen Einträgen um ein Vielfaches schneller und erzeugt kein
|
||||
riesiges WAL."""
|
||||
bind = op.get_bind()
|
||||
ddl = bind.execute(
|
||||
text("SELECT sql FROM sqlite_master WHERE type='table' AND name='filter_logs'")
|
||||
).scalar()
|
||||
if not ddl:
|
||||
return
|
||||
new_ddl = re.sub(r'^CREATE TABLE\s+"?filter_logs"?', 'CREATE TABLE filter_logs_new', ddl, count=1)
|
||||
|
||||
op.execute("DROP TABLE IF EXISTS filter_logs_new")
|
||||
op.execute(new_ddl)
|
||||
op.execute(f"INSERT INTO filter_logs_new SELECT * FROM filter_logs WHERE NOT ({NOISE})")
|
||||
for name in INDEXES:
|
||||
op.execute(f"DROP INDEX IF EXISTS {name}")
|
||||
op.execute("DROP TABLE filter_logs")
|
||||
op.execute("ALTER TABLE filter_logs_new RENAME TO filter_logs")
|
||||
for name, cols in INDEXES.items():
|
||||
op.execute(f"CREATE INDEX {name} ON filter_logs {cols}")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Gelöschte Routine-Einträge lassen sich nicht wiederherstellen.
|
||||
pass
|
||||
Reference in New Issue
Block a user