diff --git a/README.md b/README.md
index bd042e1..2c15c38 100644
--- a/README.md
+++ b/README.md
@@ -121,6 +121,12 @@ Beim Start wird `config/filters.yaml` automatisch importiert (konfigurierbar via
| `ENCRYPTION_KEY` | *(leer)* | Fernet-Key für Passwort-Verschlüsselung |
| `LOG_LEVEL` | `INFO` | Log-Level (DEBUG, INFO, WARNING, ERROR) |
| `YAML_SYNC_ON_STARTUP` | `true` | YAML-Datei beim Start importieren |
+| `LOG_RETENTION_DAYS` | `30` | Verarbeitungslog: Einträge älter als N Tage werden gelöscht |
+| `LOG_MAX_ENTRIES` | `200000` | Verarbeitungslog: höchstens so viele (neueste) Einträge behalten |
+| `MAINTENANCE_HOUR_UTC` | `2` | Stunde (UTC) der täglichen DB-Wartung; läuft zusätzlich 5 Min. nach jedem Start |
+| `LOG_VERBOSE` | `false` | Jede Nicht-Übereinstimmung loggen — erzeugt Millionen Einträge, nur zur Fehlersuche |
+
+Die tägliche Wartung löscht alte Log-Einträge, entfernt Verarbeitungsmarker gelöschter Regeln und verkleinert die SQLite-Datei per `VACUUM`, sobald mehr als 25 % ungenutzt sind (nur wenn genug Plattenplatz frei ist). Manuell auslösbar im Log unter ⋯ → „Datenbank aufräumen & verkleinern".
| `DATABASE_URL` | `sqlite:///data/mailfilter.db` | Datenbank-Pfad |
## Datenbank-Migrationen
diff --git a/app/config.py b/app/config.py
index 02edcc5..0458910 100644
--- a/app/config.py
+++ b/app/config.py
@@ -12,6 +12,12 @@ class Settings(BaseSettings):
# Ausführliches Verarbeitungslog (jede Nicht-Übereinstimmung, jede Regel pro
# Poll). Erzeugt Millionen Einträge — nur zur Fehlersuche einschalten.
log_verbose: bool = False
+ # Log-Rotation: Einträge älter als N Tage löschen und höchstens so viele
+ # behalten (die neuesten). Läuft täglich zur angegebenen Stunde (UTC) und
+ # einige Minuten nach jedem Start.
+ log_retention_days: int = 30
+ log_max_entries: int = 200_000
+ maintenance_hour_utc: int = 2
model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
diff --git a/app/routers/logs.py b/app/routers/logs.py
index 8526af0..3cb2cd2 100644
--- a/app/routers/logs.py
+++ b/app/routers/logs.py
@@ -7,6 +7,7 @@ from sqlalchemy.orm import Session
from app.database import engine, get_db
from app.models.db_models import FilterLog, LogLevel
+from app.services.maintenance import db_status, run_maintenance
logger = logging.getLogger(__name__)
@@ -134,6 +135,23 @@ def get_logs(
}
+@router.get("/maintenance")
+def maintenance_status():
+ """Größe der Datenbank, Anzahl Einträge und Ergebnis der letzten Wartung."""
+ return db_status()
+
+
+@router.post("/maintenance")
+def maintenance_run(vacuum: bool = False):
+ """Wartung sofort ausführen (Log-Rotation, verwaiste Marker, ggf. VACUUM).
+ vacuum=true erzwingt das Verkleinern der Datei."""
+ try:
+ result = run_maintenance(force_vacuum=vacuum)
+ except Exception as e:
+ raise HTTPException(500, f"Wartung fehlgeschlagen: {e}")
+ return {"result": result, "status": db_status()}
+
+
@router.delete("/")
def clear_logs(account_id: int | None = None, db: Session = Depends(get_db)):
"""Löscht Logs. Ohne account_id: schnelles Drop+Recreate der Tabelle
diff --git a/app/services/log_service.py b/app/services/log_service.py
index d80f3d2..980c5de 100644
--- a/app/services/log_service.py
+++ b/app/services/log_service.py
@@ -1,5 +1,4 @@
import logging
-from datetime import datetime, timedelta
from sqlalchemy.orm import Session
@@ -50,17 +49,3 @@ def write_log(
if close_db:
db.close()
-
-def cleanup_old_logs(days: int = 30, db: Session | None = None) -> int:
- close_db = False
- if db is None:
- db = SessionLocal()
- close_db = True
- try:
- cutoff = datetime.utcnow() - timedelta(days=days)
- count = db.query(FilterLog).filter(FilterLog.created_at < cutoff).delete()
- db.commit()
- return count
- finally:
- if close_db:
- db.close()
diff --git a/app/services/maintenance.py b/app/services/maintenance.py
new file mode 100644
index 0000000..54ca8a9
--- /dev/null
+++ b/app/services/maintenance.py
@@ -0,0 +1,141 @@
+"""Datenbank-Wartung: Log-Rotation, verwaiste Marker entfernen und bei Bedarf
+VACUUM, damit die SQLite-Datei nicht unbegrenzt wächst.
+
+SQLite gibt durch DELETE frei gewordenen Platz nicht ans Dateisystem zurück —
+die Datei schrumpft erst durch VACUUM. Das kopiert die Datenbank einmal neu
+und braucht dafür kurzzeitig freien Platz in Größe der verbleibenden Daten."""
+import logging
+import os
+import shutil
+import threading
+import time
+from datetime import datetime, timedelta
+
+from sqlalchemy import text
+
+from app.config import settings
+from app.database import engine
+
+logger = logging.getLogger(__name__)
+
+# VACUUM nur, wenn sich das lohnt: mindestens so viel Anteil bzw. Größe frei
+VACUUM_MIN_FREE_RATIO = 0.25
+VACUUM_MIN_FREE_BYTES = 50 * 1024 * 1024
+
+_lock = threading.Lock()
+last_result: dict | None = None
+
+
+def _db_path() -> str | None:
+ url = settings.database_url
+ if not url.startswith("sqlite:///"):
+ return None
+ return url.removeprefix("sqlite:///")
+
+
+def _file_size(path: str | None) -> int:
+ if not path:
+ return 0
+ total = 0
+ for suffix in ("", "-wal"):
+ try:
+ total += os.path.getsize(path + suffix)
+ except OSError:
+ pass
+ return total
+
+
+def db_status() -> dict:
+ path = _db_path()
+ with engine.connect() as conn:
+ page_size = conn.exec_driver_sql("PRAGMA page_size").scalar()
+ page_count = conn.exec_driver_sql("PRAGMA page_count").scalar()
+ free_pages = conn.exec_driver_sql("PRAGMA freelist_count").scalar()
+ log_rows = conn.execute(text("SELECT COUNT(*) FROM filter_logs")).scalar()
+ processed_rows = conn.execute(text("SELECT COUNT(*) FROM processed_mails")).scalar()
+ disk_free = shutil.disk_usage(os.path.dirname(os.path.abspath(path))).free if path else None
+ return {
+ "file_bytes": _file_size(path),
+ "used_bytes": (page_count - free_pages) * page_size,
+ "free_bytes": free_pages * page_size,
+ "log_rows": log_rows,
+ "processed_rows": processed_rows,
+ "disk_free_bytes": disk_free,
+ "retention_days": settings.log_retention_days,
+ "max_entries": settings.log_max_entries,
+ "last_run": last_result,
+ }
+
+
+def run_maintenance(force_vacuum: bool = False) -> dict:
+ """Läuft synchron (aus Thread/Executor aufrufen). Parallel-Läufe werden
+ übersprungen."""
+ global last_result
+ if not _lock.acquire(blocking=False):
+ return {"skipped": "Wartung läuft bereits"}
+ started = time.monotonic()
+ try:
+ path = _db_path()
+ size_before = _file_size(path)
+ result: dict = {"started_at": datetime.utcnow().isoformat(), "size_before": size_before}
+
+ with engine.begin() as conn:
+ cutoff = datetime.utcnow() - timedelta(days=settings.log_retention_days)
+ result["logs_deleted_age"] = conn.execute(
+ text("DELETE FROM filter_logs WHERE created_at < :cutoff"), {"cutoff": cutoff}
+ ).rowcount
+
+ # Obergrenze: nur die neuesten N behalten
+ boundary = conn.execute(
+ text("SELECT id FROM filter_logs ORDER BY id DESC LIMIT 1 OFFSET :n"),
+ {"n": settings.log_max_entries},
+ ).scalar()
+ result["logs_deleted_cap"] = (
+ conn.execute(text("DELETE FROM filter_logs WHERE id <= :b"), {"b": boundary}).rowcount
+ if boundary is not None else 0
+ )
+
+ # Marker gelöschter Regeln/Konten (SQLite erzwingt die Fremdschlüssel nicht)
+ result["processed_orphans_deleted"] = conn.execute(text(
+ "DELETE FROM processed_mails "
+ "WHERE rule_id NOT IN (SELECT id FROM filter_rules) "
+ "OR account_id NOT IN (SELECT id FROM accounts)"
+ )).rowcount
+
+ with engine.connect() as conn:
+ conn.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)")
+ page_size = conn.exec_driver_sql("PRAGMA page_size").scalar()
+ page_count = conn.exec_driver_sql("PRAGMA page_count").scalar()
+ free_pages = conn.exec_driver_sql("PRAGMA freelist_count").scalar()
+ free_bytes = free_pages * page_size
+ used_bytes = (page_count - free_pages) * page_size
+ worthwhile = free_bytes >= VACUUM_MIN_FREE_BYTES and free_bytes >= VACUUM_MIN_FREE_RATIO * page_count * page_size
+
+ result["vacuum"] = "nicht nötig"
+ if force_vacuum or worthwhile:
+ disk_free = shutil.disk_usage(os.path.dirname(os.path.abspath(path))).free if path else 0
+ # VACUUM schreibt eine Kopie der verbleibenden Daten (plus WAL)
+ if path and disk_free < used_bytes * 1.2:
+ result["vacuum"] = (
+ f"übersprungen: zu wenig Plattenplatz ({disk_free // 2**20} MB frei, "
+ f"~{int(used_bytes * 1.2) // 2**20} MB nötig)"
+ )
+ else:
+ t = time.monotonic()
+ # VACUUM darf nicht in einer Transaktion laufen
+ with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+ conn.exec_driver_sql("VACUUM")
+ conn.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)")
+ result["vacuum"] = f"ausgeführt ({time.monotonic() - t:.0f}s)"
+
+ result["size_after"] = _file_size(path)
+ result["duration_s"] = round(time.monotonic() - started, 1)
+ logger.info("DB-Wartung: %s", result)
+ last_result = result
+ return result
+ except Exception as e:
+ logger.exception("DB-Wartung fehlgeschlagen")
+ last_result = {"error": str(e), "started_at": datetime.utcnow().isoformat()}
+ raise
+ finally:
+ _lock.release()
diff --git a/app/services/scheduler.py b/app/services/scheduler.py
index 980040a..6391191 100644
--- a/app/services/scheduler.py
+++ b/app/services/scheduler.py
@@ -1,6 +1,6 @@
import asyncio
import logging
-from datetime import datetime
+from datetime import datetime, timedelta
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy.orm import Session
@@ -11,7 +11,8 @@ from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
from app.services.encryption import decrypt
from app.services.filter_engine import evaluate_conditions, execute_action
from app.services.imap_client import IMAPClient
-from app.services.log_service import cleanup_old_logs, write_log
+from app.services.log_service import write_log
+from app.services.maintenance import run_maintenance
logger = logging.getLogger(__name__)
@@ -329,6 +330,13 @@ async def poll_account(account_id: int) -> None:
await asyncio.to_thread(_poll_account_sync, account_id)
+async def maintenance_job() -> None:
+ try:
+ await asyncio.to_thread(run_maintenance)
+ except Exception:
+ pass # bereits in run_maintenance geloggt
+
+
def add_account_job(account: Account) -> None:
job_id = f"poll_account_{account.id}"
if scheduler.get_job(job_id):
@@ -364,12 +372,23 @@ def start_scheduler() -> None:
finally:
db.close()
- # Täglicher Cleanup alter Logs
+ # DB-Wartung (Log-Rotation, VACUUM): täglich zu fester Uhrzeit — ein
+ # 24h-Intervall ab Start lief bei häufigen Deploys praktisch nie — und
+ # zusätzlich kurz nach jedem Start.
scheduler.add_job(
- lambda: asyncio.get_event_loop().run_in_executor(None, cleanup_old_logs, 30),
- "interval",
- hours=24,
- id="cleanup_logs",
+ maintenance_job,
+ "cron",
+ hour=settings.maintenance_hour_utc,
+ minute=30,
+ timezone="UTC",
+ id="db_maintenance",
+ replace_existing=True,
+ )
+ scheduler.add_job(
+ maintenance_job,
+ "date",
+ run_date=datetime.now() + timedelta(minutes=5),
+ id="db_maintenance_startup",
replace_existing=True,
)
diff --git a/app/templates/logs.html b/app/templates/logs.html
index ac9c9dc..28a22ac 100644
--- a/app/templates/logs.html
+++ b/app/templates/logs.html
@@ -20,9 +20,11 @@
-