automatic DB maintenance: log rotation, orphan cleanup, VACUUM
The existing daily cleanup never effectively ran: it was an interval job counted from container start (reset on every deploy), it called asyncio.get_event_loop() from a worker thread, and even when deleting, SQLite never returns freed pages to the filesystem. Live DB: 4.1 GB, of which 2.6 GB log and 0.6 GB unused pages. New maintenance service, scheduled daily at MAINTENANCE_HOUR_UTC:30 and once 5 minutes after every start: - delete log entries older than LOG_RETENTION_DAYS (30) and keep at most LOG_MAX_ENTRIES (200k) newest ones - delete processed_mails markers of deleted rules/accounts (FKs are not enforced by SQLite) - VACUUM when >25% of the file is unused, only if free disk space covers the copy; never runs twice in parallel GET/POST /api/logs/maintenance expose status and a manual run; the log menu shows DB size, entry count and last run, with "Datenbank aufräumen & verkleinern". Removed the dead cleanup_old_logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -121,6 +121,12 @@ Beim Start wird `config/filters.yaml` automatisch importiert (konfigurierbar via
|
||||
| `ENCRYPTION_KEY` | *(leer)* | Fernet-Key für Passwort-Verschlüsselung |
|
||||
| `LOG_LEVEL` | `INFO` | Log-Level (DEBUG, INFO, WARNING, ERROR) |
|
||||
| `YAML_SYNC_ON_STARTUP` | `true` | YAML-Datei beim Start importieren |
|
||||
| `LOG_RETENTION_DAYS` | `30` | Verarbeitungslog: Einträge älter als N Tage werden gelöscht |
|
||||
| `LOG_MAX_ENTRIES` | `200000` | Verarbeitungslog: höchstens so viele (neueste) Einträge behalten |
|
||||
| `MAINTENANCE_HOUR_UTC` | `2` | Stunde (UTC) der täglichen DB-Wartung; läuft zusätzlich 5 Min. nach jedem Start |
|
||||
| `LOG_VERBOSE` | `false` | Jede Nicht-Übereinstimmung loggen — erzeugt Millionen Einträge, nur zur Fehlersuche |
|
||||
|
||||
Die tägliche Wartung löscht alte Log-Einträge, entfernt Verarbeitungsmarker gelöschter Regeln und verkleinert die SQLite-Datei per `VACUUM`, sobald mehr als 25 % ungenutzt sind (nur wenn genug Plattenplatz frei ist). Manuell auslösbar im Log unter ⋯ → „Datenbank aufräumen & verkleinern".
|
||||
| `DATABASE_URL` | `sqlite:///data/mailfilter.db` | Datenbank-Pfad |
|
||||
|
||||
## Datenbank-Migrationen
|
||||
|
||||
@@ -12,6 +12,12 @@ class Settings(BaseSettings):
|
||||
# Ausführliches Verarbeitungslog (jede Nicht-Übereinstimmung, jede Regel pro
|
||||
# Poll). Erzeugt Millionen Einträge — nur zur Fehlersuche einschalten.
|
||||
log_verbose: bool = False
|
||||
# Log-Rotation: Einträge älter als N Tage löschen und höchstens so viele
|
||||
# behalten (die neuesten). Läuft täglich zur angegebenen Stunde (UTC) und
|
||||
# einige Minuten nach jedem Start.
|
||||
log_retention_days: int = 30
|
||||
log_max_entries: int = 200_000
|
||||
maintenance_hour_utc: int = 2
|
||||
|
||||
model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import engine, get_db
|
||||
from app.models.db_models import FilterLog, LogLevel
|
||||
from app.services.maintenance import db_status, run_maintenance
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -134,6 +135,23 @@ def get_logs(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/maintenance")
|
||||
def maintenance_status():
|
||||
"""Größe der Datenbank, Anzahl Einträge und Ergebnis der letzten Wartung."""
|
||||
return db_status()
|
||||
|
||||
|
||||
@router.post("/maintenance")
|
||||
def maintenance_run(vacuum: bool = False):
|
||||
"""Wartung sofort ausführen (Log-Rotation, verwaiste Marker, ggf. VACUUM).
|
||||
vacuum=true erzwingt das Verkleinern der Datei."""
|
||||
try:
|
||||
result = run_maintenance(force_vacuum=vacuum)
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Wartung fehlgeschlagen: {e}")
|
||||
return {"result": result, "status": db_status()}
|
||||
|
||||
|
||||
@router.delete("/")
|
||||
def clear_logs(account_id: int | None = None, db: Session = Depends(get_db)):
|
||||
"""Löscht Logs. Ohne account_id: schnelles Drop+Recreate der Tabelle
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -50,17 +49,3 @@ def write_log(
|
||||
if close_db:
|
||||
db.close()
|
||||
|
||||
|
||||
def cleanup_old_logs(days: int = 30, db: Session | None = None) -> int:
|
||||
close_db = False
|
||||
if db is None:
|
||||
db = SessionLocal()
|
||||
close_db = True
|
||||
try:
|
||||
cutoff = datetime.utcnow() - timedelta(days=days)
|
||||
count = db.query(FilterLog).filter(FilterLog.created_at < cutoff).delete()
|
||||
db.commit()
|
||||
return count
|
||||
finally:
|
||||
if close_db:
|
||||
db.close()
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Datenbank-Wartung: Log-Rotation, verwaiste Marker entfernen und bei Bedarf
|
||||
VACUUM, damit die SQLite-Datei nicht unbegrenzt wächst.
|
||||
|
||||
SQLite gibt durch DELETE frei gewordenen Platz nicht ans Dateisystem zurück —
|
||||
die Datei schrumpft erst durch VACUUM. Das kopiert die Datenbank einmal neu
|
||||
und braucht dafür kurzzeitig freien Platz in Größe der verbleibenden Daten."""
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.database import engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# VACUUM nur, wenn sich das lohnt: mindestens so viel Anteil bzw. Größe frei
|
||||
VACUUM_MIN_FREE_RATIO = 0.25
|
||||
VACUUM_MIN_FREE_BYTES = 50 * 1024 * 1024
|
||||
|
||||
_lock = threading.Lock()
|
||||
last_result: dict | None = None
|
||||
|
||||
|
||||
def _db_path() -> str | None:
|
||||
url = settings.database_url
|
||||
if not url.startswith("sqlite:///"):
|
||||
return None
|
||||
return url.removeprefix("sqlite:///")
|
||||
|
||||
|
||||
def _file_size(path: str | None) -> int:
|
||||
if not path:
|
||||
return 0
|
||||
total = 0
|
||||
for suffix in ("", "-wal"):
|
||||
try:
|
||||
total += os.path.getsize(path + suffix)
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def db_status() -> dict:
|
||||
path = _db_path()
|
||||
with engine.connect() as conn:
|
||||
page_size = conn.exec_driver_sql("PRAGMA page_size").scalar()
|
||||
page_count = conn.exec_driver_sql("PRAGMA page_count").scalar()
|
||||
free_pages = conn.exec_driver_sql("PRAGMA freelist_count").scalar()
|
||||
log_rows = conn.execute(text("SELECT COUNT(*) FROM filter_logs")).scalar()
|
||||
processed_rows = conn.execute(text("SELECT COUNT(*) FROM processed_mails")).scalar()
|
||||
disk_free = shutil.disk_usage(os.path.dirname(os.path.abspath(path))).free if path else None
|
||||
return {
|
||||
"file_bytes": _file_size(path),
|
||||
"used_bytes": (page_count - free_pages) * page_size,
|
||||
"free_bytes": free_pages * page_size,
|
||||
"log_rows": log_rows,
|
||||
"processed_rows": processed_rows,
|
||||
"disk_free_bytes": disk_free,
|
||||
"retention_days": settings.log_retention_days,
|
||||
"max_entries": settings.log_max_entries,
|
||||
"last_run": last_result,
|
||||
}
|
||||
|
||||
|
||||
def run_maintenance(force_vacuum: bool = False) -> dict:
|
||||
"""Läuft synchron (aus Thread/Executor aufrufen). Parallel-Läufe werden
|
||||
übersprungen."""
|
||||
global last_result
|
||||
if not _lock.acquire(blocking=False):
|
||||
return {"skipped": "Wartung läuft bereits"}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
path = _db_path()
|
||||
size_before = _file_size(path)
|
||||
result: dict = {"started_at": datetime.utcnow().isoformat(), "size_before": size_before}
|
||||
|
||||
with engine.begin() as conn:
|
||||
cutoff = datetime.utcnow() - timedelta(days=settings.log_retention_days)
|
||||
result["logs_deleted_age"] = conn.execute(
|
||||
text("DELETE FROM filter_logs WHERE created_at < :cutoff"), {"cutoff": cutoff}
|
||||
).rowcount
|
||||
|
||||
# Obergrenze: nur die neuesten N behalten
|
||||
boundary = conn.execute(
|
||||
text("SELECT id FROM filter_logs ORDER BY id DESC LIMIT 1 OFFSET :n"),
|
||||
{"n": settings.log_max_entries},
|
||||
).scalar()
|
||||
result["logs_deleted_cap"] = (
|
||||
conn.execute(text("DELETE FROM filter_logs WHERE id <= :b"), {"b": boundary}).rowcount
|
||||
if boundary is not None else 0
|
||||
)
|
||||
|
||||
# Marker gelöschter Regeln/Konten (SQLite erzwingt die Fremdschlüssel nicht)
|
||||
result["processed_orphans_deleted"] = conn.execute(text(
|
||||
"DELETE FROM processed_mails "
|
||||
"WHERE rule_id NOT IN (SELECT id FROM filter_rules) "
|
||||
"OR account_id NOT IN (SELECT id FROM accounts)"
|
||||
)).rowcount
|
||||
|
||||
with engine.connect() as conn:
|
||||
conn.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
page_size = conn.exec_driver_sql("PRAGMA page_size").scalar()
|
||||
page_count = conn.exec_driver_sql("PRAGMA page_count").scalar()
|
||||
free_pages = conn.exec_driver_sql("PRAGMA freelist_count").scalar()
|
||||
free_bytes = free_pages * page_size
|
||||
used_bytes = (page_count - free_pages) * page_size
|
||||
worthwhile = free_bytes >= VACUUM_MIN_FREE_BYTES and free_bytes >= VACUUM_MIN_FREE_RATIO * page_count * page_size
|
||||
|
||||
result["vacuum"] = "nicht nötig"
|
||||
if force_vacuum or worthwhile:
|
||||
disk_free = shutil.disk_usage(os.path.dirname(os.path.abspath(path))).free if path else 0
|
||||
# VACUUM schreibt eine Kopie der verbleibenden Daten (plus WAL)
|
||||
if path and disk_free < used_bytes * 1.2:
|
||||
result["vacuum"] = (
|
||||
f"übersprungen: zu wenig Plattenplatz ({disk_free // 2**20} MB frei, "
|
||||
f"~{int(used_bytes * 1.2) // 2**20} MB nötig)"
|
||||
)
|
||||
else:
|
||||
t = time.monotonic()
|
||||
# VACUUM darf nicht in einer Transaktion laufen
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
conn.exec_driver_sql("VACUUM")
|
||||
conn.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
result["vacuum"] = f"ausgeführt ({time.monotonic() - t:.0f}s)"
|
||||
|
||||
result["size_after"] = _file_size(path)
|
||||
result["duration_s"] = round(time.monotonic() - started, 1)
|
||||
logger.info("DB-Wartung: %s", result)
|
||||
last_result = result
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.exception("DB-Wartung fehlgeschlagen")
|
||||
last_result = {"error": str(e), "started_at": datetime.utcnow().isoformat()}
|
||||
raise
|
||||
finally:
|
||||
_lock.release()
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -11,7 +11,8 @@ from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
|
||||
from app.services.encryption import decrypt
|
||||
from app.services.filter_engine import evaluate_conditions, execute_action
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.log_service import cleanup_old_logs, write_log
|
||||
from app.services.log_service import write_log
|
||||
from app.services.maintenance import run_maintenance
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -329,6 +330,13 @@ async def poll_account(account_id: int) -> None:
|
||||
await asyncio.to_thread(_poll_account_sync, account_id)
|
||||
|
||||
|
||||
async def maintenance_job() -> None:
|
||||
try:
|
||||
await asyncio.to_thread(run_maintenance)
|
||||
except Exception:
|
||||
pass # bereits in run_maintenance geloggt
|
||||
|
||||
|
||||
def add_account_job(account: Account) -> None:
|
||||
job_id = f"poll_account_{account.id}"
|
||||
if scheduler.get_job(job_id):
|
||||
@@ -364,12 +372,23 @@ def start_scheduler() -> None:
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Täglicher Cleanup alter Logs
|
||||
# DB-Wartung (Log-Rotation, VACUUM): täglich zu fester Uhrzeit — ein
|
||||
# 24h-Intervall ab Start lief bei häufigen Deploys praktisch nie — und
|
||||
# zusätzlich kurz nach jedem Start.
|
||||
scheduler.add_job(
|
||||
lambda: asyncio.get_event_loop().run_in_executor(None, cleanup_old_logs, 30),
|
||||
"interval",
|
||||
hours=24,
|
||||
id="cleanup_logs",
|
||||
maintenance_job,
|
||||
"cron",
|
||||
hour=settings.maintenance_hour_utc,
|
||||
minute=30,
|
||||
timezone="UTC",
|
||||
id="db_maintenance",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
maintenance_job,
|
||||
"date",
|
||||
run_date=datetime.now() + timedelta(minutes=5),
|
||||
id="db_maintenance_startup",
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
|
||||
+54
-2
@@ -20,9 +20,11 @@
|
||||
<label class="auto-refresh" title="Erste Seite alle 10 Sekunden neu laden">
|
||||
<input type="checkbox" role="switch" id="auto-refresh" onchange="toggleAutoRefresh()"> Live
|
||||
</label>
|
||||
<details class="dropdown head-menu">
|
||||
<details class="dropdown head-menu" ontoggle="if (this.open) loadDbStatus()">
|
||||
<summary>⋯</summary>
|
||||
<ul dir="rtl">
|
||||
<li class="db-status" id="db-status">Datenbank wird geprüft…</li>
|
||||
<li><a href="#" onclick="runMaintenance(); return false;" title="Alte Einträge nach Aufbewahrungsregel löschen und die Datenbankdatei verkleinern">Datenbank aufräumen & verkleinern</a></li>
|
||||
<li><a href="#" onclick="retryFailed(); return false;" title="Löscht den 'verarbeitet'-Marker für Mails mit Abruf-Fehler, damit sie beim nächsten Poll neu versucht werden">Fehlgeschlagene neu einlesen</a></li>
|
||||
<li><a href="#" onclick="clearLogs(); return false;">Log leeren</a></li>
|
||||
</ul>
|
||||
@@ -75,7 +77,9 @@
|
||||
.head-menu { margin:0; }
|
||||
.head-menu summary { padding:0.2rem 0.7rem; height:auto; font-size:1.1rem; line-height:1.2; }
|
||||
.head-menu summary::after { display:none; }
|
||||
.head-menu ul { min-width:230px; font-size:0.9rem; }
|
||||
.head-menu ul { min-width:290px; font-size:0.9rem; }
|
||||
.head-menu li.db-status { direction:ltr; text-align:left; font-size:0.78rem; color:var(--pico-muted-color); line-height:1.5;
|
||||
border-bottom:1px solid var(--app-border, #e3e6ea); padding-bottom:0.5rem; white-space:normal; }
|
||||
|
||||
.toolbar { display:flex; gap:0.5rem; align-items:center; flex-wrap:wrap; margin-bottom:0.4rem; }
|
||||
.search-wrap { position:relative; flex:1 1 360px; }
|
||||
@@ -439,6 +443,54 @@ async function retryFailed() {
|
||||
alert(`${d.unique_uids || 0} Mail(s) neu vorgemerkt. Werden beim nächsten Poll erneut geprüft.`);
|
||||
}
|
||||
|
||||
function fmtBytes(n) {
|
||||
if (n == null) return '?';
|
||||
if (n >= 1e9) return (n / 1e9).toLocaleString('de-DE', {maximumFractionDigits: 2}) + ' GB';
|
||||
return Math.round(n / 1e6).toLocaleString('de-DE') + ' MB';
|
||||
}
|
||||
|
||||
async function loadDbStatus() {
|
||||
const el = document.getElementById('db-status');
|
||||
try {
|
||||
const resp = await fetch('/api/logs/maintenance');
|
||||
const s = await readJson(resp);
|
||||
if (!resp.ok) throw new Error(s.detail || resp.status);
|
||||
const last = s.last_run && s.last_run.started_at
|
||||
? `Letzte Wartung: ${parseUtc(s.last_run.started_at).toLocaleString('de-DE', {dateStyle: 'short', timeStyle: 'short'})}`
|
||||
: 'Wartung läuft täglich automatisch';
|
||||
el.innerHTML = `Datenbank: <b>${fmtBytes(s.file_bytes)}</b> (davon ${fmtBytes(s.free_bytes)} ungenutzt)<br>
|
||||
Log: ${s.log_rows.toLocaleString('de-DE')} Einträge · Aufbewahrung ${s.retention_days} Tage, max. ${s.max_entries.toLocaleString('de-DE')}<br>
|
||||
${esc(last)}`;
|
||||
} catch (e) {
|
||||
el.textContent = 'Status nicht verfügbar: ' + e.message;
|
||||
}
|
||||
}
|
||||
|
||||
async function runMaintenance() {
|
||||
document.querySelector('.head-menu').open = false;
|
||||
if (!confirm('Datenbank jetzt aufräumen und verkleinern?\n\nAlte Log-Einträge werden nach der Aufbewahrungsregel gelöscht und die Datei wird neu geschrieben. Bei großen Datenbanken kann das einige Minuten dauern; die Mail-Verarbeitung wartet solange.')) return;
|
||||
const stats = document.getElementById('log-stats');
|
||||
const prev = stats.textContent;
|
||||
stats.innerHTML = '<span aria-busy="true">Datenbank wird aufgeräumt…</span>';
|
||||
try {
|
||||
const resp = await fetch('/api/logs/maintenance?vacuum=true', {method: 'POST'});
|
||||
const d = await readJson(resp);
|
||||
if (!resp.ok) throw new Error(d.detail || resp.status);
|
||||
const r = d.result;
|
||||
if (r.skipped) { alert(r.skipped); return; }
|
||||
alert(`Fertig in ${r.duration_s} s.\n\n` +
|
||||
`Größe: ${fmtBytes(r.size_before)} → ${fmtBytes(r.size_after)}\n` +
|
||||
`Gelöschte Log-Einträge: ${(r.logs_deleted_age + r.logs_deleted_cap).toLocaleString('de-DE')}\n` +
|
||||
`Verwaiste Marker entfernt: ${r.processed_orphans_deleted.toLocaleString('de-DE')}\n` +
|
||||
`Verkleinern: ${r.vacuum}`);
|
||||
reload();
|
||||
} catch (e) {
|
||||
alert('Wartung fehlgeschlagen: ' + e.message);
|
||||
} finally {
|
||||
if (stats.querySelector('[aria-busy]')) stats.textContent = prev;
|
||||
}
|
||||
}
|
||||
|
||||
function toggleAutoRefresh() {
|
||||
clearInterval(refreshTimer);
|
||||
refreshTimer = null;
|
||||
|
||||
Reference in New Issue
Block a user