automatic DB maintenance: log rotation, orphan cleanup, VACUUM
The existing daily cleanup never effectively ran: it was an interval job counted from container start (reset on every deploy), it called asyncio.get_event_loop() from a worker thread, and even when deleting, SQLite never returns freed pages to the filesystem. Live DB: 4.1 GB, of which 2.6 GB log and 0.6 GB unused pages. New maintenance service, scheduled daily at MAINTENANCE_HOUR_UTC:30 and once 5 minutes after every start: - delete log entries older than LOG_RETENTION_DAYS (30) and keep at most LOG_MAX_ENTRIES (200k) newest ones - delete processed_mails markers of deleted rules/accounts (FKs are not enforced by SQLite) - VACUUM when >25% of the file is unused, only if free disk space covers the copy; never runs twice in parallel GET/POST /api/logs/maintenance expose status and a manual run; the log menu shows DB size, entry count and last run, with "Datenbank aufräumen & verkleinern". Removed the dead cleanup_old_logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -11,7 +11,8 @@ from app.models.db_models import Account, FilterRule, LogLevel, ProcessedMail
|
||||
from app.services.encryption import decrypt
|
||||
from app.services.filter_engine import evaluate_conditions, execute_action
|
||||
from app.services.imap_client import IMAPClient
|
||||
from app.services.log_service import cleanup_old_logs, write_log
|
||||
from app.services.log_service import write_log
|
||||
from app.services.maintenance import run_maintenance
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -329,6 +330,13 @@ async def poll_account(account_id: int) -> None:
|
||||
await asyncio.to_thread(_poll_account_sync, account_id)
|
||||
|
||||
|
||||
async def maintenance_job() -> None:
|
||||
try:
|
||||
await asyncio.to_thread(run_maintenance)
|
||||
except Exception:
|
||||
pass # bereits in run_maintenance geloggt
|
||||
|
||||
|
||||
def add_account_job(account: Account) -> None:
|
||||
job_id = f"poll_account_{account.id}"
|
||||
if scheduler.get_job(job_id):
|
||||
@@ -364,12 +372,23 @@ def start_scheduler() -> None:
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Täglicher Cleanup alter Logs
|
||||
# DB-Wartung (Log-Rotation, VACUUM): täglich zu fester Uhrzeit — ein
|
||||
# 24h-Intervall ab Start lief bei häufigen Deploys praktisch nie — und
|
||||
# zusätzlich kurz nach jedem Start.
|
||||
scheduler.add_job(
|
||||
lambda: asyncio.get_event_loop().run_in_executor(None, cleanup_old_logs, 30),
|
||||
"interval",
|
||||
hours=24,
|
||||
id="cleanup_logs",
|
||||
maintenance_job,
|
||||
"cron",
|
||||
hour=settings.maintenance_hour_utc,
|
||||
minute=30,
|
||||
timezone="UTC",
|
||||
id="db_maintenance",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
maintenance_job,
|
||||
"date",
|
||||
run_date=datetime.now() + timedelta(minutes=5),
|
||||
id="db_maintenance_startup",
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user