Hacker-Software/imap-mail-filter-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fast log clear + robust API error handling

Browse Source 
- fix: "Log leeren" without account filter used a plain DELETE over
  millions of rows. That held the sqlite write lock long enough that
  the scheduler's own log writes started failing with
  "database is locked", and the request itself eventually 500'd back
  as HTML "Internal Server Error" — which the frontend then tried to
  JSON.parse, producing the popup the user reported. Now uses
  DROP+CREATE for full clears (near-instant) and wal_checkpoint to
  reclaim space; per-account clear still uses a normal DELETE.
- Raise sqlite busy_timeout from 10s to 60s so a legitimately slow
  writer no longer starves smaller ones.
- retry-failed: single bulk DELETE with mail_uid IN (...) per account
  instead of one query per uid.
- Both endpoints now return proper HTTPException(500, ...) with detail
  instead of letting the exception bubble as plain text.
- Frontend: parseResponse() reads text first, then tries JSON. Both
  clearLogs() and retryFailed() surface the actual server message
  instead of crashing on non-JSON responses.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
duffyduck
2026-09-01 11:52:26 +02:00
co-authored by Claude Opus 4.7
parent e1f3231365
commit e73d90747b
4 changed files with 91 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+1 -1
View File
@@ -14,7 +14,7 @@ if settings.database_url.startswith("sqlite"):
def _set_sqlite_pragmas(dbapi_connection, _):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=10000")
cursor.execute("PRAGMA busy_timeout=60000")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.close()
app/routers/filters.py
+31 -21
View File
@@ -38,29 +38,39 @@ def retry_failed_fetches(account_id: int | None = None, db: Session = Depends(ge
"""Löscht ProcessedMail-Einträge für UIDs, für die es einen ERROR-Log
mit 'Fehler beim Abrufen' gibt — damit der Scheduler sie beim nächsten
Poll neu versucht (z.B. nach einem Codec-Fix)."""
log_q = db.query(FilterLog.mail_uid, FilterLog.account_id).filter(
FilterLog.level == LogLevel.ERROR,
FilterLog.message.like("Fehler beim Abrufen%"),
FilterLog.mail_uid.isnot(None),
)
if account_id is not None:
log_q = log_q.filter(FilterLog.account_id == account_id)
try:
log_q = db.query(FilterLog.mail_uid, FilterLog.account_id).filter(
FilterLog.level == LogLevel.ERROR,
FilterLog.message.like("Fehler beim Abrufen%"),
FilterLog.mail_uid.isnot(None),
)
if account_id is not None:
log_q = log_q.filter(FilterLog.account_id == account_id)
pairs = {(row[0], row[1]) for row in log_q.all()}
if not pairs:
return {"reset": 0, "unique_uids": 0}
pairs = log_q.distinct().all()
if not pairs:
return {"reset": 0, "unique_uids": 0}
total = 0
for uid, acc_id in pairs:
q = db.query(ProcessedMail).filter(ProcessedMail.mail_uid == uid)
if acc_id is not None:
q = q.filter(ProcessedMail.account_id == acc_id)
total += q.delete(synchronize_session=False)
db.commit()
logger.info(
"Retry-Failed: %d ProcessedMail-Einträge für %d UIDs entfernt", total, len(pairs)
)
return {"reset": total, "unique_uids": len(pairs)}
# Bulk-Delete pro Account: alle UIDs in einem Query
by_account: dict[int | None, set[str]] = {}
for uid, acc_id in pairs:
by_account.setdefault(acc_id, set()).add(uid)
total = 0
for acc_id, uids in by_account.items():
q = db.query(ProcessedMail).filter(ProcessedMail.mail_uid.in_(list(uids)))
if acc_id is not None:
q = q.filter(ProcessedMail.account_id == acc_id)
total += q.delete(synchronize_session=False)
db.commit()
unique_uids = sum(len(v) for v in by_account.values())
logger.info(
"Retry-Failed: %d ProcessedMail-Einträge für %d UIDs entfernt", total, unique_uids
)
return {"reset": total, "unique_uids": unique_uids}
except Exception as e:
logger.exception("retry-failed fehlgeschlagen")
raise HTTPException(500, f"Fehler bei retry-failed: {e}")
@router.get("/account/{account_id}", response_model=list[FilterRuleResponse])
app/routers/logs.py
+31 -8
View File
@@ -1,9 +1,14 @@
from fastapi import APIRouter, Depends, Query
import logging
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.database import get_db
from app.database import engine, get_db
from app.models.db_models import FilterLog
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/logs", tags=["logs"])
@@ -78,9 +83,27 @@ def get_logs(
@router.delete("/")
def clear_logs(account_id: int | None = None, db: Session = Depends(get_db)):
query = db.query(FilterLog)
if account_id:
query = query.filter(FilterLog.account_id == account_id)
count = query.delete()
db.commit()
return {"deleted": count}
"""Löscht Logs. Ohne account_id: schnelles Drop+Recreate der Tabelle
(sekundenschnell auch bei Millionen Zeilen). Mit account_id: normales DELETE."""
try:
if account_id is None:
# Vollständiges Leeren: DROP+CREATE statt DELETE.
# DELETE FROM filter_logs würde jede Zeile einzeln ins WAL schreiben,
# bei Millionen Einträgen dauert das ewig und blockiert die DB.
db.close() # Sitzung freigeben, sonst holds sie noch Locks
row_count = 0
with engine.begin() as conn:
row_count = conn.execute(text("SELECT COUNT(*) FROM filter_logs")).scalar() or 0
FilterLog.__table__.drop(conn, checkfirst=True)
FilterLog.__table__.create(conn, checkfirst=True)
# PRAGMA wal_checkpoint(TRUNCATE) räumt die WAL-Datei auf
conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
logger.info("Log vollständig geleert (%d Einträge, Tabelle neu angelegt)", row_count)
return {"deleted": row_count}
count = db.query(FilterLog).filter(FilterLog.account_id == account_id).delete()
db.commit()
return {"deleted": count}
except Exception as e:
logger.exception("clear_logs fehlgeschlagen")
raise HTTPException(500, f"Fehler beim Leeren des Logs: {e}")
app/templates/logs.html
+28 -5
View File
@@ -228,13 +228,30 @@ function renderPaging(total, offset) {
paging.innerHTML = html;
}
async function parseResponse(resp) {
// Liest die Response robust: JSON wenn möglich, sonst als Text.
const text = await resp.text();
try { return {ok: resp.ok, status: resp.status, data: JSON.parse(text)}; }
catch { return {ok: resp.ok, status: resp.status, data: null, text}; }
}
async function clearLogs() {
if (!confirm('Log wirklich leeren?')) return;
const accountId = document.getElementById('log-account').value;
let url = '/api/logs/';
if (accountId) url += `?account_id=${accountId}`;
await fetch(url, {method: 'DELETE'});
loadLogs();
try {
const resp = await fetch(url, {method: 'DELETE'});
const r = await parseResponse(resp);
if (!r.ok) {
const msg = r.data?.detail || r.text || `HTTP ${r.status}`;
alert('Fehler beim Leeren: ' + msg);
return;
}
loadLogs();
} catch(e) {
alert('Netzwerk-Fehler: ' + e.message);
}
}
async function retryFailed() {
@@ -244,10 +261,16 @@ async function retryFailed() {
if (accountId) url += `?account_id=${accountId}`;
try {
const resp = await fetch(url, {method: 'POST'});
const data = await resp.json();
alert(`${data.unique_uids || 0} UID(s) neu vorgemerkt (${data.reset || 0} ProcessedMail-Einträge gelöscht). Werden beim nächsten Poll neu versucht.`);
const r = await parseResponse(resp);
if (!r.ok) {
const msg = r.data?.detail || r.text || `HTTP ${r.status}`;
alert('Fehler bei retry-failed: ' + msg);
return;
}
const d = r.data || {};
alert(`${d.unique_uids || 0} UID(s) neu vorgemerkt (${d.reset || 0} ProcessedMail-Einträge gelöscht). Werden beim nächsten Poll neu versucht.`);
} catch(e) {
alert('Fehler: ' + e.message);
alert('Netzwerk-Fehler: ' + e.message);
}
}