From b77b192b56754c8b960a41688f33143fdabf8db0 Mon Sep 17 00:00:00 2001 From: duffyduck Date: Mon, 18 May 2026 20:37:32 +0200 Subject: [PATCH] fix UI 500 + slow backup with millions of processed_mails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix: Starlette 1.0 changed TemplateResponse signature — request is now the first positional argument, not nested inside the context dict. All HTML routes returned 500 (unhashable dict in jinja2 template cache) after the image rebuild picked up the new starlette version. - fix: processed_mails (1.7M rows in production: 1 account × 12 rules × 141k inbox mails) made backup export hit 558 MB / 90s. Moved to opt-in checkbox in the UI alongside the logs option, default off. - yield_per for streaming the processed_mails query when included. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/main.py | 15 +++++++------- app/routers/yaml_sync.py | 12 +++++++++-- app/services/backup_service.py | 37 ++++++++++++++++++++-------------- app/templates/yaml.html | 13 ++++++++++-- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/app/main.py b/app/main.py index 796b790..d5049f6 100644 --- a/app/main.py +++ b/app/main.py @@ -89,31 +89,30 @@ def dashboard(request: Request, db: Session = Depends(get_db)): "last_poll_at": acc.last_poll_at, "filter_rule_count": len(acc.filter_rules), }) - return templates.TemplateResponse("dashboard.html", {"request": request, "accounts": account_list}) + return templates.TemplateResponse(request, "dashboard.html", {"accounts": account_list}) @app.get("/accounts") def accounts_page(request: Request, db: Session = Depends(get_db)): accs = db.query(Account).order_by(Account.name).all() - return templates.TemplateResponse("accounts.html", {"request": request, "accounts": accs}) + return templates.TemplateResponse(request, "accounts.html", {"accounts": accs}) @app.get("/accounts/new") def new_account_page(request: Request): - return templates.TemplateResponse("account_form.html", {"request": request, "account": None}) + return templates.TemplateResponse(request, "account_form.html", {"account": None}) @app.get("/accounts/{account_id}/edit") def edit_account_page(account_id: int, request: Request, db: Session = Depends(get_db)): account = db.get(Account, account_id) - return templates.TemplateResponse("account_form.html", {"request": request, "account": account}) + return templates.TemplateResponse(request, "account_form.html", {"account": account}) @app.get("/filters") def filters_page(request: Request, account_id: int = 0, db: Session = Depends(get_db)): accs = db.query(Account).order_by(Account.name).all() - return templates.TemplateResponse("filters.html", { - "request": request, + return templates.TemplateResponse(request, "filters.html", { "accounts": accs, "selected_account_id": account_id, }) @@ -121,10 +120,10 @@ def filters_page(request: Request, account_id: int = 0, db: Session = Depends(ge @app.get("/yaml") def yaml_page(request: Request): - return templates.TemplateResponse("yaml.html", {"request": request}) + return templates.TemplateResponse(request, "yaml.html", {}) @app.get("/logs") def logs_page(request: Request, db: Session = Depends(get_db)): accs = db.query(Account).order_by(Account.name).all() - return templates.TemplateResponse("logs.html", {"request": request, "accounts": accs}) + return templates.TemplateResponse(request, "logs.html", {"accounts": accs}) diff --git a/app/routers/yaml_sync.py b/app/routers/yaml_sync.py index c52bcc1..ff8c0d7 100644 --- a/app/routers/yaml_sync.py +++ b/app/routers/yaml_sync.py @@ -27,10 +27,18 @@ async def yaml_import(file: UploadFile, db: Session = Depends(get_db)): @router.get("/backup") def backup_export( include_logs: bool = Query(default=False), + include_processed: bool = Query(default=False), db: Session = Depends(get_db), ): - content = export_backup(db, include_logs=include_logs) - suffix = "-mit-logs" if include_logs else "" + content = export_backup( + db, include_logs=include_logs, include_processed=include_processed + ) + suffix_parts = [] + if include_processed: + suffix_parts.append("verarbeitung") + if include_logs: + suffix_parts.append("logs") + suffix = ("-mit-" + "-".join(suffix_parts)) if suffix_parts else "" timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S") filename = f"mailfilter-backup{suffix}-{timestamp}.json" return Response( diff --git a/app/services/backup_service.py b/app/services/backup_service.py index 05b553a..ffc452f 100644 --- a/app/services/backup_service.py +++ b/app/services/backup_service.py @@ -20,7 +20,11 @@ logger = logging.getLogger(__name__) BACKUP_VERSION = 2 -def export_backup(db: Session | None = None, include_logs: bool = False) -> str: +def export_backup( + db: Session | None = None, + include_logs: bool = False, + include_processed: bool = False, +) -> str: close_db = False if db is None: db = SessionLocal() @@ -32,6 +36,7 @@ def export_backup(db: Session | None = None, include_logs: bool = False) -> str: "version": BACKUP_VERSION, "exported_at": datetime.utcnow().isoformat(), "includes_logs": include_logs, + "includes_processed": include_processed, "accounts": [], } @@ -78,20 +83,22 @@ def export_backup(db: Session | None = None, include_logs: bool = False) -> str: "processed_mails": [], } - # Verarbeitete Mails pro Regel - processed = ( - db.query(ProcessedMail) - .filter(ProcessedMail.rule_id == rule.id) - .all() - ) - for pm in processed: - rule_data["processed_mails"].append({ - "folder": pm.folder, - "mail_uid": pm.mail_uid, - "mail_subject": pm.mail_subject, - "mail_from": pm.mail_from, - "processed_at": pm.processed_at.isoformat() if pm.processed_at else None, - }) + # Verarbeitete Mails pro Regel (nur wenn explizit gewünscht — kann sehr + # groß werden: typischerweise pro Regel × pro Mail im Quell-Ordner ein Eintrag) + if include_processed: + processed = ( + db.query(ProcessedMail) + .filter(ProcessedMail.rule_id == rule.id) + .yield_per(2000) + ) + for pm in processed: + rule_data["processed_mails"].append({ + "folder": pm.folder, + "mail_uid": pm.mail_uid, + "mail_subject": pm.mail_subject, + "mail_from": pm.mail_from, + "processed_at": pm.processed_at.isoformat() if pm.processed_at else None, + }) account_data["filter_rules"].append(rule_data) diff --git a/app/templates/yaml.html b/app/templates/yaml.html index 59ff126..e6fd64c 100644 --- a/app/templates/yaml.html +++ b/app/templates/yaml.html @@ -9,6 +9,10 @@

Backup erstellen

Gesamte Konfiguration als JSON-Datei herunterladen.

+