safe move + sqlite WAL + log indexes + backup with logs

- fix: imap folder names with spaces (RFC3501 quoting in move/create/select) - fix: move only deletes source after COPY + Message-ID verification in target - fix: backup endpoint hung on sqlite write locks — enable WAL + busy_timeout - perf: indexes on filter_logs(created_at, level, account_id+created_at) for fast log queries on millions of rows - feat: optional "logs mit sichern" checkbox in backup export, restore on import - UI: backup download uses fetch+blob with error display instead of location.href Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-18 19:54:32 +02:00
parent 66b32ded36
commit 7e7ec67e58
9 changed files with 331 additions and 27 deletions
@@ -5,14 +5,22 @@ from datetime import datetime
 from sqlalchemy.orm import Session

 from app.database import SessionLocal
-from app.models.db_models import Account, FilterAction, FilterCondition, FilterRule, ProcessedMail
+from app.models.db_models import (
+    Account,
+    FilterAction,
+    FilterCondition,
+    FilterLog,
+    FilterRule,
+    LogLevel,
+    ProcessedMail,
+)

 logger = logging.getLogger(__name__)

-BACKUP_VERSION = 1
+BACKUP_VERSION = 2


-def export_backup(db: Session | None = None) -> str:
+def export_backup(db: Session | None = None, include_logs: bool = False) -> str:
    close_db = False
    if db is None:
        db = SessionLocal()
@@ -23,6 +31,7 @@ def export_backup(db: Session | None = None) -> str:
        data = {
            "version": BACKUP_VERSION,
            "exported_at": datetime.utcnow().isoformat(),
+            "includes_logs": include_logs,
            "accounts": [],
        }

@@ -88,6 +97,30 @@ def export_backup(db: Session | None = None) -> str:

            data["accounts"].append(account_data)

+        if include_logs:
+            logs_query = (
+                db.query(FilterLog)
+                .order_by(FilterLog.id)
+                .yield_per(1000)
+            )
+            data["filter_logs"] = [
+                {
+                    "account_id": log.account_id,
+                    "account_name": log.account_name,
+                    "level": log.level.value if log.level else "info",
+                    "message": log.message,
+                    "rule_name": log.rule_name,
+                    "action_type": log.action_type,
+                    "mail_uid": log.mail_uid,
+                    "mail_subject": log.mail_subject,
+                    "mail_from": log.mail_from,
+                    "folder": log.folder,
+                    "details": log.details,
+                    "created_at": log.created_at.isoformat() if log.created_at else None,
+                }
+                for log in logs_query
+            ]
+
        return json.dumps(data, ensure_ascii=False, indent=2)
    finally:
        if close_db:
@@ -111,6 +144,7 @@ def import_backup(json_content: str, db: Session | None = None) -> dict:
            "accounts_updated": 0,
            "rules_created": 0,
            "processed_restored": 0,
+            "logs_restored": 0,
        }

        for acc_data in data["accounts"]:
@@ -205,6 +239,42 @@ def import_backup(json_content: str, db: Session | None = None) -> dict:
                    ))
                    stats["processed_restored"] += 1

+        # Logs anhängen (falls im Backup enthalten). Vorhandene Logs bleiben unberührt;
+        # die importierten werden mit neuer ID hinzugefügt.
+        log_entries = data.get("filter_logs") or []
+        if log_entries:
+            for log_data in log_entries:
+                try:
+                    level_value = log_data.get("level", "info")
+                    try:
+                        level = LogLevel(level_value)
+                    except ValueError:
+                        level = LogLevel.INFO
+                    created_raw = log_data.get("created_at")
+                    created_at = (
+                        datetime.fromisoformat(created_raw) if created_raw else None
+                    )
+                    log = FilterLog(
+                        account_id=log_data.get("account_id"),
+                        account_name=log_data.get("account_name") or "",
+                        level=level,
+                        message=log_data.get("message") or "",
+                        rule_name=log_data.get("rule_name"),
+                        action_type=log_data.get("action_type"),
+                        mail_uid=log_data.get("mail_uid"),
+                        mail_subject=log_data.get("mail_subject"),
+                        mail_from=log_data.get("mail_from"),
+                        folder=log_data.get("folder"),
+                        details=log_data.get("details"),
+                    )
+                    if created_at is not None:
+                        log.created_at = created_at
+                    db.add(log)
+                    stats["logs_restored"] += 1
+                except Exception as e:
+                    logger.warning("Log-Eintrag konnte nicht importiert werden: %s", e)
+            db.flush()
+
        db.commit()
        logger.info("Backup-Import abgeschlossen: %s", stats)
        return stats
@@ -178,16 +178,21 @@ def execute_action(
    mail: MailMessage,
    action: FilterAction,
    smtp_config: dict | None = None,
+    source_folder: str | None = None,
 ) -> bool:
    match action.action_type:
        case ActionType.MOVE:
            if not action.parameter:
                logger.error("Kein Zielordner für Move-Aktion angegeben")
                return False
-            return imap_client.move_mail(mail.uid, action.parameter)
+            return imap_client.move_mail(
+                mail.uid, action.parameter, source_folder=source_folder
+            )
        case ActionType.DELETE:
            trash = action.parameter or "Trash"
-            return imap_client.delete_mail(mail.uid, trash)
+            return imap_client.delete_mail(
+                mail.uid, trash, source_folder=source_folder
+            )
        case ActionType.MARK_READ:
            return imap_client.mark_as_read(mail.uid)
        case ActionType.FORWARD:
@@ -3,6 +3,7 @@ import email
 import email.utils
 import imaplib
 import logging
+import re
 import smtplib
 from dataclasses import dataclass, field
 from datetime import datetime
@@ -25,6 +26,13 @@ class MailMessage:
    raw: Message | None = field(default=None, repr=False)


+def _quote_mailbox(name: str) -> str:
+    """IMAP-konformes Quoting (RFC 3501 quoted-string) für Ordnernamen mit
+    Leerzeichen oder Sonderzeichen. Backslashes und Anführungszeichen werden escaped."""
+    escaped = name.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
 def _decode_header_value(value: str | None) -> str:
    if not value:
        return ""
@@ -160,7 +168,7 @@ class IMAPClient:

    def create_folder(self, folder_name: str) -> bool:
        try:
-            status, _ = self.conn.create(folder_name)
+            status, _ = self.conn.create(_quote_mailbox(folder_name))
            if status == "OK":
                logger.info("Ordner erstellt: %s", folder_name)
                return True
@@ -170,12 +178,16 @@ class IMAPClient:
            logger.error("Fehler beim Erstellen von Ordner '%s': %s", folder_name, e)
            return False

+    def select_folder(self, folder: str, readonly: bool = False) -> bool:
+        status, _ = self.conn.select(_quote_mailbox(folder), readonly=readonly)
+        return status == "OK"
+
    def fetch_unseen(self, folder: str = "INBOX") -> list[MailMessage]:
        """Legacy: Fetch unseen mails. Use fetch_all_uids + fetch_mail for processed-tracking."""
        return self.fetch_mails_by_uids(folder, self.get_all_uids(folder, search="UNSEEN"))

    def get_all_uids(self, folder: str = "INBOX", search: str = "ALL") -> list[str]:
-        self.conn.select(folder)
+        self.conn.select(_quote_mailbox(folder))
        status, data = self.conn.uid("SEARCH", None, search)
        if status != "OK" or not data[0]:
            return []
@@ -201,7 +213,7 @@ class IMAPClient:
    def fetch_mails_by_uids(self, folder: str, uids: list[str]) -> list[MailMessage]:
        if not uids:
            return []
-        self.conn.select(folder)
+        self.conn.select(_quote_mailbox(folder))
        messages = []
        for uid in uids:
            mail = self.fetch_mail(uid)
@@ -209,19 +221,120 @@ class IMAPClient:
                messages.append(mail)
        return messages

-    def move_mail(self, uid: str, target_folder: str) -> bool:
+    def _read_message_id(self, uid: str) -> str | None:
+        """Liest die Message-ID-Header für die UID im aktuell selektierten Ordner."""
        try:
-            self.conn.uid("COPY", uid, target_folder)
+            status, data = self.conn.uid(
+                "FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])"
+            )
+            if status != "OK" or not data:
+                return None
+            for part in data:
+                if isinstance(part, tuple) and len(part) >= 2:
+                    payload = part[1]
+                    text = (
+                        payload.decode("utf-8", errors="replace")
+                        if isinstance(payload, (bytes, bytearray))
+                        else str(payload)
+                    )
+                    match = re.search(r"Message-ID:\s*(<[^>\s]+>)", text, re.IGNORECASE)
+                    if match:
+                        return match.group(1)
+            return None
+        except Exception as e:
+            logger.warning("Konnte Message-ID für UID %s nicht lesen: %s", uid, e)
+            return None
+
+    def _mail_exists_in_folder(self, folder: str, message_id: str) -> bool:
+        """Sucht im Ordner nach einer Mail mit der gegebenen Message-ID (read-only)."""
+        try:
+            status, _ = self.conn.select(_quote_mailbox(folder), readonly=True)
+            if status != "OK":
+                logger.error("Konnte Ordner '%s' für Verifikation nicht öffnen", folder)
+                return False
+            # IMAP SEARCH HEADER: Wert als quoted-string übergeben
+            quoted_id = '"' + message_id.replace("\\", "\\\\").replace('"', '\\"') + '"'
+            search_status, search_data = self.conn.uid(
+                "SEARCH", None, "HEADER", "Message-ID", quoted_id
+            )
+            if search_status != "OK":
+                return False
+            return bool(search_data and search_data[0])
+        except Exception as e:
+            logger.error("Fehler bei Verifikation in '%s': %s", folder, e)
+            return False
+
+    def move_mail(
+        self,
+        uid: str,
+        target_folder: str,
+        source_folder: str | None = None,
+    ) -> bool:
+        """Sicheres Verschieben: COPY → Verifikation per Message-ID im Ziel → erst dann
+        Quelle löschen. Schlägt die Verifikation fehl, bleibt die Mail in der Quelle.
+
+        Wenn source_folder gesetzt ist, wird der Ordner nach der Verifikation wieder
+        selektiert (da die Verifikation den Zielordner aktiv schaltet)."""
+        try:
+            # 1. Message-ID aus der Quelle merken (vor COPY!) — Anker für die Verifikation
+            message_id = self._read_message_id(uid)
+            if not message_id:
+                logger.error(
+                    "Mail %s hat keine Message-ID — sicheres Verschieben nicht möglich, "
+                    "Quelle wird NICHT angetastet (Ziel: '%s')",
+                    uid, target_folder,
+                )
+                return False
+
+            # 2. COPY in den Zielordner
+            copy_status, copy_data = self.conn.uid(
+                "COPY", uid, _quote_mailbox(target_folder)
+            )
+            if copy_status != "OK":
+                logger.error(
+                    "COPY fehlgeschlagen für Mail %s nach '%s': %s %s — "
+                    "Quelle bleibt unangetastet",
+                    uid, target_folder, copy_status, copy_data,
+                )
+                # Quell-Ordner sicherheitshalber neu selektieren
+                if source_folder:
+                    self.conn.select(_quote_mailbox(source_folder))
+                return False
+
+            # 3. Verifikation: Mail im Zielordner per Message-ID suchen
+            verified = self._mail_exists_in_folder(target_folder, message_id)
+
+            # 4. Quell-Ordner wieder aktivieren (Verifikation hat ihn umgeschaltet)
+            if source_folder:
+                self.conn.select(_quote_mailbox(source_folder))
+
+            if not verified:
+                logger.error(
+                    "VERIFIKATION FEHLGESCHLAGEN: Mail %s (Message-ID %s) "
+                    "nicht in '%s' gefunden — Quelle wird NICHT gelöscht",
+                    uid, message_id, target_folder,
+                )
+                return False
+
+            # 5. Erst nach erfolgreicher Verifikation aus Quelle löschen
            self.conn.uid("STORE", uid, "+FLAGS", "(\\Deleted)")
            self.conn.expunge()
-            logger.info("Mail %s verschoben nach %s", uid, target_folder)
+            logger.info(
+                "Mail %s sicher verschoben nach '%s' (verifiziert: %s)",
+                uid, target_folder, message_id,
+            )
            return True
        except Exception as e:
            logger.error("Fehler beim Verschieben von Mail %s: %s", uid, e)
            return False

-    def delete_mail(self, uid: str, trash_folder: str = "Trash") -> bool:
-        return self.move_mail(uid, trash_folder)
+    def delete_mail(
+        self,
+        uid: str,
+        trash_folder: str = "Trash",
+        source_folder: str | None = None,
+    ) -> bool:
+        return self.move_mail(uid, trash_folder, source_folder=source_folder)

    def mark_as_read(self, uid: str) -> bool:
        try:
@@ -158,7 +158,7 @@ def _poll_account_sync(account_id: int) -> None:
                        try:
                            # Ordner muss ausgewählt sein
                            if not hasattr(client, '_current_folder') or client._current_folder != folder:
-                                client.conn.select(folder)
+                                client.select_folder(folder)
                                client._current_folder = folder
                            mail = client.fetch_mail(uid)
                        except Exception as e:
@@ -195,7 +195,10 @@ def _poll_account_sync(account_id: int) -> None:
                        total_matched += 1
                        # Aktionen ausführen
                        for action in rule.actions:
-                            success = execute_action(client, mail, action, smtp_config)
+                            success = execute_action(
+                                client, mail, action, smtp_config,
+                                source_folder=folder,
+                            )
                            action_label = action.action_type.value
                            param = action.parameter or ""
                            if param: