diff --git a/alembic/versions/3bccad0c6646_add_filter_logs_indexes.py b/alembic/versions/3bccad0c6646_add_filter_logs_indexes.py new file mode 100644 index 0000000..56427eb --- /dev/null +++ b/alembic/versions/3bccad0c6646_add_filter_logs_indexes.py @@ -0,0 +1,48 @@ +"""add indexes on filter_logs + +Revision ID: 3bccad0c6646 +Revises: c14c86cbc9c0 +Create Date: 2026-05-18 00:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op + + +revision: str = "3bccad0c6646" +down_revision: Union[str, Sequence[str], None] = "c14c86cbc9c0" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Indizes für schnelle Log-Abfragen bei großen Tabellen. + + Beschleunigt: ORDER BY created_at DESC, Filter nach account_id/level, + und die kombinierte (account_id + ORDER BY created_at)-Abfrage. + """ + op.create_index( + "ix_filter_logs_created_at", + "filter_logs", + ["created_at"], + unique=False, + ) + op.create_index( + "ix_filter_logs_level", + "filter_logs", + ["level"], + unique=False, + ) + op.create_index( + "ix_filter_logs_account_created", + "filter_logs", + ["account_id", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_filter_logs_account_created", table_name="filter_logs") + op.drop_index("ix_filter_logs_level", table_name="filter_logs") + op.drop_index("ix_filter_logs_created_at", table_name="filter_logs") diff --git a/app/database.py b/app/database.py index f13fe28..226990b 100644 --- a/app/database.py +++ b/app/database.py @@ -1,9 +1,24 @@ -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import DeclarativeBase, sessionmaker from app.config import settings engine = create_engine(settings.database_url, connect_args={"check_same_thread": False}) + + +# SQLite: WAL-Modus, damit Reader (z.B. Backup-Export) nicht von gleichzeitigen +# Writern (Scheduler/Log) blockiert werden. busy_timeout sorgt dafür, dass kurze +# Lock-Konflikte automatisch retryen statt sofort zu failen. +if settings.database_url.startswith("sqlite"): + @event.listens_for(engine, "connect") + def _set_sqlite_pragmas(dbapi_connection, _): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA busy_timeout=10000") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.close() + + SessionLocal = sessionmaker(bind=engine) diff --git a/app/models/db_models.py b/app/models/db_models.py index 11e67cb..67f6d6e 100644 --- a/app/models/db_models.py +++ b/app/models/db_models.py @@ -1,7 +1,7 @@ import enum from datetime import datetime -from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, func +from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, func from sqlalchemy.orm import Mapped, mapped_column, relationship from app.database import Base @@ -129,7 +129,7 @@ class FilterLog(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) account_id: Mapped[int | None] = mapped_column(Integer, nullable=True) account_name: Mapped[str] = mapped_column(String(100), default="") - level: Mapped[LogLevel] = mapped_column(Enum(LogLevel), default=LogLevel.INFO) + level: Mapped[LogLevel] = mapped_column(Enum(LogLevel), default=LogLevel.INFO, index=True) message: Mapped[str] = mapped_column(String(1000)) rule_name: Mapped[str | None] = mapped_column(String(200), nullable=True) action_type: Mapped[str | None] = mapped_column(String(50), nullable=True) @@ -138,7 +138,11 @@ class FilterLog(Base): mail_from: Mapped[str | None] = mapped_column(String(255), nullable=True) folder: Mapped[str | None] = mapped_column(String(255), nullable=True) details: Mapped[str | None] = mapped_column(String(2000), nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) + created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), index=True) + + __table_args__ = ( + Index("ix_filter_logs_account_created", "account_id", "created_at"), + ) class ProcessedMail(Base): diff --git a/app/routers/yaml_sync.py b/app/routers/yaml_sync.py index cf0e373..c52bcc1 100644 --- a/app/routers/yaml_sync.py +++ b/app/routers/yaml_sync.py @@ -1,4 +1,6 @@ -from fastapi import APIRouter, Depends, UploadFile +from datetime import datetime + +from fastapi import APIRouter, Depends, Query, UploadFile from fastapi.responses import PlainTextResponse, Response from sqlalchemy.orm import Session @@ -23,12 +25,18 @@ async def yaml_import(file: UploadFile, db: Session = Depends(get_db)): @router.get("/backup") -def backup_export(db: Session = Depends(get_db)): - content = export_backup(db) +def backup_export( + include_logs: bool = Query(default=False), + db: Session = Depends(get_db), +): + content = export_backup(db, include_logs=include_logs) + suffix = "-mit-logs" if include_logs else "" + timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S") + filename = f"mailfilter-backup{suffix}-{timestamp}.json" return Response( content=content, media_type="application/json", - headers={"Content-Disposition": "attachment; filename=mailfilter-backup.json"}, + headers={"Content-Disposition": f"attachment; filename={filename}"}, ) diff --git a/app/services/backup_service.py b/app/services/backup_service.py index eadab08..05b553a 100644 --- a/app/services/backup_service.py +++ b/app/services/backup_service.py @@ -5,14 +5,22 @@ from datetime import datetime from sqlalchemy.orm import Session from app.database import SessionLocal -from app.models.db_models import Account, FilterAction, FilterCondition, FilterRule, ProcessedMail +from app.models.db_models import ( + Account, + FilterAction, + FilterCondition, + FilterLog, + FilterRule, + LogLevel, + ProcessedMail, +) logger = logging.getLogger(__name__) -BACKUP_VERSION = 1 +BACKUP_VERSION = 2 -def export_backup(db: Session | None = None) -> str: +def export_backup(db: Session | None = None, include_logs: bool = False) -> str: close_db = False if db is None: db = SessionLocal() @@ -23,6 +31,7 @@ def export_backup(db: Session | None = None) -> str: data = { "version": BACKUP_VERSION, "exported_at": datetime.utcnow().isoformat(), + "includes_logs": include_logs, "accounts": [], } @@ -88,6 +97,30 @@ def export_backup(db: Session | None = None) -> str: data["accounts"].append(account_data) + if include_logs: + logs_query = ( + db.query(FilterLog) + .order_by(FilterLog.id) + .yield_per(1000) + ) + data["filter_logs"] = [ + { + "account_id": log.account_id, + "account_name": log.account_name, + "level": log.level.value if log.level else "info", + "message": log.message, + "rule_name": log.rule_name, + "action_type": log.action_type, + "mail_uid": log.mail_uid, + "mail_subject": log.mail_subject, + "mail_from": log.mail_from, + "folder": log.folder, + "details": log.details, + "created_at": log.created_at.isoformat() if log.created_at else None, + } + for log in logs_query + ] + return json.dumps(data, ensure_ascii=False, indent=2) finally: if close_db: @@ -111,6 +144,7 @@ def import_backup(json_content: str, db: Session | None = None) -> dict: "accounts_updated": 0, "rules_created": 0, "processed_restored": 0, + "logs_restored": 0, } for acc_data in data["accounts"]: @@ -205,6 +239,42 @@ def import_backup(json_content: str, db: Session | None = None) -> dict: )) stats["processed_restored"] += 1 + # Logs anhängen (falls im Backup enthalten). Vorhandene Logs bleiben unberührt; + # die importierten werden mit neuer ID hinzugefügt. + log_entries = data.get("filter_logs") or [] + if log_entries: + for log_data in log_entries: + try: + level_value = log_data.get("level", "info") + try: + level = LogLevel(level_value) + except ValueError: + level = LogLevel.INFO + created_raw = log_data.get("created_at") + created_at = ( + datetime.fromisoformat(created_raw) if created_raw else None + ) + log = FilterLog( + account_id=log_data.get("account_id"), + account_name=log_data.get("account_name") or "", + level=level, + message=log_data.get("message") or "", + rule_name=log_data.get("rule_name"), + action_type=log_data.get("action_type"), + mail_uid=log_data.get("mail_uid"), + mail_subject=log_data.get("mail_subject"), + mail_from=log_data.get("mail_from"), + folder=log_data.get("folder"), + details=log_data.get("details"), + ) + if created_at is not None: + log.created_at = created_at + db.add(log) + stats["logs_restored"] += 1 + except Exception as e: + logger.warning("Log-Eintrag konnte nicht importiert werden: %s", e) + db.flush() + db.commit() logger.info("Backup-Import abgeschlossen: %s", stats) return stats diff --git a/app/services/filter_engine.py b/app/services/filter_engine.py index a757079..c8873fb 100644 --- a/app/services/filter_engine.py +++ b/app/services/filter_engine.py @@ -178,16 +178,21 @@ def execute_action( mail: MailMessage, action: FilterAction, smtp_config: dict | None = None, + source_folder: str | None = None, ) -> bool: match action.action_type: case ActionType.MOVE: if not action.parameter: logger.error("Kein Zielordner für Move-Aktion angegeben") return False - return imap_client.move_mail(mail.uid, action.parameter) + return imap_client.move_mail( + mail.uid, action.parameter, source_folder=source_folder + ) case ActionType.DELETE: trash = action.parameter or "Trash" - return imap_client.delete_mail(mail.uid, trash) + return imap_client.delete_mail( + mail.uid, trash, source_folder=source_folder + ) case ActionType.MARK_READ: return imap_client.mark_as_read(mail.uid) case ActionType.FORWARD: diff --git a/app/services/imap_client.py b/app/services/imap_client.py index 96fccd3..1da43cd 100644 --- a/app/services/imap_client.py +++ b/app/services/imap_client.py @@ -3,6 +3,7 @@ import email import email.utils import imaplib import logging +import re import smtplib from dataclasses import dataclass, field from datetime import datetime @@ -25,6 +26,13 @@ class MailMessage: raw: Message | None = field(default=None, repr=False) +def _quote_mailbox(name: str) -> str: + """IMAP-konformes Quoting (RFC 3501 quoted-string) für Ordnernamen mit + Leerzeichen oder Sonderzeichen. Backslashes und Anführungszeichen werden escaped.""" + escaped = name.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + def _decode_header_value(value: str | None) -> str: if not value: return "" @@ -160,7 +168,7 @@ class IMAPClient: def create_folder(self, folder_name: str) -> bool: try: - status, _ = self.conn.create(folder_name) + status, _ = self.conn.create(_quote_mailbox(folder_name)) if status == "OK": logger.info("Ordner erstellt: %s", folder_name) return True @@ -170,12 +178,16 @@ class IMAPClient: logger.error("Fehler beim Erstellen von Ordner '%s': %s", folder_name, e) return False + def select_folder(self, folder: str, readonly: bool = False) -> bool: + status, _ = self.conn.select(_quote_mailbox(folder), readonly=readonly) + return status == "OK" + def fetch_unseen(self, folder: str = "INBOX") -> list[MailMessage]: """Legacy: Fetch unseen mails. Use fetch_all_uids + fetch_mail for processed-tracking.""" return self.fetch_mails_by_uids(folder, self.get_all_uids(folder, search="UNSEEN")) def get_all_uids(self, folder: str = "INBOX", search: str = "ALL") -> list[str]: - self.conn.select(folder) + self.conn.select(_quote_mailbox(folder)) status, data = self.conn.uid("SEARCH", None, search) if status != "OK" or not data[0]: return [] @@ -201,7 +213,7 @@ class IMAPClient: def fetch_mails_by_uids(self, folder: str, uids: list[str]) -> list[MailMessage]: if not uids: return [] - self.conn.select(folder) + self.conn.select(_quote_mailbox(folder)) messages = [] for uid in uids: mail = self.fetch_mail(uid) @@ -209,19 +221,120 @@ class IMAPClient: messages.append(mail) return messages - def move_mail(self, uid: str, target_folder: str) -> bool: + def _read_message_id(self, uid: str) -> str | None: + """Liest die Message-ID-Header für die UID im aktuell selektierten Ordner.""" try: - self.conn.uid("COPY", uid, target_folder) + status, data = self.conn.uid( + "FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])" + ) + if status != "OK" or not data: + return None + for part in data: + if isinstance(part, tuple) and len(part) >= 2: + payload = part[1] + text = ( + payload.decode("utf-8", errors="replace") + if isinstance(payload, (bytes, bytearray)) + else str(payload) + ) + match = re.search(r"Message-ID:\s*(<[^>\s]+>)", text, re.IGNORECASE) + if match: + return match.group(1) + return None + except Exception as e: + logger.warning("Konnte Message-ID für UID %s nicht lesen: %s", uid, e) + return None + + def _mail_exists_in_folder(self, folder: str, message_id: str) -> bool: + """Sucht im Ordner nach einer Mail mit der gegebenen Message-ID (read-only).""" + try: + status, _ = self.conn.select(_quote_mailbox(folder), readonly=True) + if status != "OK": + logger.error("Konnte Ordner '%s' für Verifikation nicht öffnen", folder) + return False + # IMAP SEARCH HEADER: Wert als quoted-string übergeben + quoted_id = '"' + message_id.replace("\\", "\\\\").replace('"', '\\"') + '"' + search_status, search_data = self.conn.uid( + "SEARCH", None, "HEADER", "Message-ID", quoted_id + ) + if search_status != "OK": + return False + return bool(search_data and search_data[0]) + except Exception as e: + logger.error("Fehler bei Verifikation in '%s': %s", folder, e) + return False + + def move_mail( + self, + uid: str, + target_folder: str, + source_folder: str | None = None, + ) -> bool: + """Sicheres Verschieben: COPY → Verifikation per Message-ID im Ziel → erst dann + Quelle löschen. Schlägt die Verifikation fehl, bleibt die Mail in der Quelle. + + Wenn source_folder gesetzt ist, wird der Ordner nach der Verifikation wieder + selektiert (da die Verifikation den Zielordner aktiv schaltet).""" + try: + # 1. Message-ID aus der Quelle merken (vor COPY!) — Anker für die Verifikation + message_id = self._read_message_id(uid) + if not message_id: + logger.error( + "Mail %s hat keine Message-ID — sicheres Verschieben nicht möglich, " + "Quelle wird NICHT angetastet (Ziel: '%s')", + uid, target_folder, + ) + return False + + # 2. COPY in den Zielordner + copy_status, copy_data = self.conn.uid( + "COPY", uid, _quote_mailbox(target_folder) + ) + if copy_status != "OK": + logger.error( + "COPY fehlgeschlagen für Mail %s nach '%s': %s %s — " + "Quelle bleibt unangetastet", + uid, target_folder, copy_status, copy_data, + ) + # Quell-Ordner sicherheitshalber neu selektieren + if source_folder: + self.conn.select(_quote_mailbox(source_folder)) + return False + + # 3. Verifikation: Mail im Zielordner per Message-ID suchen + verified = self._mail_exists_in_folder(target_folder, message_id) + + # 4. Quell-Ordner wieder aktivieren (Verifikation hat ihn umgeschaltet) + if source_folder: + self.conn.select(_quote_mailbox(source_folder)) + + if not verified: + logger.error( + "VERIFIKATION FEHLGESCHLAGEN: Mail %s (Message-ID %s) " + "nicht in '%s' gefunden — Quelle wird NICHT gelöscht", + uid, message_id, target_folder, + ) + return False + + # 5. Erst nach erfolgreicher Verifikation aus Quelle löschen self.conn.uid("STORE", uid, "+FLAGS", "(\\Deleted)") self.conn.expunge() - logger.info("Mail %s verschoben nach %s", uid, target_folder) + logger.info( + "Mail %s sicher verschoben nach '%s' (verifiziert: %s)", + uid, target_folder, message_id, + ) return True except Exception as e: logger.error("Fehler beim Verschieben von Mail %s: %s", uid, e) return False - def delete_mail(self, uid: str, trash_folder: str = "Trash") -> bool: - return self.move_mail(uid, trash_folder) + def delete_mail( + self, + uid: str, + trash_folder: str = "Trash", + source_folder: str | None = None, + ) -> bool: + return self.move_mail(uid, trash_folder, source_folder=source_folder) def mark_as_read(self, uid: str) -> bool: try: diff --git a/app/services/scheduler.py b/app/services/scheduler.py index 8e09f8b..7ed78d4 100644 --- a/app/services/scheduler.py +++ b/app/services/scheduler.py @@ -158,7 +158,7 @@ def _poll_account_sync(account_id: int) -> None: try: # Ordner muss ausgewählt sein if not hasattr(client, '_current_folder') or client._current_folder != folder: - client.conn.select(folder) + client.select_folder(folder) client._current_folder = folder mail = client.fetch_mail(uid) except Exception as e: @@ -195,7 +195,10 @@ def _poll_account_sync(account_id: int) -> None: total_matched += 1 # Aktionen ausführen for action in rule.actions: - success = execute_action(client, mail, action, smtp_config) + success = execute_action( + client, mail, action, smtp_config, + source_folder=folder, + ) action_label = action.action_type.value param = action.parameter or "" if param: diff --git a/app/templates/yaml.html b/app/templates/yaml.html index 0cbee80..59ff126 100644 --- a/app/templates/yaml.html +++ b/app/templates/yaml.html @@ -9,7 +9,12 @@

Backup erstellen

Gesamte Konfiguration als JSON-Datei herunterladen.

- + + +
@@ -47,8 +52,40 @@ {% block scripts %}