Hacker-Software/imap-mail-filter-service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

safe move + sqlite WAL + log indexes + backup with logs

Browse Source 
- fix: imap folder names with spaces (RFC3501 quoting in move/create/select)
- fix: move only deletes source after COPY + Message-ID verification in target
- fix: backup endpoint hung on sqlite write locks — enable WAL + busy_timeout
- perf: indexes on filter_logs(created_at, level, account_id+created_at) for
  fast log queries on millions of rows
- feat: optional "logs mit sichern" checkbox in backup export, restore on import
- UI: backup download uses fetch+blob with error display instead of location.href

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
duffyduck
2026-05-18 19:54:32 +02:00
parent 66b32ded36
commit 7e7ec67e58
9 changed files with 331 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+16 -1
View File
@@ -1,9 +1,24 @@
from sqlalchemy import create_engine
from sqlalchemy import create_engine, event
from sqlalchemy.orm import DeclarativeBase, sessionmaker
from app.config import settings
engine = create_engine(settings.database_url, connect_args={"check_same_thread": False})
# SQLite: WAL-Modus, damit Reader (z.B. Backup-Export) nicht von gleichzeitigen
# Writern (Scheduler/Log) blockiert werden. busy_timeout sorgt dafür, dass kurze
# Lock-Konflikte automatisch retryen statt sofort zu failen.
if settings.database_url.startswith("sqlite"):
@event.listens_for(engine, "connect")
def _set_sqlite_pragmas(dbapi_connection, _):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=10000")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.close()
SessionLocal = sessionmaker(bind=engine)
app/models/db_models.py
+7 -3
View File
@@ -1,7 +1,7 @@
import enum
from datetime import datetime
from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, func
from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
@@ -129,7 +129,7 @@ class FilterLog(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
account_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
account_name: Mapped[str] = mapped_column(String(100), default="")
level: Mapped[LogLevel] = mapped_column(Enum(LogLevel), default=LogLevel.INFO)
level: Mapped[LogLevel] = mapped_column(Enum(LogLevel), default=LogLevel.INFO, index=True)
message: Mapped[str] = mapped_column(String(1000))
rule_name: Mapped[str | None] = mapped_column(String(200), nullable=True)
action_type: Mapped[str | None] = mapped_column(String(50), nullable=True)
@@ -138,7 +138,11 @@ class FilterLog(Base):
mail_from: Mapped[str | None] = mapped_column(String(255), nullable=True)
folder: Mapped[str | None] = mapped_column(String(255), nullable=True)
details: Mapped[str | None] = mapped_column(String(2000), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), index=True)
__table_args__ = (
Index("ix_filter_logs_account_created", "account_id", "created_at"),
)
class ProcessedMail(Base):
app/routers/yaml_sync.py
+12 -4
View File
@@ -1,4 +1,6 @@
from fastapi import APIRouter, Depends, UploadFile
from datetime import datetime
from fastapi import APIRouter, Depends, Query, UploadFile
from fastapi.responses import PlainTextResponse, Response
from sqlalchemy.orm import Session
@@ -23,12 +25,18 @@ async def yaml_import(file: UploadFile, db: Session = Depends(get_db)):
@router.get("/backup")
def backup_export(db: Session = Depends(get_db)):
content = export_backup(db)
def backup_export(
include_logs: bool = Query(default=False),
db: Session = Depends(get_db),
):
content = export_backup(db, include_logs=include_logs)
suffix = "-mit-logs" if include_logs else ""
timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
filename = f"mailfilter-backup{suffix}-{timestamp}.json"
return Response(
content=content,
media_type="application/json",
headers={"Content-Disposition": "attachment; filename=mailfilter-backup.json"},
headers={"Content-Disposition": f"attachment; filename={filename}"},
)
app/services/backup_service.py
+73 -3
View File
@@ -5,14 +5,22 @@ from datetime import datetime
from sqlalchemy.orm import Session
from app.database import SessionLocal
from app.models.db_models import Account, FilterAction, FilterCondition, FilterRule, ProcessedMail
from app.models.db_models import (
Account,
FilterAction,
FilterCondition,
FilterLog,
FilterRule,
LogLevel,
ProcessedMail,
)
logger = logging.getLogger(__name__)
BACKUP_VERSION = 1
BACKUP_VERSION = 2
def export_backup(db: Session | None = None) -> str:
def export_backup(db: Session | None = None, include_logs: bool = False) -> str:
close_db = False
if db is None:
db = SessionLocal()
@@ -23,6 +31,7 @@ def export_backup(db: Session | None = None) -> str:
data = {
"version": BACKUP_VERSION,
"exported_at": datetime.utcnow().isoformat(),
"includes_logs": include_logs,
"accounts": [],
}
@@ -88,6 +97,30 @@ def export_backup(db: Session | None = None) -> str:
data["accounts"].append(account_data)
if include_logs:
logs_query = (
db.query(FilterLog)
.order_by(FilterLog.id)
.yield_per(1000)
)
data["filter_logs"] = [
{
"account_id": log.account_id,
"account_name": log.account_name,
"level": log.level.value if log.level else "info",
"message": log.message,
"rule_name": log.rule_name,
"action_type": log.action_type,
"mail_uid": log.mail_uid,
"mail_subject": log.mail_subject,
"mail_from": log.mail_from,
"folder": log.folder,
"details": log.details,
"created_at": log.created_at.isoformat() if log.created_at else None,
}
for log in logs_query
]
return json.dumps(data, ensure_ascii=False, indent=2)
finally:
if close_db:
@@ -111,6 +144,7 @@ def import_backup(json_content: str, db: Session | None = None) -> dict:
"accounts_updated": 0,
"rules_created": 0,
"processed_restored": 0,
"logs_restored": 0,
}
for acc_data in data["accounts"]:
@@ -205,6 +239,42 @@ def import_backup(json_content: str, db: Session | None = None) -> dict:
))
stats["processed_restored"] += 1
# Logs anhängen (falls im Backup enthalten). Vorhandene Logs bleiben unberührt;
# die importierten werden mit neuer ID hinzugefügt.
log_entries = data.get("filter_logs") or []
if log_entries:
for log_data in log_entries:
try:
level_value = log_data.get("level", "info")
try:
level = LogLevel(level_value)
except ValueError:
level = LogLevel.INFO
created_raw = log_data.get("created_at")
created_at = (
datetime.fromisoformat(created_raw) if created_raw else None
)
log = FilterLog(
account_id=log_data.get("account_id"),
account_name=log_data.get("account_name") or "",
level=level,
message=log_data.get("message") or "",
rule_name=log_data.get("rule_name"),
action_type=log_data.get("action_type"),
mail_uid=log_data.get("mail_uid"),
mail_subject=log_data.get("mail_subject"),
mail_from=log_data.get("mail_from"),
folder=log_data.get("folder"),
details=log_data.get("details"),
)
if created_at is not None:
log.created_at = created_at
db.add(log)
stats["logs_restored"] += 1
except Exception as e:
logger.warning("Log-Eintrag konnte nicht importiert werden: %s", e)
db.flush()
db.commit()
logger.info("Backup-Import abgeschlossen: %s", stats)
return stats
app/services/filter_engine.py
+7 -2
View File
@@ -178,16 +178,21 @@ def execute_action(
mail: MailMessage,
action: FilterAction,
smtp_config: dict | None = None,
source_folder: str | None = None,
) -> bool:
match action.action_type:
case ActionType.MOVE:
if not action.parameter:
logger.error("Kein Zielordner für Move-Aktion angegeben")
return False
return imap_client.move_mail(mail.uid, action.parameter)
return imap_client.move_mail(
mail.uid, action.parameter, source_folder=source_folder
)
case ActionType.DELETE:
trash = action.parameter or "Trash"
return imap_client.delete_mail(mail.uid, trash)
return imap_client.delete_mail(
mail.uid, trash, source_folder=source_folder
)
case ActionType.MARK_READ:
return imap_client.mark_as_read(mail.uid)
case ActionType.FORWARD:
app/services/imap_client.py
+121 -8
View File
@@ -3,6 +3,7 @@ import email
import email.utils
import imaplib
import logging
import re
import smtplib
from dataclasses import dataclass, field
from datetime import datetime
@@ -25,6 +26,13 @@ class MailMessage:
raw: Message | None = field(default=None, repr=False)
def _quote_mailbox(name: str) -> str:
"""IMAP-konformes Quoting (RFC 3501 quoted-string) für Ordnernamen mit
Leerzeichen oder Sonderzeichen. Backslashes und Anführungszeichen werden escaped."""
escaped = name.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _decode_header_value(value: str | None) -> str:
if not value:
return ""
@@ -160,7 +168,7 @@ class IMAPClient:
def create_folder(self, folder_name: str) -> bool:
try:
status, _ = self.conn.create(folder_name)
status, _ = self.conn.create(_quote_mailbox(folder_name))
if status == "OK":
logger.info("Ordner erstellt: %s", folder_name)
return True
@@ -170,12 +178,16 @@ class IMAPClient:
logger.error("Fehler beim Erstellen von Ordner '%s': %s", folder_name, e)
return False
def select_folder(self, folder: str, readonly: bool = False) -> bool:
status, _ = self.conn.select(_quote_mailbox(folder), readonly=readonly)
return status == "OK"
def fetch_unseen(self, folder: str = "INBOX") -> list[MailMessage]:
"""Legacy: Fetch unseen mails. Use fetch_all_uids + fetch_mail for processed-tracking."""
return self.fetch_mails_by_uids(folder, self.get_all_uids(folder, search="UNSEEN"))
def get_all_uids(self, folder: str = "INBOX", search: str = "ALL") -> list[str]:
self.conn.select(folder)
self.conn.select(_quote_mailbox(folder))
status, data = self.conn.uid("SEARCH", None, search)
if status != "OK" or not data[0]:
return []
@@ -201,7 +213,7 @@ class IMAPClient:
def fetch_mails_by_uids(self, folder: str, uids: list[str]) -> list[MailMessage]:
if not uids:
return []
self.conn.select(folder)
self.conn.select(_quote_mailbox(folder))
messages = []
for uid in uids:
mail = self.fetch_mail(uid)
@@ -209,19 +221,120 @@ class IMAPClient:
messages.append(mail)
return messages
def move_mail(self, uid: str, target_folder: str) -> bool:
def _read_message_id(self, uid: str) -> str | None:
"""Liest die Message-ID-Header für die UID im aktuell selektierten Ordner."""
try:
self.conn.uid("COPY", uid, target_folder)
status, data = self.conn.uid(
"FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])"
)
if status != "OK" or not data:
return None
for part in data:
if isinstance(part, tuple) and len(part) >= 2:
payload = part[1]
text = (
payload.decode("utf-8", errors="replace")
if isinstance(payload, (bytes, bytearray))
else str(payload)
)
match = re.search(r"Message-ID:\s*(<[^>\s]+>)", text, re.IGNORECASE)
if match:
return match.group(1)
return None
except Exception as e:
logger.warning("Konnte Message-ID für UID %s nicht lesen: %s", uid, e)
return None
def _mail_exists_in_folder(self, folder: str, message_id: str) -> bool:
"""Sucht im Ordner nach einer Mail mit der gegebenen Message-ID (read-only)."""
try:
status, _ = self.conn.select(_quote_mailbox(folder), readonly=True)
if status != "OK":
logger.error("Konnte Ordner '%s' für Verifikation nicht öffnen", folder)
return False
# IMAP SEARCH HEADER: Wert als quoted-string übergeben
quoted_id = '"' + message_id.replace("\\", "\\\\").replace('"', '\\"') + '"'
search_status, search_data = self.conn.uid(
"SEARCH", None, "HEADER", "Message-ID", quoted_id
)
if search_status != "OK":
return False
return bool(search_data and search_data[0])
except Exception as e:
logger.error("Fehler bei Verifikation in '%s': %s", folder, e)
return False
def move_mail(
self,
uid: str,
target_folder: str,
source_folder: str | None = None,
) -> bool:
"""Sicheres Verschieben: COPY → Verifikation per Message-ID im Ziel → erst dann
Quelle löschen. Schlägt die Verifikation fehl, bleibt die Mail in der Quelle.
Wenn source_folder gesetzt ist, wird der Ordner nach der Verifikation wieder
selektiert (da die Verifikation den Zielordner aktiv schaltet)."""
try:
# 1. Message-ID aus der Quelle merken (vor COPY!) — Anker für die Verifikation
message_id = self._read_message_id(uid)
if not message_id:
logger.error(
"Mail %s hat keine Message-ID — sicheres Verschieben nicht möglich, "
"Quelle wird NICHT angetastet (Ziel: '%s')",
uid, target_folder,
)
return False
# 2. COPY in den Zielordner
copy_status, copy_data = self.conn.uid(
"COPY", uid, _quote_mailbox(target_folder)
)
if copy_status != "OK":
logger.error(
"COPY fehlgeschlagen für Mail %s nach '%s': %s %s — "
"Quelle bleibt unangetastet",
uid, target_folder, copy_status, copy_data,
)
# Quell-Ordner sicherheitshalber neu selektieren
if source_folder:
self.conn.select(_quote_mailbox(source_folder))
return False
# 3. Verifikation: Mail im Zielordner per Message-ID suchen
verified = self._mail_exists_in_folder(target_folder, message_id)
# 4. Quell-Ordner wieder aktivieren (Verifikation hat ihn umgeschaltet)
if source_folder:
self.conn.select(_quote_mailbox(source_folder))
if not verified:
logger.error(
"VERIFIKATION FEHLGESCHLAGEN: Mail %s (Message-ID %s) "
"nicht in '%s' gefunden — Quelle wird NICHT gelöscht",
uid, message_id, target_folder,
)
return False
# 5. Erst nach erfolgreicher Verifikation aus Quelle löschen
self.conn.uid("STORE", uid, "+FLAGS", "(\\Deleted)")
self.conn.expunge()
logger.info("Mail %s verschoben nach %s", uid, target_folder)
logger.info(
"Mail %s sicher verschoben nach '%s' (verifiziert: %s)",
uid, target_folder, message_id,
)
return True
except Exception as e:
logger.error("Fehler beim Verschieben von Mail %s: %s", uid, e)
return False
def delete_mail(self, uid: str, trash_folder: str = "Trash") -> bool:
return self.move_mail(uid, trash_folder)
def delete_mail(
self,
uid: str,
trash_folder: str = "Trash",
source_folder: str | None = None,
) -> bool:
return self.move_mail(uid, trash_folder, source_folder=source_folder)
def mark_as_read(self, uid: str) -> bool:
try:
app/services/scheduler.py
+5 -2
View File
@@ -158,7 +158,7 @@ def _poll_account_sync(account_id: int) -> None:
try:
# Ordner muss ausgewählt sein
if not hasattr(client, '_current_folder') or client._current_folder != folder:
client.conn.select(folder)
client.select_folder(folder)
client._current_folder = folder
mail = client.fetch_mail(uid)
except Exception as e:
@@ -195,7 +195,10 @@ def _poll_account_sync(account_id: int) -> None:
total_matched += 1
# Aktionen ausführen
for action in rule.actions:
success = execute_action(client, mail, action, smtp_config)
success = execute_action(
client, mail, action, smtp_config,
source_folder=folder,
)
action_label = action.action_type.value
param = action.parameter or ""
if param:
app/templates/yaml.html
+42 -4
View File
@@ -9,7 +9,12 @@
<article>
<header><h3>Backup erstellen</h3></header>
<p>Gesamte Konfiguration als JSON-Datei herunterladen.</p>
<button onclick="exportBackup()">Backup herunterladen</button>
<label style="display:flex; align-items:center; gap:0.5rem; margin-bottom:0.5rem;">
<input type="checkbox" id="backup-include-logs" style="margin-bottom:0;">
Logs mit sichern <small style="opacity:0.7;">(Datei wird deutlich größer)</small>
</label>
<button id="backup-download-btn" onclick="exportBackup()">Backup herunterladen</button>
<div id="backup-export-status" style="margin-top:0.5rem;"></div>
</article>
<article>
@@ -47,8 +52,40 @@
{% block scripts %}
<script>
function exportBackup() {
window.location.href = '/api/yaml/backup';
async function exportBackup() {
const includeLogs = document.getElementById('backup-include-logs').checked;
const btn = document.getElementById('backup-download-btn');
const statusDiv = document.getElementById('backup-export-status');
btn.setAttribute('aria-busy', 'true');
btn.disabled = true;
statusDiv.innerHTML = `<small>Backup wird erstellt${includeLogs ? ' (Logs werden geladen, kann dauern...)' : ''}</small>`;
try {
const url = `/api/yaml/backup${includeLogs ? '?include_logs=true' : ''}`;
const resp = await fetch(url);
if (!resp.ok) {
const text = await resp.text().catch(() => '');
throw new Error(`HTTP ${resp.status}: ${text || resp.statusText}`);
}
const blob = await resp.blob();
const disposition = resp.headers.get('Content-Disposition') || '';
const match = disposition.match(/filename=([^;]+)/);
const filename = match ? match[1].trim().replace(/^"|"$/g, '') : 'mailfilter-backup.json';
const downloadUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = downloadUrl;
a.download = filename;
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(downloadUrl);
const sizeMb = (blob.size / 1024 / 1024).toFixed(2);
statusDiv.innerHTML = `<small>Backup heruntergeladen (${sizeMb} MB).</small>`;
} catch (e) {
statusDiv.innerHTML = `<small style="color:var(--pico-color-red-500, #b00);">Fehler: ${e.message}</small>`;
} finally {
btn.removeAttribute('aria-busy');
btn.disabled = false;
}
}
document.getElementById('backup-import-form').addEventListener('submit', async (e) => {
@@ -67,7 +104,8 @@ document.getElementById('backup-import-form').addEventListener('submit', async (
Konten erstellt: ${result.accounts_created}<br>
Konten aktualisiert: ${result.accounts_updated}<br>
Regeln erstellt: ${result.rules_created}<br>
Verarbeitungsstatus: ${result.processed_restored} Einträge
Verarbeitungsstatus: ${result.processed_restored} Einträge<br>
Logs wiederhergestellt: ${result.logs_restored ?? 0} Einträge
</article>`;
}
});