new action: attach the message as PDF and file it
For senders whose mail body is the receipt (e.g. GoCardless payouts) there is no attachment to file. The new action "Als PDF anhängen + nach <Ordner>" renders the mail like a print from a mail client (header block From/To/Date/Subject, then the content) and replaces it in the target folder with: original content + PDF + the original mail byte-for-byte as Original.eml, so the unaltered original is retained (GoBD). - Rendering with WeasyPrint. Images are fetched by us and inlined as data: URIs; WeasyPrint itself may only read data:. Fetching is restricted to http(s), public addresses (private, loopback, link-local and PDF_BLOCKED_NETWORKS, default 172.0.0.0/12, are refused; redirects are checked hop by hop), with time and size limits. cid: images come from the MIME parts. - The replacement is APPENDed with the original flags and INTERNALDATE, verified in the target via its new Message-ID, and only then is the original deleted. Already converted mails are only moved. - "PDF-Vorschau" in the rule dialog renders the newest matching mail read-only and opens it in the browser. - List/log show 📄 targets; warnings for PDF + move/delete and for PDF + "Hat Anhang = Ja"; log folder search covers PDF targets. - Docker image gets Pango and fonts (~316 MB total). Tested with the real GoCardless mail (3 remote images, rendered both locally and inside the built image) and a simulated IMAP server for success, rejected APPEND, missing copy and already converted mails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -200,6 +200,14 @@ def execute_action(
|
||||
return imap_client.delete_mail(
|
||||
mail.uid, trash, source_folder=source_folder
|
||||
)
|
||||
case ActionType.PDF:
|
||||
if not action.parameter:
|
||||
imap_client.last_error = "Kein Zielordner in der Regel angegeben"
|
||||
logger.error(imap_client.last_error)
|
||||
return False
|
||||
return imap_client.convert_pdf_and_move(
|
||||
mail.uid, action.parameter, source_folder=source_folder
|
||||
)
|
||||
case ActionType.MARK_READ:
|
||||
return imap_client.mark_as_read(mail.uid)
|
||||
case ActionType.FORWARD:
|
||||
|
||||
@@ -490,6 +490,73 @@ class IMAPClient:
|
||||
pass
|
||||
return self._fail(f"Fehler beim Verschieben von Mail {uid} nach '{target_folder}': {e}")
|
||||
|
||||
def fetch_raw(self, uid: str) -> tuple[bytes, str, str | None] | None:
|
||||
"""Rohe Mail plus Flags (ohne \\Recent/\\Deleted) und INTERNALDATE im
|
||||
IMAP-Format — für APPEND einer Ersatz-Mail mit gleichen Eigenschaften."""
|
||||
status, data = self.conn.uid("FETCH", uid, "(FLAGS INTERNALDATE BODY.PEEK[])")
|
||||
if status != "OK":
|
||||
return None
|
||||
raw = _fetch_payload(data)
|
||||
if raw is None:
|
||||
return None
|
||||
meta = b" ".join(p[0] if isinstance(p, tuple) else p for p in data if isinstance(p, (tuple, bytes)))
|
||||
flags_m = re.search(rb"FLAGS \(([^)]*)\)", meta)
|
||||
flags = [f for f in (flags_m.group(1).decode().split() if flags_m else [])
|
||||
if f.lower() not in ("\\recent", "\\deleted")]
|
||||
date_m = re.search(rb'INTERNALDATE ("[^"]+")', meta)
|
||||
return raw, "(" + " ".join(flags) + ")", date_m.group(1).decode() if date_m else None
|
||||
|
||||
def convert_pdf_and_move(self, uid: str, target_folder: str, source_folder: str | None) -> bool:
|
||||
"""Mail als PDF anhängen und in target_folder ablegen.
|
||||
|
||||
Ersatz-Mail = Original-Inhalt + PDF + Original unverändert als .eml.
|
||||
Sie wird per APPEND mit gleichen Flags und gleichem Eingangsdatum in den
|
||||
Zielordner gelegt, dort über ihre neue Message-ID nachgewiesen, und erst
|
||||
dann wird das Original in der Quelle gelöscht. Schon umgewandelte Mails
|
||||
(z.B. bei Zielordner = Quellordner) werden nur verschoben."""
|
||||
from app.services.mail_pdf import build_converted_message, is_converted, render_pdf
|
||||
|
||||
self.last_error = None
|
||||
if not source_folder:
|
||||
return self._fail(f"Mail {uid}: Quellordner unbekannt — PDF-Umwandlung abgebrochen")
|
||||
try:
|
||||
fetched = self.fetch_raw(uid)
|
||||
if not fetched:
|
||||
return self._fail(f"Mail {uid} konnte für die PDF-Umwandlung nicht geladen werden")
|
||||
raw, flags, internal_date = fetched
|
||||
if is_converted(raw):
|
||||
return self.move_mail(uid, target_folder, source_folder=source_folder)
|
||||
|
||||
try:
|
||||
pdf, _ = render_pdf(raw)
|
||||
except Exception as e:
|
||||
return self._fail(f"PDF-Erzeugung für Mail {uid} fehlgeschlagen: {e}")
|
||||
new_raw, new_id = build_converted_message(raw, pdf)
|
||||
|
||||
status, data = self.conn.append(_quote_mailbox(target_folder), flags, internal_date, new_raw)
|
||||
if status != "OK":
|
||||
self.conn.select(_quote_mailbox(source_folder))
|
||||
detail = b" ".join(d for d in (data or []) if isinstance(d, bytes)).decode(errors="replace")
|
||||
return self._fail(f"Server hat das Ablegen in '{target_folder}' abgelehnt: {status} {detail}".strip())
|
||||
|
||||
verified = self._mail_exists_in_folder(target_folder, new_id)
|
||||
self.conn.select(_quote_mailbox(source_folder))
|
||||
if not verified:
|
||||
return self._fail(
|
||||
f"PDF-Fassung von Mail {uid} im Ziel '{target_folder}' nicht gefunden — Original bleibt in der Quelle"
|
||||
)
|
||||
|
||||
self.conn.uid("STORE", uid, "+FLAGS", "(\\Deleted)")
|
||||
self.conn.expunge()
|
||||
logger.info("Mail %s als PDF angehängt und nach '%s' abgelegt (%s)", uid, target_folder, new_id)
|
||||
return True
|
||||
except Exception as e:
|
||||
try:
|
||||
self.conn.select(_quote_mailbox(source_folder))
|
||||
except Exception:
|
||||
pass
|
||||
return self._fail(f"Fehler bei PDF-Umwandlung von Mail {uid}: {e}")
|
||||
|
||||
def delete_mail(
|
||||
self,
|
||||
uid: str,
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Mail → PDF: für Absender, bei denen die Nachricht selbst der Beleg ist
|
||||
(z.B. GoCardless-Auszahlungen ohne PDF-Anhang).
|
||||
|
||||
Die Mail wird wie ein Ausdruck aus dem Mailprogramm gerendert (Kopfblock mit
|
||||
Von/An/Datum/Betreff, darunter der Inhalt). Bilder werden vorher selbst geladen
|
||||
und als data:-URIs eingebettet; WeasyPrint darf selbst nichts nachladen. Beim
|
||||
Nachladen gelten Schutzregeln, weil auch Spam-Mails Ziel einer Regel sein
|
||||
können: nur http(s), nur öffentliche Adressen (keine Anfragen ins interne
|
||||
Netz), Zeit- und Größenlimits, Redirects werden einzeln geprüft.
|
||||
|
||||
build_converted_message() baut die Ersatz-Mail: Original-Inhalt (Anzeige wie
|
||||
vorher) + PDF-Anhang + die Original-Mail Byte für Byte als .eml-Anhang, damit
|
||||
das unveränderte Original erhalten bleibt (GoBD)."""
|
||||
import base64
|
||||
import email
|
||||
import html
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from email import policy
|
||||
from email.generator import BytesGenerator
|
||||
from email.message import Message
|
||||
from email.mime.application import MIMEApplication
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.utils import make_msgid
|
||||
from io import BytesIO
|
||||
|
||||
from app.config import settings
|
||||
from app.services.imap_client import _decode_header_value, _safe_decode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONVERTED_HEADER = "X-Mailfilter-Converted"
|
||||
|
||||
FETCH_TIMEOUT = 8
|
||||
MAX_RESOURCE_BYTES = 5 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 20 * 1024 * 1024
|
||||
MAX_RESOURCES = 40
|
||||
MAX_REDIRECTS = 3
|
||||
|
||||
_PRINT_CSS = """
|
||||
@page { size: A4; margin: 14mm 12mm; }
|
||||
html, body { font-family: 'DejaVu Sans', Arial, sans-serif; }
|
||||
img { max-width: 100%; height: auto; }
|
||||
table { max-width: 100%; }
|
||||
.mf-head { font: 9.5pt 'DejaVu Sans', Arial, sans-serif; color: #222; border-bottom: 1px solid #bbb;
|
||||
padding-bottom: 6pt; margin-bottom: 10pt; background: #fff; }
|
||||
.mf-head, .mf-head * { text-align: left !important; }
|
||||
.mf-head table { border-collapse: collapse; width: auto; }
|
||||
.mf-head td { padding: 1pt 8pt 1pt 0; vertical-align: top; }
|
||||
.mf-head td.k { color: #666; white-space: nowrap; }
|
||||
.mf-head .subj { font-size: 12pt; font-weight: bold; margin-bottom: 4pt; }
|
||||
pre.mf-text { white-space: pre-wrap; word-wrap: break-word; font: 10pt 'DejaVu Sans Mono', monospace; }
|
||||
"""
|
||||
|
||||
|
||||
# ---------- Bilder sicher laden ----------
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None # Redirects selbst behandeln, damit jedes Ziel geprüft wird
|
||||
|
||||
|
||||
_opener = urllib.request.build_opener(_NoRedirect)
|
||||
|
||||
|
||||
def _blocked_networks() -> list:
|
||||
nets = []
|
||||
for part in (settings.pdf_blocked_networks or "").split(","):
|
||||
part = part.strip()
|
||||
if part:
|
||||
try:
|
||||
nets.append(ipaddress.ip_network(part, strict=False))
|
||||
except ValueError:
|
||||
logger.warning("PDF_BLOCKED_NETWORKS: ungültiges Netz '%s' ignoriert", part)
|
||||
return nets
|
||||
|
||||
|
||||
def _host_is_public(host: str) -> bool:
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
blocked = _blocked_networks()
|
||||
for info in infos:
|
||||
ip = ipaddress.ip_address(info[4][0].split("%")[0])
|
||||
if not ip.is_global or ip.is_multicast or any(ip in net for net in blocked):
|
||||
return False
|
||||
return bool(infos)
|
||||
|
||||
|
||||
def _safe_fetch(url: str, budget: dict) -> tuple[bytes, str] | None:
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
if parts.scheme not in ("http", "https") or not parts.hostname:
|
||||
return None
|
||||
if not _host_is_public(parts.hostname):
|
||||
logger.info("PDF: nicht-öffentliche Adresse übersprungen: %s", parts.hostname)
|
||||
return None
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (MailFilter PDF)"})
|
||||
try:
|
||||
resp = _opener.open(req, timeout=FETCH_TIMEOUT)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (301, 302, 303, 307, 308) and e.headers.get("Location"):
|
||||
url = urllib.parse.urljoin(url, e.headers["Location"])
|
||||
continue
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
with resp:
|
||||
ctype = (resp.headers.get_content_type() or "").lower()
|
||||
if not ctype.startswith("image/"):
|
||||
return None
|
||||
remaining = min(MAX_RESOURCE_BYTES, budget["bytes"])
|
||||
data = resp.read(remaining + 1)
|
||||
if len(data) > remaining:
|
||||
return None
|
||||
budget["bytes"] -= len(data)
|
||||
return data, ctype
|
||||
return None
|
||||
|
||||
|
||||
def _data_uri(data: bytes, ctype: str) -> str:
|
||||
return f"data:{ctype};base64,{base64.b64encode(data).decode()}"
|
||||
|
||||
|
||||
def _inline_resources(body_html: str, cid_map: dict[str, tuple[bytes, str]]) -> tuple[str, dict]:
|
||||
"""Ersetzt src="cid:…", src="http…", background="…" und CSS url(…) durch
|
||||
data:-URIs. Nicht ladbare Ressourcen werden entfernt."""
|
||||
budget = {"bytes": MAX_TOTAL_BYTES, "count": 0}
|
||||
cache: dict[str, str | None] = {}
|
||||
stats = {"loaded": 0, "skipped": 0, "cid": 0}
|
||||
|
||||
def resolve(raw_url: str) -> str | None:
|
||||
url = html.unescape(raw_url.strip())
|
||||
if url in cache:
|
||||
return cache[url]
|
||||
result = None
|
||||
if url.lower().startswith("cid:"):
|
||||
part = cid_map.get(urllib.parse.unquote(url[4:]).strip("<>"))
|
||||
if part:
|
||||
result = _data_uri(*part)
|
||||
stats["cid"] += 1
|
||||
elif url.lower().startswith("data:"):
|
||||
result = url
|
||||
elif url.lower().startswith(("http://", "https://")) and budget["count"] < MAX_RESOURCES:
|
||||
budget["count"] += 1
|
||||
fetched = _safe_fetch(url, budget)
|
||||
if fetched:
|
||||
result = _data_uri(*fetched)
|
||||
stats["loaded"] += 1
|
||||
if result is None and not url.lower().startswith("data:"):
|
||||
stats["skipped"] += 1
|
||||
cache[url] = result
|
||||
return result
|
||||
|
||||
def attr_sub(m: re.Match) -> str:
|
||||
new = resolve(m.group(3))
|
||||
return f'{m.group(1)}={m.group(2)}{new or ""}{m.group(2)}'
|
||||
|
||||
body_html = re.sub(r"\s(?:srcset)\s*=\s*([\"']).*?\1", " ", body_html, flags=re.I | re.S)
|
||||
body_html = re.sub(r"(\s(?:src|background))\s*=\s*([\"'])(.*?)\2", attr_sub, body_html, flags=re.I | re.S)
|
||||
body_html = re.sub(
|
||||
r"url\(\s*([\"']?)(.*?)\1\s*\)",
|
||||
lambda m: f"url('{resolve(m.group(2)) or ''}')",
|
||||
body_html, flags=re.I,
|
||||
)
|
||||
return body_html, stats
|
||||
|
||||
|
||||
# ---------- Rendern ----------
|
||||
|
||||
def _mail_parts(msg: Message) -> tuple[str | None, str | None, dict]:
|
||||
html_body, text_body, cid_map = None, None, {}
|
||||
for part in msg.walk():
|
||||
if part.is_multipart():
|
||||
continue
|
||||
ctype = part.get_content_type()
|
||||
disposition = str(part.get("Content-Disposition") or "").lower()
|
||||
cid = part.get("Content-ID")
|
||||
if cid:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
cid_map[cid.strip().strip("<>")] = (payload, ctype)
|
||||
if "attachment" in disposition:
|
||||
continue
|
||||
if ctype == "text/html" and html_body is None:
|
||||
html_body = _safe_decode(part.get_payload(decode=True) or b"", part.get_content_charset())
|
||||
elif ctype == "text/plain" and text_body is None:
|
||||
text_body = _safe_decode(part.get_payload(decode=True) or b"", part.get_content_charset())
|
||||
return html_body, text_body, cid_map
|
||||
|
||||
|
||||
def _header_block(msg: Message) -> str:
|
||||
rows = []
|
||||
for key, label in (("From", "Von"), ("To", "An"), ("Cc", "Cc"), ("Date", "Datum")):
|
||||
value = _decode_header_value(msg.get(key))
|
||||
if value:
|
||||
rows.append(f'<tr><td class="k">{label}:</td><td>{html.escape(value)}</td></tr>')
|
||||
subject = html.escape(_decode_header_value(msg.get("Subject")) or "(kein Betreff)")
|
||||
return f'<div class="mf-head"><div class="subj">{subject}</div><table>{"".join(rows)}</table></div>'
|
||||
|
||||
|
||||
def render_pdf(raw: bytes) -> tuple[bytes, dict]:
|
||||
"""Rendert die Mail als PDF. Liefert (pdf_bytes, stats)."""
|
||||
from weasyprint import CSS, HTML
|
||||
from weasyprint.urls import URLFetcher
|
||||
|
||||
msg = email.message_from_bytes(raw)
|
||||
html_body, text_body, cid_map = _mail_parts(msg)
|
||||
head = _header_block(msg)
|
||||
|
||||
if html_body:
|
||||
body, stats = _inline_resources(html_body, cid_map)
|
||||
body = re.sub(r"<script\b.*?</script>", "", body, flags=re.I | re.S)
|
||||
if re.search(r"<body\b[^>]*>", body, flags=re.I):
|
||||
document = re.sub(r"(<body\b[^>]*>)", lambda m: m.group(1) + head, body, count=1, flags=re.I)
|
||||
else:
|
||||
document = f"<html><head><meta charset='utf-8'></head><body>{head}{body}</body></html>"
|
||||
else:
|
||||
stats = {"loaded": 0, "skipped": 0, "cid": 0}
|
||||
text = html.escape(text_body or "")
|
||||
document = f"<html><head><meta charset='utf-8'></head><body>{head}<pre class='mf-text'>{text}</pre></body></html>"
|
||||
|
||||
# Alles ist bereits eingebettet — WeasyPrint darf nur noch data:-URIs lesen
|
||||
fetcher = URLFetcher(allowed_protocols={"data"}, allow_redirects=False)
|
||||
pdf = HTML(string=document, url_fetcher=fetcher).write_pdf(stylesheets=[CSS(string=_PRINT_CSS)])
|
||||
return pdf, stats
|
||||
|
||||
|
||||
# ---------- Ersatz-Mail bauen ----------
|
||||
|
||||
def _pdf_filename(msg: Message) -> str:
|
||||
subject = _decode_header_value(msg.get("Subject")) or "Nachricht"
|
||||
name = re.sub(r'[\\/:*?"<>|\r\n\t]+', "_", subject).strip(" ._") or "Nachricht"
|
||||
return name[:90] + ".pdf"
|
||||
|
||||
|
||||
def is_converted(raw_or_msg) -> bool:
|
||||
msg = raw_or_msg if isinstance(raw_or_msg, Message) else email.message_from_bytes(raw_or_msg)
|
||||
return msg.get(CONVERTED_HEADER) is not None
|
||||
|
||||
|
||||
def build_converted_message(raw: bytes, pdf: bytes) -> tuple[bytes, str]:
|
||||
"""Neue Mail: gleiche Kopfzeilen, gleicher sichtbarer Inhalt, plus PDF und
|
||||
das unveränderte Original als .eml. Liefert (bytes, neue Message-ID)."""
|
||||
original = email.message_from_bytes(raw)
|
||||
|
||||
new = MIMEMultipart("mixed")
|
||||
for key in ("From", "Sender", "Reply-To", "To", "Cc", "Date", "Subject"):
|
||||
for value in original.get_all(key, []):
|
||||
new[key] = value
|
||||
new_id = make_msgid(domain="mailfilter.local")
|
||||
new["Message-ID"] = new_id
|
||||
new[CONVERTED_HEADER] = "pdf"
|
||||
if original.get("Message-ID"):
|
||||
new["X-Mailfilter-Original-Message-ID"] = original["Message-ID"]
|
||||
|
||||
# Sichtbarer Inhalt wie im Original: dessen Body-Struktur ohne Kopfzeilen
|
||||
body = email.message_from_bytes(raw)
|
||||
for key in list(body.keys()):
|
||||
if not key.lower().startswith("content-"):
|
||||
del body[key]
|
||||
if not body.get("Content-Type"):
|
||||
body["Content-Type"] = "text/plain; charset=utf-8"
|
||||
new.attach(body)
|
||||
|
||||
pdf_part = MIMEApplication(pdf, _subtype="pdf")
|
||||
pdf_part.add_header("Content-Disposition", "attachment", filename=("utf-8", "", _pdf_filename(original)))
|
||||
new.attach(pdf_part)
|
||||
|
||||
# Original Byte für Byte (base64 ist verlustfrei)
|
||||
eml_part = MIMEApplication(raw, _subtype="octet-stream")
|
||||
eml_part.add_header("Content-Disposition", "attachment", filename="Original.eml")
|
||||
new.attach(eml_part)
|
||||
|
||||
buf = BytesIO()
|
||||
BytesGenerator(buf, policy=policy.compat32.clone(linesep="\r\n"), mangle_from_=False).flatten(new)
|
||||
return buf.getvalue(), new_id
|
||||
@@ -60,6 +60,37 @@ def _build_search(conditions) -> list[str]:
|
||||
return criteria
|
||||
|
||||
|
||||
def preview_pdf(account: Account, source_folder: str, conditions, limit: int = 40) -> tuple[bytes, dict]:
|
||||
"""PDF der neuesten Mail, auf die die Regel zutrifft — read-only, nichts
|
||||
wird verändert. Liefert (pdf, info)."""
|
||||
from app.services.mail_pdf import render_pdf
|
||||
|
||||
criteria = _build_search(conditions)
|
||||
client = IMAPClient(
|
||||
host=account.imap_host,
|
||||
port=account.imap_port,
|
||||
username=account.username,
|
||||
password=decrypt(account.password),
|
||||
use_ssl=account.use_ssl,
|
||||
)
|
||||
with client:
|
||||
if not client.select_folder(source_folder, readonly=True):
|
||||
raise ValueError(f"Ordner '{source_folder}' konnte nicht geöffnet werden")
|
||||
status, data = client.conn.uid("SEARCH", None, *criteria)
|
||||
uids = data[0].split() if status == "OK" and data and data[0] else []
|
||||
for raw_uid in sorted(uids, key=int, reverse=True)[:max(1, min(limit, 100))]:
|
||||
uid = raw_uid.decode() if isinstance(raw_uid, bytes) else str(raw_uid)
|
||||
mail = client.fetch_mail(uid)
|
||||
if not mail or not evaluate_conditions(mail, conditions)[0]:
|
||||
continue
|
||||
fetched = client.fetch_raw(uid)
|
||||
if not fetched:
|
||||
continue
|
||||
pdf, stats = render_pdf(fetched[0])
|
||||
return pdf, {"uid": uid, "subject": mail.subject, **stats}
|
||||
raise LookupError("Keine Mail gefunden, auf die die Regel zutrifft")
|
||||
|
||||
|
||||
def test_rule(account: Account, source_folder: str, conditions, limit: int = 40) -> dict:
|
||||
limit = max(1, min(limit, 100))
|
||||
criteria = _build_search(conditions)
|
||||
|
||||
Reference in New Issue
Block a user