added amazon importer and logging smtp
This commit is contained in:
+145
-102
@@ -5,8 +5,8 @@ import tempfile
|
||||
|
||||
import smbclient
|
||||
|
||||
from app.database import get_settings, add_log_entry
|
||||
from app.mail_processor import _connect_smtp, _build_forward_email
|
||||
from app.database import get_settings, add_log_entry, get_import_email
|
||||
from app.mail_processor import _connect_smtp, _build_forward_email, _send_with_log
|
||||
from app.scanner import detect_separator_pages, split_pdf
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -114,6 +114,119 @@ def _list_smb_folders_recursive(
|
||||
return folders
|
||||
|
||||
|
||||
async def _process_smb_folder(
|
||||
smtp_conn, settings: dict, base_path: str,
|
||||
source_rel: str, processed_rel: str,
|
||||
import_email: str, beleg_type: str, mode: str,
|
||||
) -> dict:
|
||||
"""Process one SMB folder pair. Returns counts dict."""
|
||||
smtp_from = settings.get("smtp_username", "")
|
||||
processed = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
|
||||
source_path = _smb_unc_path(base_path, source_rel)
|
||||
processed_path = _smb_unc_path(base_path, processed_rel)
|
||||
|
||||
await asyncio.to_thread(_ensure_smb_folder, processed_path)
|
||||
|
||||
pdf_files = await asyncio.to_thread(_list_pdf_files, source_path)
|
||||
if not pdf_files:
|
||||
logger.info(f"Keine PDF-Dateien im SMB-Ordner '{source_rel}' ({beleg_type})")
|
||||
return {"processed": 0, "skipped": 0, "errors": 0}
|
||||
|
||||
logger.info(f"{len(pdf_files)} PDF-Datei(en) im SMB-Ordner '{source_rel}' ({beleg_type})")
|
||||
|
||||
for filename in pdf_files:
|
||||
file_path = _smb_unc_path(source_path, filename)
|
||||
try:
|
||||
pdf_data = await asyncio.to_thread(_read_smb_file, file_path)
|
||||
|
||||
if mode == "separator":
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(pdf_data)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
separator_pages = await asyncio.to_thread(
|
||||
detect_separator_pages, tmp_path, None
|
||||
)
|
||||
documents = await asyncio.to_thread(
|
||||
split_pdf, tmp_path, separator_pages
|
||||
)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
if not documents:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
smtp_log_parts = []
|
||||
for i, doc_bytes in enumerate(documents):
|
||||
doc_filename = f"{os.path.splitext(filename)[0]}_Teil_{i + 1}.pdf"
|
||||
subject = f"SMB-Import: {filename} (Dokument {i + 1}/{len(documents)})"
|
||||
msg = _build_forward_email(
|
||||
from_addr=smtp_from,
|
||||
to_addr=import_email,
|
||||
original_subject=subject,
|
||||
original_from="SMB-Import",
|
||||
attachments=[(doc_filename, doc_bytes)],
|
||||
)
|
||||
smtp_log_parts.append(_send_with_log(smtp_conn, msg))
|
||||
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=len(documents),
|
||||
status="success",
|
||||
sent_to=import_email,
|
||||
smtp_log="\n---\n".join(smtp_log_parts),
|
||||
beleg_type=beleg_type,
|
||||
)
|
||||
logger.info(
|
||||
f"SMB verarbeitet ({beleg_type}): {filename} -> {len(documents)} Dokument(e)"
|
||||
)
|
||||
else:
|
||||
msg = _build_forward_email(
|
||||
from_addr=smtp_from,
|
||||
to_addr=import_email,
|
||||
original_subject=f"SMB-Import: {filename}",
|
||||
original_from="SMB-Import",
|
||||
attachments=[(filename, pdf_data)],
|
||||
)
|
||||
smtp_log = _send_with_log(smtp_conn, msg)
|
||||
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=1,
|
||||
status="success",
|
||||
sent_to=import_email,
|
||||
smtp_log=smtp_log,
|
||||
beleg_type=beleg_type,
|
||||
)
|
||||
logger.info(f"SMB verarbeitet ({beleg_type}): {filename}")
|
||||
|
||||
await asyncio.to_thread(_move_smb_file, file_path, processed_path, filename)
|
||||
processed += 1
|
||||
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
logger.error(f"Fehler bei SMB-Datei {filename}: {e}")
|
||||
try:
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=0,
|
||||
status="error",
|
||||
error_message=str(e),
|
||||
beleg_type=beleg_type,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"processed": processed, "skipped": skipped, "errors": errors}
|
||||
|
||||
|
||||
async def process_smb_share() -> dict:
|
||||
"""Process PDF files from SMB share - main pipeline."""
|
||||
settings = await get_settings()
|
||||
@@ -124,113 +237,43 @@ async def process_smb_share() -> dict:
|
||||
if not settings.get("smb_server") or not settings.get("smb_share"):
|
||||
return {"processed": 0, "skipped": 0, "errors": 0, "error": "SMB nicht konfiguriert"}
|
||||
|
||||
if not settings.get("import_email"):
|
||||
import_email_eingang = get_import_email(settings, "eingang")
|
||||
if not import_email_eingang:
|
||||
return {"processed": 0, "skipped": 0, "errors": 0, "error": "Import-Email nicht konfiguriert"}
|
||||
|
||||
mode = settings.get("smb_mode", "forward")
|
||||
smtp_from = settings.get("smtp_username", "")
|
||||
import_email = settings["import_email"]
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
total = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
smtp_conn = None
|
||||
|
||||
try:
|
||||
base_path = await asyncio.to_thread(_smb_register_session, settings)
|
||||
source_path = _smb_unc_path(base_path, settings.get("smb_source_path", ""))
|
||||
processed_path = _smb_unc_path(base_path, settings.get("smb_processed_path", "Verarbeitet"))
|
||||
|
||||
await asyncio.to_thread(_ensure_smb_folder, processed_path)
|
||||
|
||||
pdf_files = await asyncio.to_thread(_list_pdf_files, source_path)
|
||||
if not pdf_files:
|
||||
logger.info("Keine PDF-Dateien im SMB-Ordner gefunden")
|
||||
return {"processed": 0, "skipped": 0, "errors": 0}
|
||||
|
||||
logger.info(f"{len(pdf_files)} PDF-Datei(en) im SMB-Ordner gefunden")
|
||||
|
||||
smtp_conn = _connect_smtp(settings)
|
||||
|
||||
for filename in pdf_files:
|
||||
file_path = _smb_unc_path(source_path, filename)
|
||||
try:
|
||||
pdf_data = await asyncio.to_thread(_read_smb_file, file_path)
|
||||
# Eingangsbelege
|
||||
source = settings.get("smb_source_path", "")
|
||||
processed_rel = settings.get("smb_processed_path", "Verarbeitet")
|
||||
result = await _process_smb_folder(
|
||||
smtp_conn, settings, base_path,
|
||||
source, processed_rel,
|
||||
import_email_eingang, "eingang", mode,
|
||||
)
|
||||
for k in total:
|
||||
total[k] += result[k]
|
||||
|
||||
if mode == "separator":
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(pdf_data)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
separator_pages = await asyncio.to_thread(
|
||||
detect_separator_pages, tmp_path, None
|
||||
)
|
||||
documents = await asyncio.to_thread(
|
||||
split_pdf, tmp_path, separator_pages
|
||||
)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
if not documents:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
for i, doc_bytes in enumerate(documents):
|
||||
doc_filename = f"{os.path.splitext(filename)[0]}_Teil_{i + 1}.pdf"
|
||||
subject = f"SMB-Import: {filename} (Dokument {i + 1}/{len(documents)})"
|
||||
msg = _build_forward_email(
|
||||
from_addr=smtp_from,
|
||||
to_addr=import_email,
|
||||
original_subject=subject,
|
||||
original_from="SMB-Import",
|
||||
attachments=[(doc_filename, doc_bytes)],
|
||||
)
|
||||
smtp_conn.send_message(msg)
|
||||
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=len(documents),
|
||||
status="success",
|
||||
)
|
||||
logger.info(
|
||||
f"SMB verarbeitet: {filename} -> {len(documents)} Dokument(e) "
|
||||
f"({len(separator_pages)} Trennseite(n))"
|
||||
)
|
||||
else:
|
||||
msg = _build_forward_email(
|
||||
from_addr=smtp_from,
|
||||
to_addr=import_email,
|
||||
original_subject=f"SMB-Import: {filename}",
|
||||
original_from="SMB-Import",
|
||||
attachments=[(filename, pdf_data)],
|
||||
)
|
||||
smtp_conn.send_message(msg)
|
||||
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=1,
|
||||
status="success",
|
||||
)
|
||||
logger.info(f"SMB verarbeitet: {filename}")
|
||||
|
||||
await asyncio.to_thread(_move_smb_file, file_path, processed_path, filename)
|
||||
processed += 1
|
||||
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
logger.error(f"Fehler bei SMB-Datei {filename}: {e}")
|
||||
try:
|
||||
await add_log_entry(
|
||||
email_subject=f"SMB: {filename}",
|
||||
email_from="SMB-Import",
|
||||
attachments_count=0,
|
||||
status="error",
|
||||
error_message=str(e),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# Ausgangsbelege (optional)
|
||||
import_email_ausgang = get_import_email(settings, "ausgang")
|
||||
source_ausgang = settings.get("smb_source_path_ausgang", "")
|
||||
processed_ausgang = settings.get("smb_processed_path_ausgang", "")
|
||||
if import_email_ausgang and source_ausgang:
|
||||
if not processed_ausgang:
|
||||
processed_ausgang = source_ausgang + "/Verarbeitet"
|
||||
result = await _process_smb_folder(
|
||||
smtp_conn, settings, base_path,
|
||||
source_ausgang, processed_ausgang,
|
||||
import_email_ausgang, "ausgang", mode,
|
||||
)
|
||||
for k in total:
|
||||
total[k] += result[k]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"SMB-Verbindungsfehler: {e}")
|
||||
@@ -244,7 +287,7 @@ async def process_smb_share() -> dict:
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return {"processed": processed, "skipped": skipped, "errors": errors + 1, "error": str(e)}
|
||||
return {**total, "errors": total["errors"] + 1, "error": str(e)}
|
||||
|
||||
finally:
|
||||
if smtp_conn:
|
||||
@@ -253,8 +296,8 @@ async def process_smb_share() -> dict:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info(f"SMB fertig: {processed} verarbeitet, {skipped} übersprungen, {errors} Fehler")
|
||||
return {"processed": processed, "skipped": skipped, "errors": errors}
|
||||
logger.info(f"SMB fertig: {total['processed']} verarbeitet, {total['skipped']} übersprungen, {total['errors']} Fehler")
|
||||
return total
|
||||
|
||||
|
||||
async def test_smb_connection() -> dict:
|
||||
|
||||
Reference in New Issue
Block a user