ARIA
5c2cdd7543
Neuer Modus in der GUI: Image nur herunterladen und dauerhaft unter ./downloads/library/ ablegen, ohne USB-Stick. Baut sich so vorab eine komplette Offline-Sammlung aller macOS-Versionen auf (z.B. auf externe Platte kopierbar). Stick-Jobs pruefen jetzt IMMER zuerst die lokale Bibliothek und ueberspringen den Download bei Treffer -- jeder normale Stick-Bau traegt so nebenbei zur Bibliothek bei. Backend: installer.py bekommt job_type (stick|download), persistente Downloads in library/<name>/ mit meta.json + atomarem .part-Rename, neue /api/library. Frontend: Modus-Umschalter, Bibliotheks-Status, Offline-Badges in der Versionsliste. Live getestet auf aria-wohnung: Container neu gebaut, Katalog laedt 37 Installer, Download-Job End-to-End ueber die echte API (Cache-Miss dann Cache-Hit verifiziert), Stick-Job-Validierung weiterhin intakt.
402 lines
16 KiB
Python
402 lines
16 KiB
Python
"""
|
|
Job-Engine: Download eines macOS-Installers + Beschreiben eines USB-Sticks.
|
|
|
|
Zwei Pfade, unterschiedlich robust:
|
|
|
|
legacy_dmg (macOS bis Catalina/10.15): Apple liefert ein fertiges,
|
|
bootfaehiges BaseSystem.dmg/InstallESD.dmg. Wir wandeln es mit
|
|
dmg2img in ein rohes Image und schreiben es 1:1 mit dd auf den
|
|
Stick. Etablierter, zuverlaessiger Weg.
|
|
|
|
recovery (macOS ab Big Sur/11): Apple liefert nur noch ein
|
|
InstallAssistant.pkg. Das eigentliche createinstallmedia laeuft
|
|
nur unter echtem macOS. Wir extrahieren das .pkg (xar -> pbzx
|
|
-> cpio) und suchen darin ein SharedSupport.dmg/BaseSystem.dmg,
|
|
das wir dann genauso per dd schreiben. EXPERIMENTELL: Apple hat
|
|
den internen Aufbau von InstallAssistant.pkg mehrfach leicht
|
|
geaendert, das kann brechen. Klar als "experimentell" markiert.
|
|
|
|
Zwei Job-Typen:
|
|
|
|
"stick" Download (oder Wiederverwendung aus der lokalen Bibliothek) +
|
|
Schreiben auf einen USB-Stick.
|
|
"download" Nur herunterladen, dauerhaft in der lokalen Bibliothek ablegen.
|
|
Kein Geraet noetig. Gedacht um sich vorab eine Offline-Sammlung
|
|
aller macOS-Installer aufzubauen (z.B. auf eine externe Platte),
|
|
die man auch ohne Internetzugang zum Stick-Bauen nutzen kann.
|
|
|
|
Lokale Bibliothek (Persistenz):
|
|
|
|
Jeder erfolgreiche Download landet dauerhaft unter
|
|
/data/downloads/library/<safer-name>/ (Original-Dateiname + meta.json).
|
|
Das liegt im Bind-Mount ./downloads -- also ganz normal im Projektordner
|
|
und damit portabel (Ordner/Platte kopieren reicht). Ein "stick"-Job prueft
|
|
IMMER zuerst, ob die Datei schon lokal vorhanden ist, und ueberspringt den
|
|
Download wenn ja -- so wird aus jedem einzelnen Stick-Bau ganz nebenbei
|
|
eine wachsende Offline-Bibliothek, ganz ohne Zusatzaufwand.
|
|
|
|
Sicherheits-Leitplanken:
|
|
- Ziel-Device wird gegen die Root-Disk des Hosts geprueft (Refuse).
|
|
- Ziel-Device muss "removable" oder USB-Transport sein (Refuse sonst).
|
|
- Aufrufer muss den Device-Pfad im Request nochmal exakt bestaetigen.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from queue import Queue, Empty
|
|
|
|
import requests
|
|
|
|
from devices import list_usb_sticks
|
|
|
|
log = logging.getLogger("installer")
|
|
|
|
DOWNLOAD_DIR = "/data/downloads"
|
|
LIBRARY_DIR = os.path.join(DOWNLOAD_DIR, "library")
|
|
JOBS: dict[str, "Job"] = {}
|
|
|
|
|
|
class JobAborted(Exception):
|
|
pass
|
|
|
|
|
|
def _safe_name(product: dict) -> str:
|
|
"""Stabiler, lesbarer Ordnername fuer die Bibliothek, z.B.
|
|
'macOS_Sonoma_14.6_061-12345'. Die Produkt-ID am Ende macht ihn eindeutig,
|
|
auch wenn zwei Eintraege denselben Titel haben."""
|
|
raw = f"{product.get('title', 'macOS')}_{product.get('version', '0')}_{product.get('id', '')}"
|
|
return re.sub(r"[^A-Za-z0-9._-]+", "_", raw).strip("_")
|
|
|
|
|
|
def _payload_url(product: dict) -> str:
|
|
if product["mode"] == "legacy_dmg":
|
|
return product["base_system_url"]
|
|
return product["install_assistant_url"]
|
|
|
|
|
|
def library_entry_dir(product: dict) -> str:
|
|
return os.path.join(LIBRARY_DIR, _safe_name(product))
|
|
|
|
|
|
def library_payload_path(product: dict) -> str:
|
|
filename = os.path.basename(_payload_url(product)) or (
|
|
"BaseSystem.dmg" if product["mode"] == "legacy_dmg" else "InstallAssistant.pkg"
|
|
)
|
|
return os.path.join(library_entry_dir(product), filename)
|
|
|
|
|
|
def get_cached_payload(product: dict):
|
|
"""Gibt den Pfad zurueck wenn die Datei schon vollstaendig in der
|
|
Bibliothek liegt (meta.json + Datei vorhanden, Groesse passt), sonst
|
|
None."""
|
|
path = library_payload_path(product)
|
|
meta_path = os.path.join(library_entry_dir(product), "meta.json")
|
|
if not (os.path.isfile(path) and os.path.isfile(meta_path)):
|
|
return None
|
|
try:
|
|
with open(meta_path) as f:
|
|
meta = json.load(f)
|
|
if meta.get("complete") and os.path.getsize(path) == meta.get("size_bytes"):
|
|
return path
|
|
except (OSError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def list_library():
|
|
"""Fuer die GUI: alle vollstaendig heruntergeladenen Produkte + Gesamtgroesse."""
|
|
items = []
|
|
total = 0
|
|
if os.path.isdir(LIBRARY_DIR):
|
|
for entry in sorted(os.listdir(LIBRARY_DIR)):
|
|
meta_path = os.path.join(LIBRARY_DIR, entry, "meta.json")
|
|
if not os.path.isfile(meta_path):
|
|
continue
|
|
try:
|
|
with open(meta_path) as f:
|
|
meta = json.load(f)
|
|
except (OSError, ValueError):
|
|
continue
|
|
if not meta.get("complete"):
|
|
continue
|
|
items.append(meta)
|
|
total += meta.get("size_bytes", 0)
|
|
return {"items": items, "total_bytes": total}
|
|
|
|
|
|
class Job:
|
|
def __init__(self, product: dict, device_path: str = None, job_type: str = "stick"):
|
|
self.id = str(uuid.uuid4())
|
|
self.product = product
|
|
self.device_path = device_path
|
|
self.job_type = job_type # "stick" | "download"
|
|
self.status = "queued" # queued -> running -> done | failed | aborted
|
|
self.phase = "warten"
|
|
self.progress = 0 # 0-100 im aktuellen Phase
|
|
self.error = None
|
|
self._q: Queue = Queue()
|
|
self._lock = threading.Lock()
|
|
self.history = []
|
|
|
|
def emit(self, phase=None, progress=None, msg=None, status=None):
|
|
with self._lock:
|
|
if phase is not None:
|
|
self.phase = phase
|
|
if progress is not None:
|
|
self.progress = progress
|
|
if status is not None:
|
|
self.status = status
|
|
event = {
|
|
"phase": self.phase,
|
|
"progress": self.progress,
|
|
"status": self.status,
|
|
"msg": msg,
|
|
"ts": time.time(),
|
|
}
|
|
self.history.append(event)
|
|
self._q.put(event)
|
|
|
|
def stream(self):
|
|
# erst die bisherige Historie nachliefern, dann live weiter
|
|
for event in list(self.history):
|
|
yield event
|
|
while True:
|
|
try:
|
|
yield self._q.get(timeout=30)
|
|
except Empty:
|
|
if self.status in ("done", "failed", "aborted"):
|
|
return
|
|
yield {"phase": self.phase, "progress": self.progress, "status": self.status, "msg": None, "ts": time.time()}
|
|
|
|
def snapshot(self):
|
|
with self._lock:
|
|
return {
|
|
"id": self.id,
|
|
"job_type": self.job_type,
|
|
"status": self.status,
|
|
"phase": self.phase,
|
|
"progress": self.progress,
|
|
"error": self.error,
|
|
}
|
|
|
|
|
|
def create_job(product: dict, device_path: str = None, confirm_path: str = None, job_type: str = "stick") -> Job:
|
|
if job_type == "download":
|
|
job = Job(product, device_path=None, job_type="download")
|
|
JOBS[job.id] = job
|
|
t = threading.Thread(target=_run_job, args=(job,), daemon=True)
|
|
t.start()
|
|
return job
|
|
|
|
if job_type != "stick":
|
|
raise ValueError(f"Unbekannter job_type: {job_type}")
|
|
|
|
if confirm_path != device_path:
|
|
raise ValueError("Bestaetigungs-Pfad stimmt nicht mit dem Ziel-Geraet ueberein.")
|
|
|
|
sticks = list_usb_sticks()
|
|
match = next((s for s in sticks if s["path"] == device_path), None)
|
|
if not match:
|
|
raise ValueError(f"Geraet {device_path} nicht in der aktuellen USB-Liste gefunden.")
|
|
if match["is_root_disk"]:
|
|
raise ValueError("Verweigert: das ist die System-Platte des Hosts.")
|
|
if match["mounted"]:
|
|
raise ValueError("Geraet ist aktuell gemountet -- bitte vorher aushaengen.")
|
|
if not (match["transport"] == "usb"):
|
|
raise ValueError("Geraet ist nicht als USB-Transport erkannt -- Sicherheitsstop.")
|
|
|
|
job = Job(product, device_path, job_type="stick")
|
|
JOBS[job.id] = job
|
|
t = threading.Thread(target=_run_job, args=(job,), daemon=True)
|
|
t.start()
|
|
return job
|
|
|
|
|
|
def _run(job: Job, cmd: list[str], phase: str, progress_from_stderr=False):
|
|
log.info("job %s: %s", job.id, " ".join(cmd))
|
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
|
|
for line in proc.stdout:
|
|
line = line.strip()
|
|
if line:
|
|
job.emit(phase=phase, msg=line)
|
|
ret = proc.wait()
|
|
if ret != 0:
|
|
raise RuntimeError(f"Befehl fehlgeschlagen ({ret}): {' '.join(cmd)}")
|
|
|
|
|
|
def _download_to_library(job: Job, product: dict) -> str:
|
|
"""Laedt den Installer-Payload herunter und legt ihn dauerhaft in der
|
|
Bibliothek ab (nicht im temporaeren Scratch-Verzeichnis). Schreibt zuerst
|
|
in eine .part-Datei und benennt erst nach Erfolg um, damit ein
|
|
abgebrochener Download nicht als "vollstaendig" durchgeht."""
|
|
url = _payload_url(product)
|
|
entry_dir = library_entry_dir(product)
|
|
os.makedirs(entry_dir, exist_ok=True)
|
|
final_path = library_payload_path(product)
|
|
part_path = final_path + ".part"
|
|
|
|
job.emit(phase="download", progress=0, msg=f"Lade {url}")
|
|
with requests.get(url, stream=True, timeout=30) as r:
|
|
r.raise_for_status()
|
|
total = int(r.headers.get("Content-Length", 0)) or None
|
|
done = 0
|
|
last_pct = -1
|
|
with open(part_path, "wb") as f:
|
|
for chunk in r.iter_content(chunk_size=1024 * 1024 * 4):
|
|
if not chunk:
|
|
continue
|
|
f.write(chunk)
|
|
done += len(chunk)
|
|
if total:
|
|
pct = int(done * 100 / total)
|
|
if pct != last_pct:
|
|
job.emit(phase="download", progress=pct)
|
|
last_pct = pct
|
|
|
|
size_bytes = os.path.getsize(part_path)
|
|
os.replace(part_path, final_path)
|
|
with open(os.path.join(entry_dir, "meta.json"), "w") as f:
|
|
json.dump(
|
|
{
|
|
"id": product.get("id"),
|
|
"title": product.get("title"),
|
|
"version": product.get("version"),
|
|
"mode": product.get("mode"),
|
|
"source_url": url,
|
|
"filename": os.path.basename(final_path),
|
|
"size_bytes": size_bytes,
|
|
"downloaded_at": time.time(),
|
|
"complete": True,
|
|
},
|
|
f,
|
|
)
|
|
job.emit(phase="download", progress=100, msg=f"Download fertig, liegt dauerhaft unter {final_path}")
|
|
return final_path
|
|
|
|
|
|
def _dmg_to_raw(job: Job, dmg_path: str, work_dir: str) -> str:
|
|
raw_path = os.path.join(work_dir, os.path.basename(dmg_path) + ".img")
|
|
job.emit(phase="konvertieren", progress=0, msg="dmg2img: DMG -> rohes Image")
|
|
_run(job, ["dmg2img", "-i", dmg_path, "-o", raw_path], phase="konvertieren")
|
|
return raw_path
|
|
|
|
|
|
def _write_raw_to_device(job: Job, raw_path: str, device_path: str):
|
|
job.emit(phase="schreiben", progress=0, msg=f"Schreibe {raw_path} -> {device_path} (dd)")
|
|
_run(job, ["wipefs", "-a", device_path], phase="schreiben")
|
|
size = os.path.getsize(raw_path)
|
|
cmd = ["dd", f"if={raw_path}", f"of={device_path}", "bs=4M", "status=progress", "conv=fsync"]
|
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
|
|
for line in proc.stdout:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
job.emit(phase="schreiben", msg=line)
|
|
# dd status=progress schreibt Zeilen wie "1234567890 bytes (1.2 GB, 1.1 GiB) copied, ..."
|
|
try:
|
|
bytes_done = int(line.split(" ")[0])
|
|
pct = min(100, int(bytes_done * 100 / size))
|
|
job.emit(phase="schreiben", progress=pct)
|
|
except (ValueError, IndexError, ZeroDivisionError):
|
|
pass
|
|
ret = proc.wait()
|
|
if ret != 0:
|
|
raise RuntimeError("dd fehlgeschlagen")
|
|
job.emit(phase="schreiben", progress=100, msg="dd fertig, sync...")
|
|
subprocess.run(["sync"])
|
|
|
|
|
|
def _extract_recovery_dmg(job: Job, pkg_path: str, work_dir: str) -> str:
|
|
"""Experimentell: InstallAssistant.pkg -> xar -> pbzx -> cpio, sucht SharedSupport/BaseSystem.dmg."""
|
|
job.emit(phase="extrahieren", progress=0, msg="xar: InstallAssistant.pkg entpacken (experimentell)")
|
|
extract_dir = os.path.join(work_dir, "xar")
|
|
os.makedirs(extract_dir, exist_ok=True)
|
|
_run(job, ["xar", "-xf", pkg_path, "-C", extract_dir], phase="extrahieren")
|
|
|
|
payloads = []
|
|
for root, _dirs, files in os.walk(extract_dir):
|
|
for fn in files:
|
|
if fn == "Payload":
|
|
payloads.append(os.path.join(root, fn))
|
|
if not payloads:
|
|
raise RuntimeError("Kein Payload in InstallAssistant.pkg gefunden -- Apple-Struktur hat sich vermutlich geaendert.")
|
|
|
|
cpio_dir = os.path.join(work_dir, "cpio")
|
|
os.makedirs(cpio_dir, exist_ok=True)
|
|
for payload in payloads:
|
|
job.emit(phase="extrahieren", msg=f"pbzx+cpio: {payload}")
|
|
# pbzx dekomprimiert das payload, cpio packt den entstandenen Stream aus
|
|
pbzx_proc = subprocess.Popen(["pbzx", "-n", payload], stdout=subprocess.PIPE)
|
|
cpio_proc = subprocess.Popen(
|
|
["cpio", "-idm"], cwd=cpio_dir, stdin=pbzx_proc.stdout,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
|
)
|
|
pbzx_proc.stdout.close()
|
|
for line in cpio_proc.stdout:
|
|
line = line.strip()
|
|
if line:
|
|
job.emit(phase="extrahieren", msg=line)
|
|
cpio_proc.wait()
|
|
pbzx_proc.wait()
|
|
|
|
dmg_candidates = []
|
|
for root, _dirs, files in os.walk(cpio_dir):
|
|
for fn in files:
|
|
if fn in ("SharedSupport.dmg", "BaseSystem.dmg"):
|
|
dmg_candidates.append(os.path.join(root, fn))
|
|
if not dmg_candidates:
|
|
raise RuntimeError(
|
|
"Kein SharedSupport.dmg/BaseSystem.dmg im entpackten Installer gefunden. "
|
|
"Das ist der bekannte fragile Punkt bei macOS Big Sur+ ohne echten Mac."
|
|
)
|
|
# SharedSupport.dmg bevorzugen (enthaelt das vollstaendige Recovery-System)
|
|
dmg_candidates.sort(key=lambda p: 0 if "SharedSupport" in p else 1)
|
|
return dmg_candidates[0]
|
|
|
|
|
|
def _run_job(job: Job):
|
|
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
|
|
os.makedirs(LIBRARY_DIR, exist_ok=True)
|
|
# Scratch-Verzeichnis NUR fuer temporaere Konvertierungs-/Extraktionsdaten
|
|
# (dmg2img-Ausgabe, xar/cpio-Zwischenstand). Die eigentlichen
|
|
# Original-Downloads landen in LIBRARY_DIR und bleiben dauerhaft.
|
|
scratch_dir = tempfile.mkdtemp(dir=DOWNLOAD_DIR, prefix="scratch_")
|
|
try:
|
|
job.emit(status="running", phase="start", progress=0, msg="Job gestartet")
|
|
product = job.product
|
|
|
|
cached = get_cached_payload(product)
|
|
if cached:
|
|
payload_path = cached
|
|
job.emit(phase="download", progress=100, msg=f"Bereits lokal in der Bibliothek vorhanden -- Download uebersprungen ({cached}).")
|
|
else:
|
|
payload_path = _download_to_library(job, product)
|
|
|
|
if job.job_type == "download":
|
|
job.emit(status="done", phase="fertig", progress=100, msg="Image liegt jetzt dauerhaft in der lokalen Bibliothek (Offline nutzbar).")
|
|
return
|
|
|
|
if product["mode"] != "legacy_dmg":
|
|
job.emit(phase="hinweis", msg="Big Sur+ Pfad ist experimentell (siehe README).")
|
|
dmg_path = _extract_recovery_dmg(job, payload_path, scratch_dir)
|
|
else:
|
|
dmg_path = payload_path
|
|
|
|
raw_path = _dmg_to_raw(job, dmg_path, scratch_dir)
|
|
_write_raw_to_device(job, raw_path, job.device_path)
|
|
|
|
job.emit(status="done", phase="fertig", progress=100, msg="Stick ist fertig.")
|
|
except Exception as exc: # noqa: BLE001
|
|
log.exception("job %s failed", job.id)
|
|
job.error = str(exc)
|
|
job.emit(status="failed", msg=f"FEHLER: {exc}")
|
|
finally:
|
|
shutil.rmtree(scratch_dir, ignore_errors=True)
|