Images aus Fremd-Registries holen, loeschen und Speicher aufraeumen
Tab "Images" um Verwaltungsfunktionen erweitert: - Einzel-Pull: Image aus anderer Registry (z. B. Docker Hub) per skopeo in die eigene Registry kopieren, optional mit Benutzername/Kennwort oder Access Token fuer private Quell-Registries - Bulk-Pull: alle image-Eintraege einer docker-compose (Textfeld oder Datei-Upload) auf einmal holen; Variablen/eigene Registry werden ignoriert - Digest-Dedup: bereits mit gleichem Digest vorhandene Images werden uebersprungen (kopiert mit --all, damit Multi-Arch-Digests matchen) - Loeschen: Muelltonne hinter jedem Tag mit Rueckfrage - Garbage Collection: gibt belegten Speicher frei, laeuft per registry-Binary (Multi-Stage-Build) direkt auf dem gemounteten Speicher, kein Docker-Socket gunicorn: 1 Worker (gemeinsames Service-Passwort) + 8 Threads, Timeout 1800s fuer lange Kopiervorgaenge. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+337
@@ -1,10 +1,13 @@
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from functools import wraps
|
||||
|
||||
import bcrypt as bc
|
||||
import requests as http_client
|
||||
import yaml
|
||||
from flask import (
|
||||
Flask,
|
||||
flash,
|
||||
@@ -22,7 +25,10 @@ ADMIN_USER = os.environ.get("ADMIN_USER", "admin")
|
||||
ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "changeme")
|
||||
DB_PATH = os.environ.get("DB_PATH", "/data/users.db")
|
||||
HTPASSWD_PATH = os.environ.get("HTPASSWD_PATH", "/auth/htpasswd")
|
||||
GC_CONFIG = os.environ.get("GC_CONFIG", "/app/registry-gc-config.yml")
|
||||
REGISTRY_URL = os.environ.get("REGISTRY_URL", "http://registry:5000")
|
||||
# Host:Port ohne Schema, fuer skopeo-Ziel (z. B. "registry:5000")
|
||||
REGISTRY_HOST = REGISTRY_URL.split("://", 1)[-1]
|
||||
|
||||
# Internal service account for registry API queries (not stored in SQLite)
|
||||
_SERVICE_USER = "_service"
|
||||
@@ -86,6 +92,180 @@ def query_registry(path):
|
||||
return None
|
||||
|
||||
|
||||
# Erlaubte Zeichen in einer Image-Referenz (verhindert Shell-/Argument-Injection)
|
||||
_IMAGE_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@-]*$")
|
||||
|
||||
|
||||
def strip_registry_host(ref):
|
||||
"""Entfernt einen optionalen Registry-Host (z. B. docker.io/, ghcr.io:443/)."""
|
||||
if "/" in ref:
|
||||
first, rest = ref.split("/", 1)
|
||||
if "." in first or ":" in first or first == "localhost":
|
||||
return rest
|
||||
return ref
|
||||
|
||||
|
||||
def derive_dest(source):
|
||||
"""Leitet den Ziel-Namen in unserer Registry aus der Quell-Referenz ab.
|
||||
|
||||
Beispiele:
|
||||
nginx -> nginx:latest
|
||||
bitnami/redis:7 -> bitnami/redis:7
|
||||
ghcr.io/owner/app:v1 -> owner/app:v1
|
||||
"""
|
||||
dest = strip_registry_host(source)
|
||||
last_segment = dest.rsplit("/", 1)[-1]
|
||||
if ":" not in last_segment and "@" not in last_segment:
|
||||
dest += ":latest"
|
||||
return dest
|
||||
|
||||
|
||||
def split_dest(dest):
|
||||
"""Teilt einen Ziel-Namen "repo:tag" in (repo, tag). dest hat immer einen Tag."""
|
||||
repo, _, tag = dest.rpartition(":")
|
||||
return repo, tag
|
||||
|
||||
|
||||
def extract_compose_images(content):
|
||||
"""Liest alle 'image:'-Eintraege aus den Services einer docker-compose.
|
||||
|
||||
Gibt (images, ignored) zurueck. images ist None bei ungueltigem YAML.
|
||||
Eintraege mit Variablen (${...}) oder auf die eigene Registry werden
|
||||
uebersprungen und in ignored gesammelt.
|
||||
"""
|
||||
try:
|
||||
data = yaml.safe_load(content)
|
||||
except yaml.YAMLError:
|
||||
return None, []
|
||||
if not isinstance(data, dict):
|
||||
return None, []
|
||||
services = data.get("services")
|
||||
if not isinstance(services, dict):
|
||||
return [], []
|
||||
|
||||
images, ignored = [], []
|
||||
for svc in services.values():
|
||||
if not isinstance(svc, dict):
|
||||
continue
|
||||
image = svc.get("image")
|
||||
if not isinstance(image, str) or not image.strip():
|
||||
continue
|
||||
image = image.strip()
|
||||
host = image.split("/", 1)[0] if "/" in image else ""
|
||||
if "$" in image or "{" in image:
|
||||
ignored.append(image) # nicht aufgeloeste Variable
|
||||
elif host and host in (REGISTRY_HOST, request.host):
|
||||
ignored.append(image) # zeigt bereits auf unsere Registry
|
||||
elif not _IMAGE_REF_RE.match(image):
|
||||
ignored.append(image)
|
||||
elif image not in images:
|
||||
images.append(image)
|
||||
return images, ignored
|
||||
|
||||
|
||||
_MANIFEST_ACCEPT = ", ".join(
|
||||
[
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def get_manifest_digest(name, reference):
|
||||
"""Liefert den Manifest-Digest eines Tags (fuer das Loeschen benoetigt)."""
|
||||
try:
|
||||
resp = http_client.get(
|
||||
f"{REGISTRY_URL}/v2/{name}/manifests/{reference}",
|
||||
auth=(_SERVICE_USER, _SERVICE_PASSWORD),
|
||||
headers={"Accept": _MANIFEST_ACCEPT},
|
||||
timeout=5,
|
||||
)
|
||||
if resp.ok:
|
||||
return resp.headers.get("Docker-Content-Digest")
|
||||
except http_client.RequestException:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def delete_manifest(name, digest):
|
||||
"""Loescht ein Manifest per Digest aus der Registry."""
|
||||
try:
|
||||
resp = http_client.delete(
|
||||
f"{REGISTRY_URL}/v2/{name}/manifests/{digest}",
|
||||
auth=(_SERVICE_USER, _SERVICE_PASSWORD),
|
||||
timeout=5,
|
||||
)
|
||||
return resp.status_code in (200, 202)
|
||||
except http_client.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def remote_digest(source, src_creds=None, src_token=None):
|
||||
"""Liefert den Manifest-Digest eines Quell-Images, ohne es herunterzuladen."""
|
||||
cmd = ["skopeo", "inspect", "--format", "{{.Digest}}", f"docker://{source}"]
|
||||
if src_token:
|
||||
cmd += ["--registry-token", src_token]
|
||||
elif src_creds:
|
||||
cmd += ["--creds", src_creds]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def copy_image(source, src_creds=None, src_token=None):
|
||||
"""Kopiert ein Image in unsere Registry.
|
||||
|
||||
Gibt (status, info) zurueck:
|
||||
("pulled", dest) - kopiert
|
||||
("skipped", dest) - bereits mit gleichem Digest vorhanden
|
||||
("error", grund) - fehlgeschlagen
|
||||
"""
|
||||
if not _IMAGE_REF_RE.match(source):
|
||||
return ("error", f"{source}: ungueltige Image-Referenz")
|
||||
|
||||
dest = derive_dest(source)
|
||||
dest_name, dest_tag = split_dest(dest)
|
||||
|
||||
# Doppel-Upload vermeiden: gleicher Digest schon vorhanden? (z. B. bei "latest")
|
||||
digest = remote_digest(source, src_creds, src_token)
|
||||
if digest and get_manifest_digest(dest_name, dest_tag) == digest:
|
||||
return ("skipped", dest)
|
||||
|
||||
# --all kopiert alle Architekturen und erhaelt die Manifest-Liste,
|
||||
# damit der Digest mit der Quelle uebereinstimmt.
|
||||
cmd = [
|
||||
"skopeo",
|
||||
"copy",
|
||||
"--all",
|
||||
"--dest-tls-verify=false",
|
||||
f"docker://{source}",
|
||||
f"docker://{REGISTRY_HOST}/{dest}",
|
||||
"--dest-creds",
|
||||
f"{_SERVICE_USER}:{_SERVICE_PASSWORD}",
|
||||
]
|
||||
if src_token:
|
||||
cmd += ["--src-registry-token", src_token]
|
||||
elif src_creds:
|
||||
cmd += ["--src-creds", src_creds]
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=1800)
|
||||
if result.returncode == 0:
|
||||
return ("pulled", dest)
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
return ("error", f"{source}: {detail[-1] if detail else 'Unbekannter Fehler'}")
|
||||
except subprocess.TimeoutExpired:
|
||||
return ("error", f"{source}: Zeitueberschreitung")
|
||||
except FileNotFoundError:
|
||||
return ("error", "skopeo ist nicht installiert")
|
||||
|
||||
|
||||
def login_required(f):
|
||||
@wraps(f)
|
||||
def decorated(*args, **kwargs):
|
||||
@@ -138,6 +318,163 @@ def images():
|
||||
return render_template("images.html", repos=repos)
|
||||
|
||||
|
||||
@app.route("/pull", methods=["POST"])
|
||||
@login_required
|
||||
def pull_image():
|
||||
source = request.form.get("source", "").strip()
|
||||
use_creds = request.form.get("use_creds") == "on"
|
||||
use_token = request.form.get("use_token") == "on"
|
||||
src_user = request.form.get("src_username", "").strip()
|
||||
src_pass = request.form.get("src_password", "")
|
||||
src_token = request.form.get("src_token", "").strip()
|
||||
|
||||
if not source:
|
||||
flash("Bitte ein Quell-Image angeben.", "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
# Quell-Zugangsdaten nur anwenden, wenn die jeweilige Checkbox aktiviert ist.
|
||||
creds = f"{src_user}:{src_pass}" if (use_creds and src_user) else None
|
||||
token = src_token if (use_token and src_token) else None
|
||||
|
||||
status, info = copy_image(source, creds, token)
|
||||
if status == "pulled":
|
||||
flash(f'Image "{source}" wurde als "{info}" geholt.', "success")
|
||||
elif status == "skipped":
|
||||
flash(
|
||||
f'Image "{info}" ist bereits mit gleichem Digest vorhanden - uebersprungen.',
|
||||
"success",
|
||||
)
|
||||
else:
|
||||
flash(f"Fehler beim Holen: {info}", "error")
|
||||
|
||||
return redirect(url_for("images"))
|
||||
|
||||
|
||||
@app.route("/pull-compose", methods=["POST"])
|
||||
@login_required
|
||||
def pull_compose():
|
||||
content = request.form.get("compose", "")
|
||||
uploaded = request.files.get("compose_file")
|
||||
if uploaded and uploaded.filename:
|
||||
try:
|
||||
content = uploaded.read().decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
flash("Die hochgeladene Datei ist keine gueltige Textdatei.", "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
if not content.strip():
|
||||
flash("Bitte eine docker-compose einfuegen oder hochladen.", "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
images_list, ignored = extract_compose_images(content)
|
||||
if images_list is None:
|
||||
flash("Die docker-compose konnte nicht gelesen werden (ungueltiges YAML).", "error")
|
||||
return redirect(url_for("images"))
|
||||
if not images_list:
|
||||
flash("Keine verwertbaren Images in der docker-compose gefunden.", "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
use_creds = request.form.get("bulk_use_creds") == "on"
|
||||
use_token = request.form.get("bulk_use_token") == "on"
|
||||
src_user = request.form.get("bulk_src_username", "").strip()
|
||||
src_pass = request.form.get("bulk_src_password", "")
|
||||
src_token = request.form.get("bulk_src_token", "").strip()
|
||||
creds = f"{src_user}:{src_pass}" if (use_creds and src_user) else None
|
||||
token = src_token if (use_token and src_token) else None
|
||||
|
||||
pulled, skipped, errors = [], [], []
|
||||
for image in images_list:
|
||||
status, info = copy_image(image, creds, token)
|
||||
if status == "pulled":
|
||||
pulled.append(info)
|
||||
elif status == "skipped":
|
||||
skipped.append(info)
|
||||
else:
|
||||
errors.append(info)
|
||||
|
||||
if pulled:
|
||||
flash(f"{len(pulled)} Image(s) geholt: " + ", ".join(pulled), "success")
|
||||
if skipped:
|
||||
flash(
|
||||
f"{len(skipped)} bereits vorhanden (gleicher Digest), uebersprungen: "
|
||||
+ ", ".join(skipped),
|
||||
"success",
|
||||
)
|
||||
if ignored:
|
||||
flash(
|
||||
f"{len(ignored)} Eintrag/Eintraege ignoriert (Variable oder eigene Registry): "
|
||||
+ ", ".join(ignored),
|
||||
"error",
|
||||
)
|
||||
for err in errors:
|
||||
flash(f"Fehler: {err}", "error")
|
||||
|
||||
return redirect(url_for("images"))
|
||||
|
||||
|
||||
@app.route("/images/delete", methods=["POST"])
|
||||
@login_required
|
||||
def delete_image():
|
||||
name = request.form.get("name", "").strip()
|
||||
tag = request.form.get("tag", "").strip()
|
||||
|
||||
if not name or not tag:
|
||||
flash("Image und Tag sind erforderlich.", "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
digest = get_manifest_digest(name, tag)
|
||||
if not digest:
|
||||
flash(f'Tag "{tag}" von "{name}" wurde nicht gefunden.', "error")
|
||||
return redirect(url_for("images"))
|
||||
|
||||
if delete_manifest(name, digest):
|
||||
flash(f'Image "{name}:{tag}" wurde geloescht.', "success")
|
||||
else:
|
||||
flash(f'Image "{name}:{tag}" konnte nicht geloescht werden.', "error")
|
||||
|
||||
return redirect(url_for("images"))
|
||||
|
||||
|
||||
@app.route("/images/gc", methods=["POST"])
|
||||
@login_required
|
||||
def garbage_collect():
|
||||
# Entfernt Blobs/Layer, die von keinem Manifest mehr referenziert werden
|
||||
# (z. B. nach dem Loeschen von Tags). Nutzt die registry-Binary direkt auf
|
||||
# dem gemounteten Speicher - kein Docker-Socket noetig.
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["registry", "garbage-collect", GC_CONFIG, "--delete-untagged"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
freed = sum(
|
||||
1
|
||||
for line in result.stdout.splitlines()
|
||||
if line.startswith("blob eligible for deletion")
|
||||
)
|
||||
flash(
|
||||
"Garbage Collection abgeschlossen. "
|
||||
f"{freed} nicht mehr benoetigte Blob(s) wurden entfernt.",
|
||||
"success",
|
||||
)
|
||||
else:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
reason = detail[-1] if detail else "Unbekannter Fehler"
|
||||
# Frische, leere Registry: nichts zu tun
|
||||
if "Path not found" in reason and "repositories" in reason:
|
||||
flash("Garbage Collection: keine Images vorhanden, nichts zu tun.", "success")
|
||||
else:
|
||||
flash(f"Fehler bei der Garbage Collection: {reason}", "error")
|
||||
except subprocess.TimeoutExpired:
|
||||
flash("Zeitueberschreitung bei der Garbage Collection.", "error")
|
||||
except FileNotFoundError:
|
||||
flash("registry-Binary ist nicht verfuegbar.", "error")
|
||||
|
||||
return redirect(url_for("images"))
|
||||
|
||||
|
||||
@app.route("/help")
|
||||
@login_required
|
||||
def help_page():
|
||||
|
||||
Reference in New Issue
Block a user