"""Kernlogik: Gaeste auswaehlen, Snapshots anlegen, alte Snapshots aufraeumen.""" from __future__ import annotations import fnmatch import logging import time from dataclasses import dataclass, field from datetime import datetime from .mail import send_space_alert from .naming import build_name, effective_slug, parse_name, render_description from .proxmox import ProxmoxError, guest_volumes from .snapfs import human_bytes from .util import free_space_bytes, parse_free_space log = logging.getLogger("pvesnap.engine") # --------------------------------------------------------------------------- # Auswahl der Gaeste # --------------------------------------------------------------------------- def _matches_any_glob(text, patterns): lowered = (text or "").lower() return any(fnmatch.fnmatch(lowered, pattern.lower()) for pattern in patterns) def select_guests(group, guests): """Alle Gaeste, auf die die Auswahlregeln der Gruppe zutreffen.""" wanted_vmids = set(group.vmids) wanted_tags = set(group.tags) wanted_pools = set(p.lower() for p in group.pools) excluded_vmids = set(group.exclude_vmids) excluded_tags = set(group.exclude_tags) selected = [] for guest in guests: if group.types and guest.type not in group.types: continue if group.all: hit = True else: hit = (guest.vmid in wanted_vmids or (group.names and _matches_any_glob(guest.name, group.names)) or (wanted_tags and wanted_tags.intersection(guest.tags)) or (wanted_pools and guest.pool.lower() in wanted_pools)) if not hit: continue if guest.vmid in excluded_vmids: continue if group.exclude_names and _matches_any_glob(guest.name, group.exclude_names): continue if excluded_tags and excluded_tags.intersection(guest.tags): continue selected.append(guest) return selected # --------------------------------------------------------------------------- # Vorhaltezeit # --------------------------------------------------------------------------- def managed_snapshots(prefix, group, snapshots): """Nur die Snapshots, die pvesnap fuer *diese* Gruppe angelegt hat.""" slug = effective_slug(prefix, group.slug) result = [] for snap in snapshots: parsed = parse_name(prefix, snap.name) if not parsed or parsed["slug"] != slug: continue created = (datetime.fromtimestamp(snap.snaptime) if snap.snaptime else parsed["created"]) result.append((snap, created)) result.sort(key=lambda item: item[1], reverse=True) # neueste zuerst return result def plan_prune(prefix, group, snapshots, now): """Welche Snapshots sollen weg? Gibt eine Liste von (Snapshot, Grund) zurueck.""" doomed = [] for snap, _created in managed_snapshots(prefix, group, snapshots): if snap.snapstate: # Halbfertiger Eintrag aus einem abgebrochenen Lauf - der belegt # nur die Konfiguration und laesst sich nicht oeffnen. doomed.append((snap, "unvollstaendig (%s)" % snap.snapstate)) for index, (snap, created) in enumerate(managed_snapshots(prefix, group, snapshots)): if snap.snapstate: continue # schon oben eingeplant if index < group.keep_min: continue age = (now - created).total_seconds() if group.keep_count > 0 and index >= group.keep_count: doomed.append((snap, "Anzahl > %d" % group.keep_count)) elif group.keep_time > 0 and age > group.keep_time: doomed.append((snap, "aelter als Vorhaltezeit")) return doomed def plan_reclaim(config, group, snapshots): """Was bei Platzmangel weg darf, in der Reihenfolge des Loeschens. Anders als die Vorhaltezeit zaehlt das je Gast ueber alle pvesnap-Snapshots, auch die von Gruppen, die es nicht mehr gibt. Ein voller Pool ist schlimmer als ein verlorener alter Stand, und gerade die aeltesten halten am meisten Platz fest. Getrennt gezaehlt koennte eine Gruppe mit keep_count unter space_keep nie etwas freigeben. Halbfertige Eintraege zuerst - sie enthalten nichts. Danach die aeltesten, bis nur noch die neuesten space_keep uebrig sind. Das keep_min jeder Gruppe bleibt dabei gewahrt. """ prefix = config.globals.prefix managed = [] for snap in snapshots: parsed = parse_name(prefix, snap.name) if parsed: created = (datetime.fromtimestamp(snap.snaptime) if snap.snaptime else parsed["created"]) managed.append((created, snap)) managed.sort(key=lambda item: item[0], reverse=True) # neueste zuerst broken = [snap for _created, snap in managed if snap.snapstate] complete = [snap for _created, snap in managed if not snap.snapstate] kept = {snap.name for snap in complete[:group.space_keep]} for other in config.groups: own = [snap for snap, _created in managed_snapshots(prefix, other, complete)] kept.update(snap.name for snap in own[:other.keep_min]) return broken[::-1] + [snap for snap in complete[::-1] if snap.name not in kept] # --------------------------------------------------------------------------- # Platz vor dem Anlegen # --------------------------------------------------------------------------- SPACE_POLL = 5 # Sekunden zwischen zwei Blicken auf das Storage def _space_demand(proxmox, guest, group): """Je Storage des Gastes der Platz, den der Snapshot sofort belegt. Fuer die meisten Platten ist das nichts - ein Snapshot waechst erst mit dem, was danach geschrieben wird. Sofort kosten nur dick angelegte ZFS-Zvols und der mitgesicherte RAM. """ config = proxmox.guest_config(guest) demand = {} for volid in guest_volumes(config, guest.type): name = volid.split(":", 1)[0] demand[name] = demand.get(name, 0) + proxmox.snapshot_cost(guest.node, volid) if group.vmstate and guest.running and guest.type == "qemu": ram = int(proxmox.guest_status(guest).get("maxmem") or 0) target = config.get("vmstatestorage") for name in ([target] if target else list(demand)): demand[name] = demand.get(name, 0) + ram return demand def _space_shortages(proxmox, guest, group, demand): """Die Storages, auf denen es nicht reicht: [(Name, frei, gebraucht)]. Gebraucht wird, was der Snapshot sofort belegt, plus die Reserve, die danach noch frei bleiben muss. """ shortages = [] for name, extra in sorted(demand.items()): space = proxmox.storage_space(guest.node, name) if space is None: continue free, total = space reserve = free_space_bytes(group.space_min_free, total) needed = reserve + extra log.log(logging.INFO if proxmox.dry_run else logging.DEBUG, "%s: '%s': %s frei, Snapshot belegt sofort %s, danach frei %s, " "Reserve %s", guest.label, name, human_bytes(free), human_bytes(extra), human_bytes(max(0, free - extra)), human_bytes(reserve)) if free < needed: shortages.append((name, free, needed)) return shortages def _describe_shortages(shortages): return ", ".join("'%s': %s frei, %s gebraucht" % (name, human_bytes(free), human_bytes(needed)) for name, free, needed in shortages) def _wait_for_space(proxmox, guest, group, demand, settle, limit): """Nach dem Loeschen abwarten, bis der Platz wirklich zurueckgekommen ist. Ceph und ZFS geben ihn erst im Hintergrund frei. Saehe man sofort nach, fehlte er scheinbar noch - und der naechste Snapshot wuerde umsonst geloescht. Gewartet wird deshalb, solange der freie Platz noch waechst, und aufgegeben erst, wenn sich `settle` Sekunden lang nichts mehr tut. """ shortages = _space_shortages(proxmox, guest, group, demand) best = sum(free for _name, free, _needed in shortages) started = last_growth = time.monotonic() while (shortages and time.monotonic() - last_growth < settle and time.monotonic() - started < limit): time.sleep(min(SPACE_POLL, settle)) shortages = _space_shortages(proxmox, guest, group, demand) free_now = sum(free for _name, free, _needed in shortages) if free_now > best: best = free_now last_growth = time.monotonic() return shortages def _ensure_space(proxmox, config, group, guest, result, may_delete=True): """Sorgt vor dem Snapshot fuer Platz. False = dieser Gast wird ausgelassen. Geloescht wird einzeln und immer nur so viel, bis es reicht. Laesst sich der Platz nicht feststellen, bleibt alles stehen und der Snapshot wird versucht: Auf Verdacht wird nichts geloescht. """ if parse_free_space(group.space_min_free) == "0": return True try: demand = _space_demand(proxmox, guest, group) shortages = _space_shortages(proxmox, guest, group, demand) if not shortages: return True snapshots = proxmox.list_snapshots(guest) except ProxmoxError as exc: log.warning("Gruppe '%s': Platz fuer %s nicht pruefbar: %s", group.name, guest.label, exc) return True log.warning("Gruppe '%s': %s: zu wenig Platz (%s)", group.name, guest.label, _describe_shortages(shortages)) candidates = plan_reclaim(config, group, snapshots) if may_delete else [] for snap in candidates: try: proxmox.delete_snapshot(guest, snap.name, force=bool(snap.snapstate)) except ProxmoxError as exc: message = "%s: '%s' nicht loeschbar: %s" % (guest.label, snap.name, exc) result.errors.append(message) log.error("Gruppe '%s': %s", group.name, message) continue result.deleted.append("%s:%s" % (guest.vmid, snap.name)) log.info("Gruppe '%s': Snapshot '%s' von %s geloescht (Platzmangel)", group.name, snap.name, guest.label) if proxmox.dry_run: continue shortages = _wait_for_space(proxmox, guest, group, demand, config.globals.space_settle, config.globals.task_timeout) if not shortages: log.info("Gruppe '%s': %s: Platz reicht jetzt", group.name, guest.label) return True if proxmox.dry_run and candidates: log.info("[TESTLAUF] %s: ein echter Lauf loescht davon nur, bis der Platz " "reicht", guest.label) return True message = ("%s: kein Snapshot angelegt, zu wenig Platz (%s) und nichts mehr " "loeschbar" % (guest.label, _describe_shortages(shortages))) result.errors.append(message) result.no_space.append(message) log.error("Gruppe '%s': %s", group.name, message) return False # --------------------------------------------------------------------------- # Ausfuehrung # --------------------------------------------------------------------------- @dataclass class GroupResult: group: str matched: int = 0 created: list = field(default_factory=list) deleted: list = field(default_factory=list) skipped: list = field(default_factory=list) errors: list = field(default_factory=list) no_space: list = field(default_factory=list) # Abbrueche wegen Platzmangels @property def ok(self): return not self.errors def summary(self): return ("Gruppe '%s': %d Gast/Gaeste, %d Snapshot(s) angelegt, " "%d geloescht, %d Fehler" % (self.group, self.matched, len(self.created), len(self.deleted), len(self.errors))) def run_group(proxmox, config, group, now=None, create=True, prune=True, guests=None): """Legt fuer eine Gruppe Snapshots an und raeumt alte weg.""" now = now or datetime.now() prefix = config.globals.prefix result = GroupResult(group=group.name) if guests is None: guests = proxmox.inventory() selected = select_guests(group, guests) result.matched = len(selected) if not selected: log.warning("Gruppe '%s': keine passenden Gaeste gefunden", group.name) return result snapshot_name = build_name(prefix, group.slug, now) template = group.description or config.globals.description pause = config.globals.pause_between first = True for guest in selected: # Kurz durchatmen zwischen zwei Gaesten - entlastet den Storage-Lock, # wenn viele VMs auf demselben Storage liegen. if pause and not first and not config.globals.dry_run: time.sleep(pause) first = False if group.skip_stopped and not guest.running: result.skipped.append("%s (gestoppt)" % guest.label) log.info("Gruppe '%s': %s uebersprungen (gestoppt)", group.name, guest.label) continue if create: if not _ensure_space(proxmox, config, group, guest, result, may_delete=prune): continue description = render_description(template, group, guest, now, prefix) try: proxmox.create_snapshot( guest, snapshot_name, description, vmstate=group.vmstate and guest.running, ) result.created.append("%s:%s" % (guest.vmid, snapshot_name)) log.info("Gruppe '%s': Snapshot '%s' fuer %s angelegt", group.name, snapshot_name, guest.label) except ProxmoxError as exc: message = "%s: Snapshot fehlgeschlagen: %s" % (guest.label, exc) result.errors.append(message) log.error("Gruppe '%s': %s", group.name, message) continue # ohne neuen Snapshot nicht aufraeumen if prune: try: _prune_guest(proxmox, config, group, guest, now, result) except ProxmoxError as exc: message = "%s: Aufraeumen fehlgeschlagen: %s" % (guest.label, exc) result.errors.append(message) log.error("Gruppe '%s': %s", group.name, message) log.info(result.summary()) if result.no_space: send_space_alert(config, group.name, result.no_space) return result def _prune_guest(proxmox, config, group, guest, now, result): prefix = config.globals.prefix snapshots = proxmox.list_snapshots(guest) for snap, reason in plan_prune(prefix, group, snapshots, now): try: proxmox.delete_snapshot(guest, snap.name, force=bool(snap.snapstate)) result.deleted.append("%s:%s" % (guest.vmid, snap.name)) log.info("Gruppe '%s': Snapshot '%s' von %s geloescht (%s)", group.name, snap.name, guest.label, reason) except ProxmoxError as exc: message = "%s: '%s' nicht loeschbar: %s" % (guest.label, snap.name, exc) result.errors.append(message) log.error("Gruppe '%s': %s", group.name, message)