#!/usr/bin/env python3 """Restore prior local application images from a verified cold backup.""" from __future__ import annotations import argparse import datetime as dt import json import os import re import subprocess import sys import tempfile from pathlib import Path ROOT = Path("/opt/akyldash/current") COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" ENV_FILE = Path("/etc/akyldash/production.env") BACKUP_DIR = Path("/var/backups/akyldash") BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") APP_SERVICES = {"backend", "opensearch"} IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*") IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}") def is_digest_reference(reference: str) -> bool: return IMAGE_DIGEST_RE.fullmatch(reference) is not None def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]: images = manifest.get("images") if not isinstance(images, dict) or not APP_SERVICES <= images.keys(): raise ValueError("backup has no previous backend and OpenSearch images") plan = {} for service in sorted(APP_SERVICES): image = images[service] if not isinstance(image, dict): raise ValueError(f"invalid image record for {service}") reference = image.get("configured") image_id = image.get("image_id") if reference != configured.get(service): raise ValueError(f"Compose image reference changed for {service}; refusing rollback") if not isinstance(reference, str) or not ( IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference) ): raise ValueError(f"invalid image reference for {service}") if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"invalid previous image ID for {service}") plan[service] = (image_id, reference) return plan def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None: for image_id, reference in plan.values(): try: actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True) except subprocess.CalledProcessError as error: raise ValueError(f"previous image is unavailable: {image_id}") from error if actual != image_id: raise ValueError(f"previous image is unavailable: {image_id}") if is_digest_reference(reference): try: referenced_id = run( ["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True ) except subprocess.CalledProcessError as error: raise ValueError(f"previous digest reference is unavailable: {reference}") from error if referenced_id != image_id: raise ValueError(f"previous digest reference does not match its recorded image: {reference}") def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None: if any(is_digest_reference(reference) for _image_id, reference in plan.values()): raise ValueError( "rollback of digest-pinned images is not supported safely; no services or image references were changed" ) def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None: require_tag_rollback(plan) for image_id, reference in plan.values(): run(["docker", "image", "tag", image_id, reference]) def run(command: list[str], *, capture: bool = False) -> str: result = subprocess.run(command, check=True, text=True, capture_output=capture) return result.stdout.strip() if capture else "" def compose(*args: str, capture: bool = False) -> str: return run( ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture=capture, ) def record_backend_image() -> None: container = compose("ps", "--quiet", "backend", capture=True) if not container: raise ValueError("backend container is missing after rollback") image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True) if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError("backend returned an invalid image ID") target = Path("/opt/akyldash/deployed-image.txt") with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream: temporary = Path(stream.name) stream.write(f"{image_id}\n".encode("ascii")) os.chmod(temporary, 0o600) try: os.replace(temporary, target) finally: temporary.unlink(missing_ok=True) def rollback(backup: Path, data_compatible: bool) -> None: if not data_compatible: raise ValueError("confirm data compatibility with --data-compatible before rollback") if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve(): raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}") if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600: raise ValueError("rollback backup must be root-owned mode 0600") checksum = Path(f"{backup}.sha256") if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600: raise ValueError("rollback checksum sidecar must be root-owned mode 0600") sys.path.insert(0, str(BACKUP_TOOL.parent)) from archive import digest, verify_archive manifest = verify_archive(backup) deployment_files = { relative: digest(ROOT / relative) for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") } if deployment_files != manifest.get("deployment_files"): raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first") rendered = json.loads(compose("config", "--format", "json", capture=True)) configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES} plan = image_restore_plan(manifest, configured) verify_previous_images(plan) require_tag_rollback(plan) backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz" backup_output = BACKUP_DIR / backup_name compose("stop") try: run( [ "python3", str(BACKUP_TOOL), "create", "--compose-file", str(COMPOSE_FILE), "--env-file", str(ENV_FILE), "--output", str(backup_output), ] ) run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)]) except Exception: compose("up", "--detach") raise tag_previous_images(plan) compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") run( [ "curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", "https://akyldash.kg/review", "--output", "/dev/null", ] ) record_backend_image() print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}") def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("backup", type=Path) parser.add_argument("--data-compatible", action="store_true") args = parser.parse_args() try: if os.geteuid() != 0: raise ValueError("rollback must run through the restricted root helper") rollback(args.backup, args.data_compatible) except Exception as error: print(f"ERROR: {error}", file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())