#!/usr/bin/env python3 """Restore prior local application images from a verified cold backup.""" from __future__ import annotations import hashlib import os import re import subprocess import sys import tempfile from pathlib import Path ROOT = Path("/opt/akyldash/current") COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" ENV_FILE = Path("/etc/akyldash/production.env") STATE_FILE = Path("/opt/akyldash/release-state.json") JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json") DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt") APP_SERVICES = {"backend", "opensearch"} IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*") IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}") def is_digest_reference(reference: str) -> bool: return IMAGE_DIGEST_RE.fullmatch(reference) is not None def image_restore_plan( manifest: dict, configured: dict[str, str], expected_previous: dict[str, dict[str, str]] | None = None, ) -> dict[str, tuple[str, str]]: images = manifest.get("images") if not isinstance(images, dict) or not APP_SERVICES <= images.keys(): raise ValueError("backup has no previous backend and OpenSearch images") plan = {} for service in sorted(APP_SERVICES): image = images[service] if not isinstance(image, dict): raise ValueError(f"invalid image record for {service}") reference = image.get("configured") image_id = image.get("image_id") if expected_previous is None and reference != configured.get(service): raise ValueError(f"Compose image reference changed for {service}; refusing rollback") if not isinstance(reference, str) or not ( IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference) ): raise ValueError(f"invalid image reference for {service}") if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"invalid previous image ID for {service}") if expected_previous is not None and expected_previous.get(service) != { "configured": reference, "image_id": image_id }: raise ValueError(f"backup does not match the pinned previous {service} release") plan[service] = (image_id, reference) return plan def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None: for image_id, reference in plan.values(): try: actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True) except subprocess.CalledProcessError as error: raise ValueError(f"previous image is unavailable: {image_id}") from error if actual != image_id: raise ValueError(f"previous image is unavailable: {image_id}") if is_digest_reference(reference): try: referenced_id = run( ["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True ) except subprocess.CalledProcessError as error: raise ValueError(f"previous digest reference is unavailable: {reference}") from error if referenced_id != image_id: raise ValueError(f"previous digest reference does not match its recorded image: {reference}") def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None: if any(is_digest_reference(reference) for _image_id, reference in plan.values()): raise ValueError( "rollback of digest-pinned images is not supported safely; no services or image references were changed" ) def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None: for image_id, reference in plan.values(): if not is_digest_reference(reference): run(["docker", "image", "tag", image_id, reference]) def run(command: list[str], *, capture: bool = False) -> str: result = subprocess.run(command, check=True, text=True, capture_output=capture) return result.stdout.strip() if capture else "" def compose(*args: str, capture: bool = False) -> str: return run( ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture=capture, ) def record_backend_image() -> None: container = compose("ps", "--quiet", "backend", capture=True) if not container: raise ValueError("backend container is missing after rollback") image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True) if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError("backend returned an invalid image ID") target = DEPLOYED_IMAGE_FILE with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream: temporary = Path(stream.name) stream.write(f"{image_id}\n".encode("ascii")) os.chmod(temporary, 0o600) try: os.replace(temporary, target) directory_fd = os.open(target.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: temporary.unlink(missing_ok=True) def running_image_ids() -> dict[str, str]: result: dict[str, str] = {} for service in sorted(APP_SERVICES): container = compose("ps", "--all", "--quiet", service, capture=True) if not container or "\n" in container: raise ValueError(f"expected exactly one running {service} container") inspection = run( ["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True ) running, separator, image_id = inspection.partition("\t") if not separator or running != "true" or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"{service} container is not running with a valid image ID") result[service] = image_id return result def require_running_image_ids(expected: dict[str, str], actual: dict[str, str], label: str) -> None: if actual != expected: raise ValueError(f"running image IDs do not match the {label}") def rollback(backup: Path, data_compatible: bool) -> None: raise ValueError("local VPS backup paths are disabled; use the owner-controlled external rollback session") def rollback_external( *, state: dict, target_identity: dict, current: dict, fresh_backup: dict, data_compatible: bool, ) -> None: if os.geteuid() != 0: raise ValueError("external rollback must run in the root-controlled receiver") if not data_compatible: raise ValueError("confirm data compatibility before external rollback") from deploy_release import image_ids from release_refs import read_images, replace_images from release_state import read_json, validate_backup_identity, validate_state, write_json_atomic pinned = validate_state(state) if pinned.get("schema_version") != 2 or pinned.get("previous_backup") is None: raise ValueError("release state has no verified external previous backup") target_identity = validate_backup_identity(target_identity) fresh_backup = validate_backup_identity(fresh_backup) if target_identity != pinned["previous_backup"]: raise ValueError("rollback archive is not the root-pinned previous external backup") if image_ids(allow_stopped=True) != current: raise ValueError("stopped production images differ from the pre-rollback snapshot") current_files = { relative: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") } if target_identity["manifest"].get("deployment_files") != current_files: raise ValueError("previous backup deployment files differ from the current checkout") if app_images(target_identity["manifest"], "previous external backup") != pinned["previous"]: raise ValueError("previous backup image identities do not match root-pinned release state") if app_images(fresh_backup["manifest"], "pre-rollback external backup") != current: raise ValueError("fresh external backup does not match the current deployment") env_text, configured_values = read_images(ENV_FILE) del env_text configured = {service: configured_values[f"{service.upper()}_IMAGE"] for service in APP_SERVICES} plan = image_restore_plan(target_identity["manifest"], configured, pinned["previous"]) verify_previous_images(plan) before = {"BACKEND_IMAGE": configured["backend"], "OPENSEARCH_IMAGE": configured["opensearch"]} after = {"BACKEND_IMAGE": plan["backend"][1], "OPENSEARCH_IMAGE": plan["opensearch"][1]} journal = read_json(JOURNAL_FILE) if journal.get("status") != "external-backup-verified" or journal.get("backup") != fresh_backup: raise ValueError("fresh external backup journal is missing or does not match this rollback") try: tag_previous_images(plan) replace_images(ENV_FILE, before, after) journal["status"] = "rollback-references-switched" journal["rollback_from"] = target_identity write_json_atomic(JOURNAL_FILE, journal) compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") expected = {service: plan[service][0] for service in APP_SERVICES} require_running_image_ids(expected, running_image_ids(), "verified previous external backup") run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", "https://akyldash.kg/review", "--output", "/dev/null"]) record_backend_image() restored = { service: {"configured": plan[service][1], "image_id": plan[service][0]} for service in APP_SERVICES } write_json_atomic(STATE_FILE, { "schema_version": 2, "commit": "rollback", "manifest_sha256": "rollback", "current": restored, "previous": current, "previous_backup": fresh_backup, }) JOURNAL_FILE.unlink() except Exception as error: journal["status"] = "rollback-failed" journal["error_type"] = type(error).__name__ try: _text, current_env = read_images(ENV_FILE) active = {"BACKEND_IMAGE": current_env["BACKEND_IMAGE"], "OPENSEARCH_IMAGE": current_env["OPENSEARCH_IMAGE"]} if active == after: replace_images(ENV_FILE, after, before) elif active != before: raise ValueError("image references match neither the current nor rollback release") compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") require_running_image_ids({service: current[service]["image_id"] for service in APP_SERVICES}, running_image_ids(), "pre-rollback deployment") record_backend_image() write_json_atomic(STATE_FILE, pinned) journal["recovery"] = "pre-rollback references and image IDs restored" except Exception as recovery_error: journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__})" write_json_atomic(JOURNAL_FILE, journal) raise ValueError(f"external rollback failed; recovery journal preserved at {JOURNAL_FILE}") from error def main() -> int: print("ERROR: local VPS backup paths are disabled; use the owner-controlled external rollback session", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())