#!/usr/bin/env python3 """Install a verified staging restore onto an explicitly marked new VPS.""" from __future__ import annotations import json import os import shutil import socket import subprocess import sys from pathlib import Path from archive import COMPONENTS, check_sqlite, digest ROOT = Path("/opt/akyldash/current") COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" ENV_DEST = Path("/etc/akyldash/production.env") SECRET_DEST = Path("/etc/akyldash/review_secret") DATA_DEST = Path("/var/lib/akyldash/data") MARKER = Path("/etc/akyldash/recovery-target") PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile") VOLUME_TARGETS = { "/usr/share/opensearch/data": "opensearch-data", "/data": "caddy-data", "/config": "caddy-config", } def run(command: list[str], *, capture: bool = False) -> str: result = subprocess.run(command, check=True, text=True, capture_output=capture) return result.stdout.strip() if capture else "" def require_empty_directory(path: Path) -> None: if not path.is_dir() or path.is_symlink() or any(path.iterdir()): raise ValueError(f"restore destination must be an existing empty directory: {path}") def tree_size(root: Path) -> int: total = 0 for directory, names, files in os.walk(root, followlinks=False): names[:] = [name for name in names if not (Path(directory) / name).is_symlink()] for name in files: path = Path(directory) / name if path.is_symlink(): continue if path.is_file(): total += path.stat().st_size return total def copy_contents(source: Path, destination: Path) -> None: for child in source.iterdir(): target = destination / child.name if child.is_dir() and not child.is_symlink(): shutil.copytree(child, target, symlinks=True) else: shutil.copy2(child, target, follow_symlinks=False) def empty_directory(path: Path) -> None: for child in path.iterdir(): if child.is_dir() and not child.is_symlink(): shutil.rmtree(child) else: child.unlink() def validate_stage(staging: Path) -> dict: staging = staging.resolve(strict=True) if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"): raise ValueError("staging directory must be a direct child of /var/tmp") if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700: raise ValueError("staging directory must be root-owned mode 0700") manifest_path = staging / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if ( not isinstance(manifest, dict) or manifest.get("format") != 1 or set(manifest.get("components", [])) != COMPONENTS ): raise ValueError("staging manifest is invalid") for name in COMPONENTS - {"production-env", "review-secret"}: component = staging / name if not component.is_dir() or component.is_symlink(): raise ValueError(f"staged component must be a directory: {name}") for name in ("production-env", "review-secret"): path = staging / name if not path.is_file() or path.is_symlink(): raise ValueError(f"staged sensitive component must be a regular file: {name}") return manifest def validate_checkout(manifest: dict) -> None: if ROOT.is_symlink() or not (ROOT / ".git").exists(): raise ValueError("approved Git checkout is missing or is a symlink") if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip(): raise ValueError("recovery checkout must be clean") unsafe = subprocess.run( ["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"], check=True, capture_output=True, text=True, ).stdout.strip() if unsafe: raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}") revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True) if revision != manifest.get("git_revision"): raise ValueError("checkout Git revision does not match the backup manifest") for relative, expected in manifest.get("deployment_files", {}).items(): if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected: raise ValueError("Compose or Caddyfile does not match the backup manifest") if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES): raise ValueError("backup manifest is missing deployment file hashes") def validate_recovery_target() -> None: if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600: raise ValueError("root-only recovery-target marker is required") hostname = socket.getfqdn() addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True) if not hostname or not addresses: raise ValueError("cannot verify the recovery host identity") if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses: raise ValueError("refusing to apply recovery data to the inventoried production VPS") def compose_configuration(staging: Path) -> dict: rendered = run( [ "docker", "compose", "--env-file", str(staging / "production-env"), "-f", str(COMPOSE_FILE), "config", "--format", "json", ], capture=True, ) config = json.loads(rendered) data_source = next( mount["source"] for mount in config["services"]["backend"]["volumes"] if mount["target"] == "/app/data" ) secret_source = config["secrets"]["review_secret"]["file"] if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST: raise ValueError("backup production paths do not match the clean recovery host layout") return config def expected_volume_names(config: dict) -> set[str]: mounts = { mount["target"]: mount for service in config["services"].values() for mount in service.get("volumes", []) } names: set[str] = set() for target in VOLUME_TARGETS: mount = mounts.get(target) if not mount or mount["type"] != "volume": raise ValueError(f"expected persistent volume at {target}") names.add(config["volumes"][mount["source"]]["name"]) return names def volume_destinations(config: dict) -> dict[str, Path]: mounts = { mount["target"]: mount for service in config["services"].values() for mount in service.get("volumes", []) } destinations = {} for target, component in VOLUME_TARGETS.items(): mount = mounts.get(target) if not mount or mount["type"] != "volume": raise ValueError(f"expected persistent volume at {target}") name = config["volumes"][mount["source"]]["name"] destinations[component] = Path( run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True) ) return destinations def apply_restore(staging: Path) -> None: if os.geteuid() != 0: raise ValueError("restore apply must run as root") validate_recovery_target() manifest = validate_stage(staging) validate_checkout(manifest) config = compose_configuration(staging) require_empty_directory(DATA_DEST) if ENV_DEST.exists() or SECRET_DEST.exists(): raise ValueError("production environment and secret files must not already exist") if run(["docker", "ps", "--quiet"], capture=True): raise ValueError("recovery target must have no running Docker containers") volume_names = expected_volume_names(config) for name in volume_names: result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True) if result.returncode == 0: raise ValueError(f"recovery volume already exists: {name}") created_volumes: list[str] = [] created_files: list[Path] = [] data_started = False try: for name in volume_names: run(["docker", "volume", "create", name]) created_volumes.append(name) destinations = volume_destinations(config) destinations["data"] = DATA_DEST required: dict[int, int] = {} free: dict[int, int] = {} for component, destination in destinations.items(): device = destination.stat().st_dev required[device] = required.get(device, 0) + tree_size(staging / component) free[device] = shutil.disk_usage(destination).free for device, size in required.items(): if free[device] < size + 512 * 1024 * 1024: raise ValueError("recovery host lacks space for a safe data copy") for component, destination in destinations.items(): require_empty_directory(destination) if component == "data": data_started = True copy_contents(staging / component, destination) for directory, names, files in os.walk(destinations["opensearch"], followlinks=False): os.chown(directory, 1000, 1000, follow_symlinks=False) for name in files: os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False) for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)): shutil.copyfile(source, destination) created_files.append(destination) os.chown(destination, 0, 0) os.chmod(destination, 0o600) check_sqlite(DATA_DEST) except BaseException: for path in created_files: path.unlink(missing_ok=True) if data_started: empty_directory(DATA_DEST) for name in created_volumes: subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True) raise MARKER.unlink() print("PASS staged recovery data applied to the marked new host; services remain stopped") def main() -> int: import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--staging-dir", required=True, type=Path) args = parser.parse_args() try: apply_restore(args.staging_dir) except Exception as error: print(f"ERROR: {error}", file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())