Files
akyldash/infra/backup/apply_restore.py
agent 6f809d3196
CI / tests-and-images (pull_request) Has been cancelled
Add guarded rollback and recovery apply tools
2026-10-11 19:30:22 +03:00

268 lines
10 KiB
Python

#!/usr/bin/env python3
"""Install a verified staging restore onto an explicitly marked new VPS."""
from __future__ import annotations
import json
import os
import shutil
import socket
import subprocess
import sys
from pathlib import Path
from archive import COMPONENTS, check_sqlite, digest
ROOT = Path("/opt/akyldash/current")
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
ENV_DEST = Path("/etc/akyldash/production.env")
SECRET_DEST = Path("/etc/akyldash/review_secret")
DATA_DEST = Path("/var/lib/akyldash/data")
MARKER = Path("/etc/akyldash/recovery-target")
PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
VOLUME_TARGETS = {
"/usr/share/opensearch/data": "opensearch-data",
"/data": "caddy-data",
"/config": "caddy-config",
}
def run(command: list[str], *, capture: bool = False) -> str:
result = subprocess.run(command, check=True, text=True, capture_output=capture)
return result.stdout.strip() if capture else ""
def require_empty_directory(path: Path) -> None:
if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
raise ValueError(f"restore destination must be an existing empty directory: {path}")
def tree_size(root: Path) -> int:
total = 0
for directory, names, files in os.walk(root, followlinks=False):
names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
for name in files:
path = Path(directory) / name
if path.is_symlink():
continue
if path.is_file():
total += path.stat().st_size
return total
def copy_contents(source: Path, destination: Path) -> None:
for child in source.iterdir():
target = destination / child.name
if child.is_dir() and not child.is_symlink():
shutil.copytree(child, target, symlinks=True)
else:
shutil.copy2(child, target, follow_symlinks=False)
def empty_directory(path: Path) -> None:
for child in path.iterdir():
if child.is_dir() and not child.is_symlink():
shutil.rmtree(child)
else:
child.unlink()
def validate_stage(staging: Path) -> dict:
staging = staging.resolve(strict=True)
if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
raise ValueError("staging directory must be a direct child of /var/tmp")
if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
raise ValueError("staging directory must be root-owned mode 0700")
manifest_path = staging / "manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if (
not isinstance(manifest, dict)
or manifest.get("format") != 1
or set(manifest.get("components", [])) != COMPONENTS
):
raise ValueError("staging manifest is invalid")
for name in COMPONENTS - {"production-env", "review-secret"}:
component = staging / name
if not component.is_dir() or component.is_symlink():
raise ValueError(f"staged component must be a directory: {name}")
for name in ("production-env", "review-secret"):
path = staging / name
if not path.is_file() or path.is_symlink():
raise ValueError(f"staged sensitive component must be a regular file: {name}")
return manifest
def validate_checkout(manifest: dict) -> None:
if ROOT.is_symlink() or not (ROOT / ".git").exists():
raise ValueError("approved Git checkout is missing or is a symlink")
if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
raise ValueError("recovery checkout must be clean")
unsafe = subprocess.run(
["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
if unsafe:
raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
if revision != manifest.get("git_revision"):
raise ValueError("checkout Git revision does not match the backup manifest")
for relative, expected in manifest.get("deployment_files", {}).items():
if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
raise ValueError("Compose or Caddyfile does not match the backup manifest")
if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
raise ValueError("backup manifest is missing deployment file hashes")
def validate_recovery_target() -> None:
if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
raise ValueError("root-only recovery-target marker is required")
hostname = socket.getfqdn()
addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
if not hostname or not addresses:
raise ValueError("cannot verify the recovery host identity")
if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
raise ValueError("refusing to apply recovery data to the inventoried production VPS")
def compose_configuration(staging: Path) -> dict:
rendered = run(
[
"docker", "compose", "--env-file", str(staging / "production-env"),
"-f", str(COMPOSE_FILE), "config", "--format", "json",
],
capture=True,
)
config = json.loads(rendered)
data_source = next(
mount["source"]
for mount in config["services"]["backend"]["volumes"]
if mount["target"] == "/app/data"
)
secret_source = config["secrets"]["review_secret"]["file"]
if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
raise ValueError("backup production paths do not match the clean recovery host layout")
return config
def expected_volume_names(config: dict) -> set[str]:
mounts = {
mount["target"]: mount
for service in config["services"].values()
for mount in service.get("volumes", [])
}
names: set[str] = set()
for target in VOLUME_TARGETS:
mount = mounts.get(target)
if not mount or mount["type"] != "volume":
raise ValueError(f"expected persistent volume at {target}")
names.add(config["volumes"][mount["source"]]["name"])
return names
def volume_destinations(config: dict) -> dict[str, Path]:
mounts = {
mount["target"]: mount
for service in config["services"].values()
for mount in service.get("volumes", [])
}
destinations = {}
for target, component in VOLUME_TARGETS.items():
mount = mounts.get(target)
if not mount or mount["type"] != "volume":
raise ValueError(f"expected persistent volume at {target}")
name = config["volumes"][mount["source"]]["name"]
destinations[component] = Path(
run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
)
return destinations
def apply_restore(staging: Path) -> None:
if os.geteuid() != 0:
raise ValueError("restore apply must run as root")
validate_recovery_target()
manifest = validate_stage(staging)
validate_checkout(manifest)
config = compose_configuration(staging)
require_empty_directory(DATA_DEST)
if ENV_DEST.exists() or SECRET_DEST.exists():
raise ValueError("production environment and secret files must not already exist")
if run(["docker", "ps", "--quiet"], capture=True):
raise ValueError("recovery target must have no running Docker containers")
volume_names = expected_volume_names(config)
for name in volume_names:
result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
if result.returncode == 0:
raise ValueError(f"recovery volume already exists: {name}")
created_volumes: list[str] = []
created_files: list[Path] = []
data_started = False
try:
for name in volume_names:
run(["docker", "volume", "create", name])
created_volumes.append(name)
destinations = volume_destinations(config)
destinations["data"] = DATA_DEST
required: dict[int, int] = {}
free: dict[int, int] = {}
for component, destination in destinations.items():
device = destination.stat().st_dev
required[device] = required.get(device, 0) + tree_size(staging / component)
free[device] = shutil.disk_usage(destination).free
for device, size in required.items():
if free[device] < size + 512 * 1024 * 1024:
raise ValueError("recovery host lacks space for a safe data copy")
for component, destination in destinations.items():
require_empty_directory(destination)
if component == "data":
data_started = True
copy_contents(staging / component, destination)
for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
os.chown(directory, 1000, 1000, follow_symlinks=False)
for name in files:
os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
shutil.copyfile(source, destination)
created_files.append(destination)
os.chown(destination, 0, 0)
os.chmod(destination, 0o600)
check_sqlite(DATA_DEST)
except BaseException:
for path in created_files:
path.unlink(missing_ok=True)
if data_started:
empty_directory(DATA_DEST)
for name in created_volumes:
subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
raise
MARKER.unlink()
print("PASS staged recovery data applied to the marked new host; services remain stopped")
def main() -> int:
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--staging-dir", required=True, type=Path)
args = parser.parse_args()
try:
apply_restore(args.staging_dir)
except Exception as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())