268 lines
10 KiB
Python
268 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Install a verified staging restore onto an explicitly marked new VPS."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from archive import COMPONENTS, check_sqlite, digest
|
|
|
|
ROOT = Path("/opt/akyldash/current")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_DEST = Path("/etc/akyldash/production.env")
|
|
SECRET_DEST = Path("/etc/akyldash/review_secret")
|
|
DATA_DEST = Path("/var/lib/akyldash/data")
|
|
MARKER = Path("/etc/akyldash/recovery-target")
|
|
PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
VOLUME_TARGETS = {
|
|
"/usr/share/opensearch/data": "opensearch-data",
|
|
"/data": "caddy-data",
|
|
"/config": "caddy-config",
|
|
}
|
|
|
|
|
|
def run(command: list[str], *, capture: bool = False) -> str:
|
|
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def require_empty_directory(path: Path) -> None:
|
|
if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
|
|
raise ValueError(f"restore destination must be an existing empty directory: {path}")
|
|
|
|
|
|
def tree_size(root: Path) -> int:
|
|
total = 0
|
|
for directory, names, files in os.walk(root, followlinks=False):
|
|
names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
|
|
for name in files:
|
|
path = Path(directory) / name
|
|
if path.is_symlink():
|
|
continue
|
|
if path.is_file():
|
|
total += path.stat().st_size
|
|
return total
|
|
|
|
|
|
def copy_contents(source: Path, destination: Path) -> None:
|
|
for child in source.iterdir():
|
|
target = destination / child.name
|
|
if child.is_dir() and not child.is_symlink():
|
|
shutil.copytree(child, target, symlinks=True)
|
|
else:
|
|
shutil.copy2(child, target, follow_symlinks=False)
|
|
|
|
|
|
def empty_directory(path: Path) -> None:
|
|
for child in path.iterdir():
|
|
if child.is_dir() and not child.is_symlink():
|
|
shutil.rmtree(child)
|
|
else:
|
|
child.unlink()
|
|
|
|
|
|
def validate_stage(staging: Path) -> dict:
|
|
staging = staging.resolve(strict=True)
|
|
if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
|
|
raise ValueError("staging directory must be a direct child of /var/tmp")
|
|
if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
|
|
raise ValueError("staging directory must be root-owned mode 0700")
|
|
manifest_path = staging / "manifest.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if (
|
|
not isinstance(manifest, dict)
|
|
or manifest.get("format") != 1
|
|
or set(manifest.get("components", [])) != COMPONENTS
|
|
):
|
|
raise ValueError("staging manifest is invalid")
|
|
for name in COMPONENTS - {"production-env", "review-secret"}:
|
|
component = staging / name
|
|
if not component.is_dir() or component.is_symlink():
|
|
raise ValueError(f"staged component must be a directory: {name}")
|
|
for name in ("production-env", "review-secret"):
|
|
path = staging / name
|
|
if not path.is_file() or path.is_symlink():
|
|
raise ValueError(f"staged sensitive component must be a regular file: {name}")
|
|
return manifest
|
|
|
|
|
|
def validate_checkout(manifest: dict) -> None:
|
|
if ROOT.is_symlink() or not (ROOT / ".git").exists():
|
|
raise ValueError("approved Git checkout is missing or is a symlink")
|
|
if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
|
|
raise ValueError("recovery checkout must be clean")
|
|
unsafe = subprocess.run(
|
|
["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
if unsafe:
|
|
raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
|
|
revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
|
|
if revision != manifest.get("git_revision"):
|
|
raise ValueError("checkout Git revision does not match the backup manifest")
|
|
for relative, expected in manifest.get("deployment_files", {}).items():
|
|
if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
|
|
raise ValueError("Compose or Caddyfile does not match the backup manifest")
|
|
if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
|
|
raise ValueError("backup manifest is missing deployment file hashes")
|
|
|
|
|
|
def validate_recovery_target() -> None:
|
|
if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
|
|
raise ValueError("root-only recovery-target marker is required")
|
|
hostname = socket.getfqdn()
|
|
addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
|
|
if not hostname or not addresses:
|
|
raise ValueError("cannot verify the recovery host identity")
|
|
if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
|
|
raise ValueError("refusing to apply recovery data to the inventoried production VPS")
|
|
|
|
|
|
def compose_configuration(staging: Path) -> dict:
|
|
rendered = run(
|
|
[
|
|
"docker", "compose", "--env-file", str(staging / "production-env"),
|
|
"-f", str(COMPOSE_FILE), "config", "--format", "json",
|
|
],
|
|
capture=True,
|
|
)
|
|
config = json.loads(rendered)
|
|
data_source = next(
|
|
mount["source"]
|
|
for mount in config["services"]["backend"]["volumes"]
|
|
if mount["target"] == "/app/data"
|
|
)
|
|
secret_source = config["secrets"]["review_secret"]["file"]
|
|
if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
|
|
raise ValueError("backup production paths do not match the clean recovery host layout")
|
|
return config
|
|
|
|
|
|
def expected_volume_names(config: dict) -> set[str]:
|
|
mounts = {
|
|
mount["target"]: mount
|
|
for service in config["services"].values()
|
|
for mount in service.get("volumes", [])
|
|
}
|
|
names: set[str] = set()
|
|
for target in VOLUME_TARGETS:
|
|
mount = mounts.get(target)
|
|
if not mount or mount["type"] != "volume":
|
|
raise ValueError(f"expected persistent volume at {target}")
|
|
names.add(config["volumes"][mount["source"]]["name"])
|
|
return names
|
|
|
|
|
|
def volume_destinations(config: dict) -> dict[str, Path]:
|
|
mounts = {
|
|
mount["target"]: mount
|
|
for service in config["services"].values()
|
|
for mount in service.get("volumes", [])
|
|
}
|
|
destinations = {}
|
|
for target, component in VOLUME_TARGETS.items():
|
|
mount = mounts.get(target)
|
|
if not mount or mount["type"] != "volume":
|
|
raise ValueError(f"expected persistent volume at {target}")
|
|
name = config["volumes"][mount["source"]]["name"]
|
|
destinations[component] = Path(
|
|
run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
|
|
)
|
|
return destinations
|
|
|
|
|
|
def apply_restore(staging: Path) -> None:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("restore apply must run as root")
|
|
validate_recovery_target()
|
|
manifest = validate_stage(staging)
|
|
validate_checkout(manifest)
|
|
config = compose_configuration(staging)
|
|
require_empty_directory(DATA_DEST)
|
|
if ENV_DEST.exists() or SECRET_DEST.exists():
|
|
raise ValueError("production environment and secret files must not already exist")
|
|
if run(["docker", "ps", "--quiet"], capture=True):
|
|
raise ValueError("recovery target must have no running Docker containers")
|
|
|
|
volume_names = expected_volume_names(config)
|
|
for name in volume_names:
|
|
result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
|
|
if result.returncode == 0:
|
|
raise ValueError(f"recovery volume already exists: {name}")
|
|
|
|
created_volumes: list[str] = []
|
|
created_files: list[Path] = []
|
|
data_started = False
|
|
try:
|
|
for name in volume_names:
|
|
run(["docker", "volume", "create", name])
|
|
created_volumes.append(name)
|
|
|
|
destinations = volume_destinations(config)
|
|
destinations["data"] = DATA_DEST
|
|
|
|
required: dict[int, int] = {}
|
|
free: dict[int, int] = {}
|
|
for component, destination in destinations.items():
|
|
device = destination.stat().st_dev
|
|
required[device] = required.get(device, 0) + tree_size(staging / component)
|
|
free[device] = shutil.disk_usage(destination).free
|
|
for device, size in required.items():
|
|
if free[device] < size + 512 * 1024 * 1024:
|
|
raise ValueError("recovery host lacks space for a safe data copy")
|
|
|
|
for component, destination in destinations.items():
|
|
require_empty_directory(destination)
|
|
if component == "data":
|
|
data_started = True
|
|
copy_contents(staging / component, destination)
|
|
|
|
for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
|
|
os.chown(directory, 1000, 1000, follow_symlinks=False)
|
|
for name in files:
|
|
os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
|
|
|
|
for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
|
|
shutil.copyfile(source, destination)
|
|
created_files.append(destination)
|
|
os.chown(destination, 0, 0)
|
|
os.chmod(destination, 0o600)
|
|
check_sqlite(DATA_DEST)
|
|
except BaseException:
|
|
for path in created_files:
|
|
path.unlink(missing_ok=True)
|
|
if data_started:
|
|
empty_directory(DATA_DEST)
|
|
for name in created_volumes:
|
|
subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
|
|
raise
|
|
|
|
MARKER.unlink()
|
|
print("PASS staged recovery data applied to the marked new host; services remain stopped")
|
|
|
|
|
|
def main() -> int:
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--staging-dir", required=True, type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
apply_restore(args.staging_dir)
|
|
except Exception as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|