Files
akyldash/infra/deploy/rollback.py

200 lines
7.8 KiB
Python

#!/usr/bin/env python3
"""Restore prior local application images from a verified cold backup."""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path("/opt/akyldash/current")
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
ENV_FILE = Path("/etc/akyldash/production.env")
BACKUP_DIR = Path("/var/backups/akyldash")
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
APP_SERVICES = {"backend", "opensearch"}
IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*")
IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}")
def is_digest_reference(reference: str) -> bool:
return IMAGE_DIGEST_RE.fullmatch(reference) is not None
def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
images = manifest.get("images")
if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
raise ValueError("backup has no previous backend and OpenSearch images")
plan = {}
for service in sorted(APP_SERVICES):
image = images[service]
if not isinstance(image, dict):
raise ValueError(f"invalid image record for {service}")
reference = image.get("configured")
image_id = image.get("image_id")
if reference != configured.get(service):
raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
if not isinstance(reference, str) or not (
IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference)
):
raise ValueError(f"invalid image reference for {service}")
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError(f"invalid previous image ID for {service}")
plan[service] = (image_id, reference)
return plan
def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None:
for image_id, reference in plan.values():
try:
actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
except subprocess.CalledProcessError as error:
raise ValueError(f"previous image is unavailable: {image_id}") from error
if actual != image_id:
raise ValueError(f"previous image is unavailable: {image_id}")
if is_digest_reference(reference):
try:
referenced_id = run(
["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True
)
except subprocess.CalledProcessError as error:
raise ValueError(f"previous digest reference is unavailable: {reference}") from error
if referenced_id != image_id:
raise ValueError(f"previous digest reference does not match its recorded image: {reference}")
def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None:
if any(is_digest_reference(reference) for _image_id, reference in plan.values()):
raise ValueError(
"rollback of digest-pinned images is not supported safely; no services or image references were changed"
)
def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None:
require_tag_rollback(plan)
for image_id, reference in plan.values():
run(["docker", "image", "tag", image_id, reference])
def run(command: list[str], *, capture: bool = False) -> str:
result = subprocess.run(command, check=True, text=True, capture_output=capture)
return result.stdout.strip() if capture else ""
def compose(*args: str, capture: bool = False) -> str:
return run(
["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
capture=capture,
)
def record_backend_image() -> None:
container = compose("ps", "--quiet", "backend", capture=True)
if not container:
raise ValueError("backend container is missing after rollback")
image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError("backend returned an invalid image ID")
target = Path("/opt/akyldash/deployed-image.txt")
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
temporary = Path(stream.name)
stream.write(f"{image_id}\n".encode("ascii"))
os.chmod(temporary, 0o600)
try:
os.replace(temporary, target)
finally:
temporary.unlink(missing_ok=True)
def rollback(backup: Path, data_compatible: bool) -> None:
if not data_compatible:
raise ValueError("confirm data compatibility with --data-compatible before rollback")
if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
raise ValueError("rollback backup must be root-owned mode 0600")
checksum = Path(f"{backup}.sha256")
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
sys.path.insert(0, str(BACKUP_TOOL.parent))
from archive import digest, verify_archive
manifest = verify_archive(backup)
deployment_files = {
relative: digest(ROOT / relative)
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
}
if deployment_files != manifest.get("deployment_files"):
raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
rendered = json.loads(compose("config", "--format", "json", capture=True))
configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
plan = image_restore_plan(manifest, configured)
verify_previous_images(plan)
require_tag_rollback(plan)
backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
backup_output = BACKUP_DIR / backup_name
compose("stop")
try:
run(
[
"python3",
str(BACKUP_TOOL),
"create",
"--compose-file",
str(COMPOSE_FILE),
"--env-file",
str(ENV_FILE),
"--output",
str(backup_output),
]
)
run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
except Exception:
compose("up", "--detach")
raise
tag_previous_images(plan)
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
run(
[
"curl",
"--fail",
"--silent",
"--show-error",
"--resolve",
"akyldash.kg:443:127.0.0.1",
"https://akyldash.kg/review",
"--output",
"/dev/null",
]
)
record_backend_image()
print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("backup", type=Path)
parser.add_argument("--data-compatible", action="store_true")
args = parser.parse_args()
try:
if os.geteuid() != 0:
raise ValueError("rollback must run through the restricted root helper")
rollback(args.backup, args.data_compatible)
except Exception as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())