200 lines
7.8 KiB
Python
200 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Restore prior local application images from a verified cold backup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path("/opt/akyldash/current")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_FILE = Path("/etc/akyldash/production.env")
|
|
BACKUP_DIR = Path("/var/backups/akyldash")
|
|
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
|
|
APP_SERVICES = {"backend", "opensearch"}
|
|
IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*")
|
|
IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}")
|
|
|
|
|
|
def is_digest_reference(reference: str) -> bool:
|
|
return IMAGE_DIGEST_RE.fullmatch(reference) is not None
|
|
|
|
|
|
def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
|
|
images = manifest.get("images")
|
|
if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
|
|
raise ValueError("backup has no previous backend and OpenSearch images")
|
|
plan = {}
|
|
for service in sorted(APP_SERVICES):
|
|
image = images[service]
|
|
if not isinstance(image, dict):
|
|
raise ValueError(f"invalid image record for {service}")
|
|
reference = image.get("configured")
|
|
image_id = image.get("image_id")
|
|
if reference != configured.get(service):
|
|
raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
|
|
if not isinstance(reference, str) or not (
|
|
IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference)
|
|
):
|
|
raise ValueError(f"invalid image reference for {service}")
|
|
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError(f"invalid previous image ID for {service}")
|
|
plan[service] = (image_id, reference)
|
|
return plan
|
|
|
|
|
|
def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
|
for image_id, reference in plan.values():
|
|
try:
|
|
actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
|
|
except subprocess.CalledProcessError as error:
|
|
raise ValueError(f"previous image is unavailable: {image_id}") from error
|
|
if actual != image_id:
|
|
raise ValueError(f"previous image is unavailable: {image_id}")
|
|
if is_digest_reference(reference):
|
|
try:
|
|
referenced_id = run(
|
|
["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True
|
|
)
|
|
except subprocess.CalledProcessError as error:
|
|
raise ValueError(f"previous digest reference is unavailable: {reference}") from error
|
|
if referenced_id != image_id:
|
|
raise ValueError(f"previous digest reference does not match its recorded image: {reference}")
|
|
|
|
|
|
def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None:
|
|
if any(is_digest_reference(reference) for _image_id, reference in plan.values()):
|
|
raise ValueError(
|
|
"rollback of digest-pinned images is not supported safely; no services or image references were changed"
|
|
)
|
|
|
|
|
|
def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
|
require_tag_rollback(plan)
|
|
for image_id, reference in plan.values():
|
|
run(["docker", "image", "tag", image_id, reference])
|
|
|
|
|
|
def run(command: list[str], *, capture: bool = False) -> str:
|
|
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def compose(*args: str, capture: bool = False) -> str:
|
|
return run(
|
|
["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
|
|
capture=capture,
|
|
)
|
|
|
|
|
|
def record_backend_image() -> None:
|
|
container = compose("ps", "--quiet", "backend", capture=True)
|
|
if not container:
|
|
raise ValueError("backend container is missing after rollback")
|
|
image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError("backend returned an invalid image ID")
|
|
target = Path("/opt/akyldash/deployed-image.txt")
|
|
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
stream.write(f"{image_id}\n".encode("ascii"))
|
|
os.chmod(temporary, 0o600)
|
|
try:
|
|
os.replace(temporary, target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def rollback(backup: Path, data_compatible: bool) -> None:
|
|
if not data_compatible:
|
|
raise ValueError("confirm data compatibility with --data-compatible before rollback")
|
|
if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
|
|
raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
|
|
if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
|
|
raise ValueError("rollback backup must be root-owned mode 0600")
|
|
checksum = Path(f"{backup}.sha256")
|
|
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
|
|
raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
|
|
|
|
sys.path.insert(0, str(BACKUP_TOOL.parent))
|
|
from archive import digest, verify_archive
|
|
|
|
manifest = verify_archive(backup)
|
|
deployment_files = {
|
|
relative: digest(ROOT / relative)
|
|
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
}
|
|
if deployment_files != manifest.get("deployment_files"):
|
|
raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
|
|
rendered = json.loads(compose("config", "--format", "json", capture=True))
|
|
configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
|
|
plan = image_restore_plan(manifest, configured)
|
|
verify_previous_images(plan)
|
|
require_tag_rollback(plan)
|
|
|
|
backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
|
|
backup_output = BACKUP_DIR / backup_name
|
|
compose("stop")
|
|
try:
|
|
run(
|
|
[
|
|
"python3",
|
|
str(BACKUP_TOOL),
|
|
"create",
|
|
"--compose-file",
|
|
str(COMPOSE_FILE),
|
|
"--env-file",
|
|
str(ENV_FILE),
|
|
"--output",
|
|
str(backup_output),
|
|
]
|
|
)
|
|
run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
|
|
except Exception:
|
|
compose("up", "--detach")
|
|
raise
|
|
|
|
tag_previous_images(plan)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
run(
|
|
[
|
|
"curl",
|
|
"--fail",
|
|
"--silent",
|
|
"--show-error",
|
|
"--resolve",
|
|
"akyldash.kg:443:127.0.0.1",
|
|
"https://akyldash.kg/review",
|
|
"--output",
|
|
"/dev/null",
|
|
]
|
|
)
|
|
record_backend_image()
|
|
print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("backup", type=Path)
|
|
parser.add_argument("--data-compatible", action="store_true")
|
|
args = parser.parse_args()
|
|
try:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("rollback must run through the restricted root helper")
|
|
rollback(args.backup, args.data_compatible)
|
|
except Exception as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|