245 lines
11 KiB
Python
245 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Restore prior local application images from a verified cold backup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path("/opt/akyldash/current")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_FILE = Path("/etc/akyldash/production.env")
|
|
STATE_FILE = Path("/opt/akyldash/release-state.json")
|
|
JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json")
|
|
DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt")
|
|
APP_SERVICES = {"backend", "opensearch"}
|
|
IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*")
|
|
IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}")
|
|
|
|
|
|
def is_digest_reference(reference: str) -> bool:
|
|
return IMAGE_DIGEST_RE.fullmatch(reference) is not None
|
|
|
|
|
|
def image_restore_plan(
|
|
manifest: dict,
|
|
configured: dict[str, str],
|
|
expected_previous: dict[str, dict[str, str]] | None = None,
|
|
) -> dict[str, tuple[str, str]]:
|
|
images = manifest.get("images")
|
|
if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
|
|
raise ValueError("backup has no previous backend and OpenSearch images")
|
|
plan = {}
|
|
for service in sorted(APP_SERVICES):
|
|
image = images[service]
|
|
if not isinstance(image, dict):
|
|
raise ValueError(f"invalid image record for {service}")
|
|
reference = image.get("configured")
|
|
image_id = image.get("image_id")
|
|
if expected_previous is None and reference != configured.get(service):
|
|
raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
|
|
if not isinstance(reference, str) or not (
|
|
IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference)
|
|
):
|
|
raise ValueError(f"invalid image reference for {service}")
|
|
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError(f"invalid previous image ID for {service}")
|
|
if expected_previous is not None and expected_previous.get(service) != {
|
|
"configured": reference, "image_id": image_id
|
|
}:
|
|
raise ValueError(f"backup does not match the pinned previous {service} release")
|
|
plan[service] = (image_id, reference)
|
|
return plan
|
|
|
|
|
|
def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
|
for image_id, reference in plan.values():
|
|
try:
|
|
actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
|
|
except subprocess.CalledProcessError as error:
|
|
raise ValueError(f"previous image is unavailable: {image_id}") from error
|
|
if actual != image_id:
|
|
raise ValueError(f"previous image is unavailable: {image_id}")
|
|
if is_digest_reference(reference):
|
|
try:
|
|
referenced_id = run(
|
|
["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True
|
|
)
|
|
except subprocess.CalledProcessError as error:
|
|
raise ValueError(f"previous digest reference is unavailable: {reference}") from error
|
|
if referenced_id != image_id:
|
|
raise ValueError(f"previous digest reference does not match its recorded image: {reference}")
|
|
|
|
|
|
def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None:
|
|
if any(is_digest_reference(reference) for _image_id, reference in plan.values()):
|
|
raise ValueError(
|
|
"rollback of digest-pinned images is not supported safely; no services or image references were changed"
|
|
)
|
|
|
|
|
|
def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
|
for image_id, reference in plan.values():
|
|
if not is_digest_reference(reference):
|
|
run(["docker", "image", "tag", image_id, reference])
|
|
|
|
|
|
def run(command: list[str], *, capture: bool = False) -> str:
|
|
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def compose(*args: str, capture: bool = False) -> str:
|
|
return run(
|
|
["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
|
|
capture=capture,
|
|
)
|
|
|
|
|
|
def record_backend_image() -> None:
|
|
container = compose("ps", "--quiet", "backend", capture=True)
|
|
if not container:
|
|
raise ValueError("backend container is missing after rollback")
|
|
image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError("backend returned an invalid image ID")
|
|
target = DEPLOYED_IMAGE_FILE
|
|
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
stream.write(f"{image_id}\n".encode("ascii"))
|
|
os.chmod(temporary, 0o600)
|
|
try:
|
|
os.replace(temporary, target)
|
|
directory_fd = os.open(target.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def running_image_ids() -> dict[str, str]:
|
|
result: dict[str, str] = {}
|
|
for service in sorted(APP_SERVICES):
|
|
container = compose("ps", "--all", "--quiet", service, capture=True)
|
|
if not container or "\n" in container:
|
|
raise ValueError(f"expected exactly one running {service} container")
|
|
inspection = run(
|
|
["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True
|
|
)
|
|
running, separator, image_id = inspection.partition("\t")
|
|
if not separator or running != "true" or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError(f"{service} container is not running with a valid image ID")
|
|
result[service] = image_id
|
|
return result
|
|
|
|
|
|
def require_running_image_ids(expected: dict[str, str], actual: dict[str, str], label: str) -> None:
|
|
if actual != expected:
|
|
raise ValueError(f"running image IDs do not match the {label}")
|
|
|
|
|
|
def rollback(backup: Path, data_compatible: bool) -> None:
|
|
raise ValueError("local VPS backup paths are disabled; use the owner-controlled external rollback session")
|
|
|
|
|
|
def rollback_external(
|
|
*, state: dict, target_identity: dict, current: dict, fresh_backup: dict,
|
|
data_compatible: bool,
|
|
) -> None:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("external rollback must run in the root-controlled receiver")
|
|
if not data_compatible:
|
|
raise ValueError("confirm data compatibility before external rollback")
|
|
from deploy_release import image_ids
|
|
from release_refs import read_images, replace_images
|
|
from release_state import read_json, validate_backup_identity, validate_state, write_json_atomic
|
|
|
|
pinned = validate_state(state)
|
|
if pinned.get("schema_version") != 2 or pinned.get("previous_backup") is None:
|
|
raise ValueError("release state has no verified external previous backup")
|
|
target_identity = validate_backup_identity(target_identity)
|
|
fresh_backup = validate_backup_identity(fresh_backup)
|
|
if target_identity != pinned["previous_backup"]:
|
|
raise ValueError("rollback archive is not the root-pinned previous external backup")
|
|
if image_ids(allow_stopped=True) != current:
|
|
raise ValueError("stopped production images differ from the pre-rollback snapshot")
|
|
current_files = {
|
|
relative: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()
|
|
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
}
|
|
if target_identity["manifest"].get("deployment_files") != current_files:
|
|
raise ValueError("previous backup deployment files differ from the current checkout")
|
|
if app_images(target_identity["manifest"], "previous external backup") != pinned["previous"]:
|
|
raise ValueError("previous backup image identities do not match root-pinned release state")
|
|
if app_images(fresh_backup["manifest"], "pre-rollback external backup") != current:
|
|
raise ValueError("fresh external backup does not match the current deployment")
|
|
|
|
env_text, configured_values = read_images(ENV_FILE)
|
|
del env_text
|
|
configured = {service: configured_values[f"{service.upper()}_IMAGE"] for service in APP_SERVICES}
|
|
plan = image_restore_plan(target_identity["manifest"], configured, pinned["previous"])
|
|
verify_previous_images(plan)
|
|
before = {"BACKEND_IMAGE": configured["backend"], "OPENSEARCH_IMAGE": configured["opensearch"]}
|
|
after = {"BACKEND_IMAGE": plan["backend"][1], "OPENSEARCH_IMAGE": plan["opensearch"][1]}
|
|
journal = read_json(JOURNAL_FILE)
|
|
if journal.get("status") != "external-backup-verified" or journal.get("backup") != fresh_backup:
|
|
raise ValueError("fresh external backup journal is missing or does not match this rollback")
|
|
|
|
try:
|
|
tag_previous_images(plan)
|
|
replace_images(ENV_FILE, before, after)
|
|
journal["status"] = "rollback-references-switched"
|
|
journal["rollback_from"] = target_identity
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
expected = {service: plan[service][0] for service in APP_SERVICES}
|
|
require_running_image_ids(expected, running_image_ids(), "verified previous external backup")
|
|
run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
|
|
"https://akyldash.kg/review", "--output", "/dev/null"])
|
|
record_backend_image()
|
|
restored = {
|
|
service: {"configured": plan[service][1], "image_id": plan[service][0]}
|
|
for service in APP_SERVICES
|
|
}
|
|
write_json_atomic(STATE_FILE, {
|
|
"schema_version": 2, "commit": "rollback", "manifest_sha256": "rollback",
|
|
"current": restored, "previous": current, "previous_backup": fresh_backup,
|
|
})
|
|
JOURNAL_FILE.unlink()
|
|
except Exception as error:
|
|
journal["status"] = "rollback-failed"
|
|
journal["error_type"] = type(error).__name__
|
|
try:
|
|
_text, current_env = read_images(ENV_FILE)
|
|
active = {"BACKEND_IMAGE": current_env["BACKEND_IMAGE"], "OPENSEARCH_IMAGE": current_env["OPENSEARCH_IMAGE"]}
|
|
if active == after:
|
|
replace_images(ENV_FILE, after, before)
|
|
elif active != before:
|
|
raise ValueError("image references match neither the current nor rollback release")
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
require_running_image_ids({service: current[service]["image_id"] for service in APP_SERVICES},
|
|
running_image_ids(), "pre-rollback deployment")
|
|
record_backend_image()
|
|
write_json_atomic(STATE_FILE, pinned)
|
|
journal["recovery"] = "pre-rollback references and image IDs restored"
|
|
except Exception as recovery_error:
|
|
journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__})"
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"external rollback failed; recovery journal preserved at {JOURNAL_FILE}") from error
|
|
|
|
|
|
def main() -> int:
|
|
print("ERROR: local VPS backup paths are disabled; use the owner-controlled external rollback session", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|