#!/usr/bin/env python3 """Owner-run, locked production release cutover using a verified cold backup.""" from __future__ import annotations import argparse import hashlib import json import os import re import subprocess import sys import tempfile import urllib.parse from pathlib import Path from create_release_manifest import BUILDER_IMAGE from deployment_lock import deployment_lock from release_refs import read_images, replace_images from release_state import read_json, validate_images, validate_state, write_json_atomic from validate_release import read_json as read_manifest_json, validate_manifest ROOT = Path("/opt/akyldash/current") COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" ENV_FILE = Path("/etc/akyldash/production.env") BACKUP_DIR = Path("/var/backups/akyldash") BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") STATE_FILE = Path("/opt/akyldash/release-state.json") JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json") DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt") SERVICES = ("backend", "opensearch") SEARCH_CANARY_URL = "https://akyldash.kg/search" def run(command: list[str], capture: bool = False) -> str: result = subprocess.run(command, check=True, text=True, capture_output=capture) return result.stdout.strip() if capture else "" def compose(*args: str, capture: bool = False) -> str: return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture) def image_ids(*, allow_stopped: bool = False) -> dict[str, dict[str, str]]: configured = read_images(ENV_FILE)[1] records = {} for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")): container = compose("ps", "--all", "--quiet", service, capture=True) if not container or "\n" in container: raise ValueError(f"expected exactly one existing {service} container") inspection = run( ["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True ) running, separator, image_id = inspection.partition("\t") if not separator or (running != "true" and not (allow_stopped and running == "false")): raise ValueError(f"{service} container state is not approved for the release operation") if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"invalid running image ID for {service}") records[service] = {"configured": configured[key], "image_id": image_id} return records def write_deployed_backend_image(image_id: str) -> None: if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError("invalid backend image ID for deployment record") with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream: temporary = Path(stream.name) stream.write(f"{image_id}\n".encode("ascii")) stream.flush() os.fsync(stream.fileno()) try: os.chmod(temporary, 0o600) os.replace(temporary, DEPLOYED_IMAGE_FILE) directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: temporary.unlink(missing_ok=True) def require_backup_matches_current(current: dict, backup: dict) -> None: if current != backup: raise ValueError("backup image references or image IDs do not match the current deployment") def restore_previous_references(old_env: dict[str, str], new_env: dict[str, str]) -> None: _, actual = read_images(ENV_FILE) if actual == new_env: replace_images(ENV_FILE, new_env, old_env) elif actual != old_env: raise ValueError("production image references match neither the previous nor target release") _, restored = read_images(ENV_FILE) if restored != old_env: raise ValueError("previous production image references could not be verified") def app_images(value: dict, label: str) -> dict[str, dict[str, str]]: images = value.get("images") if not isinstance(images, dict) or not set(SERVICES) <= images.keys(): raise ValueError(f"{label} does not contain both application image records") return validate_images({service: images[service] for service in SERVICES}, label) def validate_search_response(payload: dict, language: str) -> None: results = payload.get("results") if isinstance(payload, dict) else None if not isinstance(results, list) or not results: raise ValueError(f"{language} search canary returned no results") if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip(): raise ValueError(f"{language} search canary returned a malformed first result") def check_search_canary(language: str, query: str) -> None: if language not in {"ru", "ky"} or not query.strip(): raise ValueError("both Russian and Kyrgyz search canaries must be non-empty") params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"}) output = run( ["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", f"{SEARCH_CANARY_URL}?{params}"], capture=True ) try: payload = json.loads(output) except json.JSONDecodeError as error: raise ValueError(f"{language} search canary returned invalid JSON") from error validate_search_response(payload, language) def deploy_external( manifest_value: dict, policy_value: dict, backup_value: dict, backup_identity: dict, descriptor_sha256: str, current_before_stop: dict, canary_ru: str, canary_ky: str, ) -> None: if os.geteuid() != 0: raise ValueError("production cutover is owner-run and requires root") manifest = validate_manifest(manifest_value, policy_value) if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}: raise ValueError("release descriptor uses an unapproved builder") from release_state import validate_backup_identity backup_identity = validate_backup_identity({ key: backup_identity[key] for key in ("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest") }) if backup_identity is None or backup_identity["manifest"] != backup_value: raise ValueError("release requires a source-generated, live-verified external backup") if not re.fullmatch(r"[0-9a-f]{64}", descriptor_sha256): raise ValueError("release descriptor digest is invalid") from release_handoff import latest_main_sha from release_source import require_runtime_compatibility def verify_release_still_current() -> None: if latest_main_sha(ROOT) != manifest["commit"]: raise ValueError("main advanced after the external backup; refusing a stale release cutover") require_runtime_compatibility( manifest["commit"], checkout=ROOT, policy_file=Path("/etc/akyldash/release-policy.json"), ) verify_release_still_current() if not isinstance(current_before_stop, dict) or image_ids(allow_stopped=True) != current_before_stop: raise ValueError("stopped services do not match the pre-backup image snapshot") if app_images(backup_value, "external backup images") != current_before_stop: raise ValueError("external backup image identities do not match the current deployment") current_files = { name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") } if backup_value.get("deployment_files") != current_files: raise ValueError("external backup configuration does not match the checked out Compose and Caddy files") pinned = None if STATE_FILE.exists() or STATE_FILE.is_symlink(): pinned = validate_state(read_json(STATE_FILE)) if pinned["current"] != current_before_stop: raise ValueError("current deployment does not match the recorded release state") backup_record = { "backup_id": backup_identity["backup_id"], "size": backup_identity["size"], "archive_sha256": backup_identity["archive_sha256"], "manifest_sha256": backup_identity["manifest_sha256"], "manifest": backup_value, } backup_record = validate_backup_identity(backup_record) old_env = {"BACKEND_IMAGE": current_before_stop["backend"]["configured"], "OPENSEARCH_IMAGE": current_before_stop["opensearch"]["configured"]} new_env = {"BACKEND_IMAGE": manifest["images"]["backend"], "OPENSEARCH_IMAGE": manifest["images"]["opensearch"]} journal = read_json(JOURNAL_FILE) state_written = False try: for service in SERVICES: reference = manifest["images"][service] run(["docker", "pull", reference]) pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True) if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id): raise ValueError(f"registry returned an invalid image ID for {service}") target = { service: {"configured": manifest["images"][service], "image_id": run( ["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True )} for service in SERVICES } journal.update({"status": "external-backup-verified", "target": target}) write_json_atomic(JOURNAL_FILE, journal) verify_release_still_current() replace_images(ENV_FILE, old_env, new_env) journal["status"] = "references-switched" write_json_atomic(JOURNAL_FILE, journal) compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") actual = image_ids() if actual != target: raise ValueError("started image IDs do not match the release descriptor") run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", "https://akyldash.kg/review", "--output", "/dev/null"]) check_search_canary("ru", canary_ru) check_search_canary("ky", canary_ky) write_deployed_backend_image(actual["backend"]["image_id"]) state = { "schema_version": 2, "commit": manifest["commit"], "manifest_sha256": descriptor_sha256, "current": actual, "previous": current_before_stop, "previous_backup": backup_record, } write_json_atomic(STATE_FILE, state) state_written = True JOURNAL_FILE.unlink() print(f"PASS deployed release {manifest['commit']} with --no-build", file=sys.stderr) except Exception as error: if state_written: journal["status"] = "release-active-journal-cleanup-needed" journal["error_type"] = type(error).__name__ journal["recovery"] = "release is active; verify host and resolve the recovery journal" write_json_atomic(JOURNAL_FILE, journal) raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error journal["status"] = "cutover-failed" journal["error_type"] = type(error).__name__ try: restore_previous_references(old_env, new_env) compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") restored = image_ids() if restored != current_before_stop: raise ValueError("restored running image IDs do not match the previous release") write_deployed_backend_image(current_before_stop["backend"]["image_id"]) if pinned is not None: write_json_atomic(STATE_FILE, pinned) elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink(): STATE_FILE.unlink() journal["recovery"] = "previous references and running image IDs restored and verified" except Exception as recovery_error: journal["recovery"] = ( f"manual recovery required ({type(recovery_error).__name__}); " "references, release state, and external backup identity are recorded" ) write_json_atomic(JOURNAL_FILE, journal) raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error def main() -> int: parser = argparse.ArgumentParser(description=__doc__) argparse.ArgumentParser(description=__doc__).parse_args() print("ERROR: local backup paths are disabled; use the fixed external session receiver", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())