Files
codex 710c483f34
CI / tests-and-images (pull_request) Successful in 2m15s
Runner smoke test / runner-smoke (pull_request) Successful in 3s
Harden external release preparation workflow
2026-11-03 09:38:37 +03:00

265 lines
13 KiB
Python

#!/usr/bin/env python3
"""Owner-run, locked production release cutover using a verified cold backup."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.parse
from pathlib import Path
from create_release_manifest import BUILDER_IMAGE
from deployment_lock import deployment_lock
from release_refs import read_images, replace_images
from release_state import read_json, validate_images, validate_state, write_json_atomic
from validate_release import read_json as read_manifest_json, validate_manifest
ROOT = Path("/opt/akyldash/current")
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
ENV_FILE = Path("/etc/akyldash/production.env")
BACKUP_DIR = Path("/var/backups/akyldash")
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
STATE_FILE = Path("/opt/akyldash/release-state.json")
JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json")
DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt")
SERVICES = ("backend", "opensearch")
SEARCH_CANARY_URL = "https://akyldash.kg/search"
def run(command: list[str], capture: bool = False) -> str:
result = subprocess.run(command, check=True, text=True, capture_output=capture)
return result.stdout.strip() if capture else ""
def compose(*args: str, capture: bool = False) -> str:
return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture)
def image_ids(*, allow_stopped: bool = False) -> dict[str, dict[str, str]]:
configured = read_images(ENV_FILE)[1]
records = {}
for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")):
container = compose("ps", "--all", "--quiet", service, capture=True)
if not container or "\n" in container:
raise ValueError(f"expected exactly one existing {service} container")
inspection = run(
["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True
)
running, separator, image_id = inspection.partition("\t")
if not separator or (running != "true" and not (allow_stopped and running == "false")):
raise ValueError(f"{service} container state is not approved for the release operation")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError(f"invalid running image ID for {service}")
records[service] = {"configured": configured[key], "image_id": image_id}
return records
def write_deployed_backend_image(image_id: str) -> None:
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError("invalid backend image ID for deployment record")
with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream:
temporary = Path(stream.name)
stream.write(f"{image_id}\n".encode("ascii"))
stream.flush()
os.fsync(stream.fileno())
try:
os.chmod(temporary, 0o600)
os.replace(temporary, DEPLOYED_IMAGE_FILE)
directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
temporary.unlink(missing_ok=True)
def require_backup_matches_current(current: dict, backup: dict) -> None:
if current != backup:
raise ValueError("backup image references or image IDs do not match the current deployment")
def restore_previous_references(old_env: dict[str, str], new_env: dict[str, str]) -> None:
_, actual = read_images(ENV_FILE)
if actual == new_env:
replace_images(ENV_FILE, new_env, old_env)
elif actual != old_env:
raise ValueError("production image references match neither the previous nor target release")
_, restored = read_images(ENV_FILE)
if restored != old_env:
raise ValueError("previous production image references could not be verified")
def app_images(value: dict, label: str) -> dict[str, dict[str, str]]:
images = value.get("images")
if not isinstance(images, dict) or not set(SERVICES) <= images.keys():
raise ValueError(f"{label} does not contain both application image records")
return validate_images({service: images[service] for service in SERVICES}, label)
def validate_search_response(payload: dict, language: str) -> None:
results = payload.get("results") if isinstance(payload, dict) else None
if not isinstance(results, list) or not results:
raise ValueError(f"{language} search canary returned no results")
if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip():
raise ValueError(f"{language} search canary returned a malformed first result")
def check_search_canary(language: str, query: str) -> None:
if language not in {"ru", "ky"} or not query.strip():
raise ValueError("both Russian and Kyrgyz search canaries must be non-empty")
params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"})
output = run(
["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
f"{SEARCH_CANARY_URL}?{params}"], capture=True
)
try:
payload = json.loads(output)
except json.JSONDecodeError as error:
raise ValueError(f"{language} search canary returned invalid JSON") from error
validate_search_response(payload, language)
def deploy_external(
manifest_value: dict, policy_value: dict, backup_value: dict, backup_identity: dict,
descriptor_sha256: str, current_before_stop: dict, canary_ru: str, canary_ky: str,
) -> None:
if os.geteuid() != 0:
raise ValueError("production cutover is owner-run and requires root")
manifest = validate_manifest(manifest_value, policy_value)
if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}:
raise ValueError("release descriptor uses an unapproved builder")
from release_state import validate_backup_identity
backup_identity = validate_backup_identity({
key: backup_identity[key]
for key in ("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")
})
if backup_identity is None or backup_identity["manifest"] != backup_value:
raise ValueError("release requires a source-generated, live-verified external backup")
if not re.fullmatch(r"[0-9a-f]{64}", descriptor_sha256):
raise ValueError("release descriptor digest is invalid")
from release_handoff import latest_main_sha
from release_source import require_runtime_compatibility
def verify_release_still_current() -> None:
if latest_main_sha(ROOT) != manifest["commit"]:
raise ValueError("main advanced after the external backup; refusing a stale release cutover")
require_runtime_compatibility(
manifest["commit"], checkout=ROOT,
policy_file=Path("/etc/akyldash/release-policy.json"),
)
verify_release_still_current()
if not isinstance(current_before_stop, dict) or image_ids(allow_stopped=True) != current_before_stop:
raise ValueError("stopped services do not match the pre-backup image snapshot")
if app_images(backup_value, "external backup images") != current_before_stop:
raise ValueError("external backup image identities do not match the current deployment")
current_files = {
name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
}
if backup_value.get("deployment_files") != current_files:
raise ValueError("external backup configuration does not match the checked out Compose and Caddy files")
pinned = None
if STATE_FILE.exists() or STATE_FILE.is_symlink():
pinned = validate_state(read_json(STATE_FILE))
if pinned["current"] != current_before_stop:
raise ValueError("current deployment does not match the recorded release state")
backup_record = {
"backup_id": backup_identity["backup_id"],
"size": backup_identity["size"],
"archive_sha256": backup_identity["archive_sha256"],
"manifest_sha256": backup_identity["manifest_sha256"],
"manifest": backup_value,
}
backup_record = validate_backup_identity(backup_record)
old_env = {"BACKEND_IMAGE": current_before_stop["backend"]["configured"], "OPENSEARCH_IMAGE": current_before_stop["opensearch"]["configured"]}
new_env = {"BACKEND_IMAGE": manifest["images"]["backend"], "OPENSEARCH_IMAGE": manifest["images"]["opensearch"]}
journal = read_json(JOURNAL_FILE)
state_written = False
try:
for service in SERVICES:
reference = manifest["images"][service]
run(["docker", "pull", reference])
pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True)
if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id):
raise ValueError(f"registry returned an invalid image ID for {service}")
target = {
service: {"configured": manifest["images"][service], "image_id": run(
["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True
)}
for service in SERVICES
}
journal.update({"status": "external-backup-verified", "target": target})
write_json_atomic(JOURNAL_FILE, journal)
verify_release_still_current()
replace_images(ENV_FILE, old_env, new_env)
journal["status"] = "references-switched"
write_json_atomic(JOURNAL_FILE, journal)
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
actual = image_ids()
if actual != target:
raise ValueError("started image IDs do not match the release descriptor")
run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
"https://akyldash.kg/review", "--output", "/dev/null"])
check_search_canary("ru", canary_ru)
check_search_canary("ky", canary_ky)
write_deployed_backend_image(actual["backend"]["image_id"])
state = {
"schema_version": 2, "commit": manifest["commit"],
"manifest_sha256": descriptor_sha256, "current": actual,
"previous": current_before_stop, "previous_backup": backup_record,
}
write_json_atomic(STATE_FILE, state)
state_written = True
JOURNAL_FILE.unlink()
print(f"PASS deployed release {manifest['commit']} with --no-build", file=sys.stderr)
except Exception as error:
if state_written:
journal["status"] = "release-active-journal-cleanup-needed"
journal["error_type"] = type(error).__name__
journal["recovery"] = "release is active; verify host and resolve the recovery journal"
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error
journal["status"] = "cutover-failed"
journal["error_type"] = type(error).__name__
try:
restore_previous_references(old_env, new_env)
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
restored = image_ids()
if restored != current_before_stop:
raise ValueError("restored running image IDs do not match the previous release")
write_deployed_backend_image(current_before_stop["backend"]["image_id"])
if pinned is not None:
write_json_atomic(STATE_FILE, pinned)
elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink():
STATE_FILE.unlink()
journal["recovery"] = "previous references and running image IDs restored and verified"
except Exception as recovery_error:
journal["recovery"] = (
f"manual recovery required ({type(recovery_error).__name__}); "
"references, release state, and external backup identity are recorded"
)
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
argparse.ArgumentParser(description=__doc__).parse_args()
print("ERROR: local backup paths are disabled; use the fixed external session receiver", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())