Files
akyldash/infra/deploy/deploy_release.py
T
agent 9d8ce6b67f
CI / tests-and-images (pull_request) Successful in 2m43s
Runner smoke test / runner-smoke (pull_request) Successful in 2s
fix: address production release review findings
2026-10-25 13:44:56 +03:00

262 lines
13 KiB
Python

#!/usr/bin/env python3
"""Owner-run, locked production release cutover using a verified cold backup."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.parse
from pathlib import Path
from create_release_manifest import BUILDER_IMAGE
from deployment_lock import deployment_lock
from release_refs import read_images, replace_images
from release_state import read_json, validate_images, validate_state, write_json_atomic
from validate_release import read_json as read_manifest_json, validate_manifest
ROOT = Path("/opt/akyldash/current")
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
ENV_FILE = Path("/etc/akyldash/production.env")
BACKUP_DIR = Path("/var/backups/akyldash")
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
STATE_FILE = Path("/opt/akyldash/release-state.json")
JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json")
DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt")
SERVICES = ("backend", "opensearch")
SEARCH_CANARY_URL = "https://akyldash.kg/search"
def run(command: list[str], capture: bool = False) -> str:
result = subprocess.run(command, check=True, text=True, capture_output=capture)
return result.stdout.strip() if capture else ""
def compose(*args: str, capture: bool = False) -> str:
return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture)
def image_ids() -> dict[str, dict[str, str]]:
configured = read_images(ENV_FILE)[1]
records = {}
for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")):
container = compose("ps", "--all", "--quiet", service, capture=True)
if not container or "\n" in container:
raise ValueError(f"expected exactly one existing {service} container")
inspection = run(
["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True
)
running, separator, image_id = inspection.partition("\t")
if not separator or running != "true":
raise ValueError(f"{service} is not running; refusing release cutover")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError(f"invalid running image ID for {service}")
records[service] = {"configured": configured[key], "image_id": image_id}
return records
def write_deployed_backend_image(image_id: str) -> None:
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
raise ValueError("invalid backend image ID for deployment record")
with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream:
temporary = Path(stream.name)
stream.write(f"{image_id}\n".encode("ascii"))
stream.flush()
os.fsync(stream.fileno())
try:
os.chmod(temporary, 0o600)
os.replace(temporary, DEPLOYED_IMAGE_FILE)
directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
temporary.unlink(missing_ok=True)
def backup_manifest(path: Path) -> dict:
if not path.is_file() or path.is_symlink() or path.resolve().parent != BACKUP_DIR.resolve():
raise ValueError(f"verified backup must be a regular file directly under {BACKUP_DIR}")
if path.stat().st_uid != 0 or path.stat().st_mode & 0o777 != 0o600:
raise ValueError("release backup must be root-owned mode 0600")
checksum = Path(f"{path}.sha256")
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
raise ValueError("release backup checksum must be root-owned mode 0600")
sys.path.insert(0, str(BACKUP_TOOL.parent))
from archive import digest, verify_archive
manifest = verify_archive(path)
current_files = {
name: digest(ROOT / name)
for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
}
if manifest.get("deployment_files") != current_files:
raise ValueError("backup does not match the checked out Compose and Caddy configuration")
return manifest
def require_backup_matches_current(current: dict, backup: dict) -> None:
if current != backup:
raise ValueError("backup image references or image IDs do not match the current deployment")
def restore_previous_references(old_env: dict[str, str], new_env: dict[str, str]) -> None:
_, actual = read_images(ENV_FILE)
if actual == new_env:
replace_images(ENV_FILE, new_env, old_env)
elif actual != old_env:
raise ValueError("production image references match neither the previous nor target release")
_, restored = read_images(ENV_FILE)
if restored != old_env:
raise ValueError("previous production image references could not be verified")
def app_images(value: dict, label: str) -> dict[str, dict[str, str]]:
images = value.get("images")
if not isinstance(images, dict) or not set(SERVICES) <= images.keys():
raise ValueError(f"{label} does not contain both application image records")
return validate_images({service: images[service] for service in SERVICES}, label)
def validate_search_response(payload: dict, language: str) -> None:
results = payload.get("results") if isinstance(payload, dict) else None
if not isinstance(results, list) or not results:
raise ValueError(f"{language} search canary returned no results")
if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip():
raise ValueError(f"{language} search canary returned a malformed first result")
def check_search_canary(language: str, query: str) -> None:
if language not in {"ru", "ky"} or not query.strip():
raise ValueError("both Russian and Kyrgyz search canaries must be non-empty")
params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"})
output = run(
["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
f"{SEARCH_CANARY_URL}?{params}"], capture=True
)
try:
payload = json.loads(output)
except json.JSONDecodeError as error:
raise ValueError(f"{language} search canary returned invalid JSON") from error
validate_search_response(payload, language)
def deploy(manifest_path: Path, backup_path: Path, policy_path: Path, canary_ru: str, canary_ky: str) -> None:
if os.geteuid() != 0:
raise ValueError("production cutover is owner-run and requires root")
manifest = validate_manifest(read_manifest_json(manifest_path), read_manifest_json(policy_path))
if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}:
raise ValueError("release descriptor uses an unapproved builder")
manifest_sha = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
with deployment_lock():
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
current = image_ids()
backup = backup_manifest(backup_path)
previous = app_images(backup, "backup images")
require_backup_matches_current(current, previous)
pinned = None
if STATE_FILE.exists() or STATE_FILE.is_symlink():
pinned = validate_state(read_json(STATE_FILE))
if pinned["current"] != current:
raise ValueError("current deployment does not match the recorded release state")
for service in SERVICES:
reference = manifest["images"][service]
run(["docker", "pull", reference])
pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True)
if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id):
raise ValueError(f"registry returned an invalid image ID for {service}")
target = {
service: {"configured": manifest["images"][service], "image_id": run(
["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True
)}
for service in SERVICES
}
old_env = {"BACKEND_IMAGE": current["backend"]["configured"], "OPENSEARCH_IMAGE": current["opensearch"]["configured"]}
new_env = {"BACKEND_IMAGE": target["backend"]["configured"], "OPENSEARCH_IMAGE": target["opensearch"]["configured"]}
journal = {
"schema_version": 1, "status": "prepared", "commit": manifest["commit"],
"manifest_sha256": manifest_sha, "backup": str(backup_path),
"previous": current, "target": target,
}
write_json_atomic(JOURNAL_FILE, journal)
state_written = False
try:
compose("stop")
replace_images(ENV_FILE, old_env, new_env)
journal["status"] = "references-switched"
write_json_atomic(JOURNAL_FILE, journal)
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
actual = image_ids()
if actual != target:
raise ValueError("started image IDs do not match the release descriptor")
run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
"https://akyldash.kg/review", "--output", "/dev/null"])
check_search_canary("ru", canary_ru)
check_search_canary("ky", canary_ky)
write_deployed_backend_image(actual["backend"]["image_id"])
state = {
"schema_version": 1, "commit": manifest["commit"],
"manifest_sha256": manifest_sha, "current": actual, "previous": current,
}
write_json_atomic(STATE_FILE, state)
state_written = True
JOURNAL_FILE.unlink()
print(f"PASS deployed release {manifest['commit']} with --no-build")
except Exception as error:
if state_written:
journal["status"] = "release-active-journal-cleanup-needed"
journal["error_type"] = type(error).__name__
journal["recovery"] = "target refs, release state, and health check completed; verify host and resolve journal"
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error
journal["status"] = "cutover-failed"
journal["error_type"] = type(error).__name__
try:
restore_previous_references(old_env, new_env)
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
restored = image_ids()
if restored != current:
raise ValueError("restored running image IDs do not match the previous release")
write_deployed_backend_image(current["backend"]["image_id"])
if pinned is not None:
write_json_atomic(STATE_FILE, pinned)
elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink():
STATE_FILE.unlink()
journal["recovery"] = "previous references and running image IDs restored and verified"
except Exception as recovery_error:
journal["recovery"] = (
f"manual recovery required ({type(recovery_error).__name__}); "
"references, release state, and backup are recorded; services were not restarted unless previous refs were verified"
)
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("manifest", type=Path)
parser.add_argument("backup", type=Path)
parser.add_argument("--policy", required=True, type=Path)
parser.add_argument("--canary-ru", required=True, help="known query expected to return a Russian search result")
parser.add_argument("--canary-ky", required=True, help="known query expected to return a Kyrgyz search result")
args = parser.parse_args()
try:
deploy(args.manifest, args.backup, args.policy, args.canary_ru, args.canary_ky)
except Exception as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())