262 lines
13 KiB
Python
262 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Owner-run, locked production release cutover using a verified cold backup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
from create_release_manifest import BUILDER_IMAGE
|
|
from deployment_lock import deployment_lock
|
|
from release_refs import read_images, replace_images
|
|
from release_state import read_json, validate_images, validate_state, write_json_atomic
|
|
from validate_release import read_json as read_manifest_json, validate_manifest
|
|
|
|
ROOT = Path("/opt/akyldash/current")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_FILE = Path("/etc/akyldash/production.env")
|
|
BACKUP_DIR = Path("/var/backups/akyldash")
|
|
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
|
|
STATE_FILE = Path("/opt/akyldash/release-state.json")
|
|
JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json")
|
|
DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt")
|
|
SERVICES = ("backend", "opensearch")
|
|
SEARCH_CANARY_URL = "https://akyldash.kg/search"
|
|
|
|
|
|
def run(command: list[str], capture: bool = False) -> str:
|
|
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def compose(*args: str, capture: bool = False) -> str:
|
|
return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture)
|
|
|
|
|
|
def image_ids() -> dict[str, dict[str, str]]:
|
|
configured = read_images(ENV_FILE)[1]
|
|
records = {}
|
|
for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")):
|
|
container = compose("ps", "--all", "--quiet", service, capture=True)
|
|
if not container or "\n" in container:
|
|
raise ValueError(f"expected exactly one existing {service} container")
|
|
inspection = run(
|
|
["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True
|
|
)
|
|
running, separator, image_id = inspection.partition("\t")
|
|
if not separator or running != "true":
|
|
raise ValueError(f"{service} is not running; refusing release cutover")
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError(f"invalid running image ID for {service}")
|
|
records[service] = {"configured": configured[key], "image_id": image_id}
|
|
return records
|
|
|
|
|
|
def write_deployed_backend_image(image_id: str) -> None:
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError("invalid backend image ID for deployment record")
|
|
with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
stream.write(f"{image_id}\n".encode("ascii"))
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
try:
|
|
os.chmod(temporary, 0o600)
|
|
os.replace(temporary, DEPLOYED_IMAGE_FILE)
|
|
directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def backup_manifest(path: Path) -> dict:
|
|
if not path.is_file() or path.is_symlink() or path.resolve().parent != BACKUP_DIR.resolve():
|
|
raise ValueError(f"verified backup must be a regular file directly under {BACKUP_DIR}")
|
|
if path.stat().st_uid != 0 or path.stat().st_mode & 0o777 != 0o600:
|
|
raise ValueError("release backup must be root-owned mode 0600")
|
|
checksum = Path(f"{path}.sha256")
|
|
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
|
|
raise ValueError("release backup checksum must be root-owned mode 0600")
|
|
sys.path.insert(0, str(BACKUP_TOOL.parent))
|
|
from archive import digest, verify_archive
|
|
|
|
manifest = verify_archive(path)
|
|
current_files = {
|
|
name: digest(ROOT / name)
|
|
for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
}
|
|
if manifest.get("deployment_files") != current_files:
|
|
raise ValueError("backup does not match the checked out Compose and Caddy configuration")
|
|
return manifest
|
|
|
|
|
|
def require_backup_matches_current(current: dict, backup: dict) -> None:
|
|
if current != backup:
|
|
raise ValueError("backup image references or image IDs do not match the current deployment")
|
|
|
|
|
|
def restore_previous_references(old_env: dict[str, str], new_env: dict[str, str]) -> None:
|
|
_, actual = read_images(ENV_FILE)
|
|
if actual == new_env:
|
|
replace_images(ENV_FILE, new_env, old_env)
|
|
elif actual != old_env:
|
|
raise ValueError("production image references match neither the previous nor target release")
|
|
_, restored = read_images(ENV_FILE)
|
|
if restored != old_env:
|
|
raise ValueError("previous production image references could not be verified")
|
|
|
|
|
|
def app_images(value: dict, label: str) -> dict[str, dict[str, str]]:
|
|
images = value.get("images")
|
|
if not isinstance(images, dict) or not set(SERVICES) <= images.keys():
|
|
raise ValueError(f"{label} does not contain both application image records")
|
|
return validate_images({service: images[service] for service in SERVICES}, label)
|
|
|
|
|
|
def validate_search_response(payload: dict, language: str) -> None:
|
|
results = payload.get("results") if isinstance(payload, dict) else None
|
|
if not isinstance(results, list) or not results:
|
|
raise ValueError(f"{language} search canary returned no results")
|
|
if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip():
|
|
raise ValueError(f"{language} search canary returned a malformed first result")
|
|
|
|
|
|
def check_search_canary(language: str, query: str) -> None:
|
|
if language not in {"ru", "ky"} or not query.strip():
|
|
raise ValueError("both Russian and Kyrgyz search canaries must be non-empty")
|
|
params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"})
|
|
output = run(
|
|
["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
|
|
f"{SEARCH_CANARY_URL}?{params}"], capture=True
|
|
)
|
|
try:
|
|
payload = json.loads(output)
|
|
except json.JSONDecodeError as error:
|
|
raise ValueError(f"{language} search canary returned invalid JSON") from error
|
|
validate_search_response(payload, language)
|
|
|
|
|
|
def deploy(manifest_path: Path, backup_path: Path, policy_path: Path, canary_ru: str, canary_ky: str) -> None:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("production cutover is owner-run and requires root")
|
|
manifest = validate_manifest(read_manifest_json(manifest_path), read_manifest_json(policy_path))
|
|
if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}:
|
|
raise ValueError("release descriptor uses an unapproved builder")
|
|
manifest_sha = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
|
|
|
with deployment_lock():
|
|
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
|
|
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
|
|
current = image_ids()
|
|
backup = backup_manifest(backup_path)
|
|
previous = app_images(backup, "backup images")
|
|
require_backup_matches_current(current, previous)
|
|
pinned = None
|
|
if STATE_FILE.exists() or STATE_FILE.is_symlink():
|
|
pinned = validate_state(read_json(STATE_FILE))
|
|
if pinned["current"] != current:
|
|
raise ValueError("current deployment does not match the recorded release state")
|
|
for service in SERVICES:
|
|
reference = manifest["images"][service]
|
|
run(["docker", "pull", reference])
|
|
pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True)
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id):
|
|
raise ValueError(f"registry returned an invalid image ID for {service}")
|
|
|
|
target = {
|
|
service: {"configured": manifest["images"][service], "image_id": run(
|
|
["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True
|
|
)}
|
|
for service in SERVICES
|
|
}
|
|
old_env = {"BACKEND_IMAGE": current["backend"]["configured"], "OPENSEARCH_IMAGE": current["opensearch"]["configured"]}
|
|
new_env = {"BACKEND_IMAGE": target["backend"]["configured"], "OPENSEARCH_IMAGE": target["opensearch"]["configured"]}
|
|
journal = {
|
|
"schema_version": 1, "status": "prepared", "commit": manifest["commit"],
|
|
"manifest_sha256": manifest_sha, "backup": str(backup_path),
|
|
"previous": current, "target": target,
|
|
}
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
state_written = False
|
|
try:
|
|
compose("stop")
|
|
replace_images(ENV_FILE, old_env, new_env)
|
|
journal["status"] = "references-switched"
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
actual = image_ids()
|
|
if actual != target:
|
|
raise ValueError("started image IDs do not match the release descriptor")
|
|
run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
|
|
"https://akyldash.kg/review", "--output", "/dev/null"])
|
|
check_search_canary("ru", canary_ru)
|
|
check_search_canary("ky", canary_ky)
|
|
write_deployed_backend_image(actual["backend"]["image_id"])
|
|
state = {
|
|
"schema_version": 1, "commit": manifest["commit"],
|
|
"manifest_sha256": manifest_sha, "current": actual, "previous": current,
|
|
}
|
|
write_json_atomic(STATE_FILE, state)
|
|
state_written = True
|
|
JOURNAL_FILE.unlink()
|
|
print(f"PASS deployed release {manifest['commit']} with --no-build")
|
|
except Exception as error:
|
|
if state_written:
|
|
journal["status"] = "release-active-journal-cleanup-needed"
|
|
journal["error_type"] = type(error).__name__
|
|
journal["recovery"] = "target refs, release state, and health check completed; verify host and resolve journal"
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error
|
|
journal["status"] = "cutover-failed"
|
|
journal["error_type"] = type(error).__name__
|
|
try:
|
|
restore_previous_references(old_env, new_env)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
restored = image_ids()
|
|
if restored != current:
|
|
raise ValueError("restored running image IDs do not match the previous release")
|
|
write_deployed_backend_image(current["backend"]["image_id"])
|
|
if pinned is not None:
|
|
write_json_atomic(STATE_FILE, pinned)
|
|
elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink():
|
|
STATE_FILE.unlink()
|
|
journal["recovery"] = "previous references and running image IDs restored and verified"
|
|
except Exception as recovery_error:
|
|
journal["recovery"] = (
|
|
f"manual recovery required ({type(recovery_error).__name__}); "
|
|
"references, release state, and backup are recorded; services were not restarted unless previous refs were verified"
|
|
)
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("manifest", type=Path)
|
|
parser.add_argument("backup", type=Path)
|
|
parser.add_argument("--policy", required=True, type=Path)
|
|
parser.add_argument("--canary-ru", required=True, help="known query expected to return a Russian search result")
|
|
parser.add_argument("--canary-ky", required=True, help="known query expected to return a Kyrgyz search result")
|
|
args = parser.parse_args()
|
|
try:
|
|
deploy(args.manifest, args.backup, args.policy, args.canary_ru, args.canary_ky)
|
|
except Exception as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|