265 lines
13 KiB
Python
265 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Owner-run, locked production release cutover using a verified cold backup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
from create_release_manifest import BUILDER_IMAGE
|
|
from deployment_lock import deployment_lock
|
|
from release_refs import read_images, replace_images
|
|
from release_state import read_json, validate_images, validate_state, write_json_atomic
|
|
from validate_release import read_json as read_manifest_json, validate_manifest
|
|
|
|
ROOT = Path("/opt/akyldash/current")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_FILE = Path("/etc/akyldash/production.env")
|
|
BACKUP_DIR = Path("/var/backups/akyldash")
|
|
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
|
|
STATE_FILE = Path("/opt/akyldash/release-state.json")
|
|
JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json")
|
|
DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt")
|
|
SERVICES = ("backend", "opensearch")
|
|
SEARCH_CANARY_URL = "https://akyldash.kg/search"
|
|
|
|
|
|
def run(command: list[str], capture: bool = False) -> str:
|
|
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def compose(*args: str, capture: bool = False) -> str:
|
|
return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture)
|
|
|
|
|
|
def image_ids(*, allow_stopped: bool = False) -> dict[str, dict[str, str]]:
|
|
configured = read_images(ENV_FILE)[1]
|
|
records = {}
|
|
for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")):
|
|
container = compose("ps", "--all", "--quiet", service, capture=True)
|
|
if not container or "\n" in container:
|
|
raise ValueError(f"expected exactly one existing {service} container")
|
|
inspection = run(
|
|
["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True
|
|
)
|
|
running, separator, image_id = inspection.partition("\t")
|
|
if not separator or (running != "true" and not (allow_stopped and running == "false")):
|
|
raise ValueError(f"{service} container state is not approved for the release operation")
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError(f"invalid running image ID for {service}")
|
|
records[service] = {"configured": configured[key], "image_id": image_id}
|
|
return records
|
|
|
|
|
|
def write_deployed_backend_image(image_id: str) -> None:
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError("invalid backend image ID for deployment record")
|
|
with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
stream.write(f"{image_id}\n".encode("ascii"))
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
try:
|
|
os.chmod(temporary, 0o600)
|
|
os.replace(temporary, DEPLOYED_IMAGE_FILE)
|
|
directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def require_backup_matches_current(current: dict, backup: dict) -> None:
|
|
if current != backup:
|
|
raise ValueError("backup image references or image IDs do not match the current deployment")
|
|
|
|
|
|
def restore_previous_references(old_env: dict[str, str], new_env: dict[str, str]) -> None:
|
|
_, actual = read_images(ENV_FILE)
|
|
if actual == new_env:
|
|
replace_images(ENV_FILE, new_env, old_env)
|
|
elif actual != old_env:
|
|
raise ValueError("production image references match neither the previous nor target release")
|
|
_, restored = read_images(ENV_FILE)
|
|
if restored != old_env:
|
|
raise ValueError("previous production image references could not be verified")
|
|
|
|
|
|
def app_images(value: dict, label: str) -> dict[str, dict[str, str]]:
|
|
images = value.get("images")
|
|
if not isinstance(images, dict) or not set(SERVICES) <= images.keys():
|
|
raise ValueError(f"{label} does not contain both application image records")
|
|
return validate_images({service: images[service] for service in SERVICES}, label)
|
|
|
|
|
|
def validate_search_response(payload: dict, language: str) -> None:
|
|
results = payload.get("results") if isinstance(payload, dict) else None
|
|
if not isinstance(results, list) or not results:
|
|
raise ValueError(f"{language} search canary returned no results")
|
|
if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip():
|
|
raise ValueError(f"{language} search canary returned a malformed first result")
|
|
|
|
|
|
def check_search_canary(language: str, query: str) -> None:
|
|
if language not in {"ru", "ky"} or not query.strip():
|
|
raise ValueError("both Russian and Kyrgyz search canaries must be non-empty")
|
|
params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"})
|
|
output = run(
|
|
["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
|
|
f"{SEARCH_CANARY_URL}?{params}"], capture=True
|
|
)
|
|
try:
|
|
payload = json.loads(output)
|
|
except json.JSONDecodeError as error:
|
|
raise ValueError(f"{language} search canary returned invalid JSON") from error
|
|
validate_search_response(payload, language)
|
|
|
|
|
|
def deploy_external(
|
|
manifest_value: dict, policy_value: dict, backup_value: dict, backup_identity: dict,
|
|
descriptor_sha256: str, current_before_stop: dict, canary_ru: str, canary_ky: str,
|
|
) -> None:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("production cutover is owner-run and requires root")
|
|
manifest = validate_manifest(manifest_value, policy_value)
|
|
if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}:
|
|
raise ValueError("release descriptor uses an unapproved builder")
|
|
from release_state import validate_backup_identity
|
|
|
|
backup_identity = validate_backup_identity({
|
|
key: backup_identity[key]
|
|
for key in ("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")
|
|
})
|
|
if backup_identity is None or backup_identity["manifest"] != backup_value:
|
|
raise ValueError("release requires a source-generated, live-verified external backup")
|
|
if not re.fullmatch(r"[0-9a-f]{64}", descriptor_sha256):
|
|
raise ValueError("release descriptor digest is invalid")
|
|
from release_handoff import latest_main_sha
|
|
from release_source import require_runtime_compatibility
|
|
|
|
def verify_release_still_current() -> None:
|
|
if latest_main_sha(ROOT) != manifest["commit"]:
|
|
raise ValueError("main advanced after the external backup; refusing a stale release cutover")
|
|
require_runtime_compatibility(
|
|
manifest["commit"], checkout=ROOT,
|
|
policy_file=Path("/etc/akyldash/release-policy.json"),
|
|
)
|
|
|
|
verify_release_still_current()
|
|
if not isinstance(current_before_stop, dict) or image_ids(allow_stopped=True) != current_before_stop:
|
|
raise ValueError("stopped services do not match the pre-backup image snapshot")
|
|
if app_images(backup_value, "external backup images") != current_before_stop:
|
|
raise ValueError("external backup image identities do not match the current deployment")
|
|
current_files = {
|
|
name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
|
for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
}
|
|
if backup_value.get("deployment_files") != current_files:
|
|
raise ValueError("external backup configuration does not match the checked out Compose and Caddy files")
|
|
pinned = None
|
|
if STATE_FILE.exists() or STATE_FILE.is_symlink():
|
|
pinned = validate_state(read_json(STATE_FILE))
|
|
if pinned["current"] != current_before_stop:
|
|
raise ValueError("current deployment does not match the recorded release state")
|
|
|
|
backup_record = {
|
|
"backup_id": backup_identity["backup_id"],
|
|
"size": backup_identity["size"],
|
|
"archive_sha256": backup_identity["archive_sha256"],
|
|
"manifest_sha256": backup_identity["manifest_sha256"],
|
|
"manifest": backup_value,
|
|
}
|
|
backup_record = validate_backup_identity(backup_record)
|
|
|
|
old_env = {"BACKEND_IMAGE": current_before_stop["backend"]["configured"], "OPENSEARCH_IMAGE": current_before_stop["opensearch"]["configured"]}
|
|
new_env = {"BACKEND_IMAGE": manifest["images"]["backend"], "OPENSEARCH_IMAGE": manifest["images"]["opensearch"]}
|
|
journal = read_json(JOURNAL_FILE)
|
|
state_written = False
|
|
try:
|
|
for service in SERVICES:
|
|
reference = manifest["images"][service]
|
|
run(["docker", "pull", reference])
|
|
pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True)
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id):
|
|
raise ValueError(f"registry returned an invalid image ID for {service}")
|
|
target = {
|
|
service: {"configured": manifest["images"][service], "image_id": run(
|
|
["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True
|
|
)}
|
|
for service in SERVICES
|
|
}
|
|
journal.update({"status": "external-backup-verified", "target": target})
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
verify_release_still_current()
|
|
replace_images(ENV_FILE, old_env, new_env)
|
|
journal["status"] = "references-switched"
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
actual = image_ids()
|
|
if actual != target:
|
|
raise ValueError("started image IDs do not match the release descriptor")
|
|
run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1",
|
|
"https://akyldash.kg/review", "--output", "/dev/null"])
|
|
check_search_canary("ru", canary_ru)
|
|
check_search_canary("ky", canary_ky)
|
|
write_deployed_backend_image(actual["backend"]["image_id"])
|
|
state = {
|
|
"schema_version": 2, "commit": manifest["commit"],
|
|
"manifest_sha256": descriptor_sha256, "current": actual,
|
|
"previous": current_before_stop, "previous_backup": backup_record,
|
|
}
|
|
write_json_atomic(STATE_FILE, state)
|
|
state_written = True
|
|
JOURNAL_FILE.unlink()
|
|
print(f"PASS deployed release {manifest['commit']} with --no-build", file=sys.stderr)
|
|
except Exception as error:
|
|
if state_written:
|
|
journal["status"] = "release-active-journal-cleanup-needed"
|
|
journal["error_type"] = type(error).__name__
|
|
journal["recovery"] = "release is active; verify host and resolve the recovery journal"
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error
|
|
journal["status"] = "cutover-failed"
|
|
journal["error_type"] = type(error).__name__
|
|
try:
|
|
restore_previous_references(old_env, new_env)
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
restored = image_ids()
|
|
if restored != current_before_stop:
|
|
raise ValueError("restored running image IDs do not match the previous release")
|
|
write_deployed_backend_image(current_before_stop["backend"]["image_id"])
|
|
if pinned is not None:
|
|
write_json_atomic(STATE_FILE, pinned)
|
|
elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink():
|
|
STATE_FILE.unlink()
|
|
journal["recovery"] = "previous references and running image IDs restored and verified"
|
|
except Exception as recovery_error:
|
|
journal["recovery"] = (
|
|
f"manual recovery required ({type(recovery_error).__name__}); "
|
|
"references, release state, and external backup identity are recorded"
|
|
)
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
print("ERROR: local backup paths are disabled; use the fixed external session receiver", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|