Files
codex 710c483f34
CI / tests-and-images (pull_request) Successful in 2m15s
Runner smoke test / runner-smoke (pull_request) Successful in 3s
Harden external release preparation workflow
2026-11-03 09:38:37 +03:00

161 lines
7.7 KiB
Python

#!/usr/bin/env python3
"""Fixed root-side Gitea release receiver and external-backup cutover."""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
from archive import compose_sources
from deployment_lock import deployment_lock
from deploy_release import JOURNAL_FILE, ROOT, SERVICES, compose, deploy_external, image_ids
from external_session import read_request, send_backup_session, validate_request, write_result
from release_source import fetch_release, require_runtime_compatibility
from release_state import read_json, validate_state, write_json_atomic
POLICY_FILE = Path("/etc/akyldash/release-policy.json")
CANARIES_FILE = Path("/etc/akyldash/release-canaries.json")
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
ENV_FILE = Path("/etc/akyldash/production.env")
def owner_canaries() -> tuple[str, str]:
info = CANARIES_FILE.lstat()
if (CANARIES_FILE.is_symlink() or not CANARIES_FILE.is_file() or info.st_uid != 0
or info.st_mode & 0o777 != 0o600):
raise ValueError("release canaries must be stored in a root-owned mode-0600 file")
value = json.loads(CANARIES_FILE.read_text(encoding="utf-8"))
if not isinstance(value, dict) or set(value) != {"ru", "ky"}:
raise ValueError("release canary configuration must contain exactly ru and ky")
queries = (value["ru"], value["ky"])
if any(not isinstance(query, str) or not query.strip() or len(query) > 256 for query in queries):
raise ValueError("release canaries must be non-empty strings of at most 256 characters")
return queries
def restart_current(current: dict) -> None:
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
if image_ids() != current:
raise ValueError("restarted service image IDs do not match the pre-backup deployment")
def execute(request: dict, *, reader, writer, policy: dict | None = None, api=None,
current_main: str | None = None, canaries: tuple[str, str] | None = None) -> dict:
if os.geteuid() != 0:
raise ValueError("the fixed external deployment receiver must run as root")
validate_request(request)
policy = policy if policy is not None else read_json(POLICY_FILE)
canary_ru, canary_ky = canaries or owner_canaries()
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
manifest, _manifest_bytes = fetch_release(
request, policy=policy, current_main=current_main, checkout=ROOT,
**({"api": api} if api is not None else {}),
)
require_runtime_compatibility(
manifest["commit"], checkout=ROOT, policy_file=POLICY_FILE,
**({"api": api} if api is not None else {}),
)
with deployment_lock():
require_runtime_compatibility(
manifest["commit"], checkout=ROOT, policy_file=POLICY_FILE,
**({"api": api} if api is not None else {}),
)
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
current = image_ids()
current_record = {service: current[service] for service in SERVICES}
revision = deploy_release_revision()
previous_state = None
if STATE_FILE.exists() or STATE_FILE.is_symlink():
previous_state = validate_state(read_json(STATE_FILE))
if previous_state["current"] != current_record:
raise ValueError("current deployment does not match the recorded release state")
if (previous_state["commit"] == manifest["commit"]
and previous_state["manifest_sha256"] == request["descriptor_sha256"]):
raise ValueError("release is already applied")
journal = {
"schema_version": 2, "status": "external-backup-pending", "commit": manifest["commit"],
"manifest_sha256": request["descriptor_sha256"], "previous": current_record,
}
write_json_atomic(JOURNAL_FILE, journal)
try:
compose("stop")
sources, stopped_revision, stopped_images, deployment_files = compose_sources(COMPOSE_FILE, ENV_FILE)
if stopped_revision != revision or stopped_images.get("backend") != current["backend"] or stopped_images.get("opensearch") != current["opensearch"]:
raise ValueError("stopped production inventory does not match its pre-backup snapshot")
backup_identity = send_backup_session(
writer, reader, sources=sources, revision=stopped_revision,
images=stopped_images, deployment_files=deployment_files,
context={key: request[key] for key in ("action", "run_id", "commit", "descriptor_sha256")},
)
except Exception as error:
try:
restart_current(current)
JOURNAL_FILE.unlink(missing_ok=True)
except Exception as recovery_error:
journal.update({"status": "external-backup-failed",
"error_type": type(error).__name__,
"recovery_error_type": type(recovery_error).__name__})
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"external backup failed and current services need recovery; journal at {JOURNAL_FILE}") from error
raise
backup_record = {key: backup_identity[key] for key in
("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")}
journal = {
"schema_version": 2, "status": "external-backup-verified", "commit": manifest["commit"],
"manifest_sha256": request["descriptor_sha256"], "backup": backup_record,
"previous": current_record,
}
write_json_atomic(JOURNAL_FILE, journal)
try:
deploy_external(
manifest, policy, backup_identity["manifest"], backup_record,
request["descriptor_sha256"], current_record, canary_ru, canary_ky,
)
except Exception as error:
current_journal = read_json(JOURNAL_FILE) if JOURNAL_FILE.exists() else {}
if current_journal.get("status") == "external-backup-verified":
try:
restart_current(current)
JOURNAL_FILE.unlink()
except Exception as recovery_error:
current_journal["status"] = "pre-cutover-recovery-needed"
current_journal["error_type"] = type(error).__name__
current_journal["recovery_error_type"] = type(recovery_error).__name__
write_json_atomic(JOURNAL_FILE, current_journal)
raise ValueError(f"pre-cutover recovery failed; journal at {JOURNAL_FILE}") from error
raise
return {"backup_id": backup_record["backup_id"], "commit": manifest["commit"]}
def deploy_release_revision() -> str:
from deploy_release import run
return run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
STATE_FILE = Path("/opt/akyldash/release-state.json")
def main() -> int:
try:
request = read_request(sys.stdin.buffer)
result = execute(request, reader=sys.stdin.buffer, writer=sys.stdout.buffer)
write_result(sys.stdout.buffer, status="success", message=result["backup_id"])
except Exception as error:
print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr)
try:
write_result(sys.stdout.buffer, status="error", message=f"deployment failed ({type(error).__name__})")
except (BrokenPipeError, OSError):
pass
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())