161 lines
7.7 KiB
Python
161 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Fixed root-side Gitea release receiver and external-backup cutover."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from archive import compose_sources
|
|
from deployment_lock import deployment_lock
|
|
from deploy_release import JOURNAL_FILE, ROOT, SERVICES, compose, deploy_external, image_ids
|
|
from external_session import read_request, send_backup_session, validate_request, write_result
|
|
from release_source import fetch_release, require_runtime_compatibility
|
|
from release_state import read_json, validate_state, write_json_atomic
|
|
|
|
POLICY_FILE = Path("/etc/akyldash/release-policy.json")
|
|
CANARIES_FILE = Path("/etc/akyldash/release-canaries.json")
|
|
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
|
ENV_FILE = Path("/etc/akyldash/production.env")
|
|
|
|
|
|
def owner_canaries() -> tuple[str, str]:
|
|
info = CANARIES_FILE.lstat()
|
|
if (CANARIES_FILE.is_symlink() or not CANARIES_FILE.is_file() or info.st_uid != 0
|
|
or info.st_mode & 0o777 != 0o600):
|
|
raise ValueError("release canaries must be stored in a root-owned mode-0600 file")
|
|
value = json.loads(CANARIES_FILE.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict) or set(value) != {"ru", "ky"}:
|
|
raise ValueError("release canary configuration must contain exactly ru and ky")
|
|
queries = (value["ru"], value["ky"])
|
|
if any(not isinstance(query, str) or not query.strip() or len(query) > 256 for query in queries):
|
|
raise ValueError("release canaries must be non-empty strings of at most 256 characters")
|
|
return queries
|
|
|
|
|
|
def restart_current(current: dict) -> None:
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
if image_ids() != current:
|
|
raise ValueError("restarted service image IDs do not match the pre-backup deployment")
|
|
|
|
|
|
def execute(request: dict, *, reader, writer, policy: dict | None = None, api=None,
|
|
current_main: str | None = None, canaries: tuple[str, str] | None = None) -> dict:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("the fixed external deployment receiver must run as root")
|
|
validate_request(request)
|
|
policy = policy if policy is not None else read_json(POLICY_FILE)
|
|
canary_ru, canary_ky = canaries or owner_canaries()
|
|
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
|
|
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
|
|
manifest, _manifest_bytes = fetch_release(
|
|
request, policy=policy, current_main=current_main, checkout=ROOT,
|
|
**({"api": api} if api is not None else {}),
|
|
)
|
|
require_runtime_compatibility(
|
|
manifest["commit"], checkout=ROOT, policy_file=POLICY_FILE,
|
|
**({"api": api} if api is not None else {}),
|
|
)
|
|
|
|
with deployment_lock():
|
|
require_runtime_compatibility(
|
|
manifest["commit"], checkout=ROOT, policy_file=POLICY_FILE,
|
|
**({"api": api} if api is not None else {}),
|
|
)
|
|
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
|
|
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release")
|
|
current = image_ids()
|
|
current_record = {service: current[service] for service in SERVICES}
|
|
revision = deploy_release_revision()
|
|
previous_state = None
|
|
if STATE_FILE.exists() or STATE_FILE.is_symlink():
|
|
previous_state = validate_state(read_json(STATE_FILE))
|
|
if previous_state["current"] != current_record:
|
|
raise ValueError("current deployment does not match the recorded release state")
|
|
if (previous_state["commit"] == manifest["commit"]
|
|
and previous_state["manifest_sha256"] == request["descriptor_sha256"]):
|
|
raise ValueError("release is already applied")
|
|
journal = {
|
|
"schema_version": 2, "status": "external-backup-pending", "commit": manifest["commit"],
|
|
"manifest_sha256": request["descriptor_sha256"], "previous": current_record,
|
|
}
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
try:
|
|
compose("stop")
|
|
sources, stopped_revision, stopped_images, deployment_files = compose_sources(COMPOSE_FILE, ENV_FILE)
|
|
if stopped_revision != revision or stopped_images.get("backend") != current["backend"] or stopped_images.get("opensearch") != current["opensearch"]:
|
|
raise ValueError("stopped production inventory does not match its pre-backup snapshot")
|
|
backup_identity = send_backup_session(
|
|
writer, reader, sources=sources, revision=stopped_revision,
|
|
images=stopped_images, deployment_files=deployment_files,
|
|
context={key: request[key] for key in ("action", "run_id", "commit", "descriptor_sha256")},
|
|
)
|
|
except Exception as error:
|
|
try:
|
|
restart_current(current)
|
|
JOURNAL_FILE.unlink(missing_ok=True)
|
|
except Exception as recovery_error:
|
|
journal.update({"status": "external-backup-failed",
|
|
"error_type": type(error).__name__,
|
|
"recovery_error_type": type(recovery_error).__name__})
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"external backup failed and current services need recovery; journal at {JOURNAL_FILE}") from error
|
|
raise
|
|
|
|
backup_record = {key: backup_identity[key] for key in
|
|
("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")}
|
|
journal = {
|
|
"schema_version": 2, "status": "external-backup-verified", "commit": manifest["commit"],
|
|
"manifest_sha256": request["descriptor_sha256"], "backup": backup_record,
|
|
"previous": current_record,
|
|
}
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
try:
|
|
deploy_external(
|
|
manifest, policy, backup_identity["manifest"], backup_record,
|
|
request["descriptor_sha256"], current_record, canary_ru, canary_ky,
|
|
)
|
|
except Exception as error:
|
|
current_journal = read_json(JOURNAL_FILE) if JOURNAL_FILE.exists() else {}
|
|
if current_journal.get("status") == "external-backup-verified":
|
|
try:
|
|
restart_current(current)
|
|
JOURNAL_FILE.unlink()
|
|
except Exception as recovery_error:
|
|
current_journal["status"] = "pre-cutover-recovery-needed"
|
|
current_journal["error_type"] = type(error).__name__
|
|
current_journal["recovery_error_type"] = type(recovery_error).__name__
|
|
write_json_atomic(JOURNAL_FILE, current_journal)
|
|
raise ValueError(f"pre-cutover recovery failed; journal at {JOURNAL_FILE}") from error
|
|
raise
|
|
return {"backup_id": backup_record["backup_id"], "commit": manifest["commit"]}
|
|
|
|
|
|
def deploy_release_revision() -> str:
|
|
from deploy_release import run
|
|
return run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
|
|
|
|
|
|
STATE_FILE = Path("/opt/akyldash/release-state.json")
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
request = read_request(sys.stdin.buffer)
|
|
result = execute(request, reader=sys.stdin.buffer, writer=sys.stdout.buffer)
|
|
write_result(sys.stdout.buffer, status="success", message=result["backup_id"])
|
|
except Exception as error:
|
|
print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr)
|
|
try:
|
|
write_result(sys.stdout.buffer, status="error", message=f"deployment failed ({type(error).__name__})")
|
|
except (BrokenPipeError, OSError):
|
|
pass
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|