Files
akyldash/infra/deploy/external_rollback.py

127 lines
6.0 KiB
Python

#!/usr/bin/env python3
"""Root-side rollback using a pinned external backup and a fresh external backup."""
from __future__ import annotations
import os
import sys
from archive import compose_sources
from deployment_lock import deployment_lock
from deploy_release import COMPOSE_FILE, ENV_FILE, JOURNAL_FILE, ROOT, compose, image_ids
from external_session import (nonce, read_ack, read_request,
receive_archive, send_backup_session, write_challenge,
write_result)
from release_state import (read_json, validate_state, write_json_atomic)
from rollback import rollback_external
STATE_FILE = ROOT.parent / "release-state.json"
class Discard:
def write(self, chunk: bytes) -> int:
return len(chunk)
def flush(self) -> None:
pass
def execute(*, reader, writer) -> str:
if os.geteuid() != 0:
raise ValueError("the fixed external rollback receiver must run as root")
request = read_request(reader)
if request != {"version": 1, "action": "rollback", "data_compatible": True}:
raise ValueError("external rollback request is invalid")
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before rollback")
with deployment_lock():
state = validate_state(read_json(STATE_FILE))
if state["schema_version"] != 2 or state["previous_backup"] is None:
raise ValueError("release state has no verified external previous backup")
target = state["previous_backup"]
session_nonce = nonce()
write_challenge(writer, target, session_nonce=session_nonce)
size, digest = receive_archive(reader, Discard(), maximum=target["size"])
if size != target["size"] or digest != target["archive_sha256"]:
raise ValueError("rollback bytes do not match the root-pinned external backup")
read_ack(reader, session_nonce=session_nonce, backup_id=target["backup_id"], size=size,
archive_sha256=digest, manifest_sha256=target["manifest_sha256"])
current = image_ids()
if current != state["current"]:
raise ValueError("current deployment does not match root-pinned release state")
journal = {"schema_version": 2, "status": "external-backup-pending", "previous": current}
write_json_atomic(JOURNAL_FILE, journal)
try:
compose("stop")
sources, revision, stopped_images, deployment_files = compose_sources(COMPOSE_FILE, ENV_FILE)
if any(stopped_images.get(service) != record for service, record in current.items()):
raise ValueError("stopped deployment does not match its pre-rollback snapshot")
fresh = send_backup_session(
writer, reader, sources=sources, revision=revision, images=stopped_images,
deployment_files=deployment_files,
context={"action": "rollback", "backup_id": target["backup_id"],
"archive_sha256": target["archive_sha256"],
"manifest_sha256": target["manifest_sha256"]},
)
except Exception as error:
try:
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
if image_ids() != current:
raise ValueError("restarted image IDs differ from the pre-rollback deployment")
JOURNAL_FILE.unlink(missing_ok=True)
except Exception as recovery_error:
journal.update({"status": "external-backup-failed",
"error_type": type(error).__name__,
"recovery_error_type": type(recovery_error).__name__})
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"rollback backup failed and services need recovery; journal at {JOURNAL_FILE}") from error
raise
fresh_record = {key: fresh[key] for key in
("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")}
write_json_atomic(JOURNAL_FILE, {
"schema_version": 2, "status": "external-backup-verified",
"backup": fresh_record, "previous": current,
})
try:
rollback_external(state=state, target_identity=target, current=current,
fresh_backup=fresh_record, data_compatible=True)
except Exception as error:
journal = read_json(JOURNAL_FILE) if JOURNAL_FILE.exists() else {}
if journal.get("status") == "external-backup-verified":
try:
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
if image_ids() != current:
raise ValueError("restarted image IDs differ from the pre-rollback deployment")
JOURNAL_FILE.unlink()
except Exception as recovery_error:
journal.update({
"status": "pre-cutover-recovery-needed",
"error_type": type(error).__name__,
"recovery_error_type": type(recovery_error).__name__,
})
write_json_atomic(JOURNAL_FILE, journal)
raise ValueError(f"rollback preflight failed and services need recovery; journal at {JOURNAL_FILE}") from error
raise
return fresh_record["backup_id"]
def main() -> int:
try:
result = execute(reader=sys.stdin.buffer, writer=sys.stdout.buffer)
write_result(sys.stdout.buffer, status="success", message=result)
return 0
except Exception as error:
print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr)
try:
write_result(sys.stdout.buffer, status="error", message=f"rollback failed ({type(error).__name__})")
except OSError:
pass
return 1
if __name__ == "__main__":
raise SystemExit(main())