127 lines
6.0 KiB
Python
127 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Root-side rollback using a pinned external backup and a fresh external backup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
|
|
from archive import compose_sources
|
|
from deployment_lock import deployment_lock
|
|
from deploy_release import COMPOSE_FILE, ENV_FILE, JOURNAL_FILE, ROOT, compose, image_ids
|
|
from external_session import (nonce, read_ack, read_request,
|
|
receive_archive, send_backup_session, write_challenge,
|
|
write_result)
|
|
from release_state import (read_json, validate_state, write_json_atomic)
|
|
from rollback import rollback_external
|
|
|
|
STATE_FILE = ROOT.parent / "release-state.json"
|
|
|
|
|
|
class Discard:
|
|
def write(self, chunk: bytes) -> int:
|
|
return len(chunk)
|
|
|
|
def flush(self) -> None:
|
|
pass
|
|
|
|
|
|
def execute(*, reader, writer) -> str:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("the fixed external rollback receiver must run as root")
|
|
request = read_request(reader)
|
|
if request != {"version": 1, "action": "rollback", "data_compatible": True}:
|
|
raise ValueError("external rollback request is invalid")
|
|
if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink():
|
|
raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before rollback")
|
|
|
|
with deployment_lock():
|
|
state = validate_state(read_json(STATE_FILE))
|
|
if state["schema_version"] != 2 or state["previous_backup"] is None:
|
|
raise ValueError("release state has no verified external previous backup")
|
|
target = state["previous_backup"]
|
|
session_nonce = nonce()
|
|
write_challenge(writer, target, session_nonce=session_nonce)
|
|
size, digest = receive_archive(reader, Discard(), maximum=target["size"])
|
|
if size != target["size"] or digest != target["archive_sha256"]:
|
|
raise ValueError("rollback bytes do not match the root-pinned external backup")
|
|
read_ack(reader, session_nonce=session_nonce, backup_id=target["backup_id"], size=size,
|
|
archive_sha256=digest, manifest_sha256=target["manifest_sha256"])
|
|
|
|
current = image_ids()
|
|
if current != state["current"]:
|
|
raise ValueError("current deployment does not match root-pinned release state")
|
|
journal = {"schema_version": 2, "status": "external-backup-pending", "previous": current}
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
try:
|
|
compose("stop")
|
|
sources, revision, stopped_images, deployment_files = compose_sources(COMPOSE_FILE, ENV_FILE)
|
|
if any(stopped_images.get(service) != record for service, record in current.items()):
|
|
raise ValueError("stopped deployment does not match its pre-rollback snapshot")
|
|
fresh = send_backup_session(
|
|
writer, reader, sources=sources, revision=revision, images=stopped_images,
|
|
deployment_files=deployment_files,
|
|
context={"action": "rollback", "backup_id": target["backup_id"],
|
|
"archive_sha256": target["archive_sha256"],
|
|
"manifest_sha256": target["manifest_sha256"]},
|
|
)
|
|
except Exception as error:
|
|
try:
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
if image_ids() != current:
|
|
raise ValueError("restarted image IDs differ from the pre-rollback deployment")
|
|
JOURNAL_FILE.unlink(missing_ok=True)
|
|
except Exception as recovery_error:
|
|
journal.update({"status": "external-backup-failed",
|
|
"error_type": type(error).__name__,
|
|
"recovery_error_type": type(recovery_error).__name__})
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"rollback backup failed and services need recovery; journal at {JOURNAL_FILE}") from error
|
|
raise
|
|
|
|
fresh_record = {key: fresh[key] for key in
|
|
("backup_id", "size", "archive_sha256", "manifest_sha256", "manifest")}
|
|
write_json_atomic(JOURNAL_FILE, {
|
|
"schema_version": 2, "status": "external-backup-verified",
|
|
"backup": fresh_record, "previous": current,
|
|
})
|
|
try:
|
|
rollback_external(state=state, target_identity=target, current=current,
|
|
fresh_backup=fresh_record, data_compatible=True)
|
|
except Exception as error:
|
|
journal = read_json(JOURNAL_FILE) if JOURNAL_FILE.exists() else {}
|
|
if journal.get("status") == "external-backup-verified":
|
|
try:
|
|
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
|
if image_ids() != current:
|
|
raise ValueError("restarted image IDs differ from the pre-rollback deployment")
|
|
JOURNAL_FILE.unlink()
|
|
except Exception as recovery_error:
|
|
journal.update({
|
|
"status": "pre-cutover-recovery-needed",
|
|
"error_type": type(error).__name__,
|
|
"recovery_error_type": type(recovery_error).__name__,
|
|
})
|
|
write_json_atomic(JOURNAL_FILE, journal)
|
|
raise ValueError(f"rollback preflight failed and services need recovery; journal at {JOURNAL_FILE}") from error
|
|
raise
|
|
return fresh_record["backup_id"]
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
result = execute(reader=sys.stdin.buffer, writer=sys.stdout.buffer)
|
|
write_result(sys.stdout.buffer, status="success", message=result)
|
|
return 0
|
|
except Exception as error:
|
|
print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr)
|
|
try:
|
|
write_result(sys.stdout.buffer, status="error", message=f"rollback failed ({type(error).__name__})")
|
|
except OSError:
|
|
pass
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|