"""Bounded binary protocol for one live owner-controlled external backup session.""" from __future__ import annotations import hashlib import json import re import secrets import struct import datetime as dt from typing import BinaryIO MAGIC = b"AKYL" ACK_MAGIC = b"AKYA" CHALLENGE_MAGIC = b"AKYB" RESULT_MAGIC = b"AKYR" VERSION = 1 CHUNK_BYTES = 1024 * 1024 MAX_CONTROL_BYTES = 64 * 1024 MAX_ARCHIVE_BYTES = 100 * 1024**3 BACKUP_ID_RE = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") NONCE_RE = re.compile(r"[0-9a-f]{64}\Z") def nonce() -> str: return secrets.token_hex(32) def _json_frame(writer: BinaryIO, magic: bytes, value: dict) -> None: payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") if len(payload) > MAX_CONTROL_BYTES: raise ValueError("external backup control message exceeds its size limit") writer.write(magic + struct.pack("!I", len(payload)) + payload) writer.flush() def _read_exact(reader: BinaryIO, length: int) -> bytes: parts = bytearray() while len(parts) < length: block = reader.read(length - len(parts)) if not block: raise ValueError("external backup session ended before a complete protocol frame") parts.extend(block) return bytes(parts) def _read_json_frame(reader: BinaryIO, magic: bytes) -> dict: if _read_exact(reader, len(magic)) != magic: raise ValueError("external backup protocol version or frame type is invalid") return _read_json_after_magic(reader) def _read_json_after_magic(reader: BinaryIO) -> dict: size = struct.unpack("!I", _read_exact(reader, 4))[0] if size > MAX_CONTROL_BYTES: raise ValueError("external backup control message exceeds its size limit") try: value = json.loads(_read_exact(reader, size), object_pairs_hook=_unique_object) except (UnicodeError, json.JSONDecodeError) as error: raise ValueError("external backup control message is invalid JSON") from error if not isinstance(value, dict): raise ValueError("external backup control message must be a JSON object") return value def _unique_object(pairs: list[tuple[str, object]]) -> dict: result = {} for key, value in pairs: if key in result: raise ValueError(f"duplicate external backup field: {key}") result[key] = value return result def write_request(writer: BinaryIO, request: dict) -> None: validate_request(request) _json_frame(writer, MAGIC, request) def read_request(reader: BinaryIO) -> dict: request = _read_json_frame(reader, MAGIC) validate_request(request) return request def validate_request(request: object) -> dict: if (not isinstance(request, dict) or type(request.get("version")) is not int or request["version"] != VERSION): raise ValueError("external backup request has an unsupported version") if request.get("action") == "release" and set(request) == { "version", "action", "run_id", "commit", "descriptor_sha256" }: if (not isinstance(request["run_id"], str) or not request["run_id"].isdigit() or not isinstance(request["commit"], str) or not re.fullmatch(r"[0-9a-f]{40}", request["commit"]) or not isinstance(request["descriptor_sha256"], str) or not SHA256_RE.fullmatch(request["descriptor_sha256"])): raise ValueError("external deployment request identity is invalid") return request if request.get("action") == "rollback" and set(request) == { "version", "action", "data_compatible" } and request["data_compatible"] is True: return request raise ValueError("external deployment request has unexpected fields or action") def read_header(reader: BinaryIO) -> dict: header = _read_json_frame(reader, MAGIC) return validate_header(header) def validate_header(header: object) -> dict: if not isinstance(header, dict): raise ValueError("external backup header is invalid") if set(header) != {"version", "nonce", "backup_id", "manifest", "manifest_sha256", "context"}: raise ValueError("external backup header has unexpected fields") if (type(header["version"]) is not int or header["version"] != VERSION or not isinstance(header["nonce"], str) or not NONCE_RE.fullmatch(header["nonce"]) or not isinstance(header["backup_id"], str) or not BACKUP_ID_RE.fullmatch(header["backup_id"]) or not isinstance(header["manifest"], dict) or not isinstance(header["manifest_sha256"], str) or not SHA256_RE.fullmatch(header["manifest_sha256"])): raise ValueError("external backup header is invalid") canonical = json.dumps(header["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8") if hashlib.sha256(canonical).hexdigest() != header["manifest_sha256"]: raise ValueError("external backup manifest identity does not match its header") if not isinstance(header["context"], dict): raise ValueError("external backup context is invalid") return header def read_response(reader: BinaryIO) -> tuple[str, dict]: magic = _read_exact(reader, 4) value = _read_json_after_magic(reader) if magic in {MAGIC, RESULT_MAGIC} else None if magic == MAGIC: return "header", validate_header(value) if magic == RESULT_MAGIC: result = value if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int or result["version"] != VERSION or result["status"] not in {"success", "error"} or not isinstance(result["message"], str)): raise ValueError("external backup final result is invalid") return "result", result raise ValueError("external deployment response frame is invalid") def write_header(writer: BinaryIO, *, session_nonce: str, backup_id: str, manifest: dict, context: dict | None = None) -> str: if not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id): raise ValueError("external backup session identity is invalid") manifest_bytes = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest() _json_frame(writer, MAGIC, { "version": VERSION, "nonce": session_nonce, "backup_id": backup_id, "manifest": manifest, "manifest_sha256": manifest_sha256, "context": context or {}, }) return manifest_sha256 class ArchiveFrameWriter: def __init__(self, writer: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES): self.writer = writer self.maximum = maximum self.size = 0 self.digest = hashlib.sha256() self.buffer = bytearray() self.closed = False def write(self, data: bytes) -> int: if self.closed: raise ValueError("external backup stream is already closed") self.size += len(data) if self.size > self.maximum: raise ValueError("external backup archive exceeds its size limit") self.digest.update(data) remaining = memoryview(data) while remaining: take = min(CHUNK_BYTES - len(self.buffer), len(remaining)) self.buffer.extend(remaining[:take]) remaining = remaining[take:] if len(self.buffer) == CHUNK_BYTES: self._flush_chunk(CHUNK_BYTES) return len(data) def _flush_chunk(self, size: int) -> None: block = bytes(self.buffer[:size]) del self.buffer[:size] self.writer.write(struct.pack("!I", len(block))) self.writer.write(block) def finish(self) -> tuple[int, str]: if self.closed: raise ValueError("external backup stream is already closed") if self.buffer: self._flush_chunk(len(self.buffer)) digest = self.digest.hexdigest() self.writer.write(struct.pack("!I", 0) + struct.pack("!Q", self.size) + bytes.fromhex(digest)) self.writer.flush() self.closed = True return self.size, digest def receive_archive(reader: BinaryIO, output: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES) -> tuple[int, str]: digest = hashlib.sha256() size = 0 while True: length = struct.unpack("!I", _read_exact(reader, 4))[0] if length == 0: expected_size = struct.unpack("!Q", _read_exact(reader, 8))[0] expected_digest = _read_exact(reader, 32).hex() break if length > CHUNK_BYTES or size + length > maximum: raise ValueError("external backup archive frame exceeds its size limit") chunk = _read_exact(reader, length) output.write(chunk) digest.update(chunk) size += length output.flush() actual_digest = digest.hexdigest() if size != expected_size or actual_digest != expected_digest: raise ValueError("external backup byte count or SHA-256 does not match its stream trailer") return size, actual_digest def write_ack(writer: BinaryIO, *, session_nonce: str, backup_id: str, size: int, archive_sha256: str, manifest_sha256: str, verified: bool) -> None: if (not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id) or type(size) is not int or not 0 < size <= MAX_ARCHIVE_BYTES or not SHA256_RE.fullmatch(archive_sha256) or not SHA256_RE.fullmatch(manifest_sha256) or verified is not True): raise ValueError("external backup acknowledgement is invalid") _json_frame(writer, ACK_MAGIC, { "version": VERSION, "nonce": session_nonce, "backup_id": backup_id, "size": size, "archive_sha256": archive_sha256, "manifest_sha256": manifest_sha256, "verified": verified, }) def write_challenge(writer: BinaryIO, identity: dict, *, session_nonce: str) -> None: if (not NONCE_RE.fullmatch(session_nonce) or not isinstance(identity, dict) or set(identity) != {"backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"}): raise ValueError("external rollback challenge is invalid") if (not isinstance(identity["backup_id"], str) or not BACKUP_ID_RE.fullmatch(identity["backup_id"]) or type(identity["size"]) is not int or not 0 < identity["size"] <= MAX_ARCHIVE_BYTES or not isinstance(identity["archive_sha256"], str) or not SHA256_RE.fullmatch(identity["archive_sha256"]) or not isinstance(identity["manifest_sha256"], str) or not SHA256_RE.fullmatch(identity["manifest_sha256"]) or not isinstance(identity["manifest"], dict)): raise ValueError("external rollback backup identity is invalid") canonical = json.dumps(identity["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8") if hashlib.sha256(canonical).hexdigest() != identity["manifest_sha256"]: raise ValueError("external rollback manifest identity is inconsistent") _json_frame(writer, CHALLENGE_MAGIC, {"version": VERSION, "nonce": session_nonce, **identity}) def read_challenge(reader: BinaryIO) -> dict: value = _read_json_frame(reader, CHALLENGE_MAGIC) if (set(value) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"} or type(value["version"]) is not int or value["version"] != VERSION or not isinstance(value["nonce"], str) or not NONCE_RE.fullmatch(value["nonce"]) or not isinstance(value["backup_id"], str) or not BACKUP_ID_RE.fullmatch(value["backup_id"]) or type(value["size"]) is not int or not 0 < value["size"] <= MAX_ARCHIVE_BYTES or not isinstance(value["archive_sha256"], str) or not SHA256_RE.fullmatch(value["archive_sha256"]) or not isinstance(value["manifest_sha256"], str) or not SHA256_RE.fullmatch(value["manifest_sha256"]) or not isinstance(value["manifest"], dict)): raise ValueError("external rollback challenge is invalid") canonical = json.dumps(value["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8") if hashlib.sha256(canonical).hexdigest() != value["manifest_sha256"]: raise ValueError("external rollback challenge manifest identity is inconsistent") return value def read_ack(reader: BinaryIO, *, session_nonce: str, backup_id: str, size: int, archive_sha256: str, manifest_sha256: str) -> dict: ack = _read_json_frame(reader, ACK_MAGIC) if (set(ack) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "verified"} or type(ack.get("version")) is not int or type(ack.get("size")) is not int or ack.get("verified") is not True): raise ValueError("external backup acknowledgement is invalid") expected = { "version": VERSION, "nonce": session_nonce, "backup_id": backup_id, "size": size, "archive_sha256": archive_sha256, "manifest_sha256": manifest_sha256, "verified": True, } if ack != expected: raise ValueError("external backup acknowledgement does not match this live session") return ack def write_result(writer: BinaryIO, *, status: str, message: str = "") -> None: if status not in {"success", "error"} or len(message.encode("utf-8")) > 4096: raise ValueError("external backup result is invalid") _json_frame(writer, RESULT_MAGIC, {"version": VERSION, "status": status, "message": message}) def read_result(reader: BinaryIO) -> dict: result = _read_json_frame(reader, RESULT_MAGIC) if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int or result["version"] != VERSION or result["status"] not in {"success", "error"} or not isinstance(result["message"], str)): raise ValueError("external backup final result is invalid") return result def send_backup_session(writer: BinaryIO, reader: BinaryIO, *, sources: dict, revision: str, images: dict, deployment_files: dict[str, str], context: dict) -> dict: from archive import backup_manifest, create_backup_stream manifest = backup_manifest(sources, revision, images, deployment_files) session_nonce = nonce() backup_id = "akyldash-" + dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + session_nonce[:8] manifest_sha256 = write_header( writer, session_nonce=session_nonce, backup_id=backup_id, manifest=manifest, context=context ) framed = ArchiveFrameWriter(writer) create_backup_stream( framed, sources, revision, images, deployment_files, manifest=manifest ) size, archive_sha256 = framed.finish() read_ack( reader, session_nonce=session_nonce, backup_id=backup_id, size=size, archive_sha256=archive_sha256, manifest_sha256=manifest_sha256, ) return { "backup_id": backup_id, "archive_sha256": archive_sha256, "manifest_sha256": manifest_sha256, "size": size, "manifest": manifest, }