Files
akyldash/infra/backup/external_session.py

339 lines
15 KiB
Python

"""Bounded binary protocol for one live owner-controlled external backup session."""
from __future__ import annotations
import hashlib
import json
import re
import secrets
import struct
import datetime as dt
from typing import BinaryIO
MAGIC = b"AKYL"
ACK_MAGIC = b"AKYA"
CHALLENGE_MAGIC = b"AKYB"
RESULT_MAGIC = b"AKYR"
VERSION = 1
CHUNK_BYTES = 1024 * 1024
MAX_CONTROL_BYTES = 64 * 1024
MAX_ARCHIVE_BYTES = 100 * 1024**3
BACKUP_ID_RE = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}\Z")
SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
NONCE_RE = re.compile(r"[0-9a-f]{64}\Z")
def nonce() -> str:
return secrets.token_hex(32)
def _json_frame(writer: BinaryIO, magic: bytes, value: dict) -> None:
payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
if len(payload) > MAX_CONTROL_BYTES:
raise ValueError("external backup control message exceeds its size limit")
writer.write(magic + struct.pack("!I", len(payload)) + payload)
writer.flush()
def _read_exact(reader: BinaryIO, length: int) -> bytes:
parts = bytearray()
while len(parts) < length:
block = reader.read(length - len(parts))
if not block:
raise ValueError("external backup session ended before a complete protocol frame")
parts.extend(block)
return bytes(parts)
def _read_json_frame(reader: BinaryIO, magic: bytes) -> dict:
if _read_exact(reader, len(magic)) != magic:
raise ValueError("external backup protocol version or frame type is invalid")
return _read_json_after_magic(reader)
def _read_json_after_magic(reader: BinaryIO) -> dict:
size = struct.unpack("!I", _read_exact(reader, 4))[0]
if size > MAX_CONTROL_BYTES:
raise ValueError("external backup control message exceeds its size limit")
try:
value = json.loads(_read_exact(reader, size), object_pairs_hook=_unique_object)
except (UnicodeError, json.JSONDecodeError) as error:
raise ValueError("external backup control message is invalid JSON") from error
if not isinstance(value, dict):
raise ValueError("external backup control message must be a JSON object")
return value
def _unique_object(pairs: list[tuple[str, object]]) -> dict:
result = {}
for key, value in pairs:
if key in result:
raise ValueError(f"duplicate external backup field: {key}")
result[key] = value
return result
def write_request(writer: BinaryIO, request: dict) -> None:
validate_request(request)
_json_frame(writer, MAGIC, request)
def read_request(reader: BinaryIO) -> dict:
request = _read_json_frame(reader, MAGIC)
validate_request(request)
return request
def validate_request(request: object) -> dict:
if (not isinstance(request, dict) or type(request.get("version")) is not int
or request["version"] != VERSION):
raise ValueError("external backup request has an unsupported version")
if request.get("action") == "release" and set(request) == {
"version", "action", "run_id", "commit", "descriptor_sha256"
}:
if (not isinstance(request["run_id"], str) or not request["run_id"].isdigit()
or not isinstance(request["commit"], str) or not re.fullmatch(r"[0-9a-f]{40}", request["commit"])
or not isinstance(request["descriptor_sha256"], str)
or not SHA256_RE.fullmatch(request["descriptor_sha256"])):
raise ValueError("external deployment request identity is invalid")
return request
if request.get("action") == "rollback" and set(request) == {
"version", "action", "data_compatible"
} and request["data_compatible"] is True:
return request
raise ValueError("external deployment request has unexpected fields or action")
def read_header(reader: BinaryIO) -> dict:
header = _read_json_frame(reader, MAGIC)
return validate_header(header)
def validate_header(header: object) -> dict:
if not isinstance(header, dict):
raise ValueError("external backup header is invalid")
if set(header) != {"version", "nonce", "backup_id", "manifest", "manifest_sha256", "context"}:
raise ValueError("external backup header has unexpected fields")
if (type(header["version"]) is not int or header["version"] != VERSION
or not isinstance(header["nonce"], str)
or not NONCE_RE.fullmatch(header["nonce"]) or not isinstance(header["backup_id"], str)
or not BACKUP_ID_RE.fullmatch(header["backup_id"]) or not isinstance(header["manifest"], dict)
or not isinstance(header["manifest_sha256"], str)
or not SHA256_RE.fullmatch(header["manifest_sha256"])):
raise ValueError("external backup header is invalid")
canonical = json.dumps(header["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
if hashlib.sha256(canonical).hexdigest() != header["manifest_sha256"]:
raise ValueError("external backup manifest identity does not match its header")
if not isinstance(header["context"], dict):
raise ValueError("external backup context is invalid")
return header
def read_response(reader: BinaryIO) -> tuple[str, dict]:
magic = _read_exact(reader, 4)
value = _read_json_after_magic(reader) if magic in {MAGIC, RESULT_MAGIC} else None
if magic == MAGIC:
return "header", validate_header(value)
if magic == RESULT_MAGIC:
result = value
if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int
or result["version"] != VERSION
or result["status"] not in {"success", "error"} or not isinstance(result["message"], str)):
raise ValueError("external backup final result is invalid")
return "result", result
raise ValueError("external deployment response frame is invalid")
def write_header(writer: BinaryIO, *, session_nonce: str, backup_id: str, manifest: dict,
context: dict | None = None) -> str:
if not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id):
raise ValueError("external backup session identity is invalid")
manifest_bytes = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest()
_json_frame(writer, MAGIC, {
"version": VERSION,
"nonce": session_nonce,
"backup_id": backup_id,
"manifest": manifest,
"manifest_sha256": manifest_sha256,
"context": context or {},
})
return manifest_sha256
class ArchiveFrameWriter:
def __init__(self, writer: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES):
self.writer = writer
self.maximum = maximum
self.size = 0
self.digest = hashlib.sha256()
self.buffer = bytearray()
self.closed = False
def write(self, data: bytes) -> int:
if self.closed:
raise ValueError("external backup stream is already closed")
self.size += len(data)
if self.size > self.maximum:
raise ValueError("external backup archive exceeds its size limit")
self.digest.update(data)
remaining = memoryview(data)
while remaining:
take = min(CHUNK_BYTES - len(self.buffer), len(remaining))
self.buffer.extend(remaining[:take])
remaining = remaining[take:]
if len(self.buffer) == CHUNK_BYTES:
self._flush_chunk(CHUNK_BYTES)
return len(data)
def _flush_chunk(self, size: int) -> None:
block = bytes(self.buffer[:size])
del self.buffer[:size]
self.writer.write(struct.pack("!I", len(block)))
self.writer.write(block)
def finish(self) -> tuple[int, str]:
if self.closed:
raise ValueError("external backup stream is already closed")
if self.buffer:
self._flush_chunk(len(self.buffer))
digest = self.digest.hexdigest()
self.writer.write(struct.pack("!I", 0) + struct.pack("!Q", self.size) + bytes.fromhex(digest))
self.writer.flush()
self.closed = True
return self.size, digest
def receive_archive(reader: BinaryIO, output: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES) -> tuple[int, str]:
digest = hashlib.sha256()
size = 0
while True:
length = struct.unpack("!I", _read_exact(reader, 4))[0]
if length == 0:
expected_size = struct.unpack("!Q", _read_exact(reader, 8))[0]
expected_digest = _read_exact(reader, 32).hex()
break
if length > CHUNK_BYTES or size + length > maximum:
raise ValueError("external backup archive frame exceeds its size limit")
chunk = _read_exact(reader, length)
output.write(chunk)
digest.update(chunk)
size += length
output.flush()
actual_digest = digest.hexdigest()
if size != expected_size or actual_digest != expected_digest:
raise ValueError("external backup byte count or SHA-256 does not match its stream trailer")
return size, actual_digest
def write_ack(writer: BinaryIO, *, session_nonce: str, backup_id: str, size: int,
archive_sha256: str, manifest_sha256: str, verified: bool) -> None:
if (not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id)
or type(size) is not int or not 0 < size <= MAX_ARCHIVE_BYTES
or not SHA256_RE.fullmatch(archive_sha256) or not SHA256_RE.fullmatch(manifest_sha256)
or verified is not True):
raise ValueError("external backup acknowledgement is invalid")
_json_frame(writer, ACK_MAGIC, {
"version": VERSION,
"nonce": session_nonce,
"backup_id": backup_id,
"size": size,
"archive_sha256": archive_sha256,
"manifest_sha256": manifest_sha256,
"verified": verified,
})
def write_challenge(writer: BinaryIO, identity: dict, *, session_nonce: str) -> None:
if (not NONCE_RE.fullmatch(session_nonce) or not isinstance(identity, dict)
or set(identity) != {"backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"}):
raise ValueError("external rollback challenge is invalid")
if (not isinstance(identity["backup_id"], str) or not BACKUP_ID_RE.fullmatch(identity["backup_id"])
or type(identity["size"]) is not int or not 0 < identity["size"] <= MAX_ARCHIVE_BYTES
or not isinstance(identity["archive_sha256"], str) or not SHA256_RE.fullmatch(identity["archive_sha256"])
or not isinstance(identity["manifest_sha256"], str) or not SHA256_RE.fullmatch(identity["manifest_sha256"])
or not isinstance(identity["manifest"], dict)):
raise ValueError("external rollback backup identity is invalid")
canonical = json.dumps(identity["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
if hashlib.sha256(canonical).hexdigest() != identity["manifest_sha256"]:
raise ValueError("external rollback manifest identity is inconsistent")
_json_frame(writer, CHALLENGE_MAGIC, {"version": VERSION, "nonce": session_nonce, **identity})
def read_challenge(reader: BinaryIO) -> dict:
value = _read_json_frame(reader, CHALLENGE_MAGIC)
if (set(value) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"}
or type(value["version"]) is not int or value["version"] != VERSION
or not isinstance(value["nonce"], str)
or not NONCE_RE.fullmatch(value["nonce"]) or not isinstance(value["backup_id"], str)
or not BACKUP_ID_RE.fullmatch(value["backup_id"]) or type(value["size"]) is not int
or not 0 < value["size"] <= MAX_ARCHIVE_BYTES
or not isinstance(value["archive_sha256"], str) or not SHA256_RE.fullmatch(value["archive_sha256"])
or not isinstance(value["manifest_sha256"], str) or not SHA256_RE.fullmatch(value["manifest_sha256"])
or not isinstance(value["manifest"], dict)):
raise ValueError("external rollback challenge is invalid")
canonical = json.dumps(value["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
if hashlib.sha256(canonical).hexdigest() != value["manifest_sha256"]:
raise ValueError("external rollback challenge manifest identity is inconsistent")
return value
def read_ack(reader: BinaryIO, *, session_nonce: str, backup_id: str, size: int,
archive_sha256: str, manifest_sha256: str) -> dict:
ack = _read_json_frame(reader, ACK_MAGIC)
if (set(ack) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "verified"}
or type(ack.get("version")) is not int or type(ack.get("size")) is not int
or ack.get("verified") is not True):
raise ValueError("external backup acknowledgement is invalid")
expected = {
"version": VERSION, "nonce": session_nonce, "backup_id": backup_id,
"size": size, "archive_sha256": archive_sha256,
"manifest_sha256": manifest_sha256, "verified": True,
}
if ack != expected:
raise ValueError("external backup acknowledgement does not match this live session")
return ack
def write_result(writer: BinaryIO, *, status: str, message: str = "") -> None:
if status not in {"success", "error"} or len(message.encode("utf-8")) > 4096:
raise ValueError("external backup result is invalid")
_json_frame(writer, RESULT_MAGIC, {"version": VERSION, "status": status, "message": message})
def read_result(reader: BinaryIO) -> dict:
result = _read_json_frame(reader, RESULT_MAGIC)
if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int
or result["version"] != VERSION or result["status"] not in {"success", "error"}
or not isinstance(result["message"], str)):
raise ValueError("external backup final result is invalid")
return result
def send_backup_session(writer: BinaryIO, reader: BinaryIO, *, sources: dict, revision: str,
images: dict, deployment_files: dict[str, str], context: dict) -> dict:
from archive import backup_manifest, create_backup_stream
manifest = backup_manifest(sources, revision, images, deployment_files)
session_nonce = nonce()
backup_id = "akyldash-" + dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + session_nonce[:8]
manifest_sha256 = write_header(
writer, session_nonce=session_nonce, backup_id=backup_id, manifest=manifest, context=context
)
framed = ArchiveFrameWriter(writer)
create_backup_stream(
framed, sources, revision, images, deployment_files, manifest=manifest
)
size, archive_sha256 = framed.finish()
read_ack(
reader, session_nonce=session_nonce, backup_id=backup_id, size=size,
archive_sha256=archive_sha256, manifest_sha256=manifest_sha256,
)
return {
"backup_id": backup_id,
"archive_sha256": archive_sha256,
"manifest_sha256": manifest_sha256,
"size": size,
"manifest": manifest,
}