339 lines
15 KiB
Python
339 lines
15 KiB
Python
"""Bounded binary protocol for one live owner-controlled external backup session."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import secrets
|
|
import struct
|
|
import datetime as dt
|
|
from typing import BinaryIO
|
|
|
|
MAGIC = b"AKYL"
|
|
ACK_MAGIC = b"AKYA"
|
|
CHALLENGE_MAGIC = b"AKYB"
|
|
RESULT_MAGIC = b"AKYR"
|
|
VERSION = 1
|
|
CHUNK_BYTES = 1024 * 1024
|
|
MAX_CONTROL_BYTES = 64 * 1024
|
|
MAX_ARCHIVE_BYTES = 100 * 1024**3
|
|
BACKUP_ID_RE = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}\Z")
|
|
SHA256_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
NONCE_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
|
|
|
|
def nonce() -> str:
|
|
return secrets.token_hex(32)
|
|
|
|
|
|
def _json_frame(writer: BinaryIO, magic: bytes, value: dict) -> None:
|
|
payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
if len(payload) > MAX_CONTROL_BYTES:
|
|
raise ValueError("external backup control message exceeds its size limit")
|
|
writer.write(magic + struct.pack("!I", len(payload)) + payload)
|
|
writer.flush()
|
|
|
|
|
|
def _read_exact(reader: BinaryIO, length: int) -> bytes:
|
|
parts = bytearray()
|
|
while len(parts) < length:
|
|
block = reader.read(length - len(parts))
|
|
if not block:
|
|
raise ValueError("external backup session ended before a complete protocol frame")
|
|
parts.extend(block)
|
|
return bytes(parts)
|
|
|
|
|
|
def _read_json_frame(reader: BinaryIO, magic: bytes) -> dict:
|
|
if _read_exact(reader, len(magic)) != magic:
|
|
raise ValueError("external backup protocol version or frame type is invalid")
|
|
return _read_json_after_magic(reader)
|
|
|
|
|
|
def _read_json_after_magic(reader: BinaryIO) -> dict:
|
|
size = struct.unpack("!I", _read_exact(reader, 4))[0]
|
|
if size > MAX_CONTROL_BYTES:
|
|
raise ValueError("external backup control message exceeds its size limit")
|
|
try:
|
|
value = json.loads(_read_exact(reader, size), object_pairs_hook=_unique_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("external backup control message is invalid JSON") from error
|
|
if not isinstance(value, dict):
|
|
raise ValueError("external backup control message must be a JSON object")
|
|
return value
|
|
|
|
|
|
def _unique_object(pairs: list[tuple[str, object]]) -> dict:
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError(f"duplicate external backup field: {key}")
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def write_request(writer: BinaryIO, request: dict) -> None:
|
|
validate_request(request)
|
|
_json_frame(writer, MAGIC, request)
|
|
|
|
|
|
def read_request(reader: BinaryIO) -> dict:
|
|
request = _read_json_frame(reader, MAGIC)
|
|
validate_request(request)
|
|
return request
|
|
|
|
|
|
def validate_request(request: object) -> dict:
|
|
if (not isinstance(request, dict) or type(request.get("version")) is not int
|
|
or request["version"] != VERSION):
|
|
raise ValueError("external backup request has an unsupported version")
|
|
if request.get("action") == "release" and set(request) == {
|
|
"version", "action", "run_id", "commit", "descriptor_sha256"
|
|
}:
|
|
if (not isinstance(request["run_id"], str) or not request["run_id"].isdigit()
|
|
or not isinstance(request["commit"], str) or not re.fullmatch(r"[0-9a-f]{40}", request["commit"])
|
|
or not isinstance(request["descriptor_sha256"], str)
|
|
or not SHA256_RE.fullmatch(request["descriptor_sha256"])):
|
|
raise ValueError("external deployment request identity is invalid")
|
|
return request
|
|
if request.get("action") == "rollback" and set(request) == {
|
|
"version", "action", "data_compatible"
|
|
} and request["data_compatible"] is True:
|
|
return request
|
|
raise ValueError("external deployment request has unexpected fields or action")
|
|
|
|
|
|
def read_header(reader: BinaryIO) -> dict:
|
|
header = _read_json_frame(reader, MAGIC)
|
|
return validate_header(header)
|
|
|
|
|
|
def validate_header(header: object) -> dict:
|
|
if not isinstance(header, dict):
|
|
raise ValueError("external backup header is invalid")
|
|
if set(header) != {"version", "nonce", "backup_id", "manifest", "manifest_sha256", "context"}:
|
|
raise ValueError("external backup header has unexpected fields")
|
|
if (type(header["version"]) is not int or header["version"] != VERSION
|
|
or not isinstance(header["nonce"], str)
|
|
or not NONCE_RE.fullmatch(header["nonce"]) or not isinstance(header["backup_id"], str)
|
|
or not BACKUP_ID_RE.fullmatch(header["backup_id"]) or not isinstance(header["manifest"], dict)
|
|
or not isinstance(header["manifest_sha256"], str)
|
|
or not SHA256_RE.fullmatch(header["manifest_sha256"])):
|
|
raise ValueError("external backup header is invalid")
|
|
canonical = json.dumps(header["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
if hashlib.sha256(canonical).hexdigest() != header["manifest_sha256"]:
|
|
raise ValueError("external backup manifest identity does not match its header")
|
|
if not isinstance(header["context"], dict):
|
|
raise ValueError("external backup context is invalid")
|
|
return header
|
|
|
|
|
|
def read_response(reader: BinaryIO) -> tuple[str, dict]:
|
|
magic = _read_exact(reader, 4)
|
|
value = _read_json_after_magic(reader) if magic in {MAGIC, RESULT_MAGIC} else None
|
|
if magic == MAGIC:
|
|
return "header", validate_header(value)
|
|
if magic == RESULT_MAGIC:
|
|
result = value
|
|
if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int
|
|
or result["version"] != VERSION
|
|
or result["status"] not in {"success", "error"} or not isinstance(result["message"], str)):
|
|
raise ValueError("external backup final result is invalid")
|
|
return "result", result
|
|
raise ValueError("external deployment response frame is invalid")
|
|
|
|
|
|
def write_header(writer: BinaryIO, *, session_nonce: str, backup_id: str, manifest: dict,
|
|
context: dict | None = None) -> str:
|
|
if not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id):
|
|
raise ValueError("external backup session identity is invalid")
|
|
manifest_bytes = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest()
|
|
_json_frame(writer, MAGIC, {
|
|
"version": VERSION,
|
|
"nonce": session_nonce,
|
|
"backup_id": backup_id,
|
|
"manifest": manifest,
|
|
"manifest_sha256": manifest_sha256,
|
|
"context": context or {},
|
|
})
|
|
return manifest_sha256
|
|
|
|
|
|
class ArchiveFrameWriter:
|
|
def __init__(self, writer: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES):
|
|
self.writer = writer
|
|
self.maximum = maximum
|
|
self.size = 0
|
|
self.digest = hashlib.sha256()
|
|
self.buffer = bytearray()
|
|
self.closed = False
|
|
|
|
def write(self, data: bytes) -> int:
|
|
if self.closed:
|
|
raise ValueError("external backup stream is already closed")
|
|
self.size += len(data)
|
|
if self.size > self.maximum:
|
|
raise ValueError("external backup archive exceeds its size limit")
|
|
self.digest.update(data)
|
|
remaining = memoryview(data)
|
|
while remaining:
|
|
take = min(CHUNK_BYTES - len(self.buffer), len(remaining))
|
|
self.buffer.extend(remaining[:take])
|
|
remaining = remaining[take:]
|
|
if len(self.buffer) == CHUNK_BYTES:
|
|
self._flush_chunk(CHUNK_BYTES)
|
|
return len(data)
|
|
|
|
def _flush_chunk(self, size: int) -> None:
|
|
block = bytes(self.buffer[:size])
|
|
del self.buffer[:size]
|
|
self.writer.write(struct.pack("!I", len(block)))
|
|
self.writer.write(block)
|
|
|
|
def finish(self) -> tuple[int, str]:
|
|
if self.closed:
|
|
raise ValueError("external backup stream is already closed")
|
|
if self.buffer:
|
|
self._flush_chunk(len(self.buffer))
|
|
digest = self.digest.hexdigest()
|
|
self.writer.write(struct.pack("!I", 0) + struct.pack("!Q", self.size) + bytes.fromhex(digest))
|
|
self.writer.flush()
|
|
self.closed = True
|
|
return self.size, digest
|
|
|
|
|
|
def receive_archive(reader: BinaryIO, output: BinaryIO, *, maximum: int = MAX_ARCHIVE_BYTES) -> tuple[int, str]:
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
while True:
|
|
length = struct.unpack("!I", _read_exact(reader, 4))[0]
|
|
if length == 0:
|
|
expected_size = struct.unpack("!Q", _read_exact(reader, 8))[0]
|
|
expected_digest = _read_exact(reader, 32).hex()
|
|
break
|
|
if length > CHUNK_BYTES or size + length > maximum:
|
|
raise ValueError("external backup archive frame exceeds its size limit")
|
|
chunk = _read_exact(reader, length)
|
|
output.write(chunk)
|
|
digest.update(chunk)
|
|
size += length
|
|
output.flush()
|
|
actual_digest = digest.hexdigest()
|
|
if size != expected_size or actual_digest != expected_digest:
|
|
raise ValueError("external backup byte count or SHA-256 does not match its stream trailer")
|
|
return size, actual_digest
|
|
|
|
|
|
def write_ack(writer: BinaryIO, *, session_nonce: str, backup_id: str, size: int,
|
|
archive_sha256: str, manifest_sha256: str, verified: bool) -> None:
|
|
if (not NONCE_RE.fullmatch(session_nonce) or not BACKUP_ID_RE.fullmatch(backup_id)
|
|
or type(size) is not int or not 0 < size <= MAX_ARCHIVE_BYTES
|
|
or not SHA256_RE.fullmatch(archive_sha256) or not SHA256_RE.fullmatch(manifest_sha256)
|
|
or verified is not True):
|
|
raise ValueError("external backup acknowledgement is invalid")
|
|
_json_frame(writer, ACK_MAGIC, {
|
|
"version": VERSION,
|
|
"nonce": session_nonce,
|
|
"backup_id": backup_id,
|
|
"size": size,
|
|
"archive_sha256": archive_sha256,
|
|
"manifest_sha256": manifest_sha256,
|
|
"verified": verified,
|
|
})
|
|
|
|
|
|
def write_challenge(writer: BinaryIO, identity: dict, *, session_nonce: str) -> None:
|
|
if (not NONCE_RE.fullmatch(session_nonce) or not isinstance(identity, dict)
|
|
or set(identity) != {"backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"}):
|
|
raise ValueError("external rollback challenge is invalid")
|
|
if (not isinstance(identity["backup_id"], str) or not BACKUP_ID_RE.fullmatch(identity["backup_id"])
|
|
or type(identity["size"]) is not int or not 0 < identity["size"] <= MAX_ARCHIVE_BYTES
|
|
or not isinstance(identity["archive_sha256"], str) or not SHA256_RE.fullmatch(identity["archive_sha256"])
|
|
or not isinstance(identity["manifest_sha256"], str) or not SHA256_RE.fullmatch(identity["manifest_sha256"])
|
|
or not isinstance(identity["manifest"], dict)):
|
|
raise ValueError("external rollback backup identity is invalid")
|
|
canonical = json.dumps(identity["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
if hashlib.sha256(canonical).hexdigest() != identity["manifest_sha256"]:
|
|
raise ValueError("external rollback manifest identity is inconsistent")
|
|
_json_frame(writer, CHALLENGE_MAGIC, {"version": VERSION, "nonce": session_nonce, **identity})
|
|
|
|
|
|
def read_challenge(reader: BinaryIO) -> dict:
|
|
value = _read_json_frame(reader, CHALLENGE_MAGIC)
|
|
if (set(value) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "manifest"}
|
|
or type(value["version"]) is not int or value["version"] != VERSION
|
|
or not isinstance(value["nonce"], str)
|
|
or not NONCE_RE.fullmatch(value["nonce"]) or not isinstance(value["backup_id"], str)
|
|
or not BACKUP_ID_RE.fullmatch(value["backup_id"]) or type(value["size"]) is not int
|
|
or not 0 < value["size"] <= MAX_ARCHIVE_BYTES
|
|
or not isinstance(value["archive_sha256"], str) or not SHA256_RE.fullmatch(value["archive_sha256"])
|
|
or not isinstance(value["manifest_sha256"], str) or not SHA256_RE.fullmatch(value["manifest_sha256"])
|
|
or not isinstance(value["manifest"], dict)):
|
|
raise ValueError("external rollback challenge is invalid")
|
|
canonical = json.dumps(value["manifest"], sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
if hashlib.sha256(canonical).hexdigest() != value["manifest_sha256"]:
|
|
raise ValueError("external rollback challenge manifest identity is inconsistent")
|
|
return value
|
|
|
|
|
|
def read_ack(reader: BinaryIO, *, session_nonce: str, backup_id: str, size: int,
|
|
archive_sha256: str, manifest_sha256: str) -> dict:
|
|
ack = _read_json_frame(reader, ACK_MAGIC)
|
|
if (set(ack) != {"version", "nonce", "backup_id", "size", "archive_sha256", "manifest_sha256", "verified"}
|
|
or type(ack.get("version")) is not int or type(ack.get("size")) is not int
|
|
or ack.get("verified") is not True):
|
|
raise ValueError("external backup acknowledgement is invalid")
|
|
expected = {
|
|
"version": VERSION, "nonce": session_nonce, "backup_id": backup_id,
|
|
"size": size, "archive_sha256": archive_sha256,
|
|
"manifest_sha256": manifest_sha256, "verified": True,
|
|
}
|
|
if ack != expected:
|
|
raise ValueError("external backup acknowledgement does not match this live session")
|
|
return ack
|
|
|
|
|
|
def write_result(writer: BinaryIO, *, status: str, message: str = "") -> None:
|
|
if status not in {"success", "error"} or len(message.encode("utf-8")) > 4096:
|
|
raise ValueError("external backup result is invalid")
|
|
_json_frame(writer, RESULT_MAGIC, {"version": VERSION, "status": status, "message": message})
|
|
|
|
|
|
def read_result(reader: BinaryIO) -> dict:
|
|
result = _read_json_frame(reader, RESULT_MAGIC)
|
|
if (set(result) != {"version", "status", "message"} or type(result["version"]) is not int
|
|
or result["version"] != VERSION or result["status"] not in {"success", "error"}
|
|
or not isinstance(result["message"], str)):
|
|
raise ValueError("external backup final result is invalid")
|
|
return result
|
|
|
|
|
|
def send_backup_session(writer: BinaryIO, reader: BinaryIO, *, sources: dict, revision: str,
|
|
images: dict, deployment_files: dict[str, str], context: dict) -> dict:
|
|
from archive import backup_manifest, create_backup_stream
|
|
|
|
manifest = backup_manifest(sources, revision, images, deployment_files)
|
|
session_nonce = nonce()
|
|
backup_id = "akyldash-" + dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + session_nonce[:8]
|
|
manifest_sha256 = write_header(
|
|
writer, session_nonce=session_nonce, backup_id=backup_id, manifest=manifest, context=context
|
|
)
|
|
framed = ArchiveFrameWriter(writer)
|
|
create_backup_stream(
|
|
framed, sources, revision, images, deployment_files, manifest=manifest
|
|
)
|
|
size, archive_sha256 = framed.finish()
|
|
read_ack(
|
|
reader, session_nonce=session_nonce, backup_id=backup_id, size=size,
|
|
archive_sha256=archive_sha256, manifest_sha256=manifest_sha256,
|
|
)
|
|
return {
|
|
"backup_id": backup_id,
|
|
"archive_sha256": archive_sha256,
|
|
"manifest_sha256": manifest_sha256,
|
|
"size": size,
|
|
"manifest": manifest,
|
|
}
|