310 lines
15 KiB
Python
310 lines
15 KiB
Python
"""Build a small immutable Windows receiver bundle from one Git commit."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
FILES = {
|
|
"archive.py": "infra/backup/archive.py",
|
|
"external_session.py": "infra/backup/external_session.py",
|
|
"receive_external.py": "infra/backup/receive_external.py",
|
|
"windows-release-trigger.example.xml": "infra/deploy/windows-release-trigger.example.xml",
|
|
"windows_release_launcher.py": "infra/deploy/windows_release_launcher.py",
|
|
}
|
|
MAX_BUNDLE_BYTES = 512 * 1024
|
|
MAX_FILE_BYTES = 262144
|
|
TREE_OBJECTS = {
|
|
"": "objects/root-tree.bin",
|
|
"infra": "objects/infra-tree.bin",
|
|
"infra/backup": "objects/backup-tree.bin",
|
|
"infra/deploy": "objects/deploy-tree.bin",
|
|
}
|
|
|
|
|
|
def _git(repository: Path, *arguments: str) -> bytes:
|
|
result = subprocess.run(
|
|
["git", "-C", str(repository), *arguments],
|
|
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
)
|
|
return result.stdout
|
|
|
|
|
|
def _blob_sha(data: bytes) -> str:
|
|
return _git_object_sha("blob", data)
|
|
|
|
|
|
def _git_object_sha(kind: str, data: bytes) -> str:
|
|
return hashlib.sha1(kind.encode("ascii") + b" " + str(len(data)).encode("ascii") + b"\0" + data).hexdigest()
|
|
|
|
|
|
def _tree_entry(repository: Path, tree: str, name: str) -> tuple[str, str, str]:
|
|
records = _git(repository, "ls-tree", "-z", tree, "--", name).split(b"\0")
|
|
matches = []
|
|
for record in records:
|
|
if not record:
|
|
continue
|
|
header, raw_path = record.split(b"\t", 1)
|
|
mode, kind, object_id = header.decode("ascii").split(" ")
|
|
if raw_path.decode("utf-8") == name:
|
|
matches.append((mode, kind, object_id))
|
|
if len(matches) != 1:
|
|
raise ValueError(f"Git tree is missing or duplicates required path: {name}")
|
|
return matches[0]
|
|
|
|
|
|
def _parse_git_tree(data: bytes) -> dict[str, tuple[str, str]]:
|
|
entries: dict[str, tuple[str, str]] = {}
|
|
offset = 0
|
|
while offset < len(data):
|
|
space = data.find(b" ", offset)
|
|
nul = data.find(b"\0", space + 1) if space >= 0 else -1
|
|
if space < offset or nul < 0 or nul + 21 > len(data):
|
|
raise ValueError("Git tree object is malformed")
|
|
mode = data[offset:space].decode("ascii")
|
|
name = data[space + 1:nul].decode("utf-8")
|
|
object_id = data[nul + 1:nul + 21].hex()
|
|
if not name or name in {".", ".."} or name in entries:
|
|
raise ValueError("Git tree object has an invalid or duplicate name")
|
|
entries[name] = (mode, object_id)
|
|
offset = nul + 21
|
|
return entries
|
|
|
|
|
|
def _strict_json_object(pairs):
|
|
value = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise ValueError("receiver bundle manifest has duplicate keys")
|
|
value[key] = item
|
|
return value
|
|
|
|
|
|
def verify_bundle(payload: bytes, expected_commit: str, expected_sha256: str) -> dict[str, bytes]:
|
|
"""Verify a generated or transferred bundle before an installer consumes it."""
|
|
if len(payload) <= 0 or len(payload) > MAX_BUNDLE_BYTES:
|
|
raise ValueError("receiver bundle has an invalid size")
|
|
if hashlib.sha256(payload).hexdigest() != expected_sha256:
|
|
raise ValueError("receiver bundle SHA-256 does not match")
|
|
if not re.fullmatch(r"[0-9a-f]{40}", expected_commit):
|
|
raise ValueError("expected commit must be a full lowercase Git SHA-1")
|
|
expected_entries = {
|
|
"manifest.json", "objects/commit.bin", *TREE_OBJECTS.values(),
|
|
*(f"files/{name}" for name in FILES),
|
|
}
|
|
try:
|
|
archive = zipfile.ZipFile(io.BytesIO(payload))
|
|
except (zipfile.BadZipFile, OSError) as error:
|
|
raise ValueError("receiver bundle is not a valid ZIP") from error
|
|
with archive:
|
|
infos = archive.infolist()
|
|
names = [item.filename for item in infos]
|
|
if len(names) != len(expected_entries) or len(set(names)) != len(expected_entries) or set(names) != expected_entries:
|
|
raise ValueError("receiver bundle has duplicate, missing, or unexpected entries")
|
|
for item in infos:
|
|
mode = (item.external_attr >> 16) & 0xFFFF
|
|
if mode != 0o100600 or item.compress_type != zipfile.ZIP_STORED:
|
|
raise ValueError("receiver bundle entry has an unsafe mode or compression method")
|
|
maximum = 16384 if item.filename == "manifest.json" else MAX_FILE_BYTES
|
|
if item.file_size > maximum:
|
|
raise ValueError("receiver bundle entry exceeds its size limit")
|
|
contents = {item.filename: archive.read(item) for item in infos}
|
|
|
|
try:
|
|
manifest = json.loads(contents["manifest.json"].decode("ascii"), object_pairs_hook=_strict_json_object)
|
|
except (UnicodeError, json.JSONDecodeError) as error:
|
|
raise ValueError("receiver bundle manifest is invalid") from error
|
|
if (not isinstance(manifest, dict) or set(manifest) != {"schema_version", "commit", "files", "trees"}
|
|
or type(manifest["schema_version"]) is not int or manifest["schema_version"] != 1
|
|
or manifest["commit"] != expected_commit):
|
|
raise ValueError("receiver bundle manifest does not match the expected commit and schema")
|
|
|
|
commit_object = contents["objects/commit.bin"]
|
|
if _git_object_sha("commit", commit_object) != expected_commit:
|
|
raise ValueError("receiver bundle commit object does not match expected SHA")
|
|
commit_header = commit_object.split(b"\n\n", 1)[0]
|
|
commit_trees = re.findall(rb"(?m)^tree ([0-9a-f]{40})$", commit_header)
|
|
if len(commit_trees) != 1:
|
|
raise ValueError("receiver bundle commit has no unique root tree")
|
|
|
|
if not isinstance(manifest["trees"], list) or len(manifest["trees"]) != len(TREE_OBJECTS):
|
|
raise ValueError("receiver bundle tree manifest is invalid")
|
|
tree_hashes: dict[str, str] = {}
|
|
tree_entries: dict[str, dict[str, tuple[str, str]]] = {}
|
|
for record in manifest["trees"]:
|
|
if (not isinstance(record, dict) or set(record) != {"path", "sha1"}
|
|
or not isinstance(record["path"], str) or record["path"] not in TREE_OBJECTS
|
|
or record["path"] in tree_hashes
|
|
or not isinstance(record["sha1"], str) or not re.fullmatch(r"[0-9a-f]{40}", record["sha1"])):
|
|
raise ValueError("receiver bundle tree record is invalid")
|
|
path = record["path"]
|
|
data = contents[TREE_OBJECTS[path]]
|
|
if _git_object_sha("tree", data) != record["sha1"]:
|
|
raise ValueError(f"receiver bundle tree hash does not match: {path}")
|
|
tree_hashes[path] = record["sha1"]
|
|
tree_entries[path] = _parse_git_tree(data)
|
|
if set(tree_hashes) != set(TREE_OBJECTS) or tree_hashes[""].encode("ascii") != commit_trees[0]:
|
|
raise ValueError("receiver bundle commit does not bind the declared root tree")
|
|
for parent, name, child in (("", "infra", "infra"),
|
|
("infra", "backup", "infra/backup"),
|
|
("infra", "deploy", "infra/deploy")):
|
|
mode, child_sha = tree_entries[parent].get(name, ("", ""))
|
|
if mode not in {"40000", "040000"} or child_sha != tree_hashes[child]:
|
|
raise ValueError(f"receiver bundle directory tree chain is invalid at {parent or '/'}/{name}")
|
|
|
|
if not isinstance(manifest["files"], list) or len(manifest["files"]) != len(FILES):
|
|
raise ValueError("receiver bundle file manifest is invalid")
|
|
verified: dict[str, bytes] = {}
|
|
for record in manifest["files"]:
|
|
if (not isinstance(record, dict) or set(record) != {"path", "git_blob_sha1", "sha256", "size"}
|
|
or not isinstance(record["path"], str) or record["path"] not in FILES or record["path"] in verified
|
|
or type(record["size"]) is not int or record["size"] < 0 or record["size"] > MAX_FILE_BYTES
|
|
or not isinstance(record["git_blob_sha1"], str)
|
|
or not re.fullmatch(r"[0-9a-f]{40}", record["git_blob_sha1"])
|
|
or not isinstance(record["sha256"], str)
|
|
or not re.fullmatch(r"[0-9a-f]{64}", record["sha256"])):
|
|
raise ValueError("receiver bundle file record is invalid")
|
|
name = record["path"]
|
|
data = contents[f"files/{name}"]
|
|
tree_path = "infra/backup" if FILES[name].startswith("infra/backup/") else "infra/deploy"
|
|
mode, blob_sha = tree_entries[tree_path].get(name, ("", ""))
|
|
if (mode != "100644" or blob_sha != record["git_blob_sha1"]
|
|
or _git_object_sha("blob", data) != blob_sha
|
|
or hashlib.sha256(data).hexdigest() != record["sha256"]
|
|
or len(data) != record["size"]):
|
|
raise ValueError(f"receiver bundle file does not match its commit tree: {name}")
|
|
verified[name] = data
|
|
if set(verified) != set(FILES):
|
|
raise ValueError("receiver bundle omits a required source file")
|
|
return verified
|
|
|
|
|
|
def read_commit_files(repository: Path, commit: str) -> tuple[
|
|
dict[str, bytes], list[dict[str, object]], dict[str, bytes], list[dict[str, str]]]:
|
|
if not re.fullmatch(r"[0-9a-f]{40}", commit):
|
|
raise ValueError("expected commit must be a full lowercase Git SHA-1")
|
|
if repository.is_symlink():
|
|
raise ValueError("repository must be a real directory")
|
|
repository = repository.resolve(strict=True)
|
|
if not repository.is_dir():
|
|
raise ValueError("repository must be a real directory")
|
|
if _git(repository, "cat-file", "-t", commit).strip() != b"commit":
|
|
raise ValueError("expected object is not a commit")
|
|
main_commit = _git(repository, "rev-parse", "refs/heads/main").strip().decode("ascii")
|
|
if main_commit != commit:
|
|
raise ValueError("expected commit must equal the local main branch tip")
|
|
commit_object = _git(repository, "cat-file", "commit", commit)
|
|
if _git_object_sha("commit", commit_object) != commit:
|
|
raise ValueError("commit object bytes do not match the expected SHA")
|
|
commit_header = commit_object.split(b"\n\n", 1)[0]
|
|
commit_trees = [value.decode("ascii") for value in
|
|
re.findall(rb"(?m)^tree ([0-9a-f]{40})$", commit_header)]
|
|
if len(commit_trees) != 1:
|
|
raise ValueError("commit object has no unique root tree")
|
|
tree_ids = {"": commit_trees[0]}
|
|
tree_ids["infra"] = _tree_entry(repository, tree_ids[""], "infra")[2]
|
|
tree_ids["infra/backup"] = _tree_entry(repository, tree_ids["infra"], "backup")[2]
|
|
tree_ids["infra/deploy"] = _tree_entry(repository, tree_ids["infra"], "deploy")[2]
|
|
tree_objects: dict[str, bytes] = {"objects/commit.bin": commit_object}
|
|
manifest_trees: list[dict[str, str]] = []
|
|
for tree_path, tree_sha in sorted(tree_ids.items()):
|
|
data = _git(repository, "cat-file", "tree", tree_sha)
|
|
if _git_object_sha("tree", data) != tree_sha:
|
|
raise ValueError(f"Git tree bytes do not match their object ID: {tree_path or '/'}")
|
|
tree_objects[TREE_OBJECTS[tree_path]] = data
|
|
manifest_trees.append({"path": tree_path, "sha1": tree_sha})
|
|
records = _git(repository, "ls-tree", "-z", commit, "--", *FILES.values()).split(b"\0")
|
|
tree_entries: dict[str, tuple[str, str, str]] = {}
|
|
for record in records:
|
|
if not record:
|
|
continue
|
|
header, raw_path = record.split(b"\t", 1)
|
|
mode, kind, object_id = header.decode("ascii").split(" ")
|
|
tree_entries[raw_path.decode("utf-8")] = (mode, kind, object_id)
|
|
if set(tree_entries) != set(FILES.values()):
|
|
raise ValueError("commit does not contain the exact receiver allowlist")
|
|
|
|
contents: dict[str, bytes] = {}
|
|
manifest_files: list[dict[str, object]] = []
|
|
for name, source_path in sorted(FILES.items()):
|
|
mode, kind, object_id = tree_entries[source_path]
|
|
if mode != "100644" or kind != "blob":
|
|
raise ValueError(f"receiver source is not a regular Git blob: {source_path}")
|
|
data = _git(repository, "cat-file", "blob", object_id)
|
|
if _blob_sha(data) != object_id:
|
|
raise ValueError(f"Git blob bytes do not match the commit object: {source_path}")
|
|
contents[name] = data
|
|
manifest_files.append({
|
|
"path": name,
|
|
"git_blob_sha1": object_id,
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"size": len(data),
|
|
})
|
|
for name, source_path in FILES.items():
|
|
parent_path = "infra/backup" if source_path.startswith("infra/backup/") else "infra/deploy"
|
|
_mode, _kind, expected_blob = tree_entries[source_path]
|
|
mode, kind, tree_blob = _tree_entry(repository, tree_ids[parent_path], name)
|
|
if mode != "100644" or kind != "blob" or tree_blob != expected_blob:
|
|
raise ValueError(f"Git tree path does not bind the expected regular source blob: {source_path}")
|
|
return contents, manifest_files, tree_objects, manifest_trees
|
|
|
|
|
|
def create_bundle(repository: Path, commit: str) -> bytes:
|
|
contents, manifest_files, tree_objects, manifest_trees = read_commit_files(repository, commit)
|
|
manifest = json.dumps({
|
|
"schema_version": 1,
|
|
"commit": commit,
|
|
"files": manifest_files,
|
|
"trees": manifest_trees,
|
|
}, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n"
|
|
output = io.BytesIO()
|
|
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_STORED) as bundle:
|
|
members = [("manifest.json", manifest)]
|
|
members.extend((f"files/{key}", value) for key, value in sorted(contents.items()))
|
|
members.extend(sorted(tree_objects.items()))
|
|
for name, data in members:
|
|
entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
|
|
entry.compress_type = zipfile.ZIP_STORED
|
|
entry.external_attr = 0o100600 << 16
|
|
bundle.writestr(entry, data)
|
|
result = output.getvalue()
|
|
if len(result) > MAX_BUNDLE_BYTES:
|
|
raise ValueError("receiver bundle exceeds its size limit")
|
|
verify_bundle(result, commit, hashlib.sha256(result).hexdigest())
|
|
return result
|
|
|
|
|
|
def write_bundle(repository: Path, commit: str, output_path: Path) -> str:
|
|
if output_path.exists() or output_path.is_symlink():
|
|
raise ValueError("refusing to overwrite an existing release bundle")
|
|
if not output_path.parent.is_dir() or output_path.parent.is_symlink():
|
|
raise ValueError("release bundle output directory must already exist and be a real directory")
|
|
payload = create_bundle(repository, commit)
|
|
with output_path.open("xb") as stream:
|
|
stream.write(payload)
|
|
stream.flush()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repository", type=Path, required=True)
|
|
parser.add_argument("--commit", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
digest = write_bundle(args.repository, args.commit, args.output)
|
|
print(f"Prepared receiver bundle for {args.commit}")
|
|
print(f"Bundle SHA-256: {digest}")
|
|
print(f"Bundle path: {args.output.resolve()}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|