Files
akyldash/infra/backup/archive.py

459 lines
20 KiB
Python

#!/usr/bin/env python3
"""Create and verify cold, local Akyldash backups; restore only into staging."""
from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import io
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
from pathlib import Path, PurePosixPath
from typing import BinaryIO
COMPONENTS = {
"data",
"opensearch",
"caddy-data",
"caddy-config",
"production-env",
"review-secret",
}
NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
value.update(chunk)
return value.hexdigest()
def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
members = archive.getmembers()
names: set[str] = set()
for member in members:
name = PurePosixPath(member.name)
if name.is_absolute() or ".." in name.parts or not name.parts:
raise ValueError(f"unsafe archive path: {member.name}")
if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
raise ValueError(f"non-canonical archive path: {member.name}")
if member.name in names:
raise ValueError(f"duplicate archive path: {member.name}")
names.add(member.name)
if not (member.isdir() or member.isfile() or member.issym()):
raise ValueError(f"unsupported archive entry: {member.name}")
if member.issym():
tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
if "manifest.json" not in names:
raise ValueError("backup manifest is missing")
if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
raise ValueError("backup is missing one or more required components")
manifest_member = next(item for item in members if item.name == "manifest.json")
if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
raise ValueError("backup manifest is not a regular file")
return members
def read_manifest(archive: tarfile.TarFile) -> dict:
checked_members(archive)
stream = archive.extractfile("manifest.json")
if stream is None:
raise ValueError("backup manifest is unreadable")
manifest = json.loads(stream.read().decode("utf-8"), object_pairs_hook=unique_json_object)
components = manifest.get("components") if isinstance(manifest, dict) else None
if (
not isinstance(manifest, dict)
or manifest.get("format") != 1
or not isinstance(components, list)
or not all(isinstance(item, str) for item in components)
or set(components) != COMPONENTS
):
raise ValueError("backup manifest has an unsupported format or component set")
if not isinstance(manifest.get("images"), dict):
raise ValueError("backup manifest has no image records")
if not isinstance(manifest.get("deployment_files"), dict):
raise ValueError("backup manifest has no deployment file records")
for member in archive.getmembers():
if member.name == "manifest.json":
continue
if PurePosixPath(member.name).parts[0] not in COMPONENTS:
raise ValueError(f"unexpected top-level archive path: {member.name}")
return manifest
def unique_json_object(pairs: list[tuple[str, object]]) -> dict:
result = {}
for key, value in pairs:
if key in result:
raise ValueError(f"duplicate backup manifest field: {key}")
result[key] = value
return result
def verify_archive(path: Path) -> dict:
checksum_path = Path(f"{path}.sha256")
expected = checksum_path.read_text(encoding="ascii").split()[0]
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
raise ValueError("archive SHA-256 does not match its sidecar")
with tarfile.open(path, "r:gz") as archive:
return read_manifest(archive)
def verify_external_archive(path: Path, *, expected_sha256: str, expected_manifest_sha256: str) -> dict:
if path.is_symlink() or not path.is_file():
raise ValueError("external backup must be a regular file")
if not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) or digest(path) != expected_sha256:
raise ValueError("external backup SHA-256 does not match the live session")
with tarfile.open(path, "r:gz") as archive:
manifest_member = next(member for member in checked_members(archive) if member.name == "manifest.json")
stream = archive.extractfile(manifest_member)
if stream is None:
raise ValueError("external backup manifest is unreadable")
manifest_bytes = stream.read(1024 * 1024 + 1)
manifest = read_manifest(archive)
canonical_manifest = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
if len(manifest_bytes) > 1024 * 1024 or hashlib.sha256(canonical_manifest).hexdigest() != expected_manifest_sha256:
raise ValueError("external backup manifest does not match the live session")
for member in archive.getmembers():
if member.isfile():
stream = archive.extractfile(member)
if stream is None:
raise ValueError(f"external backup member is unreadable: {member.name}")
while stream.read(1024 * 1024):
pass
return manifest
def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
sources: dict[str, Path] = {}
for value in values:
name, separator, raw_path = value.partition("=")
if not separator or not NAME_RE.fullmatch(name) or name in sources:
raise ValueError(f"invalid or duplicate source: {value}")
source = Path(raw_path)
if not source.is_absolute() or not source.exists() or source.is_symlink():
raise ValueError(f"source must be an existing absolute path: {raw_path}")
sources[name] = source.resolve(strict=True)
if set(sources) != required:
missing = ", ".join(sorted(required - sources.keys()))
extra = ", ".join(sorted(sources.keys() - required))
raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
return sources
def check_sqlite(root: Path) -> None:
for path in root.rglob("*.sqlite3"):
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
try:
result = connection.execute("PRAGMA quick_check").fetchone()
if result != ("ok",):
raise ValueError(f"SQLite integrity check failed: {path}")
finally:
connection.close()
def compose_storage(config: dict) -> dict[str, Path | None]:
mounts = {
mount["target"]: mount
for service in config["services"].values()
for mount in service.get("volumes", [])
}
volumes = subprocess.run(
["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
).stdout.split()
sources: dict[str, Path | None] = {}
for target, component in (
("/app/data", "data"),
("/usr/share/opensearch/data", "opensearch"),
("/data", "caddy-data"),
("/config", "caddy-config"),
):
mount = mounts.get(target)
if not mount:
raise ValueError(f"persistent Compose mount is missing: {target}")
if mount["type"] == "bind":
source = Path(mount["source"])
elif mount["type"] == "volume":
volume_name = config["volumes"][mount["source"]]["name"]
if volume_name not in volumes:
sources[component] = None
continue
result = subprocess.run(
["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
check=True,
capture_output=True,
text=True,
)
source = Path(result.stdout.strip())
else:
raise ValueError(f"unsupported persistent mount type at {target}")
if not source.is_dir() or source.is_symlink():
raise ValueError(f"cannot determine persistent data state for {component}: {source}")
sources[component] = source
return sources
def has_persistent_data(sources: dict[str, Path | None]) -> bool:
return any(any(path.iterdir()) for path in sources.values() if path is not None)
def is_preloaded_corpus_only(data: Path | None) -> bool:
if data is None:
return False
entries = list(data.iterdir())
return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
def compose_sources(
compose_file: Path, env_file: Path, initial_deployment: bool = False
) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
rendered = subprocess.run(
[*command, "config", "--format", "json"], check=True, capture_output=True, text=True
)
config = json.loads(rendered.stdout)
container_result = subprocess.run(
[*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
)
containers = container_result.stdout.split()
storage = compose_storage(config)
persistent_data = has_persistent_data(storage)
if not containers and not persistent_data:
raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
missing_volumes = any(path is None for path in storage.values())
if missing_volumes and containers:
raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
if missing_volumes and not initial_deployment:
raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
images: dict[str, dict[str, str]] = {}
for container in containers:
result = subprocess.run(
[
"docker",
"inspect",
"--format",
'{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
container,
],
check=True,
capture_output=True,
text=True,
)
fields = result.stdout.strip().split("\t")
if len(fields) != 4 or fields[0].lower() != "false":
raise ValueError("stop all Compose services before taking a cold backup")
images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
if containers and set(images) != set(config["services"]):
raise ValueError("not all configured Compose services have stopped containers")
secret_file = Path(config["secrets"]["review_secret"]["file"])
sources = {
**storage,
"production-env": env_file,
"review-secret": secret_file,
}
repository = compose_file.resolve().parents[2]
changed = subprocess.run(
["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
if changed:
raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
revision = subprocess.run(
["git", "-C", str(repository), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
deployment_files = {
relative: digest(repository / relative)
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
}
present = {name: path for name, path in sources.items() if path is not None}
missing = {name for name, path in sources.items() if path is None}
parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
def create_backup(
output: Path,
sources: dict[str, Path | None],
revision: str,
images: dict | None = None,
deployment_files: dict[str, str] | None = None,
) -> None:
output = output.absolute()
parent = output.parent
if not parent.is_dir() or parent.stat().st_mode & 0o077:
raise ValueError("backup directory must exist and have mode 0700 or stricter")
if output.exists() or Path(f"{output}.sha256").exists():
raise ValueError("refusing to overwrite an existing backup")
for name, source in sources.items():
if source is None:
continue
if source.is_symlink() or not (source.is_dir() or source.is_file()):
raise ValueError(f"source must be a regular file or directory: {name}")
if source.is_dir() and (output == source or source in output.parents):
raise ValueError(f"backup output cannot be inside source {name}")
for name in ("production-env", "review-secret"):
if sources[name] is None or not sources[name].is_file():
raise ValueError(f"sensitive component must be a regular file: {name}")
manifest = backup_manifest(sources, revision, images, deployment_files)
with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
temporary = Path(temp.name)
os.chmod(temporary, 0o600)
checksum_temp = Path(f"{temporary}.sha256")
try:
with tarfile.open(temporary, "w:gz") as archive:
write_backup(archive, sources, manifest)
with tarfile.open(temporary, "r:gz") as archive:
read_manifest(archive)
checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
stream.write(f"{digest(temporary)} {output.name}\n")
os.link(temporary, output)
try:
os.link(checksum_temp, Path(f"{output}.sha256"))
except Exception:
output.unlink(missing_ok=True)
raise
finally:
temporary.unlink(missing_ok=True)
checksum_temp.unlink(missing_ok=True)
def backup_manifest(
sources: dict[str, Path | None], revision: str, images: dict | None, deployment_files: dict[str, str] | None
) -> dict:
for name, source in sources.items():
if source is not None and (source.is_symlink() or not (source.is_dir() or source.is_file())):
raise ValueError(f"source must be a regular file or directory: {name}")
for name in ("production-env", "review-secret"):
if sources[name] is None or not sources[name].is_file():
raise ValueError(f"sensitive component must be a regular file: {name}")
if sources["data"] is not None and sources["data"].is_dir():
check_sqlite(sources["data"])
return {
"format": 1,
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_revision": revision,
"images": images or {},
"deployment_files": deployment_files or {},
"components": sorted(sources),
}
def write_backup(archive: tarfile.TarFile, sources: dict[str, Path | None], manifest: dict) -> None:
encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
info = tarfile.TarInfo("manifest.json")
info.size = len(encoded)
info.mode = 0o600
archive.addfile(info, io.BytesIO(encoded))
for name, source in sorted(sources.items()):
if source is None:
info = tarfile.TarInfo(f"{name}/")
info.type = tarfile.DIRTYPE
info.mode = 0o700
archive.addfile(info)
continue
def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
if name in {"production-env", "review-secret"}:
member.mode = 0o600
if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
member.mode = 0o600
return member
archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
def create_backup_stream(
output: BinaryIO, sources: dict[str, Path | None], revision: str,
images: dict | None = None, deployment_files: dict[str, str] | None = None,
manifest: dict | None = None,
) -> dict:
if os.geteuid() != 0:
raise ValueError("external production backup must run as root")
manifest = manifest or backup_manifest(sources, revision, images, deployment_files)
with tarfile.open(fileobj=output, mode="w|gz") as archive:
write_backup(archive, sources, manifest)
output.flush()
return manifest
def restore_to_staging(backup: Path, target: Path) -> None:
if not target.name.startswith("akyldash-restore-"):
raise ValueError("restore target must be a new akyldash-restore-* staging directory")
parent = target.parent.resolve()
if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
raise ValueError("restore staging directory must be under /tmp or /var/tmp")
if target.exists():
raise ValueError("restore target already exists; refusing to overwrite it")
target = parent / target.name
verify_archive(backup)
target.mkdir(mode=0o700)
try:
with tarfile.open(backup, "r:gz") as archive:
members = checked_members(archive)
required_bytes = sum(member.size for member in members if member.isfile())
if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
raise ValueError("insufficient free space for a validated restore staging copy")
for member in members:
archive.extract(member, target, filter="data")
check_sqlite(target / "data")
except Exception:
shutil.rmtree(target)
raise
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
create = subparsers.add_parser("create", help="create a cold backup archive")
create.add_argument("--output", required=True, type=Path)
create.add_argument("--compose-file", required=True, type=Path)
create.add_argument("--env-file", required=True, type=Path)
create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
verify.add_argument("archive", type=Path)
restore = subparsers.add_parser("restore", help="extract into a new staging directory")
restore.add_argument("archive", type=Path)
restore.add_argument("--target-dir", required=True, type=Path)
args = parser.parse_args()
try:
if args.command in {"create", "restore"} and os.geteuid() != 0:
raise ValueError("create and restore must run as root to protect production secrets and data")
if args.command == "create":
sources, revision, images, deployment_files = compose_sources(
args.compose_file, args.env_file, args.initial_deployment
)
create_backup(args.output, sources, revision, images, deployment_files)
elif args.command == "verify":
manifest = verify_archive(args.archive)
print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
else:
restore_to_staging(args.archive, args.target_dir)
print(f"PASS restored and validated staging data at {args.target_dir}")
except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())