Files
akyldash/infra/backup/archive.py
T
agent 6f809d3196
CI / tests-and-images (pull_request) Has been cancelled
Add guarded rollback and recovery apply tools
2026-10-11 19:30:22 +03:00

349 lines
15 KiB
Python

#!/usr/bin/env python3
"""Create and verify cold, local Akyldash backups; restore only into staging."""
from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import io
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
from pathlib import Path, PurePosixPath
COMPONENTS = {
"data",
"opensearch",
"caddy-data",
"caddy-config",
"production-env",
"review-secret",
}
NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
value.update(chunk)
return value.hexdigest()
def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
members = archive.getmembers()
names: set[str] = set()
for member in members:
name = PurePosixPath(member.name)
if name.is_absolute() or ".." in name.parts or not name.parts:
raise ValueError(f"unsafe archive path: {member.name}")
if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
raise ValueError(f"non-canonical archive path: {member.name}")
if member.name in names:
raise ValueError(f"duplicate archive path: {member.name}")
names.add(member.name)
if not (member.isdir() or member.isfile() or member.issym()):
raise ValueError(f"unsupported archive entry: {member.name}")
if member.issym():
tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
if "manifest.json" not in names:
raise ValueError("backup manifest is missing")
if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
raise ValueError("backup is missing one or more required components")
manifest_member = next(item for item in members if item.name == "manifest.json")
if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
raise ValueError("backup manifest is not a regular file")
return members
def read_manifest(archive: tarfile.TarFile) -> dict:
checked_members(archive)
stream = archive.extractfile("manifest.json")
if stream is None:
raise ValueError("backup manifest is unreadable")
manifest = json.load(stream)
components = manifest.get("components") if isinstance(manifest, dict) else None
if (
not isinstance(manifest, dict)
or manifest.get("format") != 1
or not isinstance(components, list)
or not all(isinstance(item, str) for item in components)
or set(components) != COMPONENTS
):
raise ValueError("backup manifest has an unsupported format or component set")
if not isinstance(manifest.get("images"), dict):
raise ValueError("backup manifest has no image records")
if not isinstance(manifest.get("deployment_files"), dict):
raise ValueError("backup manifest has no deployment file records")
for member in archive.getmembers():
if member.name == "manifest.json":
continue
if PurePosixPath(member.name).parts[0] not in COMPONENTS:
raise ValueError(f"unexpected top-level archive path: {member.name}")
return manifest
def verify_archive(path: Path) -> dict:
checksum_path = Path(f"{path}.sha256")
expected = checksum_path.read_text(encoding="ascii").split()[0]
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
raise ValueError("archive SHA-256 does not match its sidecar")
with tarfile.open(path, "r:gz") as archive:
return read_manifest(archive)
def parse_sources(values: list[str]) -> dict[str, Path]:
sources: dict[str, Path] = {}
for value in values:
name, separator, raw_path = value.partition("=")
if not separator or not NAME_RE.fullmatch(name) or name in sources:
raise ValueError(f"invalid or duplicate source: {value}")
source = Path(raw_path)
if not source.is_absolute() or not source.exists() or source.is_symlink():
raise ValueError(f"source must be an existing absolute path: {raw_path}")
sources[name] = source.resolve(strict=True)
if set(sources) != COMPONENTS:
missing = ", ".join(sorted(COMPONENTS - sources.keys()))
extra = ", ".join(sorted(sources.keys() - COMPONENTS))
raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
return sources
def check_sqlite(root: Path) -> None:
for path in root.rglob("*.sqlite3"):
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
try:
result = connection.execute("PRAGMA quick_check").fetchone()
if result != ("ok",):
raise ValueError(f"SQLite integrity check failed: {path}")
finally:
connection.close()
def compose_sources(
compose_file: Path, env_file: Path
) -> tuple[dict[str, Path], str, dict[str, dict[str, str]], dict[str, str]]:
command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
rendered = subprocess.run(
[*command, "config", "--format", "json"], check=True, capture_output=True, text=True
)
config = json.loads(rendered.stdout)
container_result = subprocess.run(
[*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
)
containers = container_result.stdout.split()
if not containers:
raise ValueError("Compose has no containers; refusing to create an incomplete backup")
images: dict[str, dict[str, str]] = {}
for container in containers:
result = subprocess.run(
[
"docker",
"inspect",
"--format",
'{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
container,
],
check=True,
capture_output=True,
text=True,
)
fields = result.stdout.strip().split("\t")
if len(fields) != 4 or fields[0].lower() != "false":
raise ValueError("stop all Compose services before taking a cold backup")
images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
if set(images) != set(config["services"]):
raise ValueError("not all configured Compose services have stopped containers")
mounts = {
mount["target"]: mount
for service in config["services"].values()
for mount in service.get("volumes", [])
}
volume_sources: dict[str, Path] = {}
for target, component in (
("/app/data", "data"),
("/usr/share/opensearch/data", "opensearch"),
("/data", "caddy-data"),
("/config", "caddy-config"),
):
mount = mounts.get(target)
if not mount:
raise ValueError(f"persistent Compose mount is missing: {target}")
if mount["type"] == "bind":
source = Path(mount["source"])
elif mount["type"] == "volume":
volume_name = config["volumes"][mount["source"]]["name"]
result = subprocess.run(
["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
check=True,
capture_output=True,
text=True,
)
source = Path(result.stdout.strip())
else:
raise ValueError(f"unsupported persistent mount type at {target}")
volume_sources[component] = source
secret_file = Path(config["secrets"]["review_secret"]["file"])
sources = {
**volume_sources,
"production-env": env_file,
"review-secret": secret_file,
}
repository = compose_file.resolve().parents[2]
changed = subprocess.run(
["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
if changed:
raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
revision = subprocess.run(
["git", "-C", str(repository), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
deployment_files = {
relative: digest(repository / relative)
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
}
return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images, deployment_files
def create_backup(
output: Path,
sources: dict[str, Path],
revision: str,
images: dict | None = None,
deployment_files: dict[str, str] | None = None,
) -> None:
output = output.absolute()
parent = output.parent
if not parent.is_dir() or parent.stat().st_mode & 0o077:
raise ValueError("backup directory must exist and have mode 0700 or stricter")
if output.exists() or Path(f"{output}.sha256").exists():
raise ValueError("refusing to overwrite an existing backup")
for name, source in sources.items():
if source.is_symlink() or not (source.is_dir() or source.is_file()):
raise ValueError(f"source must be a regular file or directory: {name}")
if source.is_dir() and (output == source or source in output.parents):
raise ValueError(f"backup output cannot be inside source {name}")
for name in ("production-env", "review-secret"):
if not sources[name].is_file():
raise ValueError(f"sensitive component must be a regular file: {name}")
if sources["data"].is_dir():
check_sqlite(sources["data"])
manifest = {
"format": 1,
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_revision": revision,
"images": images or {},
"deployment_files": deployment_files or {},
"components": sorted(sources),
}
with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
temporary = Path(temp.name)
os.chmod(temporary, 0o600)
checksum_temp = Path(f"{temporary}.sha256")
try:
with tarfile.open(temporary, "w:gz") as archive:
encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
info = tarfile.TarInfo("manifest.json")
info.size = len(encoded)
info.mode = 0o600
archive.addfile(info, io.BytesIO(encoded))
for name, source in sorted(sources.items()):
def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
if name in {"production-env", "review-secret"}:
member.mode = 0o600
if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
member.mode = 0o600
return member
archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
with tarfile.open(temporary, "r:gz") as archive:
read_manifest(archive)
checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
stream.write(f"{digest(temporary)} {output.name}\n")
os.link(temporary, output)
try:
os.link(checksum_temp, Path(f"{output}.sha256"))
except Exception:
output.unlink(missing_ok=True)
raise
finally:
temporary.unlink(missing_ok=True)
checksum_temp.unlink(missing_ok=True)
def restore_to_staging(backup: Path, target: Path) -> None:
if not target.name.startswith("akyldash-restore-"):
raise ValueError("restore target must be a new akyldash-restore-* staging directory")
parent = target.parent.resolve()
if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
raise ValueError("restore staging directory must be under /tmp or /var/tmp")
if target.exists():
raise ValueError("restore target already exists; refusing to overwrite it")
target = parent / target.name
verify_archive(backup)
target.mkdir(mode=0o700)
try:
with tarfile.open(backup, "r:gz") as archive:
members = checked_members(archive)
required_bytes = sum(member.size for member in members if member.isfile())
if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
raise ValueError("insufficient free space for a validated restore staging copy")
for member in members:
archive.extract(member, target, filter="data")
check_sqlite(target / "data")
except Exception:
shutil.rmtree(target)
raise
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
create = subparsers.add_parser("create", help="create a cold backup archive")
create.add_argument("--output", required=True, type=Path)
create.add_argument("--compose-file", required=True, type=Path)
create.add_argument("--env-file", required=True, type=Path)
verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
verify.add_argument("archive", type=Path)
restore = subparsers.add_parser("restore", help="extract into a new staging directory")
restore.add_argument("archive", type=Path)
restore.add_argument("--target-dir", required=True, type=Path)
args = parser.parse_args()
try:
if args.command in {"create", "restore"} and os.geteuid() != 0:
raise ValueError("create and restore must run as root to protect production secrets and data")
if args.command == "create":
sources, revision, images, deployment_files = compose_sources(args.compose_file, args.env_file)
create_backup(args.output, sources, revision, images, deployment_files)
elif args.command == "verify":
manifest = verify_archive(args.archive)
print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
else:
restore_to_staging(args.archive, args.target_dir)
print(f"PASS restored and validated staging data at {args.target_dir}")
except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())