459 lines
20 KiB
Python
459 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Create and verify cold, local Akyldash backups; restore only into staging."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import BinaryIO
|
|
|
|
COMPONENTS = {
|
|
"data",
|
|
"opensearch",
|
|
"caddy-data",
|
|
"caddy-config",
|
|
"production-env",
|
|
"review-secret",
|
|
}
|
|
NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
|
|
|
|
def digest(path: Path) -> str:
|
|
value = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
value.update(chunk)
|
|
return value.hexdigest()
|
|
|
|
|
|
def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
|
|
members = archive.getmembers()
|
|
names: set[str] = set()
|
|
for member in members:
|
|
name = PurePosixPath(member.name)
|
|
if name.is_absolute() or ".." in name.parts or not name.parts:
|
|
raise ValueError(f"unsafe archive path: {member.name}")
|
|
if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
|
|
raise ValueError(f"non-canonical archive path: {member.name}")
|
|
if member.name in names:
|
|
raise ValueError(f"duplicate archive path: {member.name}")
|
|
names.add(member.name)
|
|
if not (member.isdir() or member.isfile() or member.issym()):
|
|
raise ValueError(f"unsupported archive entry: {member.name}")
|
|
if member.issym():
|
|
tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
|
|
if "manifest.json" not in names:
|
|
raise ValueError("backup manifest is missing")
|
|
if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
|
|
raise ValueError("backup is missing one or more required components")
|
|
manifest_member = next(item for item in members if item.name == "manifest.json")
|
|
if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
|
|
raise ValueError("backup manifest is not a regular file")
|
|
return members
|
|
|
|
|
|
def read_manifest(archive: tarfile.TarFile) -> dict:
|
|
checked_members(archive)
|
|
stream = archive.extractfile("manifest.json")
|
|
if stream is None:
|
|
raise ValueError("backup manifest is unreadable")
|
|
manifest = json.loads(stream.read().decode("utf-8"), object_pairs_hook=unique_json_object)
|
|
components = manifest.get("components") if isinstance(manifest, dict) else None
|
|
if (
|
|
not isinstance(manifest, dict)
|
|
or manifest.get("format") != 1
|
|
or not isinstance(components, list)
|
|
or not all(isinstance(item, str) for item in components)
|
|
or set(components) != COMPONENTS
|
|
):
|
|
raise ValueError("backup manifest has an unsupported format or component set")
|
|
if not isinstance(manifest.get("images"), dict):
|
|
raise ValueError("backup manifest has no image records")
|
|
if not isinstance(manifest.get("deployment_files"), dict):
|
|
raise ValueError("backup manifest has no deployment file records")
|
|
for member in archive.getmembers():
|
|
if member.name == "manifest.json":
|
|
continue
|
|
if PurePosixPath(member.name).parts[0] not in COMPONENTS:
|
|
raise ValueError(f"unexpected top-level archive path: {member.name}")
|
|
return manifest
|
|
|
|
|
|
def unique_json_object(pairs: list[tuple[str, object]]) -> dict:
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError(f"duplicate backup manifest field: {key}")
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def verify_archive(path: Path) -> dict:
|
|
checksum_path = Path(f"{path}.sha256")
|
|
expected = checksum_path.read_text(encoding="ascii").split()[0]
|
|
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
|
|
raise ValueError("archive SHA-256 does not match its sidecar")
|
|
with tarfile.open(path, "r:gz") as archive:
|
|
return read_manifest(archive)
|
|
|
|
|
|
def verify_external_archive(path: Path, *, expected_sha256: str, expected_manifest_sha256: str) -> dict:
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError("external backup must be a regular file")
|
|
if not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) or digest(path) != expected_sha256:
|
|
raise ValueError("external backup SHA-256 does not match the live session")
|
|
with tarfile.open(path, "r:gz") as archive:
|
|
manifest_member = next(member for member in checked_members(archive) if member.name == "manifest.json")
|
|
stream = archive.extractfile(manifest_member)
|
|
if stream is None:
|
|
raise ValueError("external backup manifest is unreadable")
|
|
manifest_bytes = stream.read(1024 * 1024 + 1)
|
|
manifest = read_manifest(archive)
|
|
canonical_manifest = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
if len(manifest_bytes) > 1024 * 1024 or hashlib.sha256(canonical_manifest).hexdigest() != expected_manifest_sha256:
|
|
raise ValueError("external backup manifest does not match the live session")
|
|
for member in archive.getmembers():
|
|
if member.isfile():
|
|
stream = archive.extractfile(member)
|
|
if stream is None:
|
|
raise ValueError(f"external backup member is unreadable: {member.name}")
|
|
while stream.read(1024 * 1024):
|
|
pass
|
|
return manifest
|
|
|
|
|
|
def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
|
|
sources: dict[str, Path] = {}
|
|
for value in values:
|
|
name, separator, raw_path = value.partition("=")
|
|
if not separator or not NAME_RE.fullmatch(name) or name in sources:
|
|
raise ValueError(f"invalid or duplicate source: {value}")
|
|
source = Path(raw_path)
|
|
if not source.is_absolute() or not source.exists() or source.is_symlink():
|
|
raise ValueError(f"source must be an existing absolute path: {raw_path}")
|
|
sources[name] = source.resolve(strict=True)
|
|
if set(sources) != required:
|
|
missing = ", ".join(sorted(required - sources.keys()))
|
|
extra = ", ".join(sorted(sources.keys() - required))
|
|
raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
|
|
return sources
|
|
|
|
|
|
def check_sqlite(root: Path) -> None:
|
|
for path in root.rglob("*.sqlite3"):
|
|
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
|
|
try:
|
|
result = connection.execute("PRAGMA quick_check").fetchone()
|
|
if result != ("ok",):
|
|
raise ValueError(f"SQLite integrity check failed: {path}")
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def compose_storage(config: dict) -> dict[str, Path | None]:
|
|
mounts = {
|
|
mount["target"]: mount
|
|
for service in config["services"].values()
|
|
for mount in service.get("volumes", [])
|
|
}
|
|
volumes = subprocess.run(
|
|
["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
|
|
).stdout.split()
|
|
sources: dict[str, Path | None] = {}
|
|
for target, component in (
|
|
("/app/data", "data"),
|
|
("/usr/share/opensearch/data", "opensearch"),
|
|
("/data", "caddy-data"),
|
|
("/config", "caddy-config"),
|
|
):
|
|
mount = mounts.get(target)
|
|
if not mount:
|
|
raise ValueError(f"persistent Compose mount is missing: {target}")
|
|
if mount["type"] == "bind":
|
|
source = Path(mount["source"])
|
|
elif mount["type"] == "volume":
|
|
volume_name = config["volumes"][mount["source"]]["name"]
|
|
if volume_name not in volumes:
|
|
sources[component] = None
|
|
continue
|
|
result = subprocess.run(
|
|
["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
source = Path(result.stdout.strip())
|
|
else:
|
|
raise ValueError(f"unsupported persistent mount type at {target}")
|
|
if not source.is_dir() or source.is_symlink():
|
|
raise ValueError(f"cannot determine persistent data state for {component}: {source}")
|
|
sources[component] = source
|
|
return sources
|
|
|
|
|
|
def has_persistent_data(sources: dict[str, Path | None]) -> bool:
|
|
return any(any(path.iterdir()) for path in sources.values() if path is not None)
|
|
|
|
|
|
def is_preloaded_corpus_only(data: Path | None) -> bool:
|
|
if data is None:
|
|
return False
|
|
entries = list(data.iterdir())
|
|
return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
|
|
|
|
|
|
def compose_sources(
|
|
compose_file: Path, env_file: Path, initial_deployment: bool = False
|
|
) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
|
|
command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
|
|
rendered = subprocess.run(
|
|
[*command, "config", "--format", "json"], check=True, capture_output=True, text=True
|
|
)
|
|
config = json.loads(rendered.stdout)
|
|
container_result = subprocess.run(
|
|
[*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
|
|
)
|
|
containers = container_result.stdout.split()
|
|
storage = compose_storage(config)
|
|
persistent_data = has_persistent_data(storage)
|
|
if not containers and not persistent_data:
|
|
raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
|
|
missing_volumes = any(path is None for path in storage.values())
|
|
if missing_volumes and containers:
|
|
raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
|
|
if missing_volumes and not initial_deployment:
|
|
raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
|
|
if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
|
|
raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
|
|
images: dict[str, dict[str, str]] = {}
|
|
for container in containers:
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
'{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
|
|
container,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
fields = result.stdout.strip().split("\t")
|
|
if len(fields) != 4 or fields[0].lower() != "false":
|
|
raise ValueError("stop all Compose services before taking a cold backup")
|
|
images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
|
|
if containers and set(images) != set(config["services"]):
|
|
raise ValueError("not all configured Compose services have stopped containers")
|
|
|
|
secret_file = Path(config["secrets"]["review_secret"]["file"])
|
|
sources = {
|
|
**storage,
|
|
"production-env": env_file,
|
|
"review-secret": secret_file,
|
|
}
|
|
repository = compose_file.resolve().parents[2]
|
|
changed = subprocess.run(
|
|
["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
if changed:
|
|
raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
|
|
revision = subprocess.run(
|
|
["git", "-C", str(repository), "rev-parse", "HEAD"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
deployment_files = {
|
|
relative: digest(repository / relative)
|
|
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
|
}
|
|
present = {name: path for name, path in sources.items() if path is not None}
|
|
missing = {name for name, path in sources.items() if path is None}
|
|
parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
|
|
return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
|
|
|
|
|
|
def create_backup(
|
|
output: Path,
|
|
sources: dict[str, Path | None],
|
|
revision: str,
|
|
images: dict | None = None,
|
|
deployment_files: dict[str, str] | None = None,
|
|
) -> None:
|
|
output = output.absolute()
|
|
parent = output.parent
|
|
if not parent.is_dir() or parent.stat().st_mode & 0o077:
|
|
raise ValueError("backup directory must exist and have mode 0700 or stricter")
|
|
if output.exists() or Path(f"{output}.sha256").exists():
|
|
raise ValueError("refusing to overwrite an existing backup")
|
|
for name, source in sources.items():
|
|
if source is None:
|
|
continue
|
|
if source.is_symlink() or not (source.is_dir() or source.is_file()):
|
|
raise ValueError(f"source must be a regular file or directory: {name}")
|
|
if source.is_dir() and (output == source or source in output.parents):
|
|
raise ValueError(f"backup output cannot be inside source {name}")
|
|
for name in ("production-env", "review-secret"):
|
|
if sources[name] is None or not sources[name].is_file():
|
|
raise ValueError(f"sensitive component must be a regular file: {name}")
|
|
manifest = backup_manifest(sources, revision, images, deployment_files)
|
|
with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
|
|
temporary = Path(temp.name)
|
|
os.chmod(temporary, 0o600)
|
|
checksum_temp = Path(f"{temporary}.sha256")
|
|
try:
|
|
with tarfile.open(temporary, "w:gz") as archive:
|
|
write_backup(archive, sources, manifest)
|
|
with tarfile.open(temporary, "r:gz") as archive:
|
|
read_manifest(archive)
|
|
checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
|
|
stream.write(f"{digest(temporary)} {output.name}\n")
|
|
os.link(temporary, output)
|
|
try:
|
|
os.link(checksum_temp, Path(f"{output}.sha256"))
|
|
except Exception:
|
|
output.unlink(missing_ok=True)
|
|
raise
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
checksum_temp.unlink(missing_ok=True)
|
|
|
|
|
|
def backup_manifest(
|
|
sources: dict[str, Path | None], revision: str, images: dict | None, deployment_files: dict[str, str] | None
|
|
) -> dict:
|
|
for name, source in sources.items():
|
|
if source is not None and (source.is_symlink() or not (source.is_dir() or source.is_file())):
|
|
raise ValueError(f"source must be a regular file or directory: {name}")
|
|
for name in ("production-env", "review-secret"):
|
|
if sources[name] is None or not sources[name].is_file():
|
|
raise ValueError(f"sensitive component must be a regular file: {name}")
|
|
if sources["data"] is not None and sources["data"].is_dir():
|
|
check_sqlite(sources["data"])
|
|
return {
|
|
"format": 1,
|
|
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"git_revision": revision,
|
|
"images": images or {},
|
|
"deployment_files": deployment_files or {},
|
|
"components": sorted(sources),
|
|
}
|
|
|
|
|
|
def write_backup(archive: tarfile.TarFile, sources: dict[str, Path | None], manifest: dict) -> None:
|
|
encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
|
|
info = tarfile.TarInfo("manifest.json")
|
|
info.size = len(encoded)
|
|
info.mode = 0o600
|
|
archive.addfile(info, io.BytesIO(encoded))
|
|
for name, source in sorted(sources.items()):
|
|
if source is None:
|
|
info = tarfile.TarInfo(f"{name}/")
|
|
info.type = tarfile.DIRTYPE
|
|
info.mode = 0o700
|
|
archive.addfile(info)
|
|
continue
|
|
|
|
def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
|
|
if name in {"production-env", "review-secret"}:
|
|
member.mode = 0o600
|
|
if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
|
|
member.mode = 0o600
|
|
return member
|
|
|
|
archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
|
|
|
|
|
|
def create_backup_stream(
|
|
output: BinaryIO, sources: dict[str, Path | None], revision: str,
|
|
images: dict | None = None, deployment_files: dict[str, str] | None = None,
|
|
manifest: dict | None = None,
|
|
) -> dict:
|
|
if os.geteuid() != 0:
|
|
raise ValueError("external production backup must run as root")
|
|
manifest = manifest or backup_manifest(sources, revision, images, deployment_files)
|
|
with tarfile.open(fileobj=output, mode="w|gz") as archive:
|
|
write_backup(archive, sources, manifest)
|
|
output.flush()
|
|
return manifest
|
|
|
|
|
|
def restore_to_staging(backup: Path, target: Path) -> None:
|
|
if not target.name.startswith("akyldash-restore-"):
|
|
raise ValueError("restore target must be a new akyldash-restore-* staging directory")
|
|
parent = target.parent.resolve()
|
|
if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
|
|
raise ValueError("restore staging directory must be under /tmp or /var/tmp")
|
|
if target.exists():
|
|
raise ValueError("restore target already exists; refusing to overwrite it")
|
|
target = parent / target.name
|
|
verify_archive(backup)
|
|
target.mkdir(mode=0o700)
|
|
try:
|
|
with tarfile.open(backup, "r:gz") as archive:
|
|
members = checked_members(archive)
|
|
required_bytes = sum(member.size for member in members if member.isfile())
|
|
if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
|
|
raise ValueError("insufficient free space for a validated restore staging copy")
|
|
for member in members:
|
|
archive.extract(member, target, filter="data")
|
|
check_sqlite(target / "data")
|
|
except Exception:
|
|
shutil.rmtree(target)
|
|
raise
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
create = subparsers.add_parser("create", help="create a cold backup archive")
|
|
create.add_argument("--output", required=True, type=Path)
|
|
create.add_argument("--compose-file", required=True, type=Path)
|
|
create.add_argument("--env-file", required=True, type=Path)
|
|
create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
|
|
verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
|
|
verify.add_argument("archive", type=Path)
|
|
restore = subparsers.add_parser("restore", help="extract into a new staging directory")
|
|
restore.add_argument("archive", type=Path)
|
|
restore.add_argument("--target-dir", required=True, type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command in {"create", "restore"} and os.geteuid() != 0:
|
|
raise ValueError("create and restore must run as root to protect production secrets and data")
|
|
if args.command == "create":
|
|
sources, revision, images, deployment_files = compose_sources(
|
|
args.compose_file, args.env_file, args.initial_deployment
|
|
)
|
|
create_backup(args.output, sources, revision, images, deployment_files)
|
|
elif args.command == "verify":
|
|
manifest = verify_archive(args.archive)
|
|
print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
|
|
else:
|
|
restore_to_staging(args.archive, args.target_dir)
|
|
print(f"PASS restored and validated staging data at {args.target_dir}")
|
|
except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
|
|
print(f"ERROR: {error}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|