Files
akyldash/infra/backup/test_archive.py

397 lines
19 KiB
Python

import hashlib
import io
import json
import sqlite3
import tarfile
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from archive import COMPONENTS, compose_sources, create_backup, create_backup_stream, restore_to_staging, verify_archive, verify_external_archive
class BackupArchiveTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name)
self.private = self.root / "private"
self.private.mkdir(mode=0o700)
self.sources = {}
for name in COMPONENTS:
source = self.root / name
source.mkdir()
self.sources[name] = source
for name in ("production-env", "review-secret"):
path = self.root / f"{name}.txt"
path.write_text("private", encoding="utf-8")
path.chmod(0o644)
self.sources[name] = path
connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
connection.commit()
connection.close()
self.backup = self.private / "backup.tar.gz"
def tearDown(self):
self.temporary.cleanup()
def test_create_verify_and_restore_only_to_new_staging(self):
create_backup(self.backup, self.sources, "abc123")
manifest = verify_archive(self.backup)
self.assertEqual(manifest["git_revision"], "abc123")
self.assertEqual(manifest["images"], {})
target = self.root / "akyldash-restore-test"
restore_to_staging(self.backup, target)
self.assertTrue((target / "data/search-users.sqlite3").is_file())
self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
def test_external_stream_uses_no_local_archive_and_verifies_received_bytes(self):
stream = io.BytesIO()
with patch("archive.os.geteuid", return_value=0):
manifest = create_backup_stream(stream, self.sources, "a" * 40)
self.assertFalse(self.backup.exists())
received = self.private / "received.tar.gz"
received.write_bytes(stream.getvalue())
verified = verify_external_archive(
received,
expected_sha256=hashlib.sha256(stream.getvalue()).hexdigest(),
expected_manifest_sha256=hashlib.sha256(
json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest(),
)
self.assertEqual(verified["git_revision"], "a" * 40)
def test_external_archive_rejects_changed_manifest_and_duplicate_keys(self):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w:gz") as archive:
info = tarfile.TarInfo("manifest.json")
raw = b'{"format":1,"format":1,"components":[],"images":{},"deployment_files":{}}'
info.size = len(raw)
archive.addfile(info, io.BytesIO(raw))
for name in COMPONENTS:
info = tarfile.TarInfo(name + "/")
info.type = tarfile.DIRTYPE
archive.addfile(info)
received = self.private / "duplicate.tar.gz"
received.write_bytes(stream.getvalue())
with self.assertRaisesRegex(ValueError, "duplicate backup manifest"):
verify_external_archive(received, expected_sha256=hashlib.sha256(stream.getvalue()).hexdigest(),
expected_manifest_sha256="0" * 64)
def test_rejects_checksum_mismatch_without_creating_staging(self):
create_backup(self.backup, self.sources, "abc123")
self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
target = self.root / "akyldash-restore-tampered"
with self.assertRaisesRegex(ValueError, "SHA-256"):
restore_to_staging(self.backup, target)
self.assertFalse(target.exists())
def test_rejects_non_staging_and_existing_targets(self):
create_backup(self.backup, self.sources, "abc123")
with self.assertRaisesRegex(ValueError, "staging"):
restore_to_staging(self.backup, self.root / "production")
target = self.root / "akyldash-restore-existing"
target.mkdir()
with self.assertRaisesRegex(ValueError, "already exists"):
restore_to_staging(self.backup, target)
def test_create_checks_sqlite_integrity_before_writing_archive(self):
(self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
with self.assertRaises(sqlite3.DatabaseError):
create_backup(self.backup, self.sources, "abc123")
self.assertFalse(self.backup.exists())
def test_rejects_unsafe_archive_paths(self):
archive_path = self.private / "unsafe.tar.gz"
with tarfile.open(archive_path, "w:gz") as archive:
archive.addfile(tarfile.TarInfo("manifest.json"))
archive.addfile(tarfile.TarInfo("../outside"))
checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
Path(f"{archive_path}.sha256").write_text(f"{checksum} unsafe.tar.gz\n")
with self.assertRaisesRegex(ValueError, "unsafe archive path"):
verify_archive(archive_path)
def test_cold_backup_refuses_a_running_compose_container(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="container-id\n")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
return SimpleNamespace(stdout="true\n")
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "stop all Compose services"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
def compose_fixture(self):
paths = {}
mounts = {
"backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
"opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
"caddy": [
{"type": "volume", "source": "caddy-data", "target": "/data"},
{"type": "volume", "source": "caddy-config", "target": "/config"},
],
}
volumes = {}
for volume in ("os-data", "caddy-data", "caddy-config"):
name = f"akyldash_{volume}"
path = self.root / name
path.mkdir()
paths[name] = path
volumes[volume] = {"name": name}
env_file = self.root / "production.env"
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
secret = self.root / "review-secret.txt"
secret.write_text("secret", encoding="utf-8")
config = {
"services": {name: {"volumes": value} for name, value in mounts.items()},
"volumes": volumes,
"secrets": {"review_secret": {"file": str(secret)}},
}
return config, paths, env_file, secret
def test_clean_first_deployment_does_not_require_backup(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
(self.root / "data/search-users.sqlite3").unlink()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
if "status" in command:
return SimpleNamespace(stdout="")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "clean first deployment"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
config, _volume_paths, env_file, _secret = self.compose_fixture()
(self.root / "data/search-users.sqlite3").unlink()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume" and "ls" in command:
return SimpleNamespace(stdout="")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "clean first deployment"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
def test_existing_data_without_containers_is_backup_eligible(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
if "status" in command:
return SimpleNamespace(stdout="")
if "rev-parse" in command:
return SimpleNamespace(stdout="abc123\n")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
sources, revision, images, deployment_files = compose_sources(
Path.cwd() / "deploy/production/compose.yaml", env_file
)
archive = self.private / "offline-backup.tar.gz"
create_backup(archive, sources, revision, images, deployment_files)
manifest = verify_archive(archive)
self.assertEqual(set(sources), COMPONENTS)
self.assertEqual(images, {})
self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
self.assertEqual(manifest["git_revision"], "abc123")
def test_missing_persistent_mount_state_is_rejected(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
volume_paths["akyldash_os-data"].rmdir()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
def test_partial_compose_containers_are_rejected_with_existing_data(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="backend-id\n")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
if command[1] == "inspect":
return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
(self.root / "data/search-users.sqlite3").unlink()
(self.root / "data/existing.json").write_text("data", encoding="utf-8")
available = set(volume_paths) - {"akyldash_os-data"}
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume" and "ls" in command:
return SimpleNamespace(stdout="\n".join(available) + "\n")
if command[1] == "volume":
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
if "status" in command:
return SimpleNamespace(stdout="")
if "rev-parse" in command:
return SimpleNamespace(stdout="abc123\n")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
compose_sources(
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
)
def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
config, volume_paths, env_file, _secret = self.compose_fixture()
(self.root / "data/search-users.sqlite3").unlink()
corpus = self.root / "data/minjust-normalized"
corpus.mkdir()
(corpus / "document.json").write_text("corpus", encoding="utf-8")
available = set(volume_paths) - {"akyldash_os-data"}
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="")
if command[1] == "volume" and "ls" in command:
return SimpleNamespace(stdout="\n".join(available) + "\n")
if command[1] == "volume":
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
if "status" in command:
return SimpleNamespace(stdout="")
if "rev-parse" in command:
return SimpleNamespace(stdout="abc123\n")
raise AssertionError(command)
with patch("archive.subprocess.run", side_effect=run):
sources, revision, images, deployment_files = compose_sources(
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
)
self.assertIsNone(sources["opensearch"])
archive = self.private / "initial-corpus-backup.tar.gz"
create_backup(archive, sources, revision, images, deployment_files)
manifest = verify_archive(archive)
self.assertEqual(manifest["git_revision"], "abc123")
with tarfile.open(archive, "r:gz") as saved:
self.assertTrue(saved.getmember("opensearch/").isdir())
self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
def test_compose_backup_records_sources_and_image_ids(self):
data = self.root / "production-data"
data.mkdir()
secret = self.root / "secret"
secret.write_text("private", encoding="utf-8")
env_file = self.root / "production.env"
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
volumes = {}
volume_paths = {}
mounts = {
"caddy": [
{"type": "volume", "source": "caddy-data", "target": "/data"},
{"type": "volume", "source": "caddy-config", "target": "/config"},
],
"backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
"opensearch": [
{"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
],
}
for name in ("caddy-data", "caddy-config", "opensearch-data"):
path = self.root / f"volume-{name}"
path.mkdir()
volumes[name] = {"name": f"akyldash_{name}"}
volume_paths[f"akyldash_{name}"] = path
config = {
"services": {name: {"volumes": values} for name, values in mounts.items()},
"volumes": volumes,
"secrets": {"review_secret": {"file": str(secret)}},
}
service_names = ["caddy", "backend", "opensearch"]
def run(command, **_kwargs):
if "config" in command:
return SimpleNamespace(stdout=json.dumps(config))
if "ps" in command:
return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
if command[1] == "inspect":
name = command[-1].removesuffix("-id")
return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
if command[1] == "volume":
if "ls" in command:
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
return SimpleNamespace(stdout="")
with patch("archive.subprocess.run", side_effect=run):
sources, revision, images, deployment_files = compose_sources(
Path.cwd() / "deploy/production/compose.yaml", env_file
)
self.assertEqual(revision, "")
self.assertEqual(set(sources), COMPONENTS)
self.assertEqual(images["backend"]["image_id"], "sha256:backend")
self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
if __name__ == "__main__":
unittest.main()