397 lines
19 KiB
Python
397 lines
19 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import sqlite3
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from archive import COMPONENTS, compose_sources, create_backup, create_backup_stream, restore_to_staging, verify_archive, verify_external_archive
|
|
|
|
|
|
class BackupArchiveTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temporary.name)
|
|
self.private = self.root / "private"
|
|
self.private.mkdir(mode=0o700)
|
|
self.sources = {}
|
|
for name in COMPONENTS:
|
|
source = self.root / name
|
|
source.mkdir()
|
|
self.sources[name] = source
|
|
for name in ("production-env", "review-secret"):
|
|
path = self.root / f"{name}.txt"
|
|
path.write_text("private", encoding="utf-8")
|
|
path.chmod(0o644)
|
|
self.sources[name] = path
|
|
connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
|
|
connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
|
|
connection.commit()
|
|
connection.close()
|
|
self.backup = self.private / "backup.tar.gz"
|
|
|
|
def tearDown(self):
|
|
self.temporary.cleanup()
|
|
|
|
def test_create_verify_and_restore_only_to_new_staging(self):
|
|
create_backup(self.backup, self.sources, "abc123")
|
|
manifest = verify_archive(self.backup)
|
|
self.assertEqual(manifest["git_revision"], "abc123")
|
|
self.assertEqual(manifest["images"], {})
|
|
|
|
target = self.root / "akyldash-restore-test"
|
|
restore_to_staging(self.backup, target)
|
|
self.assertTrue((target / "data/search-users.sqlite3").is_file())
|
|
self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
|
|
|
|
def test_external_stream_uses_no_local_archive_and_verifies_received_bytes(self):
|
|
stream = io.BytesIO()
|
|
with patch("archive.os.geteuid", return_value=0):
|
|
manifest = create_backup_stream(stream, self.sources, "a" * 40)
|
|
self.assertFalse(self.backup.exists())
|
|
received = self.private / "received.tar.gz"
|
|
received.write_bytes(stream.getvalue())
|
|
verified = verify_external_archive(
|
|
received,
|
|
expected_sha256=hashlib.sha256(stream.getvalue()).hexdigest(),
|
|
expected_manifest_sha256=hashlib.sha256(
|
|
json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest(),
|
|
)
|
|
self.assertEqual(verified["git_revision"], "a" * 40)
|
|
|
|
def test_external_archive_rejects_changed_manifest_and_duplicate_keys(self):
|
|
stream = io.BytesIO()
|
|
with tarfile.open(fileobj=stream, mode="w:gz") as archive:
|
|
info = tarfile.TarInfo("manifest.json")
|
|
raw = b'{"format":1,"format":1,"components":[],"images":{},"deployment_files":{}}'
|
|
info.size = len(raw)
|
|
archive.addfile(info, io.BytesIO(raw))
|
|
for name in COMPONENTS:
|
|
info = tarfile.TarInfo(name + "/")
|
|
info.type = tarfile.DIRTYPE
|
|
archive.addfile(info)
|
|
received = self.private / "duplicate.tar.gz"
|
|
received.write_bytes(stream.getvalue())
|
|
with self.assertRaisesRegex(ValueError, "duplicate backup manifest"):
|
|
verify_external_archive(received, expected_sha256=hashlib.sha256(stream.getvalue()).hexdigest(),
|
|
expected_manifest_sha256="0" * 64)
|
|
|
|
def test_rejects_checksum_mismatch_without_creating_staging(self):
|
|
create_backup(self.backup, self.sources, "abc123")
|
|
self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
|
|
target = self.root / "akyldash-restore-tampered"
|
|
with self.assertRaisesRegex(ValueError, "SHA-256"):
|
|
restore_to_staging(self.backup, target)
|
|
self.assertFalse(target.exists())
|
|
|
|
def test_rejects_non_staging_and_existing_targets(self):
|
|
create_backup(self.backup, self.sources, "abc123")
|
|
with self.assertRaisesRegex(ValueError, "staging"):
|
|
restore_to_staging(self.backup, self.root / "production")
|
|
target = self.root / "akyldash-restore-existing"
|
|
target.mkdir()
|
|
with self.assertRaisesRegex(ValueError, "already exists"):
|
|
restore_to_staging(self.backup, target)
|
|
|
|
def test_create_checks_sqlite_integrity_before_writing_archive(self):
|
|
(self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
|
|
with self.assertRaises(sqlite3.DatabaseError):
|
|
create_backup(self.backup, self.sources, "abc123")
|
|
self.assertFalse(self.backup.exists())
|
|
|
|
def test_rejects_unsafe_archive_paths(self):
|
|
archive_path = self.private / "unsafe.tar.gz"
|
|
with tarfile.open(archive_path, "w:gz") as archive:
|
|
archive.addfile(tarfile.TarInfo("manifest.json"))
|
|
archive.addfile(tarfile.TarInfo("../outside"))
|
|
checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
|
|
Path(f"{archive_path}.sha256").write_text(f"{checksum} unsafe.tar.gz\n")
|
|
with self.assertRaisesRegex(ValueError, "unsafe archive path"):
|
|
verify_archive(archive_path)
|
|
|
|
def test_cold_backup_refuses_a_running_compose_container(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="container-id\n")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
return SimpleNamespace(stdout="true\n")
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "stop all Compose services"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
|
|
def compose_fixture(self):
|
|
paths = {}
|
|
mounts = {
|
|
"backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
|
|
"opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
|
|
"caddy": [
|
|
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
|
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
|
],
|
|
}
|
|
volumes = {}
|
|
for volume in ("os-data", "caddy-data", "caddy-config"):
|
|
name = f"akyldash_{volume}"
|
|
path = self.root / name
|
|
path.mkdir()
|
|
paths[name] = path
|
|
volumes[volume] = {"name": name}
|
|
env_file = self.root / "production.env"
|
|
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
|
secret = self.root / "review-secret.txt"
|
|
secret.write_text("secret", encoding="utf-8")
|
|
config = {
|
|
"services": {name: {"volumes": value} for name, value in mounts.items()},
|
|
"volumes": volumes,
|
|
"secrets": {"review_secret": {"file": str(secret)}},
|
|
}
|
|
return config, paths, env_file, secret
|
|
|
|
def test_clean_first_deployment_does_not_require_backup(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
(self.root / "data/search-users.sqlite3").unlink()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
if "status" in command:
|
|
return SimpleNamespace(stdout="")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
|
|
def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
|
|
config, _volume_paths, env_file, _secret = self.compose_fixture()
|
|
(self.root / "data/search-users.sqlite3").unlink()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume" and "ls" in command:
|
|
return SimpleNamespace(stdout="")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
|
|
def test_existing_data_without_containers_is_backup_eligible(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
if "status" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if "rev-parse" in command:
|
|
return SimpleNamespace(stdout="abc123\n")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
sources, revision, images, deployment_files = compose_sources(
|
|
Path.cwd() / "deploy/production/compose.yaml", env_file
|
|
)
|
|
archive = self.private / "offline-backup.tar.gz"
|
|
create_backup(archive, sources, revision, images, deployment_files)
|
|
manifest = verify_archive(archive)
|
|
self.assertEqual(set(sources), COMPONENTS)
|
|
self.assertEqual(images, {})
|
|
self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
|
|
self.assertEqual(manifest["git_revision"], "abc123")
|
|
|
|
def test_missing_persistent_mount_state_is_rejected(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
volume_paths["akyldash_os-data"].rmdir()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
|
|
def test_partial_compose_containers_are_rejected_with_existing_data(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="backend-id\n")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
if command[1] == "inspect":
|
|
return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
|
|
def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
(self.root / "data/search-users.sqlite3").unlink()
|
|
(self.root / "data/existing.json").write_text("data", encoding="utf-8")
|
|
available = set(volume_paths) - {"akyldash_os-data"}
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume" and "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
|
if command[1] == "volume":
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
if "status" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if "rev-parse" in command:
|
|
return SimpleNamespace(stdout="abc123\n")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
|
|
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
|
with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
|
|
compose_sources(
|
|
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
|
)
|
|
|
|
def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
|
|
config, volume_paths, env_file, _secret = self.compose_fixture()
|
|
(self.root / "data/search-users.sqlite3").unlink()
|
|
corpus = self.root / "data/minjust-normalized"
|
|
corpus.mkdir()
|
|
(corpus / "document.json").write_text("corpus", encoding="utf-8")
|
|
available = set(volume_paths) - {"akyldash_os-data"}
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if command[1] == "volume" and "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
|
if command[1] == "volume":
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
if "status" in command:
|
|
return SimpleNamespace(stdout="")
|
|
if "rev-parse" in command:
|
|
return SimpleNamespace(stdout="abc123\n")
|
|
raise AssertionError(command)
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
sources, revision, images, deployment_files = compose_sources(
|
|
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
|
)
|
|
self.assertIsNone(sources["opensearch"])
|
|
archive = self.private / "initial-corpus-backup.tar.gz"
|
|
create_backup(archive, sources, revision, images, deployment_files)
|
|
manifest = verify_archive(archive)
|
|
self.assertEqual(manifest["git_revision"], "abc123")
|
|
with tarfile.open(archive, "r:gz") as saved:
|
|
self.assertTrue(saved.getmember("opensearch/").isdir())
|
|
self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
|
|
|
|
def test_compose_backup_records_sources_and_image_ids(self):
|
|
data = self.root / "production-data"
|
|
data.mkdir()
|
|
secret = self.root / "secret"
|
|
secret.write_text("private", encoding="utf-8")
|
|
env_file = self.root / "production.env"
|
|
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
|
volumes = {}
|
|
volume_paths = {}
|
|
mounts = {
|
|
"caddy": [
|
|
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
|
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
|
],
|
|
"backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
|
|
"opensearch": [
|
|
{"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
|
|
],
|
|
}
|
|
for name in ("caddy-data", "caddy-config", "opensearch-data"):
|
|
path = self.root / f"volume-{name}"
|
|
path.mkdir()
|
|
volumes[name] = {"name": f"akyldash_{name}"}
|
|
volume_paths[f"akyldash_{name}"] = path
|
|
config = {
|
|
"services": {name: {"volumes": values} for name, values in mounts.items()},
|
|
"volumes": volumes,
|
|
"secrets": {"review_secret": {"file": str(secret)}},
|
|
}
|
|
service_names = ["caddy", "backend", "opensearch"]
|
|
|
|
def run(command, **_kwargs):
|
|
if "config" in command:
|
|
return SimpleNamespace(stdout=json.dumps(config))
|
|
if "ps" in command:
|
|
return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
|
|
if command[1] == "inspect":
|
|
name = command[-1].removesuffix("-id")
|
|
return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
|
|
if command[1] == "volume":
|
|
if "ls" in command:
|
|
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
|
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
|
return SimpleNamespace(stdout="")
|
|
|
|
with patch("archive.subprocess.run", side_effect=run):
|
|
sources, revision, images, deployment_files = compose_sources(
|
|
Path.cwd() / "deploy/production/compose.yaml", env_file
|
|
)
|
|
self.assertEqual(revision, "")
|
|
self.assertEqual(set(sources), COMPONENTS)
|
|
self.assertEqual(images["backend"]["image_id"], "sha256:backend")
|
|
self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|