146 lines
7.5 KiB
Python
146 lines
7.5 KiB
Python
import fcntl
|
|
import json
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import deploy_release
|
|
import deployment_lock
|
|
import release_refs
|
|
|
|
|
|
class DeploymentCutoverTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.current = {
|
|
"backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
|
|
"opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
|
|
}
|
|
|
|
def test_mismatched_backup_is_rejected_before_any_cutover(self):
|
|
changed = {**self.current, "backend": {**self.current["backend"], "image_id": "sha256:" + "c" * 64}}
|
|
with self.assertRaisesRegex(ValueError, "do not match"):
|
|
deploy_release.require_backup_matches_current(changed, self.current)
|
|
|
|
def test_search_canary_requires_nonempty_result_with_document_code(self):
|
|
deploy_release.validate_search_response({"results": [{"code": "DOC-1"}]}, "ru")
|
|
for payload in ({"results": []}, {"results": [{}]}, {"error": "unavailable"}):
|
|
with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "canary"):
|
|
deploy_release.validate_search_response(payload, "ky")
|
|
|
|
def test_shared_lock_rejects_concurrent_deployment(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
lock_path = Path(directory) / "deploy.lock"
|
|
with patch.object(deployment_lock, "LOCK_FILE", lock_path):
|
|
descriptor = lock_path.open("w")
|
|
os.chmod(lock_path, 0o600)
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
try:
|
|
with self.assertRaisesRegex(ValueError, "shared lock"):
|
|
with deploy_release.deployment_lock():
|
|
pass
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
descriptor.close()
|
|
|
|
def test_fsync_error_after_atomic_replace_restores_previous_references(self):
|
|
old = {"BACKEND_IMAGE": "akyldash/backend:old", "OPENSEARCH_IMAGE": "akyldash/opensearch:old"}
|
|
new = {"BACKEND_IMAGE": "registry/backend@sha256:" + "c" * 64,
|
|
"OPENSEARCH_IMAGE": "registry/opensearch@sha256:" + "d" * 64}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
env_file = Path(directory) / "production.env"
|
|
env_file.write_text("BACKEND_IMAGE=akyldash/backend:old\nOPENSEARCH_IMAGE=akyldash/opensearch:old\n")
|
|
original_fsync = release_refs.os.fsync
|
|
calls = 0
|
|
|
|
def fail_first_directory_fsync(descriptor):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 2:
|
|
raise OSError("simulated directory fsync failure after replace")
|
|
original_fsync(descriptor)
|
|
|
|
with patch.object(release_refs.os, "fsync", side_effect=fail_first_directory_fsync):
|
|
with self.assertRaisesRegex(OSError, "after replace"):
|
|
release_refs.replace_images(env_file, old, new, require_root=False)
|
|
self.assertEqual(release_refs.read_images(env_file)[1], new)
|
|
with (
|
|
patch.object(deploy_release, "ENV_FILE", env_file),
|
|
patch.object(deploy_release, "replace_images", side_effect=lambda path, expected, replacement:
|
|
release_refs.replace_images(path, expected, replacement, require_root=False)),
|
|
):
|
|
deploy_release.restore_previous_references(old, new)
|
|
self.assertEqual(release_refs.read_images(env_file)[1], old)
|
|
|
|
def test_external_cutover_rejects_unvalidated_release_before_docker_operations(self):
|
|
with patch.object(deploy_release.os, "geteuid", return_value=0), patch.object(deploy_release, "run") as run_mock, patch.object(deploy_release, "compose") as compose_mock:
|
|
with self.assertRaisesRegex(ValueError, "manifest must contain exactly"):
|
|
deploy_release.deploy_external(
|
|
{}, {"schema_version": 1, "repositories": {
|
|
"backend": ["registry/backend"], "opensearch": ["registry/search"]
|
|
}}, {}, {}, "0" * 64, self.current, "ру", "кыргызча"
|
|
)
|
|
run_mock.assert_not_called()
|
|
compose_mock.assert_not_called()
|
|
|
|
def test_main_advance_after_backup_aborts_before_image_reference_switch(self):
|
|
commit = "b" * 40
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "deploy/production").mkdir(parents=True)
|
|
for name, payload in (("compose.yaml", b"compose\n"), ("Caddyfile", b"caddy\n")):
|
|
(root / "deploy/production" / name).write_bytes(payload)
|
|
from hashlib import sha256
|
|
deployment_files = {
|
|
f"deploy/production/{name}": sha256((root / "deploy/production" / name).read_bytes()).hexdigest()
|
|
for name in ("compose.yaml", "Caddyfile")
|
|
}
|
|
backup = {"images": self.current, "deployment_files": deployment_files}
|
|
backup_identity = {
|
|
"backup_id": "20261103T000000Z-" + "a" * 32,
|
|
"size": 100, "archive_sha256": "c" * 64,
|
|
"manifest_sha256": "d" * 64, "manifest": backup,
|
|
}
|
|
manifest = {
|
|
"commit": commit,
|
|
"builder": {"name": "Kaniko", "image": deploy_release.BUILDER_IMAGE},
|
|
"images": {service: f"registry/{service}@sha256:{'e' * 64}" for service in ("backend", "opensearch")},
|
|
}
|
|
journal = {"status": "external-backup-verified"}
|
|
with (
|
|
patch.object(deploy_release.os, "geteuid", return_value=0),
|
|
patch.object(deploy_release, "ROOT", root),
|
|
patch.object(deploy_release, "STATE_FILE", root / "state.json"),
|
|
patch.object(deploy_release, "JOURNAL_FILE", root / "journal.json"),
|
|
patch.object(deploy_release, "validate_manifest", return_value=manifest),
|
|
patch("release_state.validate_backup_identity", return_value=backup_identity),
|
|
patch("release_handoff.latest_main_sha", side_effect=(commit, "f" * 40)),
|
|
patch("release_source.require_runtime_compatibility"),
|
|
patch.object(deploy_release, "image_ids", return_value=self.current),
|
|
patch.object(deploy_release, "app_images", return_value=self.current),
|
|
patch.object(deploy_release, "read_json", return_value=journal),
|
|
patch.object(deploy_release, "write_json_atomic"),
|
|
patch.object(deploy_release, "run", side_effect=(
|
|
"", "sha256:" + "1" * 64, "", "sha256:" + "2" * 64,
|
|
"sha256:" + "1" * 64, "sha256:" + "2" * 64,
|
|
)),
|
|
patch.object(deploy_release, "replace_images") as replace,
|
|
patch.object(deploy_release, "restore_previous_references"),
|
|
patch.object(deploy_release, "compose"),
|
|
patch.object(deploy_release, "write_deployed_backend_image"),
|
|
):
|
|
with self.assertRaisesRegex(ValueError, "cutover failed"):
|
|
deploy_release.deploy_external(
|
|
manifest, {}, backup, backup_identity, "9" * 64,
|
|
self.current, "ru", "ky",
|
|
)
|
|
replace.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|