126 lines
6.6 KiB
Python
126 lines
6.6 KiB
Python
import fcntl
|
|
import json
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import deploy_release
|
|
import deployment_lock
|
|
import release_refs
|
|
|
|
|
|
class DeploymentCutoverTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.current = {
|
|
"backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
|
|
"opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
|
|
}
|
|
|
|
def test_mismatched_backup_is_rejected_before_any_cutover(self):
|
|
changed = {**self.current, "backend": {**self.current["backend"], "image_id": "sha256:" + "c" * 64}}
|
|
with self.assertRaisesRegex(ValueError, "do not match"):
|
|
deploy_release.require_backup_matches_current(changed, self.current)
|
|
|
|
def test_search_canary_requires_nonempty_result_with_document_code(self):
|
|
deploy_release.validate_search_response({"results": [{"code": "DOC-1"}]}, "ru")
|
|
for payload in ({"results": []}, {"results": [{}]}, {"error": "unavailable"}):
|
|
with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "canary"):
|
|
deploy_release.validate_search_response(payload, "ky")
|
|
|
|
def test_shared_lock_rejects_concurrent_deployment(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
lock_path = Path(directory) / "deploy.lock"
|
|
with patch.object(deployment_lock, "LOCK_FILE", lock_path):
|
|
descriptor = lock_path.open("w")
|
|
os.chmod(lock_path, 0o600)
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
try:
|
|
with self.assertRaisesRegex(ValueError, "shared lock"):
|
|
with deploy_release.deployment_lock():
|
|
pass
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
descriptor.close()
|
|
|
|
def test_fsync_error_after_atomic_replace_restores_previous_references(self):
|
|
old = {"BACKEND_IMAGE": "akyldash/backend:old", "OPENSEARCH_IMAGE": "akyldash/opensearch:old"}
|
|
new = {"BACKEND_IMAGE": "registry/backend@sha256:" + "c" * 64,
|
|
"OPENSEARCH_IMAGE": "registry/opensearch@sha256:" + "d" * 64}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
env_file = Path(directory) / "production.env"
|
|
env_file.write_text("BACKEND_IMAGE=akyldash/backend:old\nOPENSEARCH_IMAGE=akyldash/opensearch:old\n")
|
|
original_fsync = release_refs.os.fsync
|
|
calls = 0
|
|
|
|
def fail_first_directory_fsync(descriptor):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 2:
|
|
raise OSError("simulated directory fsync failure after replace")
|
|
original_fsync(descriptor)
|
|
|
|
with patch.object(release_refs.os, "fsync", side_effect=fail_first_directory_fsync):
|
|
with self.assertRaisesRegex(OSError, "after replace"):
|
|
release_refs.replace_images(env_file, old, new, require_root=False)
|
|
self.assertEqual(release_refs.read_images(env_file)[1], new)
|
|
with (
|
|
patch.object(deploy_release, "ENV_FILE", env_file),
|
|
patch.object(deploy_release, "replace_images", side_effect=lambda path, expected, replacement:
|
|
release_refs.replace_images(path, expected, replacement, require_root=False)),
|
|
):
|
|
deploy_release.restore_previous_references(old, new)
|
|
self.assertEqual(release_refs.read_images(env_file)[1], old)
|
|
|
|
def test_failure_after_atomic_ref_switch_keeps_recovery_journal(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
directory = Path(directory)
|
|
manifest_file = directory / "release.json"
|
|
manifest_file.write_text("{}", encoding="utf-8")
|
|
policy_file = directory / "policy.json"
|
|
policy_file.write_text("{}", encoding="utf-8")
|
|
target = {
|
|
"backend": "registry/backend@sha256:" + "c" * 64,
|
|
"opensearch": "registry/opensearch@sha256:" + "d" * 64,
|
|
}
|
|
manifest = {"builder": {"name": "Kaniko", "image": deploy_release.BUILDER_IMAGE},
|
|
"images": target, "commit": "e" * 40}
|
|
backup_path = deploy_release.BACKUP_DIR / "backup.tar.gz"
|
|
compose_calls = [None, subprocess.CalledProcessError(1, "compose up"), None]
|
|
run_calls = ["", "sha256:" + "c" * 64, "", "sha256:" + "d" * 64,
|
|
"sha256:" + "c" * 64, "sha256:" + "d" * 64]
|
|
with (
|
|
patch.object(deploy_release.os, "geteuid", return_value=0),
|
|
patch.object(deployment_lock, "LOCK_FILE", directory / "deploy.lock"),
|
|
patch.object(deployment_lock.os, "fstat", return_value=SimpleNamespace(
|
|
st_mode=stat.S_IFREG | 0o600, st_uid=0
|
|
)),
|
|
patch.object(deploy_release, "JOURNAL_FILE", directory / "recovery.json"),
|
|
patch.object(deploy_release, "STATE_FILE", directory / "state.json"),
|
|
patch.object(deploy_release, "read_manifest_json", side_effect=[{}, {}]),
|
|
patch.object(deploy_release, "validate_manifest", return_value=manifest),
|
|
patch.object(deploy_release, "image_ids", return_value=self.current),
|
|
patch.object(deploy_release, "backup_manifest", return_value={"images": self.current}),
|
|
patch.object(deploy_release, "run", side_effect=run_calls),
|
|
patch.object(deploy_release, "compose", side_effect=compose_calls) as compose,
|
|
patch.object(deploy_release, "replace_images") as replace,
|
|
patch.object(deploy_release, "restore_previous_references") as restore,
|
|
):
|
|
with self.assertRaisesRegex(ValueError, "recovery journal preserved"):
|
|
deploy_release.deploy(manifest_file, backup_path, policy_file, "закон", "мыйзам")
|
|
self.assertEqual(replace.call_count, 1)
|
|
restore.assert_called_once()
|
|
self.assertEqual(compose.call_args_list[-1].args, ("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300"))
|
|
journal = json.loads((directory / "recovery.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(journal["status"], "cutover-failed")
|
|
self.assertIn("recovery", journal)
|
|
self.assertFalse((directory / "state.json").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|