Files
akyldash/infra/deploy/test_deploy_release.py
codex 710c483f34
CI / tests-and-images (pull_request) Successful in 2m15s
Runner smoke test / runner-smoke (pull_request) Successful in 3s
Harden external release preparation workflow
2026-11-03 09:38:37 +03:00

146 lines
7.5 KiB
Python

import fcntl
import json
import os
import stat
import subprocess
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import deploy_release
import deployment_lock
import release_refs
class DeploymentCutoverTests(unittest.TestCase):
def setUp(self):
self.current = {
"backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
"opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
}
def test_mismatched_backup_is_rejected_before_any_cutover(self):
changed = {**self.current, "backend": {**self.current["backend"], "image_id": "sha256:" + "c" * 64}}
with self.assertRaisesRegex(ValueError, "do not match"):
deploy_release.require_backup_matches_current(changed, self.current)
def test_search_canary_requires_nonempty_result_with_document_code(self):
deploy_release.validate_search_response({"results": [{"code": "DOC-1"}]}, "ru")
for payload in ({"results": []}, {"results": [{}]}, {"error": "unavailable"}):
with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "canary"):
deploy_release.validate_search_response(payload, "ky")
def test_shared_lock_rejects_concurrent_deployment(self):
with tempfile.TemporaryDirectory() as directory:
lock_path = Path(directory) / "deploy.lock"
with patch.object(deployment_lock, "LOCK_FILE", lock_path):
descriptor = lock_path.open("w")
os.chmod(lock_path, 0o600)
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
with self.assertRaisesRegex(ValueError, "shared lock"):
with deploy_release.deployment_lock():
pass
finally:
fcntl.flock(descriptor, fcntl.LOCK_UN)
descriptor.close()
def test_fsync_error_after_atomic_replace_restores_previous_references(self):
old = {"BACKEND_IMAGE": "akyldash/backend:old", "OPENSEARCH_IMAGE": "akyldash/opensearch:old"}
new = {"BACKEND_IMAGE": "registry/backend@sha256:" + "c" * 64,
"OPENSEARCH_IMAGE": "registry/opensearch@sha256:" + "d" * 64}
with tempfile.TemporaryDirectory() as directory:
env_file = Path(directory) / "production.env"
env_file.write_text("BACKEND_IMAGE=akyldash/backend:old\nOPENSEARCH_IMAGE=akyldash/opensearch:old\n")
original_fsync = release_refs.os.fsync
calls = 0
def fail_first_directory_fsync(descriptor):
nonlocal calls
calls += 1
if calls == 2:
raise OSError("simulated directory fsync failure after replace")
original_fsync(descriptor)
with patch.object(release_refs.os, "fsync", side_effect=fail_first_directory_fsync):
with self.assertRaisesRegex(OSError, "after replace"):
release_refs.replace_images(env_file, old, new, require_root=False)
self.assertEqual(release_refs.read_images(env_file)[1], new)
with (
patch.object(deploy_release, "ENV_FILE", env_file),
patch.object(deploy_release, "replace_images", side_effect=lambda path, expected, replacement:
release_refs.replace_images(path, expected, replacement, require_root=False)),
):
deploy_release.restore_previous_references(old, new)
self.assertEqual(release_refs.read_images(env_file)[1], old)
def test_external_cutover_rejects_unvalidated_release_before_docker_operations(self):
with patch.object(deploy_release.os, "geteuid", return_value=0), patch.object(deploy_release, "run") as run_mock, patch.object(deploy_release, "compose") as compose_mock:
with self.assertRaisesRegex(ValueError, "manifest must contain exactly"):
deploy_release.deploy_external(
{}, {"schema_version": 1, "repositories": {
"backend": ["registry/backend"], "opensearch": ["registry/search"]
}}, {}, {}, "0" * 64, self.current, "ру", "кыргызча"
)
run_mock.assert_not_called()
compose_mock.assert_not_called()
def test_main_advance_after_backup_aborts_before_image_reference_switch(self):
commit = "b" * 40
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "deploy/production").mkdir(parents=True)
for name, payload in (("compose.yaml", b"compose\n"), ("Caddyfile", b"caddy\n")):
(root / "deploy/production" / name).write_bytes(payload)
from hashlib import sha256
deployment_files = {
f"deploy/production/{name}": sha256((root / "deploy/production" / name).read_bytes()).hexdigest()
for name in ("compose.yaml", "Caddyfile")
}
backup = {"images": self.current, "deployment_files": deployment_files}
backup_identity = {
"backup_id": "20261103T000000Z-" + "a" * 32,
"size": 100, "archive_sha256": "c" * 64,
"manifest_sha256": "d" * 64, "manifest": backup,
}
manifest = {
"commit": commit,
"builder": {"name": "Kaniko", "image": deploy_release.BUILDER_IMAGE},
"images": {service: f"registry/{service}@sha256:{'e' * 64}" for service in ("backend", "opensearch")},
}
journal = {"status": "external-backup-verified"}
with (
patch.object(deploy_release.os, "geteuid", return_value=0),
patch.object(deploy_release, "ROOT", root),
patch.object(deploy_release, "STATE_FILE", root / "state.json"),
patch.object(deploy_release, "JOURNAL_FILE", root / "journal.json"),
patch.object(deploy_release, "validate_manifest", return_value=manifest),
patch("release_state.validate_backup_identity", return_value=backup_identity),
patch("release_handoff.latest_main_sha", side_effect=(commit, "f" * 40)),
patch("release_source.require_runtime_compatibility"),
patch.object(deploy_release, "image_ids", return_value=self.current),
patch.object(deploy_release, "app_images", return_value=self.current),
patch.object(deploy_release, "read_json", return_value=journal),
patch.object(deploy_release, "write_json_atomic"),
patch.object(deploy_release, "run", side_effect=(
"", "sha256:" + "1" * 64, "", "sha256:" + "2" * 64,
"sha256:" + "1" * 64, "sha256:" + "2" * 64,
)),
patch.object(deploy_release, "replace_images") as replace,
patch.object(deploy_release, "restore_previous_references"),
patch.object(deploy_release, "compose"),
patch.object(deploy_release, "write_deployed_backend_image"),
):
with self.assertRaisesRegex(ValueError, "cutover failed"):
deploy_release.deploy_external(
manifest, {}, backup, backup_identity, "9" * 64,
self.current, "ru", "ky",
)
replace.assert_not_called()
if __name__ == "__main__":
unittest.main()