diff --git a/CHANGELOG.md b/CHANGELOG.md index 07752dc..a43c4d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Не выпущено +- Rollback preflight проверяет digest-pinned image references по сохранённому + image ID и останавливается до изменений deployment; переключение production + на digest и полноценный digest rollback пока не реализованы. - Infrastructure: добавлен офлайн-контракт release manifest с точной схемой, digest-ссылками и repository allowlist; проверен Compose override с `--no-build` как требованием будущего deploy. Публикация, provenance и diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index 4e97162..9750ceb 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -89,6 +89,11 @@ protected file metadata remain unverified. `status` helper, guarded rollback helper, and installer with exact sudoers command rules. None is installed on the VPS. CI contains no production deployment job; starting/stopping services remains owner-run. +- Rollback preflight validates digest-pinned image references against archived + image IDs and local availability, then refuses before stopping services or + creating a backup because the current flow cannot safely switch immutable + digest references. This behavior is covered by synthetic local tests; it has + not been exercised on the VPS. - This branch adds an offline release-manifest validator and tests for the repository allowlist intended for a future root-owned host integration. The manifest format does not prove source provenance. No registry, host policy, diff --git a/docs/operations/ROLLBACK.md b/docs/operations/ROLLBACK.md index c0d7bef..c1f93dd 100644 --- a/docs/operations/ROLLBACK.md +++ b/docs/operations/ROLLBACK.md @@ -18,11 +18,18 @@ sudo -n /usr/local/sbin/akyldash-rollback \ The helper verifies archive integrity, Compose image references, deployment file hashes, and the availability of both prior local images. It makes and verifies a second cold backup, restores only the prior backend/OpenSearch image -tags, waits for Compose healthchecks, and checks HTTPS locally. It refuses to -restore if the current Compose/Caddyfile differs from the saved release. This -does not restore data or automatically undo a release with an unknown schema -change. The application has no formal migration framework; compatibility must -be checked for every change. If it is uncertain, stop and ask the owner. +tags, waits for Compose healthchecks, and checks HTTPS locally. Digest-pinned +references are validated against their recorded local image IDs, but rollback +fails before stopping services or making a backup when either image reference +uses `@sha256:`. The helper cannot safely switch immutable digest references +with the current release-recording flow. Do not retag a digest manually; digest +rollback needs a separately reviewed atomic release-reference mechanism. + +It refuses to restore if the current Compose/Caddyfile differs from the saved +release. This does not restore data or automatically undo a release with an +unknown schema change. The application has no formal migration framework; +compatibility must be checked for every change. If it is uncertain, stop and ask +the owner. The initial owner-run deployment must record its running backend image ID in `/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates diff --git a/infra/deploy/rollback.py b/infra/deploy/rollback.py index a08e5ff..16ce1bb 100644 --- a/infra/deploy/rollback.py +++ b/infra/deploy/rollback.py @@ -19,6 +19,12 @@ ENV_FILE = Path("/etc/akyldash/production.env") BACKUP_DIR = Path("/var/backups/akyldash") BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") APP_SERVICES = {"backend", "opensearch"} +IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*") +IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}") + + +def is_digest_reference(reference: str) -> bool: + return IMAGE_DIGEST_RE.fullmatch(reference) is not None def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]: @@ -34,14 +40,48 @@ def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, image_id = image.get("image_id") if reference != configured.get(service): raise ValueError(f"Compose image reference changed for {service}; refusing rollback") - if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*", reference): - raise ValueError(f"{service} image reference is not a local tag") + if not isinstance(reference, str) or not ( + IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference) + ): + raise ValueError(f"invalid image reference for {service}") if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"invalid previous image ID for {service}") plan[service] = (image_id, reference) return plan +def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None: + for image_id, reference in plan.values(): + try: + actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True) + except subprocess.CalledProcessError as error: + raise ValueError(f"previous image is unavailable: {image_id}") from error + if actual != image_id: + raise ValueError(f"previous image is unavailable: {image_id}") + if is_digest_reference(reference): + try: + referenced_id = run( + ["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True + ) + except subprocess.CalledProcessError as error: + raise ValueError(f"previous digest reference is unavailable: {reference}") from error + if referenced_id != image_id: + raise ValueError(f"previous digest reference does not match its recorded image: {reference}") + + +def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None: + if any(is_digest_reference(reference) for _image_id, reference in plan.values()): + raise ValueError( + "rollback of digest-pinned images is not supported safely; no services or image references were changed" + ) + + +def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None: + require_tag_rollback(plan) + for image_id, reference in plan.values(): + run(["docker", "image", "tag", image_id, reference]) + + def run(command: list[str], *, capture: bool = False) -> str: result = subprocess.run(command, check=True, text=True, capture_output=capture) return result.stdout.strip() if capture else "" @@ -96,10 +136,8 @@ def rollback(backup: Path, data_compatible: bool) -> None: rendered = json.loads(compose("config", "--format", "json", capture=True)) configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES} plan = image_restore_plan(manifest, configured) - for image_id, _reference in plan.values(): - actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True) - if actual != image_id: - raise ValueError(f"previous image is unavailable: {image_id}") + verify_previous_images(plan) + require_tag_rollback(plan) backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz" backup_output = BACKUP_DIR / backup_name @@ -123,8 +161,7 @@ def rollback(backup: Path, data_compatible: bool) -> None: compose("up", "--detach") raise - for image_id, reference in plan.values(): - run(["docker", "image", "tag", image_id, reference]) + tag_previous_images(plan) compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") run( [ diff --git a/infra/deploy/test_rollback.py b/infra/deploy/test_rollback.py index 7b5842b..9cb05a5 100644 --- a/infra/deploy/test_rollback.py +++ b/infra/deploy/test_rollback.py @@ -1,6 +1,12 @@ +import json +import subprocess +import sys import unittest +from types import SimpleNamespace +from unittest.mock import patch -from rollback import image_restore_plan +import rollback +from rollback import image_restore_plan, require_tag_rollback, tag_previous_images, verify_previous_images class RollbackPlanTests(unittest.TestCase): @@ -28,9 +34,96 @@ class RollbackPlanTests(unittest.TestCase): def test_refuses_digest_reference_that_cannot_be_retagged(self): self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64 self.configured["backend"] = self.manifest["images"]["backend"]["configured"] - with self.assertRaisesRegex(ValueError, "local tag"): + plan = image_restore_plan(self.manifest, self.configured) + self.assertEqual(plan["backend"], ("sha256:" + "a" * 64, self.configured["backend"])) + with self.assertRaisesRegex(ValueError, "not supported safely"): + require_tag_rollback(plan) + + def test_refuses_malformed_digest_reference(self): + self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:xyz" + self.configured["backend"] = self.manifest["images"]["backend"]["configured"] + with self.assertRaisesRegex(ValueError, "invalid image reference"): image_restore_plan(self.manifest, self.configured) + def test_refuses_invalid_recorded_image_id(self): + self.manifest["images"]["backend"]["image_id"] = "sha256:not-a-digest" + with self.assertRaisesRegex(ValueError, "invalid previous image ID"): + image_restore_plan(self.manifest, self.configured) + + def test_digest_reference_must_resolve_to_recorded_image_id(self): + digest = "registry/backend@sha256:" + "c" * 64 + self.manifest["images"]["backend"]["configured"] = digest + self.configured["backend"] = digest + plan = image_restore_plan(self.manifest, self.configured) + with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "a" * 64, + "sha256:" + "b" * 64]): + verify_previous_images(plan) + with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "d" * 64]): + with self.assertRaisesRegex(ValueError, "does not match its recorded image"): + verify_previous_images(plan) + + def test_missing_previous_image_fails_closed(self): + plan = image_restore_plan(self.manifest, self.configured) + with patch("rollback.run", side_effect=subprocess.CalledProcessError(1, "docker")): + with self.assertRaisesRegex(ValueError, "previous image is unavailable"): + verify_previous_images(plan) + + def test_digest_reference_is_never_tagged(self): + digest = "registry/backend@sha256:" + "c" * 64 + plan = {"backend": ("sha256:" + "a" * 64, digest)} + with patch("rollback.run") as run_mock: + with self.assertRaisesRegex(ValueError, "not supported safely"): + tag_previous_images(plan) + run_mock.assert_not_called() + + def test_digest_rollback_fails_before_stopping_or_changing_references(self): + backend_id = "sha256:" + "a" * 64 + opensearch_id = "sha256:" + "b" * 64 + digest = "registry/backend@sha256:" + "c" * 64 + manifest = { + "images": { + "backend": {"configured": digest, "image_id": backend_id}, + "opensearch": self.manifest["images"]["opensearch"], + }, + "deployment_files": { + "deploy/production/compose.yaml": "compose-hash", + "deploy/production/Caddyfile": "caddy-hash", + }, + } + rendered = { + "services": { + "backend": {"image": digest}, + "opensearch": {"image": self.configured["opensearch"]}, + } + } + fake_archive = SimpleNamespace( + digest=lambda path: "compose-hash" if path.name == "compose.yaml" else "caddy-hash", + verify_archive=lambda _backup: manifest, + ) + backup = rollback.BACKUP_DIR / "synthetic.tar.gz" + with ( + patch("rollback.Path.is_file", return_value=True), + patch("rollback.Path.is_symlink", return_value=False), + patch("rollback.Path.stat", return_value=SimpleNamespace(st_uid=0, st_mode=0o100600)), + patch("rollback.compose", return_value=json.dumps(rendered)) as compose_mock, + patch("rollback.run", side_effect=[backend_id, backend_id, opensearch_id]) as run_mock, + patch.dict(sys.modules, {"archive": fake_archive}), + ): + with self.assertRaisesRegex(ValueError, "not supported safely"): + rollback.rollback(backup, data_compatible=True) + compose_mock.assert_called_once_with("config", "--format", "json", capture=True) + self.assertEqual(run_mock.call_count, 3) + self.assertTrue(all(call.args[0][:3] == ["docker", "image", "inspect"] + for call in run_mock.call_args_list)) + + def test_tag_rollback_keeps_existing_image_tag_operation(self): + plan = image_restore_plan(self.manifest, self.configured) + with patch("rollback.run") as run_mock: + tag_previous_images(plan) + self.assertEqual(run_mock.call_count, 2) + run_mock.assert_any_call(["docker", "image", "tag", "sha256:" + "a" * 64, + "akyldash/backend:local"]) + if __name__ == "__main__": unittest.main()