From 852d639c455ff1c60202405e112d47b8128d66c6 Mon Sep 17 00:00:00 2001 From: agent Date: Sun, 25 Oct 2026 13:21:40 +0300 Subject: [PATCH 01/23] feat: prepare production deployment readiness --- .gitea/actions/kaniko-push/Dockerfile | 3 + .gitea/actions/kaniko-push/action.yml | 23 ++ .gitea/actions/kaniko-push/release-build.sh | 35 +++ .gitea/workflows/ci.yaml | 6 + .gitea/workflows/release-images.yaml | 58 +++++ AGENTS.md | 6 +- backend/README.md | 16 +- backend/search/minjust_opensearch.py | 73 +++++- backend/test_minjust_opensearch.py | 138 +++++++++-- deploy/production/README.md | 47 ++++ docs/operations/CI_RUNNER.md | 35 ++- docs/operations/CORPUS_HANDOFF.md | 20 +- docs/operations/CURRENT_STATE.md | 93 +++++-- docs/operations/DEPLOYMENT_OWNER_PACKET.md | 65 +++++ docs/operations/ROLLBACK.md | 50 ++-- infra/deploy/create_release_manifest.py | 69 ++++++ infra/deploy/deploy_release.py | 254 ++++++++++++++++++++ infra/deploy/deployment_lock.py | 32 +++ infra/deploy/install-helper.sh | 10 +- infra/deploy/release-policy.json | 7 + infra/deploy/release_refs.py | 79 ++++++ infra/deploy/release_state.py | 77 ++++++ infra/deploy/rollback.py | 177 ++++++++++++-- infra/deploy/test_deploy_release.py | 93 +++++++ infra/deploy/test_release_manifest.py | 52 +++- infra/deploy/test_release_refs.py | 64 +++++ infra/deploy/test_release_workflow.py | 34 +++ infra/deploy/test_rollback.py | 73 +++++- infra/deploy/validate_release.py | 18 +- 29 files changed, 1578 insertions(+), 129 deletions(-) create mode 100644 .gitea/actions/kaniko-push/Dockerfile create mode 100644 .gitea/actions/kaniko-push/action.yml create mode 100644 .gitea/actions/kaniko-push/release-build.sh create mode 100644 .gitea/workflows/release-images.yaml create mode 100644 docs/operations/DEPLOYMENT_OWNER_PACKET.md create mode 100644 infra/deploy/create_release_manifest.py create mode 100644 infra/deploy/deploy_release.py create mode 100644 infra/deploy/deployment_lock.py create mode 100644 infra/deploy/release-policy.json create mode 100644 infra/deploy/release_refs.py create mode 100644 infra/deploy/release_state.py create mode 100644 infra/deploy/test_deploy_release.py create mode 100644 infra/deploy/test_release_refs.py create mode 100644 infra/deploy/test_release_workflow.py diff --git a/.gitea/actions/kaniko-push/Dockerfile b/.gitea/actions/kaniko-push/Dockerfile new file mode 100644 index 0000000..204005e --- /dev/null +++ b/.gitea/actions/kaniko-push/Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412 +COPY release-build.sh /usr/local/bin/release-build +ENTRYPOINT ["/busybox/sh", "/usr/local/bin/release-build"] diff --git a/.gitea/actions/kaniko-push/action.yml b/.gitea/actions/kaniko-push/action.yml new file mode 100644 index 0000000..e98525b --- /dev/null +++ b/.gitea/actions/kaniko-push/action.yml @@ -0,0 +1,23 @@ +name: Publish a pinned Akyldash image +description: Build and push one commit-tagged image with pinned rootless Kaniko +inputs: + workspace: + description: Checked out source directory + required: true + dockerfile: + description: Dockerfile path relative to workspace + required: true + destination: + description: Commit-specific OCI tag + required: true + digest_file: + description: Workspace path for Kaniko's resulting digest + required: true +runs: + using: docker + image: Dockerfile + args: + - ${{ inputs.workspace }} + - ${{ inputs.dockerfile }} + - ${{ inputs.destination }} + - ${{ inputs.digest_file }} diff --git a/.gitea/actions/kaniko-push/release-build.sh b/.gitea/actions/kaniko-push/release-build.sh new file mode 100644 index 0000000..dd1265c --- /dev/null +++ b/.gitea/actions/kaniko-push/release-build.sh @@ -0,0 +1,35 @@ +#!/busybox/sh +set -eu + +workspace=$1 +dockerfile=$2 +destination=$3 +digest_file=$4 +: "${REGISTRY_USERNAME:?registry username secret is required}" +: "${REGISTRY_PASSWORD:?registry password secret is required}" + +case "$dockerfile" in + deploy/production/Dockerfile.backend) + printf '%s\n' "$destination" | grep -Eq '^git\.vakanaut\.ru/admin/akyldash/backend:[0-9a-f]{40}-[0-9]+-[1-9][0-9]*$' || { + echo 'ERROR: backend destination must be a commit/run/attempt tag in the approved registry namespace' >&2; exit 2; + } + ;; + deploy/production/Dockerfile.opensearch) + printf '%s\n' "$destination" | grep -Eq '^git\.vakanaut\.ru/admin/akyldash/opensearch:[0-9a-f]{40}-[0-9]+-[1-9][0-9]*$' || { + echo 'ERROR: OpenSearch destination must be a commit/run/attempt tag in the approved registry namespace' >&2; exit 2; + } + ;; + *) echo 'ERROR: Dockerfile is outside the approved production image set' >&2; exit 2 ;; +esac + +mkdir -p /kaniko/.docker "$(dirname "$digest_file")" +auth=$(printf '%s:%s' "$REGISTRY_USERNAME" "$REGISTRY_PASSWORD" | base64 | tr -d '\n') +umask 077 +printf '{"auths":{"git.vakanaut.ru":{"auth":"%s"}}}\n' "$auth" > /kaniko/.docker/config.json +unset auth REGISTRY_USERNAME REGISTRY_PASSWORD +/kaniko/executor \ + --context="dir://${workspace}" \ + --dockerfile="${workspace}/${dockerfile}" \ + --destination="$destination" \ + --digest-file="$digest_file" \ + --no-push-cache diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 2bd50f4..8232d13 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -44,6 +44,7 @@ jobs: infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/bootstrap/audit-host.sh \ infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \ infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \ + .gitea/actions/kaniko-push/release-build.sh \ infra/bootstrap/test-audit-host.sh infra/bootstrap/test-root-guard.sh \ infra/bootstrap/test-verify-helper.sh infra/test/run-as-unprivileged.sh \ infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \ @@ -52,6 +53,11 @@ jobs: infra/runner/test-ci-container-builds.sh python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())" python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/validate_release.py').read_text())" + python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/deploy_release.py').read_text())" + python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/create_release_manifest.py').read_text())" + python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/release_refs.py').read_text())" + python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/release_state.py').read_text())" + python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/deployment_lock.py').read_text())" - name: Run deployment contract and rollback tests run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v - name: Validate production Compose and Caddyfile diff --git a/.gitea/workflows/release-images.yaml b/.gitea/workflows/release-images.yaml new file mode 100644 index 0000000..28d04ad --- /dev/null +++ b/.gitea/workflows/release-images.yaml @@ -0,0 +1,58 @@ +name: Release production images + +on: + workflow_dispatch: + +jobs: + publish-images: + if: gitea.ref == 'refs/heads/main' + runs-on: akyldash-ci + steps: + - uses: actions/checkout@v4 + - name: Require explicitly configured Gitea registry credentials + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + test -n "$REGISTRY_USERNAME" || { echo 'REGISTRY_USERNAME repository secret is required'; exit 1; } + test -n "$REGISTRY_PASSWORD" || { echo 'REGISTRY_PASSWORD repository secret is required'; exit 1; } + - name: Build and publish backend image + uses: ./.gitea/actions/kaniko-push + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + with: + workspace: ${{ github.workspace }} + dockerfile: deploy/production/Dockerfile.backend + destination: git.vakanaut.ru/admin/akyldash/backend:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + digest_file: ${{ github.workspace }}/release/backend.digest + - name: Build and publish OpenSearch image + uses: ./.gitea/actions/kaniko-push + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + with: + workspace: ${{ github.workspace }} + dockerfile: deploy/production/Dockerfile.opensearch + destination: git.vakanaut.ru/admin/akyldash/opensearch:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + digest_file: ${{ github.workspace }}/release/opensearch.digest + - name: Create and validate release descriptor + run: | + python3 infra/deploy/create_release_manifest.py \ + --repository admin/akyldash \ + --commit '${{ github.sha }}' \ + --run-id '${{ github.run_id }}' \ + --run-attempt '${{ github.run_attempt }}' \ + --backend-digest release/backend.digest \ + --opensearch-digest release/opensearch.digest \ + --output release/release-manifest.json + python3 infra/deploy/validate_release.py release/release-manifest.json \ + --policy infra/deploy/release-policy.json + sha256sum release/release-manifest.json > release/release-manifest.json.sha256 + - name: Print verified release descriptor for the owner + run: | + sha256sum -c release/release-manifest.json.sha256 + echo 'Release descriptor (copy from this authenticated run log):' + cat release/release-manifest.json + echo 'Descriptor SHA-256:' + cat release/release-manifest.json.sha256 diff --git a/AGENTS.md b/AGENTS.md index a1e3fff..5e19ba0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,11 @@ # Акылдаш — инструкции для агента Проект: юридическая информационно-аналитическая платформа; Search API на -Python, SQLite и OpenSearch. Production пока не развернут. +Python, SQLite и OpenSearch. На 2026-10-25 публичный `/review` и прямой origin +отвечают HTTP 200; состав production-контейнеров, готовность индекса и +проверенность backup/restore не подтверждены. Сверяй актуальное состояние с +[`CURRENT_STATE.md`](docs/operations/CURRENT_STATE.md), не считай HTTP 200 +доказательством готовности поиска. - Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и [правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно. diff --git a/backend/README.md b/backend/README.md index 0aae543..6533960 100644 --- a/backend/README.md +++ b/backend/README.md @@ -171,16 +171,26 @@ python3 backend/search/minjust_opensearch.py \ После проверки production-индекса следует переключать alias, чтобы удалённые фрагменты не оставались в поиске. -Чтобы переключить alias атомарно только после успешной полной загрузки, -передайте `--alias`: +Alias переключается только после refresh, точного сравнения полного fragment +count с inventory, обеих языковых canary-проверок и совпадения ожидаемого +текущего индекса. Перед cutover обязательно укажите RU/KY запросы и точное имя +старого target: ```bash python3 backend/search/minjust_opensearch.py \ --url http://127.0.0.1:9200 \ --index akyldash-fragments-v2 \ - --alias akyldash-fragments-current + --alias akyldash-fragments-current \ + --expected-old-index akyldash-fragments-v1 \ + --canary-ru 'утверждённый русский запрос' \ + --canary-ky 'бекитилген кыргызча суроо' ``` +Wildcard removal, пустой индекс и alias с несколькими target запрещены. Если +старый target успел измениться после проверки, OpenSearch отклонит атомарную +операцию удаления точного старого alias. `--resume` повторно проверяет весь +ожидаемый inventory, а не только количество документов текущего запуска. + После каждого принятого Bulk-пакета загрузчик атомарно сохраняет checkpoint и печатает код безопасного возобновления. При временных HTTP 429/5xx, timeout и обрыве соединения запрос повторяется автоматически. Прерванную загрузку можно diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index 078fa57..b977ab8 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -17,7 +17,7 @@ from typing import Iterator from search.catalog import authority_codes, legal_force_code, source_code -APP_VERSION = "0.13.2" +APP_VERSION = "0.13.3" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") @@ -342,10 +342,17 @@ def load_bulk( resume: bool = False, checkpoint: Path = Path("data/opensearch/minjust-fragments.checkpoint.json"), alias: str | None = None, + expected_old_index: str | None = None, + canary_ru: str | None = None, + canary_ky: str | None = None, ) -> tuple[int, int]: base = url.rstrip("/") if alias is not None and (not alias or alias == index): raise ValueError("--alias must differ from --index") + if alias is not None and (not expected_old_index or expected_old_index == index): + raise ValueError("alias publication requires an exact --expected-old-index") + if alias is not None and (not canary_ru or not canary_ky): + raise ValueError("alias publication requires both Russian and Kyrgyz canary queries") index_url = f"{base}/{urllib.parse.quote(index, safe='')}" if resume: cluster_uuid, index_uuid = opensearch_identity(base, index, index_url) @@ -364,6 +371,7 @@ def load_bulk( alias, ) documents = document_count(input_root, limit, start_at) + expected_fragments = sum(1 for _ in bulk_pairs(input_root, index, limit)) if alias else None if not resume: write_json_atomic(checkpoint, state) fragments = 0 @@ -398,21 +406,72 @@ def load_bulk( write_json_atomic(checkpoint, state) print(f"checkpoint={last_code} fragments={fragments}", flush=True) if alias: - switch_alias(base, index, alias) + if not expected_fragments: + raise ValueError("refusing to publish an index with an empty expected fragment inventory") + validate_candidate_index(base, index, expected_fragments, canary_ru, canary_ky) + switch_alias(base, index, alias, expected_old_index) state["complete"] = True write_json_atomic(checkpoint, state) return documents, fragments -def switch_alias(base: str, index: str, alias: str) -> None: +def alias_target(base: str, alias: str) -> str: + alias_url = f"{base.rstrip('/')}/_alias/{urllib.parse.quote(alias, safe='')}" + try: + result = request_json(alias_url, "GET", None, "application/json") + except RuntimeError as error: + if "failed with HTTP 404:" in str(error): + raise ValueError(f"alias {alias} has no current target; explicit bootstrap is required") from error + raise + if not isinstance(result, dict) or len(result) != 1: + raise ValueError(f"alias {alias} must point to exactly one index") + target, definition = next(iter(result.items())) + if not isinstance(definition, dict) or alias not in definition.get("aliases", {}): + raise ValueError(f"alias {alias} response is malformed") + return target + + +def validate_candidate_index( + base: str, index: str, expected_fragments: int, canary_ru: str, canary_ky: str +) -> None: + if expected_fragments <= 0: + raise ValueError("expected fragment count must be positive") + encoded_index = urllib.parse.quote(index, safe='') + refreshed = request_json(f"{base.rstrip('/')}/{encoded_index}/_refresh", "POST", None, "application/json") + shards = refreshed.get("_shards", {}) + if not isinstance(shards, dict) or shards.get("failed") != 0: + raise RuntimeError(f"refresh failed for candidate index {index}") + count_result = request_json(f"{base.rstrip('/')}/{encoded_index}/_count", "GET", None, "application/json") + if count_result.get("count") != expected_fragments: + raise RuntimeError( + f"candidate index {index} has {count_result.get('count')} fragments; expected {expected_fragments}" + ) + for language, query in (("ru", canary_ru), ("ky", canary_ky)): + payload = json.dumps({"size": 1, "query": {"match": {f"text_{language}": query}}}).encode() + result = request_json( + f"{base.rstrip('/')}/{encoded_index}/_search", "POST", payload, "application/json" + ) + hits = result.get("hits", {}).get("hits", []) + if not hits or not isinstance(hits[0].get("_source", {}).get(f"text_{language}"), str): + raise RuntimeError(f"candidate index {index} failed the {language} canary query") + + +def switch_alias(base: str, index: str, alias: str, expected_old_index: str) -> None: if not alias or alias == index: raise ValueError("--alias must differ from --index") + if not expected_old_index or expected_old_index == index: + raise ValueError("--expected-old-index must name the exact previous index") + actual_old_index = alias_target(base, alias) + if actual_old_index != expected_old_index: + raise ValueError( + f"alias {alias} currently targets {actual_old_index}, expected {expected_old_index}; refusing cutover" + ) result = request_json( f"{base.rstrip('/')}/_aliases", "POST", json.dumps({ "actions": [ - {"remove": {"index": "*", "alias": alias, "must_exist": False}}, + {"remove": {"index": expected_old_index, "alias": alias, "must_exist": True}}, {"add": {"index": index, "alias": alias}}, ] }).encode(), @@ -430,6 +489,9 @@ def main() -> int: parser.add_argument("--limit", type=int) parser.add_argument("--url", help="create the index and stream bounded Bulk requests instead of writing a file") parser.add_argument("--alias", help="atomically point this alias at --index after a successful load") + parser.add_argument("--expected-old-index", help="exact current alias target required for guarded cutover") + parser.add_argument("--canary-ru", help="approved Russian query that must match the candidate index") + parser.add_argument("--canary-ky", help="approved Kyrgyz query that must match the candidate index") parser.add_argument("--mapping", type=Path, default=DEFAULT_MAPPING) parser.add_argument("--batch-mb", type=int, default=25) parser.add_argument("--resume", action="store_true", help="load into an existing index") @@ -460,6 +522,9 @@ def main() -> int: arguments.resume, checkpoint, arguments.alias, + arguments.expected_old_index, + arguments.canary_ru, + arguments.canary_ky, ) destination = arguments.url else: diff --git a/backend/test_minjust_opensearch.py b/backend/test_minjust_opensearch.py index 7e4d9c5..a409825 100644 --- a/backend/test_minjust_opensearch.py +++ b/backend/test_minjust_opensearch.py @@ -70,21 +70,88 @@ class MinjustOpenSearchTest(unittest.TestCase): self.assertEqual(flags, {"920292": True, "920492": False}) def test_switches_alias_atomically_after_successful_load(self): - with patch("search.minjust_opensearch.request_json", return_value={"acknowledged": True}) as request: - switch_alias("http://127.0.0.1:9200/", "akyldash-fragments-v2", "akyldash-fragments-current") + with patch("search.minjust_opensearch.request_json", side_effect=[ + {"akyldash-fragments-v1": {"aliases": {"akyldash-fragments-current": {}}}}, + {"acknowledged": True}, + ]) as request: + switch_alias( + "http://127.0.0.1:9200/", "akyldash-fragments-v2", + "akyldash-fragments-current", "akyldash-fragments-v1", + ) - self.assertEqual(request.call_args.args[:2], ("http://127.0.0.1:9200/_aliases", "POST")) - body = json.loads(request.call_args.args[2]) - self.assertEqual(body["actions"][0], {"remove": {"index": "*", "alias": "akyldash-fragments-current", "must_exist": False}}) + self.assertEqual(request.call_args_list[1].args[:2], ("http://127.0.0.1:9200/_aliases", "POST")) + body = json.loads(request.call_args_list[1].args[2]) + self.assertEqual(body["actions"][0], {"remove": {"index": "akyldash-fragments-v1", "alias": "akyldash-fragments-current", "must_exist": True}}) self.assertEqual(body["actions"][1], {"add": {"index": "akyldash-fragments-v2", "alias": "akyldash-fragments-current"}}) with self.assertRaisesRegex(ValueError, "differ"): - switch_alias("http://127.0.0.1:9200", "same", "same") + switch_alias("http://127.0.0.1:9200", "same", "same", "old") with self.assertRaisesRegex(ValueError, "differ"): - switch_alias("http://127.0.0.1:9200", "index", "") - with patch("search.minjust_opensearch.request_json", return_value={"acknowledged": False}): + switch_alias("http://127.0.0.1:9200", "index", "", "old") + with patch("search.minjust_opensearch.request_json", return_value={ + "akyldash-fragments-v3": {"aliases": {"alias": {}}} + }) as request: + with self.assertRaisesRegex(ValueError, "currently targets"): + switch_alias("http://127.0.0.1:9200", "new", "alias", "akyldash-fragments-v1") + request.assert_called_once() + with patch("search.minjust_opensearch.request_json", side_effect=[ + {"old": {"aliases": {"alias": {}}}}, {"acknowledged": False} + ]): with self.assertRaisesRegex(RuntimeError, "did not acknowledge"): - switch_alias("http://127.0.0.1:9200", "index", "alias") + switch_alias("http://127.0.0.1:9200", "index", "alias", "old") + + def test_zero_fragment_candidate_never_reaches_alias_switch(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "normalized" + document = root / "documents/1" + document.mkdir(parents=True) + with closing(sqlite3.connect(root / "manifest.sqlite3")) as connection: + connection.execute("CREATE TABLE documents (code TEXT, state TEXT)") + connection.execute("INSERT INTO documents VALUES ('1', 'success')") + (document / "document.json").write_text( + json.dumps({"source_code": "1", "editions": []}), encoding="utf-8" + ) + with ( + patch("search.minjust_opensearch.request_json", side_effect=[ + {}, {"cluster_uuid": "cluster"}, + {"test-index": {"settings": {"index": {"uuid": "index"}}}}, + ]) as request, + patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, + patch("search.minjust_opensearch.switch_alias") as switch, + ): + with self.assertRaisesRegex(ValueError, "empty expected fragment inventory"): + load_bulk( + root, "http://127.0.0.1:9200", "test-index", + checkpoint=Path(temporary) / "checkpoint.json", alias="current", + expected_old_index="old-index", canary_ru="закон", canary_ky="мыйзам", + ) + self.assertEqual(request.call_count, 3) + validate_candidate.assert_not_called() + switch.assert_not_called() + + def test_candidate_index_requires_exact_count_and_both_language_canaries(self): + from search.minjust_opensearch import validate_candidate_index + + with patch("search.minjust_opensearch.request_json", side_effect=[ + {"_shards": {"failed": 0}}, + {"count": 2}, + {"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}}, + {"hits": {"hits": [{"_source": {"text_ky": "сыноо"}}]}}, + ]) as request: + validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") + self.assertEqual(request.call_count, 4) + with patch("search.minjust_opensearch.request_json", side_effect=[ + {"_shards": {"failed": 0}}, {"count": 1} + ]): + with self.assertRaisesRegex(RuntimeError, "expected 2"): + validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") + with patch("search.minjust_opensearch.request_json", side_effect=[ + {"_shards": {"failed": 0}}, {"count": 2}, + {"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}}, + {"hits": {"hits": []}}, + ]): + with self.assertRaisesRegex(RuntimeError, "ky canary"): + validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") def test_exports_atomic_bulk_and_rejects_mismatched_fragment(self): with tempfile.TemporaryDirectory() as temporary: @@ -160,13 +227,16 @@ class MinjustOpenSearchTest(unittest.TestCase): document_codes(root / "normalized", 1, "8") checkpoint = root / "checkpoint.json" - with patch("search.minjust_opensearch.request_json") as request: + with ( + patch("search.minjust_opensearch.request_json") as request, + patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, + patch("search.minjust_opensearch.switch_alias") as switch, + ): request.side_effect = [ {}, {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, {"errors": False, "items": [{"index": {}}, {"index": {}}]}, - {"acknowledged": True}, ] self.assertEqual( load_bulk( @@ -176,11 +246,19 @@ class MinjustOpenSearchTest(unittest.TestCase): maximum_bytes=4096, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ), (2, 2), ) - self.assertEqual(request.call_args_list[-2].args[3], "application/x-ndjson") - self.assertEqual(request.call_args_list[-1].args[:2], ("http://127.0.0.1:9200/_aliases", "POST")) + self.assertEqual(request.call_args_list[-1].args[3], "application/x-ndjson") + validate_candidate.assert_called_once_with( + "http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам" + ) + switch.assert_called_once_with( + "http://127.0.0.1:9200", "test-index", "test-current", "test-index-old" + ) state = json.loads(checkpoint.read_text(encoding="utf-8")) self.assertEqual(state["last_document_code"], "7") self.assertTrue(state["complete"]) @@ -191,7 +269,11 @@ class MinjustOpenSearchTest(unittest.TestCase): legacy["last_document_code"] = "8" legacy["complete"] = False checkpoint.write_text(json.dumps(legacy), encoding="utf-8") - with patch("search.minjust_opensearch.request_json") as request: + with ( + patch("search.minjust_opensearch.request_json") as request, + patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, + patch("search.minjust_opensearch.switch_alias"), + ): request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, @@ -226,6 +308,9 @@ class MinjustOpenSearchTest(unittest.TestCase): resume=True, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ) with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ @@ -256,12 +341,18 @@ class MinjustOpenSearchTest(unittest.TestCase): resume=True, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ) - with patch("search.minjust_opensearch.request_json") as request: + with ( + patch("search.minjust_opensearch.request_json") as request, + patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, + patch("search.minjust_opensearch.switch_alias"), + ): request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, - {"acknowledged": True}, ] self.assertEqual( load_bulk( @@ -272,11 +363,16 @@ class MinjustOpenSearchTest(unittest.TestCase): resume=True, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ), (1, 0), ) - self.assertTrue(all(call.args[1] == "GET" for call in request.call_args_list[:-1])) - self.assertEqual(request.call_args_list[-1].args[1], "POST") + self.assertTrue(all(call.args[1] == "GET" for call in request.call_args_list)) + validate_candidate.assert_called_once_with( + "http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам" + ) state["last_document_code"] = "9" state["complete"] = False @@ -295,6 +391,9 @@ class MinjustOpenSearchTest(unittest.TestCase): resume=True, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ) failed_checkpoint = root / "failed-checkpoint.json" @@ -341,6 +440,9 @@ class MinjustOpenSearchTest(unittest.TestCase): resume=True, checkpoint=checkpoint, alias="test-current", + expected_old_index="test-index-old", + canary_ru="закон", + canary_ky="мыйзам", ) http_error = urllib.error.HTTPError( diff --git a/deploy/production/README.md b/deploy/production/README.md index b6caa67..a6264e4 100644 --- a/deploy/production/README.md +++ b/deploy/production/README.md @@ -110,3 +110,50 @@ Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md), Automated production deployment is not enabled. Deployments remain owner-run; record the release and verify service health before considering it complete. + +## Image release and owner-run cutover + +PR CI still builds both production images with Kaniko `--no-push`. The separate +`Release production images` workflow is manual and runs only when dispatched +from protected `main`. It requires repository secrets named +`REGISTRY_USERNAME` and `REGISTRY_PASSWORD`; without them it fails before any +build. Configure those secrets only after the owner confirms a dedicated +Gitea account/token with write access to the two repository packages and no +unneeded administration rights. Do not reuse a runner or production host +credential. The workflow uses the Gitea host and fixed package namespace +`git.vakanaut.ru/admin/akyldash/{backend,opensearch}`, pushes unique +commit/run/attempt tags, captures registry digests, validates the release +descriptor, and prints its JSON plus SHA-256 sidecar to the authenticated Gitea +run log for the owner to retrieve. The runner cache remains disabled. Because +the installed act_runner 0.2.13 has not yet been validated with Gitea's +v1-compatible artifact fork, the authenticated run log is the current bounded +retrieval fallback; this PR makes no runner cache changes. Registry retention must preserve referenced digests +for the full rollback window; this policy has not yet been confirmed. + +The descriptor records source repository/ref/commit, workflow run and attempt, +pinned Kaniko builder and both image digests. It is an unsigned trace record, +not a cryptographic attestation; deployment relies on retrieving it from the +authenticated Gitea run log and verifying its checksum and repository allowlist. +Do not treat the sidecar alone as proof of authenticity. + +For a future owner-approved cutover, use the root-owned checkout and run +`infra/deploy/deploy_release.py` as root with the downloaded manifest, its +matching verified cold backup, and `infra/deploy/release-policy.json`, plus +owner-verified non-empty Russian and Kyrgyz canary queries, for example +`--canary-ru '' --canary-ky ''`. +The helper checks both queries against the public Search API through local +HTTPS after Compose reports healthy, and requires at least one result with a +document code before recording the new release state. Select queries only +after corpus/index canaries have been verified; do not use generic terms that +can match unrelated test data. The helper requires the backup image references and image IDs to match the live +stack, serializes deploy and rollback with `/run/akyldash-deploy.lock`, +atomically switches both protected env image refs, and always runs Compose with +`--no-build`. It records recovery state before changing references; a failed +cutover keeps a secret-free recovery journal and refuses a subsequent release +until an owner resolves it. Rollback accepts a digest-pinned previous release +only when that backup matches the separately recorded previous refs and image +IDs. Legacy tag rollback remains supported. Neither the helper nor registry +secrets are installed on the VPS by this PR; deployment remains an owner-only +operation and must wait for host/data/backup review. The exact sanitized +evidence required from the owner is listed in +[`DEPLOYMENT_OWNER_PACKET.md`](../../docs/operations/DEPLOYMENT_OWNER_PACKET.md). diff --git a/docs/operations/CI_RUNNER.md b/docs/operations/CI_RUNNER.md index 1ac88b1..8c22777 100644 --- a/docs/operations/CI_RUNNER.md +++ b/docs/operations/CI_RUNNER.md @@ -59,10 +59,11 @@ Dockerfiles are built in a separate pinned Kaniko container action with explicit job container receives no Docker socket. Kaniko uses `--no-push` and `--no-push-cache`, so CI validates the full backend and OpenSearch builds but does not publish images or export build output. The two builds run sequentially -within one job. CI run 69 completed both builds on the runner. Actual per-action +within one job. CI run 69 completed both builds historically; the latest +verified main CI run 175 and smoke run 176 succeeded on `b5adc092`. Actual per-action resource-limit enforcement and peak memory or disk usage were not independently -measured or verified. No registry credentials, -production secrets, elevated container options, or arbitrary host mounts are configured. +measured or verified. No registry credentials, production service secrets, +elevated container options, or arbitrary host mounts are configured in PR CI. The Kaniko action uses the `linux/amd64` v1.28.5-debug manifest `sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412`, @@ -72,6 +73,25 @@ validation. act_runner starts these container actions; they do not expose its rootless Docker socket to workflow steps. Production deployment is not automated. +`release-images.yaml` is a separate manual workflow allowed only on protected +`main`. It uses the same pinned daemonless Kaniko builder sequentially and +disables Kaniko layer-cache pushes, so it adds no Docker daemon, socket, VM, or +concurrent runner slot. Its temporary peak RAM/CPU/disk use has not been +measured separately from the already successful sequential image builds; the +configured 2 CPU, 3 GiB RAM, 512-process per-job and 15 GiB storage limits still +apply. The workflow fails before building if `REGISTRY_USERNAME` or +`REGISTRY_PASSWORD` is absent. Those names are not configured or verified on +Gitea. If the owner later enables it, the credential must be package-write-only +for the two approved image names; no production env, service secret, runner +socket, or host mount is passed. The release descriptor is unsigned and the +workflow prints its small JSON and checksum to the authenticated run log. The +runner's cache service remains disabled. Gitea's runner-v2 documentation says +that its v1 cache protocol requires Gitea's upload/download artifact forks; +this repository does not yet validate a compatible artifact action against +the installed act_runner 0.2.13, so the log is the bounded retrieval fallback +and no runner cache setting is changed. The workflow has +not yet been run with registry credentials. + ## Installation and registration Review the root script and host audit output before running it. It installs @@ -160,10 +180,11 @@ that token; do not put it in command-line arguments or environment variables. present in the job container. 3. Confirm the run reports the expected runner, succeeds, and the runner service and health timer remain active. -4. Inspect the `CI` workflow run and confirm Caddy plus both Kaniko actions pass - on `akyldash-ci`. The Kaniko jobs are intentionally build-only: publishing - images requires a separately reviewed registry workflow and credentials. - Do not add production deploy workflows or secrets to this runner. +4. Inspect the `CI` workflow run and confirm Caddy plus both no-push Kaniko + actions pass on `akyldash-ci`. Image publication is a distinct manual + main-only workflow with package-only credentials; it does not deploy. + Never add production service secrets, Docker socket access, or host mounts + to this runner. ## Upgrade diff --git a/docs/operations/CORPUS_HANDOFF.md b/docs/operations/CORPUS_HANDOFF.md index 3d84ae8..12b5207 100644 --- a/docs/operations/CORPUS_HANDOFF.md +++ b/docs/operations/CORPUS_HANDOFF.md @@ -171,12 +171,14 @@ to a new staging location with owner-approved permissions, verify the copied manifest and files there, and only then consider placing it under `DATA_ROOT`. Never overwrite an existing corpus or OpenSearch volume. -The current bulk importer checks individual Bulk API item results, but it can -switch an alias after a zero-fragment input and does not enforce a refresh, -count comparison, or representative-query check before that switch. Until a -separate change adds those gates, do not pass `--alias` during production -loading. Load into a new versioned index only after the corpus transfer is -verified; require a nonzero fragment count, successful item results, an -explicit refresh, a matching OpenSearch document count, and representative -Russian/Kyrgyz queries before an owner performs an alias change. Take and -verify the required backup before any production index or service change. +The bulk importer now refuses alias publication until all Bulk item results +are successful, the full staged index is refreshed, its fragment count exactly +matches the complete normalized inventory (including on resume), and approved +Russian and Kyrgyz canaries both return text. The alias must currently target +the exact `--expected-old-index`; wildcard removals and implicit first-time +alias creation are rejected. OpenSearch performs removal of that exact target +and addition of the candidate in one alias request. A mismatch or validation +failure leaves the existing alias untouched. This is a code-level gate only: +no production index or corpus was changed, and the live inventory and canary +terms still require owner verification. See `backend/README.md` for the +operator flags and use a new versioned index for every candidate. diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index d1b450e..6ebb8c6 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -1,10 +1,53 @@ # Infrastructure current state -Read-only host recheck: 2026-10-18. Source: SSH as `codex`, local repository -inspection, and Gitea API. Root-only Docker state, UFW rule details, and -protected file metadata remain unverified. +Read-only deployment recheck: 2026-10-25. Source: SSH as `codex`, direct-origin +and public HTTPS probes, local `main` checkout, and Gitea API. No VPS files, +services, firewall, containers, volumes, production data, or credentials were +changed or inspected. Root-only Docker state, UFW rule details, protected file +metadata, current backup/restore validity, and the live corpus/index remain +unverified. The 2026-10-18 inventory below is a historical snapshot where it +conflicts with this dated recheck. -## VPS facts +## Latest read-only recheck — 2026-10-25 + +- `https://akyldash.kg/review` returned HTTP 200 with valid TLS through the + public Cloudflare edge and through direct HTTPS to origin `206.223.249.118`. + This verifies that route only; it does not prove search or index readiness. +- TCP listeners now include 22, 80, and 443. The older no-listener/Cloudflare + 521 observations below are historical, not current state. +- VPS reports 4 CPUs, 7.7 GiB RAM (about 3.7 GiB available), and 31 GiB free + on `/`. Current runner-mount free space was not rechecked. +- `/opt/akyldash/current` exists, is a root-owned directory mode `0755`. + Reading its Git SHA failed with Git's `dubious ownership` safety guard; no + `safe.directory` setting was added, so the installed revision is unknown. +- `akyldash-act-runner.service` is active under `akyldash-runner`; installed + `act_runner` is `0.2.13`. The separate `akyldash-runner-restart.service` + remains failed since 2026-10-14; its cause could not be read from the + unprivileged journal view. No restart was attempted. +- `akyldash-verify`, `akyldash-compose`, and `akyldash-rollback` are absent + from PATH in the unprivileged check; no root helper or Docker operation was + run. +- Gitea is `1.25.5`; `main` is `b5adc09203cbbbd298054248206b9f47ae4a44ca`. + CI run 175 and runner smoke 176 succeeded on that commit. Package write + access and runner inventory remain outside this API principal's permissions. +- Owner-only outstanding evidence: sanitized container/volume and protected + file metadata; current backup and separate-target restore verification; + live corpus/index counts and RU/KY canaries; Cloudflare TLS mode; registry + writer identity and digest retention. Never include secret values in that + evidence packet. The checklist is in + [`DEPLOYMENT_OWNER_PACKET.md`](DEPLOYMENT_OWNER_PACKET.md). + +The deployment PR adds a manual main-only image-publish workflow and an +owner-run `--no-build` cutover path, but neither has been executed against the +VPS. Its release descriptor is an unsigned trace record; cryptographic image +attestation remains a separate owner decision. + +## Historical host snapshot — 2026-10-18 + +Source: SSH as `codex`, local repository inspection, and Gitea API. Root-only +Docker state, UFW rule details, and protected file metadata were not verified. + +## Historical VPS facts — 2026-10-18 - Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`. - OS: Ubuntu Server `24.04.5 LTS`; kernel `6.8.0-142-generic` from the prior @@ -42,13 +85,11 @@ protected file metadata remain unverified. and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS, `https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner registration and Gitea Actions protocol connectivity are not confirmed. -- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS, - HTTPS returned Cloudflare `521` with TLS verification result `0` on - 2026-10-18. TCP listeners were SSH `22` and local DNS only, so no origin - listener on `80`/`443` was observed. Origin TLS and Cloudflare Full (strict) - are not verified. +- `akyldash.kg` resolved to Cloudflare IPv4/IPv6 edge addresses. HTTPS returned + Cloudflare `521` on this historical 2026-10-18 check; current recheck is + recorded above. Cloudflare Full (strict) mode remains unverified. -## Application and Gitea facts +## Historical application and Gitea facts — 2026-10-18 - Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`. - The repository API reports push permission, Actions and packages enabled, and @@ -87,23 +128,31 @@ protected file metadata remain unverified. The mode has not been run on a VPS. - The repository defines a root-owned read-only verifier, Compose `config` and `status` helper, guarded rollback helper, and installer with exact sudoers - command rules. None is installed on the VPS. CI contains no production - deployment job; starting/stopping services remains owner-run. -- Rollback preflight validates digest-pinned image references against archived - image IDs and local availability, then refuses before stopping services or - creating a backup because the current flow cannot safely switch immutable - digest references. This behavior is covered by synthetic local tests; it has - not been exercised on the VPS. -- This branch adds an offline release-manifest validator and tests for the - repository allowlist intended for a future root-owned host integration. The - manifest format does not prove source provenance. No registry, host policy, - publish credential, or deployment integration is configured or installed. + command rules. None is installed on the VPS. This branch adds an owner-run + release cutover helper; it is deliberately not added to the `codex` sudoers + allowlist. CI contains no production deployment job; service changes remain + owner-run. +- Digest rollback now requires release state to pin current and previous + refs/image IDs and the selected verified backup to match that previous state. + It serializes with cutover, atomically changes both protected image refs, uses + `--no-build`, and preserves a secret-free recovery journal on failure. Legacy + tag rollback remains supported. These paths have synthetic tests only; they + have not been exercised on the VPS. +- The manual image release workflow is restricted to workflow dispatch from + protected `main`; PR image builds remain no-push. It requires owner-configured + `REGISTRY_USERNAME` and `REGISTRY_PASSWORD`, validates published digests + against an explicit repository allowlist, and prints a commit/run/attempt + descriptor in the authenticated run log (runner artifact/cache support stays + disabled). The descriptor is unsigned trace metadata, not cryptographic + attestation. Registry write rights and digest retention are not configured + or verified. - A repository-scoped official `act_runner` `0.2.13` installation is configured in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB ext4 loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU, 3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048 tasks), rotated file logs, and a systemd health timer. Jobs target the - `akyldash-ci` label and do not receive either Docker socket. On 2026-10-18, + `akyldash-ci` label and do not receive either Docker socket. Latest verified + main CI run 175 and smoke run 176 succeeded on `b5adc092`. On 2026-10-18, SSH showed `akyldash-act-runner.service` active under `akyldash-runner`, with zero restarts and about 115 MiB current memory. Its rootless Docker store is mounted from `/dev/loop0` as ext4 with `rw,nosuid,nodev`; the 15 GiB mount had diff --git a/docs/operations/DEPLOYMENT_OWNER_PACKET.md b/docs/operations/DEPLOYMENT_OWNER_PACKET.md new file mode 100644 index 0000000..4003929 --- /dev/null +++ b/docs/operations/DEPLOYMENT_OWNER_PACKET.md @@ -0,0 +1,65 @@ +# Deployment owner evidence packet + +No production change is authorized by this packet. Provide the following +sanitized facts before enabling registry publishing or running an owner cutover. +Never include token/password values, environment-file contents, secret material, +or raw process environments. + +## Registry + +- Confirm the package names `admin/akyldash/backend` and + `admin/akyldash/opensearch` are available on `git.vakanaut.ru` and can be + linked to this repository as needed. +- Decide whether to configure repository Actions secrets + `REGISTRY_USERNAME` / `REGISTRY_PASSWORD`. Use a dedicated package writer + with only the required package write scope. Do not reuse Gitea administration, + VPS, runner, or production service credentials. +- Confirm the registry retention rule keeps all digest-pinned versions for at + least the rollback window. State the number of versions/days, not secret data. +- The release descriptor is printed to the authenticated workflow run log; the + runner's artifact/cache service remains disabled to avoid extra disk use. +- The current descriptor is unsigned. Decide whether Gitea run/artifact + traceability is sufficient for the first owner-run release or require a + separately provisioned signing/attestation service before publishing. + +## Host and backup + +- Have the owner provide a dated, sanitized root audit: installed checkout SHA; + Compose service names, image references and image IDs; running/health state; + persistent mount paths and read-only/read-write flags; protected env/secret + owner, group, and mode only; listener/firewall summary. Exclude all env values. +- Confirm an archive created by the documented cold-backup procedure passes + checksum and archive verification and records the current backend/OpenSearch + configured refs and image IDs. Confirm a restore was tested on a separate + target. Do not restore over production as part of this review. +- Confirm whether current data/corpus files are mounted read-only and provide + owner/mode metadata before proposing any UID, ownership, or mount changes. +- After the PR is reviewed, the owner must run the existing root-only + `infra/deploy/install-helper.sh` once from the approved root-owned checkout + to install the updated rollback helper modules (`release_refs.py`, + `release_state.py`, and `deployment_lock.py`). The installer keeps the + existing fixed sudoers command set; it does not add a deploy command for + `codex`. Do not run this installer before review or as part of CI. +- Confirm that the production host can pull the private images, or that the + published packages are intentionally public. If a pull credential is + needed, keep it in the owner-managed root Docker config; it is not a runner + secret. + +## Search data + +- Provide the current OpenSearch alias and exact target index, expected document + and fragment inventory counts, and owner-approved Russian and Kyrgyz canary + queries. Report only counts, query text, match/no-match and service health; + do not attach corpus documents or production database files. +- Confirm that the normalized corpus generation selected for publication has + passed the repository inventory verifier and that unresolved raw-source + errors have an explicit owner disposition. + +## Edge and host recovery + +- Confirm Cloudflare SSL/TLS mode (including Full/strict) and origin certificate + renewal status from the owner control plane. Read-only HTTP 200 checks do not + prove these settings. +- Report the cause or owner-approved disposition for the failed + `akyldash-runner-restart.service`; the runner itself was active during the + latest read-only check. No restart or unit change is requested here. diff --git a/docs/operations/ROLLBACK.md b/docs/operations/ROLLBACK.md index c1f93dd..5fb24a7 100644 --- a/docs/operations/ROLLBACK.md +++ b/docs/operations/ROLLBACK.md @@ -2,34 +2,44 @@ ## Current status -There is no production release, backup, or installed rollback helper. The -repository contains a root-owned rollback helper and a separate installer, but -neither has been run on the VPS. +On 2026-10-25 the public `/review` endpoint and direct origin returned HTTP +200, but current containers, data/index state, and a verified backup remain +unknown. The repository rollback helper and installer are not confirmed +installed on the VPS. This document describes code-level procedures only. -After a verified cold backup is created before a release, an owner must check -that the data format is compatible with the prior backend/OpenSearch images. -Then run the limited helper with that archive: +Before a release, the owner must verify a cold backup whose recorded backend +and OpenSearch image references and image IDs match the current deployment, +and check data compatibility with the target and prior versions. The +owner-run `infra/deploy/deploy_release.py` verifies the manifest allowlist, +backup, current image IDs, and registry digests; serializes against rollback +using `/run/akyldash-deploy.lock`; atomically replaces both image refs; +and starts Compose with `--no-build`. It is not installed as a `codex` sudo +command and was not run on the VPS. + +After a successful release, the owner can run the restricted rollback helper +with the exact verified backup recorded as the previous release: ```bash sudo -n /usr/local/sbin/akyldash-rollback \ /var/backups/akyldash/.tar.gz --data-compatible ``` -The helper verifies archive integrity, Compose image references, deployment -file hashes, and the availability of both prior local images. It makes and -verifies a second cold backup, restores only the prior backend/OpenSearch image -tags, waits for Compose healthchecks, and checks HTTPS locally. Digest-pinned -references are validated against their recorded local image IDs, but rollback -fails before stopping services or making a backup when either image reference -uses `@sha256:`. The helper cannot safely switch immutable digest references -with the current release-recording flow. Do not retag a digest manually; digest -rollback needs a separately reviewed atomic release-reference mechanism. +The helper takes the same deployment lock, validates the selected archive +against the separately pinned previous refs and image IDs, verifies image +availability, makes and verifies a second cold backup, and switches both +protected image refs atomically. Digest references are never retagged; legacy +tag images are restored by image ID. Compose runs with `--no-build`, followed +by healthcheck and local HTTPS validation. A failed cutover or rollback keeps +a secret-free recovery journal and blocks another release until an owner +resolves it. A legacy deployment without release state can still roll back +matching legacy tags, but digest rollback is refused until the state is +established by the reviewed release helper. -It refuses to restore if the current Compose/Caddyfile differs from the saved -release. This does not restore data or automatically undo a release with an -unknown schema change. The application has no formal migration framework; -compatibility must be checked for every change. If it is uncertain, stop and ask -the owner. +It refuses if the current Compose/Caddyfile differs from the backup or the +backup's image records do not match release state. This does not restore data or +automatically undo a release with an unknown schema change. The application +has no formal migration framework; compatibility must be checked for every +change. If it is uncertain, stop and ask the owner. The initial owner-run deployment must record its running backend image ID in `/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates diff --git a/infra/deploy/create_release_manifest.py b/infra/deploy/create_release_manifest.py new file mode 100644 index 0000000..d9bf684 --- /dev/null +++ b/infra/deploy/create_release_manifest.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Create a traceable, unsigned release descriptor from verified build digests.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +BUILDER_IMAGE = "ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412" + + +def digest(path: Path) -> str: + value = path.read_text(encoding="ascii").strip() + if not re.fullmatch(r"sha256:[0-9a-f]{64}", value): + raise ValueError(f"invalid Kaniko digest in {path}") + return value + + +def create(repository: str, commit: str, run_id: str, run_attempt: str, backend: Path, opensearch: Path) -> dict: + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9._-]*", repository): + raise ValueError("repository must be lowercase owner/repository") + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("commit must be a 40-character lowercase Git SHA-1") + if not re.fullmatch(r"[0-9]+", run_id): + raise ValueError("run_id must be numeric") + if not re.fullmatch(r"[1-9][0-9]*", run_attempt): + raise ValueError("run_attempt must be a positive integer") + registry = "git.vakanaut.ru/" + repository + return { + "schema_version": 2, + "source_ref": "refs/heads/main", + "repository": repository, + "commit": commit, + "workflow": ".gitea/workflows/release-images.yaml", + "run_id": run_id, + "run_attempt": run_attempt, + "builder": {"name": "Kaniko", "image": BUILDER_IMAGE}, + "images": { + "backend": f"{registry}/backend@{digest(backend)}", + "opensearch": f"{registry}/opensearch@{digest(opensearch)}", + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True) + parser.add_argument("--commit", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-attempt", required=True) + parser.add_argument("--backend-digest", required=True, type=Path) + parser.add_argument("--opensearch-digest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + manifest = create(args.repository, args.commit, args.run_id, args.run_attempt, args.backend_digest, args.opensearch_digest) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + except (OSError, UnicodeError, ValueError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print(f"PASS release descriptor for {manifest['commit']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/deploy/deploy_release.py b/infra/deploy/deploy_release.py new file mode 100644 index 0000000..0f54eaf --- /dev/null +++ b/infra/deploy/deploy_release.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Owner-run, locked production release cutover using a verified cold backup.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.parse +from pathlib import Path + +from create_release_manifest import BUILDER_IMAGE +from deployment_lock import deployment_lock +from release_refs import read_images, replace_images +from release_state import read_json, validate_images, validate_state, write_json_atomic +from validate_release import read_json as read_manifest_json, validate_manifest + +ROOT = Path("/opt/akyldash/current") +COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" +ENV_FILE = Path("/etc/akyldash/production.env") +BACKUP_DIR = Path("/var/backups/akyldash") +BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") +STATE_FILE = Path("/opt/akyldash/release-state.json") +JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json") +DEPLOYED_IMAGE_FILE = Path("/opt/akyldash/deployed-image.txt") +SERVICES = ("backend", "opensearch") +SEARCH_CANARY_URL = "https://akyldash.kg/search" + + +def run(command: list[str], capture: bool = False) -> str: + result = subprocess.run(command, check=True, text=True, capture_output=capture) + return result.stdout.strip() if capture else "" + + +def compose(*args: str, capture: bool = False) -> str: + return run(["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], capture) + + +def image_ids() -> dict[str, dict[str, str]]: + configured = read_images(ENV_FILE)[1] + records = {} + for service, key in (("backend", "BACKEND_IMAGE"), ("opensearch", "OPENSEARCH_IMAGE")): + container = compose("ps", "--all", "--quiet", service, capture=True) + if not container or "\n" in container: + raise ValueError(f"expected exactly one existing {service} container") + inspection = run( + ["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True + ) + running, separator, image_id = inspection.partition("\t") + if not separator or running != "true": + raise ValueError(f"{service} is not running; refusing release cutover") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError(f"invalid running image ID for {service}") + records[service] = {"configured": configured[key], "image_id": image_id} + return records + + +def write_deployed_backend_image(image_id: str) -> None: + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError("invalid backend image ID for deployment record") + with tempfile.NamedTemporaryFile(dir=DEPLOYED_IMAGE_FILE.parent, prefix=".deployed-image-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(f"{image_id}\n".encode("ascii")) + stream.flush() + os.fsync(stream.fileno()) + try: + os.chmod(temporary, 0o600) + os.replace(temporary, DEPLOYED_IMAGE_FILE) + directory_fd = os.open(DEPLOYED_IMAGE_FILE.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + temporary.unlink(missing_ok=True) + + +def backup_manifest(path: Path) -> dict: + if not path.is_file() or path.is_symlink() or path.resolve().parent != BACKUP_DIR.resolve(): + raise ValueError(f"verified backup must be a regular file directly under {BACKUP_DIR}") + if path.stat().st_uid != 0 or path.stat().st_mode & 0o777 != 0o600: + raise ValueError("release backup must be root-owned mode 0600") + checksum = Path(f"{path}.sha256") + if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600: + raise ValueError("release backup checksum must be root-owned mode 0600") + sys.path.insert(0, str(BACKUP_TOOL.parent)) + from archive import digest, verify_archive + + manifest = verify_archive(path) + current_files = { + name: digest(ROOT / name) + for name in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") + } + if manifest.get("deployment_files") != current_files: + raise ValueError("backup does not match the checked out Compose and Caddy configuration") + return manifest + + +def require_backup_matches_current(current: dict, backup: dict) -> None: + if current != backup: + raise ValueError("backup image references or image IDs do not match the current deployment") + + +def app_images(value: dict, label: str) -> dict[str, dict[str, str]]: + images = value.get("images") + if not isinstance(images, dict) or not set(SERVICES) <= images.keys(): + raise ValueError(f"{label} does not contain both application image records") + return validate_images({service: images[service] for service in SERVICES}, label) + + +def validate_search_response(payload: dict, language: str) -> None: + results = payload.get("results") if isinstance(payload, dict) else None + if not isinstance(results, list) or not results: + raise ValueError(f"{language} search canary returned no results") + if not isinstance(results[0], dict) or not str(results[0].get("code", "")).strip(): + raise ValueError(f"{language} search canary returned a malformed first result") + + +def check_search_canary(language: str, query: str) -> None: + if language not in {"ru", "ky"} or not query.strip(): + raise ValueError("both Russian and Kyrgyz search canaries must be non-empty") + params = urllib.parse.urlencode({"q": query, "language": language, "page_size": "1"}) + output = run( + ["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", + f"{SEARCH_CANARY_URL}?{params}"], capture=True + ) + try: + payload = json.loads(output) + except json.JSONDecodeError as error: + raise ValueError(f"{language} search canary returned invalid JSON") from error + validate_search_response(payload, language) + + +def deploy(manifest_path: Path, backup_path: Path, policy_path: Path, canary_ru: str, canary_ky: str) -> None: + if os.geteuid() != 0: + raise ValueError("production cutover is owner-run and requires root") + manifest = validate_manifest(read_manifest_json(manifest_path), read_manifest_json(policy_path)) + if manifest["builder"] != {"name": "Kaniko", "image": BUILDER_IMAGE}: + raise ValueError("release descriptor uses an unapproved builder") + manifest_sha = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + + with deployment_lock(): + if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink(): + raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before another release") + current = image_ids() + backup = backup_manifest(backup_path) + previous = app_images(backup, "backup images") + require_backup_matches_current(current, previous) + pinned = None + if STATE_FILE.exists() or STATE_FILE.is_symlink(): + pinned = validate_state(read_json(STATE_FILE)) + if pinned["current"] != current: + raise ValueError("current deployment does not match the recorded release state") + for service in SERVICES: + reference = manifest["images"][service] + run(["docker", "pull", reference]) + pulled_id = run(["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", pulled_id): + raise ValueError(f"registry returned an invalid image ID for {service}") + + target = { + service: {"configured": manifest["images"][service], "image_id": run( + ["docker", "image", "inspect", "--format", "{{.Id}}", manifest["images"][service]], capture=True + )} + for service in SERVICES + } + old_env = {"BACKEND_IMAGE": current["backend"]["configured"], "OPENSEARCH_IMAGE": current["opensearch"]["configured"]} + new_env = {"BACKEND_IMAGE": target["backend"]["configured"], "OPENSEARCH_IMAGE": target["opensearch"]["configured"]} + journal = { + "schema_version": 1, "status": "prepared", "commit": manifest["commit"], + "manifest_sha256": manifest_sha, "backup": str(backup_path), + "previous": current, "target": target, + } + write_json_atomic(JOURNAL_FILE, journal) + switched = False + state_written = False + try: + compose("stop") + replace_images(ENV_FILE, old_env, new_env) + switched = True + journal["status"] = "references-switched" + write_json_atomic(JOURNAL_FILE, journal) + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + actual = image_ids() + if actual != target: + raise ValueError("started image IDs do not match the release descriptor") + run(["curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", + "https://akyldash.kg/review", "--output", "/dev/null"]) + check_search_canary("ru", canary_ru) + check_search_canary("ky", canary_ky) + write_deployed_backend_image(actual["backend"]["image_id"]) + state = { + "schema_version": 1, "commit": manifest["commit"], + "manifest_sha256": manifest_sha, "current": actual, "previous": current, + } + write_json_atomic(STATE_FILE, state) + state_written = True + JOURNAL_FILE.unlink() + print(f"PASS deployed release {manifest['commit']} with --no-build") + except Exception as error: + if state_written: + journal["status"] = "release-active-journal-cleanup-needed" + journal["error_type"] = type(error).__name__ + journal["recovery"] = "target refs, release state, and health check completed; verify host and resolve journal" + write_json_atomic(JOURNAL_FILE, journal) + raise ValueError(f"release is active; recovery journal preserved at {JOURNAL_FILE}") from error + journal["status"] = "cutover-failed" + journal["error_type"] = type(error).__name__ + if switched: + try: + replace_images(ENV_FILE, new_env, old_env) + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + write_deployed_backend_image(current["backend"]["image_id"]) + journal["recovery"] = "previous references restored; verify service health" + except Exception as recovery_error: + journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__}); references and backup are recorded" + else: + try: + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + write_deployed_backend_image(current["backend"]["image_id"]) + journal["recovery"] = "previous stack restarted" + except Exception as recovery_error: + journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__}); backup is recorded" + if pinned is not None: + write_json_atomic(STATE_FILE, pinned) + elif (STATE_FILE.exists() or STATE_FILE.is_symlink()) and not STATE_FILE.is_symlink(): + STATE_FILE.unlink() + write_json_atomic(JOURNAL_FILE, journal) + raise ValueError(f"cutover failed; recovery journal preserved at {JOURNAL_FILE}") from error + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("backup", type=Path) + parser.add_argument("--policy", required=True, type=Path) + parser.add_argument("--canary-ru", required=True, help="known query expected to return a Russian search result") + parser.add_argument("--canary-ky", required=True, help="known query expected to return a Kyrgyz search result") + args = parser.parse_args() + try: + deploy(args.manifest, args.backup, args.policy, args.canary_ru, args.canary_ky) + except Exception as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/deploy/deployment_lock.py b/infra/deploy/deployment_lock.py new file mode 100644 index 0000000..1f8aeca --- /dev/null +++ b/infra/deploy/deployment_lock.py @@ -0,0 +1,32 @@ +"""One root-owned lock shared by image cutover and rollback.""" + +from __future__ import annotations + +import fcntl +import os +import stat +from contextlib import contextmanager +from pathlib import Path + +LOCK_FILE = Path("/run/akyldash-deploy.lock") + + +@contextmanager +def deployment_lock(): + descriptor = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o777 != 0o600: + raise ValueError("deployment lock must be a regular mode-0600 file") + if os.geteuid() == 0 and info.st_uid != 0: + raise ValueError("deployment lock must be root-owned") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ValueError("another deployment or rollback holds the shared lock") from error + try: + yield + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) diff --git a/infra/deploy/install-helper.sh b/infra/deploy/install-helper.sh index 728c134..9393e9a 100644 --- a/infra/deploy/install-helper.sh +++ b/infra/deploy/install-helper.sh @@ -8,10 +8,13 @@ fi checkout=/opt/akyldash/current rollback_source=$checkout/infra/deploy/rollback.py +release_refs_source=$checkout/infra/deploy/release_refs.py +release_state_source=$checkout/infra/deploy/release_state.py +deployment_lock_source=$checkout/infra/deploy/deployment_lock.py backup_source=$checkout/infra/backup/archive.py verify_source=$checkout/infra/bootstrap/verify-helper.sh compose_source=$checkout/infra/deploy/compose-helper.sh -if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then +if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $release_refs_source || ! -f $release_state_source || ! -f $deployment_lock_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then echo "Install the approved repository checkout at $checkout first." >&2 exit 1 fi @@ -44,7 +47,7 @@ if ! command -v visudo >/dev/null 2>&1; then exit 1 fi -for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do +for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /usr/local/lib/akyldash/release_refs.py /usr/local/lib/akyldash/release_state.py /usr/local/lib/akyldash/deployment_lock.py /etc/sudoers.d/akyldash-codex; do if [[ -L $path ]]; then echo "Refusing symlink at $path." >&2 exit 1 @@ -53,6 +56,9 @@ done install -d -o root -g root -m 0755 /usr/local/lib/akyldash install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py +install -o root -g root -m 0644 "$release_refs_source" /usr/local/lib/akyldash/release_refs.py +install -o root -g root -m 0644 "$release_state_source" /usr/local/lib/akyldash/release_state.py +install -o root -g root -m 0644 "$deployment_lock_source" /usr/local/lib/akyldash/deployment_lock.py install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback install -o root -g root -m 0750 "$compose_source" /usr/local/sbin/akyldash-compose diff --git a/infra/deploy/release-policy.json b/infra/deploy/release-policy.json new file mode 100644 index 0000000..bb945f7 --- /dev/null +++ b/infra/deploy/release-policy.json @@ -0,0 +1,7 @@ +{ + "schema_version": 1, + "repositories": { + "backend": ["git.vakanaut.ru/admin/akyldash/backend"], + "opensearch": ["git.vakanaut.ru/admin/akyldash/opensearch"] + } +} diff --git a/infra/deploy/release_refs.py b/infra/deploy/release_refs.py new file mode 100644 index 0000000..8b08601 --- /dev/null +++ b/infra/deploy/release_refs.py @@ -0,0 +1,79 @@ +"""Read and atomically replace the two application image references.""" + +from __future__ import annotations + +import os +import re +import tempfile +from pathlib import Path + +KEYS = {"BACKEND_IMAGE", "OPENSEARCH_IMAGE"} +IMAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*(?:@sha256:[0-9a-f]{64})?\Z") + + +def read_images(path: Path) -> tuple[str, dict[str, str]]: + if path.is_symlink() or not path.is_file(): + raise ValueError("production env must be a regular non-symlink file") + text = path.read_text(encoding="utf-8") + values: dict[str, str] = {} + lines = text.splitlines(keepends=True) + for line in lines: + if not re.match(r"^\s*(?:export\s+)?(?:BACKEND_IMAGE|OPENSEARCH_IMAGE)\b", line): + continue + match = re.fullmatch(r"(BACKEND_IMAGE|OPENSEARCH_IMAGE)=([^\r\n]+)(\r?\n)?", line) + if not match: + raise ValueError("application image assignments must use plain KEY=value syntax") + key, value = match.group(1), match.group(2) + if key in values or not IMAGE.fullmatch(value): + raise ValueError(f"invalid or duplicate {key} in production env") + values[key] = value + if set(values) != KEYS: + raise ValueError("production env must define exactly one backend and OpenSearch image") + return text, values + + +def replace_images( + path: Path, + expected: dict[str, str], + replacement: dict[str, str], + *, + require_root: bool = True, +) -> None: + if set(expected) != KEYS or set(replacement) != KEYS: + raise ValueError("both complete application image references are required") + if any(not isinstance(value, str) or not IMAGE.fullmatch(value) for value in (*expected.values(), *replacement.values())): + raise ValueError("invalid application image reference") + text, current = read_images(path) + info = path.stat() + if require_root and (os.geteuid() != 0 or info.st_uid != 0 or info.st_mode & 0o777 != 0o600): + raise ValueError("production env must be root-owned mode 0600 before a release switch") + if current != expected: + raise ValueError("production image references changed; refusing partial or stale release switch") + lines = text.splitlines(keepends=True) + updated = [] + for line in lines: + if not re.match(r"^\s*(?:export\s+)?(?:BACKEND_IMAGE|OPENSEARCH_IMAGE)\b", line): + updated.append(line) + continue + match = re.fullmatch(r"(BACKEND_IMAGE|OPENSEARCH_IMAGE)=([^\r\n]+)(\r?\n)?", line) + if match: + newline = match.group(3) or "" + updated.append(f"{match.group(1)}={replacement[match.group(1)]}{newline}") + else: + raise ValueError("application image assignments must use plain KEY=value syntax") + target = path.parent + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=target, prefix=f".{path.name}.", delete=False) as stream: + temporary = Path(stream.name) + stream.write("".join(updated)) + stream.flush() + os.fsync(stream.fileno()) + try: + os.chmod(temporary, path.stat().st_mode & 0o777) + os.replace(temporary, path) + directory_fd = os.open(target, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + temporary.unlink(missing_ok=True) diff --git a/infra/deploy/release_state.py b/infra/deploy/release_state.py new file mode 100644 index 0000000..4d1d27d --- /dev/null +++ b/infra/deploy/release_state.py @@ -0,0 +1,77 @@ +"""Atomic, secret-free release state and recovery journal helpers.""" + +from __future__ import annotations + +import json +import os +import re +import tempfile +from pathlib import Path + +SERVICES = {"backend", "opensearch"} + + +def validate_images(value: object, label: str) -> dict[str, dict[str, str]]: + if not isinstance(value, dict) or set(value) != SERVICES: + raise ValueError(f"{label} must contain backend and opensearch") + result = {} + for service in sorted(SERVICES): + record = value[service] + if not isinstance(record, dict) or set(record) != {"configured", "image_id"}: + raise ValueError(f"{label} has an invalid {service} record") + configured, image_id = record["configured"], record["image_id"] + if ( + not isinstance(configured, str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*(?:@sha256:[0-9a-f]{64})?", configured) + or not isinstance(image_id, str) + or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) + ): + raise ValueError(f"{label} has invalid {service} values") + result[service] = {"configured": configured, "image_id": image_id} + return result + + +def write_json_atomic(path: Path, value: dict, mode: int = 0o600) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, sort_keys=True, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + try: + os.chmod(temporary, mode) + os.replace(temporary, path) + directory_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + temporary.unlink(missing_ok=True) + + +def read_json(path: Path) -> dict: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"release state is not a regular file: {path}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"release state is unreadable: {error}") from error + if not isinstance(value, dict): + raise ValueError("release state must be a JSON object") + return value + + +def validate_state(state: object) -> dict: + if not isinstance(state, dict) or set(state) != { + "schema_version", "commit", "manifest_sha256", "current", "previous" + } or state.get("schema_version") != 1: + raise ValueError("release state has unsupported schema") + if not isinstance(state["commit"], str) or not re.fullmatch(r"[0-9a-f]{40}|rollback", state["commit"]): + raise ValueError("release state commit is invalid") + if not isinstance(state["manifest_sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}|rollback", state["manifest_sha256"]): + raise ValueError("release state provenance is invalid") + state["current"] = validate_images(state["current"], "current images") + state["previous"] = validate_images(state["previous"], "previous images") + return state diff --git a/infra/deploy/rollback.py b/infra/deploy/rollback.py index 16ce1bb..b9cd89d 100644 --- a/infra/deploy/rollback.py +++ b/infra/deploy/rollback.py @@ -18,6 +18,8 @@ COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" ENV_FILE = Path("/etc/akyldash/production.env") BACKUP_DIR = Path("/var/backups/akyldash") BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") +STATE_FILE = Path("/opt/akyldash/release-state.json") +JOURNAL_FILE = Path("/opt/akyldash/release-recovery.json") APP_SERVICES = {"backend", "opensearch"} IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*") IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}") @@ -27,7 +29,11 @@ def is_digest_reference(reference: str) -> bool: return IMAGE_DIGEST_RE.fullmatch(reference) is not None -def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]: +def image_restore_plan( + manifest: dict, + configured: dict[str, str], + expected_previous: dict[str, dict[str, str]] | None = None, +) -> dict[str, tuple[str, str]]: images = manifest.get("images") if not isinstance(images, dict) or not APP_SERVICES <= images.keys(): raise ValueError("backup has no previous backend and OpenSearch images") @@ -38,7 +44,7 @@ def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, raise ValueError(f"invalid image record for {service}") reference = image.get("configured") image_id = image.get("image_id") - if reference != configured.get(service): + if expected_previous is None and reference != configured.get(service): raise ValueError(f"Compose image reference changed for {service}; refusing rollback") if not isinstance(reference, str) or not ( IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference) @@ -46,6 +52,10 @@ def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, raise ValueError(f"invalid image reference for {service}") if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): raise ValueError(f"invalid previous image ID for {service}") + if expected_previous is not None and expected_previous.get(service) != { + "configured": reference, "image_id": image_id + }: + raise ValueError(f"backup does not match the pinned previous {service} release") plan[service] = (image_id, reference) return plan @@ -77,9 +87,9 @@ def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None: def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None: - require_tag_rollback(plan) for image_id, reference in plan.values(): - run(["docker", "image", "tag", image_id, reference]) + if not is_digest_reference(reference): + run(["docker", "image", "tag", image_id, reference]) def run(command: list[str], *, capture: bool = False) -> str: @@ -108,11 +118,26 @@ def record_backend_image() -> None: os.chmod(temporary, 0o600) try: os.replace(temporary, target) + directory_fd = os.open(target.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) finally: temporary.unlink(missing_ok=True) def rollback(backup: Path, data_compatible: bool) -> None: + sys.path.insert(0, str(BACKUP_TOOL.parent)) + from deployment_lock import deployment_lock + + with deployment_lock(): + _rollback_locked(backup, data_compatible) + + +def _rollback_locked(backup: Path, data_compatible: bool) -> None: + if JOURNAL_FILE.exists() or JOURNAL_FILE.is_symlink(): + raise ValueError(f"recovery journal exists at {JOURNAL_FILE}; resolve it before rollback") if not data_compatible: raise ValueError("confirm data compatibility with --data-compatible before rollback") if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve(): @@ -135,14 +160,39 @@ def rollback(backup: Path, data_compatible: bool) -> None: raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first") rendered = json.loads(compose("config", "--format", "json", capture=True)) configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES} - plan = image_restore_plan(manifest, configured) + state = None + expected_previous = None + if STATE_FILE.exists() or STATE_FILE.is_symlink(): + from release_state import read_json as read_release_state, validate_state + + state = validate_state(read_release_state(STATE_FILE)) + current_refs = {service: state["current"][service]["configured"] for service in APP_SERVICES} + if current_refs != configured: + raise ValueError("release state does not match current Compose image references") + for service in APP_SERVICES: + container = compose("ps", "--all", "--quiet", service, capture=True) + if not container or "\n" in container: + raise ValueError(f"expected exactly one current {service} container") + inspection = run( + ["docker", "inspect", "--format", "{{.State.Running}}\t{{.Image}}", container], capture=True + ) + running, separator, actual_id = inspection.partition("\t") + if not separator or running != "true": + raise ValueError(f"current {service} is not running; refusing rollback") + if actual_id != state["current"][service]["image_id"]: + raise ValueError(f"current {service} image ID does not match pinned release state") + expected_previous = state["previous"] + plan = image_restore_plan(manifest, configured, expected_previous) verify_previous_images(plan) - require_tag_rollback(plan) + if expected_previous is None: + require_tag_rollback(plan) backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz" backup_output = BACKUP_DIR / backup_name - compose("stop") + before = {"BACKEND_IMAGE": configured["backend"], "OPENSEARCH_IMAGE": configured["opensearch"]} + after = {"BACKEND_IMAGE": plan["backend"][1], "OPENSEARCH_IMAGE": plan["opensearch"][1]} try: + compose("stop") run( [ "python3", @@ -157,26 +207,101 @@ def rollback(backup: Path, data_compatible: bool) -> None: ] ) run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)]) - except Exception: - compose("up", "--detach") - raise + except Exception as error: + from release_state import write_json_atomic - tag_previous_images(plan) - compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") - run( - [ - "curl", - "--fail", - "--silent", - "--show-error", - "--resolve", - "akyldash.kg:443:127.0.0.1", - "https://akyldash.kg/review", - "--output", - "/dev/null", - ] - ) - record_backend_image() + journal = { + "schema_version": 1, "status": "rollback-backup-failed", "backup": str(backup), + "error_type": type(error).__name__, "target": before, + } + try: + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + journal["recovery"] = "current release restarted; verify service health" + except Exception as recovery_error: + journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__})" + write_json_atomic(JOURNAL_FILE, journal) + raise ValueError(f"rollback preflight failed; recovery journal preserved at {JOURNAL_FILE}") from error + + journal = { + "schema_version": 1, "status": "rollback-prepared", "backup": str(backup), + "previous": {service: {"configured": plan[service][1], "image_id": plan[service][0]} for service in APP_SERVICES}, + "target": {service: {"configured": configured[service]} for service in APP_SERVICES}, + } + if state is not None: + from release_state import write_json_atomic + + write_json_atomic(JOURNAL_FILE, journal) + state_written = False + try: + tag_previous_images(plan) + if state is not None: + from release_refs import replace_images + + replace_images(ENV_FILE, before, after) + journal["status"] = "rollback-references-switched" + write_json_atomic(JOURNAL_FILE, journal) + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + run( + [ + "curl", "--fail", "--silent", "--show-error", "--resolve", "akyldash.kg:443:127.0.0.1", + "https://akyldash.kg/review", "--output", "/dev/null", + ] + ) + record_backend_image() + if state is not None: + from release_state import write_json_atomic + + restored = { + service: {"configured": plan[service][1], "image_id": plan[service][0]} + for service in APP_SERVICES + } + write_json_atomic(STATE_FILE, { + "schema_version": 1, + "commit": "rollback", + "manifest_sha256": "rollback", + "current": restored, + "previous": state["current"], + }) + state_written = True + JOURNAL_FILE.unlink(missing_ok=True) + except Exception as error: + if state is not None: + from release_refs import read_images, replace_images + from release_state import write_json_atomic + + journal["status"] = "rollback-failed" + journal["error_type"] = type(error).__name__ + if state_written: + journal["status"] = "rollback-active-journal-cleanup-needed" + journal["recovery"] = "rollback refs, release state, and health check completed; verify host and resolve journal" + write_json_atomic(JOURNAL_FILE, journal) + raise ValueError(f"rollback is active; recovery journal preserved at {JOURNAL_FILE}") from error + try: + current_text, current_env = read_images(ENV_FILE) + del current_text + active = {"BACKEND_IMAGE": current_env["BACKEND_IMAGE"], "OPENSEARCH_IMAGE": current_env["OPENSEARCH_IMAGE"]} + if active == after: + replace_images(ENV_FILE, after, before) + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + record_backend_image() + write_json_atomic(STATE_FILE, state) + journal["recovery"] = "current release references restored; verify service health" + except Exception as recovery_error: + journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__})" + write_json_atomic(JOURNAL_FILE, journal) + else: + from release_state import write_json_atomic + + journal["status"] = "rollback-failed" + journal["error_type"] = type(error).__name__ + try: + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + record_backend_image() + journal["recovery"] = "legacy release restarted; verify service health" + except Exception as recovery_error: + journal["recovery"] = f"manual recovery required ({type(recovery_error).__name__})" + write_json_atomic(JOURNAL_FILE, journal) + raise print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}") diff --git a/infra/deploy/test_deploy_release.py b/infra/deploy/test_deploy_release.py new file mode 100644 index 0000000..a455916 --- /dev/null +++ b/infra/deploy/test_deploy_release.py @@ -0,0 +1,93 @@ +import fcntl +import json +import os +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import deploy_release +import deployment_lock + + +class DeploymentCutoverTests(unittest.TestCase): + def setUp(self): + self.current = { + "backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64}, + "opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64}, + } + + def test_mismatched_backup_is_rejected_before_any_cutover(self): + changed = {**self.current, "backend": {**self.current["backend"], "image_id": "sha256:" + "c" * 64}} + with self.assertRaisesRegex(ValueError, "do not match"): + deploy_release.require_backup_matches_current(changed, self.current) + + def test_search_canary_requires_nonempty_result_with_document_code(self): + deploy_release.validate_search_response({"results": [{"code": "DOC-1"}]}, "ru") + for payload in ({"results": []}, {"results": [{}]}, {"error": "unavailable"}): + with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "canary"): + deploy_release.validate_search_response(payload, "ky") + + def test_shared_lock_rejects_concurrent_deployment(self): + with tempfile.TemporaryDirectory() as directory: + lock_path = Path(directory) / "deploy.lock" + with patch.object(deployment_lock, "LOCK_FILE", lock_path): + descriptor = lock_path.open("w") + os.chmod(lock_path, 0o600) + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + with self.assertRaisesRegex(ValueError, "shared lock"): + with deploy_release.deployment_lock(): + pass + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + descriptor.close() + + def test_failure_after_atomic_ref_switch_keeps_recovery_journal(self): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + manifest_file = directory / "release.json" + manifest_file.write_text("{}", encoding="utf-8") + policy_file = directory / "policy.json" + policy_file.write_text("{}", encoding="utf-8") + target = { + "backend": "registry/backend@sha256:" + "c" * 64, + "opensearch": "registry/opensearch@sha256:" + "d" * 64, + } + manifest = {"builder": {"name": "Kaniko", "image": deploy_release.BUILDER_IMAGE}, + "images": target, "commit": "e" * 40} + backup_path = deploy_release.BACKUP_DIR / "backup.tar.gz" + compose_calls = [None, subprocess.CalledProcessError(1, "compose up"), None] + run_calls = ["", "sha256:" + "c" * 64, "", "sha256:" + "d" * 64, + "sha256:" + "c" * 64, "sha256:" + "d" * 64] + with ( + patch.object(deploy_release.os, "geteuid", return_value=0), + patch.object(deployment_lock, "LOCK_FILE", directory / "deploy.lock"), + patch.object(deployment_lock.os, "fstat", return_value=SimpleNamespace( + st_mode=stat.S_IFREG | 0o600, st_uid=0 + )), + patch.object(deploy_release, "JOURNAL_FILE", directory / "recovery.json"), + patch.object(deploy_release, "STATE_FILE", directory / "state.json"), + patch.object(deploy_release, "read_manifest_json", side_effect=[{}, {}]), + patch.object(deploy_release, "validate_manifest", return_value=manifest), + patch.object(deploy_release, "image_ids", return_value=self.current), + patch.object(deploy_release, "backup_manifest", return_value={"images": self.current}), + patch.object(deploy_release, "run", side_effect=run_calls), + patch.object(deploy_release, "compose", side_effect=compose_calls) as compose, + patch.object(deploy_release, "replace_images") as replace, + ): + with self.assertRaisesRegex(ValueError, "recovery journal preserved"): + deploy_release.deploy(manifest_file, backup_path, policy_file, "закон", "мыйзам") + self.assertEqual(replace.call_count, 2) + self.assertEqual(compose.call_args_list[-1].args, ("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")) + journal = json.loads((directory / "recovery.json").read_text(encoding="utf-8")) + self.assertEqual(journal["status"], "cutover-failed") + self.assertIn("recovery", journal) + self.assertFalse((directory / "state.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy/test_release_manifest.py b/infra/deploy/test_release_manifest.py index d924a88..a7549bd 100644 --- a/infra/deploy/test_release_manifest.py +++ b/infra/deploy/test_release_manifest.py @@ -6,25 +6,32 @@ import tempfile import unittest from pathlib import Path +from create_release_manifest import create from validate_release import validate_manifest ROOT = Path(__file__).resolve().parents[2] REPOSITORIES = { - "backend": ["registry.invalid/akyldash/backend"], - "opensearch": ["registry.invalid/akyldash/opensearch"], + "backend": ["git.vakanaut.ru/admin/akyldash/backend"], + "opensearch": ["git.vakanaut.ru/admin/akyldash/opensearch"], } +BUILDER = {"name": "Kaniko", "image": "ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412"} class ReleaseManifestTests(unittest.TestCase): def setUp(self): self.policy = {"schema_version": 1, "repositories": REPOSITORIES} self.manifest = { - "schema_version": 1, + "schema_version": 2, "source_ref": "refs/heads/main", + "repository": "admin/akyldash", "commit": "a" * 40, + "workflow": ".gitea/workflows/release-images.yaml", + "run_id": "49", + "run_attempt": "1", + "builder": BUILDER, "images": { - "backend": "registry.invalid/akyldash/backend@sha256:" + "b" * 64, - "opensearch": "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64, + "backend": "git.vakanaut.ru/admin/akyldash/backend@sha256:" + "b" * 64, + "opensearch": "git.vakanaut.ru/admin/akyldash/opensearch@sha256:" + "c" * 64, }, } @@ -37,7 +44,7 @@ class ReleaseManifestTests(unittest.TestCase): validate_manifest(self.manifest, self.policy) def test_rejects_tag_instead_of_digest(self): - self.manifest["images"]["backend"] = "registry.invalid/akyldash/backend:latest" + self.manifest["images"]["backend"] = "git.vakanaut.ru/admin/akyldash/backend:latest" with self.assertRaisesRegex(ValueError, "sha256"): validate_manifest(self.manifest, self.policy) @@ -51,8 +58,13 @@ class ReleaseManifestTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "commit"): validate_manifest(self.manifest, self.policy) + def test_rejects_manifest_from_another_repository(self): + self.manifest["repository"] = "admin/other" + with self.assertRaisesRegex(ValueError, "repository must be admin/akyldash"): + validate_manifest(self.manifest, self.policy) + def test_rejects_repository_outside_policy(self): - self.manifest["images"]["backend"] = "registry.invalid/other/backend@sha256:" + "b" * 64 + self.manifest["images"]["backend"] = "git.vakanaut.ru/other/backend@sha256:" + "b" * 64 with self.assertRaisesRegex(ValueError, "not allowed"): validate_manifest(self.manifest, self.policy) @@ -70,8 +82,8 @@ class ReleaseManifestTests(unittest.TestCase): manifest_path = Path(directory) / "manifest.json" policy_path = Path(directory) / "policy.json" manifest_path.write_text( - '{"schema_version":1,"source_ref":"refs/heads/main","commit":"' + "a" * 40 - + '","images":{"backend":"first","backend":"second","opensearch":"third"}}', + '{"schema_version":2,"source_ref":"refs/heads/main","repository":"admin/akyldash","commit":"' + "a" * 40 + + '","workflow":".gitea/workflows/release-images.yaml","run_id":"49","run_attempt":"1","builder":{},"images":{"backend":"first","backend":"second","opensearch":"third"}}', encoding="utf-8", ) policy_path.write_text(json.dumps(self.policy), encoding="utf-8") @@ -99,8 +111,8 @@ class ReleaseManifestTests(unittest.TestCase): @unittest.skipUnless(shutil.which("docker"), "Docker CLI is unavailable") def test_compose_override_renders_digest_images_and_preserves_build(self): - backend = "registry.invalid/akyldash/backend@sha256:" + "b" * 64 - opensearch = "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64 + backend = "git.vakanaut.ru/admin/akyldash/backend@sha256:" + "b" * 64 + opensearch = "git.vakanaut.ru/admin/akyldash/opensearch@sha256:" + "c" * 64 with tempfile.TemporaryDirectory() as directory: temp = Path(directory) secret = temp / "review_secret" @@ -126,6 +138,24 @@ class ReleaseManifestTests(unittest.TestCase): self.assertIn("build", services["backend"]) self.assertIn("build", services["opensearch"]) + def test_creates_commit_and_builder_traceability_from_kaniko_digests(self): + with tempfile.TemporaryDirectory() as directory: + backend, opensearch = Path(directory) / "backend", Path(directory) / "opensearch" + backend.write_text("sha256:" + "b" * 64, encoding="ascii") + opensearch.write_text("sha256:" + "c" * 64, encoding="ascii") + result = create("admin/akyldash", "a" * 40, "49", "1", backend, opensearch) + self.assertEqual(validate_manifest(result, self.policy), result) + self.assertEqual(result["workflow"], ".gitea/workflows/release-images.yaml") + self.assertEqual(result["builder"], BUILDER) + + def test_manifest_creation_rejects_missing_digest(self): + with tempfile.TemporaryDirectory() as directory: + backend, opensearch = Path(directory) / "backend", Path(directory) / "opensearch" + backend.write_text("", encoding="ascii") + opensearch.write_text("sha256:" + "c" * 64, encoding="ascii") + with self.assertRaisesRegex(ValueError, "invalid Kaniko digest"): + create("admin/akyldash", "a" * 40, "49", "1", backend, opensearch) + if __name__ == "__main__": unittest.main() diff --git a/infra/deploy/test_release_refs.py b/infra/deploy/test_release_refs.py new file mode 100644 index 0000000..35592d6 --- /dev/null +++ b/infra/deploy/test_release_refs.py @@ -0,0 +1,64 @@ +import tempfile +import unittest +from pathlib import Path + +from release_refs import read_images, replace_images + + +class ReleaseReferenceTests(unittest.TestCase): + def test_replaces_both_refs_as_one_expected_state_and_preserves_other_config(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "production.env" + path.write_text( + "DATA_ROOT=/srv/data\nBACKEND_IMAGE=akyldash/backend:local\n" + "OPENSEARCH_IMAGE=akyldash/opensearch:local\n", + encoding="utf-8", + ) + old = { + "BACKEND_IMAGE": "akyldash/backend:local", + "OPENSEARCH_IMAGE": "akyldash/opensearch:local", + } + new = { + "BACKEND_IMAGE": "git.vakanaut.ru/admin/akyldash/backend@sha256:" + "a" * 64, + "OPENSEARCH_IMAGE": "git.vakanaut.ru/admin/akyldash/opensearch@sha256:" + "b" * 64, + } + replace_images(path, old, new, require_root=False) + text, actual = read_images(path) + self.assertEqual(actual, new) + self.assertTrue(text.startswith("DATA_ROOT=/srv/data\n")) + + def test_stale_or_partial_reference_set_is_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "production.env" + original = "BACKEND_IMAGE=old/backend:tag\nOPENSEARCH_IMAGE=old/search:tag\n" + path.write_text(original, encoding="utf-8") + with self.assertRaisesRegex(ValueError, "changed"): + replace_images(path, {"BACKEND_IMAGE": "stale", "OPENSEARCH_IMAGE": "old/search:tag"}, { + "BACKEND_IMAGE": "new/backend:tag", "OPENSEARCH_IMAGE": "new/search:tag" + }, require_root=False) + self.assertEqual(path.read_text(encoding="utf-8"), original) + + def test_duplicate_and_missing_variables_fail_closed(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "production.env" + path.write_text("BACKEND_IMAGE=a/b:tag\nBACKEND_IMAGE=a/b:tag\nOPENSEARCH_IMAGE=c/d:tag\n") + with self.assertRaisesRegex(ValueError, "duplicate"): + read_images(path) + path.write_text("BACKEND_IMAGE=a/b:tag\n") + with self.assertRaisesRegex(ValueError, "exactly one"): + read_images(path) + + def test_indented_duplicate_assignment_is_not_ignored(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "production.env" + path.write_text( + "BACKEND_IMAGE=a/b:tag\n OPENSEARCH_IMAGE=attacker/search:tag\n" + "OPENSEARCH_IMAGE=c/d:tag\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "plain KEY=value"): + read_images(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy/test_release_workflow.py b/infra/deploy/test_release_workflow.py new file mode 100644 index 0000000..ba1ea22 --- /dev/null +++ b/infra/deploy/test_release_workflow.py @@ -0,0 +1,34 @@ +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class ReleaseWorkflowTests(unittest.TestCase): + def test_publisher_is_manual_main_only_and_requires_registry_credentials(self): + workflow = (ROOT / ".gitea/workflows/release-images.yaml").read_text(encoding="utf-8") + self.assertIn("workflow_dispatch:", workflow) + self.assertIn("if: gitea.ref == 'refs/heads/main'", workflow) + self.assertNotIn("pull_request:", workflow) + self.assertIn("secrets.REGISTRY_USERNAME", workflow) + self.assertIn("secrets.REGISTRY_PASSWORD", workflow) + self.assertIn("Print verified release descriptor for the owner", workflow) + self.assertIn("sha256sum -c release/release-manifest.json.sha256", workflow) + self.assertIn("cat release/release-manifest.json", workflow) + self.assertNotIn("upload-artifact", workflow) + self.assertNotIn("docker compose", workflow) + + def test_pr_build_remains_no_push_and_push_action_has_no_engine_socket(self): + build_action = (ROOT / ".gitea/actions/kaniko-build/action.yml").read_text(encoding="utf-8") + push_action = (ROOT / ".gitea/actions/kaniko-push/release-build.sh").read_text(encoding="utf-8") + self.assertIn("--no-push", build_action) + self.assertIn("--no-push-cache", build_action) + self.assertIn("REGISTRY_USERNAME:?", push_action) + self.assertIn("REGISTRY_PASSWORD:?", push_action) + self.assertNotIn("docker.sock", push_action) + self.assertNotIn("DOCKER_HOST", push_action) + self.assertIn("--digest-file=", push_action) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy/test_rollback.py b/infra/deploy/test_rollback.py index 9cb05a5..4652424 100644 --- a/infra/deploy/test_rollback.py +++ b/infra/deploy/test_rollback.py @@ -1,11 +1,13 @@ import json import subprocess import sys +import tempfile import unittest from types import SimpleNamespace from unittest.mock import patch import rollback +import deployment_lock from rollback import image_restore_plan, require_tag_rollback, tag_previous_images, verify_previous_images @@ -31,6 +33,20 @@ class RollbackPlanTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "reference changed"): image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"}) + def test_release_state_allows_guarded_digest_to_legacy_and_legacy_to_digest_restore(self): + previous = self.manifest["images"].copy() + self.configured["backend"] = "registry/backend@sha256:" + "d" * 64 + plan = image_restore_plan(self.manifest, self.configured, previous) + self.assertEqual(plan["backend"][1], "akyldash/backend:local") + old_digest = "registry/backend@sha256:" + "c" * 64 + self.manifest["images"]["backend"] = { + "configured": old_digest, "image_id": "sha256:" + "a" * 64 + } + previous["backend"] = self.manifest["images"]["backend"] + self.configured["backend"] = "akyldash/backend:local" + plan = image_restore_plan(self.manifest, self.configured, previous) + self.assertEqual(plan["backend"][1], old_digest) + def test_refuses_digest_reference_that_cannot_be_retagged(self): self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64 self.configured["backend"] = self.manifest["images"]["backend"]["configured"] @@ -68,12 +84,11 @@ class RollbackPlanTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "previous image is unavailable"): verify_previous_images(plan) - def test_digest_reference_is_never_tagged(self): + def test_digest_reference_is_restored_without_retagging(self): digest = "registry/backend@sha256:" + "c" * 64 plan = {"backend": ("sha256:" + "a" * 64, digest)} with patch("rollback.run") as run_mock: - with self.assertRaisesRegex(ValueError, "not supported safely"): - tag_previous_images(plan) + tag_previous_images(plan) run_mock.assert_not_called() def test_digest_rollback_fails_before_stopping_or_changing_references(self): @@ -101,7 +116,9 @@ class RollbackPlanTests(unittest.TestCase): verify_archive=lambda _backup: manifest, ) backup = rollback.BACKUP_DIR / "synthetic.tar.gz" + lock_dir = tempfile.TemporaryDirectory() with ( + patch.object(deployment_lock, "LOCK_FILE", rollback.Path(lock_dir.name) / "deploy.lock"), patch("rollback.Path.is_file", return_value=True), patch("rollback.Path.is_symlink", return_value=False), patch("rollback.Path.stat", return_value=SimpleNamespace(st_uid=0, st_mode=0o100600)), @@ -115,6 +132,56 @@ class RollbackPlanTests(unittest.TestCase): self.assertEqual(run_mock.call_count, 3) self.assertTrue(all(call.args[0][:3] == ["docker", "image", "inspect"] for call in run_mock.call_args_list)) + lock_dir.cleanup() + + def test_mismatched_release_backup_is_rejected_before_stopping_services(self): + current = { + "backend": {"configured": "registry/backend@sha256:" + "d" * 64, "image_id": "sha256:" + "e" * 64}, + "opensearch": {"configured": "registry/search@sha256:" + "f" * 64, "image_id": "sha256:" + "1" * 64}, + } + previous = { + "backend": {"configured": "registry/backend@sha256:" + "a" * 64, "image_id": "sha256:" + "2" * 64}, + "opensearch": {"configured": "registry/search@sha256:" + "b" * 64, "image_id": "sha256:" + "3" * 64}, + } + manifest = { + "images": { + "backend": {"configured": "registry/backend@sha256:" + "c" * 64, "image_id": previous["backend"]["image_id"]}, + "opensearch": previous["opensearch"], + }, + "deployment_files": {"deploy/production/compose.yaml": "compose-hash", "deploy/production/Caddyfile": "caddy-hash"}, + } + state = {"schema_version": 1, "commit": "a" * 40, "manifest_sha256": "b" * 64, + "current": current, "previous": previous} + rendered = {"services": {"backend": {"image": current["backend"]["configured"]}, + "opensearch": {"image": current["opensearch"]["configured"]}}} + fake_archive = SimpleNamespace( + digest=lambda path: "compose-hash" if path.name == "compose.yaml" else "caddy-hash", + verify_archive=lambda _backup: manifest, + ) + lock_dir = tempfile.TemporaryDirectory() + backup = rollback.BACKUP_DIR / "synthetic.tar.gz" + with ( + patch.object(deployment_lock, "LOCK_FILE", rollback.Path(lock_dir.name) / "deploy.lock"), + patch("rollback.Path.exists", autospec=True, + side_effect=lambda path: path == rollback.STATE_FILE), + patch("rollback.Path.is_file", return_value=True), + patch("rollback.Path.is_symlink", return_value=False), + patch("rollback.Path.stat", return_value=SimpleNamespace(st_uid=0, st_mode=0o100600)), + patch("rollback.compose", side_effect=lambda *args, **_kwargs: ( + json.dumps(rendered) if args[:2] == ("config", "--format") + else "backend-container" if args[-1] == "backend" + else "search-container" + )) as compose_mock, + patch("rollback.run", side_effect=lambda command, **_kwargs: "true\t" + current[ + "backend" if command[-1] == "backend-container" else "opensearch" + ]["image_id"]), + patch("release_state.read_json", return_value=state), + patch.dict(sys.modules, {"archive": fake_archive}), + ): + with self.assertRaisesRegex(ValueError, "backup does not match the pinned previous backend"): + rollback.rollback(backup, data_compatible=True) + self.assertFalse(any(call.args and call.args[0] == "stop" for call in compose_mock.call_args_list)) + lock_dir.cleanup() def test_tag_rollback_keeps_existing_image_tag_operation(self): plan = image_restore_plan(self.manifest, self.configured) diff --git a/infra/deploy/validate_release.py b/infra/deploy/validate_release.py index b3eccdf..f132a0e 100644 --- a/infra/deploy/validate_release.py +++ b/infra/deploy/validate_release.py @@ -10,6 +10,7 @@ import sys from pathlib import Path SERVICES = {"backend", "opensearch"} +REPOSITORY = "admin/akyldash" COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z") REPOSITORY_RE = re.compile( r"[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?" @@ -58,16 +59,27 @@ def validate_policy(policy: object) -> dict[str, list[str]]: def validate_manifest(manifest: object, policy: object) -> dict[str, object]: allowed_repositories = validate_policy(policy) if not isinstance(manifest, dict) or set(manifest) != { - "schema_version", "source_ref", "commit", "images" + "schema_version", "source_ref", "repository", "commit", "workflow", "run_id", "run_attempt", "builder", "images" }: - raise ValueError("manifest must contain exactly schema_version, source_ref, commit, and images") - if type(manifest["schema_version"]) is not int or manifest["schema_version"] != 1: + raise ValueError("manifest must contain exactly the release source, builder, run, and image fields") + if type(manifest["schema_version"]) is not int or manifest["schema_version"] != 2: raise ValueError("unsupported manifest schema_version") if manifest["source_ref"] != "refs/heads/main": raise ValueError("manifest source_ref must be refs/heads/main") commit = manifest["commit"] if not isinstance(commit, str) or not COMMIT_RE.fullmatch(commit): raise ValueError("manifest commit must be a 40-character lowercase Git SHA-1") + if manifest["repository"] != REPOSITORY: + raise ValueError(f"manifest repository must be {REPOSITORY}") + if manifest["workflow"] != ".gitea/workflows/release-images.yaml": + raise ValueError("manifest workflow is not the approved image release workflow") + if not isinstance(manifest["run_id"], str) or not re.fullmatch(r"[0-9]+", manifest["run_id"]): + raise ValueError("manifest run_id must be a numeric Gitea Actions run ID") + if not isinstance(manifest["run_attempt"], str) or not re.fullmatch(r"[1-9][0-9]*", manifest["run_attempt"]): + raise ValueError("manifest run_attempt must be a positive integer") + builder = manifest["builder"] + if builder != {"name": "Kaniko", "image": "ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412"}: + raise ValueError("manifest builder is not the pinned release builder") images = manifest["images"] if not isinstance(images, dict) or set(images) != SERVICES: raise ValueError("manifest images must contain exactly backend and opensearch") -- 2.54.0 From 9d8ce6b67fadefa1f10bda1eb05c7a285a0b0df6 Mon Sep 17 00:00:00 2001 From: agent Date: Sun, 25 Oct 2026 13:44:56 +0300 Subject: [PATCH 02/23] fix: address production release review findings --- .gitea/workflows/ci.yaml | 10 + .gitea/workflows/release-images.yaml | 15 +- backend/README.md | 5 + backend/search/minjust_opensearch.py | 46 ++-- backend/search/review.html | 10 +- backend/test_minjust_opensearch.py | 92 +++++++- deploy/production/README.md | 27 ++- docs/operations/CI_RUNNER.md | 16 +- docs/operations/CURRENT_STATE.md | 9 + docs/operations/DEPLOYMENT_OWNER_PACKET.md | 9 +- docs/operations/ROLLBACK.md | 9 +- infra/deploy/deploy_release.py | 49 +++-- infra/deploy/rollback.py | 52 +++-- infra/deploy/test_deploy_release.py | 34 ++- infra/deploy/test_release_workflow.py | 15 +- infra/deploy/test_rollback.py | 235 +++++++++++++++++---- 16 files changed, 505 insertions(+), 128 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 8232d13..1a02362 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -81,3 +81,13 @@ jobs: workspace: ${{ github.workspace }} dockerfile: deploy/production/Dockerfile.opensearch destination: akyldash-opensearch:ci + - name: Prepare a synthetic artifact compatibility marker + run: | + mkdir -p artifact-smoke + printf 'Gitea artifact upload smoke test; no release data.\n' > artifact-smoke/marker.txt + - name: Verify Gitea runtime artifact upload + uses: https://gitea.com/actions/gitea-upload-artifact@055b8b3f04a4a7ed853f2c6ab04256f83e4874dc + with: + name: artifact-smoke-${{ github.run_id }}-${{ github.run_attempt }} + path: artifact-smoke/marker.txt + retention-days: 1 diff --git a/.gitea/workflows/release-images.yaml b/.gitea/workflows/release-images.yaml index 28d04ad..442a8d8 100644 --- a/.gitea/workflows/release-images.yaml +++ b/.gitea/workflows/release-images.yaml @@ -49,10 +49,15 @@ jobs: python3 infra/deploy/validate_release.py release/release-manifest.json \ --policy infra/deploy/release-policy.json sha256sum release/release-manifest.json > release/release-manifest.json.sha256 - - name: Print verified release descriptor for the owner + - name: Upload verified release descriptor + uses: https://gitea.com/actions/gitea-upload-artifact@055b8b3f04a4a7ed853f2c6ab04256f83e4874dc + with: + name: release-descriptor-${{ github.run_id }}-${{ github.run_attempt }} + path: | + release/release-manifest.json + release/release-manifest.json.sha256 + retention-days: 90 + - name: Confirm uploaded release descriptor run: | sha256sum -c release/release-manifest.json.sha256 - echo 'Release descriptor (copy from this authenticated run log):' - cat release/release-manifest.json - echo 'Descriptor SHA-256:' - cat release/release-manifest.json.sha256 + echo 'Release descriptor and checksum are available as this run artifact.' diff --git a/backend/README.md b/backend/README.md index 6533960..d7e7e88 100644 --- a/backend/README.md +++ b/backend/README.md @@ -190,6 +190,11 @@ Wildcard removal, пустой индекс и alias с несколькими t старый target успел измениться после проверки, OpenSearch отклонит атомарную операцию удаления точного старого alias. `--resume` повторно проверяет весь ожидаемый inventory, а не только количество документов текущего запуска. +Если ответ на alias request потерян после успешного переключения или процесс +остановился до записи завершённого checkpoint, повторный запуск проверит +фактическую цель alias. Уже опубликованный candidate оставляется на месте, и +checkpoint завершается; старый target позволяет безопасно повторить cutover, +а любой другой target останавливает операцию. После каждого принятого Bulk-пакета загрузчик атомарно сохраняет checkpoint и печатает код безопасного возобновления. При временных HTTP 429/5xx, timeout и diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index b977ab8..0e67b8d 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -462,22 +462,44 @@ def switch_alias(base: str, index: str, alias: str, expected_old_index: str) -> if not expected_old_index or expected_old_index == index: raise ValueError("--expected-old-index must name the exact previous index") actual_old_index = alias_target(base, alias) + if actual_old_index == index: + return if actual_old_index != expected_old_index: raise ValueError( f"alias {alias} currently targets {actual_old_index}, expected {expected_old_index}; refusing cutover" ) - result = request_json( - f"{base.rstrip('/')}/_aliases", - "POST", - json.dumps({ - "actions": [ - {"remove": {"index": expected_old_index, "alias": alias, "must_exist": True}}, - {"add": {"index": index, "alias": alias}}, - ] - }).encode(), - "application/json", - ) - if result.get("acknowledged") is not True: + try: + result = request_json( + f"{base.rstrip('/')}/_aliases", + "POST", + json.dumps({ + "actions": [ + {"remove": {"index": expected_old_index, "alias": alias, "must_exist": True}}, + {"add": {"index": index, "alias": alias}}, + ] + }).encode(), + "application/json", + ) + except Exception as error: + try: + current = alias_target(base, alias) + except Exception: + raise RuntimeError(f"alias switch outcome is unknown; inspect {alias} before resuming") from error + if current == index: + return + if current != expected_old_index: + raise ValueError( + f"alias {alias} now targets unexpected index {current}; refusing to retry publication" + ) from error + raise RuntimeError(f"alias switch failed; {alias} still targets {expected_old_index}") from error + if not isinstance(result, dict) or result.get("acknowledged") is not True: + current = alias_target(base, alias) + if current == index: + return + if current != expected_old_index: + raise ValueError( + f"alias {alias} now targets unexpected index {current}; refusing to retry publication" + ) raise RuntimeError(f"OpenSearch did not acknowledge alias switch: {alias}") diff --git a/backend/search/review.html b/backend/search/review.html index 4caffca..d38c2c4 100644 --- a/backend/search/review.html +++ b/backend/search/review.html @@ -657,7 +657,7 @@

Документ

Выберите документ в выдаче, чтобы открыть текст.

Содержание
-
Акылдаш Правовая база КыргызстанаFrontend v0.3.3 Backend v0.13.2
+
Акылдаш Правовая база КыргызстанаFrontend v0.3.3 Backend v0.13.3

Черновик появится после поиска.

Вход в Акылдаш

Введите учетные данные, выданные администратором.

Создайте новый пароль

Для продолжения замените временный пароль. Используйте не менее 12 символов.

@@ -675,7 +675,7 @@

Документ

Источники документов

Поиск выполняется по документам, подготовленным из правовых источников Кыргызской Республики. Для каждого результата доступная редакция и ссылка на официальный источник показываются в панели документа.

Официальный источник: Централизованный банк данных правовой информации Министерства юстиции Кыргызской Республики.

Короткое знакомство

-
Акылдаш · Frontend v0.3.3 · Backend v0.13.2 · Внутренняя лаборатория релевантности
+
Акылдаш · Frontend v0.3.3 · Backend v0.13.3 · Внутренняя лаборатория релевантности