2026-10-11 20:22:23 +00:00 · 2026-10-11 19:45:56 +00:00 · 2026-10-11 19:56:01 +00:00 · 2026-10-11 20:14:50 +00:00 · 2026-10-11 19:45:56 +00:00 · 2026-10-11 19:56:01 +00:00
40 changed files with 2605 additions and 76 deletions
@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  tests-and-images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run backend tests
+        run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
+      - name: Run Telegram bot tests
+        run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
+      - name: Run infrastructure backup tests
+        run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
+      - name: Check recovery host guard
+        run: bash infra/bootstrap/test-host-guard.sh
+      - name: Check verifier helper argument guard
+        run: bash infra/bootstrap/test-verify-helper.sh
+      - name: Check infrastructure shell syntax
+        run: |
+          bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
+            infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/deploy/install-helper.sh \
+            infra/bootstrap/test-verify-helper.sh
+      - name: Run rollback helper tests
+        run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
+      - name: Validate production Compose and Caddyfile
+        run: |
+          secret_file=$(mktemp)
+          trap 'rm -f "$secret_file"' EXIT
+          REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet
+          docker run --rm --entrypoint caddy -v "$PWD/deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028 validate --config /etc/caddy/Caddyfile --adapter caddyfile
+      - name: Build production images
+        run: |
+          docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci .
+          docker build -f deploy/production/Dockerfile.opensearch -t akyldash-opensearch:ci .
@@ -0,0 +1,35 @@
+{
+  "detector": {
+    "ignoreRules": [],
+    "ignoreFiles": [],
+    "ignoreValues": [
+      {
+        "rule": "cream-palette",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:40:02.635Z",
+        "reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette"
+      },
+      {
+        "rule": "cramped-padding",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:28.434Z",
+        "reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector"
+      },
+      {
+        "rule": "design-system-font",
+        "value": "sans",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:33.020Z",
+        "reason": "Agent: DESIGN.md specifies Noto Sans with sans-serif fallback; this finding is the inherited generic fallback"
+      }
+    ]
+  }
+}
@@ -0,0 +1,29 @@
+# Акылдаш — инструкции для агента
+
+Проект: юридическая информационно-аналитическая платформа; Search API на
+Python, SQLite и OpenSearch. Production пока не развернут.
+
+- Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и
+  [правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно.
+- Архитектура и процедуры: [инфраструктура](docs/operations/INFRASTRUCTURE.md),
+  [деплой](docs/operations/DEPLOYMENT.md), [backup](docs/operations/BACKUP.md),
+  [rollback](docs/operations/ROLLBACK.md), [восстановление](docs/operations/RECOVERY.md).
+- Работай в feature/fix/refactor-ветке. `main` защищена; изменения вноси через
+  PR, не сливай его автоматически. Сохраняй незакоммиченные изменения.
+- SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай
+  `codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
+  Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
+- Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
+  `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`,
+  `bash infra/bootstrap/test-host-guard.sh`.
+  Production images собираются командами из `.gitea/workflows/ci.yaml`.
+- Перед изменениями production необходим проверенный backup. Формального
+  механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
+  SSH, firewall или production вручную без владельца и последующего PR.
+- Холодный backup требует согласованного окна простоя; восстановление допускается
+  только в новый staging-каталог, пока владелец отдельно не проверил процедуру
+  для нового хоста.
+- После каждого изменения инфраструктуры обновляй
+  `docs/operations/CURRENT_STATE.md` по фактическим проверкам.
@@ -2,6 +2,7 @@

 ## Не выпущено

+- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, Caddy HTTPS proxy с разделёнными Compose-сетями и CI-проверка конфигурации; публикация образов и production-деплой остаются выключены до проверки runner, registry, backup/restore и ограниченного deploy-helper.
 - Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
 - Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
 - Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
@@ -10,7 +10,7 @@ Telegram-бот — только часть рабочего окружения
 ## Текущее состояние

 Сейчас реализованы Telegram-бот-секретарь версии `0.2.2`, backend версии
-`0.11.2` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
+`0.11.3` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
 SQLite; первый пользователь `admin` создаётся интерактивной CLI-командой.
 Поиск расширяет аббревиатуры организационно-правовых форм и поднимает
 действующий Налоговый кодекс для запросов о налоге на прибыль.
@@ -18,7 +18,7 @@ SQLite; первый пользователь `admin` создаётся инт
 | Компонент | Версия | Состояние |
 |---|---:|---|
 | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
-| Backend | `0.11.2` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
+| Backend | `0.11.3` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
 | Frontend | `0.1.2` | лаборатория поиска, личный кабинет и управление учетными записями |
 | Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи |
 | RAG и база знаний | — | ещё не созданы |
@@ -62,4 +62,4 @@ python3 -m unittest discover -s tools/telegram-bot -v

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
@@ -1,6 +1,6 @@
 # Backend Акылдаш

-Версия: `0.11.2`
+Версия: `0.11.3`

 Первая backend-область проекта — загрузка правовых документов из официального
 Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
@@ -253,4 +253,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la

 ---

-Акылдаш · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Backend v0.11.3 · Frontend v0.1.2
@@ -451,6 +451,7 @@ def main() -> int:
    parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
    parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
    parser.add_argument("--review-secret", default=None)
+    parser.add_argument("--review-secret-file", type=Path)
    parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
    parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
    parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
@@ -458,7 +459,14 @@ def main() -> int:
    parser.add_argument("--port", type=int, default=8080)
    parser.add_argument("--version", action="version", version=APP_VERSION)
    arguments = parser.parse_args()
-    secret = arguments.review_secret.encode() if arguments.review_secret else None
+    if arguments.review_secret and arguments.review_secret_file:
+        parser.error("choose either --review-secret or --review-secret-file")
+    try:
+        secret = arguments.review_secret.encode() if arguments.review_secret else arguments.review_secret_file.read_bytes().strip() if arguments.review_secret_file else None
+    except OSError as error:
+        parser.error(f"cannot read review secret file: {error}")
+    if arguments.review_secret_file and not secret:
+        parser.error("review secret file is empty")
    api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
    if arguments.create_admin or arguments.reset_admin_password:
        users = api.auth_store.list_users()
@@ -17,7 +17,7 @@ from typing import Iterator

 from search.catalog import authority_codes, legal_force_code, source_code

-APP_VERSION = "0.11.2"
+APP_VERSION = "0.11.3"
 LANGUAGES = {"ru", "ky"}
 DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")

@@ -5,7 +5,7 @@
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Оценка поисковой выдачи · Акылдаш</title>
  <style>
-    :root { color-scheme: light; font: 16px/1.5 "Noto Sans", "Segoe UI", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", Georgia, serif; }
+    :root { color-scheme: light; font: 16px/1.5 "Noto Sans", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", serif; }
    * { box-sizing: border-box; }
    body { margin: 0; color: var(--ink); background: #F7F2E8; }
    .review__skip { position: absolute; inset-block-start: 0; inset-inline-start: -10000px; padding: .5rem 1rem; background: var(--surface); }
@@ -53,7 +53,7 @@
    .review__comment { min-height: 4rem; resize: vertical; background: var(--surface); }
    .review__document { position: sticky; inset-block-start: 1rem; min-height: 24rem; }
    .review__document-meta { margin-block-end: 1rem; padding-block-end: 1rem; border-block-end: 1px solid var(--line); color: var(--muted); }
-    .review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
+    .review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
    .review__document-body img { max-width: 100%; height: auto; }
    .review__actions { max-width: 1320px; margin: 1.5rem auto 0; padding-inline: clamp(1rem, 3vw, 3rem); }
    .review__actions-inner { display: flex; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
@@ -115,7 +115,7 @@
    .search__document { min-height: 60vh; }
    .search__document-controls { display: flex; flex-wrap: wrap; gap: .75rem; }
    .search__document-controls label { display: grid; gap: .25rem; min-width: 11rem; color: var(--muted); font-size: .875rem; }
-    .search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
+    .search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
    .search__document-body h1, .search__document-body h2, .search__document-body h3 { font-family: var(--serif); }
    .search__document-body img { max-width: 100%; height: auto; }
    .search__sr-only { position: absolute; inline-size: 1px; block-size: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
@@ -176,7 +176,7 @@
  <dialog id="users-dialog" aria-labelledby="users-heading"><section class="review__page"><h2 id="users-heading">Управление пользователями</h2><form id="create-user-form"><label>Логин<input id="new-username" autocomplete="off" minlength="3" maxlength="64" required></label><label>Временный пароль<input id="temporary-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p>Пароль — от 12 символов. Новый пользователь сменит его при первом входе. Передайте временный пароль отдельно.</p><p id="users-error" role="alert" class="review__status review__status--error"></p><button class="review__button--primary" type="submit">Создать пользователя</button></form><ul id="user-list" class="review__user-list"></ul><button type="button" data-close="users-dialog">Закрыть</button></section></dialog>
  <dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
  <div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
-  <footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.2 · Внутренняя лаборатория релевантности</div></footer>
+  <footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.3 · Внутренняя лаборатория релевантности</div></footer>
  <script>
    document.body.classList.add('review__locked');
    const DRAFT_KEY = 'akyldash-search-review-draft';
@@ -193,7 +193,7 @@
    let account = null;
    const reviewRoute = location.pathname === '/review';
    $('review').hidden = !reviewRoute; $('search-page').hidden = reviewRoute; $('review-nav').classList.toggle('is-active', reviewRoute); $('search-nav').classList.toggle('is-active', !reviewRoute);
-    if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.2'; }
+    if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.3'; }
    async function apiFetch(path, options = {}) { const response = await fetch(path, {credentials: 'same-origin', ...options}); if (response.status === 401 && path !== '/auth/login') { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); } return response; }
    const showLogin = () => { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); };
    const enterWorkspace = (user) => { account = user; $('signed-in-user').textContent = `${user.username} · ${user.role}`; $('reviewer').value = user.username; $('profile-username').textContent = user.username; $('profile-role').textContent = user.role; $('profile-created').textContent = new Date(user.created_at * 1000).toLocaleDateString('ru-RU'); $('users-open').hidden = user.username.toLowerCase() !== 'admin'; document.body.classList.remove('review__locked'); if (!reviewRoute) initSearchPage(); else if (!hasSeenTour()) setTimeout(openTutorial, 350); };
@@ -1,4 +1,6 @@
 import json
+import contextlib
+import io
 import tempfile
 import unittest
 from pathlib import Path
@@ -36,6 +38,27 @@ class SearchApiTest(unittest.TestCase):
            with self.assertRaisesRegex(ApiError, "within 10000 results"):
                api.handle("GET", "/search?q=x&page=100&page_size=100")

+    def test_review_secret_file_is_read_without_command_line_value(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.write_text("secret-value\n", encoding="utf-8")
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), patch("search.api.Api") as api, patch("search.api.ThreadingHTTPServer"):
+                main()
+            self.assertEqual(api.call_args.args[4], b"secret-value")
+
+    def test_review_secret_file_rejects_empty_content(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.touch()
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), contextlib.redirect_stderr(io.StringIO()):
+                with self.assertRaises(SystemExit) as result:
+                    main()
+            self.assertEqual(result.exception.code, 2)
+
    def test_production_api_uses_registration_intent_boosts(self):
        with tempfile.TemporaryDirectory() as temporary:
            api = self.make_api(Path(temporary))
@@ -1,9 +1,9 @@
 # Absolute path on the production host containing minjust-normalized/.
-DATA_ROOT=/volume1/docker/akyldash/data
-BACKEND_PORT=8080
+DATA_ROOT=/var/lib/akyldash/data
+REVIEW_SECRET_FILE=/etc/akyldash/review_secret
+BACKEND_IMAGE=akyldash/backend:local
+OPENSEARCH_IMAGE=akyldash/opensearch:local
 SEARCH_INDEX=akyldash-fragments-current
 OPENSEARCH_MEM_LIMIT=4g
 OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
-# Generate with: openssl rand -hex 32
-REVIEW_SECRET=replace-with-a-random-secret
 COOKIE_SECURE=true
@@ -0,0 +1,4 @@
+akyldash.kg {
+	encode zstd gzip
+	reverse_proxy backend:8080
+}
@@ -1,62 +1,104 @@
 # Production deployment

-This compose project runs the Search API and its private OpenSearch node. It
-binds the API only to `127.0.0.1`; publish it through an authenticated reverse
-proxy or VPN. OpenSearch is not published outside the compose network.
+Compose runs Caddy, the Search API, and a private OpenSearch node. Only Caddy
+publishes ports `80` and `443`; backend and OpenSearch have no host ports. The
+OpenSearch security plugin is disabled to match the current API client, so keep
+it on the private network.

-The current compose intentionally disables the OpenSearch security plugin to
-match the existing API client. Keep both services on a private host/network.
-The application provides cookie-based accounts for its search API and review
-workspace; it does not replace host-level network controls or TLS.
+## Host access and configuration

-## First deployment
-
-1. Copy this directory to the host with the repository source.
-2. Copy `.env.example` to `.env`, set an absolute `DATA_ROOT`, and replace
-   `REVIEW_SECRET` with a random value. Do not commit `.env`.
-3. Put the normalized dataset under `$DATA_ROOT/minjust-normalized/`.
-4. Check the rendered configuration:
-
-   ```bash
-   docker compose --env-file .env -f compose.yaml config
-   ```
-
-5. Start the services:
-
-   ```bash
-   docker compose --env-file .env -f compose.yaml up -d --build
-   docker compose --env-file .env -f compose.yaml ps
-   curl -fsS http://127.0.0.1:${BACKEND_PORT:-8080}/review >/dev/null
-   ```
-
-6. Load the versioned index and switch its alias only after the import and
-   validation succeed. Back up `DATA_ROOT` and the `opensearch-data` volume
-   before the first import.
-
-## Initial administrator and accounts
-
-After starting the backend for the first time, create the initial `admin`
-account with an interactive password prompt:
+The production environment file is `/etc/akyldash/production.env`; the review
+secret is `/etc/akyldash/review_secret`. Both must stay outside the checkout,
+root-owned, and mode `0600`. `DATA_ROOT` must be an absolute path. The `codex`
+account is intentionally outside the `sudo` and `docker` groups because access
+to the Docker socket grants root-equivalent control. Run host operations with
+the narrowly required `sudo` command; do not use `docker` as `codex` directly.

 ```bash
-docker compose --env-file .env -f compose.yaml exec backend python -m search.api --create-admin
+sudo install -d -o root -g root -m 0700 /etc/akyldash
+sudo install -o root -g root -m 0600 deploy/production/.env.example /etc/akyldash/production.env
+sudo sh -c 'umask 077; openssl rand -hex 32 > /etc/akyldash/review_secret'
+sudoedit /etc/akyldash/production.env
 ```

-The account database is stored at `DATA_ROOT/search-users.sqlite3`. Back it up
-alongside the review database. Sign in at `/review`; `admin` can create accounts
-with temporary passwords. Each new account must replace its temporary password
-at first sign-in. Only the `admin` login can create, reset, or delete accounts.
+Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus
+is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in
+[`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the
+root bootstrap installs packages and creates directories, but does not deploy
+the application or alter SSH, firewall, user groups, or sudoers.

-If the `admin` password is lost, reset it in an interactive terminal with:
+## First deployment and upgrades
+
+Run commands from the root-owned checkout at `/opt/akyldash/current`. Render
+and review Compose before changing services:

 ```bash
-docker compose --env-file .env -f compose.yaml exec backend python -m search.api --reset-admin-password
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml config --quiet
 ```

-The next sign-in requires another password change. Serve the backend behind
-HTTPS, rate-limit login requests at the reverse proxy, and keep
-`COOKIE_SECURE=true` in production.
+On a clean first deployment, `infra/backup/archive.py create` reports that no
+backup is required when all persistent mounts are empty and no Compose
+containers exist. Continue without a backup in that case. If any persistent
+data exists, create and verify a backup before changing schema, images, or
+starting a new application version. This also applies when containers are
+absent, including an initial installation where the normalized corpus was
+preloaded under `DATA_ROOT`; create the archive before the first `up`. When
+named volumes do not exist, pass `--initial-deployment`; this is accepted only
+without Compose containers and when `DATA_ROOT` contains only
+`minjust-normalized/`. During upgrades, missing volumes are an error because
+they can indicate lost production data. The archive tool also refuses
+ambiguous storage state, running containers, and incomplete Compose service
+containers. See
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
+commands. Do not remove an older usable backup to make room; retain the
+pre-change archive until the new deployment and a newer verified backup are
+confirmed.

-This is a deployment baseline, not a public internet exposure recipe. TLS,
-backups, monitoring, and a production OpenSearch security configuration must
-be provided by the host reverse proxy/operations setup.
+For an initial clean installation, start the stack only after config review:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d --build
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml ps
+curl -fsS https://akyldash.kg/review >/dev/null
+```
+
+Before this step, confirm DNS and have the owner review firewall rules for TCP
+`80` and `443`. Caddy manages the origin certificate; confirm HTTPS before
+setting Cloudflare to Full (strict). On the initial start, create the `admin`
+account interactively:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --create-admin
+```
+
+The account database is `$DATA_ROOT/search-users.sqlite3` and is included in
+the `data` backup. Admin password reset is also interactive:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
+```
+
+After the initial OpenSearch index is loaded and validated, the `admin` account
+is created, and `/review` works over HTTPS, create and verify the required
+post-setup baseline backup before treating deployment as complete. It captures
+the initialized index, account database, and Caddy certificate data. Follow
+the exact stop/create/verify/start sequence in
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md#initial-post-setup-baseline).
+
+## Backup, restore, and rollback
+
+Backups are cold and require downtime. Stop services, create the archive, then
+verify both its checksum and contents before proceeding. Do not run an upgrade
+or schema change until verification succeeds. Keep archives root-only under
+`/var/backups/akyldash` and copy them to separately managed encrypted storage.
+
+Restore is a staged operation on a marked replacement host; it does not
+overwrite production. For application-only rollback, use the guarded helper
+with a verified pre-release backup and explicit data-compatibility review.
+Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md),
+[`ROLLBACK.md`](../../docs/operations/ROLLBACK.md), and
+[`RECOVERY.md`](../../docs/operations/RECOVERY.md).
+
+Automated production deployment is not enabled. Deployments remain owner-run;
+record the release and verify service health before considering it complete.
@@ -1,9 +1,43 @@
+name: akyldash
+
 services:
+  caddy:
+    image: caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
+    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caddy-data:/data
+      - caddy-config:/config
+    networks:
+      - proxy
+    depends_on:
+      backend:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
  opensearch:
+    image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local}
    build:
      context: ../..
      dockerfile: deploy/production/Dockerfile.opensearch
    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
    environment:
      discovery.type: single-node
      bootstrap.memory_lock: "true"
@@ -21,6 +55,8 @@ services:
        hard: 65536
    volumes:
      - opensearch-data:/usr/share/opensearch/data
+    networks:
+      - search
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9200/_cluster/health || exit 1"]
      interval: 30s
@@ -28,12 +64,17 @@ services:
      retries: 10

  backend:
+    image: ${BACKEND_IMAGE:-akyldash/backend:local}
    build:
      context: ../..
      dockerfile: deploy/production/Dockerfile.backend
    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
    environment:
-      REVIEW_SECRET: ${REVIEW_SECRET:?set REVIEW_SECRET in .env}
      COOKIE_SECURE: ${COOKIE_SECURE:-true}
    command:
      - python
@@ -53,15 +94,31 @@ services:
      - /app/data/search-reviews.sqlite3
      - --users-db
      - /app/data/search-users.sqlite3
-      - --review-secret
-      - ${REVIEW_SECRET}
-    ports:
-      - "127.0.0.1:${BACKEND_PORT:-8080}:8080"
+      - --review-secret-file
+      - /run/secrets/review_secret
+    secrets:
+      - review_secret
+    expose:
+      - "8080"
    depends_on:
      opensearch:
        condition: service_healthy
    volumes:
      - ${DATA_ROOT:?set DATA_ROOT in .env}:/app/data
+    networks:
+      - proxy
+      - search

 volumes:
+  caddy-data:
+  caddy-config:
  opensearch-data:
+
+networks:
+  proxy:
+  search:
+    internal: true
+
+secrets:
+  review_secret:
+    file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}
@@ -26,6 +26,10 @@

 - [Текущий статус](operations/project-status.md) — выполненные шаги, риски и
  ближайшие действия.
+- [Состояние инфраструктуры](operations/CURRENT_STATE.md) — подтверждённое
+  состояние production и оставшиеся проверки.
+- [Инфраструктура и эксплуатация](operations/INFRASTRUCTURE.md) — схема,
+  доступы, deployment, backup, rollback и recovery.
 - [План Telegram-пространства](operations/telegram-workspace-plan.md) — темы,
  роли и этапы развития командного окружения.

@@ -41,4 +45,4 @@

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
@@ -0,0 +1,65 @@
+# Infrastructure access
+
+## Discover existing access first
+
+In a new Codex session, inspect the SSH alias and key file metadata without
+printing key contents, then check the connection:
+
+```bash
+ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }'
+test -r ~/.ssh/akyldash_codex && echo 'SSH key is available'
+ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps 'id; hostname'
+gitea-api GET /version
+gitea-api GET /repos/admin/akyldash
+```
+
+Do not ask for the host, login, password, token, or key again before these
+checks. Never print a private key, token, or complete environment file.
+
+## VPS
+
+- SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`.
+- SSH configuration: `~/.ssh/config`.
+- Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents).
+- `codex` can run unprivileged read-only diagnostics. The audited account has no
+  non-interactive sudo and must not be added to `sudo` or the `docker` group.
+- Host package installation, service setup, firewall changes, secret creation,
+  backup creation, recovery apply, and deployment require owner approval and a
+  reviewed root command. After the owner installs the root-owned repository
+  checkout, `infra/deploy/install-helper.sh` can grant a fixed read-only
+  verifier and guarded rollback command through sudoers; neither helper is
+  installed on the audited VPS.
+- Never connect to other VPS hosts or Synology as part of this deployment.
+
+This execution environment is WSL2. The repository is on the Windows-mounted
+`/mnt/d` filesystem, while SSH config and the identity file are in the Linux
+home directory. Do not assume Windows SSH credentials are available or copy
+keys between environments. Plain `ssh akyldash_vps` failed because the system
+SSH config include had unsafe permissions. Use `ssh -F ~/.ssh/config ...`; in a
+network-restricted session this read-only command may also require approved
+execution. If the key is missing in a future environment, report that
+limitation and do not retrieve it elsewhere.
+
+## Gitea
+
+- Repository: `https://git.vakanaut.ru/admin/akyldash`.
+- Use the installed `gitea-api` helper; it reads credentials from the
+  environment. Never inspect local credential files, print environment values,
+  or construct HTTP authorization headers.
+- The audited API token can read/write this repository and create issues, but
+  does not have `read:user` or `read:package`. An API `403` caused by a missing
+  scope is not evidence that the account itself lacks the permission.
+- Protected `main` requires one approval. Do not push directly to `main`, merge
+  automatically, or start a review. Work on a feature branch and use a PR.
+- Gitea Actions and package features are enabled for the repository, but runner
+  availability and package publishing permissions have not been confirmed.
+
+## Who can do what
+
+| Role | Operations |
+|---|---|
+| `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners |
+| Restricted verifier | Runs the root-owned read-only drift check; not installed on the audited VPS |
+| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
+| VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
+| Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
@@ -0,0 +1,131 @@
+# Backup and restore
+
+## Current status
+
+No production data or backup existed during the 2026-10-11 audit. The repository
+now contains `infra/backup/archive.py`; it is not installed or run on the VPS,
+and `/var/backups/akyldash` remains a planned root-only location.
+
+## Required release policy
+
+- Create a backup before every production change, including any future schema
+  or index migration.
+- Include the normalized corpus, `search-users.sqlite3`,
+  `search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
+  Caddy's certificate data, and required secret file in the protected backup
+  set. Store secrets and Caddy private keys with equivalent access controls.
+- Verify archive readability, checksums, and SQLite integrity before relying
+  on it. Never overwrite or delete the only usable backup.
+- Keep one current pre-release backup. If deployment fails, retain the backup
+  required to restore the previous state.
+- Test restore into an isolated target before relying on the backup. Do not
+  restore over production as a test.
+
+## Clean first deployment
+
+No backup is needed only when Compose has no containers and all four
+persistent mounts are confirmed empty. For a first install with a preloaded
+corpus and not-yet-created named volumes, use `--initial-deployment` to create
+the pre-start backup. This flag is accepted only with no Compose containers and
+when `DATA_ROOT` contains exactly `minjust-normalized/`; it archives absent
+named volumes as empty components. In every normal backup, an absent volume is
+an error because it could mean that a previous installation lost data.
+Missing mount definitions, inaccessible Docker state, or partial service
+containers are also errors and must stop deployment. Do not create placeholder
+containers to make backup pass.
+
+For the corpus-only initial case, run this before the first `up` from the
+approved checkout. Replace the timestamp with one real UTC timestamp:
+
+```bash
+cd /opt/akyldash/current
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --initial-deployment \
+  --output /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+```
+
+## Cold backup procedure
+
+The script requires every Compose container to be stopped. Schedule downtime,
+then run from the approved checkout on the VPS:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+```
+
+Replace the timestamp placeholder with one real UTC timestamp before running.
+On a clean first deployment, `create` exits with a clear message that backup
+can be skipped. With existing data and all expected volumes present, it
+archives the storage even when no Compose containers exist. A preloaded corpus
+with no named volumes requires the explicit `--initial-deployment` mode;
+otherwise missing volumes are rejected. Partial service containers or running
+services are rejected as unsafe. The create command archives the application data,
+OpenSearch volume, Caddy data/config volumes, production environment file, and
+review secret. It records the checked-out Git revision, deployment-file hashes,
+and image references/IDs for each stopped Compose service, writes a SHA-256
+sidecar, and refuses to overwrite an existing archive. It also runs SQLite
+`quick_check`. Start the stack again only after verification succeeds:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+## Initial post-setup baseline
+
+After the first index has been loaded and validated, the initial `admin` account
+has been created, and `/review` responds over HTTPS, create a second backup.
+This one captures the initialized index, account database, and newly created
+Caddy/OpenSearch volumes. Stop all services, create and verify the archive,
+then start the stack again:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+Use one real UTC timestamp in both archive paths. Do not treat initial
+production setup as complete until this baseline archive verifies. Keep the
+pre-start corpus archive as well until the baseline is verified and copied to
+separately managed encrypted storage.
+
+Keep the local archive root-only, then copy it to separately managed encrypted
+storage. A local archive alone does not protect against loss of the VPS disk.
+Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
+`/var/tmp` and validates the checksum and SQLite databases. Applying the staged
+contents requires the separate recovery-target tool; it refuses the inventoried
+production host, existing data/volumes, and an unmarked host. Example isolated
+restore:
+
+```bash
+sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
+  --target-dir /var/tmp/akyldash-restore-test
+```
+
+This validates and stages the archive. On the replacement host only, after
+running bootstrap with `--recovery-target` and checking out the matching Git
+revision, apply the staged data:
+
+```bash
+sudo python3 infra/backup/apply_restore.py \
+  --staging-dir /var/tmp/akyldash-restore-test
+```
+
+The apply command restores data and named volumes but leaves services stopped.
+Test the full procedure on a separate recovery host before relying on it. Do
+not run an unreviewed restore over existing data.
@@ -0,0 +1,97 @@
+# Infrastructure current state
+
+Last audit: 2026-10-11. Source: read-only SSH session as `codex`, local
+repository inspection, and Gitea API. This document separates measured facts
+from planned configuration.
+
+## VPS facts
+
+- Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`.
+- OS/kernel: Ubuntu Server `24.04.5 LTS`, `6.8.0-142-generic`.
+- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 93 GiB free.
+- Network: interface `ens3`, IPv4 `206.223.249.118/32`.
+- `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires
+  a password. The current WSL2 session verified the SSH alias using the
+  explicit user config and approved read-only SSH execution.
+- SSH service is running. The inspected sshd configuration has public-key and
+  password authentication enabled and `PermitRootLogin yes`; no SSH changes
+  were made.
+- Docker, Docker Compose, Git, Nginx, Caddy, Certbot, application paths, and
+  application services are absent. Only base OS services and SSH were running.
+- Public listeners observed: TCP `22`; DNS listeners are limited to loopback
+  (`127.0.0.53` and `127.0.0.54`).
+- UFW is reported active in the supplied VPS notes, but the audit account could
+  not independently read its status. Treat firewall state and rules as
+  unverified.
+- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses from the VPS.
+  Both HTTP and HTTPS return Cloudflare `521`; the origin is currently
+  unreachable through the proxy.
+
+## Application and Gitea facts
+
+- Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`.
+- The repository API reports push permission, Actions and packages enabled, and
+  PRs enabled. `main` is protected and requires one approval; the current API
+  principal cannot push or merge directly to `main`.
+- The Gitea token lacks `read:user` and `read:package`; runner status and package
+  publishing permissions could not be checked. This change adds
+  `.gitea/workflows/ci.yaml`; PR #42's run remains `Waiting to run` at the last
+  status check, so a compatible runner is not confirmed.
+- Nothing is running on the VPS. Repository Compose now describes Caddy
+  `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite account/review
+  data, and OpenSearch `3.7.0` with `analysis-icu`.
+- The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a
+  proxy network; OpenSearch and backend share a separate internal network, and
+  the backend has no host port. Caddy certificate storage, OpenSearch, and
+  application data use persistent volumes/bind mounts. TLS, registry digest
+  deployment for backend/OpenSearch, backup, rollback helper, and systemd
+  deployment are not configured on the host.
+- The repository has a cold-backup archive/verification tool and a one-shot
+  restore apply tool that requires a marked replacement host. Backup/restore
+  tests use synthetic data; neither tool has run against production. The
+  archive preflight now permits a clean first deployment to skip backup,
+  requires backup for existing persistent data even without containers,
+  allows absent named volumes only in explicit corpus-only initial-deployment
+  mode, and rejects inaccessible/ambiguous mounts and incomplete/running
+  service containers. Initial deployment docs require a second verified backup
+  after index and admin initialization.
+- Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
+  inventoried production hostname/IP; its host-identity guard has shell tests.
+  The mode has not been run on a VPS.
+- The repository defines a root-owned read-only verifier, rollback helper, and
+  owner-run installer with exact sudoers command rules. None is installed on
+  the VPS.
+- An isolated local Compose smoke test built the backend/OpenSearch images,
+  reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
+  200 for `/review` through Caddy. Its containers, networks, and volumes were
+  removed afterward. This does not verify VPS startup, DNS, or public TLS.
+- The app has no migration framework or SMTP setup. Current app versions:
+  backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
+  deployed on this VPS).
+
+## Checks not completed
+
+- SSH completed with the user config explicitly selected; plain SSH was blocked
+  by permissions on a system config include. The private key was not read.
+- UFW rules require root and remain unverified.
+- The backup archive/restore tool was tested with synthetic local data; a backup
+  and restore of real production data remain unverified because production is
+  not deployed.
+- Production instructions now use `/etc/akyldash/production.env` and root-run
+  Docker commands, consistent with the `codex` account's restricted access.
+  Initial deployment and upgrade backup ordering are documented; no production
+  host was changed for this PR.
+- The Cloudflare edge response was checked, but origin DNS configuration,
+  Cloudflare Full (strict), certificate validity, Actions runner availability,
+  registry publishing, and production deployment have not been tested.
+- No packages or services were installed or changed on the VPS.
+
+## Next owner actions
+
+1. Review and run the root bootstrap on the dedicated host.
+2. Review SSH/password/root policy and firewall rules separately; this change
+   did not touch them.
+3. Set up DNS, origin certificate, reverse proxy, protected secret files, and
+   initial corpus/index.
+4. Confirm a Gitea Actions runner and package permissions, then test CI and
+   backup/restore before considering automated production deploy.
@@ -0,0 +1,10 @@
+# Infrastructure decisions
+
+| Date | Decision | Reason and alternatives | Status / limits |
+|---|---|---|---|
+| 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
+| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only a fixed read-only verifier and root-owned rollback helper after checkout ownership and sudoers validation | Helpers are defined in Git; not installed on VPS |
+| 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
+| 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
+| 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
+| 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |
@@ -0,0 +1,62 @@
+# Deployment
+
+## Current status
+
+Production deployment is not enabled or tested on the VPS. An isolated local
+Compose smoke test passed; the VPS has no Docker, Git,
+application data, proxy, registry pull configuration, backup process, or
+restricted verification/rollback helpers. Do not deploy until the owner has
+completed the root bootstrap, provided a protected secret file and a reviewed
+proxy/firewall configuration, and a backup and restore have been tested.
+
+## First host preparation
+
+1. Review the current inventory and state, then have the owner run
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh` from a verified copy
+   of this branch on the VPS. The script installs Docker Engine/Compose and Git,
+   sets Docker log rotation, and creates root-owned paths. It refuses other OS
+   versions and does not change SSH, UFW, sudoers, or user groups. Use
+   `--recovery-target` only on a replacement host; it refuses the known
+   production host identity.
+2. Install the approved repository checkout at `/opt/akyldash/current`,
+   root-owned and not group/world-writable. Copy
+   `deploy/production/.env.example` to `/etc/akyldash/production.env`, set
+   `DATA_ROOT`, and configure
+   `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
+   Git. The secret file must be root-readable only.
+3. After reviewing the root-owned checkout, run
+   `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader,
+   read-only verifier, and rollback helper with exact sudoers command rules; it
+   does not grant general sudo or Docker access.
+4. Confirm DNS points through Cloudflare to this origin, then have the owner
+   review firewall rules and open only TCP `80` and `443` in addition to the
+   existing SSH access. The Compose Caddy service obtains and renews its HTTPS
+   certificate automatically; Cloudflare Full (strict) still needs owner-side
+   verification.
+5. For a clean first deployment, skip backup only when Compose confirms there
+   are no containers and all persistent mounts are empty. If any data exists,
+   create and verify a backup before starting a new version. For a preloaded
+   corpus with absent named volumes, use `--initial-deployment`; it is limited
+   to a container-free host whose `DATA_ROOT` contains only
+   `minjust-normalized/`. Normal backups reject absent volumes. An uncertain
+   or partial state stops deployment. Then start services, load and validate
+   the OpenSearch index, create the initial admin interactively, verify
+   `/review` over HTTPS, and create the required post-setup baseline backup
+   described in `BACKUP.md`. Record the backend image ID for drift checks:
+
+   ```bash
+   sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
+   sudo chmod 0600 /opt/akyldash/deployed-image.txt
+   ```
+
+Application deployment remains owner-run until registry permissions, a live
+Gitea runner, and a controlled deploy procedure are tested. The helper only
+supports a guarded application rollback.
+
+## CI
+
+`.gitea/workflows/ci.yaml` runs the Python unit suites and builds both
+production images on pull requests and pushes to `main`. It does not publish an
+image or deploy production. Gitea Actions is enabled at repository level, but a
+successful workflow run has not yet confirmed that a compatible runner is
+online.
@@ -0,0 +1,59 @@
+# Infrastructure
+
+## Intended layout
+
+Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
+`deploy/production/compose.yaml` defines Caddy, a Python Search API, and private
+OpenSearch 3.7. Caddy terminates HTTPS for `akyldash.kg` and proxies to the API.
+The API is reachable only on private Compose networks; OpenSearch is reachable
+only by the backend. Cloudflare should use Full (strict) only after the origin
+certificate and DNS path are verified.
+
+The API stores its account and review SQLite files beside the normalized corpus
+under the persistent data directory. OpenSearch and Caddy use named volumes.
+The API's review signing key is mounted from a host file; it is not passed in
+process arguments or committed to Git. Docker logs are bounded to 10 MiB per
+file and three files per service.
+
+## Actual state
+
+The 2026-10-11 read-only audit found a fresh Ubuntu 24.04.5 host with SSH only.
+Docker, Git, a reverse proxy, application directories, Compose services, TLS,
+and backups are not installed/configured. The `codex` account has no
+non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
+[production inventory](../../infra/inventory/production.yml).
+
+The Compose file is a repository target, not proof of deployment. The Caddy
+image is pinned by digest, while backend and OpenSearch still build locally;
+immutable registry images and automated deploy are not configured. No migration
+framework exists. The application uses SQLite and OpenSearch; SMTP
+registration/recovery is not implemented.
+
+## Repository paths and target host paths
+
+- Compose and Dockerfiles: `deploy/production/`.
+- Host checkout target: `/opt/akyldash/current`.
+- Environment configuration: `/etc/akyldash/production.env`.
+- Secret file: `/etc/akyldash/review_secret`, mode `0600`.
+- Application data: `/var/lib/akyldash/data`.
+- Intended backups: `/var/backups/akyldash`.
+
+All host paths above are targets; none existed at audit time. Root bootstrap is
+prepared at [bootstrap-ubuntu-24.04.sh](../../infra/bootstrap/bootstrap-ubuntu-24.04.sh).
+It does not edit SSH or firewall configuration, create credentials, deploy the
+application, or grant `codex` Docker/root access.
+
+## Network and operations
+
+Target public TCP listeners are SSH `22` and Caddy `80`/`443`. Backend `8080`
+and OpenSearch `9200` must remain private to Compose. The
+firewall state could not be independently verified because the SSH account has
+no sudo; do not change it without owner approval. The domain resolves to
+Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
+unreachable. Cloudflare origin mode, certificate validity, and production
+health have not been verified.
+
+Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
+[ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
+limits and procedures. Configuration changes must go through Git and review;
+do not reconcile a Git/server difference by overwriting the VPS automatically.
@@ -0,0 +1,47 @@
+# Recovery on a clean VPS
+
+## Scope and safety
+
+The bootstrap below prepares Ubuntu packages and directories only. It is not a
+complete recovery: production secrets, data, reverse proxy, TLS, registry
+credentials, backup restore, and deploy-helper still require owner-controlled
+setup. Run only on the new recovery host after verifying its identity. Never
+run recovery commands against the existing production by accident.
+
+## Procedure
+
+1. Provision a clean Ubuntu Server 24.04 host and confirm hostname, IP, console
+   access, and a tested owner recovery path.
+2. Create the SSH-authorized `codex` account without sudo or Docker group access.
+   Preserve the host's existing root SSH and firewall policy until an owner has
+   reviewed the change.
+3. Check out the approved Git revision, root-owned and not group/world-writable,
+   at `/opt/akyldash/current`, then run
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
+   recovery mode refuses the inventoried production hostname/IP and writes a
+   root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
+   Engine, Compose, Git, Docker log limits, and target directories.
+4. Install the limited read-only verifier and rollback helper with
+   `sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
+   checkout and gives `codex` access only to those fixed commands.
+5. Configure DNS to point through Cloudflare to the recovery host and allow the
+   owner to review/open TCP `80` and `443`. The production Compose Caddy service
+   obtains its origin certificate automatically. Verify Cloudflare Full
+   (strict) end to end before cutover.
+6. Transfer the protected backup to the new host through the owner's approved
+   channel. The archive contains the production environment and review secret;
+   keep it root-only and never paste secret values into chat.
+7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging
+   directory with `infra/backup/archive.py`, then run
+   `sudo python3 infra/backup/apply_restore.py --staging-dir <staging-dir>`.
+   Apply requires the recovery marker, matching clean Git revision, empty data
+   path, absent Compose volumes/config files, and no running containers. It
+   refuses the inventoried production host and leaves services stopped.
+8. Build the backend and OpenSearch images from the matching checked-out Git
+   revision and explicitly start Compose with `--wait`. Verify HTTPS and
+   container health, record the backend image ID as described in
+   [DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in
+   [CURRENT_STATE.md](CURRENT_STATE.md).
+
+Do not repoint DNS or retire the old host until the owner verifies recovery and
+approves the cutover. The restore and cutover have not been practiced yet.
@@ -0,0 +1,34 @@
+# Rollback
+
+## Current status
+
+There is no production release, backup, or installed rollback helper. The
+repository contains a root-owned rollback helper and a separate installer, but
+neither has been run on the VPS.
+
+After a verified cold backup is created before a release, an owner must check
+that the data format is compatible with the prior backend/OpenSearch images.
+Then run the limited helper with that archive:
+
+```bash
+sudo -n /usr/local/sbin/akyldash-rollback \
+  /var/backups/akyldash/<pre-release-backup>.tar.gz --data-compatible
+```
+
+The helper verifies archive integrity, Compose image references, deployment
+file hashes, and the availability of both prior local images. It makes and
+verifies a second cold backup, restores only the prior backend/OpenSearch image
+tags, waits for Compose healthchecks, and checks HTTPS locally. It refuses to
+restore if the current Compose/Caddyfile differs from the saved release. This
+does not restore data or automatically undo a release with an unknown schema
+change. The application has no formal migration framework; compatibility must
+be checked for every change. If it is uncertain, stop and ask the owner.
+
+The initial owner-run deployment must record its running backend image ID in
+`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates
+that record after a successful rollback, and `infra/bootstrap/verify.sh` checks
+it against the running container.
+
+Any data restore requires the owner, a verified pre-change backup, and a
+maintenance window. Preserve the failed release and both backups until recovery
+is confirmed.
@@ -5,7 +5,7 @@

 - Telegram-бот: `0.2.2`
 - Telegram-бот на Synology: `0.2.1`
- Backend: `0.11.2`
+- Backend: `0.11.3`
 - Frontend: `0.1.2` — поиск, лаборатория релевантности, аккаунты

 ## Краткий итог
@@ -110,8 +110,9 @@

 ### Развёртывание и сопровождение

- Docker-конфигурация развёртывания не хранится в репозитории.
- Не настроен CI/CD.
+- Production Compose, Ubuntu bootstrap и drift-проверка описаны в репозитории.
+- CI workflow добавлен в PR #42, но его Gitea run остаётся в очереди; наличие
+  runner и автоматический production-деплой не подтверждены.

 ### Хранилище

@@ -281,4 +282,4 @@

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
@@ -262,4 +262,4 @@ runtime-зависимостями frontend. Регистрация в стор

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.1
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
@@ -44,4 +44,4 @@

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+"""Install a verified staging restore onto an explicitly marked new VPS."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+from pathlib import Path
+
+from archive import COMPONENTS, check_sqlite, digest
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_DEST = Path("/etc/akyldash/production.env")
+SECRET_DEST = Path("/etc/akyldash/review_secret")
+DATA_DEST = Path("/var/lib/akyldash/data")
+MARKER = Path("/etc/akyldash/recovery-target")
+PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+VOLUME_TARGETS = {
+    "/usr/share/opensearch/data": "opensearch-data",
+    "/data": "caddy-data",
+    "/config": "caddy-config",
+}
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def require_empty_directory(path: Path) -> None:
+    if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
+        raise ValueError(f"restore destination must be an existing empty directory: {path}")
+
+
+def tree_size(root: Path) -> int:
+    total = 0
+    for directory, names, files in os.walk(root, followlinks=False):
+        names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
+        for name in files:
+            path = Path(directory) / name
+            if path.is_symlink():
+                continue
+            if path.is_file():
+                total += path.stat().st_size
+    return total
+
+
+def copy_contents(source: Path, destination: Path) -> None:
+    for child in source.iterdir():
+        target = destination / child.name
+        if child.is_dir() and not child.is_symlink():
+            shutil.copytree(child, target, symlinks=True)
+        else:
+            shutil.copy2(child, target, follow_symlinks=False)
+
+
+def empty_directory(path: Path) -> None:
+    for child in path.iterdir():
+        if child.is_dir() and not child.is_symlink():
+            shutil.rmtree(child)
+        else:
+            child.unlink()
+
+
+def validate_stage(staging: Path) -> dict:
+    staging = staging.resolve(strict=True)
+    if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
+        raise ValueError("staging directory must be a direct child of /var/tmp")
+    if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
+        raise ValueError("staging directory must be root-owned mode 0700")
+    manifest_path = staging / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or set(manifest.get("components", [])) != COMPONENTS
+    ):
+        raise ValueError("staging manifest is invalid")
+    for name in COMPONENTS - {"production-env", "review-secret"}:
+        component = staging / name
+        if not component.is_dir() or component.is_symlink():
+            raise ValueError(f"staged component must be a directory: {name}")
+    for name in ("production-env", "review-secret"):
+        path = staging / name
+        if not path.is_file() or path.is_symlink():
+            raise ValueError(f"staged sensitive component must be a regular file: {name}")
+    return manifest
+
+
+def validate_checkout(manifest: dict) -> None:
+    if ROOT.is_symlink() or not (ROOT / ".git").exists():
+        raise ValueError("approved Git checkout is missing or is a symlink")
+    if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
+        raise ValueError("recovery checkout must be clean")
+    unsafe = subprocess.run(
+        ["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if unsafe:
+        raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
+    revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
+    if revision != manifest.get("git_revision"):
+        raise ValueError("checkout Git revision does not match the backup manifest")
+    for relative, expected in manifest.get("deployment_files", {}).items():
+        if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
+            raise ValueError("Compose or Caddyfile does not match the backup manifest")
+    if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
+        raise ValueError("backup manifest is missing deployment file hashes")
+
+
+def validate_recovery_target() -> None:
+    if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("root-only recovery-target marker is required")
+    hostname = socket.getfqdn()
+    addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
+    if not hostname or not addresses:
+        raise ValueError("cannot verify the recovery host identity")
+    if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
+        raise ValueError("refusing to apply recovery data to the inventoried production VPS")
+
+
+def compose_configuration(staging: Path) -> dict:
+    rendered = run(
+        [
+            "docker", "compose", "--env-file", str(staging / "production-env"),
+            "-f", str(COMPOSE_FILE), "config", "--format", "json",
+        ],
+        capture=True,
+    )
+    config = json.loads(rendered)
+    data_source = next(
+        mount["source"]
+        for mount in config["services"]["backend"]["volumes"]
+        if mount["target"] == "/app/data"
+    )
+    secret_source = config["secrets"]["review_secret"]["file"]
+    if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
+        raise ValueError("backup production paths do not match the clean recovery host layout")
+    return config
+
+
+def expected_volume_names(config: dict) -> set[str]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    names: set[str] = set()
+    for target in VOLUME_TARGETS:
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        names.add(config["volumes"][mount["source"]]["name"])
+    return names
+
+
+def volume_destinations(config: dict) -> dict[str, Path]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    destinations = {}
+    for target, component in VOLUME_TARGETS.items():
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        name = config["volumes"][mount["source"]]["name"]
+        destinations[component] = Path(
+            run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
+        )
+    return destinations
+
+
+def apply_restore(staging: Path) -> None:
+    if os.geteuid() != 0:
+        raise ValueError("restore apply must run as root")
+    validate_recovery_target()
+    manifest = validate_stage(staging)
+    validate_checkout(manifest)
+    config = compose_configuration(staging)
+    require_empty_directory(DATA_DEST)
+    if ENV_DEST.exists() or SECRET_DEST.exists():
+        raise ValueError("production environment and secret files must not already exist")
+    if run(["docker", "ps", "--quiet"], capture=True):
+        raise ValueError("recovery target must have no running Docker containers")
+
+    volume_names = expected_volume_names(config)
+    for name in volume_names:
+        result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
+        if result.returncode == 0:
+            raise ValueError(f"recovery volume already exists: {name}")
+
+    created_volumes: list[str] = []
+    created_files: list[Path] = []
+    data_started = False
+    try:
+        for name in volume_names:
+            run(["docker", "volume", "create", name])
+            created_volumes.append(name)
+
+        destinations = volume_destinations(config)
+        destinations["data"] = DATA_DEST
+
+        required: dict[int, int] = {}
+        free: dict[int, int] = {}
+        for component, destination in destinations.items():
+            device = destination.stat().st_dev
+            required[device] = required.get(device, 0) + tree_size(staging / component)
+            free[device] = shutil.disk_usage(destination).free
+        for device, size in required.items():
+            if free[device] < size + 512 * 1024 * 1024:
+                raise ValueError("recovery host lacks space for a safe data copy")
+
+        for component, destination in destinations.items():
+            require_empty_directory(destination)
+            if component == "data":
+                data_started = True
+            copy_contents(staging / component, destination)
+
+        for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
+            os.chown(directory, 1000, 1000, follow_symlinks=False)
+            for name in files:
+                os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
+
+        for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
+            shutil.copyfile(source, destination)
+            created_files.append(destination)
+            os.chown(destination, 0, 0)
+            os.chmod(destination, 0o600)
+        check_sqlite(DATA_DEST)
+    except BaseException:
+        for path in created_files:
+            path.unlink(missing_ok=True)
+        if data_started:
+            empty_directory(DATA_DEST)
+        for name in created_volumes:
+            subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
+        raise
+
+    MARKER.unlink()
+    print("PASS staged recovery data applied to the marked new host; services remain stopped")
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--staging-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        apply_restore(args.staging_dir)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,393 @@
+#!/usr/bin/env python3
+"""Create and verify cold, local Akyldash backups; restore only into staging."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path, PurePosixPath
+
+COMPONENTS = {
+    "data",
+    "opensearch",
+    "caddy-data",
+    "caddy-config",
+    "production-env",
+    "review-secret",
+}
+NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+
+def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
+    members = archive.getmembers()
+    names: set[str] = set()
+    for member in members:
+        name = PurePosixPath(member.name)
+        if name.is_absolute() or ".." in name.parts or not name.parts:
+            raise ValueError(f"unsafe archive path: {member.name}")
+        if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
+            raise ValueError(f"non-canonical archive path: {member.name}")
+        if member.name in names:
+            raise ValueError(f"duplicate archive path: {member.name}")
+        names.add(member.name)
+        if not (member.isdir() or member.isfile() or member.issym()):
+            raise ValueError(f"unsupported archive entry: {member.name}")
+        if member.issym():
+            tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
+    if "manifest.json" not in names:
+        raise ValueError("backup manifest is missing")
+    if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
+        raise ValueError("backup is missing one or more required components")
+    manifest_member = next(item for item in members if item.name == "manifest.json")
+    if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
+        raise ValueError("backup manifest is not a regular file")
+    return members
+
+
+def read_manifest(archive: tarfile.TarFile) -> dict:
+    checked_members(archive)
+    stream = archive.extractfile("manifest.json")
+    if stream is None:
+        raise ValueError("backup manifest is unreadable")
+    manifest = json.load(stream)
+    components = manifest.get("components") if isinstance(manifest, dict) else None
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or not isinstance(components, list)
+        or not all(isinstance(item, str) for item in components)
+        or set(components) != COMPONENTS
+    ):
+        raise ValueError("backup manifest has an unsupported format or component set")
+    if not isinstance(manifest.get("images"), dict):
+        raise ValueError("backup manifest has no image records")
+    if not isinstance(manifest.get("deployment_files"), dict):
+        raise ValueError("backup manifest has no deployment file records")
+    for member in archive.getmembers():
+        if member.name == "manifest.json":
+            continue
+        if PurePosixPath(member.name).parts[0] not in COMPONENTS:
+            raise ValueError(f"unexpected top-level archive path: {member.name}")
+    return manifest
+
+
+def verify_archive(path: Path) -> dict:
+    checksum_path = Path(f"{path}.sha256")
+    expected = checksum_path.read_text(encoding="ascii").split()[0]
+    if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
+        raise ValueError("archive SHA-256 does not match its sidecar")
+    with tarfile.open(path, "r:gz") as archive:
+        return read_manifest(archive)
+
+
+def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
+    sources: dict[str, Path] = {}
+    for value in values:
+        name, separator, raw_path = value.partition("=")
+        if not separator or not NAME_RE.fullmatch(name) or name in sources:
+            raise ValueError(f"invalid or duplicate source: {value}")
+        source = Path(raw_path)
+        if not source.is_absolute() or not source.exists() or source.is_symlink():
+            raise ValueError(f"source must be an existing absolute path: {raw_path}")
+        sources[name] = source.resolve(strict=True)
+    if set(sources) != required:
+        missing = ", ".join(sorted(required - sources.keys()))
+        extra = ", ".join(sorted(sources.keys() - required))
+        raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
+    return sources
+
+
+def check_sqlite(root: Path) -> None:
+    for path in root.rglob("*.sqlite3"):
+        connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
+        try:
+            result = connection.execute("PRAGMA quick_check").fetchone()
+            if result != ("ok",):
+                raise ValueError(f"SQLite integrity check failed: {path}")
+        finally:
+            connection.close()
+
+
+def compose_storage(config: dict) -> dict[str, Path | None]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    volumes = subprocess.run(
+        ["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
+    ).stdout.split()
+    sources: dict[str, Path | None] = {}
+    for target, component in (
+        ("/app/data", "data"),
+        ("/usr/share/opensearch/data", "opensearch"),
+        ("/data", "caddy-data"),
+        ("/config", "caddy-config"),
+    ):
+        mount = mounts.get(target)
+        if not mount:
+            raise ValueError(f"persistent Compose mount is missing: {target}")
+        if mount["type"] == "bind":
+            source = Path(mount["source"])
+        elif mount["type"] == "volume":
+            volume_name = config["volumes"][mount["source"]]["name"]
+            if volume_name not in volumes:
+                sources[component] = None
+                continue
+            result = subprocess.run(
+                ["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            source = Path(result.stdout.strip())
+        else:
+            raise ValueError(f"unsupported persistent mount type at {target}")
+        if not source.is_dir() or source.is_symlink():
+            raise ValueError(f"cannot determine persistent data state for {component}: {source}")
+        sources[component] = source
+    return sources
+
+
+def has_persistent_data(sources: dict[str, Path | None]) -> bool:
+    return any(any(path.iterdir()) for path in sources.values() if path is not None)
+
+
+def is_preloaded_corpus_only(data: Path | None) -> bool:
+    if data is None:
+        return False
+    entries = list(data.iterdir())
+    return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
+
+
+def compose_sources(
+    compose_file: Path, env_file: Path, initial_deployment: bool = False
+) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
+    command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
+    rendered = subprocess.run(
+        [*command, "config", "--format", "json"], check=True, capture_output=True, text=True
+    )
+    config = json.loads(rendered.stdout)
+    container_result = subprocess.run(
+        [*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
+    )
+    containers = container_result.stdout.split()
+    storage = compose_storage(config)
+    persistent_data = has_persistent_data(storage)
+    if not containers and not persistent_data:
+        raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
+    missing_volumes = any(path is None for path in storage.values())
+    if missing_volumes and containers:
+        raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
+    if missing_volumes and not initial_deployment:
+        raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
+    if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
+        raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
+    images: dict[str, dict[str, str]] = {}
+    for container in containers:
+        result = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                '{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
+                container,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        fields = result.stdout.strip().split("\t")
+        if len(fields) != 4 or fields[0].lower() != "false":
+            raise ValueError("stop all Compose services before taking a cold backup")
+        images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
+    if containers and set(images) != set(config["services"]):
+        raise ValueError("not all configured Compose services have stopped containers")
+
+    secret_file = Path(config["secrets"]["review_secret"]["file"])
+    sources = {
+        **storage,
+        "production-env": env_file,
+        "review-secret": secret_file,
+    }
+    repository = compose_file.resolve().parents[2]
+    changed = subprocess.run(
+        ["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if changed:
+        raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
+    revision = subprocess.run(
+        ["git", "-C", str(repository), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    deployment_files = {
+        relative: digest(repository / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    present = {name: path for name, path in sources.items() if path is not None}
+    missing = {name for name, path in sources.items() if path is None}
+    parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
+    return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
+
+
+def create_backup(
+    output: Path,
+    sources: dict[str, Path | None],
+    revision: str,
+    images: dict | None = None,
+    deployment_files: dict[str, str] | None = None,
+) -> None:
+    output = output.absolute()
+    parent = output.parent
+    if not parent.is_dir() or parent.stat().st_mode & 0o077:
+        raise ValueError("backup directory must exist and have mode 0700 or stricter")
+    if output.exists() or Path(f"{output}.sha256").exists():
+        raise ValueError("refusing to overwrite an existing backup")
+    for name, source in sources.items():
+        if source is None:
+            continue
+        if source.is_symlink() or not (source.is_dir() or source.is_file()):
+            raise ValueError(f"source must be a regular file or directory: {name}")
+        if source.is_dir() and (output == source or source in output.parents):
+            raise ValueError(f"backup output cannot be inside source {name}")
+    for name in ("production-env", "review-secret"):
+        if sources[name] is None or not sources[name].is_file():
+            raise ValueError(f"sensitive component must be a regular file: {name}")
+    if sources["data"] is not None and sources["data"].is_dir():
+        check_sqlite(sources["data"])
+
+    manifest = {
+        "format": 1,
+        "created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+        "git_revision": revision,
+        "images": images or {},
+        "deployment_files": deployment_files or {},
+        "components": sorted(sources),
+    }
+    with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
+        temporary = Path(temp.name)
+    os.chmod(temporary, 0o600)
+    checksum_temp = Path(f"{temporary}.sha256")
+    try:
+        with tarfile.open(temporary, "w:gz") as archive:
+            encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
+            info = tarfile.TarInfo("manifest.json")
+            info.size = len(encoded)
+            info.mode = 0o600
+            archive.addfile(info, io.BytesIO(encoded))
+            for name, source in sorted(sources.items()):
+                if source is None:
+                    info = tarfile.TarInfo(f"{name}/")
+                    info.type = tarfile.DIRTYPE
+                    info.mode = 0o700
+                    archive.addfile(info)
+                    continue
+                def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
+                    if name in {"production-env", "review-secret"}:
+                        member.mode = 0o600
+                    if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
+                        member.mode = 0o600
+                    return member
+
+                archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
+        with tarfile.open(temporary, "r:gz") as archive:
+            read_manifest(archive)
+        checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
+            stream.write(f"{digest(temporary)}  {output.name}\n")
+        os.link(temporary, output)
+        try:
+            os.link(checksum_temp, Path(f"{output}.sha256"))
+        except Exception:
+            output.unlink(missing_ok=True)
+            raise
+    finally:
+        temporary.unlink(missing_ok=True)
+        checksum_temp.unlink(missing_ok=True)
+
+
+def restore_to_staging(backup: Path, target: Path) -> None:
+    if not target.name.startswith("akyldash-restore-"):
+        raise ValueError("restore target must be a new akyldash-restore-* staging directory")
+    parent = target.parent.resolve()
+    if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
+        raise ValueError("restore staging directory must be under /tmp or /var/tmp")
+    if target.exists():
+        raise ValueError("restore target already exists; refusing to overwrite it")
+    target = parent / target.name
+    verify_archive(backup)
+    target.mkdir(mode=0o700)
+    try:
+        with tarfile.open(backup, "r:gz") as archive:
+            members = checked_members(archive)
+            required_bytes = sum(member.size for member in members if member.isfile())
+            if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
+                raise ValueError("insufficient free space for a validated restore staging copy")
+            for member in members:
+                archive.extract(member, target, filter="data")
+        check_sqlite(target / "data")
+    except Exception:
+        shutil.rmtree(target)
+        raise
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    create = subparsers.add_parser("create", help="create a cold backup archive")
+    create.add_argument("--output", required=True, type=Path)
+    create.add_argument("--compose-file", required=True, type=Path)
+    create.add_argument("--env-file", required=True, type=Path)
+    create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
+    verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
+    verify.add_argument("archive", type=Path)
+    restore = subparsers.add_parser("restore", help="extract into a new staging directory")
+    restore.add_argument("archive", type=Path)
+    restore.add_argument("--target-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        if args.command in {"create", "restore"} and os.geteuid() != 0:
+            raise ValueError("create and restore must run as root to protect production secrets and data")
+        if args.command == "create":
+            sources, revision, images, deployment_files = compose_sources(
+                args.compose_file, args.env_file, args.initial_deployment
+            )
+            create_backup(args.output, sources, revision, images, deployment_files)
+        elif args.command == "verify":
+            manifest = verify_archive(args.archive)
+            print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
+        else:
+            restore_to_staging(args.archive, args.target_dir)
+            print(f"PASS restored and validated staging data at {args.target_dir}")
+    except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,90 @@
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import apply_restore
+
+
+class ApplyRestoreTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.staging = self.root / "akyldash-restore-test"
+        self.staging.mkdir()
+        for component in apply_restore.COMPONENTS:
+            path = self.staging / component
+            if component in {"production-env", "review-secret"}:
+                path.write_text("private", encoding="utf-8")
+            else:
+                path.mkdir()
+                (path / "payload.txt").write_text(component, encoding="utf-8")
+        self.data = self.root / "data"
+        self.data.mkdir()
+        self.volumes = {
+            name: self.root / name
+            for name in ("opensearch", "caddy-data", "caddy-config")
+        }
+        for path in self.volumes.values():
+            path.mkdir()
+        self.env = self.root / "production.env"
+        self.secret = self.root / "review_secret"
+        self.marker = self.root / "recovery-target"
+        self.marker.touch(mode=0o600)
+        self.config = {
+            "volumes": {
+                "opensearch-data": {"name": "akyldash_opensearch-data"},
+                "caddy-data": {"name": "akyldash_caddy-data"},
+                "caddy-config": {"name": "akyldash_caddy-config"},
+            }
+        }
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def run_apply(self):
+        def docker_run(command, **_kwargs):
+            if command[:3] == ["docker", "ps", "--quiet"]:
+                return ""
+            if command[1:3] == ["volume", "create"]:
+                return command[-1]
+            return ""
+
+        with (
+            patch("apply_restore.os.geteuid", return_value=0),
+            patch("apply_restore.validate_recovery_target"),
+            patch("apply_restore.validate_stage", return_value={"format": 1}),
+            patch("apply_restore.validate_checkout"),
+            patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}),
+            patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])),
+            patch("apply_restore.volume_destinations", return_value=self.volumes),
+            patch("apply_restore.DATA_DEST", self.data),
+            patch("apply_restore.ENV_DEST", self.env),
+            patch("apply_restore.SECRET_DEST", self.secret),
+            patch("apply_restore.MARKER", self.marker),
+            patch("apply_restore.run", side_effect=docker_run),
+            patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)),
+            patch("apply_restore.os.chown"),
+        ):
+            apply_restore.apply_restore(self.staging)
+
+    def test_applies_components_and_protects_secret_files(self):
+        self.run_apply()
+        self.assertEqual((self.data / "payload.txt").read_text(), "data")
+        self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch")
+        self.assertEqual(self.env.stat().st_mode & 0o777, 0o600)
+        self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600)
+        self.assertFalse(self.marker.exists())
+
+    def test_refuses_nonempty_data_destination_before_writing_secrets(self):
+        (self.data / "keep.txt").write_text("existing", encoding="utf-8")
+        with self.assertRaisesRegex(ValueError, "empty directory"):
+            self.run_apply()
+        self.assertEqual((self.data / "keep.txt").read_text(), "existing")
+        self.assertFalse(self.env.exists())
+        self.assertTrue(self.marker.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,362 @@
+import hashlib
+import json
+import sqlite3
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from archive import COMPONENTS, compose_sources, create_backup, restore_to_staging, verify_archive
+
+
+class BackupArchiveTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.private = self.root / "private"
+        self.private.mkdir(mode=0o700)
+        self.sources = {}
+        for name in COMPONENTS:
+            source = self.root / name
+            source.mkdir()
+            self.sources[name] = source
+        for name in ("production-env", "review-secret"):
+            path = self.root / f"{name}.txt"
+            path.write_text("private", encoding="utf-8")
+            path.chmod(0o644)
+            self.sources[name] = path
+        connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
+        connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
+        connection.commit()
+        connection.close()
+        self.backup = self.private / "backup.tar.gz"
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def test_create_verify_and_restore_only_to_new_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        manifest = verify_archive(self.backup)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        self.assertEqual(manifest["images"], {})
+
+        target = self.root / "akyldash-restore-test"
+        restore_to_staging(self.backup, target)
+        self.assertTrue((target / "data/search-users.sqlite3").is_file())
+        self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
+        self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
+
+    def test_rejects_checksum_mismatch_without_creating_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
+        target = self.root / "akyldash-restore-tampered"
+        with self.assertRaisesRegex(ValueError, "SHA-256"):
+            restore_to_staging(self.backup, target)
+        self.assertFalse(target.exists())
+
+    def test_rejects_non_staging_and_existing_targets(self):
+        create_backup(self.backup, self.sources, "abc123")
+        with self.assertRaisesRegex(ValueError, "staging"):
+            restore_to_staging(self.backup, self.root / "production")
+        target = self.root / "akyldash-restore-existing"
+        target.mkdir()
+        with self.assertRaisesRegex(ValueError, "already exists"):
+            restore_to_staging(self.backup, target)
+
+    def test_create_checks_sqlite_integrity_before_writing_archive(self):
+        (self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
+        with self.assertRaises(sqlite3.DatabaseError):
+            create_backup(self.backup, self.sources, "abc123")
+        self.assertFalse(self.backup.exists())
+
+    def test_rejects_unsafe_archive_paths(self):
+        archive_path = self.private / "unsafe.tar.gz"
+        with tarfile.open(archive_path, "w:gz") as archive:
+            archive.addfile(tarfile.TarInfo("manifest.json"))
+            archive.addfile(tarfile.TarInfo("../outside"))
+        checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
+        Path(f"{archive_path}.sha256").write_text(f"{checksum}  unsafe.tar.gz\n")
+        with self.assertRaisesRegex(ValueError, "unsafe archive path"):
+            verify_archive(archive_path)
+
+    def test_cold_backup_refuses_a_running_compose_container(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="container-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            return SimpleNamespace(stdout="true\n")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "stop all Compose services"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def compose_fixture(self):
+        paths = {}
+        mounts = {
+            "backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
+            "opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+        }
+        volumes = {}
+        for volume in ("os-data", "caddy-data", "caddy-config"):
+            name = f"akyldash_{volume}"
+            path = self.root / name
+            path.mkdir()
+            paths[name] = path
+            volumes[volume] = {"name": name}
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        secret = self.root / "review-secret.txt"
+        secret.write_text("secret", encoding="utf-8")
+        config = {
+            "services": {name: {"volumes": value} for name, value in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        return config, paths, env_file, secret
+
+    def test_clean_first_deployment_does_not_require_backup(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
+        config, _volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_existing_data_without_containers_is_backup_eligible(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        archive = self.private / "offline-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images, {})
+        self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
+        self.assertEqual(manifest["git_revision"], "abc123")
+
+    def test_missing_persistent_mount_state_is_rejected(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        volume_paths["akyldash_os-data"].rmdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_partial_compose_containers_are_rejected_with_existing_data(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="backend-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if command[1] == "inspect":
+                return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        (self.root / "data/existing.json").write_text("data", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+            with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
+                compose_sources(
+                    Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
+                )
+
+    def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        corpus = self.root / "data/minjust-normalized"
+        corpus.mkdir()
+        (corpus / "document.json").write_text("corpus", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
+            )
+        self.assertIsNone(sources["opensearch"])
+        archive = self.private / "initial-corpus-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        with tarfile.open(archive, "r:gz") as saved:
+            self.assertTrue(saved.getmember("opensearch/").isdir())
+            self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
+
+    def test_compose_backup_records_sources_and_image_ids(self):
+        data = self.root / "production-data"
+        data.mkdir()
+        secret = self.root / "secret"
+        secret.write_text("private", encoding="utf-8")
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        volumes = {}
+        volume_paths = {}
+        mounts = {
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+            "backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
+            "opensearch": [
+                {"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
+            ],
+        }
+        for name in ("caddy-data", "caddy-config", "opensearch-data"):
+            path = self.root / f"volume-{name}"
+            path.mkdir()
+            volumes[name] = {"name": f"akyldash_{name}"}
+            volume_paths[f"akyldash_{name}"] = path
+        config = {
+            "services": {name: {"volumes": values} for name, values in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        service_names = ["caddy", "backend", "opensearch"]
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
+            if command[1] == "inspect":
+                name = command[-1].removesuffix("-id")
+                return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            return SimpleNamespace(stdout="")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertEqual(revision, "")
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images["backend"]["image_id"], "sha256:backend")
+        self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -gt 1 || ( $# -eq 1 && $1 != --recovery-target ) ]]; then
+  echo "Usage: $0 [--recovery-target]" >&2
+  exit 2
+fi
+recovery_target=false
+if [[ ${1:-} == --recovery-target ]]; then
+  recovery_target=true
+  . "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+  recovery_hostname=$(hostname -f 2>/dev/null || true)
+  recovery_ipv4=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  if [[ -z $recovery_hostname || -z $recovery_ipv4 ]]; then
+    echo "Cannot verify recovery host identity; refusing recovery bootstrap." >&2
+    exit 1
+  fi
+  if akyldash_is_production_host "$recovery_hostname" "$recovery_ipv4"; then
+    echo "Refusing recovery bootstrap on the current production host." >&2
+    exit 1
+  fi
+fi
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run this script as root." >&2
+  exit 1
+fi
+
+. /etc/os-release
+if [[ ${ID:-} != ubuntu || ${VERSION_ID:-} != 24.04 ]]; then
+  echo "Expected Ubuntu 24.04; found ${PRETTY_NAME:-unknown}." >&2
+  exit 1
+fi
+
+if ! id codex >/dev/null 2>&1; then
+  echo "Create the SSH-authorized codex account before running bootstrap." >&2
+  exit 1
+fi
+
+if [[ -e /etc/docker/daemon.json ]] && ! python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' /etc/docker/daemon.json; then
+  echo "/etc/docker/daemon.json exists with different log settings; review it manually." >&2
+  exit 1
+fi
+
+install -d -m 0755 /etc/docker
+if [[ ! -e /etc/docker/daemon.json ]]; then
+  cat > /etc/docker/daemon.json <<'EOF'
+{
+  "log-driver": "local",
+  "log-opts": {"max-size": "10m", "max-file": "3"}
+}
+EOF
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y ca-certificates curl git
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
+  "$(dpkg --print-architecture)" "$VERSION_CODENAME" > /etc/apt/sources.list.d/docker.list
+apt-get update
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+install -d -o root -g root -m 0755 /opt/akyldash
+install -d -o root -g root -m 0700 /etc/akyldash
+install -d -o root -g root -m 0750 /var/lib/akyldash/data
+install -d -o root -g root -m 0700 /var/backups/akyldash
+if [[ $recovery_target == true ]]; then
+  install -o root -g root -m 0600 /dev/null /etc/akyldash/recovery-target
+fi
+systemctl enable --now docker
+docker compose version
+echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
+if [[ $recovery_target == true ]]; then
+  echo 'Recovery target marker created at /etc/akyldash/recovery-target.'
+fi
@@ -0,0 +1,11 @@
+akyldash_is_production_host() {
+  local hostname=${1:-}
+  local ipv4_addresses=${2:-}
+  if [[ -z $hostname ]]; then
+    hostname=$(hostname -f 2>/dev/null || true)
+  fi
+  if [[ -z $ipv4_addresses ]]; then
+    ipv4_addresses=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  fi
+  [[ $hostname == avprogs.hlab.kz ]] || grep -Fqx '206.223.249.118' <<<"$ipv4_addresses"
+}
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+
+akyldash_is_production_host avprogs.hlab.kz 127.0.0.1
+akyldash_is_production_host other.example $'10.0.0.2\n206.223.249.118'
+if akyldash_is_production_host recovery.example 10.0.0.2; then
+  echo 'FAIL unrelated recovery host identified as production' >&2
+  exit 1
+fi
+echo 'PASS recovery host identity guard'
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if bash "$(dirname "${BASH_SOURCE[0]}")/verify-helper.sh" unexpected >/dev/null 2>&1; then
+  echo 'FAIL verifier helper accepted command arguments' >&2
+  exit 1
+fi
+echo 'PASS verifier helper rejects command arguments'
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+checkout=/opt/akyldash/current
+if [[ $# -ne 0 || $EUID -ne 0 || ! -d $checkout || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
+  echo 'Refusing verification: the approved root-owned checkout is unavailable.' >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ -n $unsafe ]]; then
+  echo 'Refusing verification: the checkout contains a writable or non-root-owned path.' >&2
+  exit 1
+fi
+exec /bin/bash "$checkout/infra/bootstrap/verify.sh" --remote
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${1:-} != --remote ]]; then
+  ssh_alias=${1:-akyldash_vps}
+  exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n /usr/local/sbin/akyldash-verify'
+fi
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run the remote check as root; it is read-only." >&2
+  exit 1
+fi
+
+failed=0
+for package in git docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do
+  if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx 'install ok installed'; then
+    printf 'PASS package %s\n' "$package"
+  else
+    printf 'FAIL package %s missing\n' "$package"
+    failed=1
+  fi
+done
+
+if systemctl is-active --quiet docker; then
+  echo 'PASS Docker service active'
+else
+  echo 'FAIL Docker service inactive'
+  failed=1
+fi
+if docker compose version >/dev/null 2>&1; then
+  echo 'PASS Docker Compose responds'
+else
+  echo 'FAIL Docker Compose unavailable'
+  failed=1
+fi
+if python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' 2>/dev/null; then
+  echo 'PASS Docker log rotation configuration'
+else
+  echo 'FAIL Docker log rotation configuration differs from repository target'
+  failed=1
+fi
+if id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo 'FAIL codex has sudo or Docker group access'
+  failed=1
+else
+  echo 'PASS codex is outside sudo and docker groups'
+fi
+
+for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
+  if [[ -d $path ]]; then
+    printf 'PASS directory %s\n' "$path"
+  else
+    printf 'FAIL directory %s missing\n' "$path"
+    failed=1
+  fi
+done
+for path in /etc/akyldash /var/backups/akyldash; do
+  if [[ -d $path && $(stat -c %a "$path") == 700 ]]; then
+    printf 'PASS permissions %s mode 0700\n' "$path"
+  elif [[ -d $path ]]; then
+    printf 'FAIL permissions %s mode %s, expected 0700\n' "$path" "$(stat -c %a "$path")"
+    failed=1
+  fi
+done
+for path in /etc/akyldash/production.env /etc/akyldash/review_secret; do
+  if [[ -f $path && $(stat -c %U:%a "$path") == root:600 ]]; then
+    printf 'PASS protected file %s mode 0600\n' "$path"
+  elif [[ -e $path ]]; then
+    printf 'FAIL protected file %s must be root-owned mode 0600\n' "$path"
+    failed=1
+  else
+    printf 'FAIL protected file %s missing\n' "$path"
+    failed=1
+  fi
+done
+
+checkout=/opt/akyldash/current
+compose_file=$checkout/deploy/production/compose.yaml
+if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Caddyfile deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
+  echo 'PASS production Compose, Caddyfile, and Dockerfiles match the checked-out Git revision'
+else
+  echo 'FAIL production Compose or Dockerfile has local drift'
+  failed=1
+fi
+if docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --quiet >/dev/null 2>&1; then
+  echo 'PASS rendered production Compose configuration'
+else
+  echo 'FAIL production Compose configuration cannot be rendered'
+  failed=1
+fi
+
+if ufw status | grep -q '^Status: active$'; then
+  echo 'PASS UFW active'
+  if ! ufw status verbose | grep -q '^Default: deny (incoming)'; then
+    echo 'FAIL UFW default incoming policy is not deny'
+    failed=1
+  fi
+  while read -r rule action direction; do
+    [[ $action == ALLOW && $direction == IN ]] || continue
+    ports=${rule//\/tcp/}
+    ports=${ports//\/udp/}
+    IFS=, read -ra allowed <<< "$ports"
+    for port in "${allowed[@]}"; do
+      case "$port" in
+        22|80|443) ;;
+        *) printf 'FAIL unexpected UFW allow rule %s\n' "$rule"; failed=1 ;;
+      esac
+    done
+  done < <(ufw status | awk 'NR > 3 && NF >= 3 {print $1, $2, $3}')
+else
+  echo 'FAIL UFW inactive or unavailable'
+  failed=1
+fi
+
+tcp_listeners=$(ss -H -lnt | awk '{print $4}')
+for endpoint in $tcp_listeners; do
+  address=${endpoint%:*}
+  port=${endpoint##*:}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  case "$port" in 22|80|443) ;; *) printf 'FAIL unexpected non-loopback TCP listener %s\n' "$endpoint"; failed=1 ;; esac
+done
+printf 'INFO non-loopback TCP listeners: %s\n' "${tcp_listeners//$'\n'/,}"
+
+udp_listeners=$(ss -H -lnu | awk '{print $5}')
+for endpoint in $udp_listeners; do
+  address=${endpoint%:*}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  printf 'FAIL unexpected non-loopback UDP listener %s\n' "$endpoint"
+  failed=1
+done
+
+expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \
+      -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
+      -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
+      -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
+      $(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then
+  echo 'PASS read-only verifier, rollback helper, and exact sudoers rules present'
+else
+  echo 'FAIL restricted helper or sudoers installation missing/different'
+  failed=1
+fi
+if [[ -f /opt/akyldash/deployed-image.txt ]]; then
+  expected=$(cat /opt/akyldash/deployed-image.txt)
+  container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
+  actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)
+  if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then
+    printf 'PASS running backend image ID matches %s\n' "$expected"
+  else
+    echo 'FAIL running backend image does not match recorded digest'
+    failed=1
+  fi
+else
+  echo 'FAIL deployed image digest record missing'
+  failed=1
+fi
+
+expected_proxy=$(docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --format json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["services"]["caddy"]["image"])' 2>/dev/null || true)
+proxy_container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=caddy | head -n1)
+actual_proxy=$(docker inspect --format '{{.Config.Image}}' "$proxy_container" 2>/dev/null || true)
+if [[ $expected_proxy == *@sha256:* && $actual_proxy == "$expected_proxy" ]]; then
+  printf 'PASS running Caddy image matches %s\n' "$expected_proxy"
+else
+  echo 'FAIL running Caddy image does not match the pinned Compose digest'
+  failed=1
+fi
+exit "$failed"
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo "Run this installer as root." >&2
+  exit 1
+fi
+
+checkout=/opt/akyldash/current
+rollback_source=$checkout/infra/deploy/rollback.py
+backup_source=$checkout/infra/backup/archive.py
+verify_source=$checkout/infra/bootstrap/verify-helper.sh
+if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source ]]; then
+  echo "Install the approved repository checkout at $checkout first." >&2
+  exit 1
+fi
+if [[ -L $checkout ]]; then
+  echo "The checkout path must not be a symlink." >&2
+  exit 1
+fi
+for directory in /opt /opt/akyldash; do
+  mode=$(stat -c %a "$directory")
+  if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then
+    echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2
+    exit 1
+  fi
+done
+if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo "Create codex without sudo or Docker-group membership before installing the helper." >&2
+  exit 1
+fi
+if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then
+  echo "Existing codex sudoers rules need owner review before installing the helper." >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ $(stat -c %U "$checkout") != root || -n $unsafe ]]; then
+  echo "The checkout must be root-owned and not group/world-writable." >&2
+  exit 1
+fi
+if ! command -v visudo >/dev/null 2>&1; then
+  echo "visudo is required to validate the restricted sudoers rule." >&2
+  exit 1
+fi
+
+for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
+  if [[ -L $path ]]; then
+    echo "Refusing symlink at $path." >&2
+    exit 1
+  fi
+done
+
+install -d -o root -g root -m 0755 /usr/local/lib/akyldash
+install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
+install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify
+install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
+
+sudoers_file=/etc/sudoers.d/akyldash-codex
+expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+created=false
+if [[ -e $sudoers_file ]]; then
+  if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2
+    exit 1
+  fi
+else
+  temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX)
+  trap 'rm -f "$temp"' EXIT
+  printf '%s\n' "$expected" > "$temp"
+  chmod 0440 "$temp"
+  visudo -cf "$temp"
+  visudo -c -f /etc/sudoers
+  mv -n "$temp" "$sudoers_file"
+  if [[ $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "Could not install the expected sudoers rule." >&2
+    exit 1
+  fi
+  created=true
+fi
+
+if ! visudo -c -f /etc/sudoers; then
+  if [[ $created == true ]]; then
+    rm -f "$sudoers_file"
+  fi
+  echo "Sudoers validation failed; the newly added rule was removed." >&2
+  exit 1
+fi
+echo "Installed read-only verification and guarded rollback helpers; codex receives no general sudo or Docker access."
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Restore prior local application images from a verified cold backup."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_FILE = Path("/etc/akyldash/production.env")
+BACKUP_DIR = Path("/var/backups/akyldash")
+BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
+APP_SERVICES = {"backend", "opensearch"}
+
+
+def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
+    images = manifest.get("images")
+    if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
+        raise ValueError("backup has no previous backend and OpenSearch images")
+    plan = {}
+    for service in sorted(APP_SERVICES):
+        image = images[service]
+        if not isinstance(image, dict):
+            raise ValueError(f"invalid image record for {service}")
+        reference = image.get("configured")
+        image_id = image.get("image_id")
+        if reference != configured.get(service):
+            raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
+        if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*", reference):
+            raise ValueError(f"{service} image reference is not a local tag")
+        if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+            raise ValueError(f"invalid previous image ID for {service}")
+        plan[service] = (image_id, reference)
+    return plan
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def compose(*args: str, capture: bool = False) -> str:
+    return run(
+        ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
+        capture=capture,
+    )
+
+
+def record_backend_image() -> None:
+    container = compose("ps", "--quiet", "backend", capture=True)
+    if not container:
+        raise ValueError("backend container is missing after rollback")
+    image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
+    if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+        raise ValueError("backend returned an invalid image ID")
+    target = Path("/opt/akyldash/deployed-image.txt")
+    with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
+        temporary = Path(stream.name)
+        stream.write(f"{image_id}\n".encode("ascii"))
+    os.chmod(temporary, 0o600)
+    try:
+        os.replace(temporary, target)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def rollback(backup: Path, data_compatible: bool) -> None:
+    if not data_compatible:
+        raise ValueError("confirm data compatibility with --data-compatible before rollback")
+    if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
+        raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
+    if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback backup must be root-owned mode 0600")
+    checksum = Path(f"{backup}.sha256")
+    if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
+
+    sys.path.insert(0, str(BACKUP_TOOL.parent))
+    from archive import digest, verify_archive
+
+    manifest = verify_archive(backup)
+    deployment_files = {
+        relative: digest(ROOT / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    if deployment_files != manifest.get("deployment_files"):
+        raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
+    rendered = json.loads(compose("config", "--format", "json", capture=True))
+    configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
+    plan = image_restore_plan(manifest, configured)
+    for image_id, _reference in plan.values():
+        actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
+        if actual != image_id:
+            raise ValueError(f"previous image is unavailable: {image_id}")
+
+    backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
+    backup_output = BACKUP_DIR / backup_name
+    compose("stop")
+    try:
+        run(
+            [
+                "python3",
+                str(BACKUP_TOOL),
+                "create",
+                "--compose-file",
+                str(COMPOSE_FILE),
+                "--env-file",
+                str(ENV_FILE),
+                "--output",
+                str(backup_output),
+            ]
+        )
+        run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
+    except Exception:
+        compose("up", "--detach")
+        raise
+
+    for image_id, reference in plan.values():
+        run(["docker", "image", "tag", image_id, reference])
+    compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
+    run(
+        [
+            "curl",
+            "--fail",
+            "--silent",
+            "--show-error",
+            "--resolve",
+            "akyldash.kg:443:127.0.0.1",
+            "https://akyldash.kg/review",
+            "--output",
+            "/dev/null",
+        ]
+    )
+    record_backend_image()
+    print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("backup", type=Path)
+    parser.add_argument("--data-compatible", action="store_true")
+    args = parser.parse_args()
+    try:
+        if os.geteuid() != 0:
+            raise ValueError("rollback must run through the restricted root helper")
+        rollback(args.backup, args.data_compatible)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,36 @@
+import unittest
+
+from rollback import image_restore_plan
+
+
+class RollbackPlanTests(unittest.TestCase):
+    def setUp(self):
+        self.manifest = {
+            "images": {
+                "backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
+                "opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
+            }
+        }
+        self.configured = {
+            "backend": "akyldash/backend:local",
+            "opensearch": "akyldash/opensearch:local",
+        }
+
+    def test_plans_only_recorded_local_images(self):
+        plan = image_restore_plan(self.manifest, self.configured)
+        self.assertEqual(plan["backend"][1], "akyldash/backend:local")
+        self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local")
+
+    def test_refuses_changed_compose_image_reference(self):
+        with self.assertRaisesRegex(ValueError, "reference changed"):
+            image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"})
+
+    def test_refuses_digest_reference_that_cannot_be_retagged(self):
+        self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64
+        self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
+        with self.assertRaisesRegex(ValueError, "local tag"):
+            image_restore_plan(self.manifest, self.configured)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,90 @@
+project: akyldash
+environment: production
+ssh:
+  alias: akyldash_vps
+  host: 206.223.249.118
+  port: 22
+  user: codex
+  identity_file: ~/.ssh/akyldash_codex
+domain: akyldash.kg
+facts:
+  hostname: avprogs.hlab.kz
+  os: Ubuntu Server 24.04.5 LTS
+  kernel: 6.8.0-142-generic
+  cpu_vcpus: 4
+  memory_gib: 7.7
+  swap_gib: 3.8
+  disk_gib: 99
+  disk_free_gib: 93
+  network_interface: ens3
+  ipv4: 206.223.249.118/32
+  domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses
+  domain_http_status: Cloudflare 521 on HTTP and HTTPS from the VPS
+  user_groups: [codex, users]
+  sudo_noninteractive: false
+  installed:
+    - OpenSSH server
+    - systemd-resolved
+    - unattended-upgrades
+  absent:
+    - Docker Engine
+    - Docker Compose plugin
+    - Git
+    - reverse proxy
+  access_audit_date: 2026-10-11
+target:
+  compose_file_in_repository: deploy/production/compose.yaml
+  compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml
+  production_env_file: /etc/akyldash/production.env
+  secrets_directory: /etc/akyldash
+  data_directory: /var/lib/akyldash/data
+  backup_directory: /var/backups/akyldash
+  services:
+    caddy:
+      container: compose project service caddy
+      image: caddy:2.11.6-alpine pinned by digest in deploy/production/compose.yaml
+      host_ports: [80, 443]
+      persistent_data: named Compose volumes caddy-data and caddy-config
+    backend:
+      container: compose project service backend
+      image: built from deploy/production/Dockerfile.backend; digest deployment not configured
+      port: 8080 on private proxy and search Compose networks only
+      healthcheck: HTTP GET /review on container port 8080
+      persistent_data: /var/lib/akyldash/data (bind mount)
+    opensearch:
+      container: compose project service opensearch
+      image: opensearchproject/opensearch:3.7.0 plus analysis-icu plugin
+      port: 9200 on private Compose network only
+      persistent_data: named Compose volume opensearch-data
+  reverse_proxy:
+    target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080
+    status: repository configuration exists; not installed or running on VPS
+  external_ports_target: [22, 80, 443]
+  compose_networks:
+    proxy: caddy and backend
+    search: backend and OpenSearch; internal network
+  secrets:
+    - REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
+    - Gitea credentials from environment-backed gitea-api helper
+    - SSH key at ~/.ssh/akyldash_codex
+  restricted_helper:
+    path: /usr/local/sbin/akyldash-rollback
+    status: repository code and owner-run installer exist; not installed on VPS
+  read_only_verifier:
+    path: /usr/local/sbin/akyldash-verify
+    status: repository code and owner-run installer exist; not installed on VPS
+  checks:
+    - infra/bootstrap/verify.sh akyldash_vps
+    - docker compose -f deploy/production/compose.yaml ps
+    - curl -fsS https://akyldash.kg/review
+    - python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
+  backup_path: /var/backups/akyldash (target; archive script in infra/backup/archive.py is not installed or run)
+  deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment)
+versions:
+  application:
+    backend: 0.11.3
+    frontend: 0.1.2
+    telegram_bot: 0.2.2
+  infrastructure: unknown; no production install or release exists
+  caddy_image: 2.11.6-alpine pinned by manifest digest in Compose
+  docker_engine: not installed