From d03da40eeeab207d3800cbfb1b3719b432ef9b3e Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:26:48 +0300
Subject: [PATCH 01/17] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?=
 =?UTF-8?q?=D1=82=D1=8C=20=D0=B2=D0=BE=D1=81=D0=BF=D1=80=D0=BE=D0=B8=D0=B7?=
 =?UTF-8?q?=D0=B2=D0=BE=D0=B4=D0=B8=D0=BC=D1=83=D1=8E=20=D0=BE=D1=81=D0=BD?=
 =?UTF-8?q?=D0=BE=D0=B2=D1=83=20=D0=B8=D0=BD=D1=84=D1=80=D0=B0=D1=81=D1=82?=
 =?UTF-8?q?=D1=80=D1=83=D0=BA=D1=82=D1=83=D1=80=D1=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitea/workflows/ci.yaml                  |  20 ++++
 AGENTS.md                                 |  23 +++++
 CHANGELOG.md                              |   1 +
 README.md                                 |   6 +-
 backend/README.md                         |   4 +-
 backend/search/api.py                     |  10 +-
 backend/search/minjust_opensearch.py      |   2 +-
 backend/search/review.html                |  10 +-
 backend/test_search_api.py                |  23 +++++
 deploy/production/.env.example            |   5 +-
 deploy/production/README.md               |  11 ++-
 deploy/production/compose.yaml            |  23 ++++-
 docs/README.md                            |   6 +-
 docs/operations/ACCESS.md                 |  58 +++++++++++
 docs/operations/BACKUP.md                 |  28 ++++++
 docs/operations/CURRENT_STATE.md          |  61 ++++++++++++
 docs/operations/DECISIONS.md              |   9 ++
 docs/operations/DEPLOYMENT.md             |  37 +++++++
 docs/operations/INFRASTRUCTURE.md         |  56 +++++++++++
 docs/operations/RECOVERY.md               |  33 +++++++
 docs/operations/ROLLBACK.md               |  17 ++++
 docs/operations/project-status.md         |   4 +-
 docs/product/frontend-search-sps-plan.md  |   2 +-
 docs/product/project-overview.md          |   2 +-
 infra/bootstrap/bootstrap-ubuntu-24.04.sh |  52 ++++++++++
 infra/bootstrap/verify.sh                 | 114 ++++++++++++++++++++++
 infra/inventory/production.yml            |  72 ++++++++++++++
 27 files changed, 664 insertions(+), 25 deletions(-)
 create mode 100644 .gitea/workflows/ci.yaml
 create mode 100644 AGENTS.md
 create mode 100644 docs/operations/ACCESS.md
 create mode 100644 docs/operations/BACKUP.md
 create mode 100644 docs/operations/CURRENT_STATE.md
 create mode 100644 docs/operations/DECISIONS.md
 create mode 100644 docs/operations/DEPLOYMENT.md
 create mode 100644 docs/operations/INFRASTRUCTURE.md
 create mode 100644 docs/operations/RECOVERY.md
 create mode 100644 docs/operations/ROLLBACK.md
 create mode 100755 infra/bootstrap/bootstrap-ubuntu-24.04.sh
 create mode 100755 infra/bootstrap/verify.sh
 create mode 100644 infra/inventory/production.yml

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
new file mode 100644
index 0000000..ead6a6e
--- /dev/null
+++ b/.gitea/workflows/ci.yaml
@@ -0,0 +1,20 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  tests-and-images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run backend tests
+        run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
+      - name: Run Telegram bot tests
+        run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
+      - name: Build production images
+        run: |
+          docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci .
+          docker build -f deploy/production/Dockerfile.opensearch -t akyldash-opensearch:ci .
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..4049606
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,23 @@
+# Акылдаш — инструкции для агента
+
+Проект: юридическая информационно-аналитическая платформа; Search API на
+Python, SQLite и OpenSearch. Production пока не развернут.
+
+- Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и
+  [правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно.
+- Архитектура и процедуры: [инфраструктура](docs/operations/INFRASTRUCTURE.md),
+  [деплой](docs/operations/DEPLOYMENT.md), [backup](docs/operations/BACKUP.md),
+  [rollback](docs/operations/ROLLBACK.md), [восстановление](docs/operations/RECOVERY.md).
+- Работай в feature/fix/refactor-ветке. `main` защищена; изменения вноси через
+  PR, не сливай его автоматически. Сохраняй незакоммиченные изменения.
+- SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай
+  `codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
+  Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
+- Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`.
+  Production images собираются командами из `.gitea/workflows/ci.yaml`.
+- Перед изменениями production необходим проверенный backup. Формального
+  механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
+  SSH, firewall или production вручную без владельца и последующего PR.
+- После каждого изменения инфраструктуры обновляй
+  `docs/operations/CURRENT_STATE.md` по фактическим проверкам.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9ba8ac..ee7ec4f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Не выпущено
 
+- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, drift-проверка и CI для тестов и сборки образов; production-деплой остаётся выключен до проверки runner, backup/restore и ограниченного deploy-helper.
 - Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
 - Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
 - Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
diff --git a/README.md b/README.md
index 89fe752..58ab1d4 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ Telegram-бот — только часть рабочего окружения
 ## Текущее состояние
 
 Сейчас реализованы Telegram-бот-секретарь версии `0.2.2`, backend версии
-`0.11.2` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
+`0.11.3` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
 SQLite; первый пользователь `admin` создаётся интерактивной CLI-командой.
 Поиск расширяет аббревиатуры организационно-правовых форм и поднимает
 действующий Налоговый кодекс для запросов о налоге на прибыль.
@@ -18,7 +18,7 @@ SQLite; первый пользователь `admin` создаётся инт
 | Компонент | Версия | Состояние |
 |---|---:|---|
 | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
-| Backend | `0.11.2` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
+| Backend | `0.11.3` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
 | Frontend | `0.1.2` | лаборатория поиска, личный кабинет и управление учетными записями |
 | Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи |
 | RAG и база знаний | — | ещё не созданы |
@@ -62,4 +62,4 @@ python3 -m unittest discover -s tools/telegram-bot -v
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
diff --git a/backend/README.md b/backend/README.md
index fb4bb2d..7d03fb0 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -1,6 +1,6 @@
 # Backend Акылдаш
 
-Версия: `0.11.2`
+Версия: `0.11.3`
 
 Первая backend-область проекта — загрузка правовых документов из официального
 Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
@@ -253,4 +253,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la
 
 ---
 
-Акылдаш · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Backend v0.11.3 · Frontend v0.1.2
diff --git a/backend/search/api.py b/backend/search/api.py
index df96ae6..37337ef 100644
--- a/backend/search/api.py
+++ b/backend/search/api.py
@@ -451,6 +451,7 @@ def main() -> int:
     parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
     parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
     parser.add_argument("--review-secret", default=None)
+    parser.add_argument("--review-secret-file", type=Path)
     parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
     parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
     parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
@@ -458,7 +459,14 @@ def main() -> int:
     parser.add_argument("--port", type=int, default=8080)
     parser.add_argument("--version", action="version", version=APP_VERSION)
     arguments = parser.parse_args()
-    secret = arguments.review_secret.encode() if arguments.review_secret else None
+    if arguments.review_secret and arguments.review_secret_file:
+        parser.error("choose either --review-secret or --review-secret-file")
+    try:
+        secret = arguments.review_secret.encode() if arguments.review_secret else arguments.review_secret_file.read_bytes().strip() if arguments.review_secret_file else None
+    except OSError as error:
+        parser.error(f"cannot read review secret file: {error}")
+    if arguments.review_secret_file and not secret:
+        parser.error("review secret file is empty")
     api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
     if arguments.create_admin or arguments.reset_admin_password:
         users = api.auth_store.list_users()
diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py
index 8af8236..f62a681 100644
--- a/backend/search/minjust_opensearch.py
+++ b/backend/search/minjust_opensearch.py
@@ -17,7 +17,7 @@ from typing import Iterator
 
 from search.catalog import authority_codes, legal_force_code, source_code
 
-APP_VERSION = "0.11.2"
+APP_VERSION = "0.11.3"
 LANGUAGES = {"ru", "ky"}
 DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
 
diff --git a/backend/search/review.html b/backend/search/review.html
index 97bd25c..5817921 100644
--- a/backend/search/review.html
+++ b/backend/search/review.html
@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>Оценка поисковой выдачи · Акылдаш</title>
   <style>
-    :root { color-scheme: light; font: 16px/1.5 "Noto Sans", "Segoe UI", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", Georgia, serif; }
+    :root { color-scheme: light; font: 16px/1.5 "Noto Sans", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", serif; }
     * { box-sizing: border-box; }
     body { margin: 0; color: var(--ink); background: #F7F2E8; }
     .review__skip { position: absolute; inset-block-start: 0; inset-inline-start: -10000px; padding: .5rem 1rem; background: var(--surface); }
@@ -53,7 +53,7 @@
     .review__comment { min-height: 4rem; resize: vertical; background: var(--surface); }
     .review__document { position: sticky; inset-block-start: 1rem; min-height: 24rem; }
     .review__document-meta { margin-block-end: 1rem; padding-block-end: 1rem; border-block-end: 1px solid var(--line); color: var(--muted); }
-    .review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
+    .review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
     .review__document-body img { max-width: 100%; height: auto; }
     .review__actions { max-width: 1320px; margin: 1.5rem auto 0; padding-inline: clamp(1rem, 3vw, 3rem); }
     .review__actions-inner { display: flex; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
@@ -115,7 +115,7 @@
     .search__document { min-height: 60vh; }
     .search__document-controls { display: flex; flex-wrap: wrap; gap: .75rem; }
     .search__document-controls label { display: grid; gap: .25rem; min-width: 11rem; color: var(--muted); font-size: .875rem; }
-    .search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
+    .search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
     .search__document-body h1, .search__document-body h2, .search__document-body h3 { font-family: var(--serif); }
     .search__document-body img { max-width: 100%; height: auto; }
     .search__sr-only { position: absolute; inline-size: 1px; block-size: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
@@ -176,7 +176,7 @@
   <dialog id="users-dialog" aria-labelledby="users-heading"><section class="review__page"><h2 id="users-heading">Управление пользователями</h2><form id="create-user-form"><label>Логин<input id="new-username" autocomplete="off" minlength="3" maxlength="64" required></label><label>Временный пароль<input id="temporary-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p>Пароль — от 12 символов. Новый пользователь сменит его при первом входе. Передайте временный пароль отдельно.</p><p id="users-error" role="alert" class="review__status review__status--error"></p><button class="review__button--primary" type="submit">Создать пользователя</button></form><ul id="user-list" class="review__user-list"></ul><button type="button" data-close="users-dialog">Закрыть</button></section></dialog>
   <dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
   <div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
-  <footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.2 · Внутренняя лаборатория релевантности</div></footer>
+  <footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.3 · Внутренняя лаборатория релевантности</div></footer>
   <script>
     document.body.classList.add('review__locked');
     const DRAFT_KEY = 'akyldash-search-review-draft';
@@ -193,7 +193,7 @@
     let account = null;
     const reviewRoute = location.pathname === '/review';
     $('review').hidden = !reviewRoute; $('search-page').hidden = reviewRoute; $('review-nav').classList.toggle('is-active', reviewRoute); $('search-nav').classList.toggle('is-active', !reviewRoute);
-    if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.2'; }
+    if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.3'; }
     async function apiFetch(path, options = {}) { const response = await fetch(path, {credentials: 'same-origin', ...options}); if (response.status === 401 && path !== '/auth/login') { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); } return response; }
     const showLogin = () => { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); };
     const enterWorkspace = (user) => { account = user; $('signed-in-user').textContent = `${user.username} · ${user.role}`; $('reviewer').value = user.username; $('profile-username').textContent = user.username; $('profile-role').textContent = user.role; $('profile-created').textContent = new Date(user.created_at * 1000).toLocaleDateString('ru-RU'); $('users-open').hidden = user.username.toLowerCase() !== 'admin'; document.body.classList.remove('review__locked'); if (!reviewRoute) initSearchPage(); else if (!hasSeenTour()) setTimeout(openTutorial, 350); };
diff --git a/backend/test_search_api.py b/backend/test_search_api.py
index 41c4267..93c3606 100644
--- a/backend/test_search_api.py
+++ b/backend/test_search_api.py
@@ -1,4 +1,6 @@
 import json
+import contextlib
+import io
 import tempfile
 import unittest
 from pathlib import Path
@@ -36,6 +38,27 @@ class SearchApiTest(unittest.TestCase):
             with self.assertRaisesRegex(ApiError, "within 10000 results"):
                 api.handle("GET", "/search?q=x&page=100&page_size=100")
 
+    def test_review_secret_file_is_read_without_command_line_value(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.write_text("secret-value\n", encoding="utf-8")
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), patch("search.api.Api") as api, patch("search.api.ThreadingHTTPServer"):
+                main()
+            self.assertEqual(api.call_args.args[4], b"secret-value")
+
+    def test_review_secret_file_rejects_empty_content(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.touch()
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), contextlib.redirect_stderr(io.StringIO()):
+                with self.assertRaises(SystemExit) as result:
+                    main()
+            self.assertEqual(result.exception.code, 2)
+
     def test_production_api_uses_registration_intent_boosts(self):
         with tempfile.TemporaryDirectory() as temporary:
             api = self.make_api(Path(temporary))
diff --git a/deploy/production/.env.example b/deploy/production/.env.example
index 3d785e5..2840663 100644
--- a/deploy/production/.env.example
+++ b/deploy/production/.env.example
@@ -1,9 +1,8 @@
 # Absolute path on the production host containing minjust-normalized/.
-DATA_ROOT=/volume1/docker/akyldash/data
+DATA_ROOT=/var/lib/akyldash/data
+REVIEW_SECRET_FILE=/etc/akyldash/review_secret
 BACKEND_PORT=8080
 SEARCH_INDEX=akyldash-fragments-current
 OPENSEARCH_MEM_LIMIT=4g
 OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
-# Generate with: openssl rand -hex 32
-REVIEW_SECRET=replace-with-a-random-secret
 COOKIE_SECURE=true
diff --git a/deploy/production/README.md b/deploy/production/README.md
index ea68e44..aef98ec 100644
--- a/deploy/production/README.md
+++ b/deploy/production/README.md
@@ -12,8 +12,15 @@ workspace; it does not replace host-level network controls or TLS.
 ## First deployment
 
 1. Copy this directory to the host with the repository source.
-2. Copy `.env.example` to `.env`, set an absolute `DATA_ROOT`, and replace
-   `REVIEW_SECRET` with a random value. Do not commit `.env`.
+2. Copy `.env.example` to `.env`, set an absolute `DATA_ROOT`, and create the
+   secret file referenced by `REVIEW_SECRET_FILE` outside the repository:
+
+   ```bash
+   sudo install -d -o root -g root -m 0700 /etc/akyldash
+   sudo sh -c 'umask 077; openssl rand -hex 32 > /etc/akyldash/review_secret'
+   ```
+
+   Do not commit `.env` or the secret file.
 3. Put the normalized dataset under `$DATA_ROOT/minjust-normalized/`.
 4. Check the rendered configuration:
 
diff --git a/deploy/production/compose.yaml b/deploy/production/compose.yaml
index 6f07fcd..c510ebe 100644
--- a/deploy/production/compose.yaml
+++ b/deploy/production/compose.yaml
@@ -1,9 +1,16 @@
+name: akyldash
+
 services:
   opensearch:
     build:
       context: ../..
       dockerfile: deploy/production/Dockerfile.opensearch
     restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
     environment:
       discovery.type: single-node
       bootstrap.memory_lock: "true"
@@ -32,8 +39,12 @@ services:
       context: ../..
       dockerfile: deploy/production/Dockerfile.backend
     restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
     environment:
-      REVIEW_SECRET: ${REVIEW_SECRET:?set REVIEW_SECRET in .env}
       COOKIE_SECURE: ${COOKIE_SECURE:-true}
     command:
       - python
@@ -53,8 +64,10 @@ services:
       - /app/data/search-reviews.sqlite3
       - --users-db
       - /app/data/search-users.sqlite3
-      - --review-secret
-      - ${REVIEW_SECRET}
+      - --review-secret-file
+      - /run/secrets/review_secret
+    secrets:
+      - review_secret
     ports:
       - "127.0.0.1:${BACKEND_PORT:-8080}:8080"
     depends_on:
@@ -65,3 +78,7 @@ services:
 
 volumes:
   opensearch-data:
+
+secrets:
+  review_secret:
+    file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}
diff --git a/docs/README.md b/docs/README.md
index 00088b2..ee88ac1 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -26,6 +26,10 @@
 
 - [Текущий статус](operations/project-status.md) — выполненные шаги, риски и
   ближайшие действия.
+- [Состояние инфраструктуры](operations/CURRENT_STATE.md) — подтверждённое
+  состояние production и оставшиеся проверки.
+- [Инфраструктура и эксплуатация](operations/INFRASTRUCTURE.md) — схема,
+  доступы, deployment, backup, rollback и recovery.
 - [План Telegram-пространства](operations/telegram-workspace-plan.md) — темы,
   роли и этапы развития командного окружения.
 
@@ -41,4 +45,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md
new file mode 100644
index 0000000..b3458e7
--- /dev/null
+++ b/docs/operations/ACCESS.md
@@ -0,0 +1,58 @@
+# Infrastructure access
+
+## Discover existing access first
+
+In a new Codex session, inspect the SSH alias and key file metadata without
+printing key contents, then check the connection:
+
+```bash
+ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }'
+test -r ~/.ssh/akyldash_codex && echo 'SSH key is available'
+ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps 'id; hostname'
+gitea-api GET /version
+gitea-api GET /repos/admin/akyldash
+```
+
+Do not ask for the host, login, password, token, or key again before these
+checks. Never print a private key, token, or complete environment file.
+
+## VPS
+
+- SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`.
+- SSH configuration: `~/.ssh/config`.
+- Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents).
+- `codex` can run unprivileged read-only diagnostics. The audited account has no
+  non-interactive sudo and must not be added to `sudo` or the `docker` group.
+- Host package installation, service setup, firewall changes, secret creation,
+  backup restore, and production changes require the VPS owner to run a reviewed
+  root command. No restricted deploy-helper is installed yet.
+- Never connect to other VPS hosts or Synology as part of this deployment.
+
+This execution environment's system SSH config include had unsafe file
+permissions, so plain `ssh akyldash_vps` failed. `ssh -F ~/.ssh/config ...`
+worked. Network policy may also require an approved SSH command. If the key is
+missing in a future execution environment, report that limitation; do not
+retrieve it from another machine.
+
+## Gitea
+
+- Repository: `https://git.vakanaut.ru/admin/akyldash`.
+- Use the installed `gitea-api` helper; it reads credentials from the
+  environment. Never inspect local credential files, print environment values,
+  or construct HTTP authorization headers.
+- The audited API token can read/write this repository and create issues, but
+  does not have `read:user` or `read:package`. An API `403` caused by a missing
+  scope is not evidence that the account itself lacks the permission.
+- Protected `main` requires one approval. Do not push directly to `main`, merge
+  automatically, or start a review. Work on a feature branch and use a PR.
+- Gitea Actions and package features are enabled for the repository, but runner
+  availability and package publishing permissions have not been confirmed.
+
+## Who can do what
+
+| Role | Operations |
+|---|---|
+| `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners |
+| Restricted deploy-helper | Intended future allowlisted backup/deploy/status actions; not installed |
+| VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
+| Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
new file mode 100644
index 0000000..d0759b0
--- /dev/null
+++ b/docs/operations/BACKUP.md
@@ -0,0 +1,28 @@
+# Backup and restore
+
+## Current status
+
+No production data or backup mechanism existed during the 2026-10-11 audit.
+`/var/backups/akyldash` is a planned root-only location, not an active backup.
+Do not report backup protection as enabled until an archive and restore have
+both been checked.
+
+## Required release policy
+
+- Create a backup before every production change, including any future schema
+  or index migration.
+- Include the normalized corpus, `search-users.sqlite3`,
+  `search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
+  and required secret file in the protected backup set. Store secrets separately
+  with equivalent access controls.
+- Verify archive readability, checksums, and SQLite integrity before
+  replacing the previous known-good backup.
+- Keep one current pre-release backup. If deployment fails, retain the backup
+  required to restore the previous state.
+- Test restore into an isolated target before relying on the backup. Do not
+  restore over production as a test.
+
+The data is not yet managed by a tested backup script. Until that exists, the
+owner must pause deployment and approve a maintenance window for a reviewed
+backup/restore procedure. Copying live SQLite files with `cp` or an unchecked
+tar archive is not an approved backup.
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
new file mode 100644
index 0000000..64465fd
--- /dev/null
+++ b/docs/operations/CURRENT_STATE.md
@@ -0,0 +1,61 @@
+# Infrastructure current state
+
+Last audit: 2026-10-11. Source: read-only SSH session as `codex`, local
+repository inspection, and Gitea API. This document separates measured facts
+from planned configuration.
+
+## VPS facts
+
+- Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`.
+- OS/kernel: Ubuntu Server `24.04.5 LTS`, `6.8.0-142-generic`.
+- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 93 GiB free.
+- Network: interface `ens3`, IPv4 `206.223.249.118/32`.
+- `codex` is UID/GID 1000, groups `codex` and `users`; non-interactive sudo is
+  unavailable.
+- SSH service is running. The inspected sshd configuration has public-key and
+  password authentication enabled and `PermitRootLogin yes`; no SSH changes
+  were made.
+- Docker, Docker Compose, Git, Nginx, Caddy, Certbot, application paths, and
+  application services are absent. Only base OS services and SSH were running.
+- Public listeners observed: TCP `22`; local resolver listeners on loopback.
+- UFW is reported active in the supplied VPS notes, but the audit account could
+  not independently read its status. Treat firewall state and rules as
+  unverified.
+
+## Application and Gitea facts
+
+- Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`.
+- The repository API reports push permission, Actions and packages enabled, and
+  PRs enabled. `main` is protected and requires one approval; the current API
+  principal cannot push or merge directly to `main`.
+- The Gitea token lacks `read:user` and `read:package`; runner status and package
+  publishing permissions could not be checked. No workflow existed at audit.
+- Production Compose describes Python Search API `0.11.3`, SQLite account and
+  review data, and OpenSearch `3.7.0` with `analysis-icu`. Nothing is running.
+- Compose binds the API to `127.0.0.1:8080`, leaves OpenSearch private, uses a
+  named OpenSearch volume, and a host data bind mount. TLS/reverse proxy,
+  registry digest deployment, backup, rollback helper, and systemd deployment
+  are not configured.
+- The app has no migration framework or SMTP setup. Current app versions:
+  backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
+  deployed on this VPS).
+
+## Checks not completed
+
+- SSH completed with the user config explicitly selected; plain SSH was blocked
+  by permissions on a system config include. The private key was not read.
+- UFW rules require root and remain unverified.
+- DNS delegation, HTTPS, Cloudflare Full (strict), public application health,
+  Actions runner availability, registry publishing, and production deployment
+  have not been tested.
+- No packages or services were installed or changed on the VPS.
+
+## Next owner actions
+
+1. Review and run the root bootstrap on the dedicated host.
+2. Review SSH/password/root policy and firewall rules separately; this change
+   did not touch them.
+3. Set up DNS, origin certificate, reverse proxy, protected secret files, and
+   initial corpus/index.
+4. Confirm a Gitea Actions runner and package permissions, then test CI and
+   backup/restore before considering automated production deploy.
diff --git a/docs/operations/DECISIONS.md b/docs/operations/DECISIONS.md
new file mode 100644
index 0000000..3785da3
--- /dev/null
+++ b/docs/operations/DECISIONS.md
@@ -0,0 +1,9 @@
+# Infrastructure decisions
+
+| Date | Decision | Reason and alternatives | Status / limits |
+|---|---|---|---|
+| 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
+| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; use an owner-run bootstrap and a narrowly reviewed deploy-helper | No helper installed yet |
+| 2026-10-11 | Keep Search API on loopback and OpenSearch on the private Compose network | Minimizes host exposure; HTTPS terminates at a reviewed reverse proxy | Compose target only; no proxy or firewall audit privileges |
+| 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
+| 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
new file mode 100644
index 0000000..171d0bc
--- /dev/null
+++ b/docs/operations/DEPLOYMENT.md
@@ -0,0 +1,37 @@
+# Deployment
+
+## Current status
+
+Production deployment is not enabled or tested. The VPS has no Docker, Git,
+application data, proxy, registry pull configuration, backup process, or
+restricted deploy-helper. Do not deploy until the owner has completed the root
+bootstrap, provided a protected secret file and a reviewed proxy/firewall
+configuration, and a backup and restore have been tested.
+
+## First host preparation
+
+1. Review the current inventory and state, then have the owner run
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh` from a verified copy
+   of this branch on the VPS. The script installs Docker Engine/Compose and Git,
+   sets Docker log rotation, and creates root-owned paths. It refuses other OS
+   versions and does not change SSH, UFW, sudoers, or user groups.
+2. Install the repository checkout at `/opt/akyldash/current` and configure
+   `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
+   Git. The secret file must be root-readable only.
+3. Review DNS and certificate issuance, choose and configure the reverse proxy,
+   then ask the owner to open only the required ports after confirming current
+   firewall rules.
+4. Create and verify the first backup before starting services. Load and
+   validate the OpenSearch index, create the initial admin interactively, and
+   verify `/review` over HTTPS.
+
+The remaining steps are intentionally manual until a restricted deploy-helper,
+registry permissions, a live Gitea runner, and the backup path are tested.
+
+## CI
+
+`.gitea/workflows/ci.yaml` runs the Python unit suites and builds both
+production images on pull requests and pushes to `main`. It does not publish an
+image or deploy production. Gitea Actions is enabled at repository level, but a
+successful workflow run has not yet confirmed that a compatible runner is
+online.
diff --git a/docs/operations/INFRASTRUCTURE.md b/docs/operations/INFRASTRUCTURE.md
new file mode 100644
index 0000000..8aafd4d
--- /dev/null
+++ b/docs/operations/INFRASTRUCTURE.md
@@ -0,0 +1,56 @@
+# Infrastructure
+
+## Intended layout
+
+Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
+`deploy/production/compose.yaml` defines a Python Search API and private
+OpenSearch 3.7 node. The API binds to loopback port `8080`; OpenSearch is only
+exposed to the Compose network. The host reverse proxy should terminate HTTPS
+for `akyldash.kg` and proxy to the API. Cloudflare should use Full (strict) only
+after the origin certificate and DNS path are verified.
+
+The API stores its account and review SQLite files beside the normalized corpus
+under the persistent data directory. OpenSearch uses a named volume. The API's
+review signing key is mounted from a host file; it is not passed in process
+arguments or committed to Git. Docker logs are bounded to 10 MiB per file and
+three files per service.
+
+## Actual state
+
+The 2026-10-11 read-only audit found a fresh Ubuntu 24.04.5 host with SSH only.
+Docker, Git, a reverse proxy, application directories, Compose services, TLS,
+and backups are not installed/configured. The `codex` account has no
+non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
+[production inventory](../../infra/inventory/production.yml).
+
+The Compose file is a repository baseline, not proof of deployment. It still
+builds images locally; immutable registry images and automated deploy are not
+configured. No migration framework exists. The application uses SQLite and
+OpenSearch; SMTP registration/recovery is not implemented.
+
+## Repository paths and target host paths
+
+- Compose and Dockerfiles: `deploy/production/`.
+- Host checkout target: `/opt/akyldash/current`.
+- Environment configuration: `/etc/akyldash/production.env`.
+- Secret file: `/etc/akyldash/review_secret`, mode `0600`.
+- Application data: `/var/lib/akyldash/data`.
+- Intended backups: `/var/backups/akyldash`.
+
+All host paths above are targets; none existed at audit time. Root bootstrap is
+prepared at [bootstrap-ubuntu-24.04.sh](../../infra/bootstrap/bootstrap-ubuntu-24.04.sh).
+It does not edit SSH or firewall configuration, create credentials, deploy the
+application, or grant `codex` Docker/root access.
+
+## Network and operations
+
+Target public TCP listeners are SSH `22` and HTTPS proxy `80`/`443`. Backend
+`8080` must remain loopback-only and OpenSearch `9200` private to Compose. The
+firewall state could not be independently verified because the SSH account has
+no sudo; do not change it without owner approval. DNS, HTTPS, Cloudflare origin
+mode, and production health have not been verified.
+
+Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
+[ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
+limits and procedures. Configuration changes must go through Git and review;
+do not reconcile a Git/server difference by overwriting the VPS automatically.
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
new file mode 100644
index 0000000..c30f7f1
--- /dev/null
+++ b/docs/operations/RECOVERY.md
@@ -0,0 +1,33 @@
+# Recovery on a clean VPS
+
+## Scope and safety
+
+The bootstrap below prepares Ubuntu packages and directories only. It is not a
+complete recovery: production secrets, data, reverse proxy, TLS, registry
+credentials, backup restore, and deploy-helper still require owner-controlled
+setup. Run only on the new recovery host after verifying its identity. Never
+run recovery commands against the existing production by accident.
+
+## Procedure
+
+1. Provision a clean Ubuntu Server 24.04 host and confirm hostname, IP, console
+   access, and a tested owner recovery path.
+2. Create the SSH-authorized `codex` account without sudo or Docker group access.
+   Preserve the host's existing root SSH and firewall policy until an owner has
+   reviewed the change.
+3. Check out the approved Git revision and run
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh`. It installs Docker
+   Engine, Compose, Git, Docker log limits, and target directories.
+4. Install the reviewed reverse proxy and its HTTPS configuration after DNS
+   points to the recovery host. Verify Cloudflare Full (strict) end to end.
+5. Provide production configuration and secret files through the owner's
+   protected channel; never commit or paste secret values into chat.
+6. Restore the latest verified backup to the new host. Validate SQLite
+   databases, corpus checksums, and the OpenSearch snapshot/index before serving
+   traffic.
+7. Start the approved image digest, verify container health and
+   `https://akyldash.kg/review`, then record actual versions and checks in
+   [CURRENT_STATE.md](CURRENT_STATE.md).
+
+Do not repoint DNS or retire the old host until the owner verifies recovery and
+approves the cutover. The restore and cutover have not been practiced yet.
diff --git a/docs/operations/ROLLBACK.md b/docs/operations/ROLLBACK.md
new file mode 100644
index 0000000..8d0a83f
--- /dev/null
+++ b/docs/operations/ROLLBACK.md
@@ -0,0 +1,17 @@
+# Rollback
+
+## Current status
+
+There is no production release, pinned image digest, backup, or deploy-helper
+to roll back. Automatic rollback is not configured.
+
+After deployment automation is reviewed, rollback may switch the backend to a
+previous immutable image digest only after health checks fail and only when the
+data format remains compatible. This application currently has no formal
+migration framework, but its SQLite files and OpenSearch indexes are persistent;
+compatibility must be checked for each change. If data compatibility is
+uncertain, stop and report rather than restoring or deleting production data.
+
+Any data restore requires the owner, a verified pre-change backup, and a
+maintenance window. Preserve the failed release and its backup until recovery
+is confirmed.
diff --git a/docs/operations/project-status.md b/docs/operations/project-status.md
index e831a78..f515fef 100644
--- a/docs/operations/project-status.md
+++ b/docs/operations/project-status.md
@@ -5,7 +5,7 @@
 
 - Telegram-бот: `0.2.2`
 - Telegram-бот на Synology: `0.2.1`
-- Backend: `0.11.2`
+- Backend: `0.11.3`
 - Frontend: `0.1.2` — поиск, лаборатория релевантности, аккаунты
 
 ## Краткий итог
@@ -281,4 +281,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
diff --git a/docs/product/frontend-search-sps-plan.md b/docs/product/frontend-search-sps-plan.md
index 7238b71..ff1e880 100644
--- a/docs/product/frontend-search-sps-plan.md
+++ b/docs/product/frontend-search-sps-plan.md
@@ -262,4 +262,4 @@ runtime-зависимостями frontend. Регистрация в стор
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.1
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
diff --git a/docs/product/project-overview.md b/docs/product/project-overview.md
index 9537944..ae9e057 100644
--- a/docs/product/project-overview.md
+++ b/docs/product/project-overview.md
@@ -44,4 +44,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
diff --git a/infra/bootstrap/bootstrap-ubuntu-24.04.sh b/infra/bootstrap/bootstrap-ubuntu-24.04.sh
new file mode 100755
index 0000000..fd085d2
--- /dev/null
+++ b/infra/bootstrap/bootstrap-ubuntu-24.04.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run this script as root." >&2
+  exit 1
+fi
+
+. /etc/os-release
+if [[ ${ID:-} != ubuntu || ${VERSION_ID:-} != 24.04 ]]; then
+  echo "Expected Ubuntu 24.04; found ${PRETTY_NAME:-unknown}." >&2
+  exit 1
+fi
+
+if ! id codex >/dev/null 2>&1; then
+  echo "Create the SSH-authorized codex account before running bootstrap." >&2
+  exit 1
+fi
+
+if [[ -e /etc/docker/daemon.json ]] && ! python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' /etc/docker/daemon.json; then
+  echo "/etc/docker/daemon.json exists with different log settings; review it manually." >&2
+  exit 1
+fi
+
+install -d -m 0755 /etc/docker
+if [[ ! -e /etc/docker/daemon.json ]]; then
+  cat > /etc/docker/daemon.json <<'EOF'
+{
+  "log-driver": "local",
+  "log-opts": {"max-size": "10m", "max-file": "3"}
+}
+EOF
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y ca-certificates curl git
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
+  "$(dpkg --print-architecture)" "$VERSION_CODENAME" > /etc/apt/sources.list.d/docker.list
+apt-get update
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+install -d -o root -g root -m 0755 /opt/akyldash
+install -d -o root -g root -m 0700 /etc/akyldash
+install -d -o root -g root -m 0750 /var/lib/akyldash/data
+install -d -o root -g root -m 0700 /var/backups/akyldash
+systemctl enable --now docker
+docker compose version
+echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh
new file mode 100755
index 0000000..e80f960
--- /dev/null
+++ b/infra/bootstrap/verify.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${1:-} != --remote ]]; then
+  ssh_alias=${1:-akyldash_vps}
+  exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n bash -s -- --remote' < "$0"
+fi
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run the remote check as root; it is read-only." >&2
+  exit 1
+fi
+
+failed=0
+for package in git docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do
+  if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx 'install ok installed'; then
+    printf 'PASS package %s\n' "$package"
+  else
+    printf 'FAIL package %s missing\n' "$package"
+    failed=1
+  fi
+done
+
+if systemctl is-active --quiet docker; then
+  echo 'PASS Docker service active'
+else
+  echo 'FAIL Docker service inactive'
+  failed=1
+fi
+if docker compose version >/dev/null 2>&1; then
+  echo 'PASS Docker Compose responds'
+else
+  echo 'FAIL Docker Compose unavailable'
+  failed=1
+fi
+
+for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
+  if [[ -d $path ]]; then
+    printf 'PASS directory %s\n' "$path"
+  else
+    printf 'FAIL directory %s missing\n' "$path"
+    failed=1
+  fi
+done
+for path in /etc/akyldash /var/backups/akyldash; do
+  if [[ -d $path && $(stat -c %a "$path") == 700 ]]; then
+    printf 'PASS permissions %s mode 0700\n' "$path"
+  elif [[ -d $path ]]; then
+    printf 'FAIL permissions %s mode %s, expected 0700\n' "$path" "$(stat -c %a "$path")"
+    failed=1
+  fi
+done
+
+if ufw status | grep -q '^Status: active$'; then
+  echo 'PASS UFW active'
+  if ! ufw status verbose | grep -q '^Default: deny (incoming)'; then
+    echo 'FAIL UFW default incoming policy is not deny'
+    failed=1
+  fi
+  while read -r rule action direction; do
+    [[ $action == ALLOW && $direction == IN ]] || continue
+    ports=${rule//\/tcp/}
+    ports=${ports//\/udp/}
+    IFS=, read -ra allowed <<< "$ports"
+    for port in "${allowed[@]}"; do
+      case "$port" in
+        22|80|443) ;;
+        *) printf 'FAIL unexpected UFW allow rule %s\n' "$rule"; failed=1 ;;
+      esac
+    done
+  done < <(ufw status | awk 'NR > 3 && NF >= 3 {print $1, $2, $3}')
+else
+  echo 'FAIL UFW inactive or unavailable'
+  failed=1
+fi
+
+tcp_listeners=$(ss -H -lnt | awk '{print $4}')
+for endpoint in $tcp_listeners; do
+  address=${endpoint%:*}
+  port=${endpoint##*:}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  case "$port" in 22|80|443) ;; *) printf 'FAIL unexpected non-loopback TCP listener %s\n' "$endpoint"; failed=1 ;; esac
+done
+printf 'INFO non-loopback TCP listeners: %s\n' "${tcp_listeners//$'\n'/,}"
+
+udp_listeners=$(ss -H -lnu | awk '{print $5}')
+for endpoint in $udp_listeners; do
+  address=${endpoint%:*}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  printf 'FAIL unexpected non-loopback UDP listener %s\n' "$endpoint"
+  failed=1
+done
+
+if [[ -x /usr/local/sbin/akyldash-deploy ]]; then
+  echo 'PASS restricted deploy helper present'
+else
+  echo 'FAIL restricted deploy helper missing'
+  failed=1
+fi
+if [[ -f /opt/akyldash/deployed-image.txt ]]; then
+  expected=$(cat /opt/akyldash/deployed-image.txt)
+  container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
+  actual=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || true)
+  if [[ $expected == *@sha256:* && $actual == "$expected" ]]; then
+    printf 'PASS running backend image matches %s\n' "$expected"
+  else
+    echo 'FAIL running backend image does not match recorded digest'
+    failed=1
+  fi
+else
+  echo 'FAIL deployed image digest record missing'
+  failed=1
+fi
+exit "$failed"
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
new file mode 100644
index 0000000..1c632e0
--- /dev/null
+++ b/infra/inventory/production.yml
@@ -0,0 +1,72 @@
+project: akyldash
+environment: production
+ssh:
+  alias: akyldash_vps
+  host: 206.223.249.118
+  port: 22
+  user: codex
+  identity_file: ~/.ssh/akyldash_codex
+domain: akyldash.kg
+facts:
+  hostname: avprogs.hlab.kz
+  os: Ubuntu Server 24.04.5 LTS
+  kernel: 6.8.0-142-generic
+  cpu_vcpus: 4
+  memory_gib: 7.7
+  swap_gib: 3.8
+  disk_gib: 99
+  disk_free_gib: 93
+  network_interface: ens3
+  ipv4: 206.223.249.118/32
+  user_groups: [codex, users]
+  sudo_noninteractive: false
+  installed:
+    - OpenSSH server
+    - systemd-resolved
+    - unattended-upgrades
+  absent:
+    - Docker Engine
+    - Docker Compose plugin
+    - Git
+    - reverse proxy
+  access_audit_date: 2026-10-11
+target:
+  compose_file_in_repository: deploy/production/compose.yaml
+  compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml
+  production_env_file: /etc/akyldash/production.env
+  secrets_directory: /etc/akyldash
+  data_directory: /var/lib/akyldash/data
+  backup_directory: /var/backups/akyldash
+  services:
+    backend:
+      container: compose project service backend
+      image: built from deploy/production/Dockerfile.backend; digest deployment not configured
+      host_bind: 127.0.0.1:8080
+      healthcheck: HTTP GET /review on container port 8080
+      persistent_data: /var/lib/akyldash/data (bind mount)
+    opensearch:
+      container: compose project service opensearch
+      image: opensearchproject/opensearch:3.7.0 plus analysis-icu plugin
+      port: 9200 on private Compose network only
+      persistent_data: named Compose volume opensearch-data
+  reverse_proxy:
+    target: Caddy or another reviewed HTTPS proxy on TCP 80/443
+    status: not installed or configured
+  external_ports_target: [22, 80, 443]
+  compose_network: default private Compose network
+  secrets:
+    - REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
+    - Gitea credentials from environment-backed gitea-api helper
+    - SSH key at ~/.ssh/akyldash_codex
+  checks:
+    - infra/bootstrap/verify.sh akyldash_vps
+    - docker compose -f deploy/production/compose.yaml ps
+    - curl -fsS https://akyldash.kg/review
+  backup_path: /var/backups/akyldash (target; mechanism not configured)
+versions:
+  application:
+    backend: 0.11.3
+    frontend: 0.1.2
+    telegram_bot: 0.2.2
+  infrastructure: unknown; no production install or release exists
+  docker_engine: not installed
-- 
2.54.0


From 5c78f7ebb895b8bf3b77257fd0bbcd331d2f9629 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:32:14 +0300
Subject: [PATCH 02/17] =?UTF-8?q?=D0=A3=D1=81=D0=B8=D0=BB=D0=B8=D1=82?=
 =?UTF-8?q?=D1=8C=20read-only=20drift-=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80?=
 =?UTF-8?q?=D0=BA=D1=83=20VPS?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 infra/bootstrap/verify.sh | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh
index e80f960..c5c476f 100755
--- a/infra/bootstrap/verify.sh
+++ b/infra/bootstrap/verify.sh
@@ -33,6 +33,18 @@ else
   echo 'FAIL Docker Compose unavailable'
   failed=1
 fi
+if python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' 2>/dev/null; then
+  echo 'PASS Docker log rotation configuration'
+else
+  echo 'FAIL Docker log rotation configuration differs from repository target'
+  failed=1
+fi
+if id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo 'FAIL codex has sudo or Docker group access'
+  failed=1
+else
+  echo 'PASS codex is outside sudo and docker groups'
+fi
 
 for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
   if [[ -d $path ]]; then
@@ -50,6 +62,32 @@ for path in /etc/akyldash /var/backups/akyldash; do
     failed=1
   fi
 done
+for path in /etc/akyldash/production.env /etc/akyldash/review_secret; do
+  if [[ -f $path && $(stat -c %U:%a "$path") == root:600 ]]; then
+    printf 'PASS protected file %s mode 0600\n' "$path"
+  elif [[ -e $path ]]; then
+    printf 'FAIL protected file %s must be root-owned mode 0600\n' "$path"
+    failed=1
+  else
+    printf 'FAIL protected file %s missing\n' "$path"
+    failed=1
+  fi
+done
+
+checkout=/opt/akyldash/current
+compose_file=$checkout/deploy/production/compose.yaml
+if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
+  echo 'PASS production Compose and Dockerfiles match the checked-out Git revision'
+else
+  echo 'FAIL production Compose or Dockerfile has local drift'
+  failed=1
+fi
+if docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --quiet >/dev/null 2>&1; then
+  echo 'PASS rendered production Compose configuration'
+else
+  echo 'FAIL production Compose configuration cannot be rendered'
+  failed=1
+fi
 
 if ufw status | grep -q '^Status: active$'; then
   echo 'PASS UFW active'
-- 
2.54.0


From 5490b22275ab0f5dc440dbd0205728260c8f87a4 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:35:04 +0300
Subject: [PATCH 03/17] =?UTF-8?q?=D0=97=D0=B0=D1=84=D0=B8=D0=BA=D1=81?=
 =?UTF-8?q?=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D1=84=D0=B0=D0=BA?=
 =?UTF-8?q?=D1=82=D0=B8=D1=87=D0=B5=D1=81=D0=BA=D1=83=D1=8E=20=D0=B4=D0=BE?=
 =?UTF-8?q?=D1=81=D1=82=D1=83=D0=BF=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20produc?=
 =?UTF-8?q?tion=20=D0=B4=D0=BE=D0=BC=D0=B5=D0=BD=D0=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/operations/CURRENT_STATE.md  | 13 +++++++++----
 docs/operations/INFRASTRUCTURE.md |  6 ++++--
 infra/inventory/production.yml    |  2 ++
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 64465fd..98381d1 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -21,6 +21,9 @@ from planned configuration.
 - UFW is reported active in the supplied VPS notes, but the audit account could
   not independently read its status. Treat firewall state and rules as
   unverified.
+- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses from the VPS.
+  Both HTTP and HTTPS return Cloudflare `521`; the origin is currently
+  unreachable through the proxy.
 
 ## Application and Gitea facts
 
@@ -29,7 +32,9 @@ from planned configuration.
   PRs enabled. `main` is protected and requires one approval; the current API
   principal cannot push or merge directly to `main`.
 - The Gitea token lacks `read:user` and `read:package`; runner status and package
-  publishing permissions could not be checked. No workflow existed at audit.
+  publishing permissions could not be checked. This change adds
+  `.gitea/workflows/ci.yaml`; PR #42's run remains `Waiting to run` at the last
+  status check, so a compatible runner is not confirmed.
 - Production Compose describes Python Search API `0.11.3`, SQLite account and
   review data, and OpenSearch `3.7.0` with `analysis-icu`. Nothing is running.
 - Compose binds the API to `127.0.0.1:8080`, leaves OpenSearch private, uses a
@@ -45,9 +50,9 @@ from planned configuration.
 - SSH completed with the user config explicitly selected; plain SSH was blocked
   by permissions on a system config include. The private key was not read.
 - UFW rules require root and remain unverified.
-- DNS delegation, HTTPS, Cloudflare Full (strict), public application health,
-  Actions runner availability, registry publishing, and production deployment
-  have not been tested.
+- The Cloudflare edge response was checked, but origin DNS configuration,
+  Cloudflare Full (strict), certificate validity, Actions runner availability,
+  registry publishing, and production deployment have not been tested.
 - No packages or services were installed or changed on the VPS.
 
 ## Next owner actions
diff --git a/docs/operations/INFRASTRUCTURE.md b/docs/operations/INFRASTRUCTURE.md
index 8aafd4d..5a19667 100644
--- a/docs/operations/INFRASTRUCTURE.md
+++ b/docs/operations/INFRASTRUCTURE.md
@@ -47,8 +47,10 @@ application, or grant `codex` Docker/root access.
 Target public TCP listeners are SSH `22` and HTTPS proxy `80`/`443`. Backend
 `8080` must remain loopback-only and OpenSearch `9200` private to Compose. The
 firewall state could not be independently verified because the SSH account has
-no sudo; do not change it without owner approval. DNS, HTTPS, Cloudflare origin
-mode, and production health have not been verified.
+no sudo; do not change it without owner approval. The domain resolves to
+Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
+unreachable. Cloudflare origin mode, certificate validity, and production
+health have not been verified.
 
 Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
 [ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
index 1c632e0..8f98a1d 100644
--- a/infra/inventory/production.yml
+++ b/infra/inventory/production.yml
@@ -18,6 +18,8 @@ facts:
   disk_free_gib: 93
   network_interface: ens3
   ipv4: 206.223.249.118/32
+  domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses
+  domain_http_status: Cloudflare 521 on HTTP and HTTPS from the VPS
   user_groups: [codex, users]
   sudo_noninteractive: false
   installed:
-- 
2.54.0


From 892781522d82040fed34df5e8f2a5954df8a6d57 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:36:36 +0300
Subject: [PATCH 04/17] =?UTF-8?q?=D0=97=D0=B0=D0=BA=D1=80=D0=B5=D0=BF?=
 =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D1=8E=20Pyt?=
 =?UTF-8?q?hon=20=D0=B2=20CI?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitea/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index ead6a6e..8926c02 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -10,6 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
       - name: Run backend tests
         run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
       - name: Run Telegram bot tests
-- 
2.54.0


From 7124c82bb6485c8913d70af449e6ae509ec44f39 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:37:24 +0300
Subject: [PATCH 05/17] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8?=
 =?UTF-8?q?=D1=82=D1=8C=20=D1=81=D1=82=D0=B0=D1=82=D1=83=D1=81=20=D0=B8?=
 =?UTF-8?q?=D0=BD=D1=84=D1=80=D0=B0=D1=81=D1=82=D1=80=D1=83=D0=BA=D1=82?=
 =?UTF-8?q?=D1=83=D1=80=D0=BD=D1=8B=D1=85=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/operations/project-status.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/operations/project-status.md b/docs/operations/project-status.md
index f515fef..fa6f79f 100644
--- a/docs/operations/project-status.md
+++ b/docs/operations/project-status.md
@@ -110,8 +110,9 @@
 
 ### Развёртывание и сопровождение
 
-- Docker-конфигурация развёртывания не хранится в репозитории.
-- Не настроен CI/CD.
+- Production Compose, Ubuntu bootstrap и drift-проверка описаны в репозитории.
+- CI workflow добавлен в PR #42, но его Gitea run остаётся в очереди; наличие
+  runner и автоматический production-деплой не подтверждены.
 
 ### Хранилище
 
-- 
2.54.0


From e42c3145527e7059cde452811e993dc908bb775e Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:42:13 +0300
Subject: [PATCH 06/17] =?UTF-8?q?=D0=97=D0=B0=D1=84=D0=B8=D0=BA=D1=81?=
 =?UTF-8?q?=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=B8=D1=81=D0=BA?=
 =?UTF-8?q?=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B4=D0=B8=D0=B7?=
 =?UTF-8?q?=D0=B0=D0=B9=D0=BD-=D0=B4=D0=B5=D1=82=D0=B5=D0=BA=D1=82=D0=BE?=
 =?UTF-8?q?=D1=80=D0=B0=20=D0=B4=D0=BB=D1=8F=20=D0=BB=D0=B0=D0=B1=D0=BE?=
 =?UTF-8?q?=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=B8=D0=B8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .impeccable/config.json | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 .impeccable/config.json

diff --git a/.impeccable/config.json b/.impeccable/config.json
new file mode 100644
index 0000000..a9cc933
--- /dev/null
+++ b/.impeccable/config.json
@@ -0,0 +1,35 @@
+{
+  "detector": {
+    "ignoreRules": [],
+    "ignoreFiles": [],
+    "ignoreValues": [
+      {
+        "rule": "cream-palette",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:40:02.635Z",
+        "reason": "Agent: DESIGN.md explicitly defines #F7F2E8 as the approved product background"
+      },
+      {
+        "rule": "cramped-padding",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:28.434Z",
+        "reason": "Agent: both reported sections declare 1rem minimum padding in CSS; detector reports flush content despite that inset"
+      },
+      {
+        "rule": "design-system-font",
+        "value": "sans",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:33.020Z",
+        "reason": "Agent: DESIGN.md specifies Noto Sans with sans-serif fallback; this finding is the inherited generic fallback"
+      }
+    ]
+  }
+}
-- 
2.54.0


From fb556228c7d7e61c70a614938ab4514ae4dcbca3 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 18:57:08 +0300
Subject: [PATCH 07/17] Configure Caddy HTTPS proxy for production

---
 .gitea/workflows/ci.yaml          |  6 +++++
 CHANGELOG.md                      |  2 +-
 deploy/production/.env.example    |  1 -
 deploy/production/Caddyfile       |  4 +++
 deploy/production/README.md       | 27 ++++++++++++--------
 deploy/production/compose.yaml    | 42 +++++++++++++++++++++++++++++--
 docs/operations/BACKUP.md         |  4 +--
 docs/operations/CURRENT_STATE.md  | 15 ++++++-----
 docs/operations/DECISIONS.md      |  3 ++-
 docs/operations/DEPLOYMENT.md     | 15 ++++++-----
 docs/operations/INFRASTRUCTURE.md | 31 ++++++++++++-----------
 docs/operations/RECOVERY.md       |  6 +++--
 infra/bootstrap/verify.sh         | 14 +++++++++--
 infra/inventory/production.yml    | 16 +++++++++---
 14 files changed, 134 insertions(+), 52 deletions(-)
 create mode 100644 deploy/production/Caddyfile

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index 8926c02..8b14560 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -17,6 +17,12 @@ jobs:
         run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
       - name: Run Telegram bot tests
         run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
+      - name: Validate production Compose and Caddyfile
+        run: |
+          secret_file=$(mktemp)
+          trap 'rm -f "$secret_file"' EXIT
+          REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet
+          docker run --rm --entrypoint caddy -v "$PWD/deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028 validate --config /etc/caddy/Caddyfile --adapter caddyfile
       - name: Build production images
         run: |
           docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci .
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee7ec4f..d40a38c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 ## Не выпущено
 
-- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, drift-проверка и CI для тестов и сборки образов; production-деплой остаётся выключен до проверки runner, backup/restore и ограниченного deploy-helper.
+- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, Caddy HTTPS proxy с разделёнными Compose-сетями и CI-проверка конфигурации; публикация образов и production-деплой остаются выключены до проверки runner, registry, backup/restore и ограниченного deploy-helper.
 - Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
 - Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
 - Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
diff --git a/deploy/production/.env.example b/deploy/production/.env.example
index 2840663..e4d4a4d 100644
--- a/deploy/production/.env.example
+++ b/deploy/production/.env.example
@@ -1,7 +1,6 @@
 # Absolute path on the production host containing minjust-normalized/.
 DATA_ROOT=/var/lib/akyldash/data
 REVIEW_SECRET_FILE=/etc/akyldash/review_secret
-BACKEND_PORT=8080
 SEARCH_INDEX=akyldash-fragments-current
 OPENSEARCH_MEM_LIMIT=4g
 OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
diff --git a/deploy/production/Caddyfile b/deploy/production/Caddyfile
new file mode 100644
index 0000000..5c12485
--- /dev/null
+++ b/deploy/production/Caddyfile
@@ -0,0 +1,4 @@
+akyldash.kg {
+	encode zstd gzip
+	reverse_proxy backend:8080
+}
diff --git a/deploy/production/README.md b/deploy/production/README.md
index aef98ec..7e4918a 100644
--- a/deploy/production/README.md
+++ b/deploy/production/README.md
@@ -1,11 +1,12 @@
 # Production deployment
 
-This compose project runs the Search API and its private OpenSearch node. It
-binds the API only to `127.0.0.1`; publish it through an authenticated reverse
-proxy or VPN. OpenSearch is not published outside the compose network.
+This compose project runs Caddy, the Search API, and a private OpenSearch node.
+Caddy serves `akyldash.kg` over HTTPS and proxies to the API. Only Caddy publishes
+ports `80` and `443`; the API has no host port, and OpenSearch is reachable only
+from the backend's private network.
 
 The current compose intentionally disables the OpenSearch security plugin to
-match the existing API client. Keep both services on a private host/network.
+match the existing API client. Keep OpenSearch on its private network.
 The application provides cookie-based accounts for its search API and review
 workspace; it does not replace host-level network controls or TLS.
 
@@ -22,23 +23,29 @@ workspace; it does not replace host-level network controls or TLS.
 
    Do not commit `.env` or the secret file.
 3. Put the normalized dataset under `$DATA_ROOT/minjust-normalized/`.
-4. Check the rendered configuration:
+4. Confirm DNS points through Cloudflare to this origin. Have the owner review
+   the existing firewall rules and allow TCP `80` and `443` before Caddy starts.
+   Caddy requests and renews the origin certificate automatically; use
+   Cloudflare Full (strict) only after HTTPS succeeds.
+5. Complete and verify the backup/restore setup before starting production.
+   The repository does not yet contain a tested backup script; do not continue
+   until the owner has approved and tested one.
+6. Check the rendered configuration:
 
    ```bash
    docker compose --env-file .env -f compose.yaml config
    ```
 
-5. Start the services:
+7. Start the services:
 
    ```bash
    docker compose --env-file .env -f compose.yaml up -d --build
    docker compose --env-file .env -f compose.yaml ps
-   curl -fsS http://127.0.0.1:${BACKEND_PORT:-8080}/review >/dev/null
+   curl -fsS https://akyldash.kg/review >/dev/null
    ```
 
-6. Load the versioned index and switch its alias only after the import and
-   validation succeed. Back up `DATA_ROOT` and the `opensearch-data` volume
-   before the first import.
+8. Load the versioned index and switch its alias only after the import and
+   validation succeed, after creating another verified pre-change backup.
 
 ## Initial administrator and accounts
 
diff --git a/deploy/production/compose.yaml b/deploy/production/compose.yaml
index c510ebe..232d1a9 100644
--- a/deploy/production/compose.yaml
+++ b/deploy/production/compose.yaml
@@ -1,6 +1,32 @@
 name: akyldash
 
 services:
+  caddy:
+    image: caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
+    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caddy-data:/data
+      - caddy-config:/config
+    networks:
+      - proxy
+    depends_on:
+      backend:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
   opensearch:
     build:
       context: ../..
@@ -28,6 +54,8 @@ services:
         hard: 65536
     volumes:
       - opensearch-data:/usr/share/opensearch/data
+    networks:
+      - search
     healthcheck:
       test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9200/_cluster/health || exit 1"]
       interval: 30s
@@ -68,17 +96,27 @@ services:
       - /run/secrets/review_secret
     secrets:
       - review_secret
-    ports:
-      - "127.0.0.1:${BACKEND_PORT:-8080}:8080"
+    expose:
+      - "8080"
     depends_on:
       opensearch:
         condition: service_healthy
     volumes:
       - ${DATA_ROOT:?set DATA_ROOT in .env}:/app/data
+    networks:
+      - proxy
+      - search
 
 volumes:
+  caddy-data:
+  caddy-config:
   opensearch-data:
 
+networks:
+  proxy:
+  search:
+    internal: true
+
 secrets:
   review_secret:
     file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index d0759b0..cd2ff08 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -13,8 +13,8 @@ both been checked.
   or index migration.
 - Include the normalized corpus, `search-users.sqlite3`,
   `search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
-  and required secret file in the protected backup set. Store secrets separately
-  with equivalent access controls.
+  Caddy's certificate data, and required secret file in the protected backup
+  set. Store secrets and Caddy private keys with equivalent access controls.
 - Verify archive readability, checksums, and SQLite integrity before
   replacing the previous known-good backup.
 - Keep one current pre-release backup. If deployment fails, retain the backup
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 98381d1..107a307 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -35,12 +35,15 @@ from planned configuration.
   publishing permissions could not be checked. This change adds
   `.gitea/workflows/ci.yaml`; PR #42's run remains `Waiting to run` at the last
   status check, so a compatible runner is not confirmed.
-- Production Compose describes Python Search API `0.11.3`, SQLite account and
-  review data, and OpenSearch `3.7.0` with `analysis-icu`. Nothing is running.
-- Compose binds the API to `127.0.0.1:8080`, leaves OpenSearch private, uses a
-  named OpenSearch volume, and a host data bind mount. TLS/reverse proxy,
-  registry digest deployment, backup, rollback helper, and systemd deployment
-  are not configured.
+- Nothing is running on the VPS. Repository Compose now describes Caddy
+  `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite account/review
+  data, and OpenSearch `3.7.0` with `analysis-icu`.
+- The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a
+  proxy network; OpenSearch and backend share a separate internal network, and
+  the backend has no host port. Caddy certificate storage, OpenSearch, and
+  application data use persistent volumes/bind mounts. TLS, registry digest
+  deployment for backend/OpenSearch, backup, rollback helper, and systemd
+  deployment are not configured on the host.
 - The app has no migration framework or SMTP setup. Current app versions:
   backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
   deployed on this VPS).
diff --git a/docs/operations/DECISIONS.md b/docs/operations/DECISIONS.md
index 3785da3..e665643 100644
--- a/docs/operations/DECISIONS.md
+++ b/docs/operations/DECISIONS.md
@@ -4,6 +4,7 @@
 |---|---|---|---|
 | 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
 | 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; use an owner-run bootstrap and a narrowly reviewed deploy-helper | No helper installed yet |
-| 2026-10-11 | Keep Search API on loopback and OpenSearch on the private Compose network | Minimizes host exposure; HTTPS terminates at a reviewed reverse proxy | Compose target only; no proxy or firewall audit privileges |
+| 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
+| 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
 | 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
 | 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index 171d0bc..a3ae71e 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -15,12 +15,15 @@ configuration, and a backup and restore have been tested.
    of this branch on the VPS. The script installs Docker Engine/Compose and Git,
    sets Docker log rotation, and creates root-owned paths. It refuses other OS
    versions and does not change SSH, UFW, sudoers, or user groups.
-2. Install the repository checkout at `/opt/akyldash/current` and configure
-   `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
-   Git. The secret file must be root-readable only.
-3. Review DNS and certificate issuance, choose and configure the reverse proxy,
-   then ask the owner to open only the required ports after confirming current
-   firewall rules.
+2. Install the approved repository checkout at `/opt/akyldash/current` and
+   configure `/etc/akyldash/production.env` plus
+   `/etc/akyldash/review_secret` outside Git. The secret file must be
+   root-readable only.
+3. Confirm DNS points through Cloudflare to this origin, then have the owner
+   review firewall rules and open only TCP `80` and `443` in addition to the
+   existing SSH access. The Compose Caddy service obtains and renews its HTTPS
+   certificate automatically; Cloudflare Full (strict) still needs owner-side
+   verification.
 4. Create and verify the first backup before starting services. Load and
    validate the OpenSearch index, create the initial admin interactively, and
    verify `/review` over HTTPS.
diff --git a/docs/operations/INFRASTRUCTURE.md b/docs/operations/INFRASTRUCTURE.md
index 5a19667..74bfaea 100644
--- a/docs/operations/INFRASTRUCTURE.md
+++ b/docs/operations/INFRASTRUCTURE.md
@@ -3,17 +3,17 @@
 ## Intended layout
 
 Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
-`deploy/production/compose.yaml` defines a Python Search API and private
-OpenSearch 3.7 node. The API binds to loopback port `8080`; OpenSearch is only
-exposed to the Compose network. The host reverse proxy should terminate HTTPS
-for `akyldash.kg` and proxy to the API. Cloudflare should use Full (strict) only
-after the origin certificate and DNS path are verified.
+`deploy/production/compose.yaml` defines Caddy, a Python Search API, and private
+OpenSearch 3.7. Caddy terminates HTTPS for `akyldash.kg` and proxies to the API.
+The API is reachable only on private Compose networks; OpenSearch is reachable
+only by the backend. Cloudflare should use Full (strict) only after the origin
+certificate and DNS path are verified.
 
 The API stores its account and review SQLite files beside the normalized corpus
-under the persistent data directory. OpenSearch uses a named volume. The API's
-review signing key is mounted from a host file; it is not passed in process
-arguments or committed to Git. Docker logs are bounded to 10 MiB per file and
-three files per service.
+under the persistent data directory. OpenSearch and Caddy use named volumes.
+The API's review signing key is mounted from a host file; it is not passed in
+process arguments or committed to Git. Docker logs are bounded to 10 MiB per
+file and three files per service.
 
 ## Actual state
 
@@ -23,10 +23,11 @@ and backups are not installed/configured. The `codex` account has no
 non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
 [production inventory](../../infra/inventory/production.yml).
 
-The Compose file is a repository baseline, not proof of deployment. It still
-builds images locally; immutable registry images and automated deploy are not
-configured. No migration framework exists. The application uses SQLite and
-OpenSearch; SMTP registration/recovery is not implemented.
+The Compose file is a repository target, not proof of deployment. The Caddy
+image is pinned by digest, while backend and OpenSearch still build locally;
+immutable registry images and automated deploy are not configured. No migration
+framework exists. The application uses SQLite and OpenSearch; SMTP
+registration/recovery is not implemented.
 
 ## Repository paths and target host paths
 
@@ -44,8 +45,8 @@ application, or grant `codex` Docker/root access.
 
 ## Network and operations
 
-Target public TCP listeners are SSH `22` and HTTPS proxy `80`/`443`. Backend
-`8080` must remain loopback-only and OpenSearch `9200` private to Compose. The
+Target public TCP listeners are SSH `22` and Caddy `80`/`443`. Backend `8080`
+and OpenSearch `9200` must remain private to Compose. The
 firewall state could not be independently verified because the SSH account has
 no sudo; do not change it without owner approval. The domain resolves to
 Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
index c30f7f1..3c8e8c1 100644
--- a/docs/operations/RECOVERY.md
+++ b/docs/operations/RECOVERY.md
@@ -18,8 +18,10 @@ run recovery commands against the existing production by accident.
 3. Check out the approved Git revision and run
    `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh`. It installs Docker
    Engine, Compose, Git, Docker log limits, and target directories.
-4. Install the reviewed reverse proxy and its HTTPS configuration after DNS
-   points to the recovery host. Verify Cloudflare Full (strict) end to end.
+4. Configure DNS to point through Cloudflare to the recovery host and allow the
+   owner to review/open TCP `80` and `443`. The production Compose Caddy service
+   obtains its origin certificate automatically. Verify Cloudflare Full
+   (strict) end to end before cutover.
 5. Provide production configuration and secret files through the owner's
    protected channel; never commit or paste secret values into chat.
 6. Restore the latest verified backup to the new host. Validate SQLite
diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh
index c5c476f..dd167c2 100755
--- a/infra/bootstrap/verify.sh
+++ b/infra/bootstrap/verify.sh
@@ -76,8 +76,8 @@ done
 
 checkout=/opt/akyldash/current
 compose_file=$checkout/deploy/production/compose.yaml
-if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
-  echo 'PASS production Compose and Dockerfiles match the checked-out Git revision'
+if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Caddyfile deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
+  echo 'PASS production Compose, Caddyfile, and Dockerfiles match the checked-out Git revision'
 else
   echo 'FAIL production Compose or Dockerfile has local drift'
   failed=1
@@ -149,4 +149,14 @@ else
   echo 'FAIL deployed image digest record missing'
   failed=1
 fi
+
+expected_proxy=$(docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --format json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["services"]["caddy"]["image"])' 2>/dev/null || true)
+proxy_container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=caddy | head -n1)
+actual_proxy=$(docker inspect --format '{{.Config.Image}}' "$proxy_container" 2>/dev/null || true)
+if [[ $expected_proxy == *@sha256:* && $actual_proxy == "$expected_proxy" ]]; then
+  printf 'PASS running Caddy image matches %s\n' "$expected_proxy"
+else
+  echo 'FAIL running Caddy image does not match the pinned Compose digest'
+  failed=1
+fi
 exit "$failed"
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
index 8f98a1d..39e1a06 100644
--- a/infra/inventory/production.yml
+++ b/infra/inventory/production.yml
@@ -40,10 +40,15 @@ target:
   data_directory: /var/lib/akyldash/data
   backup_directory: /var/backups/akyldash
   services:
+    caddy:
+      container: compose project service caddy
+      image: caddy:2.11.6-alpine pinned by digest in deploy/production/compose.yaml
+      host_ports: [80, 443]
+      persistent_data: named Compose volumes caddy-data and caddy-config
     backend:
       container: compose project service backend
       image: built from deploy/production/Dockerfile.backend; digest deployment not configured
-      host_bind: 127.0.0.1:8080
+      port: 8080 on private proxy and search Compose networks only
       healthcheck: HTTP GET /review on container port 8080
       persistent_data: /var/lib/akyldash/data (bind mount)
     opensearch:
@@ -52,10 +57,12 @@ target:
       port: 9200 on private Compose network only
       persistent_data: named Compose volume opensearch-data
   reverse_proxy:
-    target: Caddy or another reviewed HTTPS proxy on TCP 80/443
-    status: not installed or configured
+    target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080
+    status: repository configuration exists; not installed or running on VPS
   external_ports_target: [22, 80, 443]
-  compose_network: default private Compose network
+  compose_networks:
+    proxy: caddy and backend
+    search: backend and OpenSearch; internal network
   secrets:
     - REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
     - Gitea credentials from environment-backed gitea-api helper
@@ -71,4 +78,5 @@ versions:
     frontend: 0.1.2
     telegram_bot: 0.2.2
   infrastructure: unknown; no production install or release exists
+  caddy_image: 2.11.6-alpine pinned by manifest digest in Compose
   docker_engine: not installed
-- 
2.54.0


From 97803ca5def939705b94277d06b836f0865f7e1b Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:12:02 +0300
Subject: [PATCH 08/17] Add cold backup and staged restore tooling

---
 .gitea/workflows/ci.yaml         |   2 +
 AGENTS.md                        |   6 +-
 docs/operations/BACKUP.md        |  53 ++++-
 docs/operations/CURRENT_STATE.md |  10 +
 docs/operations/DEPLOYMENT.md    |   3 +-
 docs/operations/RECOVERY.md      |   8 +-
 infra/backup/archive.py          | 333 +++++++++++++++++++++++++++++++
 infra/backup/test_archive.py     | 153 ++++++++++++++
 infra/inventory/production.yml   |   3 +-
 9 files changed, 557 insertions(+), 14 deletions(-)
 create mode 100644 infra/backup/archive.py
 create mode 100644 infra/backup/test_archive.py

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index 8b14560..4b0445a 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -17,6 +17,8 @@ jobs:
         run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
       - name: Run Telegram bot tests
         run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
+      - name: Run infrastructure backup tests
+        run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
       - name: Validate production Compose and Caddyfile
         run: |
           secret_file=$(mktemp)
diff --git a/AGENTS.md b/AGENTS.md
index 4049606..3a71787 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -14,10 +14,14 @@ Python, SQLite и OpenSearch. Production пока не развернут.
   `codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
   Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
 - Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
-  `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`.
+  `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
+  `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v`.
   Production images собираются командами из `.gitea/workflows/ci.yaml`.
 - Перед изменениями production необходим проверенный backup. Формального
   механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
   SSH, firewall или production вручную без владельца и последующего PR.
+- Холодный backup требует согласованного окна простоя; восстановление допускается
+  только в новый staging-каталог, пока владелец отдельно не проверил процедуру
+  для нового хоста.
 - После каждого изменения инфраструктуры обновляй
   `docs/operations/CURRENT_STATE.md` по фактическим проверкам.
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index cd2ff08..0be85c6 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -2,10 +2,9 @@
 
 ## Current status
 
-No production data or backup mechanism existed during the 2026-10-11 audit.
-`/var/backups/akyldash` is a planned root-only location, not an active backup.
-Do not report backup protection as enabled until an archive and restore have
-both been checked.
+No production data or backup existed during the 2026-10-11 audit. The repository
+now contains `infra/backup/archive.py`; it is not installed or run on the VPS,
+and `/var/backups/akyldash` remains a planned root-only location.
 
 ## Required release policy
 
@@ -22,7 +21,45 @@ both been checked.
 - Test restore into an isolated target before relying on the backup. Do not
   restore over production as a test.
 
-The data is not yet managed by a tested backup script. Until that exists, the
-owner must pause deployment and approve a maintenance window for a reviewed
-backup/restore procedure. Copying live SQLite files with `cp` or an unchecked
-tar archive is not an approved backup.
+## Cold backup procedure
+
+The script requires every Compose container to be stopped. Schedule downtime,
+then run from the approved checkout on the VPS:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+```
+
+Replace the timestamp placeholder with one real UTC timestamp before running.
+The create command refuses running services and archives the application data,
+OpenSearch volume, Caddy data/config volumes, production environment file, and
+review secret. It records the checked-out Git revision and image IDs for each
+stopped Compose service, writes a SHA-256 sidecar, and refuses to overwrite an
+existing archive. It also runs SQLite `quick_check`. Start the stack again only
+after verification succeeds:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+Keep the local archive root-only, then copy it to separately managed encrypted
+storage. A local archive alone does not protect against loss of the VPS disk.
+Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
+`/var/tmp` and validates the checksum and SQLite databases; it never writes
+production paths. Example test on an isolated host:
+
+```bash
+sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
+  --target-dir /var/tmp/akyldash-restore-test
+```
+
+This validates and stages the archive; it does not install data into Compose
+volumes. Complete and verify the host-specific restore on a separate recovery
+host before relying on it. Do not run an unreviewed restore over existing data.
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 107a307..2d11faf 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -44,6 +44,13 @@ from planned configuration.
   application data use persistent volumes/bind mounts. TLS, registry digest
   deployment for backend/OpenSearch, backup, rollback helper, and systemd
   deployment are not configured on the host.
+- The repository has a cold-backup archive/verification tool with staging-only
+  restore and unit tests. It has not run against production; do not treat this
+  as an active backup or verified recovery.
+- An isolated local Compose smoke test built the backend/OpenSearch images,
+  reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
+  200 for `/review` through Caddy. Its containers, networks, and volumes were
+  removed afterward. This does not verify VPS startup, DNS, or public TLS.
 - The app has no migration framework or SMTP setup. Current app versions:
   backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
   deployed on this VPS).
@@ -53,6 +60,9 @@ from planned configuration.
 - SSH completed with the user config explicitly selected; plain SSH was blocked
   by permissions on a system config include. The private key was not read.
 - UFW rules require root and remain unverified.
+- The backup archive/restore tool was tested with synthetic local data; a backup
+  and restore of real production data remain unverified because production is
+  not deployed.
 - The Cloudflare edge response was checked, but origin DNS configuration,
   Cloudflare Full (strict), certificate validity, Actions runner availability,
   registry publishing, and production deployment have not been tested.
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index a3ae71e..7c5e4f1 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -2,7 +2,8 @@
 
 ## Current status
 
-Production deployment is not enabled or tested. The VPS has no Docker, Git,
+Production deployment is not enabled or tested on the VPS. An isolated local
+Compose smoke test passed; the VPS has no Docker, Git,
 application data, proxy, registry pull configuration, backup process, or
 restricted deploy-helper. Do not deploy until the owner has completed the root
 bootstrap, provided a protected secret file and a reviewed proxy/firewall
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
index 3c8e8c1..fc853b4 100644
--- a/docs/operations/RECOVERY.md
+++ b/docs/operations/RECOVERY.md
@@ -24,9 +24,11 @@ run recovery commands against the existing production by accident.
    (strict) end to end before cutover.
 5. Provide production configuration and secret files through the owner's
    protected channel; never commit or paste secret values into chat.
-6. Restore the latest verified backup to the new host. Validate SQLite
-   databases, corpus checksums, and the OpenSearch snapshot/index before serving
-   traffic.
+6. Verify the archive and extract it to a new `/var/tmp/akyldash-restore-*`
+   staging directory with `infra/backup/archive.py`. The tool checks the archive
+   checksum and SQLite integrity but does not install staged files into Compose
+   volumes; have the owner review and perform that host-specific step. Validate
+   the corpus and OpenSearch index before serving traffic.
 7. Start the approved image digest, verify container health and
    `https://akyldash.kg/review`, then record actual versions and checks in
    [CURRENT_STATE.md](CURRENT_STATE.md).
diff --git a/infra/backup/archive.py b/infra/backup/archive.py
new file mode 100644
index 0000000..a20dd99
--- /dev/null
+++ b/infra/backup/archive.py
@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+"""Create and verify cold, local Akyldash backups; restore only into staging."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path, PurePosixPath
+
+COMPONENTS = {
+    "data",
+    "opensearch",
+    "caddy-data",
+    "caddy-config",
+    "production-env",
+    "review-secret",
+}
+NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+
+def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
+    members = archive.getmembers()
+    names: set[str] = set()
+    for member in members:
+        name = PurePosixPath(member.name)
+        if name.is_absolute() or ".." in name.parts or not name.parts:
+            raise ValueError(f"unsafe archive path: {member.name}")
+        if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
+            raise ValueError(f"non-canonical archive path: {member.name}")
+        if member.name in names:
+            raise ValueError(f"duplicate archive path: {member.name}")
+        names.add(member.name)
+        if not (member.isdir() or member.isfile() or member.issym()):
+            raise ValueError(f"unsupported archive entry: {member.name}")
+        if member.issym():
+            tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
+    if "manifest.json" not in names:
+        raise ValueError("backup manifest is missing")
+    if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
+        raise ValueError("backup is missing one or more required components")
+    manifest_member = next(item for item in members if item.name == "manifest.json")
+    if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
+        raise ValueError("backup manifest is not a regular file")
+    return members
+
+
+def read_manifest(archive: tarfile.TarFile) -> dict:
+    checked_members(archive)
+    stream = archive.extractfile("manifest.json")
+    if stream is None:
+        raise ValueError("backup manifest is unreadable")
+    manifest = json.load(stream)
+    components = manifest.get("components") if isinstance(manifest, dict) else None
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or not isinstance(components, list)
+        or not all(isinstance(item, str) for item in components)
+        or set(components) != COMPONENTS
+    ):
+        raise ValueError("backup manifest has an unsupported format or component set")
+    if not isinstance(manifest.get("images"), dict):
+        raise ValueError("backup manifest has no image records")
+    for member in archive.getmembers():
+        if member.name == "manifest.json":
+            continue
+        if PurePosixPath(member.name).parts[0] not in COMPONENTS:
+            raise ValueError(f"unexpected top-level archive path: {member.name}")
+    return manifest
+
+
+def verify_archive(path: Path) -> dict:
+    checksum_path = Path(f"{path}.sha256")
+    expected = checksum_path.read_text(encoding="ascii").split()[0]
+    if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
+        raise ValueError("archive SHA-256 does not match its sidecar")
+    with tarfile.open(path, "r:gz") as archive:
+        return read_manifest(archive)
+
+
+def parse_sources(values: list[str]) -> dict[str, Path]:
+    sources: dict[str, Path] = {}
+    for value in values:
+        name, separator, raw_path = value.partition("=")
+        if not separator or not NAME_RE.fullmatch(name) or name in sources:
+            raise ValueError(f"invalid or duplicate source: {value}")
+        source = Path(raw_path)
+        if not source.is_absolute() or not source.exists() or source.is_symlink():
+            raise ValueError(f"source must be an existing absolute path: {raw_path}")
+        sources[name] = source.resolve(strict=True)
+    if set(sources) != COMPONENTS:
+        missing = ", ".join(sorted(COMPONENTS - sources.keys()))
+        extra = ", ".join(sorted(sources.keys() - COMPONENTS))
+        raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
+    return sources
+
+
+def check_sqlite(root: Path) -> None:
+    for path in root.rglob("*.sqlite3"):
+        connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
+        try:
+            result = connection.execute("PRAGMA quick_check").fetchone()
+            if result != ("ok",):
+                raise ValueError(f"SQLite integrity check failed: {path}")
+        finally:
+            connection.close()
+
+
+def compose_sources(compose_file: Path, env_file: Path) -> tuple[dict[str, Path], str, dict[str, dict[str, str]]]:
+    command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
+    rendered = subprocess.run(
+        [*command, "config", "--format", "json"], check=True, capture_output=True, text=True
+    )
+    config = json.loads(rendered.stdout)
+    container_result = subprocess.run(
+        [*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
+    )
+    containers = container_result.stdout.split()
+    if not containers:
+        raise ValueError("Compose has no containers; refusing to create an incomplete backup")
+    images: dict[str, dict[str, str]] = {}
+    for container in containers:
+        result = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                '{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
+                container,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        fields = result.stdout.strip().split("\t")
+        if len(fields) != 4 or fields[0].lower() != "false":
+            raise ValueError("stop all Compose services before taking a cold backup")
+        images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
+    if set(images) != set(config["services"]):
+        raise ValueError("not all configured Compose services have stopped containers")
+
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    volume_sources: dict[str, Path] = {}
+    for target, component in (
+        ("/app/data", "data"),
+        ("/usr/share/opensearch/data", "opensearch"),
+        ("/data", "caddy-data"),
+        ("/config", "caddy-config"),
+    ):
+        mount = mounts.get(target)
+        if not mount:
+            raise ValueError(f"persistent Compose mount is missing: {target}")
+        if mount["type"] == "bind":
+            source = Path(mount["source"])
+        elif mount["type"] == "volume":
+            volume_name = config["volumes"][mount["source"]]["name"]
+            result = subprocess.run(
+                ["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            source = Path(result.stdout.strip())
+        else:
+            raise ValueError(f"unsupported persistent mount type at {target}")
+        volume_sources[component] = source
+
+    secret_file = Path(config["secrets"]["review_secret"]["file"])
+    sources = {
+        **volume_sources,
+        "production-env": env_file,
+        "review-secret": secret_file,
+    }
+    repository = compose_file.resolve().parents[2]
+    changed = subprocess.run(
+        ["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if changed:
+        raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
+    revision = subprocess.run(
+        ["git", "-C", str(repository), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images
+
+
+def create_backup(output: Path, sources: dict[str, Path], revision: str, images: dict | None = None) -> None:
+    output = output.absolute()
+    parent = output.parent
+    if not parent.is_dir() or parent.stat().st_mode & 0o077:
+        raise ValueError("backup directory must exist and have mode 0700 or stricter")
+    if output.exists() or Path(f"{output}.sha256").exists():
+        raise ValueError("refusing to overwrite an existing backup")
+    for name, source in sources.items():
+        if source.is_symlink() or not (source.is_dir() or source.is_file()):
+            raise ValueError(f"source must be a regular file or directory: {name}")
+        if source.is_dir() and (output == source or source in output.parents):
+            raise ValueError(f"backup output cannot be inside source {name}")
+    for name in ("production-env", "review-secret"):
+        if not sources[name].is_file():
+            raise ValueError(f"sensitive component must be a regular file: {name}")
+    if sources["data"].is_dir():
+        check_sqlite(sources["data"])
+
+    manifest = {
+        "format": 1,
+        "created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+        "git_revision": revision,
+        "images": images or {},
+        "components": sorted(sources),
+    }
+    with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
+        temporary = Path(temp.name)
+    os.chmod(temporary, 0o600)
+    checksum_temp = Path(f"{temporary}.sha256")
+    try:
+        with tarfile.open(temporary, "w:gz") as archive:
+            encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
+            info = tarfile.TarInfo("manifest.json")
+            info.size = len(encoded)
+            info.mode = 0o600
+            archive.addfile(info, io.BytesIO(encoded))
+            for name, source in sorted(sources.items()):
+                def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
+                    if name in {"production-env", "review-secret"}:
+                        member.mode = 0o600
+                    if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
+                        member.mode = 0o600
+                    return member
+
+                archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
+        with tarfile.open(temporary, "r:gz") as archive:
+            read_manifest(archive)
+        checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
+            stream.write(f"{digest(temporary)}  {output.name}\n")
+        os.link(temporary, output)
+        try:
+            os.link(checksum_temp, Path(f"{output}.sha256"))
+        except Exception:
+            output.unlink(missing_ok=True)
+            raise
+    finally:
+        temporary.unlink(missing_ok=True)
+        checksum_temp.unlink(missing_ok=True)
+
+
+def restore_to_staging(backup: Path, target: Path) -> None:
+    if not target.name.startswith("akyldash-restore-"):
+        raise ValueError("restore target must be a new akyldash-restore-* staging directory")
+    parent = target.parent.resolve()
+    if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
+        raise ValueError("restore staging directory must be under /tmp or /var/tmp")
+    if target.exists():
+        raise ValueError("restore target already exists; refusing to overwrite it")
+    target = parent / target.name
+    verify_archive(backup)
+    target.mkdir(mode=0o700)
+    try:
+        with tarfile.open(backup, "r:gz") as archive:
+            members = checked_members(archive)
+            required_bytes = sum(member.size for member in members if member.isfile())
+            if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
+                raise ValueError("insufficient free space for a validated restore staging copy")
+            for member in members:
+                archive.extract(member, target, filter="data")
+        check_sqlite(target / "data")
+    except Exception:
+        shutil.rmtree(target)
+        raise
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    create = subparsers.add_parser("create", help="create a cold backup archive")
+    create.add_argument("--output", required=True, type=Path)
+    create.add_argument("--compose-file", required=True, type=Path)
+    create.add_argument("--env-file", required=True, type=Path)
+    verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
+    verify.add_argument("archive", type=Path)
+    restore = subparsers.add_parser("restore", help="extract into a new staging directory")
+    restore.add_argument("archive", type=Path)
+    restore.add_argument("--target-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        if args.command in {"create", "restore"} and os.geteuid() != 0:
+            raise ValueError("create and restore must run as root to protect production secrets and data")
+        if args.command == "create":
+            sources, revision, images = compose_sources(args.compose_file, args.env_file)
+            create_backup(args.output, sources, revision, images)
+        elif args.command == "verify":
+            manifest = verify_archive(args.archive)
+            print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
+        else:
+            restore_to_staging(args.archive, args.target_dir)
+            print(f"PASS restored and validated staging data at {args.target_dir}")
+    except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
new file mode 100644
index 0000000..f00708e
--- /dev/null
+++ b/infra/backup/test_archive.py
@@ -0,0 +1,153 @@
+import hashlib
+import json
+import sqlite3
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from archive import COMPONENTS, compose_sources, create_backup, restore_to_staging, verify_archive
+
+
+class BackupArchiveTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.private = self.root / "private"
+        self.private.mkdir(mode=0o700)
+        self.sources = {}
+        for name in COMPONENTS:
+            source = self.root / name
+            source.mkdir()
+            self.sources[name] = source
+        for name in ("production-env", "review-secret"):
+            path = self.root / f"{name}.txt"
+            path.write_text("private", encoding="utf-8")
+            path.chmod(0o644)
+            self.sources[name] = path
+        connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
+        connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
+        connection.commit()
+        connection.close()
+        self.backup = self.private / "backup.tar.gz"
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def test_create_verify_and_restore_only_to_new_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        manifest = verify_archive(self.backup)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        self.assertEqual(manifest["images"], {})
+
+        target = self.root / "akyldash-restore-test"
+        restore_to_staging(self.backup, target)
+        self.assertTrue((target / "data/search-users.sqlite3").is_file())
+        self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
+        self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
+
+    def test_rejects_checksum_mismatch_without_creating_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
+        target = self.root / "akyldash-restore-tampered"
+        with self.assertRaisesRegex(ValueError, "SHA-256"):
+            restore_to_staging(self.backup, target)
+        self.assertFalse(target.exists())
+
+    def test_rejects_non_staging_and_existing_targets(self):
+        create_backup(self.backup, self.sources, "abc123")
+        with self.assertRaisesRegex(ValueError, "staging"):
+            restore_to_staging(self.backup, self.root / "production")
+        target = self.root / "akyldash-restore-existing"
+        target.mkdir()
+        with self.assertRaisesRegex(ValueError, "already exists"):
+            restore_to_staging(self.backup, target)
+
+    def test_create_checks_sqlite_integrity_before_writing_archive(self):
+        (self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
+        with self.assertRaises(sqlite3.DatabaseError):
+            create_backup(self.backup, self.sources, "abc123")
+        self.assertFalse(self.backup.exists())
+
+    def test_rejects_unsafe_archive_paths(self):
+        archive_path = self.private / "unsafe.tar.gz"
+        with tarfile.open(archive_path, "w:gz") as archive:
+            archive.addfile(tarfile.TarInfo("manifest.json"))
+            archive.addfile(tarfile.TarInfo("../outside"))
+        checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
+        Path(f"{archive_path}.sha256").write_text(f"{checksum}  unsafe.tar.gz\n")
+        with self.assertRaisesRegex(ValueError, "unsafe archive path"):
+            verify_archive(archive_path)
+
+    def test_cold_backup_refuses_a_running_compose_container(self):
+        config = {"services": {}, "volumes": {}, "secrets": {}}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="container-id\n")
+            return SimpleNamespace(stdout="true\n")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "stop all Compose services"):
+                compose_sources(Path("deploy/production/compose.yaml"), Path("/etc/akyldash/production.env"))
+
+    def test_compose_backup_records_sources_and_image_ids(self):
+        data = self.root / "production-data"
+        data.mkdir()
+        secret = self.root / "secret"
+        secret.write_text("private", encoding="utf-8")
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        volumes = {}
+        volume_paths = {}
+        mounts = {
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+            "backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
+            "opensearch": [
+                {"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
+            ],
+        }
+        for name in ("caddy-data", "caddy-config", "opensearch-data"):
+            path = self.root / f"volume-{name}"
+            path.mkdir()
+            volumes[name] = {"name": f"akyldash_{name}"}
+            volume_paths[f"akyldash_{name}"] = path
+        config = {
+            "services": {name: {"volumes": values} for name, values in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        service_names = ["caddy", "backend", "opensearch"]
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
+            if command[1] == "inspect":
+                name = command[-1].removesuffix("-id")
+                return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            return SimpleNamespace(stdout="")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertEqual(revision, "")
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images["backend"]["image_id"], "sha256:backend")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
index 39e1a06..a4b0ab3 100644
--- a/infra/inventory/production.yml
+++ b/infra/inventory/production.yml
@@ -71,7 +71,8 @@ target:
     - infra/bootstrap/verify.sh akyldash_vps
     - docker compose -f deploy/production/compose.yaml ps
     - curl -fsS https://akyldash.kg/review
-  backup_path: /var/backups/akyldash (target; mechanism not configured)
+    - python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
+  backup_path: /var/backups/akyldash (target; archive script in infra/backup/archive.py is not installed or run)
 versions:
   application:
     backend: 0.11.3
-- 
2.54.0


From 16f97d5f73527158b77d7c760b9bb234aa07f736 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:16:54 +0300
Subject: [PATCH 09/17] Guard recovery bootstrap from production host

---
 .gitea/workflows/ci.yaml                  |  2 ++
 docs/operations/CURRENT_STATE.md          |  3 +++
 docs/operations/DEPLOYMENT.md             |  4 +++-
 docs/operations/RECOVERY.md               |  4 +++-
 infra/bootstrap/bootstrap-ubuntu-24.04.sh | 26 +++++++++++++++++++++++
 infra/bootstrap/host-guard.sh             | 11 ++++++++++
 infra/bootstrap/test-host-guard.sh        | 12 +++++++++++
 7 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100644 infra/bootstrap/host-guard.sh
 create mode 100644 infra/bootstrap/test-host-guard.sh

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index 4b0445a..bcc8dc6 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -19,6 +19,8 @@ jobs:
         run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
       - name: Run infrastructure backup tests
         run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
+      - name: Check recovery host guard
+        run: bash infra/bootstrap/test-host-guard.sh
       - name: Validate production Compose and Caddyfile
         run: |
           secret_file=$(mktemp)
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 2d11faf..390c611 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -47,6 +47,9 @@ from planned configuration.
 - The repository has a cold-backup archive/verification tool with staging-only
   restore and unit tests. It has not run against production; do not treat this
   as an active backup or verified recovery.
+- Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
+  inventoried production hostname/IP; its host-identity guard has shell tests.
+  The mode has not been run on a VPS.
 - An isolated local Compose smoke test built the backend/OpenSearch images,
   reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
   200 for `/review` through Caddy. Its containers, networks, and volumes were
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index 7c5e4f1..65e1dd1 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -15,7 +15,9 @@ configuration, and a backup and restore have been tested.
    `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh` from a verified copy
    of this branch on the VPS. The script installs Docker Engine/Compose and Git,
    sets Docker log rotation, and creates root-owned paths. It refuses other OS
-   versions and does not change SSH, UFW, sudoers, or user groups.
+   versions and does not change SSH, UFW, sudoers, or user groups. Use
+   `--recovery-target` only on a replacement host; it refuses the known
+   production host identity.
 2. Install the approved repository checkout at `/opt/akyldash/current` and
    configure `/etc/akyldash/production.env` plus
    `/etc/akyldash/review_secret` outside Git. The secret file must be
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
index fc853b4..9f82d25 100644
--- a/docs/operations/RECOVERY.md
+++ b/docs/operations/RECOVERY.md
@@ -16,7 +16,9 @@ run recovery commands against the existing production by accident.
    Preserve the host's existing root SSH and firewall policy until an owner has
    reviewed the change.
 3. Check out the approved Git revision and run
-   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh`. It installs Docker
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
+   recovery mode refuses the inventoried production hostname/IP and writes a
+   root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
    Engine, Compose, Git, Docker log limits, and target directories.
 4. Configure DNS to point through Cloudflare to the recovery host and allow the
    owner to review/open TCP `80` and `443`. The production Compose Caddy service
diff --git a/infra/bootstrap/bootstrap-ubuntu-24.04.sh b/infra/bootstrap/bootstrap-ubuntu-24.04.sh
index fd085d2..403c435 100755
--- a/infra/bootstrap/bootstrap-ubuntu-24.04.sh
+++ b/infra/bootstrap/bootstrap-ubuntu-24.04.sh
@@ -1,6 +1,26 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+if [[ $# -gt 1 || ( $# -eq 1 && $1 != --recovery-target ) ]]; then
+  echo "Usage: $0 [--recovery-target]" >&2
+  exit 2
+fi
+recovery_target=false
+if [[ ${1:-} == --recovery-target ]]; then
+  recovery_target=true
+  . "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+  recovery_hostname=$(hostname -f 2>/dev/null || true)
+  recovery_ipv4=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  if [[ -z $recovery_hostname || -z $recovery_ipv4 ]]; then
+    echo "Cannot verify recovery host identity; refusing recovery bootstrap." >&2
+    exit 1
+  fi
+  if akyldash_is_production_host "$recovery_hostname" "$recovery_ipv4"; then
+    echo "Refusing recovery bootstrap on the current production host." >&2
+    exit 1
+  fi
+fi
+
 if [[ ${EUID} -ne 0 ]]; then
   echo "Run this script as root." >&2
   exit 1
@@ -47,6 +67,12 @@ install -d -o root -g root -m 0755 /opt/akyldash
 install -d -o root -g root -m 0700 /etc/akyldash
 install -d -o root -g root -m 0750 /var/lib/akyldash/data
 install -d -o root -g root -m 0700 /var/backups/akyldash
+if [[ $recovery_target == true ]]; then
+  install -o root -g root -m 0600 /dev/null /etc/akyldash/recovery-target
+fi
 systemctl enable --now docker
 docker compose version
 echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
+if [[ $recovery_target == true ]]; then
+  echo 'Recovery target marker created at /etc/akyldash/recovery-target.'
+fi
diff --git a/infra/bootstrap/host-guard.sh b/infra/bootstrap/host-guard.sh
new file mode 100644
index 0000000..950c2a5
--- /dev/null
+++ b/infra/bootstrap/host-guard.sh
@@ -0,0 +1,11 @@
+akyldash_is_production_host() {
+  local hostname=${1:-}
+  local ipv4_addresses=${2:-}
+  if [[ -z $hostname ]]; then
+    hostname=$(hostname -f 2>/dev/null || true)
+  fi
+  if [[ -z $ipv4_addresses ]]; then
+    ipv4_addresses=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  fi
+  [[ $hostname == avprogs.hlab.kz ]] || grep -Fqx '206.223.249.118' <<<"$ipv4_addresses"
+}
diff --git a/infra/bootstrap/test-host-guard.sh b/infra/bootstrap/test-host-guard.sh
new file mode 100644
index 0000000..4acba73
--- /dev/null
+++ b/infra/bootstrap/test-host-guard.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+
+akyldash_is_production_host avprogs.hlab.kz 127.0.0.1
+akyldash_is_production_host other.example $'10.0.0.2\n206.223.249.118'
+if akyldash_is_production_host recovery.example 10.0.0.2; then
+  echo 'FAIL unrelated recovery host identified as production' >&2
+  exit 1
+fi
+echo 'PASS recovery host identity guard'
-- 
2.54.0


From 6f809d3196f83bad6f777d0068cf7b68fc0f1bad Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:30:22 +0300
Subject: [PATCH 10/17] Add guarded rollback and recovery apply tools

---
 .gitea/workflows/ci.yaml           |   2 +
 .impeccable/config.json            |   4 +-
 AGENTS.md                          |   4 +-
 deploy/production/.env.example     |   2 +
 deploy/production/compose.yaml     |   2 +
 docs/operations/ACCESS.md          |   9 +-
 docs/operations/BACKUP.md          |  29 +++-
 docs/operations/CURRENT_STATE.md   |   8 +-
 docs/operations/DECISIONS.md       |   2 +-
 docs/operations/DEPLOYMENT.md      |  28 ++-
 docs/operations/RECOVERY.md        |  30 ++--
 docs/operations/ROLLBACK.md        |  35 +++-
 infra/backup/apply_restore.py      | 267 +++++++++++++++++++++++++++++
 infra/backup/archive.py            |  25 ++-
 infra/backup/test_apply_restore.py |  90 ++++++++++
 infra/backup/test_archive.py       |   3 +-
 infra/bootstrap/verify.sh          |  16 +-
 infra/deploy/install-helper.sh     |  85 +++++++++
 infra/deploy/rollback.py           | 162 +++++++++++++++++
 infra/deploy/test_rollback.py      |  36 ++++
 infra/inventory/production.yml     |   4 +
 21 files changed, 783 insertions(+), 60 deletions(-)
 create mode 100644 infra/backup/apply_restore.py
 create mode 100644 infra/backup/test_apply_restore.py
 create mode 100644 infra/deploy/install-helper.sh
 create mode 100644 infra/deploy/rollback.py
 create mode 100644 infra/deploy/test_rollback.py

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index bcc8dc6..a3cb1aa 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -21,6 +21,8 @@ jobs:
         run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
       - name: Check recovery host guard
         run: bash infra/bootstrap/test-host-guard.sh
+      - name: Run rollback helper tests
+        run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
       - name: Validate production Compose and Caddyfile
         run: |
           secret_file=$(mktemp)
diff --git a/.impeccable/config.json b/.impeccable/config.json
index a9cc933..496c0ff 100644
--- a/.impeccable/config.json
+++ b/.impeccable/config.json
@@ -10,7 +10,7 @@
           "backend/search/review.html"
         ],
         "createdAt": "2026-10-11T15:40:02.635Z",
-        "reason": "Agent: DESIGN.md explicitly defines #F7F2E8 as the approved product background"
+        "reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette"
       },
       {
         "rule": "cramped-padding",
@@ -19,7 +19,7 @@
           "backend/search/review.html"
         ],
         "createdAt": "2026-10-11T15:41:28.434Z",
-        "reason": "Agent: both reported sections declare 1rem minimum padding in CSS; detector reports flush content despite that inset"
+        "reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector"
       },
       {
         "rule": "design-system-font",
diff --git a/AGENTS.md b/AGENTS.md
index 3a71787..07077e7 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -15,7 +15,9 @@ Python, SQLite и OpenSearch. Production пока не развернут.
   Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
 - Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
   `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
-  `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v`.
+  `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`,
+  `bash infra/bootstrap/test-host-guard.sh`.
   Production images собираются командами из `.gitea/workflows/ci.yaml`.
 - Перед изменениями production необходим проверенный backup. Формального
   механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
diff --git a/deploy/production/.env.example b/deploy/production/.env.example
index e4d4a4d..63ae916 100644
--- a/deploy/production/.env.example
+++ b/deploy/production/.env.example
@@ -1,6 +1,8 @@
 # Absolute path on the production host containing minjust-normalized/.
 DATA_ROOT=/var/lib/akyldash/data
 REVIEW_SECRET_FILE=/etc/akyldash/review_secret
+BACKEND_IMAGE=akyldash/backend:local
+OPENSEARCH_IMAGE=akyldash/opensearch:local
 SEARCH_INDEX=akyldash-fragments-current
 OPENSEARCH_MEM_LIMIT=4g
 OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
diff --git a/deploy/production/compose.yaml b/deploy/production/compose.yaml
index 232d1a9..636efb4 100644
--- a/deploy/production/compose.yaml
+++ b/deploy/production/compose.yaml
@@ -28,6 +28,7 @@ services:
       retries: 3
 
   opensearch:
+    image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local}
     build:
       context: ../..
       dockerfile: deploy/production/Dockerfile.opensearch
@@ -63,6 +64,7 @@ services:
       retries: 10
 
   backend:
+    image: ${BACKEND_IMAGE:-akyldash/backend:local}
     build:
       context: ../..
       dockerfile: deploy/production/Dockerfile.backend
diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md
index b3458e7..fc48a5e 100644
--- a/docs/operations/ACCESS.md
+++ b/docs/operations/ACCESS.md
@@ -24,8 +24,11 @@ checks. Never print a private key, token, or complete environment file.
 - `codex` can run unprivileged read-only diagnostics. The audited account has no
   non-interactive sudo and must not be added to `sudo` or the `docker` group.
 - Host package installation, service setup, firewall changes, secret creation,
-  backup restore, and production changes require the VPS owner to run a reviewed
-  root command. No restricted deploy-helper is installed yet.
+  backup creation, recovery apply, and deployment require owner approval and a
+  reviewed root command. After the owner installs the root-owned repository
+  checkout, `infra/deploy/install-helper.sh` can grant only the validated
+  rollback command through sudoers; that helper is not installed on the audited
+  VPS.
 - Never connect to other VPS hosts or Synology as part of this deployment.
 
 This execution environment's system SSH config include had unsafe file
@@ -53,6 +56,6 @@ retrieve it from another machine.
 | Role | Operations |
 |---|---|
 | `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners |
-| Restricted deploy-helper | Intended future allowlisted backup/deploy/status actions; not installed |
+| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
 | VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
 | Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index 0be85c6..b14eaa8 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -39,10 +39,10 @@ sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYM
 Replace the timestamp placeholder with one real UTC timestamp before running.
 The create command refuses running services and archives the application data,
 OpenSearch volume, Caddy data/config volumes, production environment file, and
-review secret. It records the checked-out Git revision and image IDs for each
-stopped Compose service, writes a SHA-256 sidecar, and refuses to overwrite an
-existing archive. It also runs SQLite `quick_check`. Start the stack again only
-after verification succeeds:
+review secret. It records the checked-out Git revision, deployment-file hashes,
+and image references/IDs for each stopped Compose service, writes a SHA-256
+sidecar, and refuses to overwrite an existing archive. It also runs SQLite
+`quick_check`. Start the stack again only after verification succeeds:
 
 ```bash
 cd /opt/akyldash/current
@@ -52,14 +52,25 @@ sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production
 Keep the local archive root-only, then copy it to separately managed encrypted
 storage. A local archive alone does not protect against loss of the VPS disk.
 Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
-`/var/tmp` and validates the checksum and SQLite databases; it never writes
-production paths. Example test on an isolated host:
+`/var/tmp` and validates the checksum and SQLite databases. Applying the staged
+contents requires the separate recovery-target tool; it refuses the inventoried
+production host, existing data/volumes, and an unmarked host. Example isolated
+restore:
 
 ```bash
 sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
   --target-dir /var/tmp/akyldash-restore-test
 ```
 
-This validates and stages the archive; it does not install data into Compose
-volumes. Complete and verify the host-specific restore on a separate recovery
-host before relying on it. Do not run an unreviewed restore over existing data.
+This validates and stages the archive. On the replacement host only, after
+running bootstrap with `--recovery-target` and checking out the matching Git
+revision, apply the staged data:
+
+```bash
+sudo python3 infra/backup/apply_restore.py \
+  --staging-dir /var/tmp/akyldash-restore-test
+```
+
+The apply command restores data and named volumes but leaves services stopped.
+Test the full procedure on a separate recovery host before relying on it. Do
+not run an unreviewed restore over existing data.
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 390c611..d387f82 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -44,12 +44,14 @@ from planned configuration.
   application data use persistent volumes/bind mounts. TLS, registry digest
   deployment for backend/OpenSearch, backup, rollback helper, and systemd
   deployment are not configured on the host.
-- The repository has a cold-backup archive/verification tool with staging-only
-  restore and unit tests. It has not run against production; do not treat this
-  as an active backup or verified recovery.
+- The repository has a cold-backup archive/verification tool and a one-shot
+  restore apply tool that requires a marked replacement host. Backup/restore
+  tests use synthetic data; neither tool has run against production.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.
+- The repository defines a root-owned rollback helper and owner-run installer
+  with a single-command sudoers allowlist. Neither is installed on the VPS.
 - An isolated local Compose smoke test built the backend/OpenSearch images,
   reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
   200 for `/review` through Caddy. Its containers, networks, and volumes were
diff --git a/docs/operations/DECISIONS.md b/docs/operations/DECISIONS.md
index e665643..f57e17b 100644
--- a/docs/operations/DECISIONS.md
+++ b/docs/operations/DECISIONS.md
@@ -3,7 +3,7 @@
 | Date | Decision | Reason and alternatives | Status / limits |
 |---|---|---|---|
 | 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
-| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; use an owner-run bootstrap and a narrowly reviewed deploy-helper | No helper installed yet |
+| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only the root-owned rollback helper after checkout ownership and sudoers validation | Helper is defined in Git; not installed on VPS |
 | 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
 | 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
 | 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index 65e1dd1..070569d 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -18,21 +18,31 @@ configuration, and a backup and restore have been tested.
    versions and does not change SSH, UFW, sudoers, or user groups. Use
    `--recovery-target` only on a replacement host; it refuses the known
    production host identity.
-2. Install the approved repository checkout at `/opt/akyldash/current` and
-   configure `/etc/akyldash/production.env` plus
-   `/etc/akyldash/review_secret` outside Git. The secret file must be
-   root-readable only.
-3. Confirm DNS points through Cloudflare to this origin, then have the owner
+2. Install the approved repository checkout at `/opt/akyldash/current`,
+   root-owned and not group/world-writable. Configure
+   `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
+   Git. The secret file must be root-readable only.
+3. After reviewing the root-owned checkout, run
+   `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader and
+   rollback helper and adds a sudoers rule for that helper only; it does not
+   grant general sudo or Docker access.
+4. Confirm DNS points through Cloudflare to this origin, then have the owner
    review firewall rules and open only TCP `80` and `443` in addition to the
    existing SSH access. The Compose Caddy service obtains and renews its HTTPS
    certificate automatically; Cloudflare Full (strict) still needs owner-side
    verification.
-4. Create and verify the first backup before starting services. Load and
+5. Create and verify the first backup before starting services. Load and
    validate the OpenSearch index, create the initial admin interactively, and
-   verify `/review` over HTTPS.
+   verify `/review` over HTTPS. Record the backend image ID for drift checks:
 
-The remaining steps are intentionally manual until a restricted deploy-helper,
-registry permissions, a live Gitea runner, and the backup path are tested.
+   ```bash
+   sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
+   sudo chmod 0600 /opt/akyldash/deployed-image.txt
+   ```
+
+Application deployment remains owner-run until registry permissions, a live
+Gitea runner, and a controlled deploy procedure are tested. The helper only
+supports a guarded application rollback.
 
 ## CI
 
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
index 9f82d25..96b86ca 100644
--- a/docs/operations/RECOVERY.md
+++ b/docs/operations/RECOVERY.md
@@ -15,24 +15,32 @@ run recovery commands against the existing production by accident.
 2. Create the SSH-authorized `codex` account without sudo or Docker group access.
    Preserve the host's existing root SSH and firewall policy until an owner has
    reviewed the change.
-3. Check out the approved Git revision and run
+3. Check out the approved Git revision, root-owned and not group/world-writable,
+   at `/opt/akyldash/current`, then run
    `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
    recovery mode refuses the inventoried production hostname/IP and writes a
    root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
    Engine, Compose, Git, Docker log limits, and target directories.
-4. Configure DNS to point through Cloudflare to the recovery host and allow the
+4. Install the limited rollback helper with
+   `sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
+   checkout and gives `codex` access only to that helper.
+5. Configure DNS to point through Cloudflare to the recovery host and allow the
    owner to review/open TCP `80` and `443`. The production Compose Caddy service
    obtains its origin certificate automatically. Verify Cloudflare Full
    (strict) end to end before cutover.
-5. Provide production configuration and secret files through the owner's
-   protected channel; never commit or paste secret values into chat.
-6. Verify the archive and extract it to a new `/var/tmp/akyldash-restore-*`
-   staging directory with `infra/backup/archive.py`. The tool checks the archive
-   checksum and SQLite integrity but does not install staged files into Compose
-   volumes; have the owner review and perform that host-specific step. Validate
-   the corpus and OpenSearch index before serving traffic.
-7. Start the approved image digest, verify container health and
-   `https://akyldash.kg/review`, then record actual versions and checks in
+6. Transfer the protected backup to the new host through the owner's approved
+   channel. The archive contains the production environment and review secret;
+   keep it root-only and never paste secret values into chat.
+7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging
+   directory with `infra/backup/archive.py`, then run
+   `sudo python3 infra/backup/apply_restore.py --staging-dir <staging-dir>`.
+   Apply requires the recovery marker, matching clean Git revision, empty data
+   path, absent Compose volumes/config files, and no running containers. It
+   refuses the inventoried production host and leaves services stopped.
+8. Build the backend and OpenSearch images from the matching checked-out Git
+   revision and explicitly start Compose with `--wait`. Verify HTTPS and
+   container health, record the backend image ID as described in
+   [DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in
    [CURRENT_STATE.md](CURRENT_STATE.md).
 
 Do not repoint DNS or retire the old host until the owner verifies recovery and
diff --git a/docs/operations/ROLLBACK.md b/docs/operations/ROLLBACK.md
index 8d0a83f..c0d7bef 100644
--- a/docs/operations/ROLLBACK.md
+++ b/docs/operations/ROLLBACK.md
@@ -2,16 +2,33 @@
 
 ## Current status
 
-There is no production release, pinned image digest, backup, or deploy-helper
-to roll back. Automatic rollback is not configured.
+There is no production release, backup, or installed rollback helper. The
+repository contains a root-owned rollback helper and a separate installer, but
+neither has been run on the VPS.
 
-After deployment automation is reviewed, rollback may switch the backend to a
-previous immutable image digest only after health checks fail and only when the
-data format remains compatible. This application currently has no formal
-migration framework, but its SQLite files and OpenSearch indexes are persistent;
-compatibility must be checked for each change. If data compatibility is
-uncertain, stop and report rather than restoring or deleting production data.
+After a verified cold backup is created before a release, an owner must check
+that the data format is compatible with the prior backend/OpenSearch images.
+Then run the limited helper with that archive:
+
+```bash
+sudo -n /usr/local/sbin/akyldash-rollback \
+  /var/backups/akyldash/<pre-release-backup>.tar.gz --data-compatible
+```
+
+The helper verifies archive integrity, Compose image references, deployment
+file hashes, and the availability of both prior local images. It makes and
+verifies a second cold backup, restores only the prior backend/OpenSearch image
+tags, waits for Compose healthchecks, and checks HTTPS locally. It refuses to
+restore if the current Compose/Caddyfile differs from the saved release. This
+does not restore data or automatically undo a release with an unknown schema
+change. The application has no formal migration framework; compatibility must
+be checked for every change. If it is uncertain, stop and ask the owner.
+
+The initial owner-run deployment must record its running backend image ID in
+`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates
+that record after a successful rollback, and `infra/bootstrap/verify.sh` checks
+it against the running container.
 
 Any data restore requires the owner, a verified pre-change backup, and a
-maintenance window. Preserve the failed release and its backup until recovery
+maintenance window. Preserve the failed release and both backups until recovery
 is confirmed.
diff --git a/infra/backup/apply_restore.py b/infra/backup/apply_restore.py
new file mode 100644
index 0000000..d283a74
--- /dev/null
+++ b/infra/backup/apply_restore.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+"""Install a verified staging restore onto an explicitly marked new VPS."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+from pathlib import Path
+
+from archive import COMPONENTS, check_sqlite, digest
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_DEST = Path("/etc/akyldash/production.env")
+SECRET_DEST = Path("/etc/akyldash/review_secret")
+DATA_DEST = Path("/var/lib/akyldash/data")
+MARKER = Path("/etc/akyldash/recovery-target")
+PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+VOLUME_TARGETS = {
+    "/usr/share/opensearch/data": "opensearch-data",
+    "/data": "caddy-data",
+    "/config": "caddy-config",
+}
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def require_empty_directory(path: Path) -> None:
+    if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
+        raise ValueError(f"restore destination must be an existing empty directory: {path}")
+
+
+def tree_size(root: Path) -> int:
+    total = 0
+    for directory, names, files in os.walk(root, followlinks=False):
+        names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
+        for name in files:
+            path = Path(directory) / name
+            if path.is_symlink():
+                continue
+            if path.is_file():
+                total += path.stat().st_size
+    return total
+
+
+def copy_contents(source: Path, destination: Path) -> None:
+    for child in source.iterdir():
+        target = destination / child.name
+        if child.is_dir() and not child.is_symlink():
+            shutil.copytree(child, target, symlinks=True)
+        else:
+            shutil.copy2(child, target, follow_symlinks=False)
+
+
+def empty_directory(path: Path) -> None:
+    for child in path.iterdir():
+        if child.is_dir() and not child.is_symlink():
+            shutil.rmtree(child)
+        else:
+            child.unlink()
+
+
+def validate_stage(staging: Path) -> dict:
+    staging = staging.resolve(strict=True)
+    if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
+        raise ValueError("staging directory must be a direct child of /var/tmp")
+    if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
+        raise ValueError("staging directory must be root-owned mode 0700")
+    manifest_path = staging / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or set(manifest.get("components", [])) != COMPONENTS
+    ):
+        raise ValueError("staging manifest is invalid")
+    for name in COMPONENTS - {"production-env", "review-secret"}:
+        component = staging / name
+        if not component.is_dir() or component.is_symlink():
+            raise ValueError(f"staged component must be a directory: {name}")
+    for name in ("production-env", "review-secret"):
+        path = staging / name
+        if not path.is_file() or path.is_symlink():
+            raise ValueError(f"staged sensitive component must be a regular file: {name}")
+    return manifest
+
+
+def validate_checkout(manifest: dict) -> None:
+    if ROOT.is_symlink() or not (ROOT / ".git").exists():
+        raise ValueError("approved Git checkout is missing or is a symlink")
+    if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
+        raise ValueError("recovery checkout must be clean")
+    unsafe = subprocess.run(
+        ["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if unsafe:
+        raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
+    revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
+    if revision != manifest.get("git_revision"):
+        raise ValueError("checkout Git revision does not match the backup manifest")
+    for relative, expected in manifest.get("deployment_files", {}).items():
+        if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
+            raise ValueError("Compose or Caddyfile does not match the backup manifest")
+    if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
+        raise ValueError("backup manifest is missing deployment file hashes")
+
+
+def validate_recovery_target() -> None:
+    if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("root-only recovery-target marker is required")
+    hostname = socket.getfqdn()
+    addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
+    if not hostname or not addresses:
+        raise ValueError("cannot verify the recovery host identity")
+    if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
+        raise ValueError("refusing to apply recovery data to the inventoried production VPS")
+
+
+def compose_configuration(staging: Path) -> dict:
+    rendered = run(
+        [
+            "docker", "compose", "--env-file", str(staging / "production-env"),
+            "-f", str(COMPOSE_FILE), "config", "--format", "json",
+        ],
+        capture=True,
+    )
+    config = json.loads(rendered)
+    data_source = next(
+        mount["source"]
+        for mount in config["services"]["backend"]["volumes"]
+        if mount["target"] == "/app/data"
+    )
+    secret_source = config["secrets"]["review_secret"]["file"]
+    if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
+        raise ValueError("backup production paths do not match the clean recovery host layout")
+    return config
+
+
+def expected_volume_names(config: dict) -> set[str]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    names: set[str] = set()
+    for target in VOLUME_TARGETS:
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        names.add(config["volumes"][mount["source"]]["name"])
+    return names
+
+
+def volume_destinations(config: dict) -> dict[str, Path]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    destinations = {}
+    for target, component in VOLUME_TARGETS.items():
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        name = config["volumes"][mount["source"]]["name"]
+        destinations[component] = Path(
+            run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
+        )
+    return destinations
+
+
+def apply_restore(staging: Path) -> None:
+    if os.geteuid() != 0:
+        raise ValueError("restore apply must run as root")
+    validate_recovery_target()
+    manifest = validate_stage(staging)
+    validate_checkout(manifest)
+    config = compose_configuration(staging)
+    require_empty_directory(DATA_DEST)
+    if ENV_DEST.exists() or SECRET_DEST.exists():
+        raise ValueError("production environment and secret files must not already exist")
+    if run(["docker", "ps", "--quiet"], capture=True):
+        raise ValueError("recovery target must have no running Docker containers")
+
+    volume_names = expected_volume_names(config)
+    for name in volume_names:
+        result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
+        if result.returncode == 0:
+            raise ValueError(f"recovery volume already exists: {name}")
+
+    created_volumes: list[str] = []
+    created_files: list[Path] = []
+    data_started = False
+    try:
+        for name in volume_names:
+            run(["docker", "volume", "create", name])
+            created_volumes.append(name)
+
+        destinations = volume_destinations(config)
+        destinations["data"] = DATA_DEST
+
+        required: dict[int, int] = {}
+        free: dict[int, int] = {}
+        for component, destination in destinations.items():
+            device = destination.stat().st_dev
+            required[device] = required.get(device, 0) + tree_size(staging / component)
+            free[device] = shutil.disk_usage(destination).free
+        for device, size in required.items():
+            if free[device] < size + 512 * 1024 * 1024:
+                raise ValueError("recovery host lacks space for a safe data copy")
+
+        for component, destination in destinations.items():
+            require_empty_directory(destination)
+            if component == "data":
+                data_started = True
+            copy_contents(staging / component, destination)
+
+        for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
+            os.chown(directory, 1000, 1000, follow_symlinks=False)
+            for name in files:
+                os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
+
+        for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
+            shutil.copyfile(source, destination)
+            created_files.append(destination)
+            os.chown(destination, 0, 0)
+            os.chmod(destination, 0o600)
+        check_sqlite(DATA_DEST)
+    except BaseException:
+        for path in created_files:
+            path.unlink(missing_ok=True)
+        if data_started:
+            empty_directory(DATA_DEST)
+        for name in created_volumes:
+            subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
+        raise
+
+    MARKER.unlink()
+    print("PASS staged recovery data applied to the marked new host; services remain stopped")
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--staging-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        apply_restore(args.staging_dir)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/infra/backup/archive.py b/infra/backup/archive.py
index a20dd99..e623ac4 100644
--- a/infra/backup/archive.py
+++ b/infra/backup/archive.py
@@ -80,6 +80,8 @@ def read_manifest(archive: tarfile.TarFile) -> dict:
         raise ValueError("backup manifest has an unsupported format or component set")
     if not isinstance(manifest.get("images"), dict):
         raise ValueError("backup manifest has no image records")
+    if not isinstance(manifest.get("deployment_files"), dict):
+        raise ValueError("backup manifest has no deployment file records")
     for member in archive.getmembers():
         if member.name == "manifest.json":
             continue
@@ -125,7 +127,9 @@ def check_sqlite(root: Path) -> None:
             connection.close()
 
 
-def compose_sources(compose_file: Path, env_file: Path) -> tuple[dict[str, Path], str, dict[str, dict[str, str]]]:
+def compose_sources(
+    compose_file: Path, env_file: Path
+) -> tuple[dict[str, Path], str, dict[str, dict[str, str]], dict[str, str]]:
     command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
     rendered = subprocess.run(
         [*command, "config", "--format", "json"], check=True, capture_output=True, text=True
@@ -209,10 +213,20 @@ def compose_sources(compose_file: Path, env_file: Path) -> tuple[dict[str, Path]
         capture_output=True,
         text=True,
     ).stdout.strip()
-    return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images
+    deployment_files = {
+        relative: digest(repository / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images, deployment_files
 
 
-def create_backup(output: Path, sources: dict[str, Path], revision: str, images: dict | None = None) -> None:
+def create_backup(
+    output: Path,
+    sources: dict[str, Path],
+    revision: str,
+    images: dict | None = None,
+    deployment_files: dict[str, str] | None = None,
+) -> None:
     output = output.absolute()
     parent = output.parent
     if not parent.is_dir() or parent.stat().st_mode & 0o077:
@@ -235,6 +249,7 @@ def create_backup(output: Path, sources: dict[str, Path], revision: str, images:
         "created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
         "git_revision": revision,
         "images": images or {},
+        "deployment_files": deployment_files or {},
         "components": sorted(sources),
     }
     with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
@@ -315,8 +330,8 @@ def main() -> int:
         if args.command in {"create", "restore"} and os.geteuid() != 0:
             raise ValueError("create and restore must run as root to protect production secrets and data")
         if args.command == "create":
-            sources, revision, images = compose_sources(args.compose_file, args.env_file)
-            create_backup(args.output, sources, revision, images)
+            sources, revision, images, deployment_files = compose_sources(args.compose_file, args.env_file)
+            create_backup(args.output, sources, revision, images, deployment_files)
         elif args.command == "verify":
             manifest = verify_archive(args.archive)
             print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
diff --git a/infra/backup/test_apply_restore.py b/infra/backup/test_apply_restore.py
new file mode 100644
index 0000000..54f1cdb
--- /dev/null
+++ b/infra/backup/test_apply_restore.py
@@ -0,0 +1,90 @@
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import apply_restore
+
+
+class ApplyRestoreTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.staging = self.root / "akyldash-restore-test"
+        self.staging.mkdir()
+        for component in apply_restore.COMPONENTS:
+            path = self.staging / component
+            if component in {"production-env", "review-secret"}:
+                path.write_text("private", encoding="utf-8")
+            else:
+                path.mkdir()
+                (path / "payload.txt").write_text(component, encoding="utf-8")
+        self.data = self.root / "data"
+        self.data.mkdir()
+        self.volumes = {
+            name: self.root / name
+            for name in ("opensearch", "caddy-data", "caddy-config")
+        }
+        for path in self.volumes.values():
+            path.mkdir()
+        self.env = self.root / "production.env"
+        self.secret = self.root / "review_secret"
+        self.marker = self.root / "recovery-target"
+        self.marker.touch(mode=0o600)
+        self.config = {
+            "volumes": {
+                "opensearch-data": {"name": "akyldash_opensearch-data"},
+                "caddy-data": {"name": "akyldash_caddy-data"},
+                "caddy-config": {"name": "akyldash_caddy-config"},
+            }
+        }
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def run_apply(self):
+        def docker_run(command, **_kwargs):
+            if command[:3] == ["docker", "ps", "--quiet"]:
+                return ""
+            if command[1:3] == ["volume", "create"]:
+                return command[-1]
+            return ""
+
+        with (
+            patch("apply_restore.os.geteuid", return_value=0),
+            patch("apply_restore.validate_recovery_target"),
+            patch("apply_restore.validate_stage", return_value={"format": 1}),
+            patch("apply_restore.validate_checkout"),
+            patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}),
+            patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])),
+            patch("apply_restore.volume_destinations", return_value=self.volumes),
+            patch("apply_restore.DATA_DEST", self.data),
+            patch("apply_restore.ENV_DEST", self.env),
+            patch("apply_restore.SECRET_DEST", self.secret),
+            patch("apply_restore.MARKER", self.marker),
+            patch("apply_restore.run", side_effect=docker_run),
+            patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)),
+            patch("apply_restore.os.chown"),
+        ):
+            apply_restore.apply_restore(self.staging)
+
+    def test_applies_components_and_protects_secret_files(self):
+        self.run_apply()
+        self.assertEqual((self.data / "payload.txt").read_text(), "data")
+        self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch")
+        self.assertEqual(self.env.stat().st_mode & 0o777, 0o600)
+        self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600)
+        self.assertFalse(self.marker.exists())
+
+    def test_refuses_nonempty_data_destination_before_writing_secrets(self):
+        (self.data / "keep.txt").write_text("existing", encoding="utf-8")
+        with self.assertRaisesRegex(ValueError, "empty directory"):
+            self.run_apply()
+        self.assertEqual((self.data / "keep.txt").read_text(), "existing")
+        self.assertFalse(self.env.exists())
+        self.assertTrue(self.marker.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
index f00708e..77d6bcf 100644
--- a/infra/backup/test_archive.py
+++ b/infra/backup/test_archive.py
@@ -141,12 +141,13 @@ class BackupArchiveTests(unittest.TestCase):
             return SimpleNamespace(stdout="")
 
         with patch("archive.subprocess.run", side_effect=run):
-            sources, revision, images = compose_sources(
+            sources, revision, images, deployment_files = compose_sources(
                 Path.cwd() / "deploy/production/compose.yaml", env_file
             )
         self.assertEqual(revision, "")
         self.assertEqual(set(sources), COMPONENTS)
         self.assertEqual(images["backend"]["image_id"], "sha256:backend")
+        self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
 
 
 if __name__ == "__main__":
diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh
index dd167c2..e472eba 100755
--- a/infra/bootstrap/verify.sh
+++ b/infra/bootstrap/verify.sh
@@ -129,18 +129,22 @@ for endpoint in $udp_listeners; do
   failed=1
 done
 
-if [[ -x /usr/local/sbin/akyldash-deploy ]]; then
-  echo 'PASS restricted deploy helper present'
+sudoers_rule='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+if [[ -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
+      -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
+      -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
+      $(cat /etc/sudoers.d/akyldash-codex) == "$sudoers_rule" ]]; then
+  echo 'PASS limited rollback helper and sudoers rule present'
 else
-  echo 'FAIL restricted deploy helper missing'
+  echo 'FAIL limited rollback helper or sudoers rule missing/different'
   failed=1
 fi
 if [[ -f /opt/akyldash/deployed-image.txt ]]; then
   expected=$(cat /opt/akyldash/deployed-image.txt)
   container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
-  actual=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || true)
-  if [[ $expected == *@sha256:* && $actual == "$expected" ]]; then
-    printf 'PASS running backend image matches %s\n' "$expected"
+  actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)
+  if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then
+    printf 'PASS running backend image ID matches %s\n' "$expected"
   else
     echo 'FAIL running backend image does not match recorded digest'
     failed=1
diff --git a/infra/deploy/install-helper.sh b/infra/deploy/install-helper.sh
new file mode 100644
index 0000000..ec14933
--- /dev/null
+++ b/infra/deploy/install-helper.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo "Run this installer as root." >&2
+  exit 1
+fi
+
+checkout=/opt/akyldash/current
+rollback_source=$checkout/infra/deploy/rollback.py
+backup_source=$checkout/infra/backup/archive.py
+if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source ]]; then
+  echo "Install the approved repository checkout at $checkout first." >&2
+  exit 1
+fi
+if [[ -L $checkout ]]; then
+  echo "The checkout path must not be a symlink." >&2
+  exit 1
+fi
+for directory in /opt /opt/akyldash; do
+  mode=$(stat -c %a "$directory")
+  if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then
+    echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2
+    exit 1
+  fi
+done
+if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo "Create codex without sudo or Docker-group membership before installing the helper." >&2
+  exit 1
+fi
+if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then
+  echo "Existing codex sudoers rules need owner review before installing the helper." >&2
+  exit 1
+fi
+if [[ $(stat -c %U "$checkout") != root ]] || find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit | grep -q .; then
+  echo "The checkout must be root-owned and not group/world-writable." >&2
+  exit 1
+fi
+if ! command -v visudo >/dev/null 2>&1; then
+  echo "visudo is required to validate the restricted sudoers rule." >&2
+  exit 1
+fi
+
+for path in /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
+  if [[ -L $path ]]; then
+    echo "Refusing symlink at $path." >&2
+    exit 1
+  fi
+done
+
+install -d -o root -g root -m 0755 /usr/local/lib/akyldash
+install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
+install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
+
+sudoers_file=/etc/sudoers.d/akyldash-codex
+expected='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+created=false
+if [[ -e $sudoers_file ]]; then
+  if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2
+    exit 1
+  fi
+else
+  temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX)
+  trap 'rm -f "$temp"' EXIT
+  printf '%s\n' "$expected" > "$temp"
+  chmod 0440 "$temp"
+  visudo -cf "$temp"
+  visudo -c -f /etc/sudoers
+  mv -n "$temp" "$sudoers_file"
+  if [[ $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "Could not install the expected sudoers rule." >&2
+    exit 1
+  fi
+  created=true
+fi
+
+if ! visudo -c -f /etc/sudoers; then
+  if [[ $created == true ]]; then
+    rm -f "$sudoers_file"
+  fi
+  echo "Sudoers validation failed; the newly added rule was removed." >&2
+  exit 1
+fi
+echo "Installed root-owned rollback helper; codex receives no general sudo or Docker access."
diff --git a/infra/deploy/rollback.py b/infra/deploy/rollback.py
new file mode 100644
index 0000000..a08e5ff
--- /dev/null
+++ b/infra/deploy/rollback.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Restore prior local application images from a verified cold backup."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_FILE = Path("/etc/akyldash/production.env")
+BACKUP_DIR = Path("/var/backups/akyldash")
+BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
+APP_SERVICES = {"backend", "opensearch"}
+
+
+def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
+    images = manifest.get("images")
+    if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
+        raise ValueError("backup has no previous backend and OpenSearch images")
+    plan = {}
+    for service in sorted(APP_SERVICES):
+        image = images[service]
+        if not isinstance(image, dict):
+            raise ValueError(f"invalid image record for {service}")
+        reference = image.get("configured")
+        image_id = image.get("image_id")
+        if reference != configured.get(service):
+            raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
+        if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*", reference):
+            raise ValueError(f"{service} image reference is not a local tag")
+        if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+            raise ValueError(f"invalid previous image ID for {service}")
+        plan[service] = (image_id, reference)
+    return plan
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def compose(*args: str, capture: bool = False) -> str:
+    return run(
+        ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
+        capture=capture,
+    )
+
+
+def record_backend_image() -> None:
+    container = compose("ps", "--quiet", "backend", capture=True)
+    if not container:
+        raise ValueError("backend container is missing after rollback")
+    image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
+    if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+        raise ValueError("backend returned an invalid image ID")
+    target = Path("/opt/akyldash/deployed-image.txt")
+    with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
+        temporary = Path(stream.name)
+        stream.write(f"{image_id}\n".encode("ascii"))
+    os.chmod(temporary, 0o600)
+    try:
+        os.replace(temporary, target)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def rollback(backup: Path, data_compatible: bool) -> None:
+    if not data_compatible:
+        raise ValueError("confirm data compatibility with --data-compatible before rollback")
+    if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
+        raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
+    if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback backup must be root-owned mode 0600")
+    checksum = Path(f"{backup}.sha256")
+    if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
+
+    sys.path.insert(0, str(BACKUP_TOOL.parent))
+    from archive import digest, verify_archive
+
+    manifest = verify_archive(backup)
+    deployment_files = {
+        relative: digest(ROOT / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    if deployment_files != manifest.get("deployment_files"):
+        raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
+    rendered = json.loads(compose("config", "--format", "json", capture=True))
+    configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
+    plan = image_restore_plan(manifest, configured)
+    for image_id, _reference in plan.values():
+        actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
+        if actual != image_id:
+            raise ValueError(f"previous image is unavailable: {image_id}")
+
+    backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
+    backup_output = BACKUP_DIR / backup_name
+    compose("stop")
+    try:
+        run(
+            [
+                "python3",
+                str(BACKUP_TOOL),
+                "create",
+                "--compose-file",
+                str(COMPOSE_FILE),
+                "--env-file",
+                str(ENV_FILE),
+                "--output",
+                str(backup_output),
+            ]
+        )
+        run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
+    except Exception:
+        compose("up", "--detach")
+        raise
+
+    for image_id, reference in plan.values():
+        run(["docker", "image", "tag", image_id, reference])
+    compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
+    run(
+        [
+            "curl",
+            "--fail",
+            "--silent",
+            "--show-error",
+            "--resolve",
+            "akyldash.kg:443:127.0.0.1",
+            "https://akyldash.kg/review",
+            "--output",
+            "/dev/null",
+        ]
+    )
+    record_backend_image()
+    print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("backup", type=Path)
+    parser.add_argument("--data-compatible", action="store_true")
+    args = parser.parse_args()
+    try:
+        if os.geteuid() != 0:
+            raise ValueError("rollback must run through the restricted root helper")
+        rollback(args.backup, args.data_compatible)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/infra/deploy/test_rollback.py b/infra/deploy/test_rollback.py
new file mode 100644
index 0000000..7b5842b
--- /dev/null
+++ b/infra/deploy/test_rollback.py
@@ -0,0 +1,36 @@
+import unittest
+
+from rollback import image_restore_plan
+
+
+class RollbackPlanTests(unittest.TestCase):
+    def setUp(self):
+        self.manifest = {
+            "images": {
+                "backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
+                "opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
+            }
+        }
+        self.configured = {
+            "backend": "akyldash/backend:local",
+            "opensearch": "akyldash/opensearch:local",
+        }
+
+    def test_plans_only_recorded_local_images(self):
+        plan = image_restore_plan(self.manifest, self.configured)
+        self.assertEqual(plan["backend"][1], "akyldash/backend:local")
+        self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local")
+
+    def test_refuses_changed_compose_image_reference(self):
+        with self.assertRaisesRegex(ValueError, "reference changed"):
+            image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"})
+
+    def test_refuses_digest_reference_that_cannot_be_retagged(self):
+        self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64
+        self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
+        with self.assertRaisesRegex(ValueError, "local tag"):
+            image_restore_plan(self.manifest, self.configured)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
index a4b0ab3..2beb0cf 100644
--- a/infra/inventory/production.yml
+++ b/infra/inventory/production.yml
@@ -67,12 +67,16 @@ target:
     - REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
     - Gitea credentials from environment-backed gitea-api helper
     - SSH key at ~/.ssh/akyldash_codex
+  restricted_helper:
+    path: /usr/local/sbin/akyldash-rollback
+    status: repository code and owner-run installer exist; not installed on VPS
   checks:
     - infra/bootstrap/verify.sh akyldash_vps
     - docker compose -f deploy/production/compose.yaml ps
     - curl -fsS https://akyldash.kg/review
     - python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
   backup_path: /var/backups/akyldash (target; archive script in infra/backup/archive.py is not installed or run)
+  deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment)
 versions:
   application:
     backend: 0.11.3
-- 
2.54.0


From 711700fcc96655951e56a45a5d719ba09be08452 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:35:16 +0300
Subject: [PATCH 11/17] Document WSL access and verify VPS state

---
 docs/operations/ACCESS.md        | 13 ++++++++-----
 docs/operations/CURRENT_STATE.md |  8 +++++---
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md
index fc48a5e..26f3470 100644
--- a/docs/operations/ACCESS.md
+++ b/docs/operations/ACCESS.md
@@ -31,11 +31,14 @@ checks. Never print a private key, token, or complete environment file.
   VPS.
 - Never connect to other VPS hosts or Synology as part of this deployment.
 
-This execution environment's system SSH config include had unsafe file
-permissions, so plain `ssh akyldash_vps` failed. `ssh -F ~/.ssh/config ...`
-worked. Network policy may also require an approved SSH command. If the key is
-missing in a future execution environment, report that limitation; do not
-retrieve it from another machine.
+This execution environment is WSL2. The repository is on the Windows-mounted
+`/mnt/d` filesystem, while SSH config and the identity file are in the Linux
+home directory. Do not assume Windows SSH credentials are available or copy
+keys between environments. Plain `ssh akyldash_vps` failed because the system
+SSH config include had unsafe permissions. Use `ssh -F ~/.ssh/config ...`; in a
+network-restricted session this read-only command may also require approved
+execution. If the key is missing in a future environment, report that
+limitation and do not retrieve it elsewhere.
 
 ## Gitea
 
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index d387f82..8f3da6d 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -10,14 +10,16 @@ from planned configuration.
 - OS/kernel: Ubuntu Server `24.04.5 LTS`, `6.8.0-142-generic`.
 - Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 93 GiB free.
 - Network: interface `ens3`, IPv4 `206.223.249.118/32`.
-- `codex` is UID/GID 1000, groups `codex` and `users`; non-interactive sudo is
-  unavailable.
+- `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires
+  a password. The current WSL2 session verified the SSH alias using the
+  explicit user config and approved read-only SSH execution.
 - SSH service is running. The inspected sshd configuration has public-key and
   password authentication enabled and `PermitRootLogin yes`; no SSH changes
   were made.
 - Docker, Docker Compose, Git, Nginx, Caddy, Certbot, application paths, and
   application services are absent. Only base OS services and SSH were running.
-- Public listeners observed: TCP `22`; local resolver listeners on loopback.
+- Public listeners observed: TCP `22`; DNS listeners are limited to loopback
+  (`127.0.0.53` and `127.0.0.54`).
 - UFW is reported active in the supplied VPS notes, but the audit account could
   not independently read its status. Treat firewall state and rules as
   unverified.
-- 
2.54.0


From 02cf4e6e95455ddb174b4bcff77166cf52db0609 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:39:02 +0300
Subject: [PATCH 12/17] Expose drift verifier through a fixed sudo helper

---
 .gitea/workflows/ci.yaml         |  4 ++++
 docs/operations/ACCESS.md        |  7 ++++---
 docs/operations/CURRENT_STATE.md |  5 +++--
 docs/operations/DECISIONS.md     |  2 +-
 docs/operations/DEPLOYMENT.md    | 12 ++++++------
 docs/operations/RECOVERY.md      |  4 ++--
 infra/bootstrap/verify-helper.sh | 14 ++++++++++++++
 infra/bootstrap/verify.sh        | 13 +++++++------
 infra/deploy/install-helper.sh   | 13 ++++++++-----
 infra/inventory/production.yml   |  3 +++
 10 files changed, 52 insertions(+), 25 deletions(-)
 create mode 100644 infra/bootstrap/verify-helper.sh

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index a3cb1aa..5739fae 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -21,6 +21,10 @@ jobs:
         run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
       - name: Check recovery host guard
         run: bash infra/bootstrap/test-host-guard.sh
+      - name: Check infrastructure shell syntax
+        run: |
+          bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
+            infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/deploy/install-helper.sh
       - name: Run rollback helper tests
         run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
       - name: Validate production Compose and Caddyfile
diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md
index 26f3470..c33fd9b 100644
--- a/docs/operations/ACCESS.md
+++ b/docs/operations/ACCESS.md
@@ -26,9 +26,9 @@ checks. Never print a private key, token, or complete environment file.
 - Host package installation, service setup, firewall changes, secret creation,
   backup creation, recovery apply, and deployment require owner approval and a
   reviewed root command. After the owner installs the root-owned repository
-  checkout, `infra/deploy/install-helper.sh` can grant only the validated
-  rollback command through sudoers; that helper is not installed on the audited
-  VPS.
+  checkout, `infra/deploy/install-helper.sh` can grant a fixed read-only
+  verifier and guarded rollback command through sudoers; neither helper is
+  installed on the audited VPS.
 - Never connect to other VPS hosts or Synology as part of this deployment.
 
 This execution environment is WSL2. The repository is on the Windows-mounted
@@ -59,6 +59,7 @@ limitation and do not retrieve it elsewhere.
 | Role | Operations |
 |---|---|
 | `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners |
+| Restricted verifier | Runs the root-owned read-only drift check; not installed on the audited VPS |
 | Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
 | VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
 | Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 8f3da6d..18b0ade 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -52,8 +52,9 @@ from planned configuration.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.
-- The repository defines a root-owned rollback helper and owner-run installer
-  with a single-command sudoers allowlist. Neither is installed on the VPS.
+- The repository defines a root-owned read-only verifier, rollback helper, and
+  owner-run installer with exact sudoers command rules. None is installed on
+  the VPS.
 - An isolated local Compose smoke test built the backend/OpenSearch images,
   reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
   200 for `/review` through Caddy. Its containers, networks, and volumes were
diff --git a/docs/operations/DECISIONS.md b/docs/operations/DECISIONS.md
index f57e17b..32d6056 100644
--- a/docs/operations/DECISIONS.md
+++ b/docs/operations/DECISIONS.md
@@ -3,7 +3,7 @@
 | Date | Decision | Reason and alternatives | Status / limits |
 |---|---|---|---|
 | 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
-| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only the root-owned rollback helper after checkout ownership and sudoers validation | Helper is defined in Git; not installed on VPS |
+| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only a fixed read-only verifier and root-owned rollback helper after checkout ownership and sudoers validation | Helpers are defined in Git; not installed on VPS |
 | 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
 | 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
 | 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index 070569d..bb55f5f 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -5,9 +5,9 @@
 Production deployment is not enabled or tested on the VPS. An isolated local
 Compose smoke test passed; the VPS has no Docker, Git,
 application data, proxy, registry pull configuration, backup process, or
-restricted deploy-helper. Do not deploy until the owner has completed the root
-bootstrap, provided a protected secret file and a reviewed proxy/firewall
-configuration, and a backup and restore have been tested.
+restricted verification/rollback helpers. Do not deploy until the owner has
+completed the root bootstrap, provided a protected secret file and a reviewed
+proxy/firewall configuration, and a backup and restore have been tested.
 
 ## First host preparation
 
@@ -23,9 +23,9 @@ configuration, and a backup and restore have been tested.
    `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
    Git. The secret file must be root-readable only.
 3. After reviewing the root-owned checkout, run
-   `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader and
-   rollback helper and adds a sudoers rule for that helper only; it does not
-   grant general sudo or Docker access.
+   `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader,
+   read-only verifier, and rollback helper with exact sudoers command rules; it
+   does not grant general sudo or Docker access.
 4. Confirm DNS points through Cloudflare to this origin, then have the owner
    review firewall rules and open only TCP `80` and `443` in addition to the
    existing SSH access. The Compose Caddy service obtains and renews its HTTPS
diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md
index 96b86ca..03ed5e0 100644
--- a/docs/operations/RECOVERY.md
+++ b/docs/operations/RECOVERY.md
@@ -21,9 +21,9 @@ run recovery commands against the existing production by accident.
    recovery mode refuses the inventoried production hostname/IP and writes a
    root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
    Engine, Compose, Git, Docker log limits, and target directories.
-4. Install the limited rollback helper with
+4. Install the limited read-only verifier and rollback helper with
    `sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
-   checkout and gives `codex` access only to that helper.
+   checkout and gives `codex` access only to those fixed commands.
 5. Configure DNS to point through Cloudflare to the recovery host and allow the
    owner to review/open TCP `80` and `443`. The production Compose Caddy service
    obtains its origin certificate automatically. Verify Cloudflare Full
diff --git a/infra/bootstrap/verify-helper.sh b/infra/bootstrap/verify-helper.sh
new file mode 100644
index 0000000..98247ce
--- /dev/null
+++ b/infra/bootstrap/verify-helper.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+checkout=/opt/akyldash/current
+if [[ $# -ne 0 || $EUID -ne 0 || ! -d $checkout || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
+  echo 'Refusing verification: the approved root-owned checkout is unavailable.' >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ -n $unsafe ]]; then
+  echo 'Refusing verification: the checkout contains a writable or non-root-owned path.' >&2
+  exit 1
+fi
+exec /bin/bash "$checkout/infra/bootstrap/verify.sh" --remote
diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh
index e472eba..115b367 100755
--- a/infra/bootstrap/verify.sh
+++ b/infra/bootstrap/verify.sh
@@ -3,7 +3,7 @@ set -euo pipefail
 
 if [[ ${1:-} != --remote ]]; then
   ssh_alias=${1:-akyldash_vps}
-  exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n bash -s -- --remote' < "$0"
+  exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n /usr/local/sbin/akyldash-verify'
 fi
 
 if [[ ${EUID} -ne 0 ]]; then
@@ -129,14 +129,15 @@ for endpoint in $udp_listeners; do
   failed=1
 done
 
-sudoers_rule='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
-if [[ -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
+expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \
+      -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
       -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
       -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
-      $(cat /etc/sudoers.d/akyldash-codex) == "$sudoers_rule" ]]; then
-  echo 'PASS limited rollback helper and sudoers rule present'
+      $(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then
+  echo 'PASS read-only verifier, rollback helper, and exact sudoers rules present'
 else
-  echo 'FAIL limited rollback helper or sudoers rule missing/different'
+  echo 'FAIL restricted helper or sudoers installation missing/different'
   failed=1
 fi
 if [[ -f /opt/akyldash/deployed-image.txt ]]; then
diff --git a/infra/deploy/install-helper.sh b/infra/deploy/install-helper.sh
index ec14933..258af93 100644
--- a/infra/deploy/install-helper.sh
+++ b/infra/deploy/install-helper.sh
@@ -9,7 +9,8 @@ fi
 checkout=/opt/akyldash/current
 rollback_source=$checkout/infra/deploy/rollback.py
 backup_source=$checkout/infra/backup/archive.py
-if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source ]]; then
+verify_source=$checkout/infra/bootstrap/verify-helper.sh
+if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source ]]; then
   echo "Install the approved repository checkout at $checkout first." >&2
   exit 1
 fi
@@ -32,7 +33,8 @@ if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers
   echo "Existing codex sudoers rules need owner review before installing the helper." >&2
   exit 1
 fi
-if [[ $(stat -c %U "$checkout") != root ]] || find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit | grep -q .; then
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ $(stat -c %U "$checkout") != root || -n $unsafe ]]; then
   echo "The checkout must be root-owned and not group/world-writable." >&2
   exit 1
 fi
@@ -41,7 +43,7 @@ if ! command -v visudo >/dev/null 2>&1; then
   exit 1
 fi
 
-for path in /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
+for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
   if [[ -L $path ]]; then
     echo "Refusing symlink at $path." >&2
     exit 1
@@ -50,10 +52,11 @@ done
 
 install -d -o root -g root -m 0755 /usr/local/lib/akyldash
 install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
+install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify
 install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
 
 sudoers_file=/etc/sudoers.d/akyldash-codex
-expected='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
+expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *'
 created=false
 if [[ -e $sudoers_file ]]; then
   if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
@@ -82,4 +85,4 @@ if ! visudo -c -f /etc/sudoers; then
   echo "Sudoers validation failed; the newly added rule was removed." >&2
   exit 1
 fi
-echo "Installed root-owned rollback helper; codex receives no general sudo or Docker access."
+echo "Installed read-only verification and guarded rollback helpers; codex receives no general sudo or Docker access."
diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml
index 2beb0cf..0c3917c 100644
--- a/infra/inventory/production.yml
+++ b/infra/inventory/production.yml
@@ -70,6 +70,9 @@ target:
   restricted_helper:
     path: /usr/local/sbin/akyldash-rollback
     status: repository code and owner-run installer exist; not installed on VPS
+  read_only_verifier:
+    path: /usr/local/sbin/akyldash-verify
+    status: repository code and owner-run installer exist; not installed on VPS
   checks:
     - infra/bootstrap/verify.sh akyldash_vps
     - docker compose -f deploy/production/compose.yaml ps
-- 
2.54.0


From 9084fd4f16d11a64ba36d778a31b86e798f89e6d Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 19:40:58 +0300
Subject: [PATCH 13/17] Test restricted verifier argument guard

---
 .gitea/workflows/ci.yaml              | 5 ++++-
 infra/bootstrap/test-verify-helper.sh | 8 ++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 infra/bootstrap/test-verify-helper.sh

diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml
index 5739fae..e15185e 100644
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@@ -21,10 +21,13 @@ jobs:
         run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
       - name: Check recovery host guard
         run: bash infra/bootstrap/test-host-guard.sh
+      - name: Check verifier helper argument guard
+        run: bash infra/bootstrap/test-verify-helper.sh
       - name: Check infrastructure shell syntax
         run: |
           bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
-            infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/deploy/install-helper.sh
+            infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/deploy/install-helper.sh \
+            infra/bootstrap/test-verify-helper.sh
       - name: Run rollback helper tests
         run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
       - name: Validate production Compose and Caddyfile
diff --git a/infra/bootstrap/test-verify-helper.sh b/infra/bootstrap/test-verify-helper.sh
new file mode 100644
index 0000000..fde0c93
--- /dev/null
+++ b/infra/bootstrap/test-verify-helper.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if bash "$(dirname "${BASH_SOURCE[0]}")/verify-helper.sh" unexpected >/dev/null 2>&1; then
+  echo 'FAIL verifier helper accepted command arguments' >&2
+  exit 1
+fi
+echo 'PASS verifier helper rejects command arguments'
-- 
2.54.0


From d6d029d98cf9b257d6f59c6b7e162d9eb7ead588 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 22:50:39 +0300
Subject: [PATCH 14/17] Fix production backup bootstrap guidance

---
 deploy/production/README.md      | 139 ++++++++++++++------------
 docs/operations/BACKUP.md        |  18 +++-
 docs/operations/CURRENT_STATE.md |   9 +-
 docs/operations/DEPLOYMENT.md    |  13 ++-
 infra/backup/archive.py          |  87 +++++++++-------
 infra/backup/test_archive.py     | 164 ++++++++++++++++++++++++++++++-
 6 files changed, 324 insertions(+), 106 deletions(-)

diff --git a/deploy/production/README.md b/deploy/production/README.md
index 7e4918a..bd0ae4d 100644
--- a/deploy/production/README.md
+++ b/deploy/production/README.md
@@ -1,76 +1,91 @@
 # Production deployment
 
-This compose project runs Caddy, the Search API, and a private OpenSearch node.
-Caddy serves `akyldash.kg` over HTTPS and proxies to the API. Only Caddy publishes
-ports `80` and `443`; the API has no host port, and OpenSearch is reachable only
-from the backend's private network.
+Compose runs Caddy, the Search API, and a private OpenSearch node. Only Caddy
+publishes ports `80` and `443`; backend and OpenSearch have no host ports. The
+OpenSearch security plugin is disabled to match the current API client, so keep
+it on the private network.
 
-The current compose intentionally disables the OpenSearch security plugin to
-match the existing API client. Keep OpenSearch on its private network.
-The application provides cookie-based accounts for its search API and review
-workspace; it does not replace host-level network controls or TLS.
+## Host access and configuration
 
-## First deployment
-
-1. Copy this directory to the host with the repository source.
-2. Copy `.env.example` to `.env`, set an absolute `DATA_ROOT`, and create the
-   secret file referenced by `REVIEW_SECRET_FILE` outside the repository:
-
-   ```bash
-   sudo install -d -o root -g root -m 0700 /etc/akyldash
-   sudo sh -c 'umask 077; openssl rand -hex 32 > /etc/akyldash/review_secret'
-   ```
-
-   Do not commit `.env` or the secret file.
-3. Put the normalized dataset under `$DATA_ROOT/minjust-normalized/`.
-4. Confirm DNS points through Cloudflare to this origin. Have the owner review
-   the existing firewall rules and allow TCP `80` and `443` before Caddy starts.
-   Caddy requests and renews the origin certificate automatically; use
-   Cloudflare Full (strict) only after HTTPS succeeds.
-5. Complete and verify the backup/restore setup before starting production.
-   The repository does not yet contain a tested backup script; do not continue
-   until the owner has approved and tested one.
-6. Check the rendered configuration:
-
-   ```bash
-   docker compose --env-file .env -f compose.yaml config
-   ```
-
-7. Start the services:
-
-   ```bash
-   docker compose --env-file .env -f compose.yaml up -d --build
-   docker compose --env-file .env -f compose.yaml ps
-   curl -fsS https://akyldash.kg/review >/dev/null
-   ```
-
-8. Load the versioned index and switch its alias only after the import and
-   validation succeed, after creating another verified pre-change backup.
-
-## Initial administrator and accounts
-
-After starting the backend for the first time, create the initial `admin`
-account with an interactive password prompt:
+The production environment file is `/etc/akyldash/production.env`; the review
+secret is `/etc/akyldash/review_secret`. Both must stay outside the checkout,
+root-owned, and mode `0600`. `DATA_ROOT` must be an absolute path. The `codex`
+account is intentionally outside the `sudo` and `docker` groups because access
+to the Docker socket grants root-equivalent control. Run host operations with
+the narrowly required `sudo` command; do not use `docker` as `codex` directly.
 
 ```bash
-docker compose --env-file .env -f compose.yaml exec backend python -m search.api --create-admin
+sudo install -d -o root -g root -m 0700 /etc/akyldash
+sudo install -o root -g root -m 0600 deploy/production/.env.example /etc/akyldash/production.env
+sudo sh -c 'umask 077; openssl rand -hex 32 > /etc/akyldash/review_secret'
+sudoedit /etc/akyldash/production.env
 ```
 
-The account database is stored at `DATA_ROOT/search-users.sqlite3`. Back it up
-alongside the review database. Sign in at `/review`; `admin` can create accounts
-with temporary passwords. Each new account must replace its temporary password
-at first sign-in. Only the `admin` login can create, reset, or delete accounts.
+Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus
+is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in
+[`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the
+root bootstrap installs packages and creates directories, but does not deploy
+the application or alter SSH, firewall, user groups, or sudoers.
 
-If the `admin` password is lost, reset it in an interactive terminal with:
+## First deployment and upgrades
+
+Run commands from the root-owned checkout at `/opt/akyldash/current`. Render
+and review Compose before changing services:
 
 ```bash
-docker compose --env-file .env -f compose.yaml exec backend python -m search.api --reset-admin-password
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml config --quiet
 ```
 
-The next sign-in requires another password change. Serve the backend behind
-HTTPS, rate-limit login requests at the reverse proxy, and keep
-`COOKIE_SECURE=true` in production.
+On a clean first deployment, `infra/backup/archive.py create` reports that no
+backup is required when all persistent mounts are empty and no Compose
+containers exist. Continue without a backup in that case. If any persistent
+data exists, create and verify a backup before changing schema, images, or
+starting a new application version. This also applies when containers are
+absent. The archive tool refuses ambiguous storage state, running containers,
+and incomplete Compose service containers. See
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
+commands. Do not remove an older usable backup to make room; retain the
+pre-change archive until the new deployment and a newer verified backup are
+confirmed.
 
-This is a deployment baseline, not a public internet exposure recipe. TLS,
-backups, monitoring, and a production OpenSearch security configuration must
-be provided by the host reverse proxy/operations setup.
+For an initial clean installation, start the stack only after config review:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d --build
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml ps
+curl -fsS https://akyldash.kg/review >/dev/null
+```
+
+Before this step, confirm DNS and have the owner review firewall rules for TCP
+`80` and `443`. Caddy manages the origin certificate; confirm HTTPS before
+setting Cloudflare to Full (strict). On the initial start, create the `admin`
+account interactively:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --create-admin
+```
+
+The account database is `$DATA_ROOT/search-users.sqlite3` and is included in
+the `data` backup. Admin password reset is also interactive:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
+```
+
+## Backup, restore, and rollback
+
+Backups are cold and require downtime. Stop services, create the archive, then
+verify both its checksum and contents before proceeding. Do not run an upgrade
+or schema change until verification succeeds. Keep archives root-only under
+`/var/backups/akyldash` and copy them to separately managed encrypted storage.
+
+Restore is a staged operation on a marked replacement host; it does not
+overwrite production. For application-only rollback, use the guarded helper
+with a verified pre-release backup and explicit data-compatibility review.
+Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md),
+[`ROLLBACK.md`](../../docs/operations/ROLLBACK.md), and
+[`RECOVERY.md`](../../docs/operations/RECOVERY.md).
+
+Automated production deployment is not enabled. Deployments remain owner-run;
+record the release and verify service health before considering it complete.
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index b14eaa8..3c9e1db 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -14,13 +14,22 @@ and `/var/backups/akyldash` remains a planned root-only location.
   `search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
   Caddy's certificate data, and required secret file in the protected backup
   set. Store secrets and Caddy private keys with equivalent access controls.
-- Verify archive readability, checksums, and SQLite integrity before
-  replacing the previous known-good backup.
+- Verify archive readability, checksums, and SQLite integrity before relying
+  on it. Never overwrite or delete the only usable backup.
 - Keep one current pre-release backup. If deployment fails, retain the backup
   required to restore the previous state.
 - Test restore into an isolated target before relying on the backup. Do not
   restore over production as a test.
 
+## Clean first deployment
+
+No backup is needed only when Compose has no containers and all four
+persistent mounts are confirmed empty. Named volumes not created yet count as
+empty on a clean first deployment. Review the rendered config and let the
+archive tool make this determination; missing mount definitions, inaccessible
+volumes, or missing volumes alongside existing data are an error and must stop
+deployment. Do not create placeholder containers to make backup pass.
+
 ## Cold backup procedure
 
 The script requires every Compose container to be stopped. Schedule downtime,
@@ -37,7 +46,10 @@ sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYM
 ```
 
 Replace the timestamp placeholder with one real UTC timestamp before running.
-The create command refuses running services and archives the application data,
+On a clean first deployment, `create` exits with a clear message that backup
+can be skipped. With any existing persistent data it archives the available
+storage even when no Compose containers exist; partial service containers or
+running services are rejected as unsafe. The create command archives the application data,
 OpenSearch volume, Caddy data/config volumes, production environment file, and
 review secret. It records the checked-out Git revision, deployment-file hashes,
 and image references/IDs for each stopped Compose service, writes a SHA-256
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 18b0ade..20d9511 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -48,7 +48,10 @@ from planned configuration.
   deployment are not configured on the host.
 - The repository has a cold-backup archive/verification tool and a one-shot
   restore apply tool that requires a marked replacement host. Backup/restore
-  tests use synthetic data; neither tool has run against production.
+  tests use synthetic data; neither tool has run against production. The
+  archive preflight now permits a clean first deployment to skip backup,
+  requires backup for existing persistent data even without containers, and
+  rejects missing/ambiguous mounts or incomplete/running service containers.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.
@@ -71,6 +74,10 @@ from planned configuration.
 - The backup archive/restore tool was tested with synthetic local data; a backup
   and restore of real production data remain unverified because production is
   not deployed.
+- Production instructions now use `/etc/akyldash/production.env` and root-run
+  Docker commands, consistent with the `codex` account's restricted access.
+  Initial deployment and upgrade backup ordering are documented; no production
+  host was changed for this PR.
 - The Cloudflare edge response was checked, but origin DNS configuration,
   Cloudflare Full (strict), certificate validity, Actions runner availability,
   registry publishing, and production deployment have not been tested.
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index bb55f5f..c43c9a4 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -19,7 +19,9 @@ proxy/firewall configuration, and a backup and restore have been tested.
    `--recovery-target` only on a replacement host; it refuses the known
    production host identity.
 2. Install the approved repository checkout at `/opt/akyldash/current`,
-   root-owned and not group/world-writable. Configure
+   root-owned and not group/world-writable. Copy
+   `deploy/production/.env.example` to `/etc/akyldash/production.env`, set
+   `DATA_ROOT`, and configure
    `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside
    Git. The secret file must be root-readable only.
 3. After reviewing the root-owned checkout, run
@@ -31,9 +33,12 @@ proxy/firewall configuration, and a backup and restore have been tested.
    existing SSH access. The Compose Caddy service obtains and renews its HTTPS
    certificate automatically; Cloudflare Full (strict) still needs owner-side
    verification.
-5. Create and verify the first backup before starting services. Load and
-   validate the OpenSearch index, create the initial admin interactively, and
-   verify `/review` over HTTPS. Record the backend image ID for drift checks:
+5. For a clean first deployment, skip backup only when Compose confirms there
+   are no containers and all persistent mounts are empty. If any data exists,
+   create and verify a backup before starting a new version. An uncertain or
+   partial state stops deployment. Then start services, load and validate the
+   OpenSearch index, create the initial admin interactively, and verify
+   `/review` over HTTPS. Record the backend image ID for drift checks:
 
    ```bash
    sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
diff --git a/infra/backup/archive.py b/infra/backup/archive.py
index e623ac4..80a4b9c 100644
--- a/infra/backup/archive.py
+++ b/infra/backup/archive.py
@@ -127,6 +127,51 @@ def check_sqlite(root: Path) -> None:
             connection.close()
 
 
+def compose_storage(config: dict) -> dict[str, Path | None]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    volumes = subprocess.run(
+        ["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
+    ).stdout.split()
+    sources: dict[str, Path | None] = {}
+    for target, component in (
+        ("/app/data", "data"),
+        ("/usr/share/opensearch/data", "opensearch"),
+        ("/data", "caddy-data"),
+        ("/config", "caddy-config"),
+    ):
+        mount = mounts.get(target)
+        if not mount:
+            raise ValueError(f"persistent Compose mount is missing: {target}")
+        if mount["type"] == "bind":
+            source = Path(mount["source"])
+        elif mount["type"] == "volume":
+            volume_name = config["volumes"][mount["source"]]["name"]
+            if volume_name not in volumes:
+                sources[component] = None
+                continue
+            result = subprocess.run(
+                ["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            source = Path(result.stdout.strip())
+        else:
+            raise ValueError(f"unsupported persistent mount type at {target}")
+        if not source.is_dir() or source.is_symlink():
+            raise ValueError(f"cannot determine persistent data state for {component}: {source}")
+        sources[component] = source
+    return sources
+
+
+def has_persistent_data(sources: dict[str, Path | None]) -> bool:
+    return any(any(path.iterdir()) for path in sources.values() if path is not None)
+
+
 def compose_sources(
     compose_file: Path, env_file: Path
 ) -> tuple[dict[str, Path], str, dict[str, dict[str, str]], dict[str, str]]:
@@ -139,8 +184,12 @@ def compose_sources(
         [*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
     )
     containers = container_result.stdout.split()
-    if not containers:
-        raise ValueError("Compose has no containers; refusing to create an incomplete backup")
+    storage = compose_storage(config)
+    persistent_data = has_persistent_data(storage)
+    if not containers and not persistent_data:
+        raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
+    if any(path is None for path in storage.values()):
+        raise ValueError("persistent Compose volumes are missing while data exists; refusing an incomplete backup")
     images: dict[str, dict[str, str]] = {}
     for container in containers:
         result = subprocess.run(
@@ -159,42 +208,12 @@ def compose_sources(
         if len(fields) != 4 or fields[0].lower() != "false":
             raise ValueError("stop all Compose services before taking a cold backup")
         images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
-    if set(images) != set(config["services"]):
+    if containers and set(images) != set(config["services"]):
         raise ValueError("not all configured Compose services have stopped containers")
 
-    mounts = {
-        mount["target"]: mount
-        for service in config["services"].values()
-        for mount in service.get("volumes", [])
-    }
-    volume_sources: dict[str, Path] = {}
-    for target, component in (
-        ("/app/data", "data"),
-        ("/usr/share/opensearch/data", "opensearch"),
-        ("/data", "caddy-data"),
-        ("/config", "caddy-config"),
-    ):
-        mount = mounts.get(target)
-        if not mount:
-            raise ValueError(f"persistent Compose mount is missing: {target}")
-        if mount["type"] == "bind":
-            source = Path(mount["source"])
-        elif mount["type"] == "volume":
-            volume_name = config["volumes"][mount["source"]]["name"]
-            result = subprocess.run(
-                ["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
-                check=True,
-                capture_output=True,
-                text=True,
-            )
-            source = Path(result.stdout.strip())
-        else:
-            raise ValueError(f"unsupported persistent mount type at {target}")
-        volume_sources[component] = source
-
     secret_file = Path(config["secrets"]["review_secret"]["file"])
     sources = {
-        **volume_sources,
+        **storage,
         "production-env": env_file,
         "review-secret": secret_file,
     }
diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
index 77d6bcf..5e6d7ee 100644
--- a/infra/backup/test_archive.py
+++ b/infra/backup/test_archive.py
@@ -84,18 +84,176 @@ class BackupArchiveTests(unittest.TestCase):
             verify_archive(archive_path)
 
     def test_cold_backup_refuses_a_running_compose_container(self):
-        config = {"services": {}, "volumes": {}, "secrets": {}}
+        config, volume_paths, env_file, _secret = self.compose_fixture()
 
         def run(command, **_kwargs):
             if "config" in command:
                 return SimpleNamespace(stdout=json.dumps(config))
             if "ps" in command:
                 return SimpleNamespace(stdout="container-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
             return SimpleNamespace(stdout="true\n")
 
         with patch("archive.subprocess.run", side_effect=run):
             with self.assertRaisesRegex(ValueError, "stop all Compose services"):
-                compose_sources(Path("deploy/production/compose.yaml"), Path("/etc/akyldash/production.env"))
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def compose_fixture(self):
+        paths = {}
+        mounts = {
+            "backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
+            "opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+        }
+        volumes = {}
+        for volume in ("os-data", "caddy-data", "caddy-config"):
+            name = f"akyldash_{volume}"
+            path = self.root / name
+            path.mkdir()
+            paths[name] = path
+            volumes[volume] = {"name": name}
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        secret = self.root / "review-secret.txt"
+        secret.write_text("secret", encoding="utf-8")
+        config = {
+            "services": {name: {"volumes": value} for name, value in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        return config, paths, env_file, secret
+
+    def test_clean_first_deployment_does_not_require_backup(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
+        config, _volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_existing_data_without_containers_is_backup_eligible(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, _revision, images, _deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images, {})
+        self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
+
+    def test_missing_persistent_mount_state_is_rejected(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        volume_paths["akyldash_os-data"].rmdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_partial_compose_containers_are_rejected_with_existing_data(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="backend-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if command[1] == "inspect":
+                return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_existing_data_with_missing_named_volume_is_rejected_as_incomplete(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        (self.root / "data/existing.json").write_text("data", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "persistent Compose volumes are missing"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
 
     def test_compose_backup_records_sources_and_image_ids(self):
         data = self.root / "production-data"
@@ -137,6 +295,8 @@ class BackupArchiveTests(unittest.TestCase):
                 name = command[-1].removesuffix("-id")
                 return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
             if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
                 return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
             return SimpleNamespace(stdout="")
 
-- 
2.54.0


From f6c9b9883aee0e99b735c3f69b9527803887237e Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 22:51:44 +0300
Subject: [PATCH 15/17] Verify offline data backup without containers

---
 infra/backup/test_archive.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
index 5e6d7ee..105259d 100644
--- a/infra/backup/test_archive.py
+++ b/infra/backup/test_archive.py
@@ -187,12 +187,16 @@ class BackupArchiveTests(unittest.TestCase):
             raise AssertionError(command)
 
         with patch("archive.subprocess.run", side_effect=run):
-            sources, _revision, images, _deployment_files = compose_sources(
+            sources, revision, images, deployment_files = compose_sources(
                 Path.cwd() / "deploy/production/compose.yaml", env_file
             )
+        archive = self.private / "offline-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
         self.assertEqual(set(sources), COMPONENTS)
         self.assertEqual(images, {})
         self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
+        self.assertEqual(manifest["git_revision"], "abc123")
 
     def test_missing_persistent_mount_state_is_rejected(self):
         config, volume_paths, env_file, _secret = self.compose_fixture()
-- 
2.54.0


From 752266d408cca1ee46c149c2193aa6dd76e411b5 Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 22:59:23 +0300
Subject: [PATCH 16/17] Back up preloaded corpus before first deployment

---
 deploy/production/README.md      |  7 +++++--
 docs/operations/BACKUP.md        |  9 +++++----
 docs/operations/CURRENT_STATE.md |  5 +++--
 infra/backup/archive.py          | 29 ++++++++++++++++++++---------
 infra/backup/test_archive.py     | 18 +++++++++++++++---
 5 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/deploy/production/README.md b/deploy/production/README.md
index bd0ae4d..a24e153 100644
--- a/deploy/production/README.md
+++ b/deploy/production/README.md
@@ -42,8 +42,11 @@ backup is required when all persistent mounts are empty and no Compose
 containers exist. Continue without a backup in that case. If any persistent
 data exists, create and verify a backup before changing schema, images, or
 starting a new application version. This also applies when containers are
-absent. The archive tool refuses ambiguous storage state, running containers,
-and incomplete Compose service containers. See
+absent, including an initial installation where the normalized corpus was
+preloaded under `DATA_ROOT`; create the archive before the first `up`. Missing
+named volumes are recorded as empty backup components when Docker confirms
+they do not exist. The archive tool refuses ambiguous storage state, running
+containers, and incomplete Compose service containers. See
 [`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
 commands. Do not remove an older usable backup to make room; retain the
 pre-change archive until the new deployment and a newer verified backup are
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index 3c9e1db..2f77037 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -25,10 +25,11 @@ and `/var/backups/akyldash` remains a planned root-only location.
 
 No backup is needed only when Compose has no containers and all four
 persistent mounts are confirmed empty. Named volumes not created yet count as
-empty on a clean first deployment. Review the rendered config and let the
-archive tool make this determination; missing mount definitions, inaccessible
-volumes, or missing volumes alongside existing data are an error and must stop
-deployment. Do not create placeholder containers to make backup pass.
+empty on a clean first deployment. If data has already been placed in a bind
+mount, create the archive before the first `up`; missing named volumes are
+recorded as empty components. Missing mount definitions, inaccessible Docker
+state, or partial service containers are errors and must stop deployment. Do
+not create placeholder containers to make backup pass.
 
 ## Cold backup procedure
 
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 20d9511..0d4d69c 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -50,8 +50,9 @@ from planned configuration.
   restore apply tool that requires a marked replacement host. Backup/restore
   tests use synthetic data; neither tool has run against production. The
   archive preflight now permits a clean first deployment to skip backup,
-  requires backup for existing persistent data even without containers, and
-  rejects missing/ambiguous mounts or incomplete/running service containers.
+  requires backup for existing persistent data even without containers,
+  records not-yet-created named volumes as empty, and rejects inaccessible or
+  ambiguous mounts and incomplete/running service containers.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.
diff --git a/infra/backup/archive.py b/infra/backup/archive.py
index 80a4b9c..4c720e2 100644
--- a/infra/backup/archive.py
+++ b/infra/backup/archive.py
@@ -99,7 +99,7 @@ def verify_archive(path: Path) -> dict:
         return read_manifest(archive)
 
 
-def parse_sources(values: list[str]) -> dict[str, Path]:
+def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
     sources: dict[str, Path] = {}
     for value in values:
         name, separator, raw_path = value.partition("=")
@@ -109,9 +109,9 @@ def parse_sources(values: list[str]) -> dict[str, Path]:
         if not source.is_absolute() or not source.exists() or source.is_symlink():
             raise ValueError(f"source must be an existing absolute path: {raw_path}")
         sources[name] = source.resolve(strict=True)
-    if set(sources) != COMPONENTS:
-        missing = ", ".join(sorted(COMPONENTS - sources.keys()))
-        extra = ", ".join(sorted(sources.keys() - COMPONENTS))
+    if set(sources) != required:
+        missing = ", ".join(sorted(required - sources.keys()))
+        extra = ", ".join(sorted(sources.keys() - required))
         raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
     return sources
 
@@ -188,7 +188,7 @@ def compose_sources(
     persistent_data = has_persistent_data(storage)
     if not containers and not persistent_data:
         raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
-    if any(path is None for path in storage.values()):
+    if containers and any(path is None for path in storage.values()):
         raise ValueError("persistent Compose volumes are missing while data exists; refusing an incomplete backup")
     images: dict[str, dict[str, str]] = {}
     for container in containers:
@@ -236,12 +236,15 @@ def compose_sources(
         relative: digest(repository / relative)
         for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
     }
-    return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images, deployment_files
+    present = {name: path for name, path in sources.items() if path is not None}
+    missing = {name for name, path in sources.items() if path is None}
+    parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
+    return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
 
 
 def create_backup(
     output: Path,
-    sources: dict[str, Path],
+    sources: dict[str, Path | None],
     revision: str,
     images: dict | None = None,
     deployment_files: dict[str, str] | None = None,
@@ -253,14 +256,16 @@ def create_backup(
     if output.exists() or Path(f"{output}.sha256").exists():
         raise ValueError("refusing to overwrite an existing backup")
     for name, source in sources.items():
+        if source is None:
+            continue
         if source.is_symlink() or not (source.is_dir() or source.is_file()):
             raise ValueError(f"source must be a regular file or directory: {name}")
         if source.is_dir() and (output == source or source in output.parents):
             raise ValueError(f"backup output cannot be inside source {name}")
     for name in ("production-env", "review-secret"):
-        if not sources[name].is_file():
+        if sources[name] is None or not sources[name].is_file():
             raise ValueError(f"sensitive component must be a regular file: {name}")
-    if sources["data"].is_dir():
+    if sources["data"] is not None and sources["data"].is_dir():
         check_sqlite(sources["data"])
 
     manifest = {
@@ -283,6 +288,12 @@ def create_backup(
             info.mode = 0o600
             archive.addfile(info, io.BytesIO(encoded))
             for name, source in sorted(sources.items()):
+                if source is None:
+                    info = tarfile.TarInfo(f"{name}/")
+                    info.type = tarfile.DIRTYPE
+                    info.mode = 0o700
+                    archive.addfile(info)
+                    continue
                 def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
                     if name in {"production-env", "review-secret"}:
                         member.mode = 0o600
diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
index 105259d..388a693 100644
--- a/infra/backup/test_archive.py
+++ b/infra/backup/test_archive.py
@@ -238,7 +238,7 @@ class BackupArchiveTests(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
                 compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
 
-    def test_existing_data_with_missing_named_volume_is_rejected_as_incomplete(self):
+    def test_existing_data_with_uncreated_named_volume_is_backed_up_as_empty(self):
         config, volume_paths, env_file, _secret = self.compose_fixture()
         (self.root / "data/search-users.sqlite3").unlink()
         (self.root / "data/existing.json").write_text("data", encoding="utf-8")
@@ -253,11 +253,23 @@ class BackupArchiveTests(unittest.TestCase):
                 return SimpleNamespace(stdout="\n".join(available) + "\n")
             if command[1] == "volume":
                 return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
             raise AssertionError(command)
 
         with patch("archive.subprocess.run", side_effect=run):
-            with self.assertRaisesRegex(ValueError, "persistent Compose volumes are missing"):
-                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertIsNone(sources["opensearch"])
+        archive = self.private / "bind-only-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        with tarfile.open(archive, "r:gz") as saved:
+            self.assertTrue(saved.getmember("opensearch/").isdir())
 
     def test_compose_backup_records_sources_and_image_ids(self):
         data = self.root / "production-data"
-- 
2.54.0


From 3b41d1f20f0e9e80f0df450cbb9b51392a1a89cf Mon Sep 17 00:00:00 2001
From: Codex Agent <agent@vakanaut.ru>
Date: Sun, 11 Oct 2026 23:17:48 +0300
Subject: [PATCH 17/17] Guard volume gaps and back up initialized production

---
 deploy/production/README.md      | 18 +++++++---
 docs/operations/BACKUP.md        | 60 +++++++++++++++++++++++++++-----
 docs/operations/CURRENT_STATE.md |  6 ++--
 docs/operations/DEPLOYMENT.md    | 12 ++++---
 infra/backup/archive.py          | 25 ++++++++++---
 infra/backup/test_archive.py     | 38 ++++++++++++++++++--
 6 files changed, 132 insertions(+), 27 deletions(-)

diff --git a/deploy/production/README.md b/deploy/production/README.md
index a24e153..ec0b0d3 100644
--- a/deploy/production/README.md
+++ b/deploy/production/README.md
@@ -43,10 +43,13 @@ containers exist. Continue without a backup in that case. If any persistent
 data exists, create and verify a backup before changing schema, images, or
 starting a new application version. This also applies when containers are
 absent, including an initial installation where the normalized corpus was
-preloaded under `DATA_ROOT`; create the archive before the first `up`. Missing
-named volumes are recorded as empty backup components when Docker confirms
-they do not exist. The archive tool refuses ambiguous storage state, running
-containers, and incomplete Compose service containers. See
+preloaded under `DATA_ROOT`; create the archive before the first `up`. When
+named volumes do not exist, pass `--initial-deployment`; this is accepted only
+without Compose containers and when `DATA_ROOT` contains only
+`minjust-normalized/`. During upgrades, missing volumes are an error because
+they can indicate lost production data. The archive tool also refuses
+ambiguous storage state, running containers, and incomplete Compose service
+containers. See
 [`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
 commands. Do not remove an older usable backup to make room; retain the
 pre-change archive until the new deployment and a newer verified backup are
@@ -76,6 +79,13 @@ the `data` backup. Admin password reset is also interactive:
 sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
 ```
 
+After the initial OpenSearch index is loaded and validated, the `admin` account
+is created, and `/review` works over HTTPS, create and verify the required
+post-setup baseline backup before treating deployment as complete. It captures
+the initialized index, account database, and Caddy certificate data. Follow
+the exact stop/create/verify/start sequence in
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md#initial-post-setup-baseline).
+
 ## Backup, restore, and rollback
 
 Backups are cold and require downtime. Stop services, create the archive, then
diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md
index 2f77037..2a85e2e 100644
--- a/docs/operations/BACKUP.md
+++ b/docs/operations/BACKUP.md
@@ -24,12 +24,28 @@ and `/var/backups/akyldash` remains a planned root-only location.
 ## Clean first deployment
 
 No backup is needed only when Compose has no containers and all four
-persistent mounts are confirmed empty. Named volumes not created yet count as
-empty on a clean first deployment. If data has already been placed in a bind
-mount, create the archive before the first `up`; missing named volumes are
-recorded as empty components. Missing mount definitions, inaccessible Docker
-state, or partial service containers are errors and must stop deployment. Do
-not create placeholder containers to make backup pass.
+persistent mounts are confirmed empty. For a first install with a preloaded
+corpus and not-yet-created named volumes, use `--initial-deployment` to create
+the pre-start backup. This flag is accepted only with no Compose containers and
+when `DATA_ROOT` contains exactly `minjust-normalized/`; it archives absent
+named volumes as empty components. In every normal backup, an absent volume is
+an error because it could mean that a previous installation lost data.
+Missing mount definitions, inaccessible Docker state, or partial service
+containers are also errors and must stop deployment. Do not create placeholder
+containers to make backup pass.
+
+For the corpus-only initial case, run this before the first `up` from the
+approved checkout. Replace the timestamp with one real UTC timestamp:
+
+```bash
+cd /opt/akyldash/current
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --initial-deployment \
+  --output /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+```
 
 ## Cold backup procedure
 
@@ -48,9 +64,11 @@ sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYM
 
 Replace the timestamp placeholder with one real UTC timestamp before running.
 On a clean first deployment, `create` exits with a clear message that backup
-can be skipped. With any existing persistent data it archives the available
-storage even when no Compose containers exist; partial service containers or
-running services are rejected as unsafe. The create command archives the application data,
+can be skipped. With existing data and all expected volumes present, it
+archives the storage even when no Compose containers exist. A preloaded corpus
+with no named volumes requires the explicit `--initial-deployment` mode;
+otherwise missing volumes are rejected. Partial service containers or running
+services are rejected as unsafe. The create command archives the application data,
 OpenSearch volume, Caddy data/config volumes, production environment file, and
 review secret. It records the checked-out Git revision, deployment-file hashes,
 and image references/IDs for each stopped Compose service, writes a SHA-256
@@ -62,6 +80,30 @@ cd /opt/akyldash/current
 sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
 ```
 
+## Initial post-setup baseline
+
+After the first index has been loaded and validated, the initial `admin` account
+has been created, and `/review` responds over HTTPS, create a second backup.
+This one captures the initialized index, account database, and newly created
+Caddy/OpenSearch volumes. Stop all services, create and verify the archive,
+then start the stack again:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+Use one real UTC timestamp in both archive paths. Do not treat initial
+production setup as complete until this baseline archive verifies. Keep the
+pre-start corpus archive as well until the baseline is verified and copied to
+separately managed encrypted storage.
+
 Keep the local archive root-only, then copy it to separately managed encrypted
 storage. A local archive alone does not protect against loss of the VPS disk.
 Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md
index 0d4d69c..5aaf858 100644
--- a/docs/operations/CURRENT_STATE.md
+++ b/docs/operations/CURRENT_STATE.md
@@ -51,8 +51,10 @@ from planned configuration.
   tests use synthetic data; neither tool has run against production. The
   archive preflight now permits a clean first deployment to skip backup,
   requires backup for existing persistent data even without containers,
-  records not-yet-created named volumes as empty, and rejects inaccessible or
-  ambiguous mounts and incomplete/running service containers.
+  allows absent named volumes only in explicit corpus-only initial-deployment
+  mode, and rejects inaccessible/ambiguous mounts and incomplete/running
+  service containers. Initial deployment docs require a second verified backup
+  after index and admin initialization.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.
diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md
index c43c9a4..5ec86dd 100644
--- a/docs/operations/DEPLOYMENT.md
+++ b/docs/operations/DEPLOYMENT.md
@@ -35,10 +35,14 @@ proxy/firewall configuration, and a backup and restore have been tested.
    verification.
 5. For a clean first deployment, skip backup only when Compose confirms there
    are no containers and all persistent mounts are empty. If any data exists,
-   create and verify a backup before starting a new version. An uncertain or
-   partial state stops deployment. Then start services, load and validate the
-   OpenSearch index, create the initial admin interactively, and verify
-   `/review` over HTTPS. Record the backend image ID for drift checks:
+   create and verify a backup before starting a new version. For a preloaded
+   corpus with absent named volumes, use `--initial-deployment`; it is limited
+   to a container-free host whose `DATA_ROOT` contains only
+   `minjust-normalized/`. Normal backups reject absent volumes. An uncertain
+   or partial state stops deployment. Then start services, load and validate
+   the OpenSearch index, create the initial admin interactively, verify
+   `/review` over HTTPS, and create the required post-setup baseline backup
+   described in `BACKUP.md`. Record the backend image ID for drift checks:
 
    ```bash
    sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
diff --git a/infra/backup/archive.py b/infra/backup/archive.py
index 4c720e2..8175fcc 100644
--- a/infra/backup/archive.py
+++ b/infra/backup/archive.py
@@ -172,9 +172,16 @@ def has_persistent_data(sources: dict[str, Path | None]) -> bool:
     return any(any(path.iterdir()) for path in sources.values() if path is not None)
 
 
+def is_preloaded_corpus_only(data: Path | None) -> bool:
+    if data is None:
+        return False
+    entries = list(data.iterdir())
+    return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
+
+
 def compose_sources(
-    compose_file: Path, env_file: Path
-) -> tuple[dict[str, Path], str, dict[str, dict[str, str]], dict[str, str]]:
+    compose_file: Path, env_file: Path, initial_deployment: bool = False
+) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
     command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
     rendered = subprocess.run(
         [*command, "config", "--format", "json"], check=True, capture_output=True, text=True
@@ -188,8 +195,13 @@ def compose_sources(
     persistent_data = has_persistent_data(storage)
     if not containers and not persistent_data:
         raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
-    if containers and any(path is None for path in storage.values()):
-        raise ValueError("persistent Compose volumes are missing while data exists; refusing an incomplete backup")
+    missing_volumes = any(path is None for path in storage.values())
+    if missing_volumes and containers:
+        raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
+    if missing_volumes and not initial_deployment:
+        raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
+    if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
+        raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
     images: dict[str, dict[str, str]] = {}
     for container in containers:
         result = subprocess.run(
@@ -350,6 +362,7 @@ def main() -> int:
     create.add_argument("--output", required=True, type=Path)
     create.add_argument("--compose-file", required=True, type=Path)
     create.add_argument("--env-file", required=True, type=Path)
+    create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
     verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
     verify.add_argument("archive", type=Path)
     restore = subparsers.add_parser("restore", help="extract into a new staging directory")
@@ -360,7 +373,9 @@ def main() -> int:
         if args.command in {"create", "restore"} and os.geteuid() != 0:
             raise ValueError("create and restore must run as root to protect production secrets and data")
         if args.command == "create":
-            sources, revision, images, deployment_files = compose_sources(args.compose_file, args.env_file)
+            sources, revision, images, deployment_files = compose_sources(
+                args.compose_file, args.env_file, args.initial_deployment
+            )
             create_backup(args.output, sources, revision, images, deployment_files)
         elif args.command == "verify":
             manifest = verify_archive(args.archive)
diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py
index 388a693..837c698 100644
--- a/infra/backup/test_archive.py
+++ b/infra/backup/test_archive.py
@@ -238,12 +238,43 @@ class BackupArchiveTests(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
                 compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
 
-    def test_existing_data_with_uncreated_named_volume_is_backed_up_as_empty(self):
+    def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
         config, volume_paths, env_file, _secret = self.compose_fixture()
         (self.root / "data/search-users.sqlite3").unlink()
         (self.root / "data/existing.json").write_text("data", encoding="utf-8")
         available = set(volume_paths) - {"akyldash_os-data"}
 
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+            with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
+                compose_sources(
+                    Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
+                )
+
+    def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        corpus = self.root / "data/minjust-normalized"
+        corpus.mkdir()
+        (corpus / "document.json").write_text("corpus", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
         def run(command, **_kwargs):
             if "config" in command:
                 return SimpleNamespace(stdout=json.dumps(config))
@@ -261,15 +292,16 @@ class BackupArchiveTests(unittest.TestCase):
 
         with patch("archive.subprocess.run", side_effect=run):
             sources, revision, images, deployment_files = compose_sources(
-                Path.cwd() / "deploy/production/compose.yaml", env_file
+                Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
             )
         self.assertIsNone(sources["opensearch"])
-        archive = self.private / "bind-only-backup.tar.gz"
+        archive = self.private / "initial-corpus-backup.tar.gz"
         create_backup(archive, sources, revision, images, deployment_files)
         manifest = verify_archive(archive)
         self.assertEqual(manifest["git_revision"], "abc123")
         with tarfile.open(archive, "r:gz") as saved:
             self.assertTrue(saved.getmember("opensearch/").isdir())
+            self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
 
     def test_compose_backup_records_sources_and_image_ids(self):
         data = self.root / "production-data"
-- 
2.54.0