Document runner ProtectHome fix

# Conflicts:
#	CHANGELOG.md
#	docs/operations/CURRENT_STATE.md

.dockerignore

@@ -0,0 +1,16 @@
+.git
+.gitignore
+.env
+.env.*
+*.json
+!backend/
+!backend/search/
+!backend/search/*.json
+*.xlsx
+*.pdf
+*.jpg
+data/
+docs/
+tools/
+__pycache__/
+*.pyc

.gitea/actions/caddy-validate/Dockerfile

@@ -0,0 +1 @@
+FROM caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028

.gitea/actions/caddy-validate/action.yml

@@ -0,0 +1,16 @@
+name: Validate Caddyfile
+description: Run the pinned Caddy validator against the checked-out Caddyfile
+inputs:
+  workspace:
+    description: Repository workspace path supplied by the workflow
+    required: true
+runs:
+  using: docker
+  image: Dockerfile
+  entrypoint: /usr/bin/caddy
+  args:
+    - validate
+    - --config
+    - ${{ inputs.workspace }}/deploy/production/Caddyfile
+    - --adapter
+    - caddyfile

.gitea/actions/kaniko-build/Dockerfile

@@ -0,0 +1 @@
+FROM ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412

.gitea/actions/kaniko-build/action.yml

@@ -0,0 +1,22 @@
+name: Build image with Kaniko in a rootless container
+description: Validate a Dockerfile by building it without publishing the image or cache
+inputs:
+  workspace:
+    description: Repository workspace path supplied by the workflow
+    required: true
+  dockerfile:
+    description: Dockerfile path relative to the checked-out repository
+    required: true
+  destination:
+    description: Image reference required by Kaniko; the image is not pushed
+    required: true
+runs:
+  using: docker
+  image: Dockerfile
+  entrypoint: /kaniko/executor
+  args:
+    - --context=dir://${{ inputs.workspace }}
+    - --dockerfile=${{ inputs.workspace }}/${{ inputs.dockerfile }}
+    - --destination=${{ inputs.destination }}
+    - --no-push
+    - --no-push-cache

.gitea/workflows/ci.yaml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  tests-and-images:
+    runs-on: akyldash-ci
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run backend tests
+        run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
+      - name: Run Telegram bot tests
+        run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
+      - name: Run infrastructure backup tests
+        run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
+      - name: Check root guard UID decisions
+        run: bash infra/bootstrap/test-root-guard.sh
+      - name: Check recovery host guard
+        run: bash infra/bootstrap/test-host-guard.sh
+      - name: Check verifier helper argument guard
+        run: bash infra/bootstrap/test-verify-helper.sh
+      - name: Check host audit root guard
+        run: bash infra/bootstrap/test-audit-host.sh
+      - name: Check Compose helper allowlist
+        run: bash infra/deploy/test-compose-helper.sh
+      - name: Check production config root guard
+        run: bash infra/deploy/test-production-config-guard.sh
+      - name: Check runner config and isolation guards
+        run: |
+          bash infra/runner/test-runner-config.sh
+          bash infra/runner/test-install-runner-idempotency.sh
+          bash infra/runner/test-install-runner-guard.sh
+      - name: Check CI container operation isolation
+        run: bash infra/runner/test-ci-container-builds.sh
+      - name: Check infrastructure shell syntax
+        run: |
+          bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
+            infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/bootstrap/audit-host.sh \
+            infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \
+            infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \
+            infra/bootstrap/test-audit-host.sh infra/bootstrap/test-root-guard.sh \
+            infra/bootstrap/test-verify-helper.sh infra/test/run-as-unprivileged.sh \
+            infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \
+            infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \
+            infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh \
+            infra/runner/test-ci-container-builds.sh
+          python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())"
+          python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/validate_release.py').read_text())"
+      - name: Run deployment contract and rollback tests
+        run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
+      - name: Validate production Compose and Caddyfile
+        run: |
+          secret_file=$(mktemp)
+          trap 'rm -f "$secret_file"' EXIT
+          REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet
+      - name: Validate Caddyfile in isolated container action
+        uses: ./.gitea/actions/caddy-validate
+        with:
+          workspace: ${{ github.workspace }}
+      - name: Build production images
+        uses: ./.gitea/actions/kaniko-build
+        with:
+          workspace: ${{ github.workspace }}
+          dockerfile: deploy/production/Dockerfile.backend
+          destination: akyldash-backend:ci
+      - name: Build OpenSearch production image
+        uses: ./.gitea/actions/kaniko-build
+        with:
+          workspace: ${{ github.workspace }}
+          dockerfile: deploy/production/Dockerfile.opensearch
+          destination: akyldash-opensearch:ci

.gitea/workflows/runner-smoke.yaml

@@ -0,0 +1,24 @@
+name: Runner smoke test
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  runner-smoke:
+    runs-on: akyldash-ci
+    steps:
+      - name: Verify isolated runner boundary
+        run: |
+          host_uid=$(awk 'NR == 1 {print $2}' /proc/self/uid_map)
+          test "$host_uid" -gt 0
+          test ! -S /var/run/docker.sock
+          test ! -e /run/user/1000/docker.sock
+          test -z "${DOCKER_HOST:-}"
+          test ! -e /etc/akyldash/production.env
+          test ! -e /etc/akyldash/review_secret
+          test ! -e /opt/akyldash/current
+          printf 'Isolated runner: version=%s host=%s uid-map=%s\n' \
+            "$GITEA_ACTIONS_RUNNER_VERSION" "$(hostname)" "$host_uid"

.gitignore

@@ -11,5 +11,8 @@ __pycache__/
 .env.*
 !.env.example
 
-# Local bot archive
+# Local application archives
 data/
+
+# Local runtime logs
+logs/

.impeccable/config.json

@@ -0,0 +1,35 @@
+{
+  "detector": {
+    "ignoreRules": [],
+    "ignoreFiles": [],
+    "ignoreValues": [
+      {
+        "rule": "cream-palette",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:40:02.635Z",
+        "reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette"
+      },
+      {
+        "rule": "cramped-padding",
+        "value": "*",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:28.434Z",
+        "reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector"
+      },
+      {
+        "rule": "design-system-font",
+        "value": "sans",
+        "files": [
+          "backend/search/review.html"
+        ],
+        "createdAt": "2026-10-11T15:41:33.020Z",
+        "reason": "Agent: DESIGN.md specifies Noto Sans with sans-serif fallback; this finding is the inherited generic fallback"
+      }
+    ]
+  }
+}

AGENTS.md

@@ -0,0 +1,95 @@
+# Акылдаш — инструкции для агента
+
+Проект: юридическая информационно-аналитическая платформа; Search API на
+Python, SQLite и OpenSearch. Production пока не развернут.
+
+- Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и
+  [правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно.
+- Архитектура и процедуры: [инфраструктура](docs/operations/INFRASTRUCTURE.md),
+  [деплой](docs/operations/DEPLOYMENT.md), [backup](docs/operations/BACKUP.md),
+  [rollback](docs/operations/ROLLBACK.md), [восстановление](docs/operations/RECOVERY.md).
+- Работай в feature/fix/refactor-ветке. `main` защищена; изменения вноси через
+  PR. DEV не одобряет и не сливает собственный PR. Условный merge выполняет
+  только REVIEWER по правилам раздела «Автономный workflow PR» ниже. Сохраняй
+  незакоммиченные изменения.
+- SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай
+  `codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
+  Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
+- Репозиторный Gitea Actions runner описан в
+  [CI_RUNNER.md](docs/operations/CI_RUNNER.md). Регистрация выполняется
+  владельцем через Settings → Actions → Runners для `admin/akyldash`; токен
+  вводится интерактивно и не попадает в Git. Runner использует отдельный
+  rootless Docker daemon; job-контейнеры не получают Docker socket или
+  production secrets. Не включай production deployment из CI без отдельного
+  проверенного deploy-helper.
+- Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
+  `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и
+  `python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`,
+  `bash infra/bootstrap/test-host-guard.sh`.
+  Production images собираются командами из `.gitea/workflows/ci.yaml`.
+- Перед изменениями production необходим проверенный backup. Формального
+  механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
+  SSH, firewall или production вручную без владельца и последующего PR.
+- Холодный backup требует согласованного окна простоя; восстановление допускается
+  только в новый staging-каталог, пока владелец отдельно не проверил процедуру
+  для нового хоста.
+- После каждого изменения инфраструктуры обновляй
+  `docs/operations/CURRENT_STATE.md` по фактическим проверкам.
+
+## Автономный workflow PR
+
+- Маршруты ниже указаны из контекста оркестратора/root: оркестратор — thread
+  `01a144a5-8b95-77c3-b311-9a9afd062ca8`, host `local`; DEV — thread
+  `01a138f1-3b34-7c91-a9ed-4b43c3c85f1f`, host
+  `remote-ssh-codex-managed:712b8d4e-24e1-4dd5-92af-85d1fbd26449`, Gitea
+  `codex` (ID 6); REVIEWER — thread `01a144f9-5b51-7923-8398-61d4a6a98306`,
+  host `remote-ssh-codex-managed:b61555d7-8d1a-4cfa-988b-c8da4791d426`, Gitea
+  `agent` (ID 7). Если этот же чат отображает REVIEWER как `local` внутри его
+  собственного app-server, это self-label и не является маршрутом из контекста
+  оркестратора.
+- DEV принимает только явно зарегистрированную оркестратором человеческую
+  задачу с полями `task`, `repo`, `branch`, `pr`, `head_sha`, `phase`. Репозиторий
+  должен быть `admin/akyldash`; работай только в назначенной ветке и PR. Веди
+  одну задачу/PR за раз и не подхватывай старые PR самостоятельно.
+- Допустимые фазы: `initial_review` — реализация и устранение review findings;
+  `final_review_and_merge` — только после первоначального успешного review,
+  changelog-коммита и отдельного назначения оркестратора. Не переходи к следующей
+  фазе без назначения. Если задача приостановлена, останови любые действия;
+  после resume перепроверь назначение, PR, head/base, checks и review с нуля.
+- Повторная доставка одного назначения идемпотентна по ключу
+  `task`+`pr`+`head_sha`+`stage`: не дублируй commit, push, review-запрос или
+  merge; сообщи оркестратору о уже выполненном результате. Новый head или stage
+  образует новое назначение и требует повторной проверки.
+- Межчатовое чтение недоступно: оркестратор через central relay сам читает
+  результаты и выдаёт назначения. Завершай каждый этап отчётом с полями `task`,
+  `repo`, `branch`, `pr`, `head_sha`, `stage`, `checks`, `findings`,
+  `next_action`. Не настраивай отдельные webhook, daemon или транспорт.
+- DEV исследует, изменяет, проверяет, коммитит и отправляет изменения; REVIEWER
+  не меняет код и независимо проверяет полный diff от base после каждого push,
+  включая предыдущие findings. DEV не одобряет и не сливает свой PR.
+- Только после первоначального успешного формального review DEV добавляет
+  значимые изменения в существующий changelog, коммитит и отправляет их в тот же
+  PR. Затем REVIEWER полностью проверяет итоговый diff. Для docs-only задачи не
+  придумывай изменения кода; правила версии применяй только если они явно
+  относятся к этой документационной правке.
+- Слияние может выполнить только REVIEWER для зарегистрированной человеческой
+  задачи, в фазе `final_review_and_merge`, после явного назначения оркестратора.
+  Непосредственно перед ним REVIEWER подтверждает `admin/akyldash`, PR, base
+  `main`, ожидаемую ветку, автора PR `codex` (ID 6), неизменность base и head,
+  причём head должен совпадать с зарегистрированным `head_sha`, включая
+  `head_commit_id` для merge-запроса. Требуются все успешные required CI/test
+  checks и gates, отсутствие pending/failed/unknown checks, открытых
+  findings и активных `REQUEST_CHANGES`, плюс актуальный формальный `APPROVED`
+  от REVIEWER для точного текущего commit с параметрами `commit_id == head_sha`,
+  `stale == false`, `dismissed == false`. При изменении head или base review и
+  проверки выполняются заново. Комментарий с approval не является формальным
+  review.
+  Используй только обычный server-side merge с `head_commit_id`; не используй
+  `force_merge` и не обходи branch protection. Если любой существенный gate,
+  статус или защита неизвестны либо не проверяются — не сливай PR и сообщи
+  оркестратору, что именно осталось неизвестным.
+- Все Gitea API-вызовы выполняй только через `gitea-api` с личным окружением;
+  Git используй с обычным credential helper. Не читай credentials из файлов
+  репозитория. Запускай Git- и API-команды отдельно, без объединения с другими
+  командами.

CHANGELOG.md

@@ -0,0 +1,91 @@
+# История изменений
+
+## Не выпущено
+
+- Infrastructure: systemd unit Gitea Actions runner теперь использует
+  `ProtectHome=read-only`, сохраняя доступность `/run/user` и существующий
+  writable carve-out для rootless runtime; VPS не изменялся.
+- Backend `0.12.1`: добавлен ограниченный возобновляемый staging редакций из
+  текущего API Минюста с проверкой точных ID и метаданных, изоляцией от
+  legacy-корпуса и сохранением исходного порядка редакций при нормализации и
+  экспорте. Тест overlap guard задаёт безопасные права fixture независимо от
+  `umask` CI; сам guard не ослаблен.
+- Rollback preflight проверяет digest-pinned image references по сохранённому
+  image ID и останавливается до изменений deployment; переключение production
+  на digest и полноценный digest rollback пока не реализованы.
+- Infrastructure: добавлен офлайн-контракт release manifest с точной схемой,
+  digest-ссылками и repository allowlist; проверен Compose override с
+  `--no-build` как требованием будущего deploy. Публикация, provenance и
+  production-деплой не включены.
+- Добавлен read-only возобновляемый inventory корпуса с полным хешированием,
+  потоковой сверкой, индексируемыми диапазонами путей и provenance SHA-256
+  проверяющего; значения `stat` хранятся без ограничения SQLite INTEGER.
+  Сравнение source/staging сверяет путь, размер и SHA-256, удерживает shared
+  locks и отклоняет пустые корпуса без документов или фрагментов. Runbook и
+  `CURRENT_STATE.md` отражают, что полный аудит корпуса ещё выполняется; это
+  изменение не означает готовность корпуса или production.
+- CI actions теперь используют workspace, переданный Gitea, для Caddyfile и Kaniko build context; CI run 69 и smoke run 70 прошли на self-hosted runner.
+- Runner installer сохраняет `HOME` и XDG-пути в user-unit `docker.service`, перезапускает активный daemon и проверяет окружение его процесса.
+- Runner installer защищает image и mountpoint root-owned home-каталогом, оставляет регистрацию и workspace в runner-owned каталоге, безопасно форматирует образ по открытому FD и проверяет `nosuid,nodev`.
+- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, Caddy HTTPS proxy с разделёнными Compose-сетями и CI-проверка конфигурации; публикация образов и production-деплой остаются выключены до проверки runner, registry, backup/restore и ограниченного deploy-helper.
+- Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
+- Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
+- Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
+- Backend `0.11.0` и frontend `0.1.0`: лаборатория переоформлена по дизайн-системе «Түндүк»; добавлены учетные записи, обязательная смена временного пароля, личный кабинет и управление пользователями для `admin`.
+- Backend обновлён до `0.10.4`: многословный поиск получил низковесный OR fallback с требованием совпадения минимум двух терминов; fallback пропускает распознанные названия нормативных актов.
+- Backend обновлён до `0.10.3`: запрос о налоге на прибыль поднимает действующий
+  Налоговый кодекс; точное совпадение фразы дополнительно усиливает результат.
+- Backend обновлён до `0.10.2`: действующие документы ранжируются выше, а
+  утратившие силу — после остальных; из лаборатории удалён пустой блок фрагмента.
+- Backend обновлён до `0.10.1`: поисковая выдача по регистрации больше не
+  ограничивается только тремя заранее выбранными нормативными актами.
+- Backend обновлён до `0.10.0`: поисковый API использует общий планировщик
+  запросов, расширяет аббревиатуры ОсОО/ООО/ЖЧК и для запроса о регистрации
+  приоритизирует положение КР 2026 года, закон об обществах и ГК КР.
+- Backend обновлён до `0.9.4`: из поисковых фрагментов убраны технические маркеры
+  подсветки OpenSearch.
+- Backend обновлён до `0.9.3`: исправлен локальный индекс production Search API,
+  ошибки поиска показываются в общем status-блоке, а панель сохранения и footer
+  выстроены в обычном порядке документа.
+- Backend обновлён до `0.9.2`: внутренняя лаборатория получила визуальный
+  polish-проход и короткий повторно запускаемый tutorial для новых проверяющих.
+- Backend обновлён до `0.9.1`: все статусы из нормализованного корпуса доступны
+  как отдельные фильтры, варианты конституционного закона и указа сопоставляются
+  с подтверждённой юридической силой, неизвестные типы документов сохраняются
+  под общим фильтром без вывода юридической силы.
+- Уточнено, что внутренняя лаборатория оценивает выдачу собственного OpenSearch;
+  ЦБД Минюста служит источником для подтверждения документов, а прежние Excel/PDF
+  бланки удалены.
+- Исключены секреты и локальные данные из Docker build context; синхронизирована версия интерфейса.
+- Добавлен production deployment baseline для Search API и OpenSearch с
+  постоянными хранилищами и healthcheck.
+- Добавлено восстановление незавершённой разметки из `localStorage` после перезагрузки страницы.
+- Добавлена внутренняя лаборатория проверки поисковой выдачи: просмотр документов,
+  оценка релевантности 0–3, комментарии и SQLite-экспорт подписанных снимков.
+- Уточнены доступные состояния и адаптивное поведение внутреннего интерфейса
+  оценки поисковой выдачи.
+- Добавлен план внутреннего интерфейса оценки поисковой выдачи юристами.
+- Завершён Search API v1: стабильные справочники, валидный OpenAPI, безопасная
+  пагинация и проверка актуальных редакций в локальном OpenSearch.
+- Добавлен фильтр Search API v1 по укрупнённой юридической силе. Полная
+  иерархия GeneralClassifiers экспортирована в отдельный черновой каталог;
+  тематическая навигация остаётся вне API v1 до ручного утверждения рубрик и
+  подписей.
+- Добавлено безопасное переключение alias на новую версию поискового индекса
+  после полной загрузки; checkpoint защищает возобновление загрузки от смены
+  alias.
+- Добавлен roadmap готовности данных, поиска и API перед проектированием
+  frontend.
+- Исправлена выдача для явных запросов об открытии ОсОО и ЖЧК: первыми
+  показываются действующие положение о регистрации и закон о хозяйственных
+  товариществах и обществах.
+- Добавлен CLI `python3 -m search.query` для проверки текущей выдачи локального
+  OpenSearch.
+- Добавлена инструкция по подготовке и независимой проверке relevance set.
+- Улучшено ранжирование relevance-оценки: запрос теперь сопоставляет название и
+  текст документа как единое поле.
+
+## 0.5.0 — 2026-08-15
+
+- Добавлена воспроизводимая оценка качества поиска по Recall@10 и MRR@10.
+- Подготовлен шаблон relevance set из 25 русских и 25 кыргызских запросов с инструкцией по ручной разметке.

DESIGN.md

@@ -0,0 +1,282 @@
+---
+name: "Акылдаш · Түндүк"
+description: "Утверждённая дизайн-система правового портала Кыргызской Республики"
+colors:
+  bg: "#F7F2E8"
+  surface: "#FFFFFF"
+  surface-muted: "#FBF8F4"
+  text: "#38262A"
+  text-secondary: "#786469"
+  border: "#DED3CE"
+  primary: "#9D2633"
+  primary-hover: "#84202B"
+  primary-active: "#701B24"
+  on-primary: "#FFFFFF"
+  primary-subtle: "#F8EBED"
+  accent: "#A37B32"
+  highlight: "#FFF0C7"
+  focus: "#245C78"
+  success: "#246843"
+  success-bg: "#EAF4EC"
+  warning: "#82520D"
+  warning-bg: "#FFF3DA"
+  error: "#9D2633"
+  error-bg: "#FBEDEF"
+  info: "#245C78"
+  info-bg: "#EAF2F6"
+typography:
+  document-title:
+    fontFamily: "Noto Serif, serif"
+    fontSize: "32px"
+    fontWeight: 700
+    lineHeight: 1.2
+  section-title:
+    fontFamily: "Noto Serif, serif"
+    fontSize: "24px"
+    fontWeight: 600
+    lineHeight: 1.3
+  document-body:
+    fontFamily: "Noto Sans, sans-serif"
+    fontSize: "18px"
+    fontWeight: 400
+    lineHeight: 1.65
+  control:
+    fontFamily: "Noto Sans, sans-serif"
+    fontSize: "16px"
+    fontWeight: 500
+    lineHeight: 1.5
+  metadata:
+    fontFamily: "Noto Sans, sans-serif"
+    fontSize: "14px"
+    fontWeight: 400
+    lineHeight: 1.5
+rounded:
+  default: "6px"
+spacing:
+  xs: "4px"
+  sm: "8px"
+  compact: "12px"
+  md: "16px"
+  lg: "24px"
+  xl: "32px"
+  section: "48px"
+---
+
+# Design System: Акылдаш · Түндүк
+
+## Overview
+
+**Статус: утверждено пользователем 10 октября 2026 года.** Это целевая
+дизайн-система портала и внутренней лаборатории проверки поисковой выдачи.
+Она ещё не внедрена в работающий интерфейс. Утверждение дизайна не означает
+готовность публичного сервиса или разрешение на deployment.
+
+**Creative North Star: "Түндүк — спокойное пространство для работы с правом".**
+
+Профессиональный правовой сервис с кыргызской идентичностью: светлый фон
+цвета войлока, белое поле документа, глубокий красный для действий,
+тёмный сливовый текст и небольшие золотистые геометрические детали.
+Главное — читаемость длинных документов и ясная иерархия.
+
+Это единственная нормативная спецификация дизайна проекта. Требования
+исходного brief перенесены сюда; отдельный brief и ранние варианты удалены.
+Визуальный ориентир — утверждённый пользователем
+[концептуальный экран](docs/design/tunduk-desktop-concept.png).
+Точные токены и правила этой спецификации важнее случайных деталей
+генеративного изображения. Название продукта — **Акылдаш**; «УКУК» из
+ранних концепций не используется.
+
+**Key Characteristics:**
+
+- Спокойное чтение и компактные, понятные элементы управления.
+- Национальная идентичность через палитру, знак и несколько точных деталей.
+- Поиск и документ сохраняют общий контекст на широком экране.
+- Единая визуальная система публичного сервиса и внутренней лаборатории.
+- Содержимое и доступные функции определяются реальными данными и API.
+
+## Colors
+
+Нормативные значения находятся в YAML выше. При реализации ключи `colors`
+переносятся в CSS custom properties `--color-<ключ>` либо эквивалентные
+семантические токены существующей системы тем. HEX не размножаются
+по компонентам.
+
+| Роль | Токен | Применение |
+|---|---|---|
+| Светлый войлок | `bg` | Общий фон и окружение рабочей области |
+| Белый лист | `surface` | Документ, поля и основные поверхности |
+| Мягкая поверхность | `surface-muted` | Вторичные панели |
+| Сливовые чернила | `text` | Основной текст и заголовки |
+| Вторичный текст | `text-secondary` | Метаданные и пояснения |
+| Разделители | `border` | Контуры и границы панелей |
+| Глубокий красный | `primary` | Главное действие, ссылки и активная навигация |
+| Состояния красного | `primary-hover`, `primary-active` | Hover и нажатие |
+| Текст на красном | `on-primary` | Главная кнопка и активный пункт меню |
+| Мягкое красное выделение | `primary-subtle` | Выбранный результат, вторичная навигация |
+| Золотистая геометрия | `accent` | Небольшие декоративные линии |
+| Найденный фрагмент | `highlight` | Подсветка совпадения с исходным тёмным текстом |
+| Фокус | `focus` | Контрастная обводка клавиатурного фокуса |
+| Успех | `success`, `success-bg` | Подтверждение действия, подтверждённый статус |
+| Предупреждение | `warning`, `warning-bg` | Важные ограничения с текстовым объяснением |
+| Ошибка | `error`, `error-bg` | Ошибка с доступным действием для восстановления |
+| Информация | `info`, `info-bg` | Информационные сообщения |
+
+Статус и ошибка всегда имеют текстовую подпись: цвет не передаёт смысл
+самостоятельно. Не назначать юридический статус по визуальной эвристике;
+использовать данные backend. Золотистый не используется для мелкого текста.
+Контраст проверяется на фактических сочетаниях: обычный текст не менее
+4.5:1, крупный текст и необходимые графические элементы не менее 3:1.
+
+## Typography
+
+**Заголовки документа и разделов:** Noto Serif, веса 600 и 700.
+**Интерфейс и основной текст:** Noto Sans, веса 400, 500 и 600.
+Шрифты подключаются штатным способом, предпочтительно локальными WOFF2
+с `font-display: swap`. До поставки проверить лицензии и символы
+`Ө ө Ү ү Ң ң`. Файлы шрифтов пока не добавлены.
+
+| Элемент | Размер | Межстрочный интервал |
+|---|---|---|
+| Заголовок документа / страницы документа | 30–32 px; на телефоне 24–26 px | 1.2 |
+| Заголовок раздела | 22–24 px | 1.3 |
+| Текст документа | 17–18 px | 1.6–1.7 |
+| Элементы управления | 14–16 px | По токену `control` |
+| Метаданные | 13–14 px | По токену `metadata` |
+
+Мелкий текст не ниже 12 px. Ориентир ширины документа — 65–75 символов
+в строке. Длинные названия документов и кыргызские подписи переносятся,
+не обрезаются. Даты, суммы и числовые колонки используют `tabular-nums`,
+обычные данные не оформляются моноширинным шрифтом.
+
+## Layout
+
+Шкала отступов определяется `spacing`. Шапка на desktop — ориентировочно
+72–88 px. Навигация / фильтры — 220–240 px. Панель контекста, когда
+существуют соответствующие данные и функции, — 300–340 px. Центральная
+колонка занимает остаток; это адаптивные ориентиры, а не фиксированная
+ширина всей страницы.
+
+Для поискового рабочего экрана: шапка с брендом, поиском и языком;
+фильтры слева, результаты в середине, открытый документ справа. Оглавление
+можно сворачивать в подписанную кнопку, чтобы не сужать область чтения.
+Разворачивание документа и поиск внутри него реализуются только вместе
+с работающим поведением, а не как декоративные кнопки.
+
+Внутренняя лаборатория сохраняет свой двухпанельный сценарий: фактический
+порядок результатов и оценивание слева, текст выбранной редакции справа.
+Она не получает фильтры, аналитику или публичную навигацию исключительно
+для сходства с концепцией. Панель проверяющего и сохранения не перекрывает
+результаты и текст документа.
+
+При недостатке места сначала сворачивается дополнительная панель контекста,
+затем навигация. На телефоне — одна колонка; связанные материалы и
+оглавление доступны через подписанные кнопки. В лаборатории сохраняется
+существующее мобильное открытие документа и возврат фокуса при закрытии.
+Поиск, действия и метаданные переносятся. Основной текст не требует
+горизонтальной прокрутки; широкие таблицы имеют отдельную обозначенную
+область прокрутки.
+
+Проверочные размеры при реализации: **1440, 1024 и 390 px**. Это размеры
+проверки, а не заранее навязанные CSS breakpoints.
+
+## Elevation & Depth
+
+Основные поверхности плоские. Иерархия создаётся белым листом на тёплом
+фоне, отступами, типографикой и тонкими разделителями. Мягкая тень нужна
+главным образом выпадающим меню и временным панелям. Значения теней
+уточняются при реализации; декоративное размытие и обилие теней запрещены.
+
+## Shapes
+
+Основное скругление — `rounded.default` (6 px), контуры — 1 px.
+Небольшие статусные метки могут быть округлее. Кнопки не превращаются
+в капсулы. Границы интерактивных полей обозначаются явным контуром,
+а не только фоновым цветом или разделителем соседней панели.
+
+Если утверждённый логотип уже существует, он сохраняется. При отсутствии
+логотипа допускается небольшой оригинальный геометрический знак,
+вдохновлённый круглым верхним элементом юрты — түндүком. Знак из
+генеративного макета является направлением, а не готовым SVG-ассетом.
+Государственная символика не копируется.
+
+Тонкий симметричный орнамент по мотивам шырдака допустим на краях шапки
+и в пустой нижней части навигации. Он выполняется простым SVG,
+помечается `aria-hidden`, не находится под текстом и не мешает нажатию.
+На телефоне орнамент сокращается либо убирается.
+
+## Components
+
+Это требования к будущей реализации, а не перечень уже обновлённых компонентов.
+
+### Кнопки и поля
+
+- Главная кнопка: `primary` и `on-primary`, высота около 40 px.
+- Вторичная кнопка: светлая поверхность и явный контур.
+- На touch-экране область нажатия не менее 44×44 px.
+- Поля белые, с постоянной подписью; placeholder не заменяет label.
+- Видимый фокус: обводка `focus` около 2 px с отступом.
+- Hover, active, focus-visible, disabled, loading, empty и error
+  реализуются последовательно, с понятными сообщениями.
+- Переходы 120–180 мс, без анимации текста документа;
+  учитывается `prefers-reduced-motion`.
+
+### Навигация, результаты и документ
+
+- Активный пункт меню — красный фон и белая подпись; вторичная
+  навигация — `primary-subtle` и тёмно-красный текст.
+- Активная вкладка обозначена подписью и нижней линией. Длинные подписи
+  переносятся либо прокручиваются без обрезки.
+- Результаты — упорядоченные строки: название, тип, статус, дата при
+  наличии и выдержка. Каждое поле не оборачивается в отдельную карточку.
+- Выбранный результат обозначается мягким красным выделением и текстом.
+- Документ — единый белый лист: заголовок, реальные метаданные,
+  доступные действия, оглавление и текст. Абзацы не разбиваются на карточки.
+- Подсветка совпадения сохраняет исходный текст; число совпадений
+  и переходы между ними показываются только при работающем поиске.
+- Источник и выбранная редакция доступны рядом с документом.
+  Недоступный язык не подменяется выдуманным переводом.
+- Язык интерфейса и язык документа — отдельные состояния:
+  смена интерфейса не должна самовольно менять текст документа.
+- Footer сохраняет версии frontend и backend. Отсутствующий frontend
+  или макет явно обозначаются, версия не придумывается.
+
+### Внутренняя оценка выдачи
+
+Сохраняются шкала 0–3 с полными текстовыми подписями, комментарии,
+черновик, идентификация проверяющего, сохранение подписанного снимка,
+обучение и существующие ограничения доступа. Оценки не передаются
+только цветом. Служебные данные не становятся публичными из-за редизайна.
+
+### Доступность и состояния
+
+Семантические заголовки, labels, ссылки и кнопки; иконки без текста
+имеют доступное имя. Проверяются Tab, Enter, Escape, возврат фокуса
+и отсутствие клавиатурных ловушек. Статусы доступны скринридеру.
+Пустая выдача предлагает изменить запрос; ошибка предлагает повторное
+действие. Ошибка сохранения не уничтожает черновик.
+
+## Do's and Don'ts
+
+### Do:
+
+- Сохранять название «Акылдаш», реальные данные, маршруты, API и права доступа.
+- Использовать общий слой семантических токенов, существующие компоненты
+  и CSS Modules / BEM согласно правилам соответствующей части проекта.
+- Разделять официальный текст, сведения о происхождении и выводы системы.
+- Проверять чтение длинных RU/KY-документов и одноязычные редакции.
+- Использовать этот документ как нормативную спецификацию для новых экранов.
+
+### Don't:
+
+- Не переносить демонстрационные даты, судебные материалы, связи
+  и юридические формулировки из изображения в продукт.
+- Не добавлять аналитику, сравнение редакций, регистрацию и другие функции
+  только ради сходства с концепцией.
+- Не менять поисковое ранжирование или сохранение разметки ради оформления.
+- Не использовать ковровые текстуры, фотографии гор, иллюстрации юрт,
+  крупные флаги, золотые градиенты и декоративное размытие.
+- Не делать золотистыми мелкие подписи и не передавать смысл одним цветом.
+- Не добавлять отдельную тёмную тему без задачи; существующую сохранять.
+- Не считать генеративный скриншот доказательством точных HEX,
+  загрузки Noto или проверки доступности работающего интерфейса.
+- Не считать утверждение дизайна разрешением на публикацию сервиса.

README.md

@@ -9,15 +9,18 @@ Telegram-бот — только часть рабочего окружения
 
 ## Текущее состояние
 
-Сейчас реализован Telegram-бот-секретарь версии `0.2.2`. Он сохраняет сообщения
-рабочих тем, формирует Markdown-экспорты и отмечает завершённые обсуждения.
+Сейчас реализованы Telegram-бот-секретарь версии `0.2.2`, backend версии
+`0.12.2` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
+SQLite; первый пользователь `admin` создаётся интерактивной CLI-командой.
+Поиск расширяет аббревиатуры организационно-правовых форм и поднимает
+действующий Налоговый кодекс для запросов о налоге на прибыль.
 
 | Компонент | Версия | Состояние |
 |---|---:|---|
 | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
-| Backend | — | ещё не создан |
-| Frontend | — | ещё не создан |
-| Сбор и обработка правовых данных | — | ещё не созданы |
+| Backend | `0.12.2` | авторизация, поиск с фильтрами и ограниченный staging текущего API Минюста |
+| Frontend | `0.1.2` | лаборатория поиска, личный кабинет и управление учетными записями |
+| Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи |
 | RAG и база знаний | — | ещё не созданы |
 
 ## Структура репозитория
@@ -30,11 +33,15 @@ docs/                  структурированная документаци
   team/                материалы для команды
 tools/
   telegram-bot/        бот рабочего Telegram-пространства
+backend/
+  ingestion/           получение и обновление правовых источников
+  normalization/       воспроизводимая нормализация исходного архива
+  search/              Search API, авторизация и внутренний frontend
 ```
 
-Каталоги для загрузки и обработки источников, RAG, backend и frontend будут
-создаваться с первой реальной задачей в соответствующей области. Это позволит
-выбрать структуру по фактическим требованиям, а не поддерживать пустой каркас.
+Внутренний frontend отдаётся существующим Search API без отдельного
+Node-сервиса. Отдельный публичный frontend остаётся за readiness gate в
+`docs/product/frontend-design-readiness-plan.md`.
 
 Начать знакомство с проектом: [документация](docs/README.md) и
 [обзор продукта](docs/product/project-overview.md).
@@ -55,4 +62,4 @@ python3 -m unittest discover -s tools/telegram-bot -v
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend — не создан · Frontend — не создан
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.12.2 · Frontend v0.1.2

backend/README.md

@@ -0,0 +1,299 @@
+# Backend Акылдаш
+
+Версия: `0.12.2`
+
+Первая backend-область проекта — загрузка правовых документов из официального
+Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
+`ingestion/minjust_cbd.py`: его можно запускать как CLI сейчас и импортировать
+из будущего планировщика backend без запуска отдельного процесса.
+
+## Пробная загрузка
+
+Из корня репозитория:
+
+```bash
+python3 backend/ingestion/minjust_cbd.py --limit 10
+```
+
+В интерактивном терминале отображаются процент, количество документов,
+ошибки, скорость и примерное оставшееся время.
+
+Полная загрузка выполняется без `--limit`. По умолчанию архив сохраняется в
+`data/minjust-cbd`, который исключён из Git. Повторный запуск пропускает уже
+загруженные документы; `--refresh` принудительно проверяет их заново.
+Если временный идентификатор списка API истечёт или запрос страницы исчерпает
+повторы во время многодневной загрузки, скрипт пересоздаст список на текущей
+странице и продолжит автоматически.
+
+В версии `0.1.0` обновление существующих документов выполняется полной проверкой
+через `--refresh`. Инкрементальную проверку по `lastmod` из sitemap следует
+добавить вместе с backend-планировщиком, когда будет определена частота запуска.
+
+## Защита API
+
+Запросы выполняются последовательно, по умолчанию не чаще одного в секунду.
+Timeout одного ответа — 60 секунд, число попыток — 5, задержка между повторами
+растёт экспоненциально. При HTTP 429 учитывается заголовок `Retry-After`.
+Лимит действует на один процесс, поэтому одновременно следует запускать только
+один экземпляр загрузчика.
+
+Более осторожный режим:
+
+```bash
+python3 backend/ingestion/minjust_cbd.py --requests-per-second 0.5
+```
+
+## Вызов из будущего backend
+
+```python
+from ingestion.minjust_cbd import sync_archive
+
+result = sync_archive(output_path)
+```
+
+Планировщик, очередь задач и PostgreSQL пока не добавлены: модуль не зависит от
+выбора будущего backend-фреймворка.
+
+## Текущий API Минюста и отдельный staging
+
+Старый OpenData endpoint может возвращать `404`; для отдельной ограниченной
+выгрузки официальный сайт предоставляет `/api/v1/GetDocument` и
+`/api/v1/GetEdition`. Адаптер `ingestion.minjust_current` получает один
+указанный документ и сохраняет его в отдельное поколение staging:
+
+```bash
+PYTHONPATH=backend python3 -m ingestion.minjust_current \
+  --output /tmp/minjust-current-code6 \
+  --document-code 6
+```
+
+Продолжайте прерванную выгрузку только с тем же каталогом и `--resume`. По
+умолчанию лимит скачанного текста одного документа — 4 GiB, лимит запросов —
+один в секунду; каталог не должен совпадать или пересекаться с
+`data/minjust-cbd` и
+`data/minjust-normalized`. Адаптер сохраняет исходные ответы и checkpoints,
+публикует папку документа атомарно и не заменяет существующий код в staging.
+Каталог staging должен быть отдельной директорией режима 0700, принадлежащей
+текущему пользователю; его родитель должен быть существующим доверенным
+каталогом. Символические ссылки в generation, checkpoints и каталогах
+документов отклоняются. Legacy-корни вычисляются от корня репозитория, а не
+от текущего рабочего каталога.
+Для `/GetEdition?lang=kg` официальный ответ помещает кыргызский HTML в
+`contentRu` при `contentKg=null`; архивный язык определяется запрошенным
+`lang`, а не именем поля ответа.
+
+Исторические редакции запрашиваются только с `exact=true`; без этого параметра
+официальный endpoint возвращал ID последней редакции даже для другого
+запрошенного ID. Адаптер требует совпадения response ID и запрошенного ID,
+сверяет названия RU/KY с metadata `GetDocument` и сохраняет request contract в
+checkpoint. Старые staging generations без этого contract нельзя возобновить.
+`exact=true` и совпадение ID проверены на двух контрольных редакциях; это не
+подтверждает полноту всего источника.
+
+Порядок редакций сохраняется по ответу `/GetDocument` и используется как
+официальный порядок при нормализации и выборе текущей редакции в поисковом
+индексе. Вложения и встроенные изображения этим адаптером не подтверждаются:
+staging остаётся неполным до отдельной проверки медиа и не заменяет корпус.
+Получение полного списка документов не является частью этой ограниченной
+процедуры.
+
+## Нормализация архива
+
+Нормализатор читает исходный архив без изменений и создаёт отдельный набор
+данных для будущих поиска, API и RAG:
+
+```bash
+python3 backend/normalization/minjust_cbd.py --limit 10
+python3 backend/normalization/minjust_cbd.py
+```
+
+По умолчанию источник читается из `data/minjust-cbd`, а результат записывается
+в `data/minjust-normalized`. Пути можно изменить параметрами `--input` и
+`--output`; `--refresh` принудительно обрабатывает неизменившиеся документы,
+`--log-level` задаёт уровень журнала.
+Каталоги `--input` и `--output` не должны совпадать, содержать друг друга или
+пересекаться через разрешённые абсолютные пути.
+
+```text
+data/minjust-normalized/
+  manifest.sqlite3
+  documents/<code>/document.json
+  documents/<code>/editions/<edition>/edition.json
+  documents/<code>/editions/<edition>/<lang>/content.html
+  documents/<code>/editions/<edition>/<lang>/content.txt
+  documents/<code>/editions/<edition>/<lang>/fragments.json
+```
+
+`content.html` содержит только разрешённую безопасную разметку, `content.txt` —
+текст для поиска, а `fragments.json` — адресуемые блоки с детерминированными ID
+и SHA-256. Манифест пропускает документы с неизменившимися исходниками и
+повторяет документы, обработка которых завершилась ошибкой.
+
+Первая версия не выполняет OCR, перевод, юридические выводы о редакциях,
+сопоставление фрагментов, загрузку в PostgreSQL/OpenSearch и построение RAG.
+Внешние и встроенные `data:`-изображения из HTML удаляются; сведения и пути к
+локальным изображениям исходного архива сохраняются в `edition.json`.
+
+## Проверка
+
+```bash
+PYTHONPATH=backend python3 -m unittest backend/test_minjust_cbd.py -v
+PYTHONPATH=backend python3 -m unittest backend/test_minjust_normalization.py -v
+PYTHONPATH=backend python3 -m unittest backend/test_minjust_opensearch.py -v
+```
+
+## Подготовка индекса OpenSearch
+
+Mapping поискового индекса находится в
+`search/minjust-fragments-index.json`. Для кыргызского текста он использует
+`icu_analyzer`, поэтому в OpenSearch должен быть установлен плагин
+`analysis-icu`.
+
+Потоковый экспорт в формат Bulk API без внешних Python-зависимостей:
+
+```bash
+python3 backend/search/minjust_opensearch.py
+```
+
+По умолчанию создаётся `data/opensearch/minjust-fragments.ndjson`. Экспорт
+атомарный и детерминированный; для проверки можно передать `--limit 1`.
+Для прямой загрузки без большого промежуточного файла используется `--url`:
+
+```bash
+python3 backend/search/minjust_opensearch.py \
+  --url http://127.0.0.1:9200 \
+  --index akyldash-fragments-dev-v1 \
+  --limit 1
+```
+
+Запросы Bulk API ограничены 25 МБ и не разрывают пару action/source. Для
+полного прохода убрать `--limit` и выбрать новое имя версионного индекса.
+После проверки production-индекса следует переключать alias, чтобы удалённые
+фрагменты не оставались в поиске.
+
+Чтобы переключить alias атомарно только после успешной полной загрузки,
+передайте `--alias`:
+
+```bash
+python3 backend/search/minjust_opensearch.py \
+  --url http://127.0.0.1:9200 \
+  --index akyldash-fragments-v2 \
+  --alias akyldash-fragments-current
+```
+
+После каждого принятого Bulk-пакета загрузчик атомарно сохраняет checkpoint и
+печатает код безопасного возобновления. При временных HTTP 429/5xx, timeout и
+обрыве соединения запрос повторяется автоматически. Прерванную загрузку можно
+продолжить без ручного выбора документа:
+
+```bash
+python3 backend/search/minjust_opensearch.py \
+  --url http://127.0.0.1:9200 \
+  --index akyldash-fragments-v1 \
+  --resume
+```
+
+По умолчанию checkpoint хранится в
+`data/opensearch/<index>.checkpoint.json`; путь можно изменить через
+`--checkpoint`. Checkpoint привязан к URL, cluster UUID, index UUID, `--limit`,
+`--alias` и SHA-256 нормализованного manifest. При `--resume` передавайте то же
+значение `--alias`; старый checkpoint без alias можно продолжить только без
+него. Resume отклоняется при любом несовпадении:
+для обновлённого корпуса или пересозданного индекса нужно создать новый
+версионный индекс, проверить его и переключить alias. Это не оставляет
+удалённые trailing-фрагменты старых документов.
+
+## HTTP API v1
+
+Запустите публичный read-only API поверх текущего alias и нормализованного
+корпуса:
+
+```bash
+PYTHONPATH=backend python3 -m search.api
+```
+
+Он публикует OpenAPI в `GET /openapi.json` и поддерживает `GET /search`,
+`/search/filters`, `/documents/{code}`, `/documents/{code}/editions` и
+`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются со
+стабильным кодом справочника v1 и подписями RU/KY; применяйте `code` как
+параметр поиска. API не подменяет отсутствующий язык документа.
+
+## Локальный OpenSearch
+
+Стенд использует один узел OpenSearch без Dashboards, устанавливает
+`analysis-icu`, выделяет JVM 8 ГБ и доступен только на `127.0.0.1:9200`.
+Индекс хранится в `data/opensearch-node` на диске проекта.
+
+```bash
+sudo sysctl -w vm.max_map_count=262144
+docker compose -f deploy/local-opensearch/compose.yaml up -d --build
+curl http://127.0.0.1:9200/_cluster/health
+```
+
+Security plugin отключён только для локальной разработки; этот compose нельзя
+публиковать в сеть или использовать в production. Mapping локального стенда
+также задаёт одну shard и ноль replicas; для production число shard следует
+рассчитать по размеру корпуса и настроить не менее одной replica.
+
+## Оценка качества поиска
+
+`search/relevance-set-v1.template.json` содержит заготовку для 25 русских и
+25 кыргызских запросов. Для каждого запроса человек должен указать реальную
+формулировку и коды всех релевантных документов; пустая или неполная разметка
+должна быть отклонена при ручной проверке, а технически некорректная — самим
+оценщиком. Критерии выбора запросов, релевантности и двойной проверки описаны в
+`search/RELEVANCE_ANNOTATION.md`. Рабочую копию следует хранить в игнорируемом
+каталоге `data/`, пока набор не проверен и не разрешён к публикации.
+
+Baseline использует поля названия и текста соответствующего языка, оставляет в
+выдаче один результат на документ и вычисляет макро-средние Recall@10 и MRR@10:
+
+```bash
+mkdir -p data/search
+cp backend/search/relevance-set-v1.template.json data/search/relevance-set-v1.json
+PYTHONPATH=backend python3 -m search.evaluate_relevance \
+  data/search/relevance-set-v1.json \
+  > data/search/baseline-v1.json
+```
+
+Менять веса или анализаторы следует только после фиксации этого baseline и
+разбора ошибок выдачи.
+
+## Внутренняя лаборатория релевантности
+
+Создайте первую учетную запись администратора:
+
+```bash
+PYTHONPATH=backend python3 -m search.api --users-db data/search-users.sqlite3 --create-admin
+```
+
+Команда запросит и подтвердит пароль `admin` в терминале. Затем запустите
+Search API на localhost и откройте `http://127.0.0.1:8080/review`:
+
+```bash
+PYTHONPATH=backend python3 -m search.api \\
+  --reviews-db data/search-reviews.sqlite3
+```
+
+Лаборатория показывает фактический порядок выдачи OpenSearch, позволяет открыть
+текст редакции, поставить результату оценку от 0 до 3 и сохранить снимок с
+комментариями. Поиск и API доступны только после входа. Пользователь `admin`
+управляет учетными записями; новые пользователи должны заменить временный
+пароль при первом входе. Хранилище пользователей `data/search-users.sqlite3`
+нужно резервировать отдельно от поискового индекса. Экспорт доступен
+авторизованным пользователям через `GET /search-reviews/export`.
+
+При утере пароля `admin` используйте `PYTHONPATH=backend python3 -m search.api
+--users-db data/search-users.sqlite3 --reset-admin-password`; следующий вход
+потребует сменить временный пароль.
+
+Для проверки текущей выдачи без будущего HTTP API используйте CLI:
+
+```bash
+PYTHONPATH=backend python3 -m search.query "как открыть ОсОО" --language ru
+PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --language ky
+```
+
+---
+
+Акылдаш · Backend v0.12.2 · Frontend v0.1.2

backend/ingestion/minjust_cbd.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+"""Download and archive documents from the Kyrgyz Republic Ministry of Justice."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import logging
+import os
+import sqlite3
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable, Iterable
+
+APP_VERSION = "0.9.3"
+API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/"
+LANGUAGES = {"Rus": "ru", "Kyr": "ky"}
+IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"}
+LOGGER = logging.getLogger(__name__)
+
+
+class CbdClient:
+    def __init__(
+        self,
+        requests_per_second: float = 1,
+        timeout: int = 60,
+        retries: int = 5,
+    ) -> None:
+        if requests_per_second <= 0:
+            raise ValueError("requests_per_second must be greater than zero")
+        if timeout <= 0 or retries <= 0:
+            raise ValueError("timeout and retries must be greater than zero")
+        # ponytail: per-process limit; add a distributed lock before multiple workers.
+        self.interval = 1 / requests_per_second
+        self.timeout = timeout
+        self.retries = retries
+        self.last_request = 0.0
+        self.total_documents = 0
+
+    def request_json(self, method: str, parameters: Iterable[tuple[str, object]] = ()):
+        query = urllib.parse.urlencode(list(parameters), doseq=True)
+        extension = "" if method == "CheckAvailable" else ".json"
+        url = f"{API_BASE_URL}{method}{extension}" + (f"?{query}" if query else "")
+        for attempt in range(self.retries):
+            retry_delay = 2**attempt
+            wait = self.interval - (time.monotonic() - self.last_request)
+            if wait > 0:
+                time.sleep(wait)
+            request = urllib.request.Request(
+                url,
+                headers={
+                    "Accept": "application/json",
+                    "User-Agent": f"Akyldash/{APP_VERSION} (+https://cbd.minjust.gov.kg)",
+                },
+            )
+            try:
+                self.last_request = time.monotonic()
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    body = response.read()
+                    return json.loads(body) if body else None
+            except urllib.error.HTTPError as error:
+                if error.code != 429 and error.code < 500:
+                    raise
+                if error.code == 429:
+                    try:
+                        retry_delay = max(
+                            retry_delay, float(error.headers.get("Retry-After", 0))
+                        )
+                    except ValueError:
+                        pass
+            except (TimeoutError, urllib.error.URLError):
+                pass
+            if attempt + 1 < self.retries:
+                time.sleep(retry_delay)
+        raise RuntimeError(f"Ministry of Justice API request failed: {url}")
+
+    def check_available(self) -> None:
+        self.request_json("CheckAvailable")
+
+    def document_codes(self, limit: int | None = None, page_size: int = 1000):
+        def query_page(page_number: int):
+            return self.request_json(
+                "GetDocumentListByQuery",
+                (
+                    ("Property", "Code"),
+                    ("PageSize", page_size),
+                    ("PageNumber", page_number),
+                ),
+            )
+
+        first = query_page(1)
+        yielded = 0
+        page = first
+        page_number = 1
+        self.total_documents = min(first["TotalCount"], limit or first["TotalCount"])
+        while True:
+            for document in page["Documents"]:
+                yield document["Code"]
+                yielded += 1
+                if limit is not None and yielded >= limit:
+                    return
+            if yielded >= page["TotalCount"]:
+                return
+            page_number += 1
+            try:
+                page = self.request_json(
+                    "GetDocumentListById",
+                    (
+                        ("DocumentListId", first["Id"]),
+                        ("Property", "Code"),
+                        ("PageSize", page_size),
+                        ("PageNumber", page_number),
+                    ),
+                )
+            except (urllib.error.HTTPError, RuntimeError) as error:
+                if isinstance(error, urllib.error.HTTPError) and error.code != 404:
+                    raise
+                first = page = query_page(page_number)
+                self.total_documents = min(
+                    first["TotalCount"], limit or first["TotalCount"]
+                )
+
+    def document(self, code: int) -> dict:
+        return self.request_json(
+            "GetDocument",
+            (
+                ("Code", code),
+                ("Editions.Select", "all"),
+                ("Editions.Data", "all"),
+                ("Editions.Images.Select", "all"),
+                ("Editions.Images.Data", "all"),
+            ),
+        )
+
+
+@dataclass(frozen=True)
+class SyncResult:
+    discovered: int = 0
+    downloaded: int = 0
+    skipped: int = 0
+    failed: int = 0
+
+
+def format_duration(seconds: float) -> str:
+    seconds = max(0, round(seconds))
+    hours, seconds = divmod(seconds, 3600)
+    minutes, seconds = divmod(seconds, 60)
+    return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+
+def progress_line(
+    result: SyncResult, total: int, elapsed: float, width: int = 24
+) -> str:
+    processed = result.discovered
+    fraction = processed / total if total else 0
+    filled = min(width, round(width * fraction))
+    rate = processed / elapsed if elapsed > 0 else 0
+    eta = (total - processed) / rate if rate else 0
+    return (
+        f"[{'#' * filled}{'-' * (width - filled)}] {fraction:6.2%} "
+        f"{processed}/{total} downloaded={result.downloaded} "
+        f"skipped={result.skipped} failed={result.failed} "
+        f"rate={rate:.2f}/s ETA={format_duration(eta)}"
+    )
+
+
+def terminal_progress() -> Callable[[SyncResult, int], None]:
+    started = time.monotonic()
+
+    def update(result: SyncResult, total: int) -> None:
+        line = progress_line(result, total, time.monotonic() - started)
+        print(
+            f"\r{line}",
+            end="\n" if result.discovered >= total else "",
+            file=sys.stderr,
+            flush=True,
+        )
+
+    return update
+
+
+def connect_manifest(path: Path) -> sqlite3.Connection:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    connection = sqlite3.connect(path)
+    connection.execute(
+        """
+        CREATE TABLE IF NOT EXISTS documents (
+            code INTEGER PRIMARY KEY,
+            source_sha256 TEXT NOT NULL,
+            fetched_at TEXT NOT NULL
+        )
+        """
+    )
+    connection.execute(
+        """
+        CREATE TABLE IF NOT EXISTS errors (
+            code INTEGER PRIMARY KEY,
+            message TEXT NOT NULL,
+            failed_at TEXT NOT NULL
+        )
+        """
+    )
+    return connection
+
+
+def atomic_write(path: Path, content: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
+        temporary.write(content)
+        temporary_path = Path(temporary.name)
+    os.replace(temporary_path, path)
+
+
+def json_bytes(value: object) -> bytes:
+    return (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode()
+
+
+def archive_document(root: Path, document: dict) -> str:
+    code = int(document["Code"])
+    source = json.dumps(document, ensure_ascii=False, sort_keys=True).encode()
+    source_sha256 = hashlib.sha256(source).hexdigest()
+    directory = root / "documents" / str(code)
+    editions = document.get("Editions") or []
+    metadata = {key: value for key, value in document.items() if key != "Editions"}
+    atomic_write(directory / "metadata.json", json_bytes(metadata))
+
+    for edition in editions:
+        edition_directory = directory / "editions" / str(int(edition["Code"]))
+        edition_metadata = {key: value for key, value in edition.items() if key != "Data"}
+        edition_metadata["Images"] = [
+            {key: value for key, value in image.items() if key != "Data"}
+            for image in edition.get("Images") or []
+        ]
+        atomic_write(edition_directory / "metadata.json", json_bytes(edition_metadata))
+
+        for source_language, filename in LANGUAGES.items():
+            html = (edition.get("Data") or {}).get(source_language)
+            if html:
+                atomic_write(edition_directory / f"{filename}.html", html.encode())
+
+        for image in edition.get("Images") or []:
+            if not image.get("Data"):
+                continue
+            language = IMAGE_LANGUAGES.get(image.get("Lang"), "unknown")
+            filename = Path(str(image.get("Name") or "image").replace("\\", "/")).name
+            if filename in {"", ".", ".."}:
+                raise ValueError(f"Invalid image filename in document {code}")
+            atomic_write(
+                edition_directory / "images" / language / filename,
+                base64.b64decode(image["Data"], validate=True),
+            )
+    return source_sha256
+
+
+def sync_archive(
+    output: Path,
+    client: CbdClient | None = None,
+    limit: int | None = None,
+    refresh: bool = False,
+    progress: Callable[[SyncResult, int], None] | None = None,
+) -> SyncResult:
+    client = client or CbdClient()
+    output.mkdir(parents=True, exist_ok=True)
+    connection = connect_manifest(output / "manifest.sqlite3")
+    known = {row[0] for row in connection.execute("SELECT code FROM documents")}
+    discovered = downloaded = skipped = failed = 0
+
+    client.check_available()
+    for code in client.document_codes(limit):
+        discovered += 1
+        # ponytail: refresh scans all documents; use sitemap lastmod when scheduled
+        # update checks become frequent enough for the extra mapping logic to pay off.
+        if code in known and not refresh:
+            skipped += 1
+            if progress:
+                progress(
+                    SyncResult(discovered, downloaded, skipped, failed),
+                    client.total_documents,
+                )
+            continue
+        try:
+            document = client.document(code)
+            checksum = archive_document(output, document)
+            fetched_at = datetime.now(timezone.utc).isoformat()
+            with connection:
+                connection.execute(
+                    """
+                    INSERT INTO documents (code, source_sha256, fetched_at)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(code) DO UPDATE SET
+                        source_sha256 = excluded.source_sha256,
+                        fetched_at = excluded.fetched_at
+                    """,
+                    (int(document["Code"]), checksum, fetched_at),
+                )
+                connection.execute("DELETE FROM errors WHERE code = ?", (code,))
+            downloaded += 1
+        except Exception as error:  # Continue the long-running archive after one bad record.
+            with connection:
+                connection.execute(
+                    """
+                    INSERT INTO errors (code, message, failed_at) VALUES (?, ?, ?)
+                    ON CONFLICT(code) DO UPDATE SET
+                        message = excluded.message,
+                        failed_at = excluded.failed_at
+                    """,
+                    (code, str(error), datetime.now(timezone.utc).isoformat()),
+                )
+            failed += 1
+        if progress:
+            progress(
+                SyncResult(discovered, downloaded, skipped, failed),
+                client.total_documents,
+            )
+        if not progress and discovered % 100 == 0:
+            LOGGER.info(
+                "discovered=%s downloaded=%s skipped=%s failed=%s",
+                discovered,
+                downloaded,
+                skipped,
+                failed,
+            )
+    connection.close()
+    return SyncResult(discovered, downloaded, skipped, failed)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, default=Path("data/minjust-cbd"))
+    parser.add_argument("--limit", type=int, help="download only the first N documents")
+    parser.add_argument("--refresh", action="store_true", help="redownload known documents")
+    parser.add_argument("--requests-per-second", type=float, default=1)
+    parser.add_argument("--timeout", type=int, default=60)
+    parser.add_argument("--retries", type=int, default=5)
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    return parser.parse_args()
+
+
+def main() -> int:
+    arguments = parse_args()
+    if arguments.limit is not None and arguments.limit <= 0:
+        raise SystemExit("--limit must be greater than zero")
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+    result = sync_archive(
+        arguments.output,
+        CbdClient(
+            requests_per_second=arguments.requests_per_second,
+            timeout=arguments.timeout,
+            retries=arguments.retries,
+        ),
+        arguments.limit,
+        arguments.refresh,
+        terminal_progress() if sys.stderr.isatty() else None,
+    )
+    print(
+        f"discovered={result.discovered} downloaded={result.downloaded} "
+        f"skipped={result.skipped} failed={result.failed}\n"
+        f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2"
+    )
+    return int(result.failed > 0)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/ingestion/minjust_current.py

@@ -0,0 +1,720 @@
+"""Read-only staging client for the current MinJust CBD API."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import http.client
+import json
+import logging
+import os
+import sqlite3
+import stat
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+from typing import Callable
+
+from ingestion.minjust_cbd import atomic_write, connect_manifest, json_bytes
+
+API_ROOT = "https://cbd.minjust.gov.kg/api/v1"
+APP_VERSION = "0.12.2"
+REQUEST_CONTRACT = {
+    "version": 2,
+    "endpoint": "/GetEdition",
+    "edition_id_parameter": "editionId",
+    "exact_parameter": "true",
+    "language_parameter": "lang",
+    "response_id_must_match": True,
+}
+MAX_RESPONSE_BYTES = 16 * 1024 * 1024
+DEFAULT_DOCUMENT_BUDGET = 4 * 1024 * 1024 * 1024
+LANGUAGES = (("ru", "ru", "textRusType"), ("kg", "ky", "textKyrType"))
+LOGGER = logging.getLogger(__name__)
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+
+
+class SourceError(RuntimeError):
+    pass
+
+
+class SourceChanged(SourceError):
+    pass
+
+
+@dataclass(frozen=True)
+class ApiResponse:
+    value: dict
+    body: bytes
+    sha256: str
+
+
+class CurrentCbdClient:
+    def __init__(
+        self,
+        requests_per_second: float = 1,
+        timeout: int = 60,
+        retries: int = 5,
+        max_response_bytes: int = MAX_RESPONSE_BYTES,
+        opener: Callable = urllib.request.urlopen,
+        sleep: Callable[[float], None] = time.sleep,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        if requests_per_second <= 0 or timeout <= 0 or retries <= 0 or max_response_bytes <= 0:
+            raise ValueError("rate, timeout, retries, and response limit must be positive")
+        self.interval = 1 / requests_per_second
+        self.timeout = timeout
+        self.retries = retries
+        self.max_response_bytes = max_response_bytes
+        self.opener = opener
+        self.sleep = sleep
+        self.clock = clock
+        self.last_request = 0.0
+
+    def request_json(
+        self,
+        path: str,
+        *,
+        method: str = "GET",
+        parameters: tuple[tuple[str, object], ...] = (),
+        payload: dict | None = None,
+    ) -> ApiResponse:
+        if not path.startswith("/") or ".." in path.split("/"):
+            raise ValueError("API path must be an absolute endpoint path")
+        query = urllib.parse.urlencode(parameters, doseq=True)
+        url = API_ROOT + path + (f"?{query}" if query else "")
+        body = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":")).encode() if method == "POST" else None
+        for attempt in range(self.retries):
+            wait = self.interval - (self.clock() - self.last_request)
+            if wait > 0:
+                self.sleep(wait)
+            request = urllib.request.Request(
+                url,
+                data=body,
+                method=method,
+                headers={
+                    "Accept": "application/json",
+                    **({"Content-Type": "application/json"} if body is not None else {}),
+                    "User-Agent": f"Akyldash/{APP_VERSION} (+https://cbd.minjust.gov.kg)",
+                },
+            )
+            try:
+                self.last_request = self.clock()
+                with self.opener(request, timeout=self.timeout) as response:
+                    raw = response.read(self.max_response_bytes + 1)
+                if len(raw) > self.max_response_bytes:
+                    raise SourceError(f"MinJust response exceeds {self.max_response_bytes} bytes")
+                value = json.loads(raw)
+                if not isinstance(value, dict):
+                    raise SourceError("MinJust response must be a JSON object")
+                return ApiResponse(value, raw, hashlib.sha256(raw).hexdigest())
+            except urllib.error.HTTPError as error:
+                if error.code != 429 and error.code < 500:
+                    raise SourceError(f"MinJust returned HTTP {error.code} for {path}") from error
+                delay = 2**attempt
+                if error.code == 429:
+                    delay = max(delay, retry_after_seconds(error.headers.get("Retry-After")))
+            except (TimeoutError, urllib.error.URLError, http.client.IncompleteRead, json.JSONDecodeError) as error:
+                delay = 2**attempt
+                LOGGER.warning("MinJust request failed for %s (attempt %s/%s)", path, attempt + 1, self.retries)
+                last_error = error
+            if attempt + 1 < self.retries:
+                self.sleep(delay)
+        raise SourceError(f"MinJust request failed after {self.retries} attempts: {path}") from locals().get("last_error")
+
+    def document(self, alias: str) -> ApiResponse:
+        response = self.request_json("/GetDocument", parameters=(("DocumentCode", alias),))
+        value = response.value
+        canonical = value.get("documentCode")
+        editions = value.get("editions")
+        if not isinstance(canonical, int) or isinstance(canonical, bool) or canonical < 1:
+            raise SourceError("MinJust document has an invalid canonical documentCode")
+        if not isinstance(editions, list):
+            raise SourceError("MinJust document is missing editions")
+        ids = [edition_id(item) for item in editions]
+        if len(ids) != len(set(ids)):
+            raise SourceError(f"MinJust document {canonical} contains duplicate edition IDs")
+        return response
+
+    def edition(self, identifier: int, language: str) -> ApiResponse:
+        if identifier < 1 or language not in {"ru", "kg"}:
+            raise ValueError("edition ID or requested language is invalid")
+        response = self.request_json(
+            "/GetEdition",
+            parameters=(("editionId", identifier), ("lang", language), ("exact", "true")),
+        )
+        returned_id = response.value.get("id")
+        if not isinstance(returned_id, int) or isinstance(returned_id, bool) or returned_id < 1:
+            raise SourceError(f"MinJust returned an invalid edition response ID for {identifier}")
+        if returned_id != identifier:
+            raise SourceChanged(f"MinJust exact edition ID {returned_id} does not match requested ID {identifier}")
+        return response
+
+
+def retry_after_seconds(value: str | None) -> float:
+    if not value:
+        return 0
+    try:
+        return max(0, float(value))
+    except ValueError:
+        try:
+            return max(0, (parsedate_to_datetime(value) - datetime.now(timezone.utc)).total_seconds())
+        except (TypeError, ValueError, OverflowError):
+            return 0
+
+
+def edition_id(value: object) -> int:
+    if not isinstance(value, dict):
+        raise SourceError("MinJust document contains an invalid edition")
+    identifier = value.get("id")
+    if not isinstance(identifier, int) or isinstance(identifier, bool) or identifier < 1:
+        raise SourceError("MinJust edition has an invalid id")
+    return identifier
+
+
+def open_manifest(path: Path) -> sqlite3.Connection:
+    connection = sqlite3.connect(path)
+    connection.execute("PRAGMA journal_mode=DELETE")
+    connection.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS documents (
+            alias TEXT PRIMARY KEY,
+            canonical_id INTEGER,
+            response_sha256 TEXT,
+            expected_editions INTEGER,
+            state TEXT NOT NULL,
+            error TEXT,
+            updated_at TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS editions (
+            canonical_id INTEGER NOT NULL,
+            edition_id INTEGER NOT NULL,
+            source_order INTEGER NOT NULL,
+            response_sha256 TEXT NOT NULL,
+            PRIMARY KEY (canonical_id, edition_id)
+        );
+        CREATE TABLE IF NOT EXISTS languages (
+            canonical_id INTEGER NOT NULL,
+            edition_id INTEGER NOT NULL,
+            request_language TEXT NOT NULL,
+            archive_language TEXT NOT NULL,
+            response_sha256 TEXT,
+            content_sha256 TEXT,
+            byte_count INTEGER NOT NULL DEFAULT 0,
+            state TEXT NOT NULL,
+            error TEXT,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (canonical_id, edition_id, request_language)
+        );
+        CREATE TABLE IF NOT EXISTS generation (
+            contract_sha256 TEXT NOT NULL
+        );
+        """
+    )
+    return connection
+
+
+def reject_symlink_components(path: Path) -> None:
+    current = Path(path.anchor)
+    for part in path.parts[1:]:
+        current /= part
+        try:
+            mode = current.lstat().st_mode
+        except FileNotFoundError:
+            continue
+        if stat.S_ISLNK(mode):
+            raise ValueError(f"staging path must not contain a symbolic link: {current}")
+        if current != path and not stat.S_ISDIR(mode):
+            raise ValueError(f"staging parent must be a directory: {current}")
+
+
+def ensure_stage_path(root: Path, path: Path) -> Path:
+    root = root.absolute()
+    path = path.absolute()
+    try:
+        relative = path.relative_to(root)
+    except ValueError as error:
+        raise ValueError("staging file escaped its private generation") from error
+    current = root
+    if current.is_symlink():
+        raise ValueError("staging root must not be a symbolic link")
+    for part in relative.parts:
+        current /= part
+        try:
+            mode = current.lstat().st_mode
+        except FileNotFoundError:
+            continue
+        if stat.S_ISLNK(mode):
+            raise ValueError(f"staging path must not contain a symbolic link: {relative}")
+        if current != path and not stat.S_ISDIR(mode):
+            raise ValueError(f"staging parent must be a directory: {relative}")
+    return path
+
+
+def stage_write(root: Path, path: Path, content: bytes) -> None:
+    path = ensure_stage_path(root, path)
+    atomic_write(path, content)
+    ensure_stage_path(root, path)
+
+
+def stage_mkdir(root: Path, path: Path) -> None:
+    path = ensure_stage_path(root, path)
+    path.mkdir(parents=True, exist_ok=True)
+    ensure_stage_path(root, path)
+    if not path.is_dir():
+        raise ValueError("staging path must be a directory")
+
+
+def validate_stage_tree(output: Path) -> None:
+    for relative in (
+        Path("current-api.sqlite3"), Path("current-api.sqlite3-journal"),
+        Path("current-api.sqlite3-wal"), Path("current-api.sqlite3-shm"),
+        Path("manifest.sqlite3"), Path("manifest.sqlite3-journal"),
+        Path("manifest.sqlite3-wal"), Path("manifest.sqlite3-shm"),
+        Path("request-contract.json"),
+    ):
+        path = ensure_stage_path(output, output / relative)
+        if path.exists() and not path.is_file():
+            raise ValueError(f"staging checkpoint must be a regular file: {relative}")
+    for relative in (Path(".work"), Path("documents")):
+        root = ensure_stage_path(output, output / relative)
+        if root.exists() and not root.is_dir():
+            raise ValueError(f"staging path must be a directory: {relative}")
+        if root.is_dir():
+            for directory, directories, files in os.walk(root, followlinks=False):
+                for name in directories + files:
+                    ensure_stage_path(output, Path(directory) / name)
+
+
+def validate_stage_root(output: Path, resume: bool) -> Path:
+    requested = Path(output)
+    if ".." in requested.parts:
+        raise ValueError("staging output path must not contain '..'")
+    destination = Path(os.path.abspath(requested))
+    reject_symlink_components(destination)
+    parent = destination.parent
+    if not parent.is_dir():
+        raise ValueError("staging output parent must already exist")
+    for ancestor in (parent, *parent.parents):
+        info = ancestor.lstat()
+        mode = stat.S_IMODE(info.st_mode)
+        if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
+            raise ValueError("staging output ancestors must be real directories")
+        sticky_parent = bool(mode & stat.S_ISVTX)
+        if mode & 0o022 and not sticky_parent:
+            raise ValueError("staging output parent is not trusted")
+    for relative in (Path("data/minjust-cbd"), Path("data/minjust-normalized")):
+        legacy = (REPOSITORY_ROOT / relative).resolve()
+        if destination == legacy or destination.is_relative_to(legacy) or legacy.is_relative_to(destination):
+            raise ValueError("current API staging must not overlap the existing corpus")
+    if destination.exists():
+        if destination.is_symlink() or not destination.is_dir():
+            raise ValueError("staging output must be a real directory")
+    else:
+        destination.mkdir(mode=0o700)
+    root_info = destination.lstat()
+    root_mode = stat.S_IMODE(root_info.st_mode)
+    if (
+        stat.S_ISLNK(root_info.st_mode)
+        or not stat.S_ISDIR(root_info.st_mode)
+        or root_info.st_uid != os.geteuid()
+        or root_mode & 0o077
+        or (root_mode & 0o700) != 0o700
+    ):
+        raise ValueError("staging output must be a private directory owned by the current user")
+    validate_stage_tree(destination)
+    if any(destination.iterdir()) and not resume:
+        raise ValueError("staging output is not empty; use --resume for this generation")
+    contract_path = destination / "request-contract.json"
+    if contract_path.exists():
+        try:
+            contract = json.loads(contract_path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError) as error:
+            raise SourceChanged("staging generation request contract is invalid") from error
+        if contract != REQUEST_CONTRACT:
+            raise SourceChanged("staging generation uses a different request contract")
+    else:
+        if any(destination.iterdir()):
+            raise SourceChanged("staging generation has no exact-edition request contract; start a new generation")
+        stage_write(destination, contract_path, json_bytes(REQUEST_CONTRACT))
+    return destination
+
+
+def stage_document(
+    output: Path,
+    alias: str,
+    client: CurrentCbdClient,
+    *,
+    resume: bool = False,
+    max_document_bytes: int = DEFAULT_DOCUMENT_BUDGET,
+) -> dict:
+    """Stage one document and its official editions without touching legacy roots."""
+    if not alias or len(alias) > 64 or "/" in alias or "\\" in alias:
+        raise ValueError("document code is invalid")
+    if max_document_bytes <= 0:
+        raise ValueError("document byte budget must be positive")
+    output = validate_stage_root(output, resume)
+    work_root = output / ".work" / "documents"
+    target_root = output / "documents"
+    connection = open_manifest(output / "current-api.sqlite3")
+    raw_manifest = connect_manifest(output / "manifest.sqlite3")
+    try:
+        contract_sha = hashlib.sha256(json.dumps(REQUEST_CONTRACT, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+        saved_contract = connection.execute("SELECT contract_sha256 FROM generation").fetchone()
+        if saved_contract and saved_contract[0] != contract_sha:
+            raise SourceChanged("checkpoint request contract changed; start a new staging generation")
+        if not saved_contract:
+            if connection.execute("SELECT COUNT(*) FROM documents").fetchone()[0]:
+                raise SourceChanged("existing checkpoints have no request contract; start a new staging generation")
+            with connection:
+                connection.execute("INSERT INTO generation (contract_sha256) VALUES (?)", (contract_sha,))
+        response = client.document(alias)
+        source = response.value
+        canonical = source["documentCode"]
+        record_sha = response.sha256
+        editions = source["editions"]
+        previous = connection.execute(
+            "SELECT canonical_id, response_sha256, expected_editions FROM documents WHERE alias = ?",
+            (alias,),
+        ).fetchone()
+        if previous and previous[1] is not None and (
+            previous[0] != canonical or previous[1] != record_sha or previous[2] != len(editions)
+        ):
+            raise SourceChanged("document metadata changed; start a new staging generation")
+        working_document = work_root / str(canonical)
+        final_document = target_root / str(canonical)
+        ensure_stage_path(output, working_document)
+        ensure_stage_path(output, final_document)
+        if final_document.is_symlink():
+            raise ValueError("staging target must not be a symbolic link")
+        if final_document.exists() and not previous:
+            raise ValueError("staging target already exists without a matching checkpoint")
+        if final_document.exists():
+            final_metadata = ensure_stage_path(output, final_document / "metadata.json")
+            source_document = ensure_stage_path(output, final_document / "source-document.json")
+            metadata = json.loads(final_metadata.read_text(encoding="utf-8"))
+            if (
+                metadata.get("SourceApi", {}).get("response_sha256") != record_sha
+                or hashlib.sha256(source_document.read_bytes()).hexdigest() != record_sha
+            ):
+                raise SourceChanged("published staging document does not match the current source record")
+            for edition in editions:
+                identifier = edition_id(edition)
+                for request_language, archive_language, type_field in LANGUAGES:
+                    html_path = ensure_stage_path(
+                        output,
+                        final_document / "editions" / str(identifier) / f"{archive_language}.html",
+                    )
+                    checkpoint = connection.execute(
+                        "SELECT state, content_sha256 FROM languages WHERE canonical_id = ? AND edition_id = ? AND request_language = ?",
+                        (canonical, identifier, request_language),
+                    ).fetchone()
+                    if not edition.get(type_field):
+                        if not checkpoint or checkpoint[0] != "unavailable":
+                            raise SourceError(f"published staging document has no checkpoint for edition {identifier} {request_language}")
+                    elif (
+                        not checkpoint
+                        or checkpoint[0] != "complete"
+                        or not html_path.is_file()
+                        or sha256_file(html_path) != checkpoint[1]
+                    ):
+                        raise SourceError(f"published staging document failed verification for edition {identifier} {request_language}")
+            document_sha = hash_tree(final_document)
+            fetched_at = utc_now()
+            manifest_row = raw_manifest.execute(
+                "SELECT source_sha256 FROM documents WHERE code = ?", (canonical,)
+            ).fetchone()
+            if manifest_row and manifest_row[0] != document_sha:
+                raise SourceError("published staging document does not match its raw manifest")
+            with raw_manifest:
+                raw_manifest.execute(
+                    "INSERT OR IGNORE INTO documents (code, source_sha256, fetched_at) VALUES (?, ?, ?)",
+                    (canonical, document_sha, fetched_at),
+                )
+                raw_manifest.execute(
+                    "INSERT INTO errors (code, message, failed_at) VALUES (?, ?, ?) "
+                    "ON CONFLICT(code) DO UPDATE SET message=excluded.message, failed_at=excluded.failed_at",
+                    (canonical, "attachments and embedded media are not independently verified", fetched_at),
+                )
+            with connection:
+                connection.execute(
+                    "UPDATE documents SET state='text_complete_incomplete_attachments', error=NULL, updated_at=? WHERE alias=?",
+                    (fetched_at, alias),
+                )
+            return {
+                "alias": alias,
+                "canonical_id": canonical,
+                "editions": len(editions),
+                "source_sha256": document_sha,
+                "state": "text_complete_incomplete_attachments",
+                "resumed": True,
+            }
+        stage_mkdir(output, working_document)
+        document_metadata = map_document_metadata(source, alias, response.sha256)
+        stage_write(output, working_document / "metadata.json", json_bytes(document_metadata))
+        stage_write(output, working_document / "source-document.json", response.body)
+        with connection:
+            connection.execute(
+                "INSERT INTO documents (alias, canonical_id, response_sha256, expected_editions, state, updated_at) VALUES (?, ?, ?, ?, 'fetching', ?) "
+                "ON CONFLICT(alias) DO UPDATE SET canonical_id=excluded.canonical_id, response_sha256=excluded.response_sha256, "
+                "expected_editions=excluded.expected_editions, state='fetching', error=NULL, updated_at=excluded.updated_at",
+                (alias, canonical, record_sha, len(editions), utc_now()),
+            )
+            for order, edition in enumerate(editions):
+                identifier = edition_id(edition)
+                existing = connection.execute(
+                    "SELECT source_order, response_sha256 FROM editions WHERE canonical_id = ? AND edition_id = ?",
+                    (canonical, identifier),
+                ).fetchone()
+                if existing and existing != (order, hashlib.sha256(json_bytes(edition)).hexdigest()):
+                    raise SourceChanged(f"edition metadata changed for {identifier}")
+                connection.execute(
+                    "INSERT OR IGNORE INTO editions (canonical_id, edition_id, source_order, response_sha256) VALUES (?, ?, ?, ?)",
+                    (canonical, identifier, order, hashlib.sha256(json_bytes(edition)).hexdigest()),
+                )
+
+        incomplete = []
+        for order, edition in enumerate(editions):
+            identifier = edition_id(edition)
+            directory = working_document / "editions" / str(identifier)
+            stage_mkdir(output, directory)
+            stage_write(output, directory / "metadata.json", json_bytes(map_edition_metadata(edition, order)))
+            for request_language, archive_language, type_field in LANGUAGES:
+                html_path = ensure_stage_path(output, directory / f"{archive_language}.html")
+                checkpoint = connection.execute(
+                    "SELECT state, content_sha256, byte_count FROM languages WHERE canonical_id = ? AND edition_id = ? AND request_language = ?",
+                    (canonical, identifier, request_language),
+                ).fetchone()
+                if checkpoint and checkpoint[0] == "complete":
+                    if not html_path.is_file() or sha256_file(html_path) != checkpoint[1]:
+                        raise SourceError(f"staged content hash mismatch for edition {identifier} {request_language}")
+                    continue
+                if not edition.get(type_field):
+                    save_language_checkpoint(
+                        connection, canonical, identifier, request_language, archive_language,
+                        None, None, 0, "unavailable", "source metadata reports no language file",
+                    )
+                    continue
+                try:
+                    language_response = client.edition(identifier, request_language)
+                    response_id = language_response.value.get("id")
+                    if isinstance(response_id, bool) or not isinstance(response_id, int) or response_id != identifier:
+                        raise SourceChanged(f"MinJust exact edition ID mismatch for {identifier}")
+                    for field in ("nameRus", "nameKyr"):
+                        if edition.get(field) is not None and language_response.value.get(field) != edition[field]:
+                            raise SourceChanged(f"MinJust exact edition metadata mismatch for {identifier}: {field}")
+                    content = language_response.value.get("contentRu")
+                    if not isinstance(content, str) or not content:
+                        raise SourceError(f"MinJust returned no content for edition {identifier} language {request_language}")
+                    encoded = content.encode("utf-8")
+                    completed_bytes = connection.execute(
+                        "SELECT COALESCE(SUM(byte_count), 0) FROM languages WHERE canonical_id = ? AND state = 'complete'",
+                        (canonical,),
+                    ).fetchone()[0]
+                    if completed_bytes + len(encoded) > max_document_bytes:
+                        raise SourceError(f"document {canonical} exceeds its {max_document_bytes}-byte staging budget")
+                    response_metadata = {
+                        key: value
+                        for key, value in language_response.value.items()
+                        if key not in {"contentRu", "contentKg"}
+                    }
+                    response_metadata["requested_language"] = request_language
+                    response_metadata["archive_language"] = archive_language
+                    response_metadata["requested_edition_id"] = identifier
+                    response_metadata["exact"] = True
+                    response_metadata["response_sha256"] = language_response.sha256
+                    stage_write(
+                        output,
+                        directory / "source" / f"{request_language}.json",
+                        json_bytes(response_metadata),
+                    )
+                    stage_write(output, html_path, encoded)
+                    save_language_checkpoint(
+                        connection, canonical, identifier, request_language, archive_language,
+                        language_response.sha256, hashlib.sha256(encoded).hexdigest(), len(encoded), "complete", None,
+                    )
+                except Exception as error:
+                    message = str(error)[:500]
+                    save_language_checkpoint(
+                        connection, canonical, identifier, request_language, archive_language,
+                        None, None, 0, "error", message,
+                    )
+                    incomplete.append({"edition_id": identifier, "language": request_language, "error": message})
+                    if isinstance(error, SourceChanged) or (isinstance(error, SourceError) and "budget" in message):
+                        raise
+
+        if incomplete:
+            with connection:
+                connection.execute(
+                    "UPDATE documents SET state='incomplete', error=?, updated_at=? WHERE alias=?",
+                    (json.dumps(incomplete, ensure_ascii=False), utc_now(), alias),
+                )
+            return {"alias": alias, "canonical_id": canonical, "editions": len(editions), "incomplete": incomplete}
+
+        working_metadata = ensure_stage_path(output, working_document / "metadata.json")
+        metadata = json.loads(working_metadata.read_text(encoding="utf-8"))
+        metadata["SourceCompleteness"] = {
+            "text": "complete",
+            "attachments": "not independently verified",
+        }
+        stage_write(output, working_metadata, json_bytes(metadata))
+        stage_mkdir(output, target_root)
+        ensure_stage_path(output, working_document)
+        ensure_stage_path(output, final_document)
+        if final_document.exists():
+            raise ValueError("refusing to replace a document in this staging generation")
+        os.replace(working_document, final_document)
+        validate_stage_tree(output)
+        document_sha = hash_tree(final_document)
+        fetched_at = utc_now()
+        with raw_manifest:
+            raw_manifest.execute(
+                "INSERT INTO documents (code, source_sha256, fetched_at) VALUES (?, ?, ?)",
+                (canonical, document_sha, fetched_at),
+            )
+            raw_manifest.execute(
+                "INSERT INTO errors (code, message, failed_at) VALUES (?, ?, ?)",
+                (canonical, "attachments and embedded media are not independently verified", fetched_at),
+            )
+        with connection:
+            connection.execute(
+                "UPDATE documents SET state='text_complete_incomplete_attachments', updated_at=? WHERE alias=?",
+                (fetched_at, alias),
+            )
+        return {
+            "alias": alias,
+            "canonical_id": canonical,
+            "editions": len(editions),
+            "source_sha256": document_sha,
+            "state": "text_complete_incomplete_attachments",
+        }
+    except Exception as error:
+        with connection:
+            connection.execute(
+                "INSERT INTO documents (alias, state, error, updated_at) VALUES (?, 'error', ?, ?) "
+                "ON CONFLICT(alias) DO UPDATE SET state='error', error=excluded.error, updated_at=excluded.updated_at",
+                (alias, str(error)[:500], utc_now()),
+            )
+        raise
+    finally:
+        raw_manifest.close()
+        connection.close()
+
+
+def map_document_metadata(source: dict, alias: str, response_sha256: str) -> dict:
+    canonical = source["documentCode"]
+    return {
+        "Code": canonical,
+        "Name": {"Rus": source.get("nameRus"), "Kyr": source.get("nameKyr")},
+        "Title": {"Rus": source.get("headerRu"), "Kyr": source.get("headerKg")},
+        "Number": source.get("number"),
+        "DateAdopted": source.get("dateAdopted"),
+        "DateInclusion": source.get("dateInclusion"),
+        "DateOfEntry": source.get("dateOfEntry"),
+        "DatePublication": source.get("datePublication"),
+        "DateRegistration": source.get("dateRegistration"),
+        "IsPublicInRegister": source.get("isPublicInRegister"),
+        "SourceStatus": source.get("status"),
+        "SourceClassId": source.get("classId"),
+        "SourceRefTypeId": source.get("refTypeId"),
+        "SourceAuthorityIds": source.get("authoritiesId"),
+        "SourcePublicationIds": source.get("sourcePublicationsId"),
+        "SourceMetadata": {key: value for key, value in source.items() if key != "editions"},
+        "SourceDocumentCode": alias,
+        "SourceMediaStatus": "not independently verified",
+        "SourceApi": {"endpoint": "/GetDocument", "response_sha256": response_sha256},
+    }
+
+
+def map_edition_metadata(source: dict, source_order: int) -> dict:
+    return {
+        "Code": edition_id(source),
+        "Name": {"Rus": source.get("nameRus"), "Kyr": source.get("nameKyr")},
+        "SourceOrder": source_order,
+        "SourceTextTypes": {
+            "ru": source.get("textRusType"),
+            "ky": source.get("textKyrType"),
+        },
+        "SourceMediaStatus": "not independently verified",
+        "SourceMetadata": source,
+    }
+
+
+def save_language_checkpoint(
+    connection: sqlite3.Connection,
+    canonical: int,
+    edition: int,
+    request_language: str,
+    archive_language: str,
+    response_sha256: str | None,
+    content_sha256: str | None,
+    byte_count: int,
+    state: str,
+    error: str | None,
+) -> None:
+    with connection:
+        connection.execute(
+            "INSERT INTO languages (canonical_id, edition_id, request_language, archive_language, response_sha256, content_sha256, byte_count, state, error, updated_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(canonical_id, edition_id, request_language) DO UPDATE SET "
+            "archive_language=excluded.archive_language, response_sha256=excluded.response_sha256, content_sha256=excluded.content_sha256, "
+            "byte_count=excluded.byte_count, state=excluded.state, error=excluded.error, updated_at=excluded.updated_at",
+            (canonical, edition, request_language, archive_language, response_sha256, content_sha256, byte_count, state, error, utc_now()),
+        )
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for block in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def hash_tree(root: Path) -> str:
+    digest = hashlib.sha256()
+    for path in sorted(item for item in root.rglob("*") if item.is_file()):
+        digest.update(path.relative_to(root).as_posix().encode())
+        digest.update(b"\0")
+        digest.update(sha256_file(path).encode("ascii"))
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True, help="new, separate staging generation")
+    parser.add_argument("--document-code", required=True, help="one official document alias or canonical code")
+    parser.add_argument("--resume", action="store_true", help="resume only this staging generation")
+    parser.add_argument("--requests-per-second", type=float, default=1)
+    parser.add_argument("--timeout", type=int, default=60)
+    parser.add_argument("--retries", type=int, default=5)
+    parser.add_argument("--max-document-bytes", type=int, default=DEFAULT_DOCUMENT_BUDGET)
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    return parser.parse_args()
+
+
+def main() -> int:
+    arguments = parse_args()
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+    result = stage_document(
+        arguments.output,
+        arguments.document_code,
+        CurrentCbdClient(arguments.requests_per_second, arguments.timeout, arguments.retries),
+        resume=arguments.resume,
+        max_document_bytes=arguments.max_document_bytes,
+    )
+    print(json.dumps(result, ensure_ascii=False, sort_keys=True))
+    return int(result.get("state") != "text_complete_incomplete_attachments")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/normalization/minjust_cbd.py

@@ -0,0 +1,741 @@
+#!/usr/bin/env python3
+"""Normalize the local Ministry of Justice CBD archive."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import html
+import json
+import logging
+import os
+import re
+import shutil
+import sqlite3
+import sys
+import tempfile
+import time
+import unicodedata
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Callable
+from urllib.parse import urlsplit
+
+APP_VERSION = "0.10.0"
+SCHEMA_VERSION = "1"
+NORMALIZER_VERSION = "1.1.0"
+LANGUAGES = ("ru", "ky")
+LOGGER = logging.getLogger(__name__)
+
+ALLOWED_TAGS = {
+    "a", "b", "blockquote", "br", "div", "em", "h1", "h2", "h3", "h4",
+    "h5", "h6", "i", "img", "li", "ol", "p", "pre", "span", "strong",
+    "sub", "sup", "table", "tbody", "td", "tfoot", "th", "thead", "tr",
+    "u", "ul",
+}
+VOID_TAGS = {"br", "img"}
+DROP_CONTENT_TAGS = {"applet", "iframe", "noscript", "object", "script", "style", "svg"}
+DROP_ELEMENT_TAGS = {"link", "meta"}
+BLOCK_TAGS = {"blockquote", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "pre", "td", "th"}
+AUTO_CLOSE = {
+    "li": {"li"},
+    "p": {"blockquote", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ol", "p", "pre", "table", "ul"},
+    "td": {"td", "th"},
+    "th": {"td", "th"},
+    "tr": {"tr"},
+}
+
+
+@dataclass(frozen=True)
+class NormalizeResult:
+    discovered: int = 0
+    normalized: int = 0
+    skipped: int = 0
+    failed: int = 0
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def atomic_write(path: Path, content: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
+        temporary.write(content)
+        temporary_path = Path(temporary.name)
+    os.replace(temporary_path, path)
+
+
+def json_bytes(value: object) -> bytes:
+    return (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+
+
+def sha256_bytes(content: bytes) -> str:
+    return hashlib.sha256(content).hexdigest()
+
+
+def source_inventory(document_directory: Path, input_root: Path) -> tuple[list[dict], str]:
+    files = []
+    combined = hashlib.sha256()
+    for path in sorted(item for item in document_directory.rglob("*") if item.is_file()):
+        relative = path.relative_to(input_root).as_posix()
+        digest = hashlib.sha256()
+        with path.open("rb") as source:
+            for chunk in iter(lambda: source.read(1024 * 1024), b""):
+                digest.update(chunk)
+        checksum = digest.hexdigest()
+        files.append({"path": relative, "sha256": checksum})
+        combined.update(relative.encode("utf-8"))
+        combined.update(b"\0")
+        combined.update(checksum.encode("ascii"))
+        combined.update(b"\0")
+    return files, combined.hexdigest()
+
+
+def clean_value(value):
+    if isinstance(value, str):
+        normalized = unicodedata.normalize("NFC", value)
+        return normalized if normalized.strip() else None
+    if isinstance(value, dict):
+        return {key: clean_value(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [clean_value(item) for item in value]
+    return value
+
+
+def bilingual(value) -> dict[str, object]:
+    value = value if isinstance(value, dict) else {}
+    return {"ru": clean_value(value.get("Rus")), "ky": clean_value(value.get("Kyr"))}
+
+
+def hierarchy_paths(items: object, child_key: str) -> list[dict]:
+    paths: list[dict] = []
+
+    def visit(nodes: object, ancestors: dict[str, list[str]]) -> None:
+        for node in nodes if isinstance(nodes, list) else []:
+            if not isinstance(node, dict):
+                continue
+            names = bilingual(node.get("Name"))
+            current = {language: list(ancestors[language]) for language in LANGUAGES}
+            for language in LANGUAGES:
+                name = names[language]
+                if name:
+                    current[language].append(str(name))
+            children = node.get(child_key)
+            if children:
+                visit(children, current)
+            else:
+                paths.append(current)
+
+    visit(items, {"ru": [], "ky": []})
+    return paths
+
+
+class SafeHtmlParser(HTMLParser):
+    def __init__(self, edition_directory: Path) -> None:
+        super().__init__(convert_charrefs=True)
+        self.edition_directory = edition_directory.resolve()
+        self.parts: list[str] = []
+        self.stack: list[str] = []
+        self.drop_depth = 0
+        self.removed_elements = 0
+        self.removed_attributes = 0
+        self.removed_images = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        if self.drop_depth:
+            if tag in DROP_CONTENT_TAGS:
+                self.drop_depth += 1
+            return
+        if tag in DROP_CONTENT_TAGS:
+            self.drop_depth = 1
+            self.removed_elements += 1
+            return
+        if tag in DROP_ELEMENT_TAGS:
+            self.removed_elements += 1
+            return
+        if tag not in ALLOWED_TAGS:
+            self.removed_elements += 1
+            return
+        for open_tag, closing_tags in AUTO_CLOSE.items():
+            if tag in closing_tags and open_tag in self.stack:
+                self._close(open_tag)
+        safe_attrs = self._attributes(tag, attrs)
+        if tag == "img" and not any(name == "src" for name, _ in safe_attrs):
+            self.removed_images += 1
+            return
+        rendered = "".join(
+            f' {name}="{html.escape(value, quote=True)}"' for name, value in safe_attrs
+        )
+        self.parts.append(f"<{tag}{rendered}>")
+        if tag not in VOID_TAGS:
+            self.stack.append(tag)
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.handle_starttag(tag, attrs)
+        if tag.lower() not in VOID_TAGS:
+            self.handle_endtag(tag)
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if self.drop_depth:
+            if tag in DROP_CONTENT_TAGS:
+                self.drop_depth -= 1
+            return
+        if tag in self.stack:
+            self._close(tag)
+
+    def handle_data(self, data: str) -> None:
+        if not self.drop_depth:
+            self.parts.append(html.escape(unicodedata.normalize("NFC", data), quote=False))
+
+    def close(self) -> None:
+        super().close()
+        while self.stack:
+            self.parts.append(f"</{self.stack.pop()}>")
+
+    def _close(self, tag: str) -> None:
+        while self.stack:
+            current = self.stack.pop()
+            self.parts.append(f"</{current}>")
+            if current == tag:
+                break
+
+    def _attributes(self, tag: str, attrs: list[tuple[str, str | None]]) -> list[tuple[str, str]]:
+        allowed = {"title"}
+        if tag == "a":
+            allowed |= {"href"}
+        elif tag == "img":
+            allowed |= {"alt", "src"}
+        elif tag in {"td", "th"}:
+            allowed |= {"colspan", "rowspan"}
+        safe = []
+        for raw_name, raw_value in attrs:
+            name = raw_name.lower()
+            value = unicodedata.normalize("NFC", raw_value or "")
+            if name not in allowed:
+                self.removed_attributes += 1
+                continue
+            if name == "href" and not safe_link(value):
+                self.removed_attributes += 1
+                continue
+            if name == "src" and not self.safe_image(value):
+                self.removed_attributes += 1
+                continue
+            if name in {"colspan", "rowspan"} and not value.isdigit():
+                self.removed_attributes += 1
+                continue
+            safe.append((name, value))
+        if tag == "a" and any(name == "href" and urlsplit(value).scheme in {"http", "https"} for name, value in safe):
+            safe.append(("rel", "noopener noreferrer"))
+        return safe
+
+    def safe_image(self, value: str) -> bool:
+        try:
+            parsed = urlsplit(value)
+        except ValueError:
+            return False
+        if parsed.scheme or parsed.netloc or not parsed.path or parsed.path.startswith(("/", "\\")):
+            return False
+        candidate = (self.edition_directory / parsed.path.replace("\\", "/")).resolve()
+        try:
+            candidate.relative_to(self.edition_directory)
+        except ValueError:
+            return False
+        return candidate.is_file()
+
+
+def safe_link(value: str) -> bool:
+    value = value.strip()
+    if not value or value.startswith(("//", "\\\\")):
+        return False
+    try:
+        parsed = urlsplit(value)
+    except ValueError:
+        return False
+    return parsed.scheme.lower() in {"", "http", "https", "mailto"} and not (
+        not parsed.scheme and parsed.netloc
+    )
+
+
+def sanitize_html(source: str, edition_directory: Path) -> tuple[str, dict]:
+    parser = SafeHtmlParser(edition_directory)
+    parser.feed(source)
+    parser.close()
+    return unicodedata.normalize("NFC", "".join(parser.parts)), {
+        "removed_elements": parser.removed_elements,
+        "removed_attributes": parser.removed_attributes,
+        "removed_images": parser.removed_images,
+    }
+
+
+class TextBlockParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.blocks: list[tuple[str, str]] = []
+        self.active_tag: str | None = None
+        self.active: list[str] = []
+        self.loose: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        if tag in BLOCK_TAGS:
+            self._flush_active()
+            self._flush_loose()
+            self.active_tag = tag
+        elif tag == "br":
+            (self.active if self.active_tag else self.loose).append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == self.active_tag:
+            self._flush_active()
+
+    def handle_data(self, data: str) -> None:
+        (self.active if self.active_tag else self.loose).append(data)
+
+    def close(self) -> None:
+        super().close()
+        self._flush_active()
+        self._flush_loose()
+
+    def _flush_active(self) -> None:
+        if self.active_tag:
+            text = clean_text("".join(self.active))
+            if text:
+                self.blocks.append((self.active_tag, text))
+        self.active_tag = None
+        self.active = []
+
+    def _flush_loose(self) -> None:
+        text = clean_text("".join(self.loose))
+        if text:
+            self.blocks.append(("p", text))
+        self.loose = []
+
+
+def clean_text(value: str) -> str:
+    lines = []
+    for line in unicodedata.normalize("NFC", value).replace("\xa0", " ").splitlines():
+        line = re.sub(r"[ \t\f\v]+", " ", line).strip()
+        if line:
+            lines.append(line)
+    return "\n".join(lines)
+
+
+def fragment_type(tag: str, text: str, language: str) -> str:
+    lowered = text.casefold()
+    article_words = ("статья", "ст.") if language == "ru" else ("берене", "статья")
+    if any(re.match(rf"^{re.escape(word)}\s*\d", lowered) for word in article_words):
+        return "article"
+    if re.match(r"^\d+(?:\.\d+)*[.)]?\s+", text):
+        return "point"
+    if tag.startswith("h"):
+        return "heading"
+    return {"li": "list_item", "td": "table_cell", "th": "table_header"}.get(tag, "paragraph")
+
+
+def extract_text_and_fragments(
+    sanitized: str,
+    document_code: str,
+    edition_code: str,
+    language: str,
+    source_path: str,
+    source_sha256: str,
+) -> tuple[str, list[dict]]:
+    parser = TextBlockParser()
+    parser.feed(sanitized)
+    parser.close()
+    fragments = []
+    for position, (tag, text) in enumerate(parser.blocks, 1):
+        fragments.append(
+            {
+                "id": f"document:{document_code}:edition:{edition_code}:lang:{language}:fragment:{position}",
+                "document_code": document_code,
+                "edition_code": edition_code,
+                "language": language,
+                "position": position,
+                "type": fragment_type(tag, text, language),
+                "text": text,
+                "text_sha256": sha256_bytes(text.encode("utf-8")),
+                "source_path": source_path,
+                "source_sha256": source_sha256,
+            }
+        )
+    return "\n\n".join(fragment["text"] for fragment in fragments), fragments
+
+
+def load_json(path: Path) -> dict:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError(f"Expected JSON object: {path}")
+    return value
+
+
+def edition_summary(edition_directory: Path, input_root: Path) -> dict:
+    metadata = load_json(edition_directory / "metadata.json")
+    languages = [language for language in LANGUAGES if (edition_directory / f"{language}.html").is_file()]
+    summary = {
+        "source_code": str(metadata.get("Code", edition_directory.name)),
+        "name": bilingual(metadata.get("Name")),
+        "source_type": clean_value(metadata.get("Type")),
+        "available_languages": languages,
+        "source_path": edition_directory.relative_to(input_root).as_posix(),
+    }
+    if "SourceOrder" in metadata:
+        summary["source_order"] = metadata["SourceOrder"]
+    return summary
+
+
+def normalize_document(
+    document_directory: Path,
+    input_root: Path,
+    destination: Path,
+    files: list[dict] | None = None,
+    source_checksum: str | None = None,
+) -> None:
+    metadata_path = document_directory / "metadata.json"
+    metadata = load_json(metadata_path)
+    document_code = str(metadata.get("Code", document_directory.name))
+    if document_code != document_directory.name:
+        raise ValueError(f"Document code mismatch in {metadata_path}")
+    if files is None or source_checksum is None:
+        files, source_checksum = source_inventory(document_directory, input_root)
+    file_checksums = {item["path"]: item["sha256"] for item in files}
+    edition_root = document_directory / "editions"
+    edition_directories = sorted(
+        (path for path in edition_root.iterdir() if path.is_dir()),
+        key=lambda path: (not path.name.isdigit(), int(path.name) if path.name.isdigit() else path.name),
+    ) if edition_root.is_dir() else []
+    summaries = [edition_summary(path, input_root) for path in edition_directories]
+    source_orders = [item.get("source_order") for item in summaries]
+    if any(order is not None for order in source_orders):
+        if any(not isinstance(order, int) or isinstance(order, bool) or order < 0 for order in source_orders):
+            raise ValueError(f"Invalid source edition order in {document_directory}")
+        if len(set(source_orders)) != len(source_orders):
+            raise ValueError(f"Duplicate source edition order in {document_directory}")
+        ordered = sorted(zip(source_orders, edition_directories, summaries), key=lambda item: item[0])
+        edition_directories = [item[1] for item in ordered]
+        summaries = [item[2] for item in ordered]
+    available_languages = [language for language in LANGUAGES if any(language in item["available_languages"] for item in summaries)]
+    normalized_metadata = clean_value(metadata)
+    document = {
+        "schema_version": SCHEMA_VERSION,
+        "source_code": document_code,
+        "class": bilingual(metadata.get("Class")),
+        "type": bilingual(metadata.get("Type")),
+        "title": bilingual(metadata.get("Title")),
+        "name": bilingual(metadata.get("Name")),
+        "status": bilingual(metadata.get("Status")),
+        "number": clean_value(metadata.get("Number")),
+        "dates": {key: value for key, value in normalized_metadata.items() if key.startswith("Date")},
+        "registration_number": clean_value(metadata.get("NumberRegistration")),
+        "publication_number": clean_value(metadata.get("NumberPublication")),
+        "is_public_in_cdb": metadata.get("IsPublicInCdb"),
+        "is_public_in_register": metadata.get("IsPublicInRegister"),
+        "authorities": normalized_metadata.get("Authorities") or [],
+        "authority_paths": hierarchy_paths(metadata.get("Authorities"), "Authorities"),
+        "source_publications": normalized_metadata.get("SourcePublications") or [],
+        "source_publication_paths": hierarchy_paths(metadata.get("SourcePublications"), "SourcePublications"),
+        "keywords": normalized_metadata.get("Keywords") or [],
+        "keyword_paths": hierarchy_paths(metadata.get("Keywords"), "Keywords"),
+        "general_classifiers": normalized_metadata.get("GeneralClassifiers") or [],
+        "general_classifier_paths": hierarchy_paths(metadata.get("GeneralClassifiers"), "GeneralClassifiers"),
+        "references": normalized_metadata.get("References") or [],
+        "source_metadata": normalized_metadata,
+        "available_languages": available_languages,
+        "editions": summaries,
+        "source": {
+            "path": document_directory.relative_to(input_root).as_posix(),
+            "files": files,
+            "sha256": source_checksum,
+        },
+        "normalizer": {"version": NORMALIZER_VERSION, "processed_at": utc_now()},
+    }
+    atomic_write(destination / "document.json", json_bytes(document))
+
+    for edition_directory, summary in zip(edition_directories, summaries):
+        edition_code = summary["source_code"]
+        edition_metadata = load_json(edition_directory / "metadata.json")
+        edition_destination = destination / "editions" / edition_directory.name
+        image_records = []
+        for image in edition_metadata.get("Images") or []:
+            language = {"Russian": "ru", "Kyrgyz": "ky"}.get(image.get("Lang"), "unknown")
+            name = Path(str(image.get("Name") or "").replace("\\", "/")).name
+            source_path = edition_directory / "images" / language / name
+            relative = source_path.relative_to(input_root).as_posix()
+            image_records.append(
+                {
+                    "language": language,
+                    "name": clean_value(image.get("Name")),
+                    "source_path": relative if source_path.is_file() else None,
+                    "source_sha256": file_checksums.get(relative),
+                    "source_metadata": clean_value(image),
+                }
+            )
+        quality = {"has_html": bool(summary["available_languages"]), "languages": {}}
+        for language in summary["available_languages"]:
+            html_path = edition_directory / f"{language}.html"
+            relative = html_path.relative_to(input_root).as_posix()
+            raw = html_path.read_text(encoding="utf-8")
+            sanitized, sanitizer_quality = sanitize_html(raw, edition_directory)
+            text, fragments = extract_text_and_fragments(
+                sanitized, document_code, edition_code, language, relative, file_checksums[relative]
+            )
+            language_destination = edition_destination / language
+            atomic_write(language_destination / "content.html", sanitized.encode("utf-8"))
+            atomic_write(language_destination / "content.txt", (text + ("\n" if text else "")).encode("utf-8"))
+            atomic_write(language_destination / "fragments.json", json_bytes(fragments))
+            quality["languages"][language] = {
+                **sanitizer_quality,
+                "empty_text": not bool(text),
+                "fragment_count": len(fragments),
+            }
+        edition = {
+            "schema_version": SCHEMA_VERSION,
+            "source_code": edition_code,
+            "name": summary["name"],
+            "source_type": summary["source_type"],
+            "available_languages": summary["available_languages"],
+            "images": image_records,
+            "source_metadata": clean_value(edition_metadata),
+            "source": {
+                "path": summary["source_path"],
+                "files": [item for item in files if item["path"].startswith(summary["source_path"] + "/")],
+            },
+            "quality": quality,
+        }
+        atomic_write(edition_destination / "edition.json", json_bytes(edition))
+
+
+def connect_manifest(path: Path) -> sqlite3.Connection:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    connection = sqlite3.connect(path)
+    connection.execute(
+        """
+        CREATE TABLE IF NOT EXISTS documents (
+            code TEXT PRIMARY KEY,
+            source_sha256 TEXT,
+            schema_version TEXT NOT NULL,
+            normalizer_version TEXT NOT NULL,
+            processed_at TEXT,
+            state TEXT NOT NULL,
+            error TEXT,
+            failed_at TEXT
+        )
+        """
+    )
+    columns = {row[1] for row in connection.execute("PRAGMA table_info(documents)")}
+    if "failed_at" not in columns:
+        connection.execute("ALTER TABLE documents ADD COLUMN failed_at TEXT")
+    return connection
+
+
+def publish_directory(staged: Path, target: Path) -> None:
+    target.parent.mkdir(parents=True, exist_ok=True)
+    backup = target.parent / f".{target.name}.previous"
+    if backup.exists():
+        shutil.rmtree(backup)
+    if target.exists():
+        os.replace(target, backup)
+    try:
+        os.replace(staged, target)
+    except Exception:
+        if backup.exists():
+            os.replace(backup, target)
+        raise
+    if backup.exists():
+        shutil.rmtree(backup)
+
+
+def recover_directory(target: Path) -> None:
+    backup = target.parent / f".{target.name}.previous"
+    if not backup.exists():
+        return
+    if target.exists():
+        shutil.rmtree(backup)
+    else:
+        os.replace(backup, target)
+
+
+def validate_roots(input_root: Path, output: Path) -> None:
+    source = input_root.resolve()
+    destination = output.resolve()
+    if source == destination or source.is_relative_to(destination) or destination.is_relative_to(source):
+        raise ValueError("--input and --output must not overlap")
+
+
+def normalize_archive(
+    input_root: Path = Path("data/minjust-cbd"),
+    output: Path = Path("data/minjust-normalized"),
+    limit: int | None = None,
+    refresh: bool = False,
+    progress: Callable[[NormalizeResult, int], None] | None = None,
+) -> NormalizeResult:
+    validate_roots(input_root, output)
+    document_root = input_root / "documents"
+    if not document_root.is_dir():
+        raise FileNotFoundError(f"Document directory not found: {document_root}")
+    output.mkdir(parents=True, exist_ok=True)
+    connection = connect_manifest(output / "manifest.sqlite3")
+    known = {
+        row[0]: (row[1], row[2], row[3], row[4])
+        for row in connection.execute(
+            "SELECT code, source_sha256, schema_version, normalizer_version, state FROM documents"
+        )
+    }
+    directories = sorted(
+        (path for path in document_root.iterdir() if path.is_dir()),
+        key=lambda path: (not path.name.isdigit(), int(path.name) if path.name.isdigit() else path.name),
+    )
+    if limit is not None:
+        directories = directories[:limit]
+    total = len(directories)
+    discovered = normalized = skipped = failed = 0
+    staging_root = output / ".staging"
+    staging_root.mkdir(exist_ok=True)
+    try:
+        for source_directory in directories:
+            discovered += 1
+            code = source_directory.name
+            target = output / "documents" / code
+            checksum = None
+            try:
+                recover_directory(target)
+                files, checksum = source_inventory(source_directory, input_root)
+                if target.is_dir() and not refresh and known.get(code) == (
+                    checksum, SCHEMA_VERSION, NORMALIZER_VERSION, "success"
+                ):
+                    skipped += 1
+                else:
+                    with tempfile.TemporaryDirectory(dir=staging_root) as temporary:
+                        staged = Path(temporary) / code
+                        normalize_document(source_directory, input_root, staged, files, checksum)
+                        publish_directory(staged, target)
+                    with connection:
+                        connection.execute(
+                            """
+                            INSERT INTO documents (
+                                code, source_sha256, schema_version,
+                                normalizer_version, processed_at, state, error,
+                                failed_at
+                            ) VALUES (?, ?, ?, ?, ?, 'success', NULL, NULL)
+                            ON CONFLICT(code) DO UPDATE SET
+                                source_sha256=excluded.source_sha256,
+                                schema_version=excluded.schema_version,
+                                normalizer_version=excluded.normalizer_version,
+                                processed_at=excluded.processed_at,
+                                state='success', error=NULL, failed_at=NULL
+                            """,
+                            (code, checksum, SCHEMA_VERSION, NORMALIZER_VERSION, utc_now()),
+                        )
+                    normalized += 1
+            except Exception as error:  # Keep a corpus run alive after one malformed record.
+                LOGGER.exception("Failed to normalize document %s", code)
+                with connection:
+                    connection.execute(
+                        """
+                        INSERT INTO documents (
+                            code, source_sha256, schema_version,
+                            normalizer_version, processed_at, state, error,
+                            failed_at
+                        ) VALUES (?, ?, ?, ?, NULL, 'error', ?, ?)
+                        ON CONFLICT(code) DO UPDATE SET
+                            source_sha256=excluded.source_sha256,
+                            schema_version=excluded.schema_version,
+                            normalizer_version=excluded.normalizer_version,
+                            state='error', error=excluded.error,
+                            failed_at=excluded.failed_at
+                        """,
+                        (
+                            code,
+                            checksum,
+                            SCHEMA_VERSION,
+                            NORMALIZER_VERSION,
+                            str(error),
+                            utc_now(),
+                        ),
+                    )
+                failed += 1
+            result = NormalizeResult(discovered, normalized, skipped, failed)
+            if progress:
+                progress(result, total)
+            elif discovered % 100 == 0:
+                LOGGER.info("discovered=%s normalized=%s skipped=%s failed=%s", discovered, normalized, skipped, failed)
+    finally:
+        connection.close()
+        try:
+            staging_root.rmdir()
+        except OSError:
+            pass
+    return NormalizeResult(discovered, normalized, skipped, failed)
+
+
+def format_duration(seconds: float) -> str:
+    seconds = max(0, round(seconds))
+    hours, seconds = divmod(seconds, 3600)
+    minutes, seconds = divmod(seconds, 60)
+    return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+
+def progress_line(result: NormalizeResult, total: int, elapsed: float, width: int = 24) -> str:
+    fraction = result.discovered / total if total else 0
+    filled = min(width, round(width * fraction))
+    rate = result.discovered / elapsed if elapsed > 0 else 0
+    eta = (total - result.discovered) / rate if rate else 0
+    return (
+        f"[{'#' * filled}{'-' * (width - filled)}] {fraction:6.2%} "
+        f"{result.discovered}/{total} normalized={result.normalized} "
+        f"skipped={result.skipped} failed={result.failed} "
+        f"rate={rate:.2f}/s ETA={format_duration(eta)}"
+    )
+
+
+def terminal_progress() -> Callable[[NormalizeResult, int], None]:
+    started = time.monotonic()
+
+    def update(result: NormalizeResult, total: int) -> None:
+        print(
+            f"\r{progress_line(result, total, time.monotonic() - started)}",
+            end="\n" if result.discovered >= total else "",
+            file=sys.stderr,
+            flush=True,
+        )
+
+    return update
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, default=Path("data/minjust-cbd"))
+    parser.add_argument("--output", type=Path, default=Path("data/minjust-normalized"))
+    parser.add_argument("--limit", type=int, help="normalize only the first N documents")
+    parser.add_argument("--refresh", action="store_true", help="renormalize unchanged documents")
+    parser.add_argument("--log-level", choices=("DEBUG", "INFO", "WARNING", "ERROR"), default="INFO")
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    return parser.parse_args()
+
+
+def main() -> int:
+    arguments = parse_args()
+    if arguments.limit is not None and arguments.limit <= 0:
+        raise SystemExit("--limit must be greater than zero")
+    logging.basicConfig(level=arguments.log_level, format="%(asctime)s %(levelname)s %(message)s")
+    result = normalize_archive(
+        arguments.input,
+        arguments.output,
+        arguments.limit,
+        arguments.refresh,
+        terminal_progress() if sys.stderr.isatty() else None,
+    )
+    print(
+        f"discovered={result.discovered} normalized={result.normalized} "
+        f"skipped={result.skipped} failed={result.failed}\n"
+        f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2"
+    )
+    return int(result.failed > 0)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/search/RELEVANCE_ANNOTATION.md

@@ -0,0 +1,108 @@
+# Инструкция по разметке relevance set v1
+
+## Цель
+
+Набор проверяет, находит ли поиск нужные документы по реальным формулировкам
+пользователей. Он не должен подгоняться под текущую выдачу: сначала фиксируются
+запросы и релевантные документы, затем считается baseline и меняется
+ранжирование.
+
+## Подготовка
+
+Создайте игнорируемую Git рабочую копию:
+
+```bash
+mkdir -p data/search
+cp backend/search/relevance-set-v1.template.json data/search/relevance-set-v1.json
+```
+
+Сохраните идентификаторы `ru-01`–`ru-25` и `ky-01`–`ky-25`. Заполняйте
+`query` и `relevant_document_codes`; остальные поля и структуру JSON не меняйте.
+
+## Выбор запросов
+
+- Используйте 25 русских и 25 кыргызских запросов, реально заданных или
+  сформулированных носителем языка для практической юридической задачи.
+- Не переводите русский набор дословно на кыргызский: оба набора должны
+  отражать естественные формулировки своего языка.
+- Записывайте исходную формулировку без улучшения под поисковик. Допустимы
+  разговорные слова, распространённые сокращения и опечатки.
+- Не используйте персональные данные, закрытые материалы и сведения, которых
+  нет в публичном корпусе Минюста.
+- Не включайте запрос, если нельзя установить хотя бы один релевантный документ.
+- Не повторяйте один информационный запрос в нескольких близких формулировках.
+
+Проверьте разнообразие набора: названия и номера актов, вопросы по жизненной
+или рабочей ситуации, короткие тематические запросы, органы принятия, статусы и
+даты. Это ориентир, а не квота: реальные запросы важнее искусственного баланса.
+
+## Критерий релевантности
+
+Документ релевантен, если его текст или реквизиты непосредственно отвечают
+информационной потребности запроса. Добавляйте все такие документы, а не только
+первый результат.
+
+Не отмечайте документ релевантным только потому, что он:
+
+- содержит отдельные слова запроса;
+- упоминает нужный акт без ответа на запрос;
+- относится к близкой теме;
+- является утратившей силу редакцией, когда запрос явно требует действующую
+  норму, либо наоборот.
+
+Если запрос допускает несколько самостоятельных правильных документов,
+добавьте коды каждого из них. Код берите из поля `document_code`, а не из ID
+фрагмента или редакции.
+
+## Разметка одного запроса
+
+1. Зафиксируйте исходную формулировку `query` и информационную потребность до
+   оценки выдачи нашего поиска.
+2. Юрист устанавливает релевантные акты по содержанию и реквизитам документов.
+   Используйте ЦБД Минюста как авторитетный источник для проверки текста,
+   статуса и редакции акта. Порядок выдачи ЦБД не является объектом оценки и не
+   переносится в эталон.
+3. Запишите уникальные `document_code` всех документов, которые прямо отвечают
+   на запрос. Спорные документы передайте на независимую проверку.
+4. Зафиксируйте согласованный набор до просмотра результатов OpenSearch и
+   сохраните его отдельно от рабочих данных.
+5. Проверьте запрос во внутренней лаборатории (`/review`): оцените фактические
+   результаты OpenSearch в исходном порядке, сохраните снимок и комментарии.
+   Лаборатория проверяет качество нашей поисковой системы, а не ЦБД Минюста.
+
+Пример структуры (код условный):
+
+```json
+{
+  "id": "ru-01",
+  "language": "ru",
+  "query": "как зарегистрировать общественное объединение",
+  "relevant_document_codes": ["12345"]
+}
+```
+
+## Проверка качества
+
+Второй человек должен проверить формулировку, язык и полный список релевантных
+документов для каждого запроса. Спорные случаи обсуждаются до единого решения;
+результат голосования или непроверенную разметку в baseline не включайте.
+
+Перед запуском убедитесь, что:
+
+- заполнены ровно 50 записей: 25 `ru` и 25 `ky`;
+- все запросы непустые и различаются по информационной потребности;
+- у каждой записи есть хотя бы один уникальный `document_code`;
+- язык запроса совпадает с `language`;
+- JSON не содержит комментариев и дополнительных полей.
+
+Оценщик дополнительно проверит структуру файла. После ручной проверки
+зафиксируйте копию набора и не меняйте её при настройке поиска:
+
+```bash
+PYTHONPATH=backend python3 -m search.evaluate_relevance \
+  data/search/relevance-set-v1.json \
+  > data/search/baseline-v1.json
+```
+
+Разбирайте запросы с низкими Recall@10 и MRR@10 по отдельности. Меняйте веса,
+анализаторы или словари только после сохранения исходного baseline.

backend/search/api.py

@@ -0,0 +1,498 @@
+"""Minimal HTTP API for the normalized legal-document corpus."""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import getpass
+import http.cookies
+import json
+import os
+import re
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from search.auth import AuthError, AuthStore
+from search.minjust_opensearch import APP_VERSION, request_json
+from search.catalog import CATALOGS, labels
+from search.query import build_search_query
+from search.reviews import ReviewSnapshots, ReviewStore
+
+
+API_VERSION = "v1"
+SEARCH_ALGORITHM_VERSION = "search-6"
+LANGUAGES = {"ru", "ky"}
+CODE = re.compile(r"^[0-9]+$")
+MAX_PAGE_SIZE = 100
+MAX_RESULT_WINDOW = 10_000
+
+
+class ApiError(Exception):
+    def __init__(self, status: int, message: str):
+        self.status = status
+        self.message = message
+
+
+def parse_positive(value: str | None, name: str, default: int, maximum: int) -> int:
+    if value is None:
+        return default
+    try:
+        parsed = int(value)
+    except ValueError as error:
+        raise ApiError(400, f"{name} must be an integer") from error
+    if not 1 <= parsed <= maximum:
+        raise ApiError(400, f"{name} must be between 1 and {maximum}")
+    return parsed
+
+
+def one(query: dict[str, list[str]], name: str) -> str | None:
+    values = query.get(name, [])
+    if len(values) > 1:
+        raise ApiError(400, f"{name} must be specified once")
+    return values[0] if values else None
+
+
+def date(value: str | None, name: str) -> str | None:
+    if value is None:
+        return None
+    try:
+        datetime.date.fromisoformat(value)
+    except ValueError as error:
+        raise ApiError(400, f"{name} must be an ISO date") from error
+    return value
+
+
+def openapi() -> dict:
+    responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "401": {"description": "Authentication required"}, "403": {"description": "Additional access required"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
+    return {
+        "openapi": "3.0.3",
+        "info": {"title": "Akyldash Search API", "version": API_VERSION},
+        "components": {"securitySchemes": {"SessionCookie": {"type": "apiKey", "in": "cookie", "name": "akyldash_session"}}},
+        "security": [{"SessionCookie": []}],
+        "paths": {
+            "/auth/login": {"post": {"security": [], "responses": {"200": {"description": "Signed in and session cookie issued"}, "401": {"description": "Invalid login or password"}}}},
+            "/auth/logout": {"post": {"responses": {"200": {"description": "Signed out"}}}},
+            "/auth/me": {"get": {"responses": {"200": {"description": "Current account"}, "401": {"description": "Authentication required"}}}},
+            "/auth/change-password": {"post": {"description": "Requires the current password and a different new password", "responses": {"200": {"description": "Password changed"}, "400": {"description": "Current password is missing or invalid, or the new password is invalid or unchanged"}}}},
+            "/admin/users": {"get": {"responses": {"200": {"description": "User list"}}}, "post": {"responses": {"201": {"description": "User created with a temporary password"}}}},
+            "/admin/users/{username}": {"delete": {"responses": {"200": {"description": "User deleted"}}}},
+            "/admin/users/{username}/reset-password": {"post": {"responses": {"200": {"description": "Temporary password set"}}}},
+            "/search": {"get": {"responses": responses, "parameters": [
+                {"name": "q", "in": "query", "required": True, "schema": {"type": "string"}},
+                {"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}},
+                {"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}},
+                {"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": MAX_PAGE_SIZE}},
+                {"name": "document_type", "in": "query", "schema": {"type": "string"}},
+                {"name": "status", "in": "query", "schema": {"type": "string"}},
+                {"name": "authority", "in": "query", "schema": {"type": "string"}},
+                {"name": "legal_force", "in": "query", "schema": {"type": "string"}},
+                {"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}},
+                {"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}},
+                {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}},
+            ]}},
+            "/search/filters": {"get": {"responses": responses}},
+            "/search-reviews": {"post": {"responses": {"201": {"description": "Review saved"}, "400": {"description": "Invalid review"}}}},
+            "/search-reviews/export": {"get": {"responses": responses}},
+            "/review": {"get": {"responses": {"200": {"description": "Review interface"}}}},
+            "/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
+            "/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
+            "/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
+        },
+    }
+
+
+class Api:
+    def __init__(self, base_url: str, index: str, data_root: Path, reviews_db: Path | str = ":memory:", review_secret: bytes | None = None, users_db: Path | str = ":memory:", cookie_secure: bool = False):
+        self.base_url = base_url.rstrip("/")
+        self.index = index
+        self.data_root = data_root
+        self.review_store = ReviewStore(reviews_db)
+        self.review_snapshots = ReviewSnapshots(review_secret)
+        self.auth_store = AuthStore(users_db)
+        self.cookie_secure = cookie_secure
+
+    def search_url(self, suffix: str) -> str:
+        return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}"
+
+    def query_opensearch(self, body: dict) -> dict:
+        try:
+            return request_json(self.search_url("_search"), "POST", json.dumps(body, ensure_ascii=False).encode(), "application/json")
+        except RuntimeError as error:
+            raise ApiError(502, "search backend is unavailable") from error
+
+    def search(self, query: dict[str, list[str]]) -> dict:
+        text = one(query, "q")
+        if not text or not text.strip():
+            raise ApiError(400, "q is required")
+        if len(text) > 500:
+            raise ApiError(400, "q must not exceed 500 characters")
+        language = one(query, "language") or "ru"
+        if language not in LANGUAGES:
+            raise ApiError(400, "language must be ru or ky")
+        page = parse_positive(one(query, "page"), "page", 1, 1_000_000)
+        page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE)
+        if page * page_size >= MAX_RESULT_WINDOW:
+            raise ApiError(400, f"page and page_size must stay within {MAX_RESULT_WINDOW} results")
+        sort = one(query, "sort") or "relevance"
+        if sort not in {"relevance", "date"}:
+            raise ApiError(400, "sort must be relevance or date")
+        filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}]
+        fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes", "legal_force": "legal_force_code"}
+        for parameter, field in fields.items():
+            value = one(query, parameter)
+            if value:
+                if value not in CATALOGS[parameter]:
+                    raise ApiError(400, f"{parameter} must be a catalog code")
+                filters.append({"term": {field: value}})
+        date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
+        if date_from and date_to and date_from > date_to:
+            raise ApiError(400, "date_from must not be later than date_to")
+        if date_from or date_to:
+            date_range = {key: value for key, value in (("gte", date_from), ("lte", date_to)) if value}
+            filters.append({"range": {"date_adopted": date_range}})
+        body = {
+            "from": (page - 1) * page_size,
+            "size": page_size + 1,
+            "_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "legal_force_code", "date_adopted", "number"],
+            "query": {"bool": {"filter": filters, "must": build_search_query(language, text)}},
+            "collapse": {"field": "document_code"},
+            "highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}},
+        }
+        body["sort"] = [{"_script": {
+            "type": "number",
+            "order": "asc",
+            "script": {"lang": "painless", "source": "doc['status_code'].size() == 0 ? 1 : (doc['status_code'].value == 'active' ? 0 : (doc['status_code'].value == 'repealed' ? 2 : 1))"},
+        }}]
+        if sort == "date":
+            body["sort"].append({"date_adopted": "desc"})
+        body["sort"].append({"_score": "desc"})
+        response = self.query_opensearch(body)
+        try:
+            hits = response["hits"]["hits"]
+        except (KeyError, TypeError) as error:
+            raise ApiError(502, "search backend returned an incomplete response") from error
+        results = [self.search_hit(hit, language) for hit in hits[:page_size]]
+        snapshot_results = [{"rank": rank, **result} for rank, result in enumerate(results, 1)]
+        concrete_indexes = {hit.get("_index") for hit in hits[:page_size] if hit.get("_index")}
+        index_name = next(iter(concrete_indexes)) if len(concrete_indexes) == 1 else self.index
+        return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": results, "review_token": self.review_snapshots.create(text, language, index_name, snapshot_results, SEARCH_ALGORITHM_VERSION)}
+
+    @staticmethod
+    def search_hit(hit: dict, language: str) -> dict:
+        source = hit.get("_source")
+        if not isinstance(source, dict) or not source.get("document_code"):
+            raise ApiError(502, "search backend returned an incomplete result")
+        highlight = hit.get("highlight", {}).get(f"text_{language}", [])
+        legal_force = source.get("legal_force_code")
+        snippet = highlight[0].replace("<em>", "").replace("</em>", "") if highlight else None
+        return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "legal_force": labels("legal_force", legal_force)[language] if legal_force else None, "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": snippet}
+
+    def filters(self, query: dict[str, list[str]]) -> dict:
+        language = one(query, "language") or "ru"
+        if language not in LANGUAGES:
+            raise ApiError(400, "language must be ru or ky")
+        fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes"), "legal_forces": ("legal_force", "legal_force_code")}
+        body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
+        response = self.query_opensearch(body)
+        try:
+            aggregations = response["aggregations"]
+            values = {
+                name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
+                for name, pair in fields.items()
+            }
+        except (KeyError, TypeError) as error:
+            raise ApiError(502, "search backend returned incomplete filters") from error
+        return {"api_version": API_VERSION, "language": language, **values}
+
+    def directory(self, code: str) -> Path:
+        if not CODE.fullmatch(code):
+            raise ApiError(404, "document not found")
+        path = self.data_root / "documents" / code
+        if not path.is_dir():
+            raise ApiError(404, "document not found")
+        return path
+
+    @staticmethod
+    def read_json(path: Path, message: str) -> dict:
+        try:
+            value = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError) as error:
+            raise ApiError(500, message) from error
+        if not isinstance(value, dict):
+            raise ApiError(500, message)
+        return value
+
+    def document(self, code: str) -> dict:
+        document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
+        editions = document.get("editions")
+        if not isinstance(editions, list):
+            raise ApiError(500, "document data is unavailable")
+        return {"api_version": API_VERSION, "document": document, "current_edition": editions[-1] if editions else None}
+
+    def editions(self, code: str) -> dict:
+        document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
+        return {"api_version": API_VERSION, "code": code, "available_languages": document.get("available_languages", []), "editions": document.get("editions", [])}
+
+    def edition(self, code: str, edition: str, query: dict[str, list[str]]) -> dict:
+        if not CODE.fullmatch(edition):
+            raise ApiError(404, "edition not found")
+        directory = self.directory(code) / "editions" / edition
+        if not directory.is_dir():
+            raise ApiError(404, "edition not found")
+        metadata = self.read_json(directory / "edition.json", "edition data is unavailable")
+        language = one(query, "language")
+        if language is not None and language not in LANGUAGES:
+            raise ApiError(400, "language must be ru or ky")
+        languages = [language] if language else metadata.get("available_languages", [])
+        content = {}
+        for item in languages:
+            if item not in metadata.get("available_languages", []):
+                continue
+            try:
+                content[item] = {"html": (directory / item / "content.html").read_text(encoding="utf-8"), "text": (directory / item / "content.txt").read_text(encoding="utf-8")}
+            except (OSError, UnicodeError) as error:
+                raise ApiError(500, "edition content is unavailable") from error
+        if language and language not in content:
+            raise ApiError(404, "edition language not found")
+        return {"api_version": API_VERSION, "edition": metadata, "content": content}
+
+    def save_review(self, body: dict, username: str) -> dict:
+        if not isinstance(body, dict):
+            raise ApiError(400, "request body must be an object")
+        try:
+            snapshot = self.review_snapshots.verify(body["review_token"])
+            reviewer = username
+            overall_comment = body.get("overall_comment", "").strip()
+            submitted = body["results"]
+        except (KeyError, AttributeError, TypeError, ValueError) as error:
+            raise ApiError(400, "review_token, reviewer and results are required") from error
+        if not reviewer or len(reviewer) > 120:
+            raise ApiError(400, "reviewer must be between 1 and 120 characters")
+        if len(overall_comment) > 4000:
+            raise ApiError(400, "overall_comment is too long")
+        if not isinstance(submitted, list):
+            raise ApiError(400, "results must be an array")
+        by_rank = {item["rank"]: item for item in snapshot["results"]}
+        if len(submitted) != len(by_rank) or {item.get("rank") for item in submitted if isinstance(item, dict)} != set(by_rank):
+            raise ApiError(400, "all search results must be reviewed exactly once")
+        results = []
+        for item in submitted:
+            if not isinstance(item, dict) or not isinstance(item.get("rank"), int) or item["rank"] not in by_rank:
+                raise ApiError(400, "review result rank is invalid")
+            source = by_rank[item["rank"]]
+            if item.get("code") != source["code"]:
+                raise ApiError(400, "review result document does not match the search snapshot")
+            rating = item.get("rating")
+            if rating is not None and (isinstance(rating, bool) or not isinstance(rating, int) or not 0 <= rating <= 3):
+                raise ApiError(400, "rating must be an integer from 0 to 3")
+            comment = item.get("comment", "")
+            if not isinstance(comment, str) or len(comment) > 4000:
+                raise ApiError(400, "result comment is too long")
+            results.append({**source, "rating": rating, "comment": comment.strip()})
+        if not results:
+            raise ApiError(400, "at least one result must be reviewed")
+        review = {"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "reviewer": reviewer, "query": snapshot["query"], "language": snapshot["language"], "index_name": snapshot["index_name"], "algorithm_version": snapshot["algorithm_version"], "top_result_code": snapshot["results"][0]["code"] if snapshot["results"] else None, "results": results, "overall_comment": overall_comment}
+        review_id = self.review_store.save(review)
+        return {"api_version": API_VERSION, "id": review_id, "created_at": review["created_at"]}
+
+    @staticmethod
+    def review_page() -> str:
+        try:
+            return Path(__file__).with_name("review.html").read_text(encoding="utf-8")
+        except (OSError, UnicodeError) as error:
+            raise ApiError(500, "review interface is unavailable") from error
+
+    def handle(self, method: str, path: str, body: dict | None = None, user: dict | None = None) -> tuple[int, dict]:
+        parsed = urllib.parse.urlsplit(path)
+        query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
+        parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
+        if method == "POST" and parts == ["auth", "login"]:
+            try:
+                token, account = self.auth_store.login(body.get("username"), body.get("password"))
+                return 200, {"account": account, "session_token": token}
+            except AuthError as error:
+                raise ApiError(401, str(error)) from error
+        if method == "POST" and parts == ["auth", "logout"]:
+            return 200, {"ok": True}
+        if method == "GET" and parts == ["auth", "me"]:
+            return 200, {"account": user}
+        if method == "POST" and parts == ["auth", "change-password"]:
+            try:
+                self.auth_store.change_password(user["username"], body.get("current_password"), body.get("new_password"))
+                token, account = self.auth_store.login(user["username"], body.get("new_password"))
+                return 200, {"account": account, "session_token": token}
+            except AuthError as error:
+                raise ApiError(400, str(error)) from error
+        if user and user["username"].casefold() == "admin" and method == "GET" and parts == ["admin", "users"]:
+            return 200, {"users": self.auth_store.list_users()}
+        if user and user["username"].casefold() == "admin" and method == "POST" and parts == ["admin", "users"]:
+            try:
+                self.auth_store.create_user(body.get("username"), body.get("password"))
+                return 201, {"ok": True}
+            except AuthError as error:
+                raise ApiError(400, str(error)) from error
+        if user and user["username"].casefold() == "admin" and method == "DELETE" and len(parts) == 3 and parts[:2] == ["admin", "users"]:
+            try:
+                self.auth_store.delete_user(parts[2])
+                return 200, {"ok": True}
+            except AuthError as error:
+                raise ApiError(400, str(error)) from error
+        if user and user["username"].casefold() == "admin" and method == "POST" and len(parts) == 4 and parts[:2] == ["admin", "users"] and parts[3] == "reset-password":
+            try:
+                self.auth_store.reset_password(parts[2], body.get("password"))
+                return 200, {"ok": True}
+            except AuthError as error:
+                raise ApiError(400, str(error)) from error
+        if method == "POST" and parts == ["search-reviews"]:
+            return 201, self.save_review(body, user["username"] if user else body.get("reviewer", ""))
+        if method == "GET" and parts == ["search-reviews", "export"]:
+            return 200, {"api_version": API_VERSION, "reviews": self.review_store.export()}
+        if method != "GET":
+            raise ApiError(405, "method not allowed")
+        if parts == ["openapi.json"]:
+            return 200, openapi()
+        if parts == ["search"]:
+            return 200, self.search(query)
+        if parts == ["search", "filters"]:
+            return 200, self.filters(query)
+        if len(parts) == 2 and parts[0] == "documents":
+            return 200, self.document(parts[1])
+        if len(parts) == 3 and parts[:1] == ["documents"] and parts[2] == "editions":
+            return 200, self.editions(parts[1])
+        if len(parts) == 4 and parts[:1] == ["documents"] and parts[2] == "editions":
+            return 200, self.edition(parts[1], parts[3], query)
+        raise ApiError(404, "endpoint not found")
+
+
+def handler(api: Api):
+    class RequestHandler(BaseHTTPRequestHandler):
+        def respond(self, method: str):
+            try:
+                route = urllib.parse.urlsplit(self.path).path
+                cookies = http.cookies.SimpleCookie(self.headers.get("Cookie", ""))
+                session = cookies.get("akyldash_session")
+                token = session.value if session else None
+                user = api.auth_store.user_for_token(token)
+                if method == "GET" and route in {"/", "/review"}:
+                    body = api.review_page().encode()
+                    self.send_response(200)
+                    self.send_header("Content-Type", "text/html; charset=utf-8")
+                    self.send_header("Cache-Control", "no-store")
+                    self.send_header("X-Content-Type-Options", "nosniff")
+                    self.send_header("X-Frame-Options", "DENY")
+                    self.send_header("Referrer-Policy", "no-referrer")
+                    self.send_header("Content-Length", str(len(body)))
+                    self.end_headers()
+                    self.wfile.write(body)
+                    return
+                if not (method == "POST" and route in {"/auth/login", "/auth/logout"}):
+                    if not user:
+                        raise ApiError(401, "authentication required")
+                    if user["must_change_password"] and not (method == "POST" and route == "/auth/change-password") and not (method == "GET" and route == "/auth/me"):
+                        raise ApiError(403, "change password before continuing")
+                    if route.startswith("/admin/users") and user["username"].casefold() != "admin":
+                        raise ApiError(403, "admin account required")
+                body = None
+                if method in {"POST", "DELETE"}:
+                    length = int(self.headers.get("Content-Length", "0"))
+                    if length > 1_000_000:
+                        raise ApiError(413, "request body is too large")
+                    body = json.loads(self.rfile.read(length) or b"{}")
+                    if not isinstance(body, dict):
+                        raise ApiError(400, "request body must be a JSON object")
+                if method == "POST" and route == "/auth/logout":
+                    api.auth_store.logout(token)
+                status, payload = api.handle(method, self.path, body, user)
+            except json.JSONDecodeError:
+                status, payload = 400, {"api_version": API_VERSION, "error": "request body must be valid JSON"}
+            except ApiError as error:
+                status, payload = error.status, {"api_version": API_VERSION, "error": error.message}
+            except (ValueError, http.cookies.CookieError):
+                status, payload = 400, {"api_version": API_VERSION, "error": "invalid request"}
+            session_token = payload.pop("session_token", None) if isinstance(payload, dict) else None
+            body = json.dumps(payload, ensure_ascii=False).encode()
+            self.send_response(status)
+            if session_token:
+                secure = "; Secure" if api.cookie_secure else ""
+                self.send_header("Set-Cookie", f"akyldash_session={session_token}; Path=/; HttpOnly; SameSite=Strict; Max-Age=43200{secure}")
+            elif method == "POST" and urllib.parse.urlsplit(self.path).path == "/auth/logout":
+                secure = "; Secure" if api.cookie_secure else ""
+                self.send_header("Set-Cookie", f"akyldash_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0{secure}")
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Cache-Control", "no-store")
+            self.send_header("X-Content-Type-Options", "nosniff")
+            self.send_header("X-Frame-Options", "DENY")
+            self.send_header("Referrer-Policy", "no-referrer")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self):
+            self.respond("GET")
+
+        def do_POST(self):
+            self.respond("POST")
+
+        def do_DELETE(self):
+            self.respond("DELETE")
+
+        def log_message(self, format: str, *args):
+            return
+
+    return RequestHandler
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--url", default="http://127.0.0.1:9200")
+    parser.add_argument("--index", default="akyldash-fragments-current")
+    parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized"))
+    parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
+    parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
+    parser.add_argument("--review-secret", default=None)
+    parser.add_argument("--review-secret-file", type=Path)
+    parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
+    parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
+    parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8080)
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    arguments = parser.parse_args()
+    if arguments.review_secret and arguments.review_secret_file:
+        parser.error("choose either --review-secret or --review-secret-file")
+    try:
+        secret = arguments.review_secret.encode() if arguments.review_secret else arguments.review_secret_file.read_bytes().strip() if arguments.review_secret_file else None
+    except OSError as error:
+        parser.error(f"cannot read review secret file: {error}")
+    if arguments.review_secret_file and not secret:
+        parser.error("review secret file is empty")
+    api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
+    if arguments.create_admin or arguments.reset_admin_password:
+        users = api.auth_store.list_users()
+        admin_exists = any(user["username"].casefold() == "admin" for user in users)
+        if arguments.create_admin and users:
+            parser.error("accounts already exist")
+        if arguments.reset_admin_password and not admin_exists:
+            parser.error("admin account does not exist")
+        if arguments.create_admin and arguments.reset_admin_password:
+            parser.error("choose either --create-admin or --reset-admin-password")
+        password = getpass.getpass("Password for admin (12+ characters): ")
+        confirm = getpass.getpass("Confirm password: ")
+        if password != confirm:
+            parser.error("passwords do not match")
+        try:
+            if arguments.create_admin:
+                api.auth_store.create_admin(password)
+            else:
+                api.auth_store.reset_password("admin", password)
+        except AuthError as error:
+            parser.error(str(error))
+        print("Initial admin account created." if arguments.create_admin else "Admin password reset. Change it at next sign-in.")
+        return 0
+    ThreadingHTTPServer((arguments.host, arguments.port), handler(api)).serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/search/auth.py

@@ -0,0 +1,178 @@
+"""Local user accounts and cookie sessions for the internal search workspace."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import re
+import secrets
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+
+USERNAME = re.compile(r"^[A-Za-z0-9._-]{3,64}$")
+PASSWORD_ITERATIONS = 310_000
+SESSION_TTL = 60 * 60 * 12
+
+
+class AuthError(ValueError):
+    pass
+
+
+class AuthStore:
+    def __init__(self, path: Path | str):
+        if path != ":memory:":
+            database = Path(path)
+            database.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                descriptor = os.open(database, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
+                os.close(descriptor)
+            except FileExistsError:
+                pass
+        self.connection = sqlite3.connect(path, check_same_thread=False)
+        self.connection.row_factory = sqlite3.Row
+        self._lock = threading.RLock()
+        self.connection.executescript("""
+            CREATE TABLE IF NOT EXISTS users (
+                username TEXT PRIMARY KEY COLLATE NOCASE,
+                password_hash BLOB NOT NULL,
+                password_salt BLOB NOT NULL,
+                must_change_password INTEGER NOT NULL DEFAULT 1,
+                created_at INTEGER NOT NULL,
+                first_login_at INTEGER,
+                last_login_at INTEGER,
+                password_reset_at INTEGER
+            );
+            CREATE TABLE IF NOT EXISTS sessions (
+                token_hash BLOB PRIMARY KEY,
+                username TEXT NOT NULL REFERENCES users(username) ON DELETE CASCADE,
+                expires_at INTEGER NOT NULL
+            );
+        """)
+        user_columns = {row["name"] for row in self.connection.execute("PRAGMA table_info(users)")}
+        for column in ("first_login_at", "last_login_at", "password_reset_at"):
+            if column not in user_columns:
+                self.connection.execute(f"ALTER TABLE users ADD COLUMN {column} INTEGER")
+        self.connection.execute("PRAGMA foreign_keys = ON")
+        self.connection.commit()
+
+    @staticmethod
+    def _validate(username: str, password: str) -> tuple[str, str]:
+        if not isinstance(username, str) or not USERNAME.fullmatch(username):
+            raise AuthError("login must be 3–64 letters, numbers, dots, underscores or hyphens")
+        if not isinstance(password, str) or not 12 <= len(password) <= 256:
+            raise AuthError("password must be between 12 and 256 characters")
+        return username, password
+
+    @staticmethod
+    def _hash(password: str, salt: bytes) -> bytes:
+        return hashlib.pbkdf2_hmac("sha256", password.encode(), salt, PASSWORD_ITERATIONS)
+
+    def create_admin(self, password: str) -> None:
+        self._validate("admin", password)
+        salt = secrets.token_bytes(16)
+        with self._lock:
+            if self.connection.execute("SELECT 1 FROM users LIMIT 1").fetchone():
+                raise AuthError("accounts already exist")
+            self.connection.execute("INSERT INTO users(username,password_hash,password_salt,must_change_password,created_at) VALUES('admin',?,?,0,?)", (self._hash(password, salt), salt, int(time.time())))
+            self.connection.commit()
+
+    # shortcut: login throttling belongs to the HTTPS reverse proxy; add local limits before direct public exposure.
+    def login(self, username: str, password: str) -> tuple[str, dict]:
+        if not isinstance(username, str) or not isinstance(password, str):
+            raise AuthError("invalid login or password")
+        with self._lock:
+            user = self.connection.execute("SELECT * FROM users WHERE username = ? COLLATE NOCASE", (username,)).fetchone()
+            salt = user["password_salt"] if user else b"\0" * 16
+            expected_hash = user["password_hash"] if user else b"\0" * 32
+        valid = hmac.compare_digest(self._hash(password, salt), expected_hash)
+        if not user or not valid:
+            raise AuthError("invalid login or password")
+        with self._lock:
+            current = self.connection.execute("SELECT * FROM users WHERE username=?", (user["username"],)).fetchone()
+            if not current or not hmac.compare_digest(current["password_hash"], expected_hash):
+                raise AuthError("invalid login or password")
+            token = secrets.token_urlsafe(32)
+            now = int(time.time())
+            self.connection.execute("DELETE FROM sessions WHERE expires_at <= ?", (now,))
+            self.connection.execute("INSERT INTO sessions(token_hash,username,expires_at) VALUES(?,?,?)", (hashlib.sha256(token.encode()).digest(), user["username"], now + SESSION_TTL))
+            self.connection.execute("UPDATE users SET first_login_at=COALESCE(first_login_at,?),last_login_at=? WHERE username=?", (now, now, user["username"]))
+            self.connection.commit()
+            current = self.connection.execute("SELECT * FROM users WHERE username=?", (user["username"],)).fetchone()
+            return token, self._public(current)
+
+    @staticmethod
+    def _public(user: sqlite3.Row) -> dict:
+        return {"username": user["username"], "role": "Администратор", "must_change_password": bool(user["must_change_password"]), "created_at": user["created_at"]}
+
+    def user_for_token(self, token: str | None) -> dict | None:
+        if not token:
+            return None
+        with self._lock:
+            user = self.connection.execute("SELECT users.* FROM sessions JOIN users USING(username) WHERE token_hash = ? AND expires_at > ?", (hashlib.sha256(token.encode()).digest(), int(time.time()))).fetchone()
+            return self._public(user) if user else None
+
+    def logout(self, token: str | None) -> None:
+        if token:
+            with self._lock:
+                self.connection.execute("DELETE FROM sessions WHERE token_hash = ?", (hashlib.sha256(token.encode()).digest(),))
+                self.connection.commit()
+
+    def change_password(self, username: str, old_password: str, new_password: str) -> None:
+        if not isinstance(old_password, str):
+            raise AuthError("current password is required")
+        self._validate(username, new_password)
+        with self._lock:
+            user = self.connection.execute("SELECT * FROM users WHERE username = ?", (username,)).fetchone()
+            if not user:
+                raise AuthError("account not found")
+            if not hmac.compare_digest(self._hash(old_password, user["password_salt"]), user["password_hash"]):
+                raise AuthError("current password is incorrect")
+            if hmac.compare_digest(self._hash(new_password, user["password_salt"]), user["password_hash"]):
+                raise AuthError("new password must be different")
+            salt = secrets.token_bytes(16)
+            password_hash = self._hash(new_password, salt)
+            self.connection.execute("UPDATE users SET password_hash=?,password_salt=?,must_change_password=0 WHERE username=?", (password_hash, salt, username))
+            self.connection.execute("DELETE FROM sessions WHERE username=?", (username,))
+            self.connection.commit()
+
+    def create_user(self, username: str, password: str) -> None:
+        self._validate(username, password)
+        salt = secrets.token_bytes(16)
+        password_hash = self._hash(password, salt)
+        try:
+            with self._lock:
+                self.connection.execute("INSERT INTO users(username,password_hash,password_salt,must_change_password,created_at) VALUES(?,?,?,1,?)", (username, password_hash, salt, int(time.time())))
+                self.connection.commit()
+        except sqlite3.IntegrityError as error:
+            raise AuthError("login already exists") from error
+
+    def delete_user(self, username: str) -> None:
+        if username.casefold() == "admin":
+            raise AuthError("admin account cannot be deleted")
+        with self._lock:
+            self.connection.execute("DELETE FROM users WHERE username=?", (username,))
+            self.connection.commit()
+
+    def reset_password(self, username: str, password: str) -> None:
+        self._validate(username, password)
+        salt = secrets.token_bytes(16)
+        password_hash = self._hash(password, salt)
+        with self._lock:
+            cursor = self.connection.execute("UPDATE users SET password_hash=?,password_salt=?,must_change_password=1,password_reset_at=? WHERE username=?", (password_hash, salt, int(time.time()), username))
+            if not cursor.rowcount:
+                raise AuthError("account not found")
+            self.connection.execute("DELETE FROM sessions WHERE username=?", (username,))
+            self.connection.commit()
+
+    def list_users(self) -> list[dict]:
+        with self._lock:
+            users = []
+            for row in self.connection.execute("SELECT * FROM users ORDER BY username COLLATE NOCASE"):
+                account = self._public(row)
+                account.update(first_login_at=row["first_login_at"], last_login_at=row["last_login_at"], password_reset_at=row["password_reset_at"])
+                users.append(account)
+            return users

backend/search/catalog.py

@@ -0,0 +1,79 @@
+"""Immutable v1 search catalogs."""
+
+DOCUMENT_TYPES = {
+    "constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "other": ("Прочие документы", "Башка документтер"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"),
+}
+STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "not_yet_effective": ("Не вступил в силу", "Күчүнө кире элек"), "canceled": ("Отменено", "Жокко чыгарылды"), "inactive": ("Не действует", "Күчүндө эмес же Колдонулбайт"), "terminated": ("Прекратило действие", "Күчүн токтотту же Колдонуу токтотулган"), "suspended": ("Действие приостановлено", "Күчүнө кирүүсү токтотулган"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")}
+AUTHORITIES = {"president": ("Президент", "Президент"), "parliament": ("Органы законодательной власти", "Мыйзам чыгаруу бийлик органдары"), "cabinet": ("Правительство и Кабинет Министров", "Өкмөт жана Министрлер Кабинети"), "ministries_and_committees": ("Министерства и государственные комитеты", "Министрликтер жана мамлекеттик комитеттер"), "administrative_agencies": ("Административные ведомства", "Административдик ведомстволор"), "national_bank": ("Национальный банк", "Улуттук банк"), "other_state_bodies": ("Иные государственные органы", "Башка мамлекеттик органдар"), "local_representative_bodies": ("Представительные органы местного самоуправления", "Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары"), "other": ("Прочие органы", "Башка органдар")}
+LEGAL_FORCE_LEVELS = {
+    "constitutional": ("Конституционный уровень", "Конституциялык деңгээл"),
+    "legislative": ("Законодательный уровень", "Мыйзам деңгээли"),
+    "subordinate": ("Подзаконный уровень", "Мыйзам алдындагы деңгээл"),
+}
+LEGAL_FORCE_BY_DOCUMENT_TYPE = {
+    "constitution": "constitutional",
+    "constitutional_law": "constitutional",
+    "code": "legislative",
+    "law": "legislative",
+    "decree": "subordinate",
+    "resolution": "subordinate",
+}
+CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES, "legal_force": LEGAL_FORCE_LEVELS}
+DOCUMENT_TYPE_ALIASES = {
+    ("Положение", "Положение"): "provision",
+    ("Конституционный закон", "Конституционный закон"): "constitutional_law",
+    ("Указ", "Жарлыгы"): "decree",
+    ("устав", None): "charter",
+}
+
+
+def labels(category: str, code: str) -> dict[str, str]:
+    try:
+        ru, ky = CATALOGS[category][code]
+    except KeyError as error:
+        raise ValueError(f"Unknown {category} catalog code: {code}") from error
+    return {"ru": ru, "ky": ky}
+
+
+def source_code(category: str, value: dict | None) -> str:
+    pair = ((value or {}).get("ru"), (value or {}).get("ky"))
+    if pair == (None, None):
+        return "unspecified"
+    if category == "document_type" and pair in DOCUMENT_TYPE_ALIASES:
+        return DOCUMENT_TYPE_ALIASES[pair]
+    for code, expected in CATALOGS[category].items():
+        if pair == expected:
+            return code
+    # ponytail: uncurated source types share one filter; add reviewed codes when separate filtering is needed.
+    if category == "document_type":
+        return "other"
+    raise ValueError(f"Unmapped {category} catalog value: {pair!r}")
+
+
+def authority_codes(paths: list[dict]) -> list[str]:
+    codes = set()
+    for path in paths:
+        text = " ".join(path.get("ru", []) + path.get("ky", [])).lower()
+        if "президент" in text:
+            codes.add("president")
+        elif "жогорку кенеш" in text or "верховный совет" in text or "мыйзам чыгаруу" in text:
+            codes.add("parliament")
+        elif "кабинет министров" in text or "правительство" in text or "өкмөт" in text:
+            codes.add("cabinet")
+        elif "министер" in text or "мамлекеттик комитет" in text:
+            codes.add("ministries_and_committees")
+        elif "административ" in text:
+            codes.add("administrative_agencies")
+        elif "национальн" in text and "банк" in text or "улуттук банк" in text:
+            codes.add("national_bank")
+        elif "кенеш" in text or "кеңеш" in text or "айыл" in text or "местного самоуправления" in text:
+            codes.add("local_representative_bodies")
+        elif "иные государственные" in text or "башка мамлекеттик" in text:
+            codes.add("other_state_bodies")
+        else:
+            codes.add("other")
+    return sorted(codes) or ["other"]
+
+
+def legal_force_code(document_type: dict | None) -> str | None:
+    return LEGAL_FORCE_BY_DOCUMENT_TYPE.get(source_code("document_type", document_type))

backend/search/evaluate_relevance.py

@@ -0,0 +1,90 @@
+"""Measure document search Recall@K and MRR@K against a relevance set."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from search.minjust_opensearch import APP_VERSION
+from search.query import search_documents
+
+
+def load_queries(path: Path) -> list[dict]:
+    with path.open(encoding="utf-8") as source:
+        queries = json.load(source)
+    if not isinstance(queries, list) or not queries:
+        raise ValueError("Relevance set must be a non-empty JSON array")
+
+    seen = set()
+    for item in queries:
+        if not isinstance(item, dict) or set(item) != {
+            "id", "language", "query", "relevant_document_codes"
+        }:
+            raise ValueError("Each query must contain id, language, query and relevant_document_codes")
+        codes = item["relevant_document_codes"]
+        if (
+            not isinstance(item["id"], str)
+            or not item["id"].strip()
+            or item["id"] in seen
+            or not isinstance(item["language"], str)
+            or item["language"] not in {"ru", "ky"}
+            or not isinstance(item["query"], str)
+            or not item["query"].strip()
+            or not isinstance(codes, list)
+            or not codes
+            or any(not isinstance(code, str) or not code for code in codes)
+            or len(codes) != len(set(codes))
+        ):
+            raise ValueError(f"Invalid relevance query: {item.get('id', '<unknown>')}")
+        seen.add(item["id"])
+    return queries
+
+
+def search(base_url: str, index: str, item: dict, top_k: int) -> list[str]:
+    return search_documents(base_url, index, item["language"], item["query"], top_k)
+
+
+def evaluate(queries: list[dict], base_url: str, index: str, top_k: int) -> dict:
+    results = []
+    for item in queries:
+        retrieved = search(base_url, index, item, top_k)
+        relevant = set(item["relevant_document_codes"])
+        matches = [rank for rank, code in enumerate(retrieved, 1) if code in relevant]
+        results.append({
+            "id": item["id"],
+            "language": item["language"],
+            "query": item["query"],
+            "retrieved_document_codes": retrieved,
+            f"recall_at_{top_k}": len(relevant.intersection(retrieved)) / len(relevant),
+            f"reciprocal_rank_at_{top_k}": 1 / matches[0] if matches else 0.0,
+        })
+    return {
+        "summary": {
+            "query_count": len(results),
+            f"recall_at_{top_k}": sum(item[f"recall_at_{top_k}"] for item in results) / len(results),
+            f"mrr_at_{top_k}": sum(item[f"reciprocal_rank_at_{top_k}"] for item in results) / len(results),
+        },
+        "queries": results,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("relevance_set", type=Path)
+    parser.add_argument("--url", default="http://127.0.0.1:9200")
+    parser.add_argument("--index", default="akyldash-fragments-v1")
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    arguments = parser.parse_args()
+    if arguments.top_k <= 0:
+        raise SystemExit("--top-k must be greater than zero")
+    result = evaluate(load_queries(arguments.relevance_set), arguments.url, arguments.index, arguments.top_k)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    print(f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/search/minjust-fragments-index.json

@@ -0,0 +1,40 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 1,
+      "number_of_replicas": 0,
+      "refresh_interval": "30s"
+    }
+  },
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "schema_version": { "type": "keyword" },
+      "document_code": { "type": "keyword" },
+      "edition_code": { "type": "keyword" },
+      "is_current_edition": { "type": "boolean" },
+      "language": { "type": "keyword" },
+      "position": { "type": "integer" },
+      "fragment_type": { "type": "keyword" },
+      "text_ru": { "type": "text", "analyzer": "russian" },
+      "text_ky": { "type": "text", "analyzer": "icu_analyzer" },
+      "document_name_ru": { "type": "text", "analyzer": "russian", "fields": { "keyword": { "type": "keyword", "ignore_above": 1024 } } },
+      "document_name_ky": { "type": "text", "analyzer": "icu_analyzer", "fields": { "keyword": { "type": "keyword", "ignore_above": 1024 } } },
+      "document_type_ru": { "type": "keyword" },
+      "document_type_ky": { "type": "keyword" },
+      "document_type_code": { "type": "keyword" },
+      "legal_force_code": { "type": "keyword" },
+      "status_ru": { "type": "keyword" },
+      "status_ky": { "type": "keyword" },
+      "status_code": { "type": "keyword" },
+      "number": { "type": "keyword" },
+      "date_adopted": { "type": "date", "format": "strict_date" },
+      "authority_paths_ru": { "type": "keyword", "ignore_above": 2048 },
+      "authority_paths_ky": { "type": "keyword", "ignore_above": 2048 },
+      "authority_codes": { "type": "keyword" },
+      "source_path": { "type": "keyword", "index": false },
+      "source_sha256": { "type": "keyword", "index": false },
+      "text_sha256": { "type": "keyword", "index": false }
+    }
+  }
+}

backend/search/minjust_opensearch.py

@@ -0,0 +1,473 @@
+"""Export normalized Ministry fragments for the OpenSearch Bulk API."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sqlite3
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Iterator
+
+from search.catalog import authority_codes, legal_force_code, source_code
+
+APP_VERSION = "0.12.2"
+LANGUAGES = {"ru", "ky"}
+DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
+
+
+def read_json(path: Path):
+    def reject_constant(value: str):
+        raise ValueError(f"Invalid JSON constant: {value}")
+
+    for attempt in range(3):
+        try:
+            with path.open(encoding="utf-8") as source:
+                return json.load(source, parse_constant=reject_constant)
+        except (OSError, UnicodeError, json.JSONDecodeError) as error:
+            if attempt == 2:
+                raise ValueError(f"Cannot read JSON {path}: {error}") from error
+            time.sleep(0.1)
+
+
+def localized(value: object, language: str):
+    return value.get(language) if isinstance(value, dict) else None
+
+
+def paths(document: dict, field: str, language: str) -> list[str]:
+    return [" > ".join(item[language]) for item in document.get(field, []) if item.get(language)]
+
+
+def search_document(document: dict, fragment: dict, expected: tuple[str, str, str, int], current_edition: str) -> dict:
+    document_code, edition_code, language, position = expected
+    fragment_id = f"document:{document_code}:edition:{edition_code}:lang:{language}:fragment:{position}"
+    required = ("id", "document_code", "edition_code", "language", "position", "type", "text", "text_sha256", "source_path", "source_sha256")
+    if not isinstance(fragment, dict) or any(key not in fragment for key in required):
+        raise ValueError(f"Incomplete fragment {fragment_id}")
+    if (fragment["document_code"], fragment["edition_code"], fragment["language"], fragment["position"], fragment["id"]) != (*expected, fragment_id):
+        raise ValueError(f"Fragment identity does not match its path: {fragment_id}")
+    if language not in LANGUAGES or not isinstance(fragment["text"], str) or not fragment["text"]:
+        raise ValueError(f"Invalid fragment content: {fragment_id}")
+    for key in ("text_sha256", "source_sha256"):
+        value = fragment[key]
+        if not isinstance(value, str) or len(value) != 64 or any(char not in "0123456789abcdef" for char in value):
+            raise ValueError(f"Invalid {key}: {fragment_id}")
+    if hashlib.sha256(fragment["text"].encode()).hexdigest() != fragment["text_sha256"]:
+        raise ValueError(f"Text checksum mismatch: {fragment_id}")
+
+    dates = document.get("dates") or {}
+    result = {
+        "schema_version": document["schema_version"],
+        "document_code": document_code,
+        "edition_code": edition_code,
+        "is_current_edition": edition_code == current_edition,
+        "language": language,
+        "position": position,
+        "fragment_type": fragment["type"],
+        f"text_{language}": fragment["text"],
+        "document_name_ru": localized(document.get("name"), "ru"),
+        "document_name_ky": localized(document.get("name"), "ky"),
+        "document_type_ru": localized(document.get("type"), "ru"),
+        "document_type_ky": localized(document.get("type"), "ky"),
+        "document_type_code": source_code("document_type", document.get("type")),
+        "legal_force_code": legal_force_code(document.get("type")),
+        "status_ru": localized(document.get("status"), "ru"),
+        "status_ky": localized(document.get("status"), "ky"),
+        "status_code": source_code("status", document.get("status")),
+        "number": document.get("number"),
+        "date_adopted": dates.get("DateAdopted"),
+        "authority_paths_ru": paths(document, "authority_paths", "ru"),
+        "authority_paths_ky": paths(document, "authority_paths", "ky"),
+        "authority_codes": authority_codes(document.get("authority_paths", [])),
+        "source_path": fragment["source_path"],
+        "source_sha256": fragment["source_sha256"],
+        "text_sha256": fragment["text_sha256"],
+    }
+    return {key: value for key, value in result.items() if value is not None}
+
+
+def document_codes(input_root: Path, limit: int | None = None, start_at: str | None = None) -> list[str]:
+    connection = sqlite3.connect(f"file:{input_root / 'manifest.sqlite3'}?mode=ro", uri=True)
+    try:
+        query = "SELECT code FROM documents WHERE state = 'success' ORDER BY CAST(code AS INTEGER), code"
+        parameters: list[int] = []
+        if limit is not None:
+            query += " LIMIT ?"
+            parameters.append(limit)
+        codes = [str(code) for (code,) in connection.execute(query, parameters)]
+    finally:
+        connection.close()
+    if start_at is None:
+        return codes
+    try:
+        return codes[codes.index(start_at):]
+    except ValueError as error:
+        raise ValueError(f"Resume document not found in selected range: {start_at}") from error
+
+
+def bulk_pairs(
+    input_root: Path,
+    index: str,
+    limit: int | None = None,
+    start_at: str | None = None,
+) -> Iterator[tuple[str, bytes]]:
+    document_root = input_root / "documents"
+    if not document_root.is_dir():
+        raise FileNotFoundError(f"Document directory not found: {document_root}")
+    for code in document_codes(input_root, limit, start_at):
+        directory = document_root / code
+        document = read_json(directory / "document.json")
+        if document.get("source_code") != directory.name:
+            raise ValueError(f"Document identity does not match its path: {directory}")
+        edition_root = directory / "editions"
+        editions = [path.name for path in edition_root.iterdir() if path.is_dir()] if edition_root.is_dir() else []
+        if not editions:
+            continue
+        official_editions = document.get("editions") or []
+        current_edition = (
+            str(official_editions[-1]["source_code"])
+            if official_editions
+            else max(editions, key=lambda value: (not value.isdigit(), int(value) if value.isdigit() else value))
+        )
+        for fragment_path in sorted(directory.glob("editions/*/*/fragments.json")):
+            edition_code, language = fragment_path.parts[-3:-1]
+            values = read_json(fragment_path)
+            if not isinstance(values, list):
+                raise ValueError(f"Fragments must be a list: {fragment_path}")
+            for position, fragment in enumerate(values, 1):
+                source = search_document(document, fragment, (directory.name, edition_code, language, position), current_edition)
+                action = json.dumps({"index": {"_index": index, "_id": fragment["id"]}}, ensure_ascii=False, allow_nan=False)
+                body = json.dumps(source, ensure_ascii=False, allow_nan=False)
+                yield directory.name, f"{action}\n{body}\n".encode()
+
+
+def document_count(input_root: Path, limit: int | None, start_at: str | None = None) -> int:
+    return len(document_codes(input_root, limit, start_at))
+
+
+def bulk_batches(pairs: Iterator[tuple[str, bytes]], maximum_bytes: int) -> Iterator[tuple[str, bytes]]:
+    batch = bytearray()
+    last_code = ""
+    for code, pair in pairs:
+        if len(pair) > maximum_bytes:
+            raise ValueError(f"One Bulk pair exceeds the {maximum_bytes}-byte batch limit")
+        if batch and len(batch) + len(pair) > maximum_bytes:
+            yield last_code, bytes(batch)
+            batch.clear()
+        last_code = code
+        batch.extend(pair)
+    if batch:
+        yield last_code, bytes(batch)
+
+
+def export_bulk(input_root: Path, output: Path, index: str, limit: int | None = None) -> tuple[int, int]:
+    source = input_root.resolve()
+    destination = output.resolve()
+    if destination == source or destination.is_relative_to(source):
+        raise ValueError("--output must not be inside --input")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fragments = 0
+    with tempfile.NamedTemporaryFile("wb", dir=output.parent, delete=False) as target:
+        temporary = Path(target.name)
+        try:
+            for code, pair in bulk_pairs(input_root, index, limit):
+                target.write(pair)
+                fragments += 1
+            target.flush()
+            os.fsync(target.fileno())
+            os.replace(temporary, output)
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+    return document_count(input_root, limit), fragments
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_json_atomic(path: Path, value: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile("wb", dir=path.parent, delete=False) as target:
+        temporary = Path(target.name)
+        try:
+            target.write((json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode())
+            target.flush()
+            os.fsync(target.fileno())
+            os.replace(temporary, path)
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+
+
+def request_json(
+    url: str,
+    method: str,
+    body: bytes | None,
+    content_type: str,
+    attempts: int = 5,
+    retry_invalid_json: bool = False,
+) -> dict:
+    request = urllib.request.Request(url, data=body, method=method, headers={"Content-Type": content_type})
+    for attempt in range(attempts):
+        try:
+            with urllib.request.urlopen(request, timeout=120) as response:
+                return json.load(response)
+        except (UnicodeError, json.JSONDecodeError) as error:
+            if not retry_invalid_json or attempt == attempts - 1:
+                raise RuntimeError(f"{method} {url} returned invalid JSON: {error}") from error
+            delay = 2**attempt
+        except urllib.error.HTTPError as error:
+            response_body = error.read(500).decode("utf-8", errors="replace")
+            error.close()
+            retryable = error.code == 429 or 500 <= error.code < 600
+            if not retryable or attempt == attempts - 1:
+                raise RuntimeError(
+                    f"{method} {url} failed with HTTP {error.code}: {response_body}"
+                ) from error
+            retry_after = error.headers.get("Retry-After")
+            delay = float(retry_after) if retry_after and retry_after.isdigit() else 2**attempt
+        except (urllib.error.URLError, TimeoutError, ConnectionResetError) as error:
+            if attempt == attempts - 1:
+                raise RuntimeError(f"{method} {url} failed after {attempts} attempts: {error}") from error
+            delay = 2**attempt
+        time.sleep(delay)
+    raise AssertionError("unreachable")
+
+
+def opensearch_identity(base: str, index: str, index_url: str) -> tuple[str, str]:
+    cluster = request_json(f"{base}/", "GET", None, "application/json")
+    definition = request_json(index_url, "GET", None, "application/json")
+    try:
+        return cluster["cluster_uuid"], definition[index]["settings"]["index"]["uuid"]
+    except (KeyError, TypeError) as error:
+        raise RuntimeError("OpenSearch identity response is incomplete") from error
+
+
+def checkpoint_state(
+    input_root: Path,
+    index: str,
+    checkpoint: Path,
+    resume: bool,
+    url: str,
+    cluster_uuid: str,
+    index_uuid: str,
+    limit: int | None,
+    alias: str | None,
+) -> tuple[dict, str | None]:
+    source = input_root.resolve()
+    destination = checkpoint.resolve()
+    if destination == source or destination.is_relative_to(source):
+        raise ValueError("--checkpoint must not be inside --input")
+    manifest_sha256 = file_sha256(input_root / "manifest.sqlite3")
+    if not resume:
+        return {
+            "schema_version": 2,
+            "url": url,
+            "cluster_uuid": cluster_uuid,
+            "index": index,
+            "index_uuid": index_uuid,
+            "input": str(source),
+            "manifest_sha256": manifest_sha256,
+            "limit": limit,
+            "alias": alias,
+            "last_document_code": None,
+            "complete": False,
+        }, None
+
+    state = read_json(checkpoint)
+    expected = {
+        "schema_version",
+        "url",
+        "cluster_uuid",
+        "index",
+        "index_uuid",
+        "input",
+        "manifest_sha256",
+        "limit",
+        "alias",
+        "last_document_code",
+        "complete",
+    }
+    legacy_expected = expected - {"alias"}
+    if not isinstance(state, dict):
+        raise ValueError(f"Invalid checkpoint: {checkpoint}")
+    if set(state) == legacy_expected:
+        if state["schema_version"] != 1 or alias is not None:
+            raise ValueError(f"Legacy checkpoint does not support --alias: {checkpoint}")
+        state["schema_version"] = 2
+        state["alias"] = None
+    elif set(state) != expected:
+        raise ValueError(f"Invalid checkpoint: {checkpoint}")
+    if (
+        state["schema_version"] != 2
+        or state["url"] != url
+        or state["cluster_uuid"] != cluster_uuid
+        or state["index"] != index
+        or state["index_uuid"] != index_uuid
+        or state["input"] != str(source)
+    ):
+        raise ValueError(f"Checkpoint does not match this load: {checkpoint}")
+    if state["limit"] != limit:
+        raise ValueError(f"Checkpoint limit does not match --limit: {checkpoint}")
+    if state["alias"] != alias:
+        raise ValueError(f"Checkpoint alias does not match --alias: {checkpoint}")
+    if state["manifest_sha256"] != manifest_sha256:
+        raise ValueError("Normalized manifest changed; create a new versioned index")
+    if state["complete"] is not False:
+        raise ValueError(f"Checkpoint is already complete: {checkpoint}")
+    start_at = state["last_document_code"]
+    if start_at is not None and not isinstance(start_at, str):
+        raise ValueError(f"Invalid checkpoint document code: {checkpoint}")
+    return state, start_at
+
+
+def load_bulk(
+    input_root: Path,
+    url: str,
+    index: str,
+    mapping: Path = DEFAULT_MAPPING,
+    maximum_bytes: int = 25 * 1024 * 1024,
+    limit: int | None = None,
+    resume: bool = False,
+    checkpoint: Path = Path("data/opensearch/minjust-fragments.checkpoint.json"),
+    alias: str | None = None,
+) -> tuple[int, int]:
+    base = url.rstrip("/")
+    if alias is not None and (not alias or alias == index):
+        raise ValueError("--alias must differ from --index")
+    index_url = f"{base}/{urllib.parse.quote(index, safe='')}"
+    if resume:
+        cluster_uuid, index_uuid = opensearch_identity(base, index, index_url)
+    else:
+        request_json(index_url, "PUT", mapping.read_bytes(), "application/json")
+        cluster_uuid, index_uuid = opensearch_identity(base, index, index_url)
+    state, start_at = checkpoint_state(
+        input_root,
+        index,
+        checkpoint,
+        resume,
+        base,
+        cluster_uuid,
+        index_uuid,
+        limit,
+        alias,
+    )
+    documents = document_count(input_root, limit, start_at)
+    if not resume:
+        write_json_atomic(checkpoint, state)
+    fragments = 0
+    for last_code, batch in bulk_batches(bulk_pairs(input_root, index, limit, start_at), maximum_bytes):
+        result = request_json(
+            f"{base}/_bulk",
+            "POST",
+            batch,
+            "application/x-ndjson",
+            retry_invalid_json=True,
+        )
+        expected = batch.count(b"\n") // 2
+        items = result.get("items", [])
+        if result.get("errors"):
+            failures = [item.get("index", {}) for item in items if item.get("index", {}).get("error")]
+            details = "; ".join(
+                f"{item.get('_id', '<unknown>')}: {item['error'].get('type', 'error')}: "
+                f"{item['error'].get('reason', '<no reason>')}"
+                for item in failures[:5]
+            )
+            raise RuntimeError(
+                f"OpenSearch Bulk API failed for {len(failures)} item(s); "
+                f"resume from {checkpoint}: {details}"
+            )
+        if len(items) != expected:
+            raise RuntimeError(
+                f"OpenSearch Bulk API returned {len(items)} of {expected} item result(s); "
+                f"resume from {checkpoint}"
+            )
+        fragments += len(items)
+        state["last_document_code"] = last_code
+        write_json_atomic(checkpoint, state)
+        print(f"checkpoint={last_code} fragments={fragments}", flush=True)
+    if alias:
+        switch_alias(base, index, alias)
+    state["complete"] = True
+    write_json_atomic(checkpoint, state)
+    return documents, fragments
+
+
+def switch_alias(base: str, index: str, alias: str) -> None:
+    if not alias or alias == index:
+        raise ValueError("--alias must differ from --index")
+    result = request_json(
+        f"{base.rstrip('/')}/_aliases",
+        "POST",
+        json.dumps({
+            "actions": [
+                {"remove": {"index": "*", "alias": alias, "must_exist": False}},
+                {"add": {"index": index, "alias": alias}},
+            ]
+        }).encode(),
+        "application/json",
+    )
+    if result.get("acknowledged") is not True:
+        raise RuntimeError(f"OpenSearch did not acknowledge alias switch: {alias}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, default=Path("data/minjust-normalized"))
+    parser.add_argument("--output", type=Path, default=Path("data/opensearch/minjust-fragments.ndjson"))
+    parser.add_argument("--index", default="akyldash-fragments-v1")
+    parser.add_argument("--limit", type=int)
+    parser.add_argument("--url", help="create the index and stream bounded Bulk requests instead of writing a file")
+    parser.add_argument("--alias", help="atomically point this alias at --index after a successful load")
+    parser.add_argument("--mapping", type=Path, default=DEFAULT_MAPPING)
+    parser.add_argument("--batch-mb", type=int, default=25)
+    parser.add_argument("--resume", action="store_true", help="load into an existing index")
+    parser.add_argument("--checkpoint", type=Path, help="persistent resume checkpoint path")
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    arguments = parser.parse_args()
+    if arguments.limit is not None and arguments.limit <= 0:
+        raise SystemExit("--limit must be greater than zero")
+    if arguments.batch_mb <= 0:
+        raise SystemExit("--batch-mb must be greater than zero")
+    if arguments.resume and not arguments.url:
+        raise SystemExit("--resume requires --url")
+    if arguments.checkpoint and not arguments.url:
+        raise SystemExit("--checkpoint requires --url")
+    if arguments.alias == "":
+        raise SystemExit("--alias must not be empty")
+    if arguments.alias is not None and not arguments.url:
+        raise SystemExit("--alias requires --url")
+    if arguments.url:
+        checkpoint = arguments.checkpoint or Path("data/opensearch") / f"{arguments.index}.checkpoint.json"
+        documents, fragments = load_bulk(
+            arguments.input,
+            arguments.url,
+            arguments.index,
+            arguments.mapping,
+            arguments.batch_mb * 1024 * 1024,
+            arguments.limit,
+            arguments.resume,
+            checkpoint,
+            arguments.alias,
+        )
+        destination = arguments.url
+    else:
+        documents, fragments = export_bulk(arguments.input, arguments.output, arguments.index, arguments.limit)
+        destination = arguments.output
+    print(f"documents={documents} fragments={fragments} destination={destination}\nAkyldash · Backend v{APP_VERSION} · Frontend v0.1.2")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/search/query.py

@@ -0,0 +1,216 @@
+"""Run document searches against the local OpenSearch index."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import urllib.parse
+
+from search.minjust_opensearch import APP_VERSION, request_json
+
+
+LEGAL_FORM_ALIASES = {
+    "ru": {
+        "осоо": ("ооо", "общество с ограниченной ответственностью"),
+        "ооо": ("осоо", "общество с ограниченной ответственностью"),
+        "общество с ограниченной ответственностью": ("осоо", "ооо"),
+        "общества с ограниченной ответственностью": ("осоо", "ооо"),
+    },
+    "ky": {
+        "жчк": ("жоопкерчилиги чектелген коом",),
+        "жоопкерчилиги чектелген коом": ("жчк",),
+    },
+}
+COMPANY_REGISTRATION_DOCUMENTS = (("230044970", 3000), ("667", 2000), ("4", 1000))
+TAX_PROFIT_DOCUMENTS = (("112340", 3000),)
+BOILERPLATE_PREFIXES = {
+    "ru": (
+        "как регулируется", "правовое регулирование", "судебная практика", "сроки и ответственность",
+        "действующая редакция", "последние изменения", "основные положения", "комментарий к акту",
+    ),
+    "ky": (),
+}
+ACT_TITLE_PREFIXES = {
+    "ru": (
+        r"(?:гражданский\s+)?процессуальный\s+кодекс(?:\s+(?:кыргызской\s+республики|кр))?",
+        r"(?:гражданск\w*|уголовн\w*|налогов\w*|трудов\w*|семейн\w*)\s+кодекс\w*(?:\s+(?:кыргызской\s+республики|кр))?",
+        r"кодекс\s+о\s+правонарушениях(?:\s+(?:кыргызской\s+республики|кр))?",
+        r"конституци\w*(?:\s+(?:кыргызской\s+республики|кр))?",
+        r"(?:гк|ук|нк|коап|гпк|упк|уик)(?:\s+(?:кыргызской\s+республики|кр))?",
+        r"закон(?:\s+(?:кыргызской\s+республики|кр))?\s+о\s+(?:защит\w*\s+прав\w*\s+потребител\w*|государственн\w+\s+регистраци\w+\s+юридическ\w+\s+лиц|труд\w*)",
+    ),
+    "ky": (
+        r"кыргыз\w*\s+республикасынын\s+конституци\w*",
+        r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жарандык|кылмыш\w*|салык|эмгек|үй-бүлө|административдик|жарандык\s+процесстик)\s+кодекс\w*",
+        r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жк|үк)(?:\s+кр)?",
+    ),
+}
+
+
+def relaxed_query_text(language: str, query: str) -> str:
+    value = query.strip()
+    for prefix in BOILERPLATE_PREFIXES[language]:
+        if value.casefold().startswith(prefix):
+            value = value[len(prefix):].strip()
+            break
+    for pattern in ACT_TITLE_PREFIXES[language]:
+        match = re.match(rf"\s*{pattern}(?=\s|$|[.,;:!?])", value, re.IGNORECASE)
+        if match:
+            value = value[match.end():].lstrip(" \t,;:—-")
+            break
+    return value
+
+
+def query_variants(language: str, query: str) -> list[str]:
+    variants = [query]
+    for alias, expansions in LEGAL_FORM_ALIASES[language].items():
+        pattern = re.compile(rf"(?<!\w){re.escape(alias)}(?!\w)", re.IGNORECASE)
+        if pattern.search(query):
+            variants.extend(pattern.sub(expansion, query) for expansion in expansions)
+    return list(dict.fromkeys(variants))
+
+
+def document_priority_clauses(language: str, documents: tuple[tuple[str, int], ...]) -> list[dict]:
+    status = {"ru": "Действует", "ky": "Күчүндө"}[language]
+    return [
+        {
+            "constant_score": {
+                "filter": {
+                    "bool": {
+                        "filter": [
+                            {"term": {"document_code": code}},
+                            {"term": {f"status_{language}": status}},
+                        ]
+                    }
+                },
+                "boost": boost,
+            }
+        }
+        for code, boost in documents
+    ]
+
+
+def company_registration_clauses(language: str, query: str) -> list[dict]:
+    non_registration_context = {
+        "ru": (r"\bсч[её]т\w*", r"\bбанк\w*", r"\bфилиал\w*", r"\bпредставительств\w*"),
+        "ky": (r"\bбанк\w*", r"\bэсеп\w*", r"\bфилиал\w*", r"\bөкүлчүлүк\w*"),
+    }[language]
+    normalized_query = query.casefold()
+    if any(re.search(pattern, normalized_query) for pattern in non_registration_context):
+        return []
+
+    forms, patterns = {
+        "ru": (
+            r"(?:осоо|ооо|обществ\w*\s+с\s+ограниченной\s+ответственностью)",
+            (
+                r"\bкак\s+(?:откры\w*|созда\w*|зарегистр\w*)\s+{form}\b",
+                r"\b(?:порядок|процедура)\s+(?:открыти\w*|создани\w*|регистраци\w*)\s+{form}\b",
+                r"\b(?:государственн\w+\s+)?регистраци\w*\s+{form}\b",
+                r"\bсоздани\w*\s+{form}\b",
+            ),
+        ),
+        "ky": (
+            r"(?:жчк|жоопкерчилиги\s+чектелген\s+коом)",
+            (
+                r"\bжчк\s+ач\w*\s+тартиби\b",
+                r"\bжчк\s+кантип\s+ач\w*\b",
+                r"\b{form}\s+(?:ачуу|ачуунун|каттоо|каттоонун)\b",
+                r"\b(?:ачуу|каттоо)\s+{form}\b",
+            ),
+        ),
+    }[language]
+    if not any(re.search(pattern.format(form=forms), normalized_query) for pattern in patterns):
+        return []
+
+    return document_priority_clauses(language, COMPANY_REGISTRATION_DOCUMENTS)
+
+
+def tax_profit_clauses(language: str, query: str) -> list[dict]:
+    if language != "ru" or not re.search(r"\bналог\w*\s+(?:на\s+)?прибыл\w*\b", query.casefold()):
+        return []
+    # ponytail: exact topic anchor; expand only with verified tax-code sources.
+    return document_priority_clauses(language, TAX_PROFIT_DOCUMENTS)
+
+
+def build_search_query(language: str, query: str) -> dict:
+    registration_clauses = company_registration_clauses(language, query)
+    intent_clauses = [*registration_clauses, *tax_profit_clauses(language, query)]
+    variants = query_variants(language, query)
+    full_text = [
+        {
+            "multi_match": {
+                "query": variant,
+                "fields": [f"document_name_{language}^4", f"text_{language}"],
+                "type": "cross_fields",
+                "operator": "OR" if registration_clauses else "AND",
+            }
+        }
+        for variant in variants
+    ]
+    relaxed_text = []
+    if not registration_clauses:
+        seen_relaxed = set()
+        for variant in variants:
+            value = relaxed_query_text(language, variant)
+            words = value.split()
+            if value in seen_relaxed or len(words) < 2 or (value == variant.strip() and len(words) < 3):
+                continue
+            if len(words) < 3 and not any(len(word) >= 5 for word in words):
+                continue
+            seen_relaxed.add(value)
+            relaxed_text.append({
+                "multi_match": {
+                    "query": value,
+                    "fields": [f"document_name_{language}^4", f"text_{language}"],
+                    "type": "cross_fields",
+                    "operator": "OR",
+                    "minimum_should_match": 2,
+                    "boost": 0.1,
+                }
+            })
+    phrase_matches = [
+        {"match_phrase": {f"text_{language}": {"query": variant, "boost": 3}}}
+        for variant in query_variants(language, query)
+    ]
+    planned = {"should": [*full_text, *phrase_matches, *relaxed_text, *intent_clauses], "minimum_should_match": 1}
+    return {"bool": planned}
+
+
+def build_search_body(language: str, query: str, top_k: int) -> bytes:
+    return json.dumps({
+        "size": top_k,
+        "track_total_hits": False,
+        "_source": ["document_code"],
+        "query": {"bool": {"filter": [{"term": {"language": language}}, {"term": {"is_current_edition": True}}], "must": build_search_query(language, query)}},
+        "collapse": {"field": "document_code"},
+    }, ensure_ascii=False).encode()
+
+
+def search_documents(base_url: str, index: str, language: str, query: str, top_k: int) -> list[str]:
+    url = f"{base_url.rstrip('/')}/{urllib.parse.quote(index, safe='')}/_search"
+    response = request_json(url, "POST", build_search_body(language, query, top_k), "application/json")
+    try:
+        return [hit["_source"]["document_code"] for hit in response["hits"]["hits"]]
+    except (KeyError, TypeError) as error:
+        raise RuntimeError("OpenSearch search response is incomplete") from error
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("query")
+    parser.add_argument("--language", choices=("ru", "ky"), required=True)
+    parser.add_argument("--url", default="http://127.0.0.1:9200")
+    parser.add_argument("--index", default="akyldash-fragments-v1")
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--version", action="version", version=APP_VERSION)
+    arguments = parser.parse_args()
+    if arguments.top_k <= 0:
+        raise SystemExit("--top-k must be greater than zero")
+    print(json.dumps(search_documents(arguments.url, arguments.index, arguments.language, arguments.query, arguments.top_k), ensure_ascii=False))
+    print(f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/search/relevance-set-v1.template.json

@@ -0,0 +1,52 @@
+[
+  {"id": "ru-01", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-02", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-03", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-04", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-05", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-06", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-07", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-08", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-09", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-10", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-11", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-12", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-13", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-14", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-15", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-16", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-17", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-18", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-19", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-20", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-21", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-22", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-23", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-24", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ru-25", "language": "ru", "query": "", "relevant_document_codes": []},
+  {"id": "ky-01", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-02", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-03", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-04", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-05", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-06", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-07", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-08", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-09", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-10", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-11", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-12", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-13", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-14", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-15", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-16", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-17", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-18", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-19", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-20", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-21", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-22", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-23", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-24", "language": "ky", "query": "", "relevant_document_codes": []},
+  {"id": "ky-25", "language": "ky", "query": "", "relevant_document_codes": []}
+]

backend/search/review.html

@@ -0,0 +1,335 @@
+<!doctype html>
+<html lang="ru">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Оценка поисковой выдачи · Акылдаш</title>
+  <style>
+    :root { color-scheme: light; font: 16px/1.5 "Noto Sans", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", serif; }
+    * { box-sizing: border-box; }
+    body { margin: 0; color: var(--ink); background: #F7F2E8; }
+    .review__skip { position: absolute; inset-block-start: 0; inset-inline-start: -10000px; padding: .5rem 1rem; background: var(--surface); }
+    .review__skip:focus { inset-inline-start: 1rem; z-index: 2; }
+    .review__header, .review__main { max-width: 1320px; margin: auto; padding-inline: clamp(1rem, 3vw, 3rem); }
+    .review__header { padding-block: 0 1.5rem; background: var(--surface); border-block-end: 1px solid var(--line); }
+    .review__header h1 { max-width: 24ch; margin: 1.5rem 0 .4rem; font: 700 clamp(1.75rem, 3vw, 2rem)/1.2 var(--serif); }
+    .review__header p { max-width: 65ch; margin: 0; color: var(--muted); }
+    .review__header-top { display: flex; justify-content: space-between; gap: 1rem; align-items: start; }
+    .review__eyebrow { display: inline-flex; align-items: center; gap: .5rem; color: var(--ink); font: 600 1.1rem/1.25 var(--serif); }
+    .review__eyebrow::before { content: ''; width: 1.5rem; height: 1.5rem; border: 2px solid var(--brand); border-radius: 50%; background: linear-gradient(90deg, transparent 45%, var(--brand) 45% 55%, transparent 55%), linear-gradient(0deg, transparent 45%, var(--brand) 45% 55%, transparent 55%); }
+    .review__main { padding-block-end: 2rem; }
+    .review__search { display: grid; grid-template-columns: minmax(18rem, 1fr) 9rem 9rem auto; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
+    .review__field { display: grid; gap: .35rem; min-width: 0; }
+    .review__field span, .review__actions label { color: var(--muted); font-size: .875rem; font-weight: 700; }
+    .review__field--small { min-width: 0; }
+    input, select, textarea, button { font: inherit; }
+    input, select, textarea { width: 100%; min-height: 44px; border: 1px solid var(--line); border-radius: 6px; padding: .7rem .8rem; background: var(--surface); color: var(--ink); }
+    input::placeholder, textarea::placeholder { color: var(--muted); }
+    input:focus-visible, select:focus-visible, textarea:focus-visible, button:focus-visible { outline: 3px solid var(--focus); outline-offset: 2px; }
+    button { min-height: 44px; cursor: pointer; border: 1px solid var(--line); border-radius: 6px; padding: .55rem .9rem; background: var(--surface); color: var(--ink); font-weight: 600; transition-property: background-color, border-color, color, scale; transition-duration: 150ms; transition-timing-function: cubic-bezier(.2, 0, 0, 1); }
+    button:active { scale: .96; }
+    button:disabled { cursor: wait; opacity: .55; }
+    .review__button--primary { background: var(--brand); color: #fff; border-color: var(--brand); }
+    .review__status { min-height: 1.7rem; margin-block: 1rem; color: var(--muted); }
+    .review__status--error { color: #9D2633; }
+    .review__workspace { display: grid; grid-template-columns: minmax(24rem, .9fr) minmax(28rem, 1.1fr); gap: 1rem; align-items: start; }
+    .review__results, .review__document { min-width: 0; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; padding: clamp(1rem, 2vw, 1.5rem); }
+    .review__panel-heading { display: flex; justify-content: space-between; gap: 1rem; align-items: baseline; margin-block-end: 1rem; }
+    .review__panel-heading h2 { margin: 0; font: 600 1.5rem/1.3 var(--serif); }
+    .review__panel-hint { color: var(--muted); font-size: .875rem; }
+    .review__results-list { display: grid; gap: .75rem; margin: 0; padding: 0; list-style: none; }
+    .review__result { padding: 1rem 0; border: 0; border-block-end: 1px solid var(--line); border-radius: 0; background: var(--surface); transition-property: background-color, border-color; transition-duration: 150ms; transition-timing-function: cubic-bezier(.2, 0, 0, 1); }
+    .review__result--selected { background: var(--subtle); box-shadow: none; }
+    .review__result:first-child { border-block-start: 1px solid var(--line); }
+    .review__result-title { display: flex; gap: .65rem; align-items: baseline; width: 100%; min-height: 44px; text-align: start; font-weight: 700; border: 0; padding: 0; background: none; color: var(--brand-dark); }
+    .review__rank { flex: 0 0 auto; color: var(--brand); font-variant-numeric: tabular-nums; }
+    .review__meta { margin-block: .5rem; color: var(--muted); font-size: .875rem; }
+    .review__rating { display: grid; grid-template-columns: repeat(4, 1fr); gap: .4rem; margin-block: 1rem .75rem; padding: 0; border: 0; }
+    .review__rating legend { width: 100%; margin-block-end: .35rem; font-size: .875rem; font-weight: 700; color: var(--muted); grid-column: 1 / -1; }
+    .review__rating label { display: grid; place-items: center; min-width: 0; min-height: 44px; padding: .45rem .25rem; border: 1px solid var(--line); border-radius: 6px; color: var(--muted); text-align: center; font-size: .875rem; cursor: pointer; }
+    .review__rating label:has(input:checked) { border-color: var(--brand); background: var(--brand); color: #fff; }
+    .review__rating label:has(input:focus-visible) { outline: 3px solid var(--focus); outline-offset: 2px; }
+    .review__rating input { position: absolute; opacity: 0; inline-size: 1px; min-height: 1px; block-size: 1px; }
+    .review__comment { min-height: 4rem; resize: vertical; background: var(--surface); }
+    .review__document { position: sticky; inset-block-start: 1rem; min-height: 24rem; }
+    .review__document-meta { margin-block-end: 1rem; padding-block-end: 1rem; border-block-end: 1px solid var(--line); color: var(--muted); }
+    .review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
+    .review__document-body img { max-width: 100%; height: auto; }
+    .review__actions { max-width: 1320px; margin: 1.5rem auto 0; padding-inline: clamp(1rem, 3vw, 3rem); }
+    .review__actions-inner { display: flex; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
+    .review__actions label { display: grid; gap: .35rem; flex: 1; color: var(--muted); font-size: .875rem; font-weight: 700; }
+    .review__actions label:last-of-type { flex: 2; }
+    dialog { max-width: min(38rem, calc(100% - 2rem)); max-height: calc(100% - 2rem); border: 1px solid var(--line); border-radius: 6px; padding: 1.5rem; box-shadow: 0 1rem 2.5rem rgb(10 20 30 / .18); }
+    dialog::backdrop { background: rgb(10 20 30 / .5); }
+    .review__dialog-close { float: inline-end; }
+    .review__empty { padding: 3rem 1rem; color: var(--muted); text-align: center; }
+    .review__empty strong { display: block; margin-block-end: .35rem; color: var(--ink); }
+    .review__tour-spotlight { position: fixed; z-index: 1; pointer-events: none; border: 4px solid var(--accent); border-radius: 6px; box-shadow: 0 0 0 100vmax rgb(56 38 42 / .58), 0 0 0 8px rgb(163 123 50 / .25); }
+    .review__tour { width: min(34rem, calc(100% - 2rem)); padding: .5rem; color: var(--ink); }
+    .review__tour::backdrop { background: transparent; }
+    .review__tour-content { padding: clamp(1.5rem, 4vw, 2rem); }
+    .review__tour h2 { margin: 0 0 .65rem; font-size: clamp(1.5rem, 3vw, 2rem); line-height: 1.2; letter-spacing: -.02em; }
+    .review__tour p { margin: 0; color: var(--muted); }
+    .review__tour-footer { display: flex; justify-content: space-between; gap: .75rem; align-items: center; margin-block-start: 1.5rem; }
+    .review__tour-progress { color: var(--muted); font-size: .8125rem; }
+    .review__tour-actions { display: flex; gap: .5rem; }
+    .review__nav { display: flex; align-items: center; justify-content: space-between; gap: 1rem; max-width: 1440px; margin: auto; padding: .75rem clamp(1rem, 3vw, 3rem); border-block-end: 1px solid var(--line); }
+    .review__nav-links, .review__nav-account { display: flex; align-items: center; gap: .5rem; }
+    .review__nav button, .review__nav a { min-height: 44px; }
+    .review__nav .is-active { color: var(--brand); border-color: var(--brand); background: var(--subtle); }
+    .review__page { max-width: 38rem; }
+    .review__page h2 { font: 600 1.5rem/1.3 var(--serif); }
+    .review__page label { display: grid; gap: .35rem; margin-block: .9rem; color: var(--muted); font-size: .9rem; font-weight: 600; }
+    .review__page-actions { display: flex; flex-wrap: wrap; gap: .5rem; margin-block-start: 1rem; }
+    .review__notice { padding: .75rem; background: #FFF3DA; color: #82520D; border-radius: 6px; }
+    .review__user-list { padding: 0; list-style: none; }
+    .review__user-row { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: .75rem; padding-block: .75rem; border-block-end: 1px solid var(--line); }
+    .review__user-meta { display: grid; gap: .25rem; color: var(--muted); white-space: pre-line; }
+    .review__user-reset { display: flex; flex-wrap: wrap; gap: .5rem; align-items: end; }
+    .review__user-reset label { display: grid; gap: .25rem; margin: 0; }
+    .review__user-reset input { width: min(15rem, 100%); }
+    .review__header-top { padding-block: .75rem; align-items: center; }
+    .review__header-content { max-width: 1440px; margin: auto; padding-inline: clamp(1rem, 3vw, 3rem); }
+    body.review__locked .review__header, body.review__locked .review__main, body.review__locked .review__actions, body.review__locked footer, body.review__locked .search { visibility: hidden; }
+    .search { max-width: 1600px; margin: auto; }
+    .search__query { padding: 1rem clamp(1rem, 3vw, 3rem); background: var(--surface); border-block-end: 1px solid var(--line); }
+    .search__query > label, .search__filters > label { display: grid; gap: .35rem; color: var(--muted); font-size: .85rem; font-weight: 600; }
+    .search__query-row { display: grid; grid-template-columns: minmax(15rem, 1fr) auto 9rem; gap: .75rem; align-items: end; }
+    .search__language { display: grid; gap: .25rem; color: var(--muted); font-size: .8125rem; }
+    .search__status { margin-inline: clamp(1rem, 3vw, 3rem); }
+    .search__layout { display: grid; grid-template-columns: 230px minmax(320px, .9fr) minmax(420px, 1.2fr); min-height: 65vh; border-block: 1px solid var(--line); background: var(--surface); }
+    .search__filters, .search__results, .search__document { min-width: 0; padding: 1rem; }
+    .search__filters, .search__results { border-inline-end: 1px solid var(--line); }
+    .search__filters h2, .search__results h2, .search__document h2 { margin: 0; font: 600 1.5rem/1.3 var(--serif); }
+    .search__filters .review__panel-heading, .search__results .review__panel-heading { align-items: center; }
+    .search__filter { margin-block: 1.25rem; }
+    .search__filter h3 { margin: 0 0 .4rem; font-size: 1rem; }
+    .search__filter label { display: flex; align-items: center; gap: .5rem; min-height: 44px; }
+    .search__filter input { width: 24px; min-height: 24px; height: 24px; accent-color: var(--brand); }
+    .search__filters > label { margin-block: .75rem; }
+    .search__result-list { margin: 0; padding: 0; list-style: none; }
+    .search__result { padding: 1rem .5rem; border-block-end: 1px solid var(--line); }
+    .search__result--selected { background: var(--subtle); }
+    .search__result-title { min-height: 44px; border: 0; padding: 0; text-align: start; background: none; color: var(--brand); font-weight: 700; }
+    .search__pagination { display: flex; justify-content: space-between; align-items: center; gap: .5rem; padding-block-start: 1rem; }
+    .search__document { min-height: 60vh; }
+    .search__document-controls { display: flex; flex-wrap: wrap; gap: .75rem; }
+    .search__document-controls label { display: grid; gap: .25rem; min-width: 11rem; color: var(--muted); font-size: .875rem; }
+    .search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
+    .search__document-body h1, .search__document-body h2, .search__document-body h3 { font-family: var(--serif); }
+    .search__document-body img { max-width: 100%; height: auto; }
+    .search__sr-only { position: absolute; inline-size: 1px; block-size: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
+    @media (max-width: 969px) { .search__layout { grid-template-columns: 1fr; } }
+    @media (max-width: 800px) {
+      .search__filters, .search__results { border-inline-end: 0; border-block-end: 1px solid var(--line); }
+      .search__document { min-height: 24rem; }
+      .review__search { grid-template-columns: 1fr 1fr; }
+      .review__search .review__field:first-child { grid-column: 1 / -1; }
+      .review__workspace { grid-template-columns: 1fr; }
+      .review__main { padding-block-end: 2rem; }
+      .review__document { display: none; }
+      .review__document--tour-visible { display: block; }
+      .review__search button { grid-column: 1 / -1; }
+      .review__actions { padding-inline: 1rem; }
+      .review__actions-inner { flex-wrap: wrap; }
+      .review__actions label { flex: 1 1 14rem !important; }
+      .review__actions button { flex: 1 1 100%; }
+    }
+    @media (max-width: 480px) { .review__search { grid-template-columns: 1fr; } .review__search .review__field:first-child, .review__search button { grid-column: auto; } .review__rating { grid-template-columns: repeat(2, minmax(0, 1fr)); } .search__query-row { grid-template-columns: 1fr 1fr; } .search__query-row > input { grid-column: 1 / -1; } .review__nav { align-items: start; flex-wrap: wrap; } .review__nav-links { order: 3; width: 100%; flex-wrap: wrap; } .review__nav-account { margin-inline-start: auto; } }
+    @media (hover: hover) { button:hover { background: var(--surface-muted); border-color: var(--muted); } .review__button--primary:hover { background: var(--brand-dark); border-color: var(--brand-dark); } .review__result:hover { background: var(--subtle); } .review__result-title:hover, .search__result-title:hover { background: none; color: var(--brand); } .review__rating label:hover { border-color: var(--brand); } }
+    @media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; transition: none !important; } }
+  </style>
+</head>
+<body>
+  <a class="review__skip" href="#results">Перейти к результатам</a>
+  <header class="review__header">
+    <div class="review__nav"><div class="review__eyebrow">Акылдаш</div><nav class="review__nav-links" aria-label="Основная навигация"><a id="search-nav" href="/">Поиск</a><a id="review-nav" href="/review">Лаборатория</a><button id="account-open" type="button">Личный кабинет</button><button id="users-open" type="button" hidden>Пользователи</button></nav><div class="review__nav-account"><span id="signed-in-user"></span><button id="logout" type="button">Выйти</button></div></div>
+    <div class="review__header-content"><div class="review__header-top"><button id="tutorial-open" type="button">Как это работает</button></div>
+    <h1>Проверка качества поиска</h1><p>Сопоставляйте запрос с текстом документов и сохраняйте оценку выдачи.</p></div>
+  </header>
+  <main class="review__main" id="review">
+    <form class="review__search" id="search-form">
+      <label class="review__field">Юридический запрос<input id="query" name="q" maxlength="500" required autocomplete="off" placeholder="Например, как открыть ОсОО?"></label>
+      <label class="review__field review__field--small">Язык запроса<select id="language" name="language"><option value="ru">Русский</option><option value="ky">Кыргызский</option></select></label>
+      <label class="review__field review__field--small">Результатов<select id="page-size" name="page_size"><option>10</option><option>20</option></select></label>
+      <button class="review__button--primary" type="submit">Показать результаты</button>
+    </form>
+    <div class="review__status" id="status" role="status" aria-live="polite"></div>
+    <div class="review__workspace">
+      <section class="review__results" aria-labelledby="results-heading"><div class="review__panel-heading"><h2 id="results-heading">Результаты поиска</h2><span class="review__panel-hint">Оцените каждый документ</span></div><ol class="review__results-list" id="results"><li class="review__empty"><strong>Начните с запроса</strong>Введите юридический вопрос выше, чтобы увидеть результаты.</li></ol></section>
+      <section class="review__document" id="document-panel" aria-labelledby="document-heading"><div class="review__panel-heading"><h2 id="document-heading">Просмотр документа</h2><span class="review__panel-hint">Откройте результат</span></div><div class="review__document-meta" id="document-meta">Выберите название в списке, чтобы прочитать документ.</div><article class="review__document-body" id="document-body"></article></section>
+    </div>
+  </main>
+  <main class="search" id="search-page" hidden>
+    <h1 class="search__sr-only">Поиск по правовой базе</h1>
+    <form class="search__query" id="public-search-form"><label for="public-query">Поиск по правовой базе</label><div class="search__query-row"><input id="public-query" maxlength="500" autocomplete="off" placeholder="Введите правовой вопрос или название документа"><button class="review__button--primary" type="submit">Найти</button><label class="search__language">Язык<select id="public-language"><option value="ru">Русский</option><option value="ky">Кыргызский</option></select></label></div></form>
+    <p class="review__status" id="public-status" role="status" aria-live="polite"></p>
+    <div class="search__layout"><aside class="search__filters" aria-labelledby="filters-heading"><div class="review__panel-heading"><h2 id="filters-heading">Фильтры</h2><button id="clear-filters" type="button">Сбросить</button></div><div id="filter-fields"></div><label>Дата принятия от<input id="date-from" type="date"></label><label>Дата принятия до<input id="date-to" type="date"></label></aside>
+      <section class="search__results" aria-labelledby="search-results-heading"><div class="review__panel-heading"><div><h2 id="search-results-heading">Результаты</h2><span class="review__panel-hint" id="result-count">Введите запрос</span></div><label>Сортировка<select id="sort-order"><option value="relevance">По релевантности</option><option value="date">По дате</option></select></label></div><ol id="public-results" class="search__result-list"><li class="review__empty"><strong>Начните с поиска</strong>Введите запрос, чтобы найти документы.</li></ol><div class="search__pagination"><button id="page-prev" type="button" disabled>Назад</button><span id="page-label">Страница 1</span><button id="page-next" type="button" disabled>Далее</button></div></section>
+      <section class="search__document" aria-labelledby="public-document-heading"><div class="review__panel-heading"><h2 id="public-document-heading">Документ</h2></div><div class="search__document-controls"><label>Редакция<select id="document-edition" disabled></select></label><label>Язык документа<select id="document-language" disabled></select></label></div><p id="public-document-meta" class="review__document-meta">Выберите результат, чтобы прочитать доступную редакцию.</p><article id="public-document-body" class="review__document-body"></article></section>
+    </div>
+  </main>
+  <div class="review__actions"><div class="review__actions-inner"><label>Проверяющий<input id="reviewer" readonly aria-label="Проверяющий"></label><label>Комментарий к выдаче <input id="overall-comment" maxlength="4000" placeholder="Что важно учесть? (необязательно)"></label><button class="review__button--primary" id="save" type="button">Сохранить оценку</button></div></div>
+  <dialog id="login-dialog" aria-labelledby="login-heading"><form class="review__page" id="login-form"><h2 id="login-heading">Вход в Акылдаш</h2><p>Введите учетные данные, выданные администратором.</p><label>Логин<input id="login-username" name="username" autocomplete="username" required minlength="3" maxlength="64"></label><label>Пароль<input id="login-password" name="password" type="password" autocomplete="current-password" required></label><p id="login-error" class="review__status review__status--error" role="alert"></p><button class="review__button--primary" type="submit">Войти</button></form></dialog>
+  <dialog id="password-dialog" aria-labelledby="password-heading"><form class="review__page" id="password-form"><h2 id="password-heading">Создайте новый пароль</h2><p class="review__notice">Для продолжения замените временный пароль. Используйте не менее 12 символов.</p><label>Текущий пароль<input id="current-password" type="password" autocomplete="current-password" required></label><label>Новый пароль<input id="new-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><label>Повторите новый пароль<input id="confirm-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p id="password-error" class="review__status review__status--error" role="alert"></p><button class="review__button--primary" type="submit">Сохранить пароль</button></form></dialog>
+  <dialog id="account-dialog" aria-labelledby="account-heading"><section class="review__page"><h2 id="account-heading">Личный кабинет</h2><dl><dt>Логин</dt><dd id="profile-username"></dd><dt>Роль</dt><dd id="profile-role">Администратор</dd><dt>Дата создания</dt><dd id="profile-created"></dd></dl><form id="profile-password-form"><h3>Изменить пароль</h3><p>Новый пароль должен содержать не менее 12 символов.</p><label>Текущий пароль<input id="profile-current-password" type="password" autocomplete="current-password" required></label><label>Новый пароль<input id="profile-new-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p id="profile-error" role="alert" class="review__status review__status--error"></p><div class="review__page-actions"><button class="review__button--primary" type="submit">Изменить пароль</button><button type="button" data-close="account-dialog">Закрыть</button></div></form></section></dialog>
+  <dialog id="users-dialog" aria-labelledby="users-heading"><section class="review__page"><h2 id="users-heading">Управление пользователями</h2><form id="create-user-form"><label>Логин<input id="new-username" autocomplete="off" minlength="3" maxlength="64" required></label><label>Временный пароль<input id="temporary-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p>Пароль — от 12 символов. Новый пользователь сменит его при первом входе. Передайте временный пароль отдельно.</p><p id="users-error" role="alert" class="review__status review__status--error"></p><button class="review__button--primary" type="submit">Создать пользователя</button></form><ul id="user-list" class="review__user-list"></ul><button type="button" data-close="users-dialog">Закрыть</button></section></dialog>
+  <dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
+  <div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
+  <footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.12.2 · Внутренняя лаборатория релевантности</div></footer>
+  <script>
+    document.body.classList.add('review__locked');
+    const DRAFT_KEY = 'akyldash-search-review-draft';
+    const draftStorageKey = () => `${DRAFT_KEY}:${account?.username || 'anonymous'}`;
+    const TOUR_KEY = 'akyldash-search-review-tour-seen';
+    const TOUR_STEPS = [
+      { target: 'query', title: 'Начните с юридического вопроса', text: 'Введите практический запрос, например «как открыть ОсОО?», выберите язык и нажмите «Показать результаты».', label: 'Шаг 1 из 4' },
+      { target: 'results', title: 'Смотрите порядок выдачи', text: 'Документы расположены в том порядке, в котором их нашёл поиск. Номер слева — позиция результата.', label: 'Шаг 2 из 4' },
+      { target: 'document-panel', title: 'Читайте документ перед оценкой', text: 'Нажмите на название результата. Справа откроется текст документа и ссылка на официальный источник.', label: 'Шаг 3 из 4' },
+      { target: 'save', title: 'Сохраните свою разметку', text: 'Поставьте оценку каждому просмотренному документу, добавьте комментарии и сохраните результат внизу страницы.', label: 'Шаг 4 из 4' }
+    ];
+    const state = { results: [], token: '', selected: null, query: '' };
+    const $ = (id) => document.getElementById(id);
+    let account = null;
+    const reviewRoute = location.pathname === '/review';
+    $('review').hidden = !reviewRoute; $('search-page').hidden = reviewRoute; $('review-nav').classList.toggle('is-active', reviewRoute); $('search-nav').classList.toggle('is-active', !reviewRoute);
+    if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.12.2'; }
+    async function apiFetch(path, options = {}) { const response = await fetch(path, {credentials: 'same-origin', ...options}); if (response.status === 401 && path !== '/auth/login') { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); } return response; }
+    const showLogin = () => { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); };
+    const enterWorkspace = (user) => { account = user; $('signed-in-user').textContent = `${user.username} · ${user.role}`; $('reviewer').value = user.username; $('profile-username').textContent = user.username; $('profile-role').textContent = user.role; $('profile-created').textContent = new Date(user.created_at * 1000).toLocaleDateString('ru-RU'); $('users-open').hidden = user.username.toLowerCase() !== 'admin'; document.body.classList.remove('review__locked'); if (!reviewRoute) initSearchPage(); else if (!hasSeenTour()) setTimeout(openTutorial, 350); };
+    const filterFields = [['document_types', 'Вид документа', 'document_type'], ['statuses', 'Статус', 'status'], ['authorities', 'Орган принятия', 'authority'], ['legal_forces', 'Юридическая сила', 'legal_force']];
+    let filterCatalogs = {};
+    let publicPage = 1;
+    let selectedPublicDocument = null;
+    const selectedFilters = () => Object.fromEntries(filterFields.map(([key,, parameter]) => [parameter, document.querySelector(`input[name="filter-${parameter}"]:checked`)?.value || '']));
+    function renderFilterCatalogs(payload) { filterCatalogs = payload; const root = $('filter-fields'); root.replaceChildren(); filterFields.forEach(([key, title, parameter]) => { const items = payload[key] || []; if (!items.length) return; const group = document.createElement('section'); group.className = 'search__filter'; const heading = document.createElement('h3'); heading.textContent = title; group.append(heading); items.forEach((item) => { const label = document.createElement('label'); const input = document.createElement('input'); input.type = 'radio'; input.name = `filter-${parameter}`; input.value = item.code; input.addEventListener('change', () => runSearch(1)); const text = document.createElement('span'); text.textContent = `${item.labels.ru || item.labels.ky || item.code} (${item.count})`; label.append(input, text); group.append(label); }); root.append(group); }); }
+    function renderPublicResults(results) { const list = $('public-results'); list.replaceChildren(); if (!results.length) { list.innerHTML = '<li class="review__empty"><strong>Документы не найдены</strong>Измените запрос или сбросьте фильтры.</li>'; return; } results.forEach((result, index) => { const row = document.createElement('li'); row.className = 'search__result'; const title = document.createElement('button'); title.type = 'button'; title.className = 'search__result-title'; title.textContent = result.name || 'Название не указано'; title.addEventListener('click', () => openPublicDocument(result, row)); const meta = document.createElement('p'); meta.className = 'review__meta'; meta.textContent = [result.type, result.status, result.date_adopted, result.number, result.legal_force].filter(Boolean).join(' · '); const snippet = document.createElement('p'); snippet.textContent = result.snippet || 'Для документа нет выдержки из текста.'; row.append(title, meta, snippet); list.append(row); }); }
+    async function runSearch(page = 1) { const query = $('public-query').value.trim(); if (!query) { $('public-status').textContent = 'Введите запрос, чтобы начать поиск.'; return; } publicPage = page; $('public-status').textContent = 'Выполняется поиск…'; const params = new URLSearchParams({q: query, language: $('public-language').value, page: String(page), page_size: '20', sort: $('sort-order').value}); for (const [key, value] of Object.entries(selectedFilters())) if (value) params.set(key, value); if ($('date-from').value) params.set('date_from', $('date-from').value); if ($('date-to').value) params.set('date_to', $('date-to').value); history.replaceState(null, '', `/?${params}`); try { const response = await apiFetch(`/search?${params}`); const payload = await response.json(); if (!response.ok) throw new Error(payload.error || 'Не удалось выполнить поиск.'); renderPublicResults(payload.results); $('result-count').textContent = `${payload.results.length} документов`; $('public-status').textContent = payload.results.length ? `Результаты для запроса «${query}».` : `По запросу «${query}» документов нет. Измените запрос или фильтры.`; $('page-label').textContent = `Страница ${page}`; $('page-prev').disabled = page <= 1; $('page-next').disabled = !payload.has_next; } catch (error) { $('public-status').textContent = error.message || 'Не удалось выполнить поиск. Повторите попытку.'; $('result-count').textContent = 'Поиск недоступен'; } }
+    async function loadPublicEdition() {
+      if (!selectedPublicDocument) return;
+      const {code} = selectedPublicDocument;
+      const edition = $('document-edition').value;
+      const editionInfo = selectedPublicDocument.editions.find((item) => item.source_code === edition);
+      const languages = editionInfo?.available_languages || [];
+      const languageSelect = $('document-language');
+      [...languageSelect.options].forEach((option) => { option.disabled = !languages.includes(option.value); });
+      if (!languages.length) { $('public-document-meta').textContent = 'В этой редакции нет доступного текста.'; $('public-document-body').replaceChildren(); return; }
+      if (!languages.includes(languageSelect.value)) languageSelect.value = languages[0];
+      const language = languageSelect.value;
+      $('public-document-meta').textContent = 'Загрузка редакции…';
+      try {
+        const response = await apiFetch(`/documents/${encodeURIComponent(code)}/editions/${encodeURIComponent(edition)}?language=${encodeURIComponent(language)}`);
+        const payload = await response.json();
+        if (!response.ok) throw new Error(payload.error || 'Редакция недоступна.');
+        const content = payload.content[language];
+        const meta = $('public-document-meta'); meta.replaceChildren();
+        const details = document.createElement('span'); details.textContent = [selectedPublicDocument.type, selectedPublicDocument.status, selectedPublicDocument.date_adopted].filter(Boolean).join(' · ');
+        const source = document.createElement('a'); source.href = `https://cbd.minjust.gov.kg/${encodeURIComponent(code)}/edition/${encodeURIComponent(edition)}/${encodeURIComponent(language)}`; source.target = '_blank'; source.rel = 'noreferrer'; source.textContent = 'Официальный источник';
+        meta.append(details, document.createTextNode(' · '), source);
+        $('public-document-body').innerHTML = content.html;
+      } catch (error) { $('public-document-meta').textContent = error.message || 'Не удалось загрузить документ.'; $('public-document-body').replaceChildren(); }
+    }
+    async function openPublicDocument(result, row) {
+      document.querySelectorAll('.search__result--selected').forEach((item) => item.classList.remove('search__result--selected'));
+      row.classList.add('search__result--selected'); selectedPublicDocument = result;
+      $('public-document-heading').textContent = result.name || 'Документ'; $('public-document-body').replaceChildren();
+      try {
+        const response = await apiFetch(`/documents/${encodeURIComponent(result.code)}/editions`);
+        const payload = await response.json();
+        if (!response.ok) throw new Error(payload.error || 'Не удалось получить список редакций.');
+        selectedPublicDocument.editions = payload.editions || [];
+        const editionSelect = $('document-edition'); editionSelect.replaceChildren();
+        (payload.editions || []).forEach((edition) => { const option = document.createElement('option'); option.value = edition.source_code; const name = edition.name?.[$('public-language').value] || edition.name?.ru || edition.name?.ky; option.textContent = [name, edition.source_code].filter(Boolean).join(' · '); editionSelect.append(option); });
+        const matchedEdition = [...editionSelect.options].find((option) => option.value === String(result.edition));
+        if (matchedEdition) editionSelect.value = matchedEdition.value;
+        editionSelect.disabled = !editionSelect.options.length;
+        const languages = payload.available_languages || [];
+        const languageSelect = $('document-language'); languageSelect.replaceChildren();
+        languages.forEach((language) => { const option = document.createElement('option'); option.value = language; option.textContent = language === 'ru' ? 'Русский' : 'Кыргызский'; languageSelect.append(option); });
+        languageSelect.disabled = !languages.length;
+        if (!languages.length || !editionSelect.options.length) throw new Error('Для документа нет доступных редакций с текстом.');
+        languageSelect.value = languages.includes($('public-language').value) ? $('public-language').value : languages[0];
+        await loadPublicEdition();
+      } catch (error) { $('public-document-meta').textContent = error.message || 'Документ недоступен.'; }
+    }
+    $('document-edition').addEventListener('change', loadPublicEdition); $('document-language').addEventListener('change', loadPublicEdition);
+    async function initSearchPage() { if (reviewRoute || filterCatalogs.length) return; const params = new URLSearchParams(location.search); $('public-query').value = params.get('q') || ''; $('public-language').value = params.get('language') || 'ru'; $('sort-order').value = params.get('sort') || 'relevance'; $('date-from').value = params.get('date_from') || ''; $('date-to').value = params.get('date_to') || ''; const response = await apiFetch(`/search/filters?language=${encodeURIComponent($('public-language').value)}`); if (response.ok) renderFilterCatalogs(await response.json()); for (const [key,, parameter] of filterFields) { const value = params.get(parameter); const input = value && document.querySelector(`input[name="filter-${parameter}"][value="${CSS.escape(value)}"]`); if (input) input.checked = true; } const query = params.get('q'); if (query) runSearch(Number(params.get('page') || 1)); }
+    $('public-search-form').addEventListener('submit', (event) => { event.preventDefault(); runSearch(1); });
+    $('sort-order').addEventListener('change', () => runSearch(1)); $('date-from').addEventListener('change', () => runSearch(1)); $('date-to').addEventListener('change', () => runSearch(1)); $('public-language').addEventListener('change', async () => { const response = await apiFetch(`/search/filters?language=${encodeURIComponent($('public-language').value)}`); if (response.ok) renderFilterCatalogs(await response.json()); if ($('public-query').value.trim()) runSearch(1); });
+    $('page-prev').addEventListener('click', () => runSearch(publicPage - 1)); $('page-next').addEventListener('click', () => runSearch(publicPage + 1));
+    $('clear-filters').addEventListener('click', () => { document.querySelectorAll('#filter-fields input:checked').forEach((input) => { input.checked = false; }); $('date-from').value = ''; $('date-to').value = ''; $('sort-order').value = 'relevance'; runSearch(1); });
+    async function loadUsers() {
+      $('users-error').textContent = '';
+      const response = await apiFetch('/admin/users');
+      if (!response.ok) { $('users-error').textContent = 'Не удалось загрузить список пользователей.'; return; }
+      const {users} = await response.json();
+      const formatUserDate = (timestamp) => timestamp ? new Intl.DateTimeFormat('ru-RU', {dateStyle: 'medium', timeStyle: 'short'}).format(new Date(timestamp * 1000)) : '—';
+      const list = $('user-list'); list.replaceChildren();
+      users.forEach((user) => {
+        const row = document.createElement('li'); row.className = 'review__user-row';
+        const label = document.createElement('span'); label.textContent = `${user.username} · Администратор${user.must_change_password ? ' · требуется сменить пароль' : ''}`; row.append(label);
+        const dates = document.createElement('small'); dates.className = 'review__user-meta'; dates.textContent = `Первый вход: ${formatUserDate(user.first_login_at)}\nПоследний вход: ${formatUserDate(user.last_login_at)}\nСброс пароля: ${formatUserDate(user.password_reset_at)}`; row.append(dates);
+        if (user.username.toLowerCase() !== 'admin') {
+          const resetForm = document.createElement('form'); resetForm.className = 'review__user-reset';
+          const password = document.createElement('input'); password.type = 'password'; password.autocomplete = 'new-password'; password.minLength = 12; password.maxLength = 256; password.required = true;
+          const passwordLabel = document.createElement('label'); passwordLabel.append(document.createTextNode('Новый временный пароль (от 12 символов)'), password);
+          const reset = document.createElement('button'); reset.type = 'submit'; reset.textContent = 'Сбросить пароль'; resetForm.append(passwordLabel, reset);
+          resetForm.addEventListener('submit', async (event) => { event.preventDefault(); const result = await apiFetch(`/admin/users/${encodeURIComponent(user.username)}/reset-password`, {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({password: password.value})}); if (!result.ok) { $('users-error').textContent = (await result.json()).error || 'Не удалось сбросить пароль.'; return; } await loadUsers(); });
+          const remove = document.createElement('button'); remove.type = 'button'; remove.textContent = 'Удалить'; remove.addEventListener('click', async () => { if (!confirm(`Удалить учетную запись ${user.username}? Сохраненная разметка останется в истории.`)) return; const result = await apiFetch(`/admin/users/${encodeURIComponent(user.username)}`, {method: 'DELETE'}); if (!result.ok) { $('users-error').textContent = 'Не удалось удалить учетную запись.'; return; } await loadUsers(); });
+          row.append(resetForm, remove);
+        }
+        list.append(row);
+      });
+    }
+    $('login-form').addEventListener('submit', async (event) => { event.preventDefault(); $('login-error').textContent = ''; const response = await fetch('/auth/login', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({username: $('login-username').value, password: $('login-password').value})}); const payload = await response.json(); if (!response.ok) { $('login-error').textContent = payload.error || 'Не удалось войти. Проверьте логин и пароль.'; $('login-password').focus(); return; } $('login-dialog').close(); $('login-password').value = ''; if (payload.account.must_change_password) { $('current-password').value = sessionTemporaryPassword; $('password-dialog').showModal(); $('new-password').focus(); } else { enterWorkspace(payload.account); } });
+    let sessionTemporaryPassword = '';
+    $('login-form').addEventListener('submit', () => { sessionTemporaryPassword = $('login-password').value; }, true);
+    $('password-form').addEventListener('submit', async (event) => { event.preventDefault(); $('password-error').textContent = ''; if ($('new-password').value !== $('confirm-password').value) { $('password-error').textContent = 'Новые пароли не совпадают.'; $('confirm-password').focus(); return; } const response = await fetch('/auth/change-password', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({current_password: $('current-password').value, new_password: $('new-password').value})}); const payload = await response.json(); if (!response.ok) { $('password-error').textContent = payload.error || 'Не удалось изменить пароль.'; return; } $('password-dialog').close(); sessionTemporaryPassword = ''; $('password-form').reset(); enterWorkspace(payload.account); });
+    $('profile-password-form').addEventListener('submit', async (event) => { event.preventDefault(); $('profile-error').textContent = ''; const response = await apiFetch('/auth/change-password', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({current_password: $('profile-current-password').value, new_password: $('profile-new-password').value})}); const payload = await response.json(); if (!response.ok) { $('profile-error').textContent = payload.error || 'Не удалось изменить пароль.'; return; } $('account-dialog').close(); $('profile-password-form').reset(); enterWorkspace(payload.account); });
+    $('create-user-form').addEventListener('submit', async (event) => { event.preventDefault(); $('users-error').textContent = ''; const response = await apiFetch('/admin/users', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({username: $('new-username').value, password: $('temporary-password').value})}); const payload = await response.json(); if (!response.ok) { $('users-error').textContent = payload.error || 'Не удалось создать учетную запись.'; return; } $('create-user-form').reset(); await loadUsers(); });
+    $('account-open').addEventListener('click', () => $('account-dialog').showModal());
+    $('users-open').addEventListener('click', async () => { await loadUsers(); $('users-dialog').showModal(); });
+    document.querySelectorAll('[data-close]').forEach((button) => button.addEventListener('click', () => $(button.dataset.close).close()));
+    $('logout').addEventListener('click', async () => { await fetch('/auth/logout', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: '{}'}); account = null; document.body.classList.add('review__locked'); showLogin(); });
+    (async () => { const response = await fetch('/auth/me', {credentials: 'same-origin'}); if (!response.ok) { showLogin(); return; } const payload = await response.json(); if (payload.account.must_change_password) { $('password-dialog').showModal(); $('current-password').focus(); return; } enterWorkspace(payload.account); })();
+    const setStatus = (text, error = false) => { $('status').textContent = text; $('status').classList.toggle('review__status--error', error); $('status').setAttribute('role', error ? 'alert' : 'status'); $('status').setAttribute('aria-live', error ? 'assertive' : 'polite'); };
+    const setError = (text) => setStatus(text, true);
+    const escapeText = (value) => value == null ? '' : String(value);
+    const readDraft = () => { try { return JSON.parse(localStorage.getItem(draftStorageKey()) || 'null'); } catch (_) { return null; } };
+    const hasSeenTour = () => { try { return localStorage.getItem(TOUR_KEY) === '1'; } catch (_) { return false; } };
+    const markTourSeen = () => { try { localStorage.setItem(TOUR_KEY, '1'); } catch (_) {} };
+    const saveDraft = () => { if (!state.token) return; const results = [...$('results').children].map((item, index) => ({rank: index + 1, code: item.dataset.code, rating: item.querySelector('input:checked')?.value ?? null, comment: item.querySelector('textarea').value})); try { localStorage.setItem(draftStorageKey(), JSON.stringify({query: state.query, language: $('language').value, pageSize: $('page-size').value, overallComment: $('overall-comment').value, results})); } catch (_) {} };
+    const applyDraft = () => { const draft = readDraft(); if (!draft || draft.query !== state.query || draft.language !== $('language').value || draft.pageSize !== $('page-size').value) return; $('reviewer').value = account?.username || ''; $('overall-comment').value = draft.overallComment || ''; (draft.results || []).forEach((saved) => { const item = [...$('results').children].find((candidate) => candidate.dataset.rank === String(saved.rank) && candidate.dataset.code === saved.code); if (!item) return; if (saved.rating != null) { const input = item.querySelector(`input[value="${CSS.escape(String(saved.rating))}"]`); if (input) input.checked = true; } item.querySelector('textarea').value = saved.comment || ''; }); };
+    function renderResults() {
+      $('results').replaceChildren();
+      if (!state.results.length) { $('results').innerHTML = '<li class="review__empty"><strong>Результатов пока нет</strong>Введите другой запрос и попробуйте снова.</li>'; return; }
+      state.results.forEach((result, index) => {
+        const item = document.createElement('li'); item.className = 'review__result'; item.dataset.rank = index + 1; item.dataset.code = result.code;
+        const title = document.createElement('button'); title.type = 'button'; title.className = 'review__result-title'; title.innerHTML = `<span class="review__rank">#${index + 1}</span><span></span>`; title.lastElementChild.textContent = escapeText(result.name || 'Название не указано'); title.addEventListener('click', () => openDocument(index, title));
+        const meta = document.createElement('div'); meta.className = 'review__meta'; meta.textContent = [result.type, result.status, result.date_adopted, result.number].filter(Boolean).join(' · ');
+        const rating = document.createElement('fieldset'); rating.className = 'review__rating'; rating.innerHTML = `<legend>Оценка результата</legend>`;
+        [['0', 'нерелевантен'], ['1', 'косвенно полезен'], ['2', 'частично полезен'], ['3', 'прямо отвечает']].forEach(([value, label]) => { const id = `rating-${index}-${value}`; const wrapper = document.createElement('label'); wrapper.htmlFor = id; wrapper.textContent = `${value} — ${label}`; const input = document.createElement('input'); input.type = 'radio'; input.name = `rating-${index}`; input.id = id; input.value = value; wrapper.prepend(input); rating.append(wrapper); });
+        const comment = document.createElement('textarea'); comment.className = 'review__comment'; comment.maxLength = 4000; comment.placeholder = 'Комментарий к результату (необязательно)'; comment.setAttribute('aria-label', `Комментарий к результату #${index + 1}`);
+        item.append(title, meta, rating, comment); $('results').append(item);
+      });
+      applyDraft();
+    }
+    async function openDocument(index, trigger) {
+      const result = state.results[index]; state.selected = trigger; [...$('results').children].forEach((item) => item.classList.remove('review__result--selected')); trigger.closest('.review__result')?.classList.add('review__result--selected'); setStatus('Загрузка документа…');
+      try { const response = await apiFetch(`/documents/${encodeURIComponent(result.code)}/editions/${encodeURIComponent(result.edition)}?language=${encodeURIComponent($('language').value)}`); if (!response.ok) throw new Error('Документ недоступен'); const payload = await response.json(); const content = payload.content[$('language').value]; const meta = $('document-meta'); meta.textContent = [result.name, result.status, result.date_adopted].filter(Boolean).join(' · '); const source = document.createElement('a'); source.href = `https://cbd.minjust.gov.kg/${encodeURIComponent(result.code)}/edition/${encodeURIComponent(result.edition)}/${encodeURIComponent($('language').value)}`; source.target = '_blank'; source.rel = 'noreferrer'; source.textContent = ' Официальный источник'; meta.append(source); $('document-body').innerHTML = content.html; $('dialog-heading').textContent = result.name || 'Документ'; $('dialog-body').innerHTML = content.html; if (matchMedia('(max-width: 800px)').matches) $('document-dialog').showModal(); setStatus('Документ загружен.'); } catch (error) { setError('Не удалось загрузить документ. Повторите попытку.'); }
+    }
+    $('dialog-close').addEventListener('click', () => { $('document-dialog').close(); if (state.selected) state.selected.focus(); });
+    let tutorialStep = 0; let tutorialReturnFocus = null;
+    function clearTourTarget() { $('tutorial-spotlight').hidden = true; $('document-panel').classList.remove('review__document--tour-visible'); }
+    function renderTutorial() { const step = TOUR_STEPS[tutorialStep]; const target = $(step.target); if (step.target === 'document-panel' && getComputedStyle(target).display === 'none') target.classList.add('review__document--tour-visible'); target.scrollIntoView({block: 'nearest', inline: 'nearest'}); const bounds = target.getBoundingClientRect(); $('tutorial-spotlight').style.cssText = `inset-block-start: ${Math.max(8, bounds.top - 4)}px; inset-inline-start: ${Math.max(8, bounds.left - 4)}px; inline-size: ${bounds.width + 8}px; block-size: ${bounds.height + 8}px;`; $('tutorial-spotlight').hidden = false; $('tutorial-heading').textContent = step.title; $('tutorial-text').textContent = step.text; $('tutorial-progress').textContent = step.label; $('tutorial-next').textContent = tutorialStep === TOUR_STEPS.length - 1 ? 'Понятно' : 'Далее'; }
+    function closeTutorial() { clearTourTarget(); $('tutorial-dialog').close(); markTourSeen(); tutorialReturnFocus?.focus(); }
+    function openTutorial() { tutorialReturnFocus = document.activeElement; tutorialStep = 0; renderTutorial(); $('tutorial-dialog').showModal(); $('tutorial-next').focus(); }
+    $('tutorial-open').addEventListener('click', openTutorial); $('tutorial-next').addEventListener('click', () => { if (tutorialStep === TOUR_STEPS.length - 1) closeTutorial(); else { tutorialStep += 1; renderTutorial(); } }); $('tutorial-skip').addEventListener('click', closeTutorial); $('tutorial-dialog').addEventListener('cancel', (event) => { event.preventDefault(); closeTutorial(); });
+    $('search-form').addEventListener('submit', async (event) => { event.preventDefault(); const query = $('query').value.trim(); if (!query) return; state.query = query; setStatus('Поиск выполняется…'); $('save').disabled = true; try { const params = new URLSearchParams({q: query, language: $('language').value, page_size: $('page-size').value}); const response = await apiFetch(`/search?${params}`); if (!response.ok) throw new Error(); const payload = await response.json(); state.results = payload.results; state.token = payload.review_token; renderResults(); setStatus(state.results.length ? `Найдено результатов: ${state.results.length}.` : `По запросу «${query}» ничего не найдено. Измените запрос.`); } catch (error) { state.results = []; state.token = ''; renderResults(); setError('Не удалось выполнить поиск. Повторите поиск.'); } finally { $('save').disabled = false; } });
+    document.addEventListener('input', saveDraft); document.addEventListener('change', saveDraft);
+    $('save').addEventListener('click', async () => { if (!state.token) { setError('Сначала выполните поиск.'); return; } const reviewer = $('reviewer').value.trim(); const results = [...$('results').children].map((item, index) => ({rank: index + 1, code: item.dataset.code, rating: item.querySelector('input:checked') ? Number(item.querySelector('input:checked').value) : null, comment: item.querySelector('textarea').value})); $('save').disabled = true; setStatus('Сохранение выполняется…'); try { const response = await apiFetch('/search-reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({review_token: state.token, reviewer, overall_comment: $('overall-comment').value, results})}); if (!response.ok) throw new Error(); const payload = await response.json(); localStorage.removeItem(draftStorageKey()); setStatus(`Оценка сохранена · № ${payload.id}.`); } catch (error) { setError('Не удалось сохранить. Проверьте подключение и повторите.'); } finally { $('save').disabled = false; } });
+    const draft = readDraft(); if (draft) { $('query').value = draft.query || ''; $('language').value = draft.language || 'ru'; $('page-size').value = draft.pageSize || '20'; $('reviewer').value = account?.username || ''; $('overall-comment').value = draft.overallComment || ''; }
+    if (reviewRoute) $('tutorial-open').hidden = false;
+  </script>
+</body>
+</html>

backend/search/reviews.py

@@ -0,0 +1,83 @@
+"""Persistence and signed snapshots for search relevance reviews."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import secrets
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+
+MAX_COMMENT = 4000
+MAX_REVIEWER = 120
+
+
+class ReviewStore:
+    def __init__(self, path: Path | str = ":memory:"):
+        if path != ":memory:":
+            Path(path).parent.mkdir(parents=True, exist_ok=True)
+        self.connection = sqlite3.connect(path, check_same_thread=False)
+        self.connection.row_factory = sqlite3.Row
+        # ponytail: one SQLite lock; split connections only if review throughput matters.
+        self._lock = threading.Lock()
+        self.connection.execute("""
+            CREATE TABLE IF NOT EXISTS search_reviews (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                created_at TEXT NOT NULL,
+                reviewer TEXT NOT NULL,
+                query TEXT NOT NULL,
+                language TEXT NOT NULL,
+                index_name TEXT NOT NULL,
+                algorithm_version TEXT NOT NULL,
+                top_result_code TEXT,
+                results_json TEXT NOT NULL,
+                overall_comment TEXT NOT NULL
+            )
+        """)
+        self.connection.commit()
+
+    def save(self, review: dict) -> int:
+        with self._lock:
+            cursor = self.connection.execute(
+                "INSERT INTO search_reviews(created_at, reviewer, query, language, index_name, algorithm_version, top_result_code, results_json, overall_comment) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (review["created_at"], review["reviewer"], review["query"], review["language"], review["index_name"], review["algorithm_version"], review["top_result_code"], json.dumps(review["results"], ensure_ascii=False), review["overall_comment"]),
+            )
+            self.connection.commit()
+            return int(cursor.lastrowid)
+
+    def export(self) -> list[dict]:
+        with self._lock:
+            return [
+                {**dict(row), "results": json.loads(row["results_json"])}
+                for row in self.connection.execute("SELECT * FROM search_reviews ORDER BY id")
+            ]
+
+
+class ReviewSnapshots:
+    def __init__(self, secret: bytes | None = None, ttl: int = 3600):
+        self.secret = secret or secrets.token_bytes(32)
+        self.ttl = ttl
+
+    def create(self, query: str, language: str, index: str, results: list[dict], algorithm_version: str) -> str:
+        payload = {"query": query, "language": language, "index_name": index, "algorithm_version": algorithm_version, "results": results, "expires_at": int(time.time()) + self.ttl}
+        encoded = base64.urlsafe_b64encode(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).decode().rstrip("=")
+        signature = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest()
+        return f"{encoded}.{signature}"
+
+    def verify(self, token: str) -> dict:
+        try:
+            encoded, signature = token.split(".", 1)
+            expected = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest()
+            if not hmac.compare_digest(signature, expected):
+                raise ValueError
+            payload = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)))
+            if payload["expires_at"] < int(time.time()):
+                raise ValueError
+            return payload
+        except (ValueError, KeyError, TypeError, json.JSONDecodeError, UnicodeError) as error:
+            raise ValueError("invalid or expired search snapshot") from error

backend/test_minjust_cbd.py

@@ -0,0 +1,156 @@
+import base64
+import tempfile
+import unittest
+import urllib.error
+from email.message import Message
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from ingestion.minjust_cbd import CbdClient, SyncResult, progress_line, sync_archive
+
+
+class MinjustCbdTest(unittest.TestCase):
+    def test_archives_document_and_resumes(self):
+        image = base64.b64encode(b"image").decode()
+        document = {
+            "Code": 1,
+            "Name": {"Rus": "Закон", "Kyr": "Мыйзам"},
+            "Editions": [
+                {
+                    "Code": 10,
+                    "Name": {"Rus": "01.01.2026", "Kyr": "01.01.2026"},
+                    "Data": {"Rus": "<p>Закон</p>", "Kyr": "<p>Мыйзам</p>"},
+                    "Images": [
+                        {
+                            "Lang": "Russian",
+                            "Name": "image001.jpg",
+                            "Data": image,
+                        }
+                    ],
+                }
+            ],
+        }
+        client = Mock()
+        client.total_documents = 1
+        client.document_codes.return_value = [1]
+        client.document.return_value = document
+
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            first = sync_archive(output, client)
+            second = sync_archive(output, client)
+            refreshed = sync_archive(output, client, refresh=True)
+
+            self.assertEqual(first.downloaded, 1)
+            self.assertEqual(second.skipped, 1)
+            self.assertEqual(refreshed.downloaded, 1)
+            self.assertEqual(client.document.call_count, 2)
+            self.assertEqual(
+                (output / "documents/1/editions/10/ru.html").read_text(),
+                "<p>Закон</p>",
+            )
+            self.assertEqual(
+                (output / "documents/1/editions/10/images/ru/image001.jpg").read_bytes(),
+                b"image",
+            )
+
+    def test_uses_stable_list_id_for_next_page(self):
+        client = CbdClient(requests_per_second=1000)
+        client.request_json = Mock(
+            side_effect=[
+                {
+                    "Id": "list-id",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 1}, {"Code": 2}],
+                },
+                {
+                    "Id": "list-id",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 3}],
+                },
+            ]
+        )
+
+        self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3])
+        self.assertEqual(
+            client.request_json.call_args_list[1].args[0], "GetDocumentListById"
+        )
+
+    def test_recreates_expired_list_at_current_page(self):
+        client = CbdClient(requests_per_second=1000)
+        client.request_json = Mock(
+            side_effect=[
+                {
+                    "Id": "expired-list",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 1}, {"Code": 2}],
+                },
+                urllib.error.HTTPError("https://example.test", 404, "", {}, None),
+                {
+                    "Id": "new-list",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 3}],
+                },
+            ]
+        )
+
+        self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3])
+        recovery_call = client.request_json.call_args_list[2]
+        self.assertEqual(recovery_call.args[0], "GetDocumentListByQuery")
+        self.assertIn(("PageNumber", 2), recovery_call.args[1])
+
+    def test_recreates_list_after_page_retries_are_exhausted(self):
+        client = CbdClient(requests_per_second=1000)
+        client.request_json = Mock(
+            side_effect=[
+                {
+                    "Id": "failed-list",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 1}, {"Code": 2}],
+                },
+                RuntimeError("Ministry of Justice API request failed"),
+                {
+                    "Id": "new-list",
+                    "TotalCount": 3,
+                    "Documents": [{"Code": 3}],
+                },
+            ]
+        )
+
+        self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3])
+        recovery_call = client.request_json.call_args_list[2]
+        self.assertEqual(recovery_call.args[0], "GetDocumentListByQuery")
+        self.assertIn(("PageNumber", 2), recovery_call.args[1])
+
+    def test_formats_progress_with_rate_and_eta(self):
+        line = progress_line(
+            SyncResult(discovered=50, downloaded=48, skipped=1, failed=1),
+            total=100,
+            elapsed=25,
+        )
+
+        self.assertIn("50.00%", line)
+        self.assertIn("rate=2.00/s", line)
+        self.assertIn("ETA=00:00:25", line)
+
+    @patch("ingestion.minjust_cbd.time.sleep")
+    @patch("ingestion.minjust_cbd.urllib.request.urlopen")
+    def test_respects_retry_after_on_rate_limit(self, urlopen, sleep):
+        headers = Message()
+        headers["Retry-After"] = "7"
+        response = Mock()
+        response.read.return_value = b"{}"
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        urlopen.side_effect = [
+            urllib.error.HTTPError("https://example.test", 429, "", headers, None),
+            response,
+        ]
+
+        CbdClient(requests_per_second=1000, retries=2).request_json("Example")
+
+        self.assertIn(((7.0,), {}), [(call.args, call.kwargs) for call in sleep.call_args_list])
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_minjust_current.py

@@ -0,0 +1,405 @@
+import json
+import os
+import sqlite3
+import tempfile
+import unittest
+import urllib.error
+from email.message import Message
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from ingestion import minjust_current
+from ingestion.minjust_current import (
+    ApiResponse,
+    CurrentCbdClient,
+    SourceChanged,
+    SourceError,
+    stage_document,
+)
+
+
+def api_response(value):
+    body = json.dumps(value, ensure_ascii=False).encode()
+    import hashlib
+
+    return ApiResponse(value, body, hashlib.sha256(body).hexdigest())
+
+
+def document_record(alias="6", canonical=6, editions=None):
+    return api_response(
+        {
+            "documentCode": canonical,
+            "nameRus": "Закон",
+            "nameKyr": "Мыйзам",
+            "headerRu": "Заголовок",
+            "headerKg": "Баш сөз",
+            "number": 114,
+            "dateAdopted": "1998-08-04",
+            "classId": 1,
+            "refTypeId": 10,
+            "status": 1,
+            "editions": editions if editions is not None else [
+                {"id": 920492, "nameRus": "Редакция A", "nameKyr": "Редакция A", "textRusType": "docx", "textKyrType": "docx"},
+                {"id": 920292, "nameRus": "Редакция B", "nameKyr": "Редакция B", "textRusType": "docx", "textKyrType": "docx"},
+            ],
+        }
+    )
+
+
+class CurrentCbdClientTests(unittest.TestCase):
+    def test_rejects_legacy_roots_when_called_from_an_unrelated_working_directory(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            repository = base / "repository"
+            elsewhere = base / "elsewhere"
+            elsewhere.mkdir()
+            legacy_roots = [
+                repository / "data/minjust-cbd",
+                repository / "data/minjust-normalized",
+            ]
+            for legacy in legacy_roots:
+                legacy.mkdir(parents=True)
+            # Root CI uses umask 000, so make the trusted fixture's parent permissions explicit.
+            for directory in (repository, repository / "data", *legacy_roots):
+                directory.chmod(0o755)
+            original_cwd = Path.cwd()
+            client = Mock()
+            try:
+                os.chdir(elsewhere)
+                with patch.object(minjust_current, "REPOSITORY_ROOT", repository):
+                    for legacy in legacy_roots:
+                        with self.subTest(legacy=legacy), self.assertRaisesRegex(ValueError, "must not overlap"):
+                            stage_document(legacy / "staging", "6", client)
+                        self.assertFalse((legacy / "staging").exists())
+            finally:
+                os.chdir(original_cwd)
+            client.document.assert_not_called()
+
+    def test_requires_private_owned_staging_root(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "shared"
+            output.mkdir(mode=0o755)
+            output.chmod(0o755)
+
+            with self.assertRaisesRegex(ValueError, "private directory"):
+                stage_document(output, "6", Mock())
+
+    def test_resume_rejects_symlinked_checkpoints_and_document_paths(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            targets = [
+                ("request-contract.json", False),
+                ("current-api.sqlite3", False),
+                ("manifest.sqlite3", False),
+                (".work/documents/6", True),
+                (".work/documents/6/editions/920492/ru.html", False),
+                ("documents/6", True),
+            ]
+            client = Mock()
+            for index, (relative, directory) in enumerate(targets):
+                with self.subTest(relative=relative):
+                    output = base / f"generation-{index}"
+                    output.mkdir(mode=0o700)
+                    contract = output / "request-contract.json"
+                    if relative != "request-contract.json":
+                        contract.write_text(json.dumps(minjust_current.REQUEST_CONTRACT), encoding="utf-8")
+                    outside = base / f"outside-{index}"
+                    if directory:
+                        outside.mkdir()
+                    else:
+                        outside.write_text("preserve", encoding="utf-8")
+                    link = output / relative
+                    link.parent.mkdir(parents=True, exist_ok=True)
+                    try:
+                        link.symlink_to(outside, target_is_directory=directory)
+                    except (NotImplementedError, OSError):
+                        self.skipTest("symlinks are unavailable")
+
+                    with self.assertRaisesRegex(ValueError, "symbolic link"):
+                        stage_document(output, "6", client, resume=True)
+
+                    client.document.assert_not_called()
+                    if not directory:
+                        self.assertEqual(outside.read_text(encoding="utf-8"), "preserve")
+                    else:
+                        self.assertEqual(list(outside.iterdir()), [])
+
+    def test_respects_retry_after_and_retries_server_errors(self):
+        headers = Message()
+        headers["Retry-After"] = "7"
+        response = Mock()
+        response.read.return_value = b"{}"
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        opener = Mock(side_effect=[
+            urllib.error.HTTPError("https://example.test", 429, "", headers, None),
+            urllib.error.HTTPError("https://example.test", 503, "", Message(), None),
+            response,
+        ])
+        delays = []
+
+        CurrentCbdClient(opener=opener, sleep=delays.append, clock=lambda: 0, retries=3).request_json("/GetEdition")
+
+        self.assertIn(7.0, delays)
+        self.assertIn(2, delays)
+        self.assertEqual(opener.call_count, 3)
+
+    def test_enforces_request_interval(self):
+        now = [0.0]
+        starts = []
+        response = Mock()
+        response.read.return_value = b"{}"
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+
+        def opener(_request, timeout):
+            starts.append(now[0])
+            return response
+
+        client = CurrentCbdClient(
+            opener=opener,
+            clock=lambda: now[0],
+            sleep=lambda delay: now.__setitem__(0, now[0] + delay),
+        )
+        client.request_json("/GetEdition")
+        client.request_json("/GetEdition")
+
+        self.assertGreaterEqual(starts[1] - starts[0], 1.0)
+
+    def test_rejects_oversized_response(self):
+        response = Mock()
+        response.read.return_value = b"x" * 11
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        with self.assertRaisesRegex(SourceError, "exceeds 10 bytes"):
+            CurrentCbdClient(opener=lambda *_args, **_kwargs: response, max_response_bytes=10).request_json("/GetEdition")
+
+    def test_get_edition_uses_exact_query_and_checks_requested_id(self):
+        response = Mock()
+        response.read.return_value = json.dumps({"id": 920292, "contentRu": "text", "contentKg": None}).encode()
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        requests = []
+
+        def opener(request, timeout):
+            requests.append(request)
+            return response
+
+        result = CurrentCbdClient(opener=opener, sleep=lambda _delay: None).edition(920292, "kg")
+
+        self.assertEqual(requests[0].full_url, "https://cbd.minjust.gov.kg/api/v1/GetEdition?editionId=920292&lang=kg&exact=true")
+        self.assertEqual(requests[0].method, "GET")
+        self.assertEqual(result.value["id"], 920292)
+
+    def test_rejects_response_id_that_does_not_match_exact_request(self):
+        response = Mock()
+        response.read.return_value = json.dumps({"id": 920492, "contentRu": "text", "contentKg": None}).encode()
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+
+        with self.assertRaisesRegex(SourceChanged, "does not match requested ID"):
+            CurrentCbdClient(opener=lambda *_args, **_kwargs: response).edition(920292, "ru")
+
+    def test_stages_alias_mapping_language_quirk_and_official_edition_order(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            output = base / "generation"
+            old_raw = base / "legacy" / "documents" / "6" / "sentinel"
+            old_raw.parent.mkdir(parents=True)
+            old_raw.write_text("preserve", encoding="utf-8")
+
+            class Client:
+                def document(self, alias):
+                    self.alias = alias
+                    return document_record(alias=alias, canonical=6)
+
+                def edition(self, identifier, language):
+                    text = "<p>Russian</p>" if language == "ru" else "<p>Кыргызча</p>"
+                    name = "Редакция A" if identifier == 920492 else "Редакция B"
+                    return api_response({
+                        "id": identifier, "nameRus": name, "nameKyr": name,
+                        "contentRu": text, "contentKg": None,
+                    })
+
+            client = Client()
+            result = stage_document(output, "6", client)
+            raw = output / "documents" / "6"
+
+            self.assertEqual(result["state"], "text_complete_incomplete_attachments")
+            self.assertEqual((raw / "editions/920492/ru.html").read_text(), "<p>Russian</p>")
+            self.assertEqual((raw / "editions/920492/ky.html").read_text(), "<p>Кыргызча</p>")
+            first = json.loads((raw / "editions/920492/metadata.json").read_text())
+            second = json.loads((raw / "editions/920292/metadata.json").read_text())
+            self.assertEqual((first["SourceOrder"], second["SourceOrder"]), (0, 1))
+            language_meta = json.loads((raw / "editions/920292/source/ru.json").read_text())
+            self.assertEqual((language_meta["requested_edition_id"], language_meta["exact"]), (920292, True))
+            with sqlite3.connect(output / "current-api.sqlite3") as connection:
+                self.assertTrue(connection.execute("SELECT contract_sha256 FROM generation").fetchone()[0])
+            self.assertEqual(old_raw.read_text(), "preserve")
+            with sqlite3.connect(output / "current-api.sqlite3") as connection:
+                rows = connection.execute(
+                    "SELECT request_language, archive_language, state FROM languages ORDER BY request_language, edition_id"
+                ).fetchall()
+                self.assertEqual(rows, [("kg", "ky", "complete")] * 2 + [("ru", "ru", "complete")] * 2)
+
+    def test_resolves_official_alias_to_canonical_staging_directory(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                def document(self, alias):
+                    return document_record(alias=alias, canonical=205064, editions=[])
+
+            result = stage_document(output, "1-4", Client())
+
+            self.assertEqual(result["canonical_id"], 205064)
+            self.assertTrue((output / "documents/205064/metadata.json").is_file())
+            self.assertFalse((output / "documents/1-4").exists())
+
+    def test_rejects_non_exact_edition_response_id_without_publishing(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                def document(self, _alias):
+                    return document_record(editions=[{"id": 920292, "textRusType": "docx", "textKyrType": "docx"}])
+
+                def edition(self, _identifier, language):
+                    return api_response({"id": 920492, "contentRu": f"<{language} edition>", "contentKg": None})
+
+            with self.assertRaisesRegex(SourceChanged, "exact edition ID mismatch"):
+                stage_document(output, "6", Client())
+
+            self.assertFalse((output / "documents/6").exists())
+            with sqlite3.connect(output / "current-api.sqlite3") as connection:
+                self.assertEqual(connection.execute("SELECT state FROM languages").fetchone()[0], "error")
+
+    def test_rejects_exact_edition_metadata_contradiction(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+            client = CurrentCbdClient()
+            client.document = lambda _alias: document_record(editions=[{
+                "id": 920292, "nameRus": "04.08.1998", "nameKyr": "04.08.1998",
+                "textRusType": "docx", "textKyrType": "docx",
+            }])
+            client.request_json = lambda *_args, **_kwargs: api_response({
+                "id": 920292, "nameRus": "13.04.2017", "nameKyr": "13.04.2017",
+                "contentRu": "<wrong edition>", "contentKg": None,
+            })
+
+            with self.assertRaisesRegex(SourceChanged, "metadata mismatch"):
+                stage_document(output, "6", client)
+
+            self.assertFalse((output / "documents/6").exists())
+
+    def test_recovers_atomic_publish_before_manifest_checkpoint(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                def document(self, _alias):
+                    return document_record(editions=[{"id": 920492, "textRusType": "docx", "textKyrType": "docx"}])
+
+                def edition(self, identifier, language):
+                    return api_response({"id": identifier, "contentRu": f"<{language}>", "contentKg": None})
+
+            client = Client()
+            first = stage_document(output, "6", client)
+            with sqlite3.connect(output / "current-api.sqlite3") as connection:
+                connection.execute("UPDATE documents SET state='fetching' WHERE alias='6'")
+            with sqlite3.connect(output / "manifest.sqlite3") as connection:
+                connection.execute("DELETE FROM documents WHERE code=6")
+                connection.execute("DELETE FROM errors WHERE code=6")
+
+            resumed = stage_document(output, "6", client, resume=True)
+
+            self.assertEqual(resumed["source_sha256"], first["source_sha256"])
+            self.assertTrue(resumed["resumed"])
+            with sqlite3.connect(output / "manifest.sqlite3") as connection:
+                self.assertEqual(connection.execute("SELECT COUNT(*) FROM documents WHERE code=6").fetchone()[0], 1)
+                self.assertEqual(connection.execute("SELECT COUNT(*) FROM errors WHERE code=6").fetchone()[0], 1)
+
+    def test_retries_after_document_metadata_request_failed_before_checkpoint(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                failed = False
+
+                def document(self, _alias):
+                    if not self.failed:
+                        self.failed = True
+                        raise SourceError("temporary DNS failure")
+                    return document_record(editions=[])
+
+            client = Client()
+            with self.assertRaisesRegex(SourceError, "DNS"):
+                stage_document(output, "6", client)
+
+            result = stage_document(output, "6", client, resume=True)
+
+            self.assertEqual(result["canonical_id"], 6)
+            with sqlite3.connect(output / "current-api.sqlite3") as connection:
+                row = connection.execute(
+                    "SELECT canonical_id, response_sha256, expected_editions FROM documents WHERE alias='6'"
+                ).fetchone()
+                self.assertEqual(row, (6, document_record(editions=[]).sha256, 0))
+
+    def test_does_not_resume_generation_without_exact_request_contract(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "legacy-generation"
+            output.mkdir(mode=0o700)
+            sentinel = output / "old-response.json"
+            sentinel.write_text("preserve", encoding="utf-8")
+            client = Mock()
+
+            with self.assertRaisesRegex(SourceChanged, "no exact-edition request contract"):
+                stage_document(output, "6", client, resume=True)
+
+            client.document.assert_not_called()
+            self.assertEqual(sentinel.read_text(encoding="utf-8"), "preserve")
+
+    def test_incomplete_translation_resumes_without_rewriting_successful_language(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                failed = False
+
+                def document(self, _alias):
+                    return document_record(editions=[{"id": 920492, "textRusType": "docx", "textKyrType": "docx"}])
+
+                def edition(self, identifier, language):
+                    if language == "kg" and not self.failed:
+                        self.failed = True
+                        raise SourceError("simulated interrupted response")
+                    return api_response({"id": identifier, "contentRu": f"<{language}>", "contentKg": None})
+
+            client = Client()
+            first = stage_document(output, "6", client)
+            self.assertIn("incomplete", first)
+            self.assertFalse((output / "documents/6").exists())
+            second = stage_document(output, "6", client, resume=True)
+            self.assertEqual(second["state"], "text_complete_incomplete_attachments")
+            self.assertEqual((output / "documents/6/editions/920492/ru.html").read_text(), "<ru>")
+            self.assertEqual((output / "documents/6/editions/920492/ky.html").read_text(), "<kg>")
+            self.assertEqual(client.failed, True)
+
+    def test_refuses_budget_overrun_without_publishing_partial_document(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            output = Path(temporary) / "generation"
+
+            class Client:
+                def document(self, _alias):
+                    return document_record(editions=[{"id": 920492, "textRusType": "docx", "textKyrType": "docx"}])
+
+                def edition(self, identifier, _language):
+                    return api_response({"id": identifier, "contentRu": "123456", "contentKg": None})
+
+            with self.assertRaisesRegex(SourceError, "budget"):
+                stage_document(output, "6", Client(), max_document_bytes=5)
+            self.assertFalse((output / "documents/6").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_minjust_normalization.py

@@ -0,0 +1,193 @@
+import json
+import os
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+from normalization.minjust_cbd import normalize_archive
+
+
+class MinjustNormalizationTest(unittest.TestCase):
+    def write_json(self, path: Path, value: object) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
+
+    def edition(self, root: Path, code: int, languages: dict[str, str]) -> None:
+        directory = root / "editions" / str(code)
+        self.write_json(
+            directory / "metadata.json",
+            {"Code": code, "Name": {"Rus": "Редакция", "Kyr": "Редакция"}, "Type": "edition", "Images": []},
+        )
+        for language, content in languages.items():
+            (directory / f"{language}.html").write_text(content, encoding="utf-8")
+
+    def document(self, root: Path, code: int = 1) -> Path:
+        directory = root / "documents" / str(code)
+        self.write_json(
+            directory / "metadata.json",
+            {
+                "Code": code,
+                "Class": {"Rus": "Акты", "Kyr": "Актылар"},
+                "Type": {"Rus": "Закон", "Kyr": "Мыйзам"},
+                "Title": {"Rus": " ", "Kyr": None},
+                "Name": {"Rus": "Документ", "Kyr": "Документ"},
+                "Status": {"Rus": "Действует", "Kyr": "Күчүндө"},
+                "Number": "1",
+                "DateAdopted": "2026-01-01",
+                "IsPublicInCdb": True,
+                "IsPublicInRegister": True,
+                "Authorities": [],
+                "SourcePublications": [],
+                "Keywords": [],
+                "GeneralClassifiers": [],
+                "References": [],
+            },
+        )
+        return directory
+
+    def test_languages_safety_empty_document_and_deterministic_fragments(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            source = base / "source"
+            output = base / "normalized"
+            document = self.document(source)
+            self.edition(
+                document,
+                10,
+                {
+                    "ru": '<meta charset=unicode><style>x</style><p style="color:red">Статья 1 Закон<script>bad()</script></p><a href="javascript:bad">ссылка</a><a href="http://[">сломанная ссылка</a><img src="http://[">',
+                },
+            )
+            self.edition(document, 20, {"ky": "<p>1. Кыргызча жобо</p>"})
+            self.edition(document, 30, {"ru": "<p>Русский</p>", "ky": "<p>Кыргызча</p>"})
+            self.edition(document, 40, {})
+
+            first = normalize_archive(source, output)
+            fragments_path = output / "documents/1/editions/10/ru/fragments.json"
+            fragments = fragments_path.read_bytes()
+            second = normalize_archive(source, output)
+
+            self.assertEqual((first.normalized, second.skipped), (1, 1))
+            self.assertEqual(fragments, fragments_path.read_bytes())
+            safe_html = (output / "documents/1/editions/10/ru/content.html").read_text(encoding="utf-8")
+            self.assertNotIn("script", safe_html)
+            self.assertNotIn("style=", safe_html)
+            self.assertNotIn("javascript:", safe_html)
+            self.assertNotIn("http://[", safe_html)
+            self.assertIn("Статья 1 Закон", safe_html)
+            parsed = json.loads(fragments)
+            self.assertEqual(parsed[0]["type"], "article")
+            self.assertEqual(parsed[0]["id"], "document:1:edition:10:lang:ru:fragment:1")
+            self.assertEqual(len(parsed[0]["text_sha256"]), 64)
+            canonical = json.loads((output / "documents/1/document.json").read_text(encoding="utf-8"))
+            self.assertEqual(canonical["available_languages"], ["ru", "ky"])
+            self.assertIsNone(canonical["title"]["ru"])
+            empty = json.loads((output / "documents/1/editions/40/edition.json").read_text(encoding="utf-8"))
+            self.assertFalse(empty["quality"]["has_html"])
+
+    def test_preserves_official_edition_order_instead_of_sorting_ids(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            source = base / "source"
+            output = base / "normalized"
+            document = self.document(source, 6)
+            self.edition(document, 920492, {"ru": "<p>Первая редакция</p>"})
+            self.edition(document, 920292, {"ru": "<p>Последняя редакция</p>"})
+            for code, order in ((920492, 0), (920292, 1)):
+                metadata_path = document / "editions" / str(code) / "metadata.json"
+                metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+                metadata["SourceOrder"] = order
+                metadata_path.write_text(json.dumps(metadata, ensure_ascii=False), encoding="utf-8")
+
+            normalize_archive(source, output)
+            normalized = json.loads((output / "documents/6/document.json").read_text(encoding="utf-8"))
+
+            self.assertEqual([item["source_code"] for item in normalized["editions"]], ["920492", "920292"])
+            self.assertEqual([item["source_order"] for item in normalized["editions"]], [0, 1])
+
+    def test_rejects_partial_source_edition_order(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            source = base / "source"
+            output = base / "normalized"
+            document = self.document(source, 6)
+            self.edition(document, 10, {"ru": "<p>Один</p>"})
+            self.edition(document, 20, {"ru": "<p>Два</p>"})
+            metadata_path = document / "editions/10/metadata.json"
+            metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+            metadata["SourceOrder"] = 0
+            metadata_path.write_text(json.dumps(metadata), encoding="utf-8")
+
+            with self.assertLogs("normalization.minjust_cbd", level="ERROR"):
+                result = normalize_archive(source, output)
+
+            self.assertEqual((result.failed, result.normalized), (1, 0))
+            with sqlite3.connect(output / "manifest.sqlite3") as connection:
+                error = connection.execute("SELECT error FROM documents WHERE code='6'").fetchone()[0]
+            self.assertIn("Invalid source edition order", error)
+
+    def test_continues_after_bad_document_and_clears_repaired_error(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            source = base / "source"
+            output = base / "normalized"
+            bad = source / "documents/1"
+            bad.mkdir(parents=True)
+            (bad / "metadata.json").write_text("not json", encoding="utf-8")
+            good = self.document(source, 2)
+            self.edition(good, 10, {"ky": "<p>Берене 1 Текст</p>"})
+
+            with self.assertLogs("normalization.minjust_cbd", level="ERROR"):
+                failed = normalize_archive(source, output)
+            with sqlite3.connect(output / "manifest.sqlite3") as connection:
+                state, failed_at = connection.execute(
+                    "SELECT state, failed_at FROM documents WHERE code='1'"
+                ).fetchone()
+                self.assertEqual(state, "error")
+                self.assertIsNotNone(failed_at)
+            self.write_json(bad / "metadata.json", {"Code": 1, "Name": {"Rus": "Исправлен", "Kyr": None}})
+            repaired = normalize_archive(source, output)
+
+            self.assertEqual((failed.failed, failed.normalized), (1, 1))
+            self.assertEqual((repaired.normalized, repaired.skipped, repaired.failed), (1, 1, 0))
+            with sqlite3.connect(output / "manifest.sqlite3") as connection:
+                self.assertEqual(
+                    connection.execute(
+                        "SELECT state, error, failed_at FROM documents WHERE code='1'"
+                    ).fetchone(),
+                    ("success", None, None),
+                )
+
+    def test_rejects_overlapping_input_and_output(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            source = Path(temporary) / "source"
+            metadata = self.document(source) / "metadata.json"
+            original = metadata.read_bytes()
+
+            for output in (source, source / "normalized", source.parent):
+                with self.subTest(output=output):
+                    with self.assertRaisesRegex(ValueError, "must not overlap"):
+                        normalize_archive(source, output)
+                    self.assertEqual(metadata.read_bytes(), original)
+
+    def test_recovers_interrupted_directory_publication_before_skip(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            base = Path(temporary)
+            source = base / "source"
+            output = base / "normalized"
+            self.document(source)
+            first = normalize_archive(source, output)
+            target = output / "documents/1"
+            backup = output / "documents/.1.previous"
+            os.replace(target, backup)
+
+            second = normalize_archive(source, output)
+
+            self.assertEqual((first.normalized, second.skipped), (1, 1))
+            self.assertTrue((target / "document.json").is_file())
+            self.assertFalse(backup.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_minjust_opensearch.py

@@ -0,0 +1,418 @@
+import hashlib
+import io
+import json
+import sqlite3
+import tempfile
+import unittest
+import urllib.error
+from contextlib import closing
+from pathlib import Path
+from unittest.mock import patch
+
+from search.catalog import legal_force_code, labels, source_code
+from search.minjust_opensearch import bulk_batches, document_codes, export_bulk, load_bulk, request_json, switch_alias
+
+
+class MinjustOpenSearchTest(unittest.TestCase):
+    def test_unknown_document_type_keeps_source_label_and_has_no_inferred_force(self):
+        value = {"ru": "Решение", "ky": "Решение"}
+
+        self.assertEqual(source_code("document_type", value), "other")
+        self.assertEqual(labels("document_type", "other"), {"ru": "Прочие документы", "ky": "Башка документтер"})
+        self.assertIsNone(legal_force_code(value))
+        self.assertEqual(source_code("status", {"ru": "Не вступил в силу", "ky": "Күчүнө кире элек"}), "not_yet_effective")
+        self.assertEqual(legal_force_code({"ru": "Конституционный закон", "ky": "Конституционный закон"}), "constitutional")
+        self.assertEqual(legal_force_code({"ru": "Указ", "ky": "Жарлыгы"}), "subordinate")
+        self.assertEqual(source_code("document_type", {"ru": "устав"}), "charter")
+
+    def test_export_marks_last_official_edition_current_even_when_ids_are_out_of_order(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            input_root = root / "normalized"
+            document_root = input_root / "documents/6"
+            document_root.mkdir(parents=True)
+            with closing(sqlite3.connect(input_root / "manifest.sqlite3")) as connection:
+                with connection:
+                    connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                    connection.execute("INSERT INTO documents VALUES ('6', 'success')")
+            (document_root / "document.json").write_text(
+                json.dumps({"schema_version": "1", "source_code": "6", "editions": [
+                    {"source_code": "920492", "source_order": 0},
+                    {"source_code": "920292", "source_order": 1},
+                ]}),
+                encoding="utf-8",
+            )
+            for edition in ("920292", "920492"):
+                text = f"Текст {edition}"
+                fragment_path = document_root / f"editions/{edition}/ru/fragments.json"
+                fragment_path.parent.mkdir(parents=True)
+                fragment_path.write_text(json.dumps([{
+                    "id": f"document:6:edition:{edition}:lang:ru:fragment:1",
+                    "document_code": "6",
+                    "edition_code": edition,
+                    "language": "ru",
+                    "position": 1,
+                    "type": "paragraph",
+                    "text": text,
+                    "text_sha256": hashlib.sha256(text.encode()).hexdigest(),
+                    "source_path": f"documents/6/editions/{edition}/ru.html",
+                    "source_sha256": "b" * 64,
+                }]), encoding="utf-8")
+
+            output = root / "bulk.ndjson"
+            self.assertEqual(export_bulk(input_root, output, "test-index"), (1, 2))
+            lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()]
+            flags = {
+                lines[index]["index"]["_id"].split(":")[3]: lines[index + 1]["is_current_edition"]
+                for index in range(0, len(lines), 2)
+            }
+
+            self.assertEqual(flags, {"920292": True, "920492": False})
+
+    def test_switches_alias_atomically_after_successful_load(self):
+        with patch("search.minjust_opensearch.request_json", return_value={"acknowledged": True}) as request:
+            switch_alias("http://127.0.0.1:9200/", "akyldash-fragments-v2", "akyldash-fragments-current")
+
+        self.assertEqual(request.call_args.args[:2], ("http://127.0.0.1:9200/_aliases", "POST"))
+        body = json.loads(request.call_args.args[2])
+        self.assertEqual(body["actions"][0], {"remove": {"index": "*", "alias": "akyldash-fragments-current", "must_exist": False}})
+        self.assertEqual(body["actions"][1], {"add": {"index": "akyldash-fragments-v2", "alias": "akyldash-fragments-current"}})
+
+        with self.assertRaisesRegex(ValueError, "differ"):
+            switch_alias("http://127.0.0.1:9200", "same", "same")
+        with self.assertRaisesRegex(ValueError, "differ"):
+            switch_alias("http://127.0.0.1:9200", "index", "")
+        with patch("search.minjust_opensearch.request_json", return_value={"acknowledged": False}):
+            with self.assertRaisesRegex(RuntimeError, "did not acknowledge"):
+                switch_alias("http://127.0.0.1:9200", "index", "alias")
+
+    def test_exports_atomic_bulk_and_rejects_mismatched_fragment(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            document_root = root / "normalized/documents/7"
+            document_root.mkdir(parents=True)
+            with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection:
+                with connection:
+                    connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                    connection.execute("INSERT INTO documents VALUES ('7', 'success')")
+                    connection.execute("INSERT INTO documents VALUES ('8', 'success')")
+            (document_root / "document.json").write_text(
+                json.dumps(
+                    {
+                        "schema_version": "1",
+                        "source_code": "7",
+                        "name": {"ru": "Закон", "ky": "Мыйзам"},
+                        "type": {"ru": "Закон", "ky": "Мыйзам"},
+                        "status": {"ru": "Действует", "ky": "Күчүндө"},
+                        "number": "1",
+                        "dates": {"DateAdopted": "2026-01-01"},
+                        "authority_paths": [{"ru": ["Кабинет"], "ky": ["Кабинет"]}],
+                    },
+                    ensure_ascii=False,
+                ),
+                encoding="utf-8",
+            )
+            empty = root / "normalized/documents/8"
+            empty.mkdir()
+            (empty / "document.json").write_text(
+                json.dumps({"schema_version": "1", "source_code": "8"}), encoding="utf-8"
+            )
+            for language, text in (("ru", "Текст \"RU\"\nстрока"), ("ky", "Кыргызча текст")):
+                path = document_root / f"editions/10/{language}/fragments.json"
+                path.parent.mkdir(parents=True)
+                path.write_text(
+                    json.dumps(
+                        [{
+                            "id": f"document:7:edition:10:lang:{language}:fragment:1",
+                            "document_code": "7",
+                            "edition_code": "10",
+                            "language": language,
+                            "position": 1,
+                            "type": "paragraph",
+                            "text": text,
+                            "text_sha256": hashlib.sha256(text.encode()).hexdigest(),
+                            "source_path": f"documents/7/editions/10/{language}.html",
+                            "source_sha256": "b" * 64,
+                        }],
+                        ensure_ascii=False,
+                    ),
+                    encoding="utf-8",
+                )
+
+            output = root / "bulk.ndjson"
+            self.assertEqual(export_bulk(root / "normalized", output, "test-index"), (2, 2))
+            content = output.read_bytes()
+            self.assertTrue(content.endswith(b"\n"))
+            lines = [json.loads(line) for line in content.splitlines()]
+            self.assertEqual(len(lines), 4)
+            self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1")
+            self.assertIn("text_ky", lines[1])
+            self.assertTrue(lines[1]["is_current_edition"])
+            self.assertNotIn("text_ru", lines[1])
+            self.assertEqual(lines[1]["legal_force_code"], "legislative")
+            self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока")
+            self.assertEqual(
+                list(bulk_batches(iter((("7", b"a\nb\n"), ("8", b"c\nd\n"))), 4)),
+                [("7", b"a\nb\n"), ("8", b"c\nd\n")],
+            )
+            self.assertEqual(document_codes(root / "normalized", 2, "8"), ["8"])
+            with self.assertRaisesRegex(ValueError, "selected range"):
+                document_codes(root / "normalized", 1, "8")
+
+            checkpoint = root / "checkpoint.json"
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {},
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                    {"errors": False, "items": [{"index": {}}, {"index": {}}]},
+                    {"acknowledged": True},
+                ]
+                self.assertEqual(
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    ),
+                    (2, 2),
+                )
+                self.assertEqual(request.call_args_list[-2].args[3], "application/x-ndjson")
+                self.assertEqual(request.call_args_list[-1].args[:2], ("http://127.0.0.1:9200/_aliases", "POST"))
+            state = json.loads(checkpoint.read_text(encoding="utf-8"))
+            self.assertEqual(state["last_document_code"], "7")
+            self.assertTrue(state["complete"])
+
+            legacy = state.copy()
+            legacy.pop("alias")
+            legacy["schema_version"] = 1
+            legacy["last_document_code"] = "8"
+            legacy["complete"] = False
+            checkpoint.write_text(json.dumps(legacy), encoding="utf-8")
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                self.assertEqual(
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                    ),
+                    (1, 0),
+                )
+            self.assertEqual(json.loads(checkpoint.read_text(encoding="utf-8"))["schema_version"], 2)
+
+            state["last_document_code"] = "8"
+            state["complete"] = False
+            checkpoint.write_text(json.dumps(state), encoding="utf-8")
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-2"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                with self.assertRaisesRegex(ValueError, "does not match"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    )
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                with self.assertRaisesRegex(ValueError, "alias"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                    )
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                with self.assertRaisesRegex(ValueError, "limit"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        limit=1,
+                        resume=True,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    )
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                    {"acknowledged": True},
+                ]
+                self.assertEqual(
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    ),
+                    (1, 0),
+                )
+                self.assertTrue(all(call.args[1] == "GET" for call in request.call_args_list[:-1]))
+                self.assertEqual(request.call_args_list[-1].args[1], "POST")
+
+            state["last_document_code"] = "9"
+            state["complete"] = False
+            checkpoint.write_text(json.dumps(state), encoding="utf-8")
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                with self.assertRaisesRegex(ValueError, "Resume document not found"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    )
+
+            failed_checkpoint = root / "failed-checkpoint.json"
+            failure = {
+                "errors": True,
+                "items": [{"index": {"_id": "bad-id", "error": {"type": "mapper", "reason": "bad value"}}}],
+            }
+            failed_requests = [
+                {},
+                {"cluster_uuid": "cluster-1"},
+                {"failed-index": {"settings": {"index": {"uuid": "index-2"}}}},
+                failure,
+            ]
+            with patch("search.minjust_opensearch.request_json", side_effect=failed_requests):
+                with self.assertRaisesRegex(RuntimeError, "bad-id: mapper: bad value"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "failed-index",
+                        maximum_bytes=4096,
+                        checkpoint=failed_checkpoint,
+                    )
+            failed_state = json.loads(failed_checkpoint.read_text(encoding="utf-8"))
+            self.assertIsNone(failed_state["last_document_code"])
+            self.assertFalse(failed_state["complete"])
+
+            state["last_document_code"] = "8"
+            state["complete"] = False
+            checkpoint.write_text(json.dumps(state), encoding="utf-8")
+            with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection:
+                with connection:
+                    connection.execute("INSERT INTO documents VALUES ('9', 'success')")
+            with patch("search.minjust_opensearch.request_json") as request:
+                request.side_effect = [
+                    {"cluster_uuid": "cluster-1"},
+                    {"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
+                ]
+                with self.assertRaisesRegex(ValueError, "manifest changed"):
+                    load_bulk(
+                        root / "normalized",
+                        "http://127.0.0.1:9200",
+                        "test-index",
+                        maximum_bytes=4096,
+                        resume=True,
+                        checkpoint=checkpoint,
+                        alias="test-current",
+                    )
+
+            http_error = urllib.error.HTTPError(
+                "http://127.0.0.1:9200/test",
+                429,
+                "busy",
+                {},
+                io.BytesIO(b"busy"),
+            )
+            with (
+                patch("search.minjust_opensearch.urllib.request.urlopen", side_effect=[http_error, io.BytesIO(b"{}")]),
+                patch("search.minjust_opensearch.time.sleep") as sleep,
+            ):
+                self.assertEqual(
+                    request_json("http://127.0.0.1:9200/test", "GET", None, "application/json", attempts=2),
+                    {},
+                )
+                sleep.assert_called_once_with(1)
+
+            with (
+                patch(
+                    "search.minjust_opensearch.urllib.request.urlopen",
+                    side_effect=[io.BytesIO(b"{"), io.BytesIO(b"{}")],
+                ),
+                patch("search.minjust_opensearch.time.sleep") as sleep,
+            ):
+                self.assertEqual(
+                    request_json(
+                        "http://127.0.0.1:9200/_bulk",
+                        "POST",
+                        b"{}\n{}\n",
+                        "application/x-ndjson",
+                        attempts=2,
+                        retry_invalid_json=True,
+                    ),
+                    {},
+                )
+                sleep.assert_called_once_with(1)
+
+            bad = document_root / "editions/10/ru/fragments.json"
+            fragments = json.loads(bad.read_text(encoding="utf-8"))
+            fragments[0]["document_code"] = "8"
+            bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "identity"):
+                export_bulk(root / "normalized", output, "test-index")
+            self.assertEqual(output.read_bytes(), content)
+
+            fragments[0]["document_code"] = "7"
+            fragments[0]["text"] = "Повреждено"
+            bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "checksum"):
+                export_bulk(root / "normalized", output, "test-index")
+            with self.assertRaisesRegex(ValueError, "inside --input"):
+                export_bulk(root / "normalized", root / "normalized/manifest.sqlite3", "test-index")
+            self.assertEqual(output.read_bytes(), content)
+
+            bad.write_text("", encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "fragments.json"):
+                export_bulk(root / "normalized", output, "test-index")
+            self.assertEqual(output.read_bytes(), content)
+
+            definition = json.loads(
+                (Path(__file__).parent / "search/minjust-fragments-index.json").read_text(encoding="utf-8")
+            )
+            mapping = definition["mappings"]
+            self.assertEqual(definition["settings"]["index"]["number_of_shards"], 1)
+            self.assertEqual(definition["settings"]["index"]["number_of_replicas"], 0)
+            self.assertEqual(mapping["dynamic"], "strict")
+            self.assertEqual(mapping["properties"]["position"]["type"], "integer")
+            self.assertEqual(mapping["properties"]["text_ru"]["analyzer"], "russian")
+            self.assertEqual(mapping["properties"]["text_ky"]["analyzer"], "icu_analyzer")
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_search_api.py

@@ -0,0 +1,132 @@
+import json
+import contextlib
+import io
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from search.api import Api, ApiError
+
+
+class SearchApiTest(unittest.TestCase):
+    def make_api(self, root: Path) -> Api:
+        document = root / "documents/7"
+        edition = document / "editions/10"
+        edition.mkdir(parents=True)
+        (document / "document.json").write_text(json.dumps({
+            "source_code": "7", "available_languages": ["ru"],
+            "editions": [{"source_code": "10", "available_languages": ["ru"]},],
+        }), encoding="utf-8")
+        (edition / "edition.json").write_text(json.dumps({"source_code": "10", "available_languages": ["ru"]}), encoding="utf-8")
+        (edition / "ru").mkdir()
+        (edition / "ru/content.html").write_text("<p>Текст</p>", encoding="utf-8")
+        (edition / "ru/content.txt").write_text("Текст\n", encoding="utf-8")
+        return Api("http://opensearch:9200", "current", root)
+
+    def test_search_pagination_filters_and_highlight(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["<em>Закон</em>"]}}]}}
+            with patch("search.api.request_json", return_value=response) as request:
+                status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=active")
+            self.assertEqual(status, 200)
+            self.assertEqual(payload["results"][0]["snippet"], "Закон")
+            body = json.loads(request.call_args.args[2])
+            self.assertEqual((body["from"], body["size"]), (5, 6))
+            self.assertIn({"term": {"status_code": "active"}}, body["query"]["bool"]["filter"])
+            with self.assertRaisesRegex(ApiError, "within 10000 results"):
+                api.handle("GET", "/search?q=x&page=100&page_size=100")
+
+    def test_review_secret_file_is_read_without_command_line_value(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.write_text("secret-value\n", encoding="utf-8")
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), patch("search.api.Api") as api, patch("search.api.ThreadingHTTPServer"):
+                main()
+            self.assertEqual(api.call_args.args[4], b"secret-value")
+
+    def test_review_secret_file_rejects_empty_content(self):
+        from search.api import main
+
+        with tempfile.TemporaryDirectory() as temporary:
+            secret_file = Path(temporary) / "secret"
+            secret_file.touch()
+            with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), contextlib.redirect_stderr(io.StringIO()):
+                with self.assertRaises(SystemExit) as result:
+                    main()
+            self.assertEqual(result.exception.code, 2)
+
+    def test_production_api_uses_registration_intent_boosts(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            response = {"hits": {"hits": []}}
+            with patch("search.api.request_json", return_value=response) as request:
+                api.handle("GET", "/search?q=%D0%BA%D0%B0%D0%BA+%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D1%8C+%D0%9E%D1%81%D0%9E%D0%9E&language=ru")
+            body = json.loads(request.call_args.args[2])
+            planned = body["query"]["bool"]["must"]["bool"]
+            self.assertEqual(planned["minimum_should_match"], 1)
+            self.assertNotIn("filter", planned)
+            self.assertTrue(any("constant_score" in clause for clause in planned["should"]))
+            self.assertTrue(any("общество с ограниченной ответственностью" in clause.get("multi_match", {}).get("query", "") for clause in planned["should"]))
+
+    def test_production_api_boosts_tax_code_for_profit_tax_query(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            with patch("search.api.request_json", return_value={"hits": {"hits": []}}) as request:
+                api.handle("GET", "/search?q=%D0%BD%D0%B0%D0%BB%D0%BE%D0%B3+%D0%BD%D0%B0+%D0%BF%D1%80%D0%B8%D0%B1%D1%8B%D0%BB%D1%8C&language=ru")
+            body = json.loads(request.call_args.args[2])
+            planned = body["query"]["bool"]["must"]["bool"]
+            anchors = [clause["constant_score"]["filter"]["bool"]["filter"][0] for clause in planned["should"] if "constant_score" in clause]
+            self.assertIn({"term": {"document_code": "112340"}}, anchors)
+
+    def test_search_sorts_repealed_documents_after_other_statuses(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            with patch("search.api.request_json", return_value={"hits": {"hits": []}}) as request:
+                api.handle("GET", "/search?q=НДС+на+импорт&language=ru")
+                body = json.loads(request.call_args.args[2])
+                source = body["sort"][0]["_script"]["script"]["source"]
+                self.assertIn("'active' ? 0", source)
+                self.assertIn("'repealed' ? 2", source)
+                self.assertEqual(body["sort"][1], {"_score": "desc"})
+
+                api.handle("GET", "/search?q=НДС+на+импорт&language=ru&sort=date")
+                body = json.loads(request.call_args.args[2])
+                self.assertEqual(body["sort"][1], {"date_adopted": "desc"})
+
+    def test_document_editions_openapi_and_validation(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            specification = api.handle("GET", "/openapi.json")[1]
+            self.assertEqual(specification["info"]["version"], "v1")
+            self.assertEqual(specification["paths"]["/documents/{code}"]["get"]["parameters"][0]["required"], True)
+            self.assertEqual(specification["paths"]["/documents/{code}/editions/{edition}"]["get"]["parameters"][1]["name"], "edition")
+            self.assertEqual(api.handle("GET", "/documents/7")[1]["current_edition"]["source_code"], "10")
+            self.assertEqual(api.handle("GET", "/documents/7/editions/10?language=ru")[1]["content"]["ru"]["text"], "Текст\n")
+            with self.assertRaisesRegex(ApiError, "q is required"):
+                api.handle("GET", "/search")
+            with self.assertRaisesRegex(ApiError, "date_from must be an ISO date"):
+                api.handle("GET", "/search?q=x&date_from=tomorrow")
+            with self.assertRaisesRegex(ApiError, "document not found"):
+                api.handle("GET", "/documents/%2E%2E")
+
+    def test_filters_count_documents_and_return_bilingual_labels(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = self.make_api(Path(temporary))
+            keys = {"document_types": "law", "statuses": "active", "authorities": "parliament", "legal_forces": "legislative"}
+            response = {"aggregations": {name: {"buckets": [{"key": keys[name], "documents": {"value": 3}}]} for name in keys}}
+            with patch("search.api.request_json", return_value=response) as request:
+                payload = api.handle("GET", "/search/filters?language=ky")[1]
+            self.assertEqual(payload["document_types"][0], {"code": "law", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
+            self.assertEqual(payload["legal_forces"][0]["labels"]["ky"], "Мыйзам деңгээли")
+            body = json.loads(request.call_args.args[2])
+            self.assertIn("terms", body["aggs"]["document_types"])
+            self.assertIn("legal_force_code", body["aggs"]["legal_forces"]["terms"]["field"])
+            self.assertEqual(body["query"], {"term": {"is_current_edition": True}})
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_search_api_opensearch.py

@@ -0,0 +1,41 @@
+import json
+import os
+import unittest
+import uuid
+from pathlib import Path
+
+from search.api import Api
+from search.minjust_opensearch import DEFAULT_MAPPING, request_json
+
+
+@unittest.skipUnless(os.getenv("AKYLDASH_OPENSEARCH_URL"), "set AKYLDASH_OPENSEARCH_URL to run against local OpenSearch")
+class SearchApiOpenSearchTest(unittest.TestCase):
+    def test_current_editions_filters_and_catalogs(self):
+        base_url = os.environ["AKYLDASH_OPENSEARCH_URL"].rstrip("/")
+        index = f"akyldash-api-test-{uuid.uuid4().hex}"
+        request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json")
+        try:
+            documents = [
+                {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "legal_force_code": "legislative", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "status_code": "repealed", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
+                {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "legal_force_code": "legislative", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
+                {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "document_type_code": "decree", "legal_force_code": "subordinate", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"], "authority_codes": ["president"]},
+            ]
+            for number, document in enumerate(documents):
+                request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json")
+            request_json(f"{base_url}/{index}/_refresh", "POST", None, "application/json")
+            api = Api(base_url, index, Path("."))
+            self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], [])
+            filtered = api.handle("GET", "/search?q=needle&document_type=law")[1]
+            self.assertEqual([result["code"] for result in filtered["results"]], ["1"])
+            force_filtered = api.handle("GET", "/search?q=needle&legal_force=subordinate")[1]
+            self.assertEqual([result["code"] for result in force_filtered["results"]], ["2"])
+            self.assertEqual(force_filtered["results"][0]["legal_force"], "Подзаконный уровень")
+            filters = api.handle("GET", "/search/filters")[1]
+            self.assertEqual({item["count"] for item in filters["statuses"]}, {2})
+            self.assertEqual({item["code"] for item in filters["legal_forces"]}, {"legislative", "subordinate"})
+        finally:
+            request_json(f"{base_url}/{index}", "DELETE", None, "application/json")
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_search_relevance.py

@@ -0,0 +1,172 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from search.evaluate_relevance import evaluate, load_queries
+from search.query import COMPANY_REGISTRATION_DOCUMENTS, TAX_PROFIT_DOCUMENTS, build_search_body, query_variants
+
+
+class SearchRelevanceTest(unittest.TestCase):
+    def test_loads_queries_and_calculates_document_metrics(self):
+        queries = [
+            {
+                "id": "ru-01",
+                "language": "ru",
+                "query": "трудовой договор",
+                "relevant_document_codes": ["7", "8"],
+            },
+            {
+                "id": "ky-01",
+                "language": "ky",
+                "query": "эмгек келишими",
+                "relevant_document_codes": ["9"],
+            },
+        ]
+        with tempfile.TemporaryDirectory() as temporary:
+            path = Path(temporary) / "queries.json"
+            path.write_text(json.dumps(queries, ensure_ascii=False), encoding="utf-8")
+            loaded = load_queries(path)
+
+        with patch("search.evaluate_relevance.search_documents", side_effect=[["7", "10", "8"], ["10", "9"]]):
+            result = evaluate(loaded, "http://127.0.0.1:9200", "test", 10)
+
+        self.assertEqual(result["summary"], {"query_count": 2, "recall_at_10": 1.0, "mrr_at_10": 0.75})
+        self.assertEqual(result["queries"][0]["reciprocal_rank_at_10"], 1.0)
+        body = json.loads(build_search_body("ru", "трудовой договор", 10))
+        self.assertFalse(body["track_total_hits"])
+        self.assertEqual(body["collapse"], {"field": "document_code"})
+        self.assertIn({"term": {"is_current_edition": True}}, body["query"]["bool"]["filter"])
+        lexical_query = body["query"]["bool"]["must"]["bool"]["should"][0]["multi_match"]
+        self.assertEqual(lexical_query["type"], "cross_fields")
+        self.assertEqual(lexical_query["fields"], ["document_name_ru^4", "text_ru"])
+        self.assertEqual(lexical_query["operator"], "AND")
+        self.assertTrue(any("match_phrase" in clause for clause in body["query"]["bool"]["must"]["bool"]["should"]))
+
+    def test_legal_form_abbreviations_expand_in_both_directions(self):
+        self.assertIn("как открыть общество с ограниченной ответственностью", query_variants("ru", "как открыть ОсОО"))
+        self.assertIn("регистрация осоо", [variant.casefold() for variant in query_variants("ru", "регистрация общества с ограниченной ответственностью")])
+        self.assertIn("жчк ачуу тартиби", [variant.casefold() for variant in query_variants("ky", "жоопкерчилиги чектелген коом ачуу тартиби")])
+
+    def test_relaxed_match_drops_ru_template_prefix_and_stays_low_weight(self):
+        body = json.loads(build_search_body("ru", "правовое регулирование расторжение договора аренды", 10))
+        clauses = body["query"]["bool"]["must"]["bool"]["should"]
+        strict = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("operator") == "AND"]
+        relaxed = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("boost") == 0.1]
+        self.assertEqual(strict[0]["query"], "правовое регулирование расторжение договора аренды")
+        self.assertEqual(relaxed[0]["query"], "расторжение договора аренды")
+        self.assertEqual(relaxed[0]["operator"], "OR")
+        self.assertEqual(relaxed[0]["minimum_should_match"], 2)
+        self.assertNotIn("boost", strict[0])
+
+    def test_relaxed_match_supports_kyrgyz_and_avoids_short_named_acts(self):
+        ky = json.loads(build_search_body("ky", "эмгек келишимин бузуу шарттары", 10))
+        ky_clauses = ky["query"]["bool"]["must"]["bool"]["should"]
+        self.assertTrue(any(item.get("multi_match", {}).get("operator") == "OR" and item["multi_match"].get("boost") == 0.1 for item in ky_clauses))
+
+        named_act = json.loads(build_search_body("ru", "Гражданский кодекс", 10))
+        act_clauses = named_act["query"]["bool"]["must"]["bool"]["should"]
+        self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in act_clauses))
+
+        long_named_act = json.loads(build_search_body("ru", "последние изменения Гражданского кодекса Кыргызской Республики", 10))
+        long_act_clauses = long_named_act["query"]["bool"]["must"]["bool"]["should"]
+        self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in long_act_clauses))
+
+        short_template = json.loads(build_search_body("ru", "как регулируется взыскание алиментов", 10))
+        short_clauses = short_template["query"]["bool"]["must"]["bool"]["should"]
+        short_fallback = next(item["multi_match"] for item in short_clauses if item.get("multi_match", {}).get("boost") == 0.1)
+        self.assertEqual(short_fallback["query"], "взыскание алиментов")
+        self.assertEqual(short_fallback["minimum_should_match"], 2)
+
+        stopword_query = json.loads(build_search_body("ru", "правовое регулирование права и обязанности", 10))
+        stopword_clauses = stopword_query["query"]["bool"]["must"]["bool"]["should"]
+        stopword_fallback = next(item["multi_match"] for item in stopword_clauses if item.get("multi_match", {}).get("boost") == 0.1)
+        self.assertEqual(stopword_fallback["minimum_should_match"], 2)
+
+        for language, query in (("ru", "последние изменения ГК КР"), ("ky", "Кыргыз Республикасынын Конституциясы")):
+            act_body = json.loads(build_search_body(language, query, 10))
+            act_clauses = act_body["query"]["bool"]["must"]["bool"]["should"]
+            self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in act_clauses))
+
+        for language, query, expected in (
+            ("ru", "Налоговый кодекс ответственность за неуплату налога", "ответственность за неуплату налога"),
+            ("ru", "Налоговый кодекс налоговая ставка", "налоговая ставка"),
+            ("ru", "Налоговый кодекс: налоговая ставка", "налоговая ставка"),
+            ("ru", "закон о труде порядок выплаты зарплаты", "порядок выплаты зарплаты"),
+            ("ky", "Кыргыз Республикасынын Конституциясы укуктарды чектөө тартиби", "укуктарды чектөө тартиби"),
+        ):
+            body = json.loads(build_search_body(language, query, 10))
+            clauses = body["query"]["bool"]["must"]["bool"]["should"]
+            fallback = next(item["multi_match"] for item in clauses if item.get("multi_match", {}).get("boost") == 0.1)
+            self.assertEqual(fallback["query"], expected)
+
+        for query in (
+            "Закон о защите прав потребителей",
+            "Закон Кыргызской Республики о защите прав потребителей",
+            "Закон Кыргызской Республики о государственной регистрации юридических лиц",
+        ):
+            body = json.loads(build_search_body("ru", query, 10))
+            clauses = body["query"]["bool"]["must"]["bool"]["should"]
+            self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in clauses))
+
+        consumer_rights = json.loads(build_search_body("ru", "Закон о защите прав потребителей возврат денег", 10))
+        consumer_clauses = consumer_rights["query"]["bool"]["must"]["bool"]["should"]
+        consumer_fallback = next(item["multi_match"] for item in consumer_clauses if item.get("multi_match", {}).get("boost") == 0.1)
+        self.assertEqual(consumer_fallback["query"], "возврат денег")
+
+    def test_company_registration_intent_boosts_current_documents(self):
+        for language, query, status in (
+            ("ru", "как открыть ОсОО", "Действует"),
+            ("ky", "ЖЧК ачуу тартиби", "Күчүндө"),
+        ):
+            body = json.loads(build_search_body(language, query, 10))
+            search_query = body["query"]["bool"]["must"]["bool"]
+            self.assertEqual(search_query["minimum_should_match"], 1)
+            self.assertNotIn("filter", search_query)
+            lexical = [clause["multi_match"] for clause in search_query["should"] if "multi_match" in clause]
+            self.assertTrue(all(clause["type"] == "cross_fields" and clause["operator"] == "OR" for clause in lexical))
+            boosts = [clause["constant_score"] for clause in search_query["should"] if "constant_score" in clause]
+            self.assertEqual([item["boost"] for item in boosts], [boost for _, boost in COMPANY_REGISTRATION_DOCUMENTS])
+            self.assertEqual(
+                [item["filter"]["bool"]["filter"][0]["term"]["document_code"] for item in boosts],
+                [code for code, _ in COMPANY_REGISTRATION_DOCUMENTS],
+            )
+            self.assertTrue(all(item["filter"]["bool"]["filter"][1] == {"term": {f"status_{language}": status}} for item in boosts))
+
+    def test_profit_tax_intent_prioritizes_the_current_tax_code(self):
+        for query in ("налог на прибыль", "налога на прибыль организаций", "налогообложение прибыли"):
+            body = json.loads(build_search_body("ru", query, 10))
+            clauses = body["query"]["bool"]["must"]["bool"]["should"]
+            boosts = [clause["constant_score"] for clause in clauses if "constant_score" in clause]
+            self.assertEqual([item["boost"] for item in boosts], [boost for _, boost in TAX_PROFIT_DOCUMENTS])
+            self.assertEqual([item["filter"]["bool"]["filter"][0]["term"]["document_code"] for item in boosts], ["112340"])
+            self.assertTrue(all(item["filter"]["bool"]["filter"][1] == {"term": {"status_ru": "Действует"}} for item in boosts))
+
+        for query in ("прибыль организации", "налог на доходы", "налог на имущество"):
+            body = json.loads(build_search_body("ru", query, 10))
+            clauses = body["query"]["bool"]["must"]["bool"]["should"]
+            self.assertFalse(any("constant_score" in clause for clause in clauses))
+
+    def test_company_registration_intent_ignores_non_procedural_queries(self):
+        for language, query in (
+            ("ru", "ОсОО зарегистрирован?"),
+            ("ru", "кто зарегистрировал ОсОО"),
+            ("ru", "как открыть счет ОсОО"),
+            ("ru", "как открыть ОсОО банковский счет"),
+            ("ru", "как открыть ОсОО банковский счёт"),
+            ("ru", "как открыть филиал ОсОО"),
+            ("ru", "как создать договор для ОсОО"),
+            ("ru", "порядок создания логотипа ОсОО"),
+            ("ky", "ЖЧК ачык маалымат"),
+            ("ky", "ЖЧК кантип банк эсебин ачуу"),
+            ("ky", "ЖЧК кантип келишим түзүү"),
+            ("ky", "ЖЧК кантип логотип түзүү"),
+        ):
+            body = json.loads(build_search_body(language, query, 10))
+            clauses = body["query"]["bool"]["must"]["bool"]["should"]
+            self.assertFalse(any("constant_score" in clause for clause in clauses))
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_search_reviews.py

@@ -0,0 +1,46 @@
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from search.api import Api, ApiError
+
+
+class SearchReviewTest(unittest.TestCase):
+    def test_review_must_cover_each_snapshot_rank_once(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret")
+            response = {"hits": {"hits": [{"_index": "search-20260827", "_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}, {"_index": "search-20260827", "_source": {"document_code": "8", "edition_code": "11", "document_name_ru": "Кодекс"}}]}}
+            with patch("search.api.request_json", return_value=response):
+                result = api.handle("GET", "/search?q=test&language=ru&page_size=2")[1]
+            with self.assertRaisesRegex(ApiError, "exactly once"):
+                api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3}]})
+
+    def test_saves_signed_search_snapshot_and_rejects_tampering(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret")
+            response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}]}}
+            with patch("search.api.request_json", return_value=response):
+                result = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru")[1]
+            saved = api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3, "comment": "Прямой ответ"}]})
+            self.assertEqual(saved[0], 201)
+            exported = api.handle("GET", "/search-reviews/export")[1]["reviews"]
+            self.assertEqual(exported[0]["top_result_code"], "7")
+            self.assertEqual(exported[0]["results"][0]["rating"], 3)
+            with self.assertRaisesRegex(ApiError, "does not match"):
+                api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "8", "rating": 3}]})
+
+    def test_snapshot_and_review_page_are_available(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            api = Api("http://opensearch:9200", "current", Path(temporary))
+            page = api.review_page()
+            self.assertIn("Оценка поисковой выдачи", page)
+            self.assertIn("akyldash-search-review-draft", page)
+            self.assertIn("tutorial-dialog", page)
+            self.assertIn("Как это работает", page)
+            self.assertNotIn('id="error"', page)
+            self.assertIn("localStorage", page)
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/test_search_topic_taxonomy.py

@@ -0,0 +1,102 @@
+import json
+import sqlite3
+import tempfile
+import unittest
+from contextlib import closing
+from pathlib import Path
+
+from tools.build_search_topic_taxonomy import build, group_for
+
+
+class SearchTopicTaxonomyTest(unittest.TestCase):
+    def test_ambiguous_roots_are_left_for_manual_review(self):
+        self.assertEqual(group_for("Иностранные инвестиции"), "review_required")
+
+    def test_preserves_source_tree_counts_and_marks_unclassified_roots(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            (root / "documents/1").mkdir(parents=True)
+            (root / "documents/2").mkdir(parents=True)
+            with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
+                db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                db.executemany("INSERT INTO documents VALUES (?, 'success')", [("1",), ("2",)])
+                db.commit()
+            classifier = {
+                "Code": None,
+                "Name": {"Rus": "ЗАКОНОДАТЕЛЬСТВО О ТРУДЕ", "Kyr": None},
+                "GeneralClassifiers": [{
+                    "Code": "child-1",
+                    "Name": {"Rus": "Рабочее время", "Kyr": "Иш убактысы"},
+                    "GeneralClassifiers": [],
+                }],
+            }
+            unknown = {
+                "Code": None,
+                "Name": {"Rus": "НЕИЗВЕСТНАЯ РУБРИКА", "Kyr": None},
+                "GeneralClassifiers": [],
+            }
+            (root / "documents/1/document.json").write_text(
+                json.dumps({"general_classifiers": [classifier]}, ensure_ascii=False),
+                encoding="utf-8",
+            )
+            (root / "documents/2/document.json").write_text(
+                json.dumps({"general_classifiers": [classifier, unknown]}, ensure_ascii=False),
+                encoding="utf-8",
+            )
+
+            catalog = build(root)
+            groups = {group["code"]: group for group in catalog["groups"]}
+            labor_root = groups["labor_social"]["children"][0]
+
+            self.assertEqual((catalog["source_document_count"], catalog["source_node_count"], catalog["source_root_count"]), (2, 3, 2))
+            self.assertEqual(labor_root["document_count"], 2)
+            self.assertEqual(labor_root["children"][0]["document_count"], 2)
+            self.assertEqual(groups["review_required"]["children"][0]["grouping_status"], "manual_review")
+            self.assertEqual(catalog["nodes_missing_kyrgyz_label"], 2)
+
+    def test_rebuild_fails_instead_of_omitting_successful_manifest_document(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            (root / "documents/1").mkdir(parents=True)
+            with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
+                db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                db.execute("INSERT INTO documents VALUES ('1', 'success')")
+                db.commit()
+
+            with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
+                build(root)
+
+    def test_rebuild_rejects_normalized_document_without_classifier_list(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            document = root / "documents/1"
+            document.mkdir(parents=True)
+            with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
+                db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                db.execute("INSERT INTO documents VALUES ('1', 'success')")
+                db.commit()
+            (document / "document.json").write_text("{}", encoding="utf-8")
+
+            with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
+                build(root)
+
+    def test_rebuild_rejects_invalid_nested_classifier_branch(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            document = root / "documents/1"
+            document.mkdir(parents=True)
+            with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
+                db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
+                db.execute("INSERT INTO documents VALUES ('1', 'success')")
+                db.commit()
+            (document / "document.json").write_text(
+                json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
+                encoding="utf-8",
+            )
+
+            with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
+                build(root)
+
+
+if __name__ == "__main__":
+    unittest.main()

deploy/local-opensearch/Dockerfile

@@ -0,0 +1,3 @@
+FROM opensearchproject/opensearch:3.7.0
+
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu

deploy/local-opensearch/compose.yaml

@@ -0,0 +1,21 @@
+services:
+  opensearch:
+    build: .
+    container_name: akyldash-opensearch
+    environment:
+      discovery.type: single-node
+      bootstrap.memory_lock: "true"
+      DISABLE_SECURITY_PLUGIN: "true"
+      OPENSEARCH_JAVA_OPTS: -Xms8g -Xmx8g
+    mem_limit: 12g
+    ports:
+      - 127.0.0.1:9200:9200
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - ../../data/opensearch-node:/usr/share/opensearch/data

deploy/production/.env.example

@@ -0,0 +1,9 @@
+# Absolute path on the production host containing minjust-normalized/.
+DATA_ROOT=/var/lib/akyldash/data
+REVIEW_SECRET_FILE=/etc/akyldash/review_secret
+BACKEND_IMAGE=akyldash/backend:local
+OPENSEARCH_IMAGE=akyldash/opensearch:local
+SEARCH_INDEX=akyldash-fragments-current
+OPENSEARCH_MEM_LIMIT=4g
+OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
+COOKIE_SECURE=true

deploy/production/Caddyfile

@@ -0,0 +1,4 @@
+akyldash.kg {
+	encode zstd gzip
+	reverse_proxy backend:8080
+}

deploy/production/Dockerfile.backend

@@ -0,0 +1,11 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+COPY backend /app/backend
+
+ENV PYTHONPATH=/app/backend
+EXPOSE 8080
+
+CMD ["python", "-m", "search.api", "--host", "0.0.0.0", "--port", "8080", "--url", "http://opensearch:9200", "--index", "akyldash-fragments-current", "--data", "/app/data/minjust-normalized", "--reviews-db", "/app/data/search-reviews.sqlite3"]
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/review', timeout=3)"]

deploy/production/Dockerfile.opensearch

@@ -0,0 +1,3 @@
+FROM opensearchproject/opensearch:3.7.0
+
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu

deploy/production/README.md

@@ -0,0 +1,112 @@
+# Production deployment
+
+Compose runs Caddy, the Search API, and a private OpenSearch node. Only Caddy
+publishes ports `80` and `443`; backend and OpenSearch have no host ports. The
+OpenSearch security plugin is disabled to match the current API client, so keep
+it on the private network.
+
+## Host access and configuration
+
+The production environment file is `/etc/akyldash/production.env`; the review
+secret is `/etc/akyldash/review_secret`. Both must stay outside the checkout,
+root-owned, and mode `0600`. `DATA_ROOT` must be an absolute path. The `codex`
+account is intentionally outside the `sudo` and `docker` groups because access
+to the Docker socket grants root-equivalent control. Run host operations with
+the narrowly required `sudo` command; do not use `docker` as `codex` directly.
+
+After reviewing the host audit and confirming these files do not already exist,
+the owner can create them from the approved checkout with
+`sudo bash infra/deploy/prepare-production-config.sh`. The script refuses to
+overwrite existing configuration, installs the tracked env example with mode
+`0600`, and generates a random review secret with mode `0600`. It does not
+start Compose or create volumes. Review and set `DATA_ROOT` in the protected
+env file before deployment.
+
+Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus
+is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in
+[`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the
+root bootstrap installs packages and creates directories, but does not deploy
+the application or alter SSH, firewall, user groups, or sudoers. The restricted
+`akyldash-compose` helper allows only root-owned Compose `config` and `status`;
+service start/stop remains an owner-run root operation.
+
+The repository Gitea Actions runner is an isolated CI worker described in
+[`docs/operations/CI_RUNNER.md`](../../docs/operations/CI_RUNNER.md). It has no
+production Docker socket, production env files, or application checkout. CI
+does not deploy or change production; all deployment remains owner-run.
+
+## First deployment and upgrades
+
+Run commands from the root-owned checkout at `/opt/akyldash/current`. Render
+and review Compose before changing services:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml config --quiet
+```
+
+On a clean first deployment, `infra/backup/archive.py create` reports that no
+backup is required when all persistent mounts are empty and no Compose
+containers exist. Continue without a backup in that case. If any persistent
+data exists, create and verify a backup before changing schema, images, or
+starting a new application version. This also applies when containers are
+absent, including an initial installation where the normalized corpus was
+preloaded under `DATA_ROOT`; create the archive before the first `up`. When
+named volumes do not exist, pass `--initial-deployment`; this is accepted only
+without Compose containers and when `DATA_ROOT` contains only
+`minjust-normalized/`. During upgrades, missing volumes are an error because
+they can indicate lost production data. The archive tool also refuses
+ambiguous storage state, running containers, and incomplete Compose service
+containers. See
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
+commands. Do not remove an older usable backup to make room; retain the
+pre-change archive until the new deployment and a newer verified backup are
+confirmed.
+
+For an initial clean installation, start the stack only after config review:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d --build
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml ps
+curl -fsS https://akyldash.kg/review >/dev/null
+```
+
+Before this step, confirm DNS and have the owner review firewall rules for TCP
+`80` and `443`. Caddy manages the origin certificate; confirm HTTPS before
+setting Cloudflare to Full (strict). On the initial start, create the `admin`
+account interactively:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --create-admin
+```
+
+The account database is `$DATA_ROOT/search-users.sqlite3` and is included in
+the `data` backup. Admin password reset is also interactive:
+
+```bash
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
+```
+
+After the initial OpenSearch index is loaded and validated, the `admin` account
+is created, and `/review` works over HTTPS, create and verify the required
+post-setup baseline backup before treating deployment as complete. It captures
+the initialized index, account database, and Caddy certificate data. Follow
+the exact stop/create/verify/start sequence in
+[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md#initial-post-setup-baseline).
+
+## Backup, restore, and rollback
+
+Backups are cold and require downtime. Stop services, create the archive, then
+verify both its checksum and contents before proceeding. Do not run an upgrade
+or schema change until verification succeeds. Keep archives root-only under
+`/var/backups/akyldash` and copy them to separately managed encrypted storage.
+
+Restore is a staged operation on a marked replacement host; it does not
+overwrite production. For application-only rollback, use the guarded helper
+with a verified pre-release backup and explicit data-compatibility review.
+Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md),
+[`ROLLBACK.md`](../../docs/operations/ROLLBACK.md), and
+[`RECOVERY.md`](../../docs/operations/RECOVERY.md).
+
+Automated production deployment is not enabled. Deployments remain owner-run;
+record the release and verify service health before considering it complete.

deploy/production/compose.yaml

@@ -0,0 +1,124 @@
+name: akyldash
+
+services:
+  caddy:
+    image: caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
+    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caddy-data:/data
+      - caddy-config:/config
+    networks:
+      - proxy
+    depends_on:
+      backend:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
+  opensearch:
+    image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local}
+    build:
+      context: ../..
+      dockerfile: deploy/production/Dockerfile.opensearch
+    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
+    environment:
+      discovery.type: single-node
+      bootstrap.memory_lock: "true"
+      DISABLE_SECURITY_PLUGIN: "true"
+      OPENSEARCH_JAVA_OPTS: ${OPENSEARCH_JAVA_OPTS:--Xms2g -Xmx2g}
+    mem_limit: ${OPENSEARCH_MEM_LIMIT:-4g}
+    expose:
+      - "9200"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    networks:
+      - search
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9200/_cluster/health || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+
+  backend:
+    image: ${BACKEND_IMAGE:-akyldash/backend:local}
+    build:
+      context: ../..
+      dockerfile: deploy/production/Dockerfile.backend
+    restart: unless-stopped
+    logging:
+      driver: local
+      options:
+        max-size: 10m
+        max-file: "3"
+    environment:
+      COOKIE_SECURE: ${COOKIE_SECURE:-true}
+    command:
+      - python
+      - -m
+      - search.api
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8080"
+      - --url
+      - http://opensearch:9200
+      - --index
+      - ${SEARCH_INDEX:-akyldash-fragments-current}
+      - --data
+      - /app/data/minjust-normalized
+      - --reviews-db
+      - /app/data/search-reviews.sqlite3
+      - --users-db
+      - /app/data/search-users.sqlite3
+      - --review-secret-file
+      - /run/secrets/review_secret
+    secrets:
+      - review_secret
+    expose:
+      - "8080"
+    depends_on:
+      opensearch:
+        condition: service_healthy
+    volumes:
+      - ${DATA_ROOT:?set DATA_ROOT in .env}:/app/data
+    networks:
+      - proxy
+      - search
+
+volumes:
+  caddy-data:
+  caddy-config:
+  opensearch-data:
+
+networks:
+  proxy:
+  search:
+    internal: true
+
+secrets:
+  review_secret:
+    file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}

docs/README.md

@@ -2,8 +2,20 @@
 
 ## Продукт
 
+- [Дизайн-система «Түндүк»](../DESIGN.md) — утверждённые палитра,
+  типографика, компоненты и правила адаптивности портала и лаборатории.
 - [Обзор проекта](product/project-overview.md) — назначение, основные области и
   правила работы с данными.
+- [План frontend поисковой СПС](product/frontend-search-sps-plan.md) — границы
+  MVP, зависимости и спринты.
+- [Готовность к проектированию frontend](product/frontend-design-readiness-plan.md) —
+  обязательные работы и критерии перехода к frontend.
+- [План интерфейса оценки поисковой выдачи](product/search-relevance-review-interface-plan.md) —
+  внутренняя лаборатория сбора оценок юристов для настройки OpenSearch.
+- [Разметка relevance set](../backend/search/RELEVANCE_ANNOTATION.md) — подготовка
+  эталонных документов и воспроизводимая оценка собственного поиска.
+- [Задание по нормализации документов](product/minjust-document-normalization-agent-task.md) —
+  требования и критерии приёмки нормализатора ЦБД Минюста КР.
 
 ## Решения
 
@@ -14,9 +26,18 @@
 
 - [Текущий статус](operations/project-status.md) — выполненные шаги, риски и
   ближайшие действия.
+- [Состояние инфраструктуры](operations/CURRENT_STATE.md) — подтверждённое
+  состояние production и оставшиеся проверки.
+- [Инфраструктура и эксплуатация](operations/INFRASTRUCTURE.md) — схема,
+  доступы, deployment, backup, rollback и recovery.
 - [План Telegram-пространства](operations/telegram-workspace-plan.md) — темы,
   роли и этапы развития командного окружения.
 
+## Backend
+
+- [Выгрузка и нормализация ЦБД Минюста КР](../backend/README.md) — запуск,
+  хранение и проверка конвейера правовых документов.
+
 ## Команда
 
 - [Навыки работы с ИИ](team/ai-skills-for-beginners.md) — короткие практики для
@@ -24,4 +45,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend — не создан · Frontend — не создан
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.12.2 · Frontend v0.1.2

docs/decisions/001-telegram-workspace-mvp.md

@@ -78,4 +78,4 @@ Telegram позволяет запретить пользователям отп
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан

docs/operations/ACCESS.md

@@ -0,0 +1,84 @@
+# Infrastructure access
+
+## Discover existing access first
+
+In a new Codex session, inspect the SSH alias and key file metadata without
+printing key contents, then check the connection:
+
+```bash
+ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }'
+test -r ~/.ssh/akyldash_codex && echo 'SSH key is available'
+ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps id
+ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps hostname -f
+gitea-api GET /version
+gitea-api GET /repos/admin/akyldash
+```
+
+Do not ask for the host, login, password, token, or key again before these
+checks. Never print a private key, token, or complete environment file.
+
+## VPS
+
+- SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`.
+- SSH configuration: `~/.ssh/config`.
+- Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents).
+- `codex` can run unprivileged read-only diagnostics. It has no sudo or Docker
+  group membership; Docker CLI access to the daemon is denied. Do not add it to
+  either group.
+- Bootstrap is complete. Docker and Compose are active, but no root-owned
+  repository checkout or restricted helper is installed yet. The owner can run
+  `infra/bootstrap/audit-host.sh` from the reviewed root-owned checkout to
+  report Docker containers/volumes, protected-file metadata, UFW rules, ports,
+  and ISPmanager status without showing secret contents or changing state.
+- Host package/service configuration, firewall changes, protected env/secret
+  creation, backup creation, recovery apply, and production deployment require
+  a reviewed owner/root command. `infra/deploy/install-helper.sh` grants the
+  fixed verifier, read-only Compose `config`/`status`, and guarded rollback
+  commands. It does not grant general sudo or Docker access; it is not
+  installed on the VPS.
+- `infra/deploy/prepare-production-config.sh` is an owner-run root operation.
+  It refuses to overwrite existing files, copies only the tracked env example,
+  generates a random review secret, and writes both files with mode `0600`.
+  It does not start Compose or create volumes.
+- Never connect to other VPS hosts or Synology as part of this deployment.
+
+This execution environment is WSL2. The repository is on the Windows-mounted
+`/mnt/d` filesystem, while SSH config and the identity file are in the Linux
+home directory. Do not assume Windows SSH credentials are available or copy
+keys between environments. Plain `ssh akyldash_vps` failed because the system
+SSH config include had unsafe permissions. Use `ssh -F ~/.ssh/config ...`; in a
+network-restricted session this read-only command may also require approved
+execution. If the key is missing in a future environment, report that
+limitation and do not retrieve it elsewhere.
+
+## Gitea
+
+- Repository: `https://git.vakanaut.ru/admin/akyldash`.
+- Use the installed `gitea-api` helper; it reads credentials from the
+  environment. Never inspect local credential files, print environment values,
+  or construct HTTP authorization headers.
+- The audited API token can read/write this repository and create issues, but
+  does not have `read:user` or `read:package`. Listing repository Actions
+  runners returns `403` because the API principal is not the repository owner.
+  An API `403` caused by missing scope/ownership is not evidence that no runner
+  exists.
+- Protected `main` requires one approval. Do not push directly to `main`, merge
+  automatically, or start a review. Work on a feature branch and use a PR.
+- Gitea Actions and package features are enabled for the repository. The Actions
+  task API currently returns no runs, but runner availability and package
+  publishing permissions have not been confirmed. The repository-level runner
+  is prepared by `infra/runner/install-runner.sh`, but is not installed or
+  registered yet. Only a repository owner may obtain its registration token in
+  `admin/akyldash` Settings → Actions → Runners; do not use organization or
+  instance tokens and never put the token in Git, a command argument, or a
+  report. See [CI_RUNNER.md](CI_RUNNER.md).
+
+## Who can do what
+
+| Role | Operations |
+|---|---|
+| `codex` over SSH | Unprivileged diagnostics; Docker daemon, UFW rules, and protected paths require the root audit script |
+| Restricted helpers | Read-only drift check and Compose `config`/`status`, plus guarded rollback; not installed on the audited VPS |
+| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
+| VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
+| Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |

docs/operations/BACKUP.md

@@ -0,0 +1,131 @@
+# Backup and restore
+
+## Current status
+
+No production data or backup existed during the 2026-10-11 audit. The repository
+now contains `infra/backup/archive.py`; it is not installed or run on the VPS,
+and `/var/backups/akyldash` remains a planned root-only location.
+
+## Required release policy
+
+- Create a backup before every production change, including any future schema
+  or index migration.
+- Include the normalized corpus, `search-users.sqlite3`,
+  `search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
+  Caddy's certificate data, and required secret file in the protected backup
+  set. Store secrets and Caddy private keys with equivalent access controls.
+- Verify archive readability, checksums, and SQLite integrity before relying
+  on it. Never overwrite or delete the only usable backup.
+- Keep one current pre-release backup. If deployment fails, retain the backup
+  required to restore the previous state.
+- Test restore into an isolated target before relying on the backup. Do not
+  restore over production as a test.
+
+## Clean first deployment
+
+No backup is needed only when Compose has no containers and all four
+persistent mounts are confirmed empty. For a first install with a preloaded
+corpus and not-yet-created named volumes, use `--initial-deployment` to create
+the pre-start backup. This flag is accepted only with no Compose containers and
+when `DATA_ROOT` contains exactly `minjust-normalized/`; it archives absent
+named volumes as empty components. In every normal backup, an absent volume is
+an error because it could mean that a previous installation lost data.
+Missing mount definitions, inaccessible Docker state, or partial service
+containers are also errors and must stop deployment. Do not create placeholder
+containers to make backup pass.
+
+For the corpus-only initial case, run this before the first `up` from the
+approved checkout. Replace the timestamp with one real UTC timestamp:
+
+```bash
+cd /opt/akyldash/current
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --initial-deployment \
+  --output /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
+```
+
+## Cold backup procedure
+
+The script requires every Compose container to be stopped. Schedule downtime,
+then run from the approved checkout on the VPS:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
+```
+
+Replace the timestamp placeholder with one real UTC timestamp before running.
+On a clean first deployment, `create` exits with a clear message that backup
+can be skipped. With existing data and all expected volumes present, it
+archives the storage even when no Compose containers exist. A preloaded corpus
+with no named volumes requires the explicit `--initial-deployment` mode;
+otherwise missing volumes are rejected. Partial service containers or running
+services are rejected as unsafe. The create command archives the application data,
+OpenSearch volume, Caddy data/config volumes, production environment file, and
+review secret. It records the checked-out Git revision, deployment-file hashes,
+and image references/IDs for each stopped Compose service, writes a SHA-256
+sidecar, and refuses to overwrite an existing archive. It also runs SQLite
+`quick_check`. Start the stack again only after verification succeeds:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+## Initial post-setup baseline
+
+After the first index has been loaded and validated, the initial `admin` account
+has been created, and `/review` responds over HTTPS, create a second backup.
+This one captures the initialized index, account database, and newly created
+Caddy/OpenSearch volumes. Stop all services, create and verify the archive,
+then start the stack again:
+
+```bash
+cd /opt/akyldash/current
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
+sudo python3 infra/backup/archive.py create \
+  --compose-file deploy/production/compose.yaml \
+  --env-file /etc/akyldash/production.env \
+  --output /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
+sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
+```
+
+Use one real UTC timestamp in both archive paths. Do not treat initial
+production setup as complete until this baseline archive verifies. Keep the
+pre-start corpus archive as well until the baseline is verified and copied to
+separately managed encrypted storage.
+
+Keep the local archive root-only, then copy it to separately managed encrypted
+storage. A local archive alone does not protect against loss of the VPS disk.
+Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
+`/var/tmp` and validates the checksum and SQLite databases. Applying the staged
+contents requires the separate recovery-target tool; it refuses the inventoried
+production host, existing data/volumes, and an unmarked host. Example isolated
+restore:
+
+```bash
+sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
+  --target-dir /var/tmp/akyldash-restore-test
+```
+
+This validates and stages the archive. On the replacement host only, after
+running bootstrap with `--recovery-target` and checking out the matching Git
+revision, apply the staged data:
+
+```bash
+sudo python3 infra/backup/apply_restore.py \
+  --staging-dir /var/tmp/akyldash-restore-test
+```
+
+The apply command restores data and named volumes but leaves services stopped.
+Test the full procedure on a separate recovery host before relying on it. Do
+not run an unreviewed restore over existing data.

docs/operations/CI_RUNNER.md

@@ -0,0 +1,220 @@
+# AkylDash Gitea Actions runner
+
+## Scope and isolation
+
+The deployment follows the official [Gitea 1.25 act_runner guide](https://docs.gitea.com/1.25/usage/actions/act-runner/)
+for repository-scoped registration. Gitea warns that access to a Docker daemon
+is root-equivalent; the dedicated rootless daemon is used instead of the
+production daemon. The pinned binary and checksum are published in the
+[official Gitea download directory](https://dl.gitea.com/gitea-runner/0.2.13/).
+
+The runner is reserved to repository `admin/akyldash`, using the token under
+`https://git.vakanaut.ru/admin/akyldash/settings/actions/runners`. Do not use
+organization or instance runner settings. Pin the official `act_runner`
+`0.2.13` x86_64 binary from `dl.gitea.com`; the installer verifies its SHA-256
+before placing it on the host.
+
+CI jobs run in `docker.gitea.com/runner-images:ubuntu-24.04` containers with
+capacity 1, per-job limits of 2 CPU, 3 GiB RAM and 512 processes, and a 15 GiB
+dedicated ext4 loop-backed Docker data filesystem. The runner account's home
+directory is root-owned mode `0711`, so CI cannot rename the image or mountpoint
+entries. Docker config remains under `.config`; registration state and the
+runner workspace live in `.config/act-runner`, which is runner-owned. The
+rootless Docker setup and user service use `.config/act-runner` as writable
+`HOME`, `.config` as `XDG_CONFIG_HOME`, and
+`.config/act-runner/.local/share` as `XDG_DATA_HOME`. This keeps Docker config
+and its systemd user unit in the existing `.config` tree without granting
+write access to the protected home.
+The runner systemd unit uses `ProtectHome=read-only` so `/run/user` remains
+visible; `ReadWritePaths` retains the existing `/run/user/$uid` writable
+carve-out for the runner's rootless runtime. `ProtectHome=yes` would hide
+`/run/user` and break the configured runtime and Docker socket paths. See the
+[systemd.exec documentation](https://www.freedesktop.org/software/systemd/man/255/systemd.exec.html#ProtectHome=).
+The rootless Docker daemon and runner user are separate from production Docker;
+`codex` is not added to
+sudo or Docker groups. Jobs do not receive the rootless Docker socket, the
+production Docker socket, production environment/secrets, or arbitrary host
+bind mounts. The runner account has no access to `/etc/akyldash` or the
+root-owned application checkout. These are host permission boundaries, not a
+claim that untrusted project code is safe; review workflow changes and actions
+before merging them.
+
+The runner service restarts on failure and boot. Systemd caps the runner
+account's shared slice to 2.5 CPU, 4 GiB RAM and 2048 tasks; the rootless
+daemon is separately capped to 2 CPU, 4 GiB and 1024 tasks, while each job has
+the limits above. Runner logs go to a rotated file (10 MiB, 7 copies). A
+systemd timer checks the runner service and the rootless Docker API each minute;
+on failure it restarts the fixed runner service. No UFW changes or public
+metrics ports are needed. Runner cache is disabled.
+
+The CI workflow and `runner-smoke.yaml` target the repository runner label
+`akyldash-ci`. CI keeps the backend, Telegram, backup/restore, deployment,
+bootstrap, runner, Compose, and Caddy checks. Caddy validation runs in a pinned
+container action with explicit `/usr/bin/caddy` entrypoint. The workflow passes
+Gitea's `${{ github.workspace }}` value into each container action; Caddy and
+Kaniko use that supplied path for repository files instead of assuming a
+container workspace location. Both production
+Dockerfiles are built in a separate pinned Kaniko container action with explicit
+`/kaniko/executor` entrypoint using the runner's rootless container engine; the
+job container receives no Docker socket. Kaniko uses `--no-push` and
+`--no-push-cache`, so CI validates the full backend and OpenSearch builds but
+does not publish images or export build output. The two builds run sequentially
+within one job. CI run 69 completed both builds on the runner. Actual per-action
+resource-limit enforcement and peak memory or disk usage were not independently
+measured or verified. No registry credentials,
+production secrets, elevated container options, or arbitrary host mounts are configured.
+
+The Kaniko action uses the `linux/amd64` v1.28.5-debug manifest
+`sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412`,
+listed in the [official GitHub Packages entry](https://github.com/osscontainertools/kaniko/pkgs/container/kaniko/1271366984?tag=v1.28.5-debug).
+The pinned Caddy image digest remains the same as the former `docker run`
+validation. act_runner starts these container actions; they do not expose its
+rootless Docker socket to workflow steps.
+Production deployment is not automated.
+
+## Installation and registration
+
+Review the root script and host audit output before running it. It installs
+`uidmap`, `dbus-user-session`, `curl`, `logrotate`, and Docker's rootless extras;
+creates the dedicated system user and selects the first free, shared 65536-ID
+range from `/etc/subuid` and `/etc/subgid`; reserves 15 GiB
+for runner images/workspaces; and configures systemd services. It starts only
+the separate rootless Docker daemon; the runner and health timer stay disabled
+until repository registration. It does not change
+SSH/UFW, production Docker configuration, application data, or production
+secrets. It can be rerun after partial setup: it validates and reuses an
+existing runner account, subordinate ID mappings, image, and mount unit. An
+existing image must be exactly 15 GiB and contain ext4; a mismatch stops the
+installer without resizing or formatting it. The runner account must retain
+a non-root system UID below `UID_MIN`, its expected home, primary group, and
+sole supplementary group. After validating the account, the installer locks
+its home directory to root ownership and mode `0711`; it refuses to rerun while
+the runner service is active, so it cannot interrupt a job. The mount
+is started only when it is not already active, and an active mount must be the
+expected ext4 loop device backed by the runner image with both `nosuid` and
+`nodev` active. Runner-home directories and user configuration files are
+created in runner-owned subdirectories. Root creates only the first `.config`
+entry beneath the already locked home; subsequent directory and config writes
+run as `akyldash-runner`. The installer rejects static symlinks and validates
+ownership and mode after setup. This also normalizes the `0755` intermediate
+directories left by the first installer version. Conflicting files or state
+stop the installer for manual inspection. A new image requires at least 25 GiB
+free on `/`. Image creation opens each parent component without following
+symlinks, creates the image exclusively relative to the opened directory, and
+passes the open file descriptor to `mkfs.ext4`; validation checks the same inode
+and rejects path replacement. Systemd mounts the image under the root-owned home
+entry, which the runner cannot replace after validation.
+
+If the reviewed checkout is not present at the inventoried path, first verify
+that the target is absent and clone the reviewed `main` there. Do not overwrite
+an existing checkout.
+
+```bash
+sudo test ! -e /opt/akyldash/current
+sudo git clone --branch main \
+  --single-branch https://git.vakanaut.ru/admin/akyldash.git \
+  /opt/akyldash/current
+```
+
+From that root-owned reviewed checkout on the VPS:
+
+```bash
+sudo bash /opt/akyldash/current/infra/runner/install-runner.sh
+```
+
+Then, as a repository owner, open repository **Settings → Actions → Runners**,
+create a repository-level runner, and copy its registration token locally. Do
+not send the token in chat or save it in the checkout. Register interactively
+in a terminal so it is not placed in shell history or process arguments:
+
+```bash
+sudo -u akyldash-runner env HOME=/var/lib/akyldash-runner/.config/act-runner \
+  /usr/local/bin/act_runner \
+  --config /etc/akyldash-runner/config.yaml register
+```
+
+Enter `https://git.vakanaut.ru/`, the token, runner name `akyldash-ci-vps-01`,
+and label `akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04`
+when prompted. The config sets the label and capacity; verify them in the
+repository runner settings. The token is saved only in the mode-0700 runner
+runtime directory, outside Git. Start/enable the runner after registration:
+
+```bash
+sudo systemctl enable --now akyldash-act-runner.service
+sudo systemctl enable --now akyldash-runner-healthcheck.timer
+sudo systemctl status akyldash-act-runner.service
+sudo systemctl status akyldash-runner-healthcheck.timer
+```
+
+The interactive command's terminal behavior should be confirmed before pasting
+the token. If it echoes input, stop and use the repository settings to revoke
+that token; do not put it in command-line arguments or environment variables.
+
+## Validation
+
+1. Confirm the runner is **Online** in the repository runner settings, with the
+   expected name and `akyldash-ci` label.
+2. Confirm **Runner smoke test** runs for the open PR (or dispatch it manually).
+   It checks that production
+   sockets, files, environment secrets, and writable application paths are not
+   present in the job container.
+3. Confirm the run reports the expected runner, succeeds, and the runner
+   service and health timer remain active.
+4. Inspect the `CI` workflow run and confirm Caddy plus both Kaniko actions pass
+   on `akyldash-ci`. The Kaniko jobs are intentionally build-only: publishing
+   images requires a separately reviewed registry workflow and credentials.
+   Do not add production deploy workflows or secrets to this runner.
+
+## Upgrade
+
+Update `runner_version` and its expected official SHA-256 together in
+`infra/runner/install-runner.sh`, review upstream release compatibility with
+the installed Gitea, and open a PR. Do not replace the binary while a job is
+active. Stop the runner, install/verify the new binary, then start it and
+confirm Online plus a smoke run. Preserve
+`/var/lib/akyldash-runner/.config/act-runner/.runner`.
+
+## Recovery and rollback
+
+The runner is disposable: production data and secrets are not on its filesystem.
+For a runner process failure, inspect `systemctl status` and
+`journalctl -u akyldash-act-runner.service`; restart the fixed runner service.
+If installation stops after creating or enabling the mount unit, inspect the
+image and mount with `stat`, `blkid`, `findmnt`, and `losetup`, then rerun the
+reviewed installer only when the image is 15 GiB ext4 and the active mount is
+backed by it. The installer recognizes that state and does not format or
+restart the mounted filesystem. If size, filesystem, unit contents, or backing
+device differ, stop and resolve the state manually; never format the existing
+image as a retry.
+For lost/corrupt registration state, stop the service and obtain a fresh
+repository-level token from Settings → Actions → Runners. Preserve the 15 GiB
+data image and state for diagnosis; re-register only after the repository
+owner confirms/revokes the old runner in Gitea. The images/cache can be rebuilt
+from the workflow. Never format or remove an existing runner store as an
+automatic recovery step.
+
+## Status
+
+The owner reports that the repository-scoped runner is registered, appears
+Idle in Gitea, and its rootless Docker daemon works; the report date was not
+recorded. Gitea status checked 2026-10-18 showed smoke run 62 successful and CI
+run 61 pending on head `69c62466`. That workflow requested `ubuntu-latest`,
+while the smoke workflow used `akyldash-ci`; the label difference is a
+plausible explanation for the pending status, but the runner inventory API
+returned 403 and the scheduler's reason was not independently confirmed.
+
+On head `3a89dc21`, CI run 63 failed at the host audit root guard and smoke run
+64 succeeded. On head `436806a8`, CI run 65 failed in the Caddy container action
+with `exec: "validate": executable file not found`; Kaniko was skipped and smoke
+run 66 succeeded. On head `d96fc4d3`, CI run 67 failed because the Caddy action
+looked for `/github/workspace/deploy/production/Caddyfile`, which was not
+mounted at that fixed path; Kaniko was skipped and smoke run 68 succeeded. On
+head `5d72bb09`, CI run 69 and smoke run 70 succeeded. CI validated the Caddyfile
+and built the backend and OpenSearch images using `${{ github.workspace }}`.
+Actual per-action resource-limit enforcement and peak usage were not
+independently measured or verified. Image publishing, host-wide resource
+metrics, and production deployment remain unverified. Root-guard tests
+use `setpriv` for negative cases when jobs start as UID 0 and separately test
+the host-audit guard with simulated UID 0 and UID 65534; the real host audit,
+production config setup, and runner installer are never executed as root by
+those tests. No VPS changes are part of this PR.

docs/operations/CORPUS_HANDOFF.md

@@ -0,0 +1,182 @@
+# Corpus inventory and production handoff
+
+The repository contains a read-only verifier for the ignored local corpus at
+`data/minjust-cbd/` and `data/minjust-normalized/`. It does not download,
+normalize, copy, index, or modify source files. Keep inventory artifacts outside
+the repository and both source trees.
+
+## Run the inventory
+
+From the repository root, create a private artifact directory and run:
+
+```bash
+artifact_dir=$(mktemp -d /tmp/akyldash-corpus-audit.XXXXXXXX)
+python3 infra/deploy/verify_corpus.py \
+  --raw-root data/minjust-cbd \
+  --normalized-root data/minjust-normalized \
+  --output-dir "$artifact_dir"
+```
+
+The tool writes `result.json`, `inventory.sqlite3`, and a persistent lock file
+into that directory. Keep the directory private: the inventory contains source
+paths and checksums. A second run against the same artifact directory is
+rejected while a scan is active. Exit
+status `0` means the scan and structural checks completed without errors; `1`
+means findings were recorded or the run is incomplete; `2` means the arguments
+or artifact path were invalid. Read `result.json` to distinguish those states.
+Warnings, including documents with zero text fragments, are reported
+separately and do not turn the scan into a failure.
+
+For an interrupted run, use the same artifact directory and add `--resume`.
+Resume rehashes every source file instead of trusting cached timestamps, then
+reports added, changed, and removed paths relative to entries already stored
+in the inventory. After resuming an interrupted first pass, those counters are
+relative to the partial saved inventory; the full rehash and final checks still
+cover every path. A successful result therefore describes the latest complete
+pass. Run against a quiescent corpus; do not normalize, fetch, or copy files
+into either source tree during the scan. Do not point `--output-dir` inside
+either source tree or through a symlink. The verifier rejects such paths before
+creating or changing them; it also flags symlinks, reparse points, hard links,
+unsupported file types, and files that change while being read.
+
+The inventory database schema is versioned. Resume requires the same root
+metadata and verifier SHA-256 recorded in `result.json`; start a new artifact
+directory after changing the verifier. `entries` counts files plus directories;
+`files` and `directories` are separate counts. Document-prefix checks use
+indexed path ranges, and stale-path reporting streams rows from SQLite.
+
+The audit checks SQLite integrity, manifest/document directory agreement,
+normalized schema and normalizer versions, source inventory checksums,
+fragment identities and text hashes, required output files, and text assembled
+from fragments. It does not establish the legal authority, completeness, or
+semantic correctness of MinJust content.
+
+## Compare source and staging before transfer
+
+Run a complete inventory of the approved normalized source and the copied
+staging tree into separate private artifact directories, using
+`--normalized-root` for each. Require `status: complete` and
+`verification_status: passed` in both results. The comparison command also
+requires each manifest to contain at least one document and each normalized
+tree to contain at least one fragment; an empty source or staging tree is a
+failure.
+
+```bash
+python3 infra/deploy/compare_corpus_inventories.py \
+  "$source_artifacts/result.json" "$staging_artifacts/result.json"
+```
+
+The checker streams every relative path, size, and SHA-256 from both SQLite
+inventories, and requires the same verifier/schema version and different
+normalized roots. It opens both existing private `.inventory.lock` files
+read-only and holds shared nonblocking locks through comparison; a missing,
+unsafe, or busy lock fails closed. A zero exit status confirms a non-empty
+staging tree matches the source. Keep both trees quiescent during their scans
+and comparison. This is an equality check only; the owner still reviews the
+code `6` disposition and approves the transfer destination.
+
+## Current data disposition
+
+As measured on 2026-10-18, both local manifests contain 209,958 documents and
+pass SQLite `quick_check`. All normalized manifest entries are `success` at
+schema `1` and normalizer `1.0.0`. The raw manifest records one fetch error for
+code `6`, which is absent from its `documents` table; this does not prove that
+the source document is absent. On 2026-10-18, bounded calls through the
+repository's rate-limited MinJust client received HTTP `404` for
+`GetDocumentListByQuery`, `GetDocument(6)`, control `GetDocument(1096)`, and
+`CheckAvailable`. The listing call failed before returning a code list, and
+both document calls failed identically. The repeated `404` is consistent with
+the OpenData endpoint being unavailable or its route having changed, but does
+not identify which. It is not evidence that code `6` is absent. The [public
+document listing page](https://cbd.minjust.gov.kg/list-docs/kg) was reachable,
+but it does not confirm this numeric API code. Do not describe the corpus as
+fully verified or silently waive this error. Recheck the official listing and
+API when the endpoint responds, then record an explicit owner disposition.
+From the VPS, a metadata-only HTTPS check to `CheckAvailable` failed at the
+HTTP/2 transport layer (`curl` exit 92, `INTERNAL_ERROR`, HTTP status `000`,
+TLS verification `0`). An HTTP/1.1 retry was not completed.
+
+## Current API and bounded code 6 recovery (2026-10-20)
+
+The current official site uses `POST /api/v1/GetDocuments` (page size 12),
+`GET /api/v1/GetDocument?DocumentCode=...`, and
+`GET /api/v1/GetEdition?editionId=...&lang=ru|kg`. A bounded first-page probe
+reported 210,639 records; this recovery does not enumerate the full list. The
+official alias `1-4` resolves to canonical code `205064`. Code `6` is present
+and has 201 editions. The edition order in `GetDocument` is authoritative;
+numeric IDs do not define currentness.
+
+For `lang=kg`, the API returns Kyrgyz HTML in `contentRu` and `contentKg=null`.
+The bounded adapter maps that response by the requested language to archive
+`ky`, preserves the response and checkpoints, limits requests to one per
+second, caps JSON responses at 16 MiB, and caps one document's text at 4 GiB.
+It writes only to a separate generation under `/tmp`; it neither resumes nor
+overwrites the existing ignored corpus.
+
+Without `exact=true`, a request for `920292` returned response field
+`id=920492` despite different text length/hash. Two exact probes for `920292`
+and `920492` returned matching IDs and names/dates (`04.08.1998` and
+`13.04.2017`) matching `GetDocument`; their RU text lengths/hashes were
+485,487 / `2b51c2ff4e9b91cbe2a7ed052a5a9ed9150bda993fc15c120ac948580926b718`
+and 1,139,908 / `e5f0fdbb47766e5a5dbb971183d765e4c1dcaf9c969a218a84206244ce9132e8`.
+The adapter now requires `exact=true`, returned-ID equality, and matching RU/KY
+names before a language checkpoint can complete. Its versioned request
+contract is part of both staging metadata and the SQLite resume identity;
+pre-contract generations cannot be resumed.
+
+The adapter does not independently validate attachments or embedded media.
+Even after the text stage completes, the raw manifest records this as an
+unresolved error. Do not merge this single document into the existing corpus,
+claim corpus completeness, or transfer it to production without a separate
+review and comparison. The prior no-exact pass at
+`/tmp/akyldash-current-code6.ZAMC37u4ex` is marked quarantined in
+`qualification.json`; its 240 text checkpoints and 304,493,520 bytes are
+retained as unverified, with no document directory published. Its preserved
+work-tree SHA-256 is
+`1e874c1d7fab1e00e7e45ff1b16422ea474e22ee13c901ac9a0612f47aa5d24a`. It must
+not be reused or normalized.
+
+The exact-only generation at `/tmp/akyldash-current-code6-exact.L4k3AvnOlP`
+contains 201 editions and 402 complete RU/KY text checkpoints (604,377,885
+raw bytes). The raw manifest passes `quick_check`; its stored source SHA-256
+matches the staged document tree at
+`81d8e1a53b2ebef61f08b56d04b3f0a80077d587a4fadae0152bb4b7f13055ea`. Separate
+normalization at `/tmp/akyldash-current-code6-exact.L4k3AvnOlP-normalized`
+passes `quick_check`, records normalizer `1.1.0`, and preserves first/last
+edition order `920292` (`04.08.1998`) to `920492` (`13.04.2017`). The normalized
+generation is about 2.05 GB. Neither temporary generation has been merged into
+the existing corpus. Attachments and embedded media remain unverified.
+
+The first full pass (run `b4a4556e30cc464d90bc8d75ef1673e9`) stopped before
+validation with `Python int too large to convert to SQLite INTEGER` after
+2,120,420 raw entries and 397,856 normalized entries. This was an inventory
+metadata storage failure, not a corpus finding. The verifier now stores file
+identity values as text and records its own SHA-256. A complete rehash with the
+updated verifier is required; do not treat the failed pass or progress output
+as a completed audit.
+
+A fresh full rehash is now running in `/tmp/akyldash-corpus-audit-final.gtsD87vc`,
+run `f4dd5e53020f442bbbecee2d0e4986a6`, verifier SHA-256
+`83dac230099fc91972cd14e86e9f8ec72d13bca08f53d402f13a169e096d98f5`. Check
+its private `result.json` for final counters, findings, and manifest comparison.
+
+## Production transfer and indexing gates
+
+Production deployment is not enabled. Before any transfer, the owner must
+review the root-only host audit, confirm the target data location is empty or
+backed up, resolve the raw code `6` finding, and approve the exact normalized
+corpus to use. Transfer only `minjust-normalized/`; keep the raw source archive
+offline. Do not preserve the source's Windows/WSL file permission bits. Copy
+to a new staging location with owner-approved permissions, verify the copied
+manifest and files there, and only then consider placing it under `DATA_ROOT`.
+Never overwrite an existing corpus or OpenSearch volume.
+
+The current bulk importer checks individual Bulk API item results, but it can
+switch an alias after a zero-fragment input and does not enforce a refresh,
+count comparison, or representative-query check before that switch. Until a
+separate change adds those gates, do not pass `--alias` during production
+loading. Load into a new versioned index only after the corpus transfer is
+verified; require a nonzero fragment count, successful item results, an
+explicit refresh, a matching OpenSearch document count, and representative
+Russian/Kyrgyz queries before an owner performs an alias change. Take and
+verify the required backup before any production index or service change.

docs/operations/CURRENT_STATE.md

@@ -0,0 +1,273 @@
+# Infrastructure current state
+
+Read-only host recheck: 2026-10-18. Source: SSH as `codex`, local repository
+inspection, and Gitea API. Root-only Docker state, UFW rule details, and
+protected file metadata remain unverified.
+
+## VPS facts
+
+- Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`.
+- OS: Ubuntu Server `24.04.5 LTS`; kernel `6.8.0-142-generic` from the prior
+  inventory audit.
+- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB root filesystem with
+  84 GiB free on 2026-10-18. Runner storage is a separate 15 GiB ext4 loop
+  mount with 12 GiB free at that check.
+- Network: interface `ens3`, IPv4 `206.223.249.118/32`.
+- `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires
+  a password. Docker CLI cannot access `/var/run/docker.sock` as `codex`.
+- SSH service is running. The inspected sshd configuration has public-key and
+  password authentication enabled and `PermitRootLogin yes`; no SSH changes
+  were made.
+- Bootstrap completed. Docker CLI `29.9.0` and Compose `5.6.0` were rechecked
+  over SSH on 2026-10-18; containerd `2.4.1` and runc `1.5.2` are from the
+  prior package audit. The Docker daemon API remains root-only.
+- `/etc/docker/daemon.json` has the expected `local` log driver and `10m` / `3`
+  rotation limits. Bootstrap paths exist with root ownership: `/opt/akyldash`
+  mode `0755`, `/etc/akyldash` and `/var/backups/akyldash` mode `0700`, and
+  `/var/lib/akyldash/data` mode `0750`.
+- `/opt/akyldash/current`, verifier/rollback binaries, and the sudoers file are
+  absent. `codex` has not received sudo or Docker-group access. Metadata for
+  `/etc/akyldash/production.env` and `/etc/akyldash/review_secret` could not be
+  checked because access is denied. The target
+  `/var/lib/akyldash/data/minjust-normalized` and its manifest metadata were
+  also denied; their presence, ownership, and contents are unknown. No sudo was
+  used.
+- TCP listeners observed: SSH `22` and loopback DNS only; no listener on `1500`,
+  `80`, or `443`. No ISPmanager package or service unit was found through the
+  unprivileged checks; root can confirm with the audit script below.
+- Owner reports UFW is active and allows TCP `22`, `80`, `443`, and `1500`.
+  Rule details were not independently read; no firewall changes were made.
+- Additional read-only runner checks on 2026-10-12 confirmed `/` is ext4 with
+  about 92 GiB free at the prior audit, `/usr/bin/dockerd-rootless.sh` exists,
+  and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS,
+  `https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner
+  registration and Gitea Actions protocol connectivity are not confirmed.
+- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS,
+  HTTPS returned Cloudflare `521` with TLS verification result `0` on
+  2026-10-18. TCP listeners were SSH `22` and local DNS only, so no origin
+  listener on `80`/`443` was observed. Origin TLS and Cloudflare Full (strict)
+  are not verified.
+
+## Application and Gitea facts
+
+- Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`.
+- The repository API reports push permission, Actions and packages enabled, and
+  PRs enabled. `main` is protected and requires one approval; the current API
+  principal cannot push or merge directly to `main`.
+- The repository runner-list API returns `403` because this API principal is
+  not the repository owner. The latest verified CI run 75 and smoke run 76 both
+  succeeded on `main` head `0e276ae6`. The Actions run-list endpoint returned
+  `403` for this API principal during this recheck, so no newer run state was
+  independently retrieved. The owner reports the runner is registered and
+  Idle; repository runner inventory and package publishing permissions remain
+  unverified.
+- No public application listener is observed on TCP `80`/`443`. Docker
+  container/volume state still requires the root audit. Repository Compose describes
+  Caddy `2.11.6` pinned by digest, Python Search API `0.12.2` in this branch,
+  SQLite account/review data, and OpenSearch `3.7.0` with `analysis-icu`.
+- The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a
+  proxy network; OpenSearch and backend share a separate internal network, and
+  backend has no host port. Caddy certificate storage, OpenSearch, and
+  application data use persistent volumes/bind mounts.
+- The root-owned repository checkout, production env/secret files, Compose
+  volumes/containers, backup, verifier/rollback helpers, and application are
+  not confirmed installed. TLS and registry digest deployment for
+  backend/OpenSearch are not configured.
+- The repository has a cold-backup archive/verification tool and a one-shot
+  restore apply tool that requires a marked replacement host. Backup/restore
+  tests use synthetic data; neither tool has run against production. The
+  archive preflight permits a clean first deployment to skip backup, requires
+  backup for existing persistent data even without containers, allows absent
+  named volumes only in explicit corpus-only initial-deployment mode, and
+  rejects inaccessible/ambiguous mounts and incomplete/running service
+  containers. Initial deployment docs require a second verified backup after
+  index and admin initialization.
+- Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
+  inventoried production hostname/IP; its host-identity guard has shell tests.
+  The mode has not been run on a VPS.
+- The repository defines a root-owned read-only verifier, Compose `config` and
+  `status` helper, guarded rollback helper, and installer with exact sudoers
+  command rules. None is installed on the VPS. CI contains no production
+  deployment job; starting/stopping services remains owner-run.
+- Rollback preflight validates digest-pinned image references against archived
+  image IDs and local availability, then refuses before stopping services or
+  creating a backup because the current flow cannot safely switch immutable
+  digest references. This behavior is covered by synthetic local tests; it has
+  not been exercised on the VPS.
+- This branch adds an offline release-manifest validator and tests for the
+  repository allowlist intended for a future root-owned host integration. The
+  manifest format does not prove source provenance. No registry, host policy,
+  publish credential, or deployment integration is configured or installed.
+- A repository-scoped official `act_runner` `0.2.13` installation is
+  configured in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB
+  ext4 loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU,
+  3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048
+  tasks), rotated file logs, and a systemd health timer. Jobs target the
+  `akyldash-ci` label and do not receive either Docker socket. On 2026-10-18,
+  SSH showed `akyldash-act-runner.service` active under `akyldash-runner`, with
+  zero restarts and about 115 MiB current memory. Its rootless Docker store is
+  mounted from `/dev/loop0` as ext4 with `rw,nosuid,nodev`; the 15 GiB mount had
+  12 GiB free. The separate `akyldash-runner-restart.service` remains failed
+  with exit status 1 since 2026-10-14; its journal had no entries, so the cause
+  is unknown. The latest verified main CI run 75 and smoke run 76 succeeded on
+  head `0e276ae6`. The Actions API returned 403 during this recheck, so no newer
+  run state was independently retrieved. A read-only SSH recheck on 2026-10-21
+  confirmed the active `akyldash-act-runner.service` runs as
+  `akyldash-runner` with `ProtectHome=read-only`; `ReadWritePaths` includes the
+  runner runtime/config/log paths and `/run/user/999`. Its `ExecStart` uses
+  `/etc/akyldash-runner/config.yaml`. The live unit retains a writable carveout
+  for `/run/user/999`; the installer keeps that existing carveout and changes
+  `ProtectHome` from `yes` to `read-only` so the runtime path remains visible.
+  No VPS unit was changed. Main commit
+  `d2fb4b81` passed CI run 115 and smoke run 116. On
+  head `3a89dc21`, CI run 63 failed at the host audit root guard and smoke run 64
+  succeeded. On head `436806a8`, CI run 65 failed in the Caddy container action
+  with `exec: "validate": executable file not found`; Kaniko was skipped and
+  smoke run 66 succeeded. On head `d96fc4d3`, CI run 67 failed because the
+  Caddy action's fixed `/github/workspace/...` path did not exist; Kaniko was
+  skipped and smoke run 68 succeeded. On head `5d72bb09`, CI run 69 and smoke
+  run 70 succeeded. CI validated the Caddyfile and built the backend and
+  OpenSearch images using `${{ github.workspace }}`. Actual per-action
+  resource-limit enforcement and peak usage were not independently measured or
+  verified. Image publishing, host-wide resource metrics, and production
+  deployment remain unverified. No VPS changes are made.
+- An isolated local Compose smoke test built backend/OpenSearch images,
+  reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
+  200 for `/review` through Caddy. Its containers, networks, and volumes were
+  removed afterward. This does not verify VPS startup, DNS, or public TLS.
+- The app has no migration framework or SMTP setup. Repository versions on this
+  branch are:
+  backend `0.12.2`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
+  deployed on this VPS).
+
+## Corpus readiness (2026-10-18)
+
+- The local raw archive and normalized corpus are ignored under `data/`; no
+  corpus files were changed. Their read-only SQLite manifests both pass
+  `quick_check` and each report 209,958 documents. The raw manifest is
+  28,295,168 bytes with SHA-256
+  `f8e3dd2fdec607583ca09cfdb5b76171bb7bfa1e41a0e07ee5aca8481e18e17e`; it has
+  one fetch error, code `6`, absent from `documents`. The normalized manifest
+  is 31,571,968 bytes with SHA-256
+  `1679c20e6d0f3e5a9b50737c006c4417a28ebcaf7467d194d3bf8243538f83bf`; all
+  entries are `success`, schema `1`, normalizer `1.0.0`.
+- The read-only full-file inventory in `/tmp/akyldash-corpus-audit.Mjr6aa`
+  stopped before validation in run `b4a4556e30cc464d90bc8d75ef1673e9` with
+  `Python int too large to convert to SQLite INTEGER`. At stop it had processed
+  2,120,420 raw entries (1,081,428 files, 1,038,992 directories) and 397,856
+  normalized entries (246,321 files, 151,535 directories). This is a verifier
+  metadata storage failure, not a corpus finding. A full rehash with the
+  corrected, versioned verifier is required before readiness can be stated.
+- A fresh full rehash is running in `/tmp/akyldash-corpus-audit-final.gtsD87vc`,
+  run `f4dd5e53020f442bbbecee2d0e4986a6`, with verifier SHA-256
+  `83dac230099fc91972cd14e86e9f8ec72d13bca08f53d402f13a169e096d98f5`.
+  `result.json` currently reports `scan_status: running`; its live counters
+  must be recorded after completion. Sources remain read-only.
+- Through the repository client, bounded MinJust OpenData requests for
+  `GetDocumentListByQuery`, `GetDocument(6)`, control `GetDocument(1096)`, and
+  `CheckAvailable` each returned HTTP `404`. The list API returned no code list;
+  the direct requests do not establish whether code `6` exists. The public
+  listing page is reachable, but it does not map the numeric API code. A
+  metadata-only `CheckAvailable` request from the VPS failed in HTTP/2 with
+  `curl` exit 92 (`INTERNAL_ERROR`), HTTP status `000`, and TLS verification 0;
+  the HTTP/1.1 retry was not completed. Treat the code `6` disposition and
+  full corpus readiness as unresolved.
+- On the VPS, read-only `stat` of `/var/lib/akyldash/data/minjust-normalized`
+  and its manifest was denied. Production corpus presence and metadata remain
+  unknown. No corpus was transferred and no production service, Docker socket,
+  runner policy, or VPS configuration was changed.
+
+## Current MinJust API staging (2026-10-20)
+
+- The public site's current API uses `POST /api/v1/GetDocuments` with page size
+  12, `GET /api/v1/GetDocument?DocumentCode=...`, and
+  `GET /api/v1/GetEdition?editionId=...&lang=ru|kg`. A bounded page-1 probe
+  reported 210,639 filtered records; no full listing was enumerated for this
+  recovery. Official alias `1-4` resolves to canonical document code `205064`.
+- Document code `6` has 201 official editions. The API's `lang=kg` response
+  places Kyrgyz HTML in `contentRu` while returning `contentKg=null`; staging
+  maps by requested language (`kg` to archive `ky`) and preserves the edition
+  order from the document response. Normalization and search export now retain
+  and use that order instead of sorting IDs.
+- Without `exact=true`, one probe for edition `920292` returned response field
+  `id=920492` despite a different text length/hash. Two bounded `exact=true`
+  probes returned matching IDs and names/dates from `GetDocument`: edition
+  `920292` / `04.08.1998` (485,487 RU characters, SHA-256
+  `2b51c2ff4e9b91cbe2a7ed052a5a9ed9150bda993fc15c120ac948580926b718`) and
+  edition `920492` / `13.04.2017` (1,139,908 RU characters, SHA-256
+  `e5f0fdbb47766e5a5dbb971183d765e4c1dcaf9c969a218a84206244ce9132e8`).
+  Every edition request now uses `exact=true`, requires returned ID equality,
+  and checks RU/KY names against `GetDocument` metadata.
+- `backend/ingestion/minjust_current.py` stages one document into a separate
+  generation, at no more than one request per second, with a 16 MiB response
+  limit, a 4 GiB text budget per document, contract-bound checkpoints, and
+  atomic publication. Staging requires a private 0700 directory owned by the
+  current user and rejects symlinks under checkpoints/document paths; legacy
+  corpus exclusions are anchored to the repository root. The corpus verifier
+  continues to validate unchanged normalized manifests from versions 1.0.0
+  and 1.1.0. The earlier non-exact pass is quarantined at
+  `/tmp/akyldash-current-code6.ZAMC37u4ex/qualification.json`: its 240 language
+  responses (304,493,520 bytes) remain unverified and are not reusable. The
+  retained work-tree SHA-256 is
+  `1e874c1d7fab1e00e7e45ff1b16422ea474e22ee13c901ac9a0612f47aa5d24a`.
+- Exact-only code-6 staging is complete at
+  `/tmp/akyldash-current-code6-exact.L4k3AvnOlP`: 201 editions and 402
+  complete language checkpoints, 604,377,885 raw bytes. The raw manifest
+  passes `quick_check`; its document checksum matches the staged directory at
+  `81d8e1a53b2ebef61f08b56d04b3f0a80077d587a4fadae0152bb4b7f13055ea`.
+  Separate normalization at
+  `/tmp/akyldash-current-code6-exact.L4k3AvnOlP-normalized` contains 201
+  editions in official response order, passes `quick_check`, and records
+  normalizer `1.1.0` (2,048,797,024 bytes). The prior `data/minjust-cbd` and
+  `data/minjust-normalized` trees and production were not used as input or
+  modified. The raw staging manifest still records attachments and embedded
+  media as unverified; this single document does not establish corpus
+  completeness or production readiness.
+
+## Checks not completed
+
+- Docker Engine version from the server API, container and volume inventory,
+  protected env/secret metadata, sudoers, and detailed UFW rules require root.
+  `infra/bootstrap/audit-host.sh` is provided for an owner-run read-only audit.
+- The backup archive/restore tool was tested with synthetic local data; a backup
+  and restore of real production data remain unverified because production is
+  not deployed.
+- Cloudflare edge response was checked; origin DNS target, origin certificate,
+  Full (strict), registry publishing, host-wide runner metrics, and production
+  deployment have not been verified. The
+  owner-reported Idle state has not been independently checked through runner
+  inventory because the API returned 403.
+- This audit made no host changes. No package or service was installed by this
+  audit; the owner performed the initial bootstrap beforehand.
+
+## Next owner actions
+
+1. Install the approved root-owned checkout after its PR is reviewed, then run
+   `sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh` and share
+   the output. The script prints no secret values and does not mutate the host.
+2. After reviewing the audit output and confirming the target files are absent,
+   run
+   `sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`;
+   it refuses to overwrite existing files. Set the approved `DATA_ROOT` and
+   place the approved corpus/index and Cloudflare origin configuration. Do not
+   deploy until data/service state and the required backup path are reviewed.
+3. Run `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh` after
+   reviewing the checkout; it grants only fixed verification, Compose
+   `config`/`status`, and guarded rollback commands.
+4. After merging the reviewed PR, confirm a subsequent CI and smoke run on
+   `akyldash-ci`. Image publishing and production deployment remain manual and
+   are not part of CI.
+5. Test backup/restore and rollback only with synthetic data in an isolated
+   environment before any production deployment.
+
+## Change in this PR
+
+Adds a read-only, resumable full-file corpus inventory and normalized-data
+cross-check, a source/staging comparator requiring nonzero documents and
+fragments plus held shared artifact locks, synthetic regression tests, and an
+operator handoff runbook. CI's
+existing `infra/deploy` unittest discovery includes the new tests. No workflow,
+runner, VPS, production data, or Docker settings are changed. The first real
+corpus scan stopped on an integer-storage bug; full validation with the
+corrected verifier is still required. The source API's code `6` outcome remains
+unconfirmed.

docs/operations/DECISIONS.md

@@ -0,0 +1,10 @@
+# Infrastructure decisions
+
+| Date | Decision | Reason and alternatives | Status / limits |
+|---|---|---|---|
+| 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
+| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only a fixed read-only verifier and root-owned rollback helper after checkout ownership and sudoers validation | Helpers are defined in Git; not installed on VPS |
+| 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
+| 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
+| 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
+| 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |

docs/operations/DEPLOYMENT.md

@@ -0,0 +1,126 @@
+# Deployment
+
+## Current status
+
+Production deployment is not enabled or tested on the VPS. The owner completed
+the root bootstrap: Docker/Compose are active and target directories exist.
+The `codex` account cannot access the Docker socket. No root-owned checkout or
+restricted helpers are installed; protected env/secret metadata and Docker
+container/volume state remain unverified because they require root.
+The unprivileged audit sees no listener on `80`, `443`, or `1500`; Cloudflare
+still returns `521`. Do not deploy until the owner has reviewed root-only host
+state, supplied protected configuration and the approved corpus/index, and
+backup/restore have been tested.
+
+## First host preparation
+
+1. The initial root bootstrap is complete. After reviewing and merging the
+   infrastructure PR, the owner must install that approved `main` checkout at
+   `/opt/akyldash/current` (the path was absent during the unprivileged audit):
+
+   ```bash
+   sudo git clone --branch main --single-branch https://git.vakanaut.ru/admin/akyldash.git /opt/akyldash/current
+   ```
+
+   The checkout is root-owned by default. Then run
+   `sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh`. This
+   read-only audit reports Docker state, protected-file metadata, UFW rules,
+   listeners, and ISPmanager without printing secret contents. Resolve any
+   existing containers, volumes, or data before proceeding; do not overwrite
+   or delete them.
+2. After confirming the existing env and secret do not exist and reviewing the
+   tracked defaults, run
+   `sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`.
+   It creates `/etc/akyldash/production.env` from the tracked example and
+   generates `/etc/akyldash/review_secret`; both are root-owned mode `0600`.
+   It refuses to overwrite either file and does not start services or create
+   volumes. Review `DATA_ROOT` and follow the
+   [corpus inventory and handoff gates](CORPUS_HANDOFF.md) before placing the
+   approved normalized corpus there.
+3. After reviewing the root-owned checkout and audit output, run
+   `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh`. It installs
+   the backup reader, read-only verifier, `akyldash-compose config|status`, and
+   rollback helper with exact sudoers command rules. The Compose helper has no
+   start/stop/build operation and grants no direct Docker access. Actual
+   deployments remain owner-run through the documented Compose commands.
+4. Confirm DNS points through Cloudflare to this origin. The owner reports UFW
+   already allows TCP `80` and `443`; read and document the full rules before
+   any firewall decision. TCP `1500` is allowed by owner report but has no
+   observed listener and no ISPmanager package/service was found; the owner
+   must identify its purpose. Do not change UFW as part of this procedure. The
+   Compose Caddy service obtains and renews HTTPS certificates automatically;
+   Cloudflare Full (strict) still needs end-to-end verification.
+5. For a clean first deployment, skip backup only when Compose confirms there
+   are no containers and all persistent mounts are empty. If any data exists,
+   create and verify a backup before starting a new version. For a preloaded
+   corpus with absent named volumes, use `--initial-deployment`; it is limited
+   to a container-free host whose `DATA_ROOT` contains only
+   `minjust-normalized/`. Normal backups reject absent volumes. An uncertain
+   or partial state stops deployment. Then start services, load and validate
+   the OpenSearch index, create the initial admin interactively, verify
+   `/review` over HTTPS, and create the required post-setup baseline backup
+   described in `BACKUP.md`. Record the backend image ID for drift checks:
+
+   ```bash
+   sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
+   sudo chmod 0600 /opt/akyldash/deployed-image.txt
+   ```
+
+Application deployment remains owner-run. The repository-scoped `act_runner`
+`0.2.13` service is active under `akyldash-runner` and uses rootless Docker on a
+separate loop-backed filesystem. CI run 75 and smoke run 76 succeeded on main
+head `0e276ae6`. Runner job containers have neither Docker socket nor production
+secrets. CI has no image publishing or production deployment step. The
+repository defines sudo helpers for fixed verification, read-only Compose
+`config`/`status`, and guarded rollback, but none is installed on the VPS. They
+do not start or stop services.
+
+## Release image contract (preparation only)
+
+`infra/deploy/validate_release.py` checks an offline JSON release manifest
+against an explicitly supplied repository allowlist. Schema version 1 requires
+exactly `schema_version`, `source_ref`, `commit`, and `images`; the source ref
+must be `refs/heads/main`, the commit must be a 40-character lowercase Git
+SHA-1, and the images must contain exactly `backend` and `opensearch` as
+`repository@sha256:<64 lowercase hex>` references. The policy has schema
+version 1 and one non-empty repository allowlist for each service. Unknown or
+duplicate JSON keys are rejected.
+
+The validator takes the manifest path and an explicit policy path:
+
+```bash
+python3 infra/deploy/validate_release.py release.json --policy /etc/akyldash/release-policy.json
+```
+
+The policy shape is `{"schema_version":1,"repositories":{"backend":["registry.invalid/akyldash/backend"],"opensearch":["registry.invalid/akyldash/opensearch"]}}`.
+These `registry.invalid` names are format examples only; this change does not
+create a policy file or approve a registry.
+
+The repository name in the policy must be approved for the registry. No real
+registry namespace or root-owned host policy is configured by this change; do
+not use the synthetic `registry.invalid` names in tests as production values.
+The host integration must load policy only from its fixed root-owned path.
+This validator checks format and allowlist only: a manifest can claim a main
+ref and commit, but is not proof of source, build provenance, signature, or
+trusted publication. CI still has no publish credentials, image publication,
+host helper, or deploy trigger.
+
+Compose merges an image override with the existing service `build` entries.
+The regression test renders fake digest references and confirms both entries
+remain; any future deploy command must use `docker compose up --no-build` so
+the host cannot silently build a different image. This does not make digest
+rollback available: the current rollback helper expects local tags. A separate
+host-side deploy/rollback design, owner-approved registry permissions and
+retention, and manual backup/health/rollback end-to-end validation are still
+required before publishing or enabling CD.
+
+## CI
+
+`.gitea/workflows/ci.yaml` runs the Python unit suites and builds both
+production images on pull requests and pushes to `main`. It does not publish an
+image or deploy production. `.gitea/workflows/runner-smoke.yaml` exercises the
+`akyldash-ci` runner on pull requests, pushes to `main`, or manual dispatch.
+Both workflows passed on main head `0e276ae6` in runs 75 and 76 respectively.
+The repository runner-list endpoint is not available to the current API
+principal, so its registration details cannot be independently read through
+Gitea API.

docs/operations/INFRASTRUCTURE.md

@@ -0,0 +1,59 @@
+# Infrastructure
+
+## Intended layout
+
+Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
+`deploy/production/compose.yaml` defines Caddy, a Python Search API, and private
+OpenSearch 3.7. Caddy terminates HTTPS for `akyldash.kg` and proxies to the API.
+The API is reachable only on private Compose networks; OpenSearch is reachable
+only by the backend. Cloudflare should use Full (strict) only after the origin
+certificate and DNS path are verified.
+
+The API stores its account and review SQLite files beside the normalized corpus
+under the persistent data directory. OpenSearch and Caddy use named volumes.
+The API's review signing key is mounted from a host file; it is not passed in
+process arguments or committed to Git. Docker logs are bounded to 10 MiB per
+file and three files per service.
+
+## Actual state
+
+The 2026-10-11 read-only audit found a fresh Ubuntu 24.04.5 host with SSH only.
+Docker, Git, a reverse proxy, application directories, Compose services, TLS,
+and backups are not installed/configured. The `codex` account has no
+non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
+[production inventory](../../infra/inventory/production.yml).
+
+The Compose file is a repository target, not proof of deployment. The Caddy
+image is pinned by digest, while backend and OpenSearch still build locally;
+immutable registry images and automated deploy are not configured. No migration
+framework exists. The application uses SQLite and OpenSearch; SMTP
+registration/recovery is not implemented.
+
+## Repository paths and target host paths
+
+- Compose and Dockerfiles: `deploy/production/`.
+- Host checkout target: `/opt/akyldash/current`.
+- Environment configuration: `/etc/akyldash/production.env`.
+- Secret file: `/etc/akyldash/review_secret`, mode `0600`.
+- Application data: `/var/lib/akyldash/data`.
+- Intended backups: `/var/backups/akyldash`.
+
+All host paths above are targets; none existed at audit time. Root bootstrap is
+prepared at [bootstrap-ubuntu-24.04.sh](../../infra/bootstrap/bootstrap-ubuntu-24.04.sh).
+It does not edit SSH or firewall configuration, create credentials, deploy the
+application, or grant `codex` Docker/root access.
+
+## Network and operations
+
+Target public TCP listeners are SSH `22` and Caddy `80`/`443`. Backend `8080`
+and OpenSearch `9200` must remain private to Compose. The
+firewall state could not be independently verified because the SSH account has
+no sudo; do not change it without owner approval. The domain resolves to
+Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
+unreachable. Cloudflare origin mode, certificate validity, and production
+health have not been verified.
+
+Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
+[ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
+limits and procedures. Configuration changes must go through Git and review;
+do not reconcile a Git/server difference by overwriting the VPS automatically.

docs/operations/RECOVERY.md

@@ -0,0 +1,47 @@
+# Recovery on a clean VPS
+
+## Scope and safety
+
+The bootstrap below prepares Ubuntu packages and directories only. It is not a
+complete recovery: production secrets, data, reverse proxy, TLS, registry
+credentials, backup restore, and deploy-helper still require owner-controlled
+setup. Run only on the new recovery host after verifying its identity. Never
+run recovery commands against the existing production by accident.
+
+## Procedure
+
+1. Provision a clean Ubuntu Server 24.04 host and confirm hostname, IP, console
+   access, and a tested owner recovery path.
+2. Create the SSH-authorized `codex` account without sudo or Docker group access.
+   Preserve the host's existing root SSH and firewall policy until an owner has
+   reviewed the change.
+3. Check out the approved Git revision, root-owned and not group/world-writable,
+   at `/opt/akyldash/current`, then run
+   `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
+   recovery mode refuses the inventoried production hostname/IP and writes a
+   root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
+   Engine, Compose, Git, Docker log limits, and target directories.
+4. Install the limited read-only verifier and rollback helper with
+   `sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
+   checkout and gives `codex` access only to those fixed commands.
+5. Configure DNS to point through Cloudflare to the recovery host and allow the
+   owner to review/open TCP `80` and `443`. The production Compose Caddy service
+   obtains its origin certificate automatically. Verify Cloudflare Full
+   (strict) end to end before cutover.
+6. Transfer the protected backup to the new host through the owner's approved
+   channel. The archive contains the production environment and review secret;
+   keep it root-only and never paste secret values into chat.
+7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging
+   directory with `infra/backup/archive.py`, then run
+   `sudo python3 infra/backup/apply_restore.py --staging-dir <staging-dir>`.
+   Apply requires the recovery marker, matching clean Git revision, empty data
+   path, absent Compose volumes/config files, and no running containers. It
+   refuses the inventoried production host and leaves services stopped.
+8. Build the backend and OpenSearch images from the matching checked-out Git
+   revision and explicitly start Compose with `--wait`. Verify HTTPS and
+   container health, record the backend image ID as described in
+   [DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in
+   [CURRENT_STATE.md](CURRENT_STATE.md).
+
+Do not repoint DNS or retire the old host until the owner verifies recovery and
+approves the cutover. The restore and cutover have not been practiced yet.

docs/operations/ROLLBACK.md

@@ -0,0 +1,41 @@
+# Rollback
+
+## Current status
+
+There is no production release, backup, or installed rollback helper. The
+repository contains a root-owned rollback helper and a separate installer, but
+neither has been run on the VPS.
+
+After a verified cold backup is created before a release, an owner must check
+that the data format is compatible with the prior backend/OpenSearch images.
+Then run the limited helper with that archive:
+
+```bash
+sudo -n /usr/local/sbin/akyldash-rollback \
+  /var/backups/akyldash/<pre-release-backup>.tar.gz --data-compatible
+```
+
+The helper verifies archive integrity, Compose image references, deployment
+file hashes, and the availability of both prior local images. It makes and
+verifies a second cold backup, restores only the prior backend/OpenSearch image
+tags, waits for Compose healthchecks, and checks HTTPS locally. Digest-pinned
+references are validated against their recorded local image IDs, but rollback
+fails before stopping services or making a backup when either image reference
+uses `@sha256:`. The helper cannot safely switch immutable digest references
+with the current release-recording flow. Do not retag a digest manually; digest
+rollback needs a separately reviewed atomic release-reference mechanism.
+
+It refuses to restore if the current Compose/Caddyfile differs from the saved
+release. This does not restore data or automatically undo a release with an
+unknown schema change. The application has no formal migration framework;
+compatibility must be checked for every change. If it is uncertain, stop and ask
+the owner.
+
+The initial owner-run deployment must record its running backend image ID in
+`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates
+that record after a successful rollback, and `infra/bootstrap/verify.sh` checks
+it against the running container.
+
+Any data restore requires the owner, a verified pre-change backup, and a
+maintenance window. Preserve the failed release and both backups until recovery
+is confirmed.

docs/operations/project-status.md

@@ -1,18 +1,37 @@
 # Статус проекта
 
-Последняя проверка: 2026-08-03
+Последняя проверка: 2026-10-10
 Назначение документа: быстро восстановить контекст проекта для участников команды и будущих агентов.
 
 - Telegram-бот: `0.2.2`
 - Telegram-бот на Synology: `0.2.1`
-- Backend: не создан
-- Frontend: не создан
+- Backend: `0.12.2`
+- Frontend: `0.1.2` — поиск, лаборатория релевантности, аккаунты
 
 ## Краткий итог
 
-Репозиторий переориентирован с отдельного бота на весь проект юридической информационно-аналитической платформы. Telegram-бот выделен в инструмент рабочего окружения: он сохраняет сообщения разрешённых тем в SQLite, экспортирует обсуждения в Markdown и постоянно запущен в Container Manager на Synology. Версия `0.2.2` будет развёрнута после слияния изменений; сейчас на Synology работает `0.2.1`.
+Репозиторий переориентирован с отдельного бота на весь проект юридической информационно-аналитической платформы. Telegram-бот выделен в инструмент рабочего окружения. Реализованы возобновляемая выгрузка документов из официального Open Data API ЦБД Минюста КР и их локальная воспроизводимая нормализация.
 
-Ближайшая цель — 2026-08-04 зафиксировать MVP с командой, настроить резервное копирование SQLite и затем проверить полный рабочий цикл на реальном обсуждении.
+Поисковая система и внутренняя лаборатория оценки выдачи реализованы. Ближайшая
+цель — проверить собственную выдачу на практических RU/KY-запросах, разобрать
+оценки юристов и зафиксировать baseline. ЦБД Минюста служит источником для
+проверки документов и редакций, а не системой для сравнения поисковой выдачи.
+
+### Точка остановки на 2026-09-18
+
+- В PR #37 из ветки `feature/tax-profit-search-intent` тематический запрос
+  «налог на прибыль» поднимает действующий НК КР; для обычных запросов сохранён
+  `cross_fields`, добавлено усиление точной фразы. Нужна дальнейшая проверка на
+  контрольном наборе запросов.
+- Независимое ревью поискового кода не выявило дефектов; замечание об устаревшем
+  handoff исправлено. Backend-тесты: 33 пройдены, 1 пропущен;
+  `git diff --check` чистый.
+- Текущий локальный backend-контейнер при последней проверке был `0.10.2`;
+  изменения ветки (`0.10.3`) не развёрнуты. Не деплоить без отдельного запроса.
+- В рабочем дереве есть постороннее пользовательское изменение `.gitignore`;
+  не включать его в коммиты и не удалять.
+- Следующий шаг: повторно проверить полный diff PR #37 и после успешных проверок
+  передать его на слияние; затем продолжить оценку поиска на контрольном наборе.
 
 ## Уже сделано
 
@@ -67,6 +86,17 @@
 - Локальный Git-репозиторий восстановлен и привязан к Gitea.
 - Репозиторий организован как основа всего проекта, а не отдельного бота.
 - Бот развёрнут в Container Manager на Synology; автозапуск после перезапуска менеджера проверен.
+- Реализован backend-загрузчик ЦБД Минюста КР версии `0.2.2` без внешних зависимостей.
+- Загрузчик сохраняет метаданные, редакции RU/KY и изображения, а прогресс — в SQLite.
+- Пилотная выгрузка двух документов и возобновление без повторного скачивания проверены на живом API.
+- Реализован backend-нормализатор версии `0.2.2` без внешних зависимостей.
+- Нормализатор создаёт канонические метаданные, безопасный HTML, чистый текст и адресуемые фрагменты RU/KY.
+- SQLite-манифест обеспечивает возобновление, повтор ошибок и пропуск неизменившихся документов.
+- Полный проход завершён: 209 958 документов нормализованы без ошибок.
+- Контрольная выборка RU/KY прошла проверки текста, фрагментов, ID и SHA-256.
+- Добавлены строгий mapping и атомарный Bulk NDJSON-экспорт для OpenSearch.
+- На предыдущей итерации использовались Excel/PDF-бланки юридической проверки;
+  текущий процесс опирается на relevance set и внутреннюю лабораторию.
 
 ### Развёртывание
 
@@ -80,8 +110,9 @@
 
 ### Развёртывание и сопровождение
 
-- Docker-конфигурация развёртывания не хранится в репозитории.
-- Не настроен CI/CD.
+- Production Compose, Ubuntu bootstrap и drift-проверка описаны в репозитории.
+- CI workflow добавлен в PR #42, но его Gitea run остаётся в очереди; наличие
+  runner и автоматический production-деплой не подтверждены.
 
 ### Хранилище
 
@@ -102,6 +133,20 @@
 - Не настроены уведомления Gitea.
 - Не проведён полный пользовательский тест: сообщение → сохранение → экспорт → анализ в ChatGPT → возврат отчёта.
 
+### Готовность поиска к публичному frontend
+
+- Реализована внутренняя лаборатория для просмотра и оценки фактической
+  выдачи OpenSearch; пилот с юристами ещё не проведён.
+- Не завершена независимая юридическая проверка всех 50 запросов relevance set
+  v1; спорные строки нельзя включать в baseline.
+- Не зафиксированы неизменяемая копия подтверждённого relevance set и baseline
+  `Recall@10`/`MRR@10`.
+- Публичный readiness gate остаётся открытым: нужны обновление корпуса,
+  переиндексация, проверка выдачи и API-сценарии для RU, KY и одноязычных актов.
+- Создан внутренний авторизованный frontend поиска и лаборатории; первая
+  учетная запись `admin` создаётся CLI-командой, новых пользователей добавляет
+  только `admin`.
+
 ## Важные неизвестные
 
 По мере реального использования нужно принять решения по следующим вопросам:
@@ -112,19 +157,94 @@
 
 ## Следующий этап
 
-### Фиксация MVP и проверка полного цикла
+### Проверка собственной поисковой выдачи
 
-Рекомендуемый порядок:
-
-1. Настроить резервное копирование `/volume1/docker/akyldash/data`.
-2. Зафиксировать состав MVP с командой и закрыть тему `MVP`.
-3. Провести одно реальное обсуждение с ответами, правками и вложением.
-4. Экспортировать обсуждение и вручную проанализировать Markdown в ChatGPT.
-5. Вернуть подтверждённый итог в тему `Решения` и зафиксировать его в Git/Gitea.
-6. Записать обнаруженные неудобства и только после этого выбирать следующую доработку.
+1. Подготовить согласованный RU/KY relevance set: фиксировать потребность
+   запроса и подтверждённые `document_code`, не ориентируясь на порядок выдачи.
+2. Проверить практические запросы во внутренней лаборатории `/review`, сохранить
+   снимки и оценки фактических результатов OpenSearch.
+3. Разобрать ошибки ранжирования, сохранить baseline и повторить ту же проверку
+   после изменений.
+4. Затем пройти readiness gate issue #21 для корпуса, индекса и Search API перед
+   подготовкой публичного frontend.
 
 ## История изменений статуса
 
+### 2026-09-12
+
+- Внутренняя лаборатория оценивает фактическую выдачу собственного OpenSearch;
+  ЦБД Минюста используется только для проверки источников и редакций актов.
+- Backend `0.8.3`; закрытая внутренняя лаборатория готова к пилоту.
+
+### 2026-09-06
+
+- Подготовлены бланки независимой юридической проверки relevance set v1 для
+  русского и кыргызского языков и инструкция для юриста.
+- Ближайшим этапом зафиксированы юридическая верификация, baseline качества
+  поиска и финальный readiness gate перед frontend.
+
+### 2026-08-18
+
+- Оценщик поиска использует `cross_fields` для совместного сопоставления
+  названия и текста документа.
+- На размеченном наборе из 50 запросов Recall@10 вырос с `0.23` до `0.30`,
+  MRR@10 — с `0.1854` до `0.2272`.
+- Версия backend обновлена до `0.5.1`.
+
+### 2026-08-14
+
+- Добавлены шаблон relevance set v1 и воспроизводимый расчёт Recall@K/MRR@K.
+- Версия backend обновлена до `0.5.0`.
+
+- Полный локальный индекс содержит 56 295 965 фрагментов и успешно отвечает на
+  RU/KY-запросы.
+- Добавлены атомарный checkpoint и безопасное продолжение прерванной загрузки
+  существующего индекса.
+- Добавлены retry/backoff для временных HTTP-сбоев и подробные ошибки Bulk API.
+- Ошибки чтения JSON теперь содержат точный путь и повторяются при временном сбое.
+- Версия backend обновлена до `0.4.1`.
+
+### 2026-08-13
+
+- Добавлен локальный одноузловой OpenSearch с `analysis-icu` без Dashboards.
+- Добавлена прямая потоковая загрузка корпуса пакетами до 25 МБ.
+- Версия backend обновлена до `0.4.0`.
+
+### 2026-08-13
+
+- Подтверждена полная успешная нормализация 209 958 загруженных документов.
+- Единственный отсутствующий документ `6` повторно не отдан API Минюста.
+- Добавлены mapping фрагментов и потоковый экспорт для OpenSearch Bulk API.
+- Версия backend обновлена до `0.3.0`.
+
+### 2026-08-12
+
+- Нормализатор запрещает пересекающиеся каталоги источника и результата,
+  восстанавливает прерванную публикацию и отбрасывает некорректные URL.
+- Версия backend обновлена до `0.2.2`.
+- Загрузчик пересоздаёт временный список документов на текущей странице не
+  только после HTTP 404, но и после исчерпания повторов запроса списка.
+- Версия backend обновлена до `0.2.1`.
+
+### 2026-08-10
+
+- Добавлена первая версия воспроизводимой нормализации локального архива ЦБД.
+- Добавлены атомарная публикация результатов, контрольные суммы, карантин ошибок и терминальный прогресс.
+- Версия backend обновлена до `0.2.0`.
+
+### 2026-08-06
+
+- Добавлен терминальный прогрессбар со скоростью и расчётным временем завершения.
+- Обработка HTTP 429 учитывает рекомендованную сервером задержку `Retry-After`.
+- Добавлено автоматическое восстановление после истечения серверного списка документов.
+- Версия backend обновлена до `0.1.2`.
+
+### 2026-08-05
+
+- Создана область `backend/ingestion` для получения правовых источников.
+- Реализована возобновляемая выгрузка ЦБД Минюста КР версии `0.1.0`.
+- Проверены двуязычные редакции, изображения, SQLite-манифест и повторный запуск.
+
 ### 2026-08-03
 
 - Репозиторий перестроен под весь проект юридической платформы; Telegram-бот перенесён в `tools/telegram-bot`.
@@ -162,4 +282,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend — не создан · Frontend — не создан
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.12.2 · Frontend v0.1.2

docs/operations/telegram-workspace-plan.md

@@ -398,4 +398,4 @@ Git сохраняет актуальную версию
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан

docs/product/frontend-design-readiness-plan.md

@@ -0,0 +1,128 @@
+# Готовность к публичному frontend
+
+Этот документ определяет обязательный результат до начала проектирования и
+разработки публичной пользовательской системы. Он дополняет
+[план frontend поисковой СПС](frontend-search-sps-plan.md): тот описывает MVP и
+спринты frontend, этот — критерий перехода к ним.
+
+## Правило перехода
+
+Публичный frontend и его выпуск начинаются, когда выполнены все обязательные
+пункты этого плана и пройден финальный readiness gate. Внутренняя
+авторизованная поисковая страница и лаборатория могут развиваться отдельно по
+явной задаче и используют только реальные существующие API.
+
+Персональные функции вне этого этапа: закладки, заметки, уведомления, RAG,
+судебная практика и внешние коммерческие сервисы. Внутренние учетные записи
+для лаборатории не включают публичную регистрацию.
+
+## 1. Данные и поисковый индекс
+
+### Результат
+
+В локальном OpenSearch доступен воспроизводимо собранный корпус актуальных
+редакций ЦБД Минюста КР, пригодный для поиска на русском и кыргызском языках.
+
+### Обязательные работы
+
+1. Подтвердить для каждого индексируемого документа наличие канонических
+   реквизитов: код, редакция, язык, статус, тип, орган, дата, номер, название
+   и ссылка на официальный источник.
+2. Зафиксировать правила для документов без текста и одноязычных редакций:
+   они не скрываются и не получают выдуманный перевод.
+3. Проверить versioned-индекс, mapping, анализаторы RU/KY, полноту актуальных
+   редакций и процедуру безопасной переиндексации с переключением alias.
+4. Описать и выполнить воспроизводимый сценарий обновления:
+   выгрузка → нормализация → новый индекс → проверка → переключение alias.
+
+### Критерий приёмки
+
+Команда может повторить обновление на чистом локальном окружении, проверить
+количество документов и фрагментов, а затем безопасно переключить поисковый
+alias без смешивания старого и нового корпуса.
+
+## 2. Relevance set и качество поиска
+
+### Результат
+
+Есть замороженный набор `relevance set v1` из 50 практических запросов:
+минимум 25 на русском и 25 на кыргызском языках.
+
+### Обязательные работы
+
+1. Заполнить рабочую копию из `backend/search/relevance-set-v1.template.json`
+   естественными формулировками пользователей и подтверждёнными
+   `document_code` релевантных действующих актов.
+2. Подтвердить документы по официальной ЦБД и проверить их наличие в локальном
+   индексе по `document_code`, не оценивая порядок поисковой выдачи. Не включать
+   запросы без установленного эталонного результата.
+3. Провести независимую вторую проверку языка, формулировки и полного списка
+   кодов. Спорные результаты не включать до согласования.
+4. После независимой проверки сохранить неизменяемую копию набора, затем
+   оценить собственную выдачу и сохранить baseline `Recall@10`/`MRR@10`. Новые
+   правила ранжирования проверять на том же наборе и сравнивать с baseline.
+5. Отдельно проверить распространённые сокращения, опечатки и запросы о
+   практическом действии. Правило принимается, только если улучшает
+   подтверждённые запросы и не создаёт ложных срабатываний в негативных
+   сценариях.
+
+### Критерий приёмки
+
+Оценщик проходит на полном наборе, выводит метрики и выдачу для каждого
+запроса; baseline и результаты его повторного запуска воспроизводимы.
+
+## 3. Справочники и контракт API
+
+### Результат
+
+Frontend получает все юридически значимые данные через версионированный
+OpenAPI-контракт, а не реконструирует их из текста фрагментов.
+
+### Обязательные работы
+
+1. Утвердить справочники v1: типы документов, органы принятия, статусы,
+   уровни действия и темы. Для каждого значения определить стабильный код и
+   подписи RU/KY.
+2. Реализовать и описать OpenAPI для:
+   - `GET /search` — запрос, язык, фильтры, сортировка, серверная пагинация,
+     выдержка и подсветка;
+   - `GET /search/filters` — допустимые значения фильтров;
+   - `GET /documents/{code}` — карточка актуального документа;
+   - `GET /documents/{code}/editions` и
+     `GET /documents/{code}/editions/{edition}` — редакции и их содержимое.
+3. Зафиксировать единые ответы для пустой выдачи, неизвестного документа,
+   недоступной редакции и ошибки upstream; для документов с одним языком
+   вернуть доступные языки явно.
+4. Добавить контрактные и интеграционные проверки API на локальном OpenSearch:
+   поиск, фильтры, пагинация, сортировка, документ, редакции и одноязычные
+   акты.
+
+### Критерий приёмки
+
+OpenAPI опубликован вместе с backend, тестовый клиент получает реальные данные
+по всем endpoint без mock-слоя, а результаты поиска открывают актуальный
+документ и выбранную редакцию.
+
+## 4. Финальный readiness gate
+
+Перед началом frontend выполнить и зафиксировать один сквозной сценарий:
+
+1. Обновить корпус из официальной ЦБД.
+2. Нормализовать данные и собрать новый versioned-индекс.
+3. Прогнать проверки индекса и relevance baseline.
+4. Переключить alias на проверенный индекс.
+5. Выполнить API-сценарии поиска, фильтрации, просмотра документа и редакции
+   на RU, KY и одноязычном документе.
+
+Gate считается пройденным, если все проверки успешны, зафиксированы версии
+backend и индекса, опубликованы известные ограничения и назначен ответственный
+за юридическую проверку relevance set.
+
+## Порядок issues
+
+1. Собрать и независимо проверить relevance set v1.
+2. Завершить проверку полноты данных и воспроизводимую переиндексацию с alias.
+3. Утвердить справочники v1.
+4. Реализовать OpenAPI и интеграционные проверки.
+5. Провести финальный readiness gate и только затем открыть задачу на
+   проектирование frontend.

docs/product/frontend-search-sps-plan.md

@@ -0,0 +1,265 @@
+# План реализации frontend поисковой СПС
+
+Основание: `docs/Функциональные_возможности_поисковой_СПС.docx`.
+
+Документ с требованиями описывает не только frontend, но и поиск, юридическую
+обработку, персональные данные, уведомления и внешние источники. Поэтому
+frontend следует начинать после появления нормализованной базы, поискового API
+и API документов.
+
+Требования необходимо адаптировать под Кыргызскую Республику: в исходном
+документе используются примеры ТК РФ и деление
+«федеральный/региональный/муниципальный», которое нельзя переносить без
+изменений.
+
+## Задачи до начала frontend-разработки
+
+### Обязательные для MVP
+
+1. Нормализовать архив Минюста:
+   - очистить HTML;
+   - выделить структуру документа и редакций;
+   - унифицировать статусы, органы, виды документов и даты;
+   - сохранить ссылки на официальный источник и дату получения;
+   - определить правила отображения документов без текста.
+2. Подготовить backend API:
+   - `GET /search`;
+   - `GET /search/filters`;
+   - `GET /documents/{code}`;
+   - `GET /documents/{code}/editions`;
+   - `GET /documents/{code}/editions/{edition}`;
+   - описание API в OpenAPI;
+   - серверную пагинацию, фильтрацию и сортировку.
+3. Развернуть поисковый сервис. Рекомендуемый вариант — self-hosted OpenSearch:
+   - отдельные поля и анализаторы для русского и кыргызского текстов;
+   - русский морфологический анализ;
+   - ICU-нормализация кыргызского текста;
+   - словари синонимов и сокращений;
+   - подсветка совпадений;
+   - индексирование всех редакций.
+4. Подготовить тестовый набор из 50–100 реальных запросов на русском и
+   кыргызском языках и вручную отметить ожидаемые результаты.
+5. Утвердить справочники:
+   - виды документов;
+   - органы принятия;
+   - статусы;
+   - уровни действия;
+   - тематический классификатор первой версии.
+
+OpenSearch имеет встроенный
+[русский морфологический анализатор](https://docs.opensearch.org/latest/analyzers/language-analyzers/russian/),
+поддерживает [синонимы и нечёткий поиск](https://docs.opensearch.org/latest/query-dsl/full-text/match/)
+и [подсветку результатов](https://docs.opensearch.org/latest/search-plugins/searching-data/highlight).
+Встроенного кыргызского морфологического анализатора в перечне нет, поэтому
+нужно отдельно проверить ICU и словари на реальных запросах.
+[ICU-анализатор](https://docs.opensearch.org/latest/analyzers/language-analyzers/icu/)
+обеспечивает Unicode-нормализацию, но сам по себе не гарантирует кыргызскую
+морфологию.
+
+### Сторонние сервисы, не нужные для MVP
+
+Их не следует подключать заранее:
+
+- Keycloak или другой OIDC-провайдер — перед закладками, папками и ролями;
+- SMTP, Telegram или Web Push — перед «документами на контроле»;
+- LibreOffice или Gotenberg — перед экспортом в Word, PDF и RTF;
+- поставщики судебной практики и экспертных комментариев — после проверки
+  лицензий;
+- источники курсов, календарей и справочных данных — перед соответствующим
+  разделом;
+- Sentry или аналог — опционально перед публичным запуском.
+
+## Граница MVP
+
+MVP — публичная справочно-поисковая система без регистрации и персональных
+функций.
+
+В MVP входят:
+
+- интерфейс на русском и кыргызском языках;
+- строка полнотекстового поиска;
+- исправление распространённых опечаток;
+- базовые синонимы и сокращения;
+- список результатов с подсвеченными фрагментами;
+- фильтры по языку, виду документа, органу, статусу и дате;
+- сортировка по релевантности и дате;
+- пагинация;
+- карточка документа;
+- актуальная редакция, статус и дата актуальности;
+- переключение между доступными языками;
+- поиск внутри открытого документа;
+- список редакций и открытие выбранной редакции;
+- ссылка на официальный источник и сведения о происхождении данных;
+- адаптивность, доступность, состояния загрузки и ошибок.
+
+Сравнение редакций, аккаунты, заметки, уведомления, RAG и судебная практика в
+MVP не входят.
+
+Интерфейс не должен предполагать наличие обоих языков. На момент полного
+скачивания архива распределение следующее:
+
+- только русский язык — 29 433 документа;
+- только кыргызский язык — 98 905 документов;
+- оба языка — 80 930 документов;
+- нет HTML-текста — 690 документов.
+
+## Рекомендуемая основа frontend
+
+Для внутреннего frontend лаборатории и авторизованного поиска используется
+одна HTML-страница, которую отдаёт существующий Search API, и браузерный
+JavaScript без сборщика и runtime-зависимостей. Это сохраняет текущую схему
+развёртывания. Публичный frontend после readiness gate можно оценить отдельно.
+
+- Next.js App Router и TypeScript;
+- CSS Modules с BEM-именованием;
+- дизайн-токены для цветов, отступов, типографики и состояний;
+- серверный `fetch` и URL-параметры вместо отдельного глобального хранилища;
+- Playwright для основных пользовательских сценариев;
+- адаптивный web-интерфейс без отдельного мобильного приложения.
+
+Next.js App Router поддерживает серверные компоненты, маршрутизацию и
+TypeScript в стандартной конфигурации. См.
+[официальную документацию](https://nextjs.org/docs/app).
+
+## Дизайн-процесс и внешние ориентиры
+
+При проектировании и проверке интерфейса используются следующие источники:
+
+- [jakubkrehel/skills](https://github.com/jakubkrehel/skills) — обязательная
+  комплексная проверка интерфейса через `better-interface`, включая UI,
+  типографику, цвета, доступность, layout и тексты;
+- [UI Skills](https://www.ui-skills.com/) — каталог практик и узких skills,
+  которые подключаются только под конкретную задачу после проверки их
+  содержания и лицензии;
+- [Refero Styles](https://styles.refero.design/) — библиотека визуальных
+  направлений и примеров `DESIGN.md` для поиска референсов.
+
+Правила применения:
+
+1. Визуальное направление «Түндүк» утверждено пользователем. Применять
+   корневой [DESIGN.md](../../DESIGN.md); повторный выбор направления не нужен.
+2. Не копировать чужую дизайн-систему целиком. Цвета, типографика, плотность и
+   компоненты должны учитывать длинные юридические тексты, два языка и
+   доступность.
+3. Перенести значения из корневого `DESIGN.md` в дизайн-токены проекта.
+   Не создавать отдельный `frontend/DESIGN.md` или конкурирующие спецификации.
+4. Дизайн-токены и компоненты Акылдаша являются источником истины. Внешние
+   рекомендации не могут отменять BEM, доступность, требования безопасности и
+   продуктовые ограничения проекта.
+5. Каждый завершённый пользовательский сценарий проходит `better-interface`
+   review. Перед выпуском MVP выполняется полный review поиска, фильтров и
+   просмотра документа.
+6. UI Skills используется для точечного поиска решения, а не для одновременного
+   смешивания нескольких визуальных стилей.
+
+Эти ресурсы используются на этапе проектирования и review и не являются
+runtime-зависимостями frontend. Регистрация в стороннем SaaS для MVP не нужна.
+
+## План спринтов MVP
+
+### Спринт 0 — фундамент, 1 неделя
+
+- создать `frontend/`;
+- настроить Next.js, TypeScript, lint и сборку;
+- применить утверждённую дизайн-систему «Түндүк» из корневого `DESIGN.md`;
+- установить полный набор `jakubkrehel/skills` для проектных design review;
+- определить маршруты и типы API;
+- создать дизайн-токены;
+- реализовать базовые компоненты: кнопка, поле, селект, статус, карточка,
+  пагинация;
+- создать общий layout и двуязычную навигацию;
+- добавить footer с версиями frontend и backend;
+- подготовить макеты поиска, результатов и документа;
+- провести первый `better-interface` review макетов;
+- настроить CI.
+
+Результат: интерфейсный каркас работает на mock-ответах API.
+
+### Спринт 1 — быстрый поиск, 2 недели
+
+- главная страница с поиском;
+- интеграция с `/search`;
+- список результатов;
+- подсветка совпадений;
+- URL, которым можно поделиться;
+- переключение RU/KY;
+- состояния загрузки, отсутствия результатов и ошибки API;
+- базовая мобильная версия.
+
+Результат: пользователь может найти документ и открыть результат.
+
+### Спринт 2 — точный отбор, 2 недели
+
+- фильтры по реквизитам;
+- сортировка;
+- пагинация;
+- отображение числа результатов;
+- сброс отдельных и всех фильтров;
+- сохранение состояния в URL;
+- доступное управление с клавиатуры;
+- адаптивная панель фильтров.
+
+Результат: поддерживается быстрый и реквизитный поиск.
+
+### Спринт 3 — просмотр документа, 2 недели
+
+- заголовок, реквизиты, статус и дата актуальности;
+- безопасное отображение очищенного HTML;
+- переключение языка;
+- поиск внутри документа;
+- навигация по найденным фрагментам;
+- список редакций;
+- открытие предыдущей редакции;
+- ссылка на ЦБД Минюста;
+- печать средствами браузера.
+
+Результат: пользователь может проверить текст и его происхождение.
+
+### Спринт 4 — стабилизация и выпуск, 2 недели
+
+- сквозные тесты поиска и просмотра;
+- проверка русских, кыргызских и одноязычных документов;
+- соответствие WCAG 2.2 AA;
+- защита от внедрения небезопасного HTML;
+- проверка производительности;
+- корректные метаданные страниц;
+- обработка недоступности API;
+- production-сборка и развёртывание;
+- пользовательское тестирование на 10–15 реальных юридических задачах.
+
+Результат: публичный MVP.
+
+Оценка frontend-части после готовности API: **9 недель**.
+
+## Спринты после MVP
+
+### Спринт 5 — персональные функции
+
+Базовый вход и смена пароля для внутреннего frontend уже реализованы. Для
+публичного продукта остаются закладки, заметки, подборки и сохранённые фильтры.
+
+### Спринт 6 — контроль изменений
+
+Документы на контроле, подписки на редакции и уведомления.
+
+### Спринт 7 — юридические связи
+
+Сравнение редакций, прямые и обратные ссылки, утратившие силу фрагменты.
+
+### Спринт 8 — практические материалы
+
+Формы, образцы, инструкции, чек-листы, календари и справочные данные.
+
+### Спринт 9 — расширенный анализ
+
+Судебная практика, экспертные комментарии, дерево связей и RAG с обязательными
+ссылками на источники.
+
+### Спринт 10 — корпоративные функции
+
+Роли, журналирование, API, интеграция с СЭД, расширенный экспорт и
+персонализация.
+
+---
+
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.12.2 · Frontend v0.1.2

docs/product/minjust-document-normalization-agent-task.md

@@ -0,0 +1,222 @@
+# Задание агенту: нормализация документов ЦБД Минюста КР
+
+## Цель
+
+Реализовать первую рабочую версию воспроизводимого нормализатора локального
+архива `data/minjust-cbd`. Нормализованные данные должны быть пригодны для
+последующей загрузки в OpenSearch, backend API и RAG, но подключение этих
+сервисов в текущую задачу не входит.
+
+## Текущее состояние
+
+- загрузчик находится в `backend/ingestion/minjust_cbd.py`;
+- сырой архив хранится в `data/minjust-cbd` и исключён из Git;
+- в read-only проверенном манифесте 209 958 документов;
+- один код, `6`, остаётся в таблице `errors` и отсутствует в `documents`;
+  это не подтверждает отсутствие документа в источнике: при последней
+  проверке OpenData API возвращал HTTP `404` для списка, `GetDocument(6)`,
+  контрольного `GetDocument(1096)` и `CheckAvailable`;
+- в нормализованном манифесте 209 958 успешных записей, schema `1`,
+  normalizer `1.0.0`; полная потоковая проверка всех файлов ещё выполняется.
+- документ содержит `metadata.json` и каталог `editions`;
+- редакция содержит `metadata.json`, `ru.html` и/или `ky.html`, иногда
+  изображения;
+- HTML создан Microsoft Word, может содержать некорректный
+  `<meta charset=unicode>`, служебные стили и неполную разметку;
+- файлы архива записаны загрузчиком в UTF-8;
+- часть документов одноязычная, а часть не содержит HTML-текста.
+
+## Обязательные ограничения
+
+1. Не изменять и не перезаписывать `data/minjust-cbd`.
+2. Не запускать полный проход по архиву во время автоматических тестов.
+3. Не подключать PostgreSQL, OpenSearch, OCR, embeddings, машинный перевод и
+   сетевые API.
+4. Сначала использовать стандартную библиотеку. Новая зависимость допустима
+   только если на реальных образцах доказано, что стандартный HTML-парсер не
+   обеспечивает корректность или безопасность.
+5. Все записи выполнять атомарно.
+6. Ошибка одного документа не должна останавливать длительный прогон.
+7. Повторный запуск должен пропускать неизменившиеся документы.
+8. Не изменять пользовательские файлы `logs/`, исходный DOCX и несвязанные
+   незакоммиченные изменения.
+
+## Размещение
+
+Использовать существующую backend-структуру:
+
+```text
+backend/
+  normalization/
+    minjust_cbd.py
+```
+
+Результат по умолчанию:
+
+```text
+data/minjust-normalized/
+  manifest.sqlite3
+  documents/<document_code>/document.json
+  documents/<document_code>/editions/<edition_code>/edition.json
+  documents/<document_code>/editions/<edition_code>/<lang>/content.html
+  documents/<document_code>/editions/<edition_code>/<lang>/content.txt
+  documents/<document_code>/editions/<edition_code>/<lang>/fragments.json
+```
+
+Каталог уже покрывается правилом игнорирования `data/`.
+
+## Канонические данные
+
+### `document.json`
+
+Сохранить как минимум:
+
+- `schema_version`;
+- `source_code`;
+- двуязычные `class`, `type`, `title`, `name`, `status`;
+- номера и даты без юридически неподтверждённых выводов;
+- флаги публичности;
+- органы, публикации, ключевые слова и классификаторы с сохранением дерева;
+- пути листьев иерархий для будущих фильтров;
+- ссылки из `References` без выдумывания связей;
+- `available_languages`;
+- список редакций;
+- путь к источнику и SHA-256 исходных файлов;
+- версию нормализатора и время обработки.
+
+Пустые строки привести к `null`, но не переводить значения и не заменять
+официальные формулировки собственными.
+
+### `edition.json`
+
+Сохранить:
+
+- исходный код редакции;
+- двуязычное название;
+- исходный тип;
+- доступные языки;
+- изображения без бинарных данных;
+- контрольные суммы источников;
+- признаки качества.
+
+Не считать дату из `Name` датой вступления редакции в силу и не назначать
+актуальную редакцию без подтверждённого правила источника.
+
+### Языковой вариант
+
+Для каждого имеющегося `ru.html` или `ky.html` сформировать:
+
+- `content.html` — безопасный HTML для frontend;
+- `content.txt` — извлечённый текст с сохранением смысловых переносов;
+- `fragments.json` — упорядоченные адресуемые блоки.
+
+Сырой HTML всегда читать как UTF-8, не доверяя его meta charset.
+
+## Очистка HTML
+
+- удалить `script`, `style`, `meta`, `link`, комментарии и служебные элементы;
+- удалить обработчики событий, inline-стили и опасные URL;
+- разрешить минимальный набор структурных тегов: заголовки, абзацы, `pre`,
+  списки, таблицы, безопасные ссылки, изображения и базовое текстовое
+  выделение;
+- нормализовать Unicode в NFC;
+- преобразовать неразрывные пробелы и избыточные пробелы только в
+  `content.txt`, не искажая отображаемый юридический текст;
+- не загружать внешние ресурсы;
+- относительные изображения связывать только с файлами внутри редакции;
+- неизвестную или сломанную разметку сохранять как текст, а не терять молча.
+
+## Фрагменты
+
+Минимальная версия должна создавать фрагмент для каждого содержательного
+блочного элемента. Распознавание статей и пунктов допускается только простыми
+проверяемыми правилами RU/KY; обычный абзац является fallback.
+
+Каждый фрагмент содержит:
+
+- стабильный `id`;
+- `document_code`, `edition_code`, `language`;
+- порядковую позицию;
+- тип блока;
+- чистый текст;
+- SHA-256 текста.
+
+Идентификатор должен быть детерминированным и включать документ, редакцию,
+язык и позицию. Не добавлять сложное сопоставление фрагментов между
+редакциями — это отдельная будущая задача.
+
+## Манифест и возобновление
+
+SQLite-манифест должен хранить:
+
+- код документа;
+- SHA-256 набора исходных файлов;
+- версию схемы и нормализатора;
+- время успешной обработки;
+- состояние и текст последней ошибки.
+
+Если checksum и версия нормализатора не изменились, документ пропускается.
+После успешной повторной обработки ошибка удаляется. Добавить `--limit` и
+понятный терминальный прогресс, пригодный для долгого запуска.
+
+## CLI и импорт из будущего backend
+
+CLI запускается из корня репозитория:
+
+```bash
+python3 backend/normalization/minjust_cbd.py
+```
+
+Предусмотреть параметры:
+
+- `--input`;
+- `--output`;
+- `--limit`;
+- `--refresh`;
+- `--log-level`.
+
+Основную функцию можно импортировать без запуска CLI. Не создавать
+планировщик, очередь задач или framework-интеграцию.
+
+## Проверки
+
+Добавить один компактный тестовый модуль, который проверяет:
+
+1. русскую редакцию;
+2. кыргызскую редакцию;
+3. двуязычную редакцию;
+4. Word HTML с опасным `script`, inline-стилем и `javascript:` URL;
+5. документ без HTML;
+6. повторный запуск и пропуск неизменившегося документа;
+7. продолжение после ошибки одного документа;
+8. детерминированные fragment ID и checksums.
+
+Провести пилотный read-only запуск на небольшой реальной выборке через
+`--limit`, не нормализовать весь архив в рамках разработки.
+
+## Документация и версия
+
+- описать запуск, структуру результата и ограничения в `backend/README.md`;
+- отметить реализацию в `docs/operations/project-status.md`;
+- это новая обратно совместимая backend-функция: увеличить minor-версию
+  backend по SemVer;
+- обновить все отображаемые backend-версии и footer, не меняя версию
+  Telegram-бота;
+- frontend по-прежнему помечать как не созданный.
+
+## Критерии приёмки
+
+- сырой архив не изменён;
+- тесты проходят;
+- `git diff --check` проходит;
+- пилотный запуск завершается без остановки на отдельных ошибках;
+- повторный пилотный запуск пропускает неизменившиеся документы;
+- unsafe HTML не попадает в `content.html`;
+- каждый фрагмент прослеживается до документа, редакции, языка и исходного
+  файла;
+- отсутствуют молча потерянные HTML или ошибки;
+- реализация не содержит PostgreSQL/OpenSearch/RAG-кода «на будущее».
+
+---
+
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан

docs/product/project-overview.md

@@ -44,4 +44,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2 · Backend — не создан · Frontend — не создан
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.12.2 · Frontend v0.1.2

docs/product/search-catalog-v1.md

@@ -0,0 +1,128 @@
+# Справочники Search API v1
+
+Статус: рабочий контракт Search API v1. Тематическая иерархия вынесена в
+отдельный черновой каталог и не входит в фильтры API v1.
+
+`code` — неизменяемый идентификатор, независимый от языка. Клиент хранит и
+передаёт только `code`; русское и кыргызское названия являются подписями и могут
+исправляться без изменения кода.
+
+## Типы документов
+
+| Code | RU | KY |
+| --- | --- | --- |
+| `constitution` | Конституция | Конституция |
+| `constitutional_law` | Конституционный Закон | Конституциалык Мыйзам |
+| `code` | Кодекс | Кодекс |
+| `law` | Закон | Мыйзам |
+| `decree` | Указ | Жарлык |
+| `resolution` | Постановление | Токтом |
+| `order` | Распоряжение | Распоряжение |
+| `instruction` | Инструкция | Инструкция |
+| `rules` | Правила | Правила |
+| `procedure` | Порядок | Порядок |
+| `provision` | Положение | Жобо |
+| `regulation` | Регламент | Регламент |
+| `charter` | Устав | Жобо (Устав) |
+| `program` | Программа | Программа |
+| `plan` | План | План |
+| `strategy` | Стратегия | Стратегия |
+| `concept` | Концепция | Концепция |
+| `doctrine` | Доктрина | Доктрина |
+| `agreement` | Соглашение | Соглашение |
+| `declaration` | Декларация | Декларация |
+| `registry` | Реестр | Реестр |
+| `norms` | Нормативы | Нормативы |
+| `model` | Модель | Модель |
+| `matrix` | Матрица | Матрица |
+| `study` | Исследование | Исследование |
+| `report` | Доклад | Доклад |
+| `principles` | Основные принципы | Основные принципы |
+| `unspecified` | Не указан | Көрсөтүлгөн эмес |
+
+## Статусы
+
+| Code | RU | KY |
+| --- | --- | --- |
+| `active` | Действует | Күчүндө |
+| `repealed` | Утратил силу | Күчүн жоготту |
+| `unspecified` | Не указан | Көрсөтүлгөн эмес |
+
+## Юридическая сила
+
+Это укрупнённая группировка для фильтра Search API, а не полная иерархия
+нормативных правовых актов из статьи 6 Закона КР «О нормативных правовых
+актах». В этой группировке Конституция и конституционные законы относятся к
+конституционному уровню, кодексы и законы — к законодательному, указы и
+постановления, перечисленные законом как виды НПА, — к подзаконному.
+
+| Code | RU | KY |
+| --- | --- | --- |
+| `constitutional` | Конституционный уровень | Конституциялык деңгээл |
+| `legislative` | Законодательный уровень | Мыйзам деңгээли |
+| `subordinate` | Подзаконный уровень | Мыйзам алдындагы деңгээл |
+
+Для прочих типов значение не назначается автоматически: одного названия типа
+недостаточно, чтобы установить юридическую силу утверждающего НПА. Основание:
+[статьи 4 и 6 Закона КР «О нормативных правовых актах»](https://cbd.minjust.gov.kg/4-3987/edition/53304/ru).
+Эта группировка реализована как фильтр `legal_force` Search API v1.
+
+## Органы принятия
+
+Орган содержит два уровня: стабильную группу и конкретный орган. В v1 группы
+следующие:
+
+| Code | RU | KY |
+| --- | --- | --- |
+| `president` | Президент | Президент |
+| `parliament` | Органы законодательной власти | Мыйзам чыгаруу бийлик органдары |
+| `cabinet` | Правительство и Кабинет Министров | Өкмөт жана Министрлер Кабинети |
+| `ministries_and_committees` | Министерства и государственные комитеты | Министрликтер жана мамлекеттик комитеттер |
+| `administrative_agencies` | Административные ведомства | Административдик ведомстволор |
+| `national_bank` | Национальный банк | Улуттук банк |
+| `other_state_bodies` | Иные государственные органы | Башка мамлекеттик органдар |
+| `local_representative_bodies` | Представительные органы местного самоуправления | Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары |
+| `other` | Прочие органы | Башка органдар |
+
+Конкретные министерства, муниципальные и айылные кенеши не получают ID из
+подписи: в выгрузке ЦБД их поле `Code` часто равно `null`. До появления
+первичного неизменяемого идентификатора API v1 выдаёт и принимает только код
+группы органа. Полный каталог конкретных органов — отдельная версия (`v2`),
+когда источник предоставит такие идентификаторы либо будет утверждён вручную
+поддерживаемый реестр.
+
+## Темы
+
+Полная исходная иерархия `GeneralClassifiers` сохранена отдельно в
+[`search-topic-taxonomy-v1.json`](search-topic-taxonomy-v1.json). Над исходными
+ветвями добавлены предложенные смысловые группы; исходные подписи и вложенность
+не переставляются. Поле `document_count` показывает число документов, где узел
+встречается. Предлагаются группы: конституционный строй и права; государство и
+публичное управление; право, суд и правопорядок; международные отношения;
+оборона и безопасность; экономика, финансы и предпринимательство; труд и
+социальная защита; здравоохранение; образование, наука и культура; земля,
+природные ресурсы и окружающая среда; сельское хозяйство; транспорт, связь,
+строительство и жильё; информация и СМИ; гражданские, семейные и жилищные
+отношения; общие и межотраслевые вопросы.
+
+Каталог является черновиком: группа корневой рубрики предложена по русской
+подписи, рубрики с несколькими совпадениями правил или без совпадений помещены
+в `review_required` и отмечены `manual_review`; остальные помечены
+`provisional_lexical`. В корпусе 2 065 узлов и 687 вариантов корневых рубрик;
+328 корневых вариантов ожидают ручной классификации. Отсутствующие в источнике
+кыргызские подписи оставлены пустыми: они отсутствуют у 1 524 узлов, новые
+переводы не придуманы. Внутренние `source-*` ID являются
+временными хешами кода источника и цепочки подписей; при `Code: null` они
+зависят от подписей и не утверждаются как публичные стабильные коды.
+
+Тематическая навигация и фильтр не входят в Search API v1. До включения в API
+нужно вручную утвердить смысловые группы, идентификаторы и недостающие KY-
+подписи. Источник — нормализованные данные ЦБД Минюста; команда для пересборки
+каталога находится в `tools/build_search_topic_taxonomy.py`.
+
+## Правила совместимости
+
+- Код из этой таблицы нельзя переиспользовать и нельзя менять его значение.
+- Новое значение добавляется только новой записью; удалённое остаётся доступно
+  для старых документов.
+- Запрос с неизвестным кодом возвращает `400`.

docs/product/search-relevance-review-interface-plan.md

@@ -0,0 +1,179 @@
+# План внутреннего интерфейса оценки поисковой выдачи
+
+## Цель
+
+Создать закрытую лабораторию релевантности: юрист оценивает фактическую выдачу
+нашего OpenSearch по практическим запросам. Цель — улучшать собственное
+ранжирование, а не воспроизводить алгоритм сайта Минюста КР.
+
+ЦБД Минюста используется как официальный источник текста, реквизитов, статуса
+и редакции акта. Порядок результатов и оценка их полезности определяются в
+нашем сервисе.
+
+Это внутренний рабочий инструмент, а не публичный frontend MVP. Он не включает
+регистрацию, личные кабинеты, публичный дизайн, сложные фильтры или сравнение
+редакций.
+
+## Сценарий юриста
+
+1. Указать поисковый запрос и язык.
+2. Получить первые 10–20 результатов в точном порядке OpenSearch.
+3. Увидеть позицию каждого результата: `#1`, `#2` и далее.
+4. Открыть выбранный документ по клику на название в правой панели страницы.
+5. Поставить каждому просмотренному результату оценку и комментарий.
+6. Сохранить снимок выдачи и оценок.
+7. Передать накопленные записи на анализ ранжирования.
+
+## Оценка результата
+
+| Балл | Значение |
+| ---: | --- |
+| 0 | Нерелевантен: совпали слова, но акт не отвечает на вопрос. |
+| 1 | Косвенно полезен: относится к теме, но прямого ответа нет. |
+| 2 | Частично полезен: отвечает не полностью или требует другого акта. |
+| 3 | Прямо и достаточно отвечает на запрос. |
+
+Оценка относится к отдельному документу, а не ко всей выдаче. Результат без
+оценки считается непросмотренным, а не нерелевантным.
+
+## Интерфейс
+
+Рекомендуемый экран — две панели.
+
+- Вверху: поле запроса, переключатель RU/KY, выбор числа результатов и кнопка
+  «Найти».
+- Слева: карточки результатов в порядке выдачи. Карточка содержит позицию,
+  название, тип, статус, дату, номер и фрагмент текста.
+- Справа: заголовок, реквизиты, очищенный HTML выбранной редакции и ссылка на
+  официальный источник.
+- В карточке: кнопки оценки `0`, `1`, `2`, `3` и раскрываемое поле
+  комментария.
+- Внизу: имя или псевдоним проверяющего, общий комментарий и кнопка
+  «Сохранить оценку».
+
+Правая панель предпочтительнее popup: она не блокируется браузером, сохраняет
+контекст выдачи и работает на одном экране с оценкой.
+
+### Доступность оценки и документа
+
+Оценка реализуется нативной группой `radio` внутри `fieldset` с `legend`
+«Оценка результата». У каждого значения есть видимая подпись: «0 —
+нерелевантен», «1 — косвенно полезен», «2 — частично полезен», «3 — прямо
+отвечает». Нельзя передавать смысл оценки только цветом. Все элементы управления
+доступны с клавиатуры, имеют видимый `:focus-visible`; выбранный результат
+обозначается текстом и визуальным состоянием.
+
+На узком экране список результатов занимает всю страницу. Кнопка «Открыть
+документ» открывает полноэкранный нативный `<dialog>` с явной кнопкой
+«Закрыть». При закрытии фокус возвращается на исходную кнопку «Открыть
+документ».
+
+### Состояния
+
+- Во время поиска и сохранения показывается состояние загрузки; повторная
+  отправка на это время недоступна. Стабильная пустая область `role="status"`
+  в DOM объявляет начало поиска, число результатов и успешное сохранение.
+- Пустая выдача сообщает: «По запросу „…“ ничего не найдено» и предлагает
+  «Изменить запрос».
+- Ошибка поиска сообщает причину и предлагает «Повторить поиск»; текст ошибки
+  выводится в `role="alert"`.
+- Ошибка сохранения сообщает: «Не удалось сохранить. Проверьте подключение и
+  повторите». Черновик остаётся в браузере, текст ошибки выводится в
+  `role="alert"`.
+- После сохранения выводится: «Оценка сохранена · № … · дата и время» в
+  указанной стабильной области `role="status"`.
+
+До сохранения черновик хранится в `localStorage`. После успешного сохранения
+интерфейс показывает ID записи и время сохранения.
+
+## Backend и хранение
+
+Использовать существующие endpoint:
+
+- `GET /search` — получить ранжированный список результатов;
+- `GET /documents/{code}/editions/{edition}` — получить текст выбранной
+  редакции.
+
+Добавить два endpoint:
+
+- `POST /search-reviews` — валидирует и сохраняет оценку;
+- `GET /search-reviews/export` — отдаёт накопленные записи в JSON.
+
+Для первой версии достаточно отдельной SQLite-базы. Это стандартная библиотека
+Python, данные переживают перезапуск и легко выгружаются для анализа. Доступ к
+интерфейсу и всем endpoint `search-reviews`, включая экспорт, должен быть
+ограничен локальной сетью/VPN или аутентификацией reverse proxy.
+
+Одна запись представляет один сохранённый поисковый сеанс:
+
+| Поле | Назначение |
+| --- | --- |
+| `id`, `created_at` | Идентификатор и время сохранения. |
+| `reviewer` | Имя или псевдоним проверяющего. |
+| `query`, `language` | Исходный запрос и язык поиска. |
+| `index_name`, `algorithm_version` | Версия индекса и алгоритма на момент оценки. |
+| `top_result_code` | Код документа в позиции `#1`. |
+| `results_json` | Снимок результатов в исходном порядке с оценками и комментариями. |
+| `overall_comment` | Общий комментарий к выдаче. |
+
+В `results_json` для каждого результата сохраняются: `rank`, `document_code`,
+`edition_code`, название, реквизиты, фрагмент, оценка и комментарий. Снимок
+выдачи обязателен: после изменения алгоритма можно будет восстановить именно
+тот результат, который видел юрист.
+
+## Правила сохранения
+
+- Запрос не пустой, не длиннее 500 символов.
+- Оценка может быть только целым числом от 0 до 3 либо отсутствовать у
+  непросмотренного результата.
+- Комментарии имеют ограничение длины; пользовательские значения не вставляются
+  в HTML.
+- `GET /search` возвращает краткоживущий HMAC-подписанный снимок выдачи:
+  запрос, язык, индекс, результаты и их позиции. `POST /search-reviews`
+  принимает этот снимок и только оценки с комментариями. Сервер проверяет
+  подпись и срок, самостоятельно формирует `results_json` и отклоняет оценки
+  для отсутствующих либо подменённых позиций и документов.
+- Нельзя передавать персональные данные или закрытые материалы в комментариях.
+- Кнопка «Сохранить оценку» остаётся доступной до отправки: отсутствующие
+  обязательные поля проверяются после нажатия, ошибка показана рядом с полем и
+  фокус переводится на первое некорректное поле.
+
+## Анализ данных
+
+Экспорт должен содержать исходный JSON-снимок, чтобы его можно было обработать
+скриптом или открыть в табличном инструменте. Первый отчёт строит:
+
+- среднюю оценку для каждой позиции выдачи;
+- долю результатов с оценкой `3` в top-1, top-3 и top-10;
+- запросы, где нет результатов с оценкой `2` или `3`;
+- документы, которые часто получают низкую оценку в первых позициях;
+- комментарии для ручного разбора ошибок.
+
+Сырые оценки не должны автоматически менять веса поиска. Сначала команда
+разбирает причины: анализатор, синонимы, статус, отсутствие документа,
+неправильная формулировка запроса или юридическая неоднозначность.
+
+## Этапы реализации
+
+1. Утвердить шкалу 0–3, обязательность имени проверяющего и правила доступа.
+2. Добавить SQLite-хранилище, валидацию, сохранение и JSON-экспорт.
+3. Добавить статическую внутреннюю страницу в существующий Python-сервер, без
+   Next.js и отдельного публичного приложения.
+4. Подключить поиск, правую панель документа, черновик, адаптивный режим и
+   сохранение в закреплённой панели действий на широком экране.
+5. Добавить минимальные backend-проверки сохранения, повторного запуска,
+   экспорта и недопустимых оценок.
+6. Провести ручный прогон на десяти русскоязычных практических запросах с
+   двумя юристами.
+7. На собранных записях настроить OpenSearch и повторить тот же набор запросов.
+
+## Критерий готовности
+
+Юрист вводит запрос, видит порядок выдачи, открывает документ, выставляет
+оценки и комментарии, сохраняет их. Экспорт содержит запрос, язык, документ
+на позиции `#1`, полный порядок результатов, оценки, комментарии и версию
+индекса. Данные можно сравнить до и после изменения алгоритма.
+
+---
+
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан

docs/team/ai-skills-for-beginners.md

@@ -180,4 +180,4 @@
 
 ---
 
-Акылдаш · Telegram-бот v0.2.2
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан

infra/backup/apply_restore.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+"""Install a verified staging restore onto an explicitly marked new VPS."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+from pathlib import Path
+
+from archive import COMPONENTS, check_sqlite, digest
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_DEST = Path("/etc/akyldash/production.env")
+SECRET_DEST = Path("/etc/akyldash/review_secret")
+DATA_DEST = Path("/var/lib/akyldash/data")
+MARKER = Path("/etc/akyldash/recovery-target")
+PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+VOLUME_TARGETS = {
+    "/usr/share/opensearch/data": "opensearch-data",
+    "/data": "caddy-data",
+    "/config": "caddy-config",
+}
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def require_empty_directory(path: Path) -> None:
+    if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
+        raise ValueError(f"restore destination must be an existing empty directory: {path}")
+
+
+def tree_size(root: Path) -> int:
+    total = 0
+    for directory, names, files in os.walk(root, followlinks=False):
+        names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
+        for name in files:
+            path = Path(directory) / name
+            if path.is_symlink():
+                continue
+            if path.is_file():
+                total += path.stat().st_size
+    return total
+
+
+def copy_contents(source: Path, destination: Path) -> None:
+    for child in source.iterdir():
+        target = destination / child.name
+        if child.is_dir() and not child.is_symlink():
+            shutil.copytree(child, target, symlinks=True)
+        else:
+            shutil.copy2(child, target, follow_symlinks=False)
+
+
+def empty_directory(path: Path) -> None:
+    for child in path.iterdir():
+        if child.is_dir() and not child.is_symlink():
+            shutil.rmtree(child)
+        else:
+            child.unlink()
+
+
+def validate_stage(staging: Path) -> dict:
+    staging = staging.resolve(strict=True)
+    if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
+        raise ValueError("staging directory must be a direct child of /var/tmp")
+    if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
+        raise ValueError("staging directory must be root-owned mode 0700")
+    manifest_path = staging / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or set(manifest.get("components", [])) != COMPONENTS
+    ):
+        raise ValueError("staging manifest is invalid")
+    for name in COMPONENTS - {"production-env", "review-secret"}:
+        component = staging / name
+        if not component.is_dir() or component.is_symlink():
+            raise ValueError(f"staged component must be a directory: {name}")
+    for name in ("production-env", "review-secret"):
+        path = staging / name
+        if not path.is_file() or path.is_symlink():
+            raise ValueError(f"staged sensitive component must be a regular file: {name}")
+    return manifest
+
+
+def validate_checkout(manifest: dict) -> None:
+    if ROOT.is_symlink() or not (ROOT / ".git").exists():
+        raise ValueError("approved Git checkout is missing or is a symlink")
+    if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
+        raise ValueError("recovery checkout must be clean")
+    unsafe = subprocess.run(
+        ["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if unsafe:
+        raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
+    revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
+    if revision != manifest.get("git_revision"):
+        raise ValueError("checkout Git revision does not match the backup manifest")
+    for relative, expected in manifest.get("deployment_files", {}).items():
+        if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
+            raise ValueError("Compose or Caddyfile does not match the backup manifest")
+    if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
+        raise ValueError("backup manifest is missing deployment file hashes")
+
+
+def validate_recovery_target() -> None:
+    if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("root-only recovery-target marker is required")
+    hostname = socket.getfqdn()
+    addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
+    if not hostname or not addresses:
+        raise ValueError("cannot verify the recovery host identity")
+    if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
+        raise ValueError("refusing to apply recovery data to the inventoried production VPS")
+
+
+def compose_configuration(staging: Path) -> dict:
+    rendered = run(
+        [
+            "docker", "compose", "--env-file", str(staging / "production-env"),
+            "-f", str(COMPOSE_FILE), "config", "--format", "json",
+        ],
+        capture=True,
+    )
+    config = json.loads(rendered)
+    data_source = next(
+        mount["source"]
+        for mount in config["services"]["backend"]["volumes"]
+        if mount["target"] == "/app/data"
+    )
+    secret_source = config["secrets"]["review_secret"]["file"]
+    if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
+        raise ValueError("backup production paths do not match the clean recovery host layout")
+    return config
+
+
+def expected_volume_names(config: dict) -> set[str]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    names: set[str] = set()
+    for target in VOLUME_TARGETS:
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        names.add(config["volumes"][mount["source"]]["name"])
+    return names
+
+
+def volume_destinations(config: dict) -> dict[str, Path]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    destinations = {}
+    for target, component in VOLUME_TARGETS.items():
+        mount = mounts.get(target)
+        if not mount or mount["type"] != "volume":
+            raise ValueError(f"expected persistent volume at {target}")
+        name = config["volumes"][mount["source"]]["name"]
+        destinations[component] = Path(
+            run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
+        )
+    return destinations
+
+
+def apply_restore(staging: Path) -> None:
+    if os.geteuid() != 0:
+        raise ValueError("restore apply must run as root")
+    validate_recovery_target()
+    manifest = validate_stage(staging)
+    validate_checkout(manifest)
+    config = compose_configuration(staging)
+    require_empty_directory(DATA_DEST)
+    if ENV_DEST.exists() or SECRET_DEST.exists():
+        raise ValueError("production environment and secret files must not already exist")
+    if run(["docker", "ps", "--quiet"], capture=True):
+        raise ValueError("recovery target must have no running Docker containers")
+
+    volume_names = expected_volume_names(config)
+    for name in volume_names:
+        result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
+        if result.returncode == 0:
+            raise ValueError(f"recovery volume already exists: {name}")
+
+    created_volumes: list[str] = []
+    created_files: list[Path] = []
+    data_started = False
+    try:
+        for name in volume_names:
+            run(["docker", "volume", "create", name])
+            created_volumes.append(name)
+
+        destinations = volume_destinations(config)
+        destinations["data"] = DATA_DEST
+
+        required: dict[int, int] = {}
+        free: dict[int, int] = {}
+        for component, destination in destinations.items():
+            device = destination.stat().st_dev
+            required[device] = required.get(device, 0) + tree_size(staging / component)
+            free[device] = shutil.disk_usage(destination).free
+        for device, size in required.items():
+            if free[device] < size + 512 * 1024 * 1024:
+                raise ValueError("recovery host lacks space for a safe data copy")
+
+        for component, destination in destinations.items():
+            require_empty_directory(destination)
+            if component == "data":
+                data_started = True
+            copy_contents(staging / component, destination)
+
+        for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
+            os.chown(directory, 1000, 1000, follow_symlinks=False)
+            for name in files:
+                os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
+
+        for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
+            shutil.copyfile(source, destination)
+            created_files.append(destination)
+            os.chown(destination, 0, 0)
+            os.chmod(destination, 0o600)
+        check_sqlite(DATA_DEST)
+    except BaseException:
+        for path in created_files:
+            path.unlink(missing_ok=True)
+        if data_started:
+            empty_directory(DATA_DEST)
+        for name in created_volumes:
+            subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
+        raise
+
+    MARKER.unlink()
+    print("PASS staged recovery data applied to the marked new host; services remain stopped")
+
+
+def main() -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--staging-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        apply_restore(args.staging_dir)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infra/backup/archive.py

@@ -0,0 +1,393 @@
+#!/usr/bin/env python3
+"""Create and verify cold, local Akyldash backups; restore only into staging."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path, PurePosixPath
+
+COMPONENTS = {
+    "data",
+    "opensearch",
+    "caddy-data",
+    "caddy-config",
+    "production-env",
+    "review-secret",
+}
+NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+
+def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
+    members = archive.getmembers()
+    names: set[str] = set()
+    for member in members:
+        name = PurePosixPath(member.name)
+        if name.is_absolute() or ".." in name.parts or not name.parts:
+            raise ValueError(f"unsafe archive path: {member.name}")
+        if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
+            raise ValueError(f"non-canonical archive path: {member.name}")
+        if member.name in names:
+            raise ValueError(f"duplicate archive path: {member.name}")
+        names.add(member.name)
+        if not (member.isdir() or member.isfile() or member.issym()):
+            raise ValueError(f"unsupported archive entry: {member.name}")
+        if member.issym():
+            tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
+    if "manifest.json" not in names:
+        raise ValueError("backup manifest is missing")
+    if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
+        raise ValueError("backup is missing one or more required components")
+    manifest_member = next(item for item in members if item.name == "manifest.json")
+    if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
+        raise ValueError("backup manifest is not a regular file")
+    return members
+
+
+def read_manifest(archive: tarfile.TarFile) -> dict:
+    checked_members(archive)
+    stream = archive.extractfile("manifest.json")
+    if stream is None:
+        raise ValueError("backup manifest is unreadable")
+    manifest = json.load(stream)
+    components = manifest.get("components") if isinstance(manifest, dict) else None
+    if (
+        not isinstance(manifest, dict)
+        or manifest.get("format") != 1
+        or not isinstance(components, list)
+        or not all(isinstance(item, str) for item in components)
+        or set(components) != COMPONENTS
+    ):
+        raise ValueError("backup manifest has an unsupported format or component set")
+    if not isinstance(manifest.get("images"), dict):
+        raise ValueError("backup manifest has no image records")
+    if not isinstance(manifest.get("deployment_files"), dict):
+        raise ValueError("backup manifest has no deployment file records")
+    for member in archive.getmembers():
+        if member.name == "manifest.json":
+            continue
+        if PurePosixPath(member.name).parts[0] not in COMPONENTS:
+            raise ValueError(f"unexpected top-level archive path: {member.name}")
+    return manifest
+
+
+def verify_archive(path: Path) -> dict:
+    checksum_path = Path(f"{path}.sha256")
+    expected = checksum_path.read_text(encoding="ascii").split()[0]
+    if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
+        raise ValueError("archive SHA-256 does not match its sidecar")
+    with tarfile.open(path, "r:gz") as archive:
+        return read_manifest(archive)
+
+
+def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
+    sources: dict[str, Path] = {}
+    for value in values:
+        name, separator, raw_path = value.partition("=")
+        if not separator or not NAME_RE.fullmatch(name) or name in sources:
+            raise ValueError(f"invalid or duplicate source: {value}")
+        source = Path(raw_path)
+        if not source.is_absolute() or not source.exists() or source.is_symlink():
+            raise ValueError(f"source must be an existing absolute path: {raw_path}")
+        sources[name] = source.resolve(strict=True)
+    if set(sources) != required:
+        missing = ", ".join(sorted(required - sources.keys()))
+        extra = ", ".join(sorted(sources.keys() - required))
+        raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
+    return sources
+
+
+def check_sqlite(root: Path) -> None:
+    for path in root.rglob("*.sqlite3"):
+        connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
+        try:
+            result = connection.execute("PRAGMA quick_check").fetchone()
+            if result != ("ok",):
+                raise ValueError(f"SQLite integrity check failed: {path}")
+        finally:
+            connection.close()
+
+
+def compose_storage(config: dict) -> dict[str, Path | None]:
+    mounts = {
+        mount["target"]: mount
+        for service in config["services"].values()
+        for mount in service.get("volumes", [])
+    }
+    volumes = subprocess.run(
+        ["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
+    ).stdout.split()
+    sources: dict[str, Path | None] = {}
+    for target, component in (
+        ("/app/data", "data"),
+        ("/usr/share/opensearch/data", "opensearch"),
+        ("/data", "caddy-data"),
+        ("/config", "caddy-config"),
+    ):
+        mount = mounts.get(target)
+        if not mount:
+            raise ValueError(f"persistent Compose mount is missing: {target}")
+        if mount["type"] == "bind":
+            source = Path(mount["source"])
+        elif mount["type"] == "volume":
+            volume_name = config["volumes"][mount["source"]]["name"]
+            if volume_name not in volumes:
+                sources[component] = None
+                continue
+            result = subprocess.run(
+                ["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            source = Path(result.stdout.strip())
+        else:
+            raise ValueError(f"unsupported persistent mount type at {target}")
+        if not source.is_dir() or source.is_symlink():
+            raise ValueError(f"cannot determine persistent data state for {component}: {source}")
+        sources[component] = source
+    return sources
+
+
+def has_persistent_data(sources: dict[str, Path | None]) -> bool:
+    return any(any(path.iterdir()) for path in sources.values() if path is not None)
+
+
+def is_preloaded_corpus_only(data: Path | None) -> bool:
+    if data is None:
+        return False
+    entries = list(data.iterdir())
+    return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
+
+
+def compose_sources(
+    compose_file: Path, env_file: Path, initial_deployment: bool = False
+) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
+    command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
+    rendered = subprocess.run(
+        [*command, "config", "--format", "json"], check=True, capture_output=True, text=True
+    )
+    config = json.loads(rendered.stdout)
+    container_result = subprocess.run(
+        [*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
+    )
+    containers = container_result.stdout.split()
+    storage = compose_storage(config)
+    persistent_data = has_persistent_data(storage)
+    if not containers and not persistent_data:
+        raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
+    missing_volumes = any(path is None for path in storage.values())
+    if missing_volumes and containers:
+        raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
+    if missing_volumes and not initial_deployment:
+        raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
+    if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
+        raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
+    images: dict[str, dict[str, str]] = {}
+    for container in containers:
+        result = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                '{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
+                container,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        fields = result.stdout.strip().split("\t")
+        if len(fields) != 4 or fields[0].lower() != "false":
+            raise ValueError("stop all Compose services before taking a cold backup")
+        images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
+    if containers and set(images) != set(config["services"]):
+        raise ValueError("not all configured Compose services have stopped containers")
+
+    secret_file = Path(config["secrets"]["review_secret"]["file"])
+    sources = {
+        **storage,
+        "production-env": env_file,
+        "review-secret": secret_file,
+    }
+    repository = compose_file.resolve().parents[2]
+    changed = subprocess.run(
+        ["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if changed:
+        raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
+    revision = subprocess.run(
+        ["git", "-C", str(repository), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    deployment_files = {
+        relative: digest(repository / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    present = {name: path for name, path in sources.items() if path is not None}
+    missing = {name for name, path in sources.items() if path is None}
+    parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
+    return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
+
+
+def create_backup(
+    output: Path,
+    sources: dict[str, Path | None],
+    revision: str,
+    images: dict | None = None,
+    deployment_files: dict[str, str] | None = None,
+) -> None:
+    output = output.absolute()
+    parent = output.parent
+    if not parent.is_dir() or parent.stat().st_mode & 0o077:
+        raise ValueError("backup directory must exist and have mode 0700 or stricter")
+    if output.exists() or Path(f"{output}.sha256").exists():
+        raise ValueError("refusing to overwrite an existing backup")
+    for name, source in sources.items():
+        if source is None:
+            continue
+        if source.is_symlink() or not (source.is_dir() or source.is_file()):
+            raise ValueError(f"source must be a regular file or directory: {name}")
+        if source.is_dir() and (output == source or source in output.parents):
+            raise ValueError(f"backup output cannot be inside source {name}")
+    for name in ("production-env", "review-secret"):
+        if sources[name] is None or not sources[name].is_file():
+            raise ValueError(f"sensitive component must be a regular file: {name}")
+    if sources["data"] is not None and sources["data"].is_dir():
+        check_sqlite(sources["data"])
+
+    manifest = {
+        "format": 1,
+        "created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+        "git_revision": revision,
+        "images": images or {},
+        "deployment_files": deployment_files or {},
+        "components": sorted(sources),
+    }
+    with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
+        temporary = Path(temp.name)
+    os.chmod(temporary, 0o600)
+    checksum_temp = Path(f"{temporary}.sha256")
+    try:
+        with tarfile.open(temporary, "w:gz") as archive:
+            encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
+            info = tarfile.TarInfo("manifest.json")
+            info.size = len(encoded)
+            info.mode = 0o600
+            archive.addfile(info, io.BytesIO(encoded))
+            for name, source in sorted(sources.items()):
+                if source is None:
+                    info = tarfile.TarInfo(f"{name}/")
+                    info.type = tarfile.DIRTYPE
+                    info.mode = 0o700
+                    archive.addfile(info)
+                    continue
+                def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
+                    if name in {"production-env", "review-secret"}:
+                        member.mode = 0o600
+                    if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
+                        member.mode = 0o600
+                    return member
+
+                archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
+        with tarfile.open(temporary, "r:gz") as archive:
+            read_manifest(archive)
+        checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
+            stream.write(f"{digest(temporary)}  {output.name}\n")
+        os.link(temporary, output)
+        try:
+            os.link(checksum_temp, Path(f"{output}.sha256"))
+        except Exception:
+            output.unlink(missing_ok=True)
+            raise
+    finally:
+        temporary.unlink(missing_ok=True)
+        checksum_temp.unlink(missing_ok=True)
+
+
+def restore_to_staging(backup: Path, target: Path) -> None:
+    if not target.name.startswith("akyldash-restore-"):
+        raise ValueError("restore target must be a new akyldash-restore-* staging directory")
+    parent = target.parent.resolve()
+    if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
+        raise ValueError("restore staging directory must be under /tmp or /var/tmp")
+    if target.exists():
+        raise ValueError("restore target already exists; refusing to overwrite it")
+    target = parent / target.name
+    verify_archive(backup)
+    target.mkdir(mode=0o700)
+    try:
+        with tarfile.open(backup, "r:gz") as archive:
+            members = checked_members(archive)
+            required_bytes = sum(member.size for member in members if member.isfile())
+            if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
+                raise ValueError("insufficient free space for a validated restore staging copy")
+            for member in members:
+                archive.extract(member, target, filter="data")
+        check_sqlite(target / "data")
+    except Exception:
+        shutil.rmtree(target)
+        raise
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    create = subparsers.add_parser("create", help="create a cold backup archive")
+    create.add_argument("--output", required=True, type=Path)
+    create.add_argument("--compose-file", required=True, type=Path)
+    create.add_argument("--env-file", required=True, type=Path)
+    create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
+    verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
+    verify.add_argument("archive", type=Path)
+    restore = subparsers.add_parser("restore", help="extract into a new staging directory")
+    restore.add_argument("archive", type=Path)
+    restore.add_argument("--target-dir", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        if args.command in {"create", "restore"} and os.geteuid() != 0:
+            raise ValueError("create and restore must run as root to protect production secrets and data")
+        if args.command == "create":
+            sources, revision, images, deployment_files = compose_sources(
+                args.compose_file, args.env_file, args.initial_deployment
+            )
+            create_backup(args.output, sources, revision, images, deployment_files)
+        elif args.command == "verify":
+            manifest = verify_archive(args.archive)
+            print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
+        else:
+            restore_to_staging(args.archive, args.target_dir)
+            print(f"PASS restored and validated staging data at {args.target_dir}")
+    except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infra/backup/test_apply_restore.py

@@ -0,0 +1,90 @@
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import apply_restore
+
+
+class ApplyRestoreTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.staging = self.root / "akyldash-restore-test"
+        self.staging.mkdir()
+        for component in apply_restore.COMPONENTS:
+            path = self.staging / component
+            if component in {"production-env", "review-secret"}:
+                path.write_text("private", encoding="utf-8")
+            else:
+                path.mkdir()
+                (path / "payload.txt").write_text(component, encoding="utf-8")
+        self.data = self.root / "data"
+        self.data.mkdir()
+        self.volumes = {
+            name: self.root / name
+            for name in ("opensearch", "caddy-data", "caddy-config")
+        }
+        for path in self.volumes.values():
+            path.mkdir()
+        self.env = self.root / "production.env"
+        self.secret = self.root / "review_secret"
+        self.marker = self.root / "recovery-target"
+        self.marker.touch(mode=0o600)
+        self.config = {
+            "volumes": {
+                "opensearch-data": {"name": "akyldash_opensearch-data"},
+                "caddy-data": {"name": "akyldash_caddy-data"},
+                "caddy-config": {"name": "akyldash_caddy-config"},
+            }
+        }
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def run_apply(self):
+        def docker_run(command, **_kwargs):
+            if command[:3] == ["docker", "ps", "--quiet"]:
+                return ""
+            if command[1:3] == ["volume", "create"]:
+                return command[-1]
+            return ""
+
+        with (
+            patch("apply_restore.os.geteuid", return_value=0),
+            patch("apply_restore.validate_recovery_target"),
+            patch("apply_restore.validate_stage", return_value={"format": 1}),
+            patch("apply_restore.validate_checkout"),
+            patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}),
+            patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])),
+            patch("apply_restore.volume_destinations", return_value=self.volumes),
+            patch("apply_restore.DATA_DEST", self.data),
+            patch("apply_restore.ENV_DEST", self.env),
+            patch("apply_restore.SECRET_DEST", self.secret),
+            patch("apply_restore.MARKER", self.marker),
+            patch("apply_restore.run", side_effect=docker_run),
+            patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)),
+            patch("apply_restore.os.chown"),
+        ):
+            apply_restore.apply_restore(self.staging)
+
+    def test_applies_components_and_protects_secret_files(self):
+        self.run_apply()
+        self.assertEqual((self.data / "payload.txt").read_text(), "data")
+        self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch")
+        self.assertEqual(self.env.stat().st_mode & 0o777, 0o600)
+        self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600)
+        self.assertFalse(self.marker.exists())
+
+    def test_refuses_nonempty_data_destination_before_writing_secrets(self):
+        (self.data / "keep.txt").write_text("existing", encoding="utf-8")
+        with self.assertRaisesRegex(ValueError, "empty directory"):
+            self.run_apply()
+        self.assertEqual((self.data / "keep.txt").read_text(), "existing")
+        self.assertFalse(self.env.exists())
+        self.assertTrue(self.marker.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/backup/test_archive.py

@@ -0,0 +1,362 @@
+import hashlib
+import json
+import sqlite3
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from archive import COMPONENTS, compose_sources, create_backup, restore_to_staging, verify_archive
+
+
+class BackupArchiveTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        self.private = self.root / "private"
+        self.private.mkdir(mode=0o700)
+        self.sources = {}
+        for name in COMPONENTS:
+            source = self.root / name
+            source.mkdir()
+            self.sources[name] = source
+        for name in ("production-env", "review-secret"):
+            path = self.root / f"{name}.txt"
+            path.write_text("private", encoding="utf-8")
+            path.chmod(0o644)
+            self.sources[name] = path
+        connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
+        connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
+        connection.commit()
+        connection.close()
+        self.backup = self.private / "backup.tar.gz"
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def test_create_verify_and_restore_only_to_new_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        manifest = verify_archive(self.backup)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        self.assertEqual(manifest["images"], {})
+
+        target = self.root / "akyldash-restore-test"
+        restore_to_staging(self.backup, target)
+        self.assertTrue((target / "data/search-users.sqlite3").is_file())
+        self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
+        self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
+        self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
+
+    def test_rejects_checksum_mismatch_without_creating_staging(self):
+        create_backup(self.backup, self.sources, "abc123")
+        self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
+        target = self.root / "akyldash-restore-tampered"
+        with self.assertRaisesRegex(ValueError, "SHA-256"):
+            restore_to_staging(self.backup, target)
+        self.assertFalse(target.exists())
+
+    def test_rejects_non_staging_and_existing_targets(self):
+        create_backup(self.backup, self.sources, "abc123")
+        with self.assertRaisesRegex(ValueError, "staging"):
+            restore_to_staging(self.backup, self.root / "production")
+        target = self.root / "akyldash-restore-existing"
+        target.mkdir()
+        with self.assertRaisesRegex(ValueError, "already exists"):
+            restore_to_staging(self.backup, target)
+
+    def test_create_checks_sqlite_integrity_before_writing_archive(self):
+        (self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
+        with self.assertRaises(sqlite3.DatabaseError):
+            create_backup(self.backup, self.sources, "abc123")
+        self.assertFalse(self.backup.exists())
+
+    def test_rejects_unsafe_archive_paths(self):
+        archive_path = self.private / "unsafe.tar.gz"
+        with tarfile.open(archive_path, "w:gz") as archive:
+            archive.addfile(tarfile.TarInfo("manifest.json"))
+            archive.addfile(tarfile.TarInfo("../outside"))
+        checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
+        Path(f"{archive_path}.sha256").write_text(f"{checksum}  unsafe.tar.gz\n")
+        with self.assertRaisesRegex(ValueError, "unsafe archive path"):
+            verify_archive(archive_path)
+
+    def test_cold_backup_refuses_a_running_compose_container(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="container-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            return SimpleNamespace(stdout="true\n")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "stop all Compose services"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def compose_fixture(self):
+        paths = {}
+        mounts = {
+            "backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
+            "opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+        }
+        volumes = {}
+        for volume in ("os-data", "caddy-data", "caddy-config"):
+            name = f"akyldash_{volume}"
+            path = self.root / name
+            path.mkdir()
+            paths[name] = path
+            volumes[volume] = {"name": name}
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        secret = self.root / "review-secret.txt"
+        secret.write_text("secret", encoding="utf-8")
+        config = {
+            "services": {name: {"volumes": value} for name, value in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        return config, paths, env_file, secret
+
+    def test_clean_first_deployment_does_not_require_backup(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
+        config, _volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "clean first deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_existing_data_without_containers_is_backup_eligible(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        archive = self.private / "offline-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images, {})
+        self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
+        self.assertEqual(manifest["git_revision"], "abc123")
+
+    def test_missing_persistent_mount_state_is_rejected(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        volume_paths["akyldash_os-data"].rmdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_partial_compose_containers_are_rejected_with_existing_data(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (volume_paths["akyldash_os-data"] / "node-data").mkdir()
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="backend-id\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if command[1] == "inspect":
+                return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+
+    def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        (self.root / "data/existing.json").write_text("data", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
+                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+            with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
+                compose_sources(
+                    Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
+                )
+
+    def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
+        config, volume_paths, env_file, _secret = self.compose_fixture()
+        (self.root / "data/search-users.sqlite3").unlink()
+        corpus = self.root / "data/minjust-normalized"
+        corpus.mkdir()
+        (corpus / "document.json").write_text("corpus", encoding="utf-8")
+        available = set(volume_paths) - {"akyldash_os-data"}
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="")
+            if command[1] == "volume" and "ls" in command:
+                return SimpleNamespace(stdout="\n".join(available) + "\n")
+            if command[1] == "volume":
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
+            raise AssertionError(command)
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
+            )
+        self.assertIsNone(sources["opensearch"])
+        archive = self.private / "initial-corpus-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        with tarfile.open(archive, "r:gz") as saved:
+            self.assertTrue(saved.getmember("opensearch/").isdir())
+            self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
+
+    def test_compose_backup_records_sources_and_image_ids(self):
+        data = self.root / "production-data"
+        data.mkdir()
+        secret = self.root / "secret"
+        secret.write_text("private", encoding="utf-8")
+        env_file = self.root / "production.env"
+        env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
+        volumes = {}
+        volume_paths = {}
+        mounts = {
+            "caddy": [
+                {"type": "volume", "source": "caddy-data", "target": "/data"},
+                {"type": "volume", "source": "caddy-config", "target": "/config"},
+            ],
+            "backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
+            "opensearch": [
+                {"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
+            ],
+        }
+        for name in ("caddy-data", "caddy-config", "opensearch-data"):
+            path = self.root / f"volume-{name}"
+            path.mkdir()
+            volumes[name] = {"name": f"akyldash_{name}"}
+            volume_paths[f"akyldash_{name}"] = path
+        config = {
+            "services": {name: {"volumes": values} for name, values in mounts.items()},
+            "volumes": volumes,
+            "secrets": {"review_secret": {"file": str(secret)}},
+        }
+        service_names = ["caddy", "backend", "opensearch"]
+
+        def run(command, **_kwargs):
+            if "config" in command:
+                return SimpleNamespace(stdout=json.dumps(config))
+            if "ps" in command:
+                return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
+            if command[1] == "inspect":
+                name = command[-1].removesuffix("-id")
+                return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
+            if command[1] == "volume":
+                if "ls" in command:
+                    return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
+                return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            return SimpleNamespace(stdout="")
+
+        with patch("archive.subprocess.run", side_effect=run):
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertEqual(revision, "")
+        self.assertEqual(set(sources), COMPONENTS)
+        self.assertEqual(images["backend"]["image_id"], "sha256:backend")
+        self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/bootstrap/audit-host.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo "Run this read-only audit as root." >&2
+  exit 1
+fi
+
+echo '== Host =='
+hostname -f
+grep -E '^(PRETTY_NAME|VERSION_ID)=' /etc/os-release
+nproc
+free -h
+df -h /
+
+echo '== Packages and services =='
+dpkg-query -W docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+docker version --format 'Docker Engine {{.Server.Version}}'
+docker compose version
+containerd --version
+runc --version
+systemctl is-active docker
+python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); print("Docker logs:", d.get("log-driver"), d.get("log-opts"))'
+
+echo '== Account and paths =='
+id codex
+for path in /opt/akyldash /opt/akyldash/current /etc/akyldash /etc/akyldash/production.env /etc/akyldash/review_secret /var/lib/akyldash/data /var/backups/akyldash /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /etc/sudoers.d/akyldash-codex; do
+  if [[ -e $path ]]; then
+    stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
+  else
+    printf '%s absent\n' "$path"
+  fi
+done
+
+echo '== ISPmanager =='
+dpkg-query -W -f='${binary:Package} ${Version}\n' 'ispmanager*' 'ispmgr*' 2>/dev/null || true
+systemctl list-unit-files --type=service --no-legend | grep -iE 'ispmanager|ispmgr' || echo 'No ISPmanager service unit found.'
+
+echo '== Docker state =='
+docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
+docker volume ls
+
+echo '== Gitea Actions runner (read-only) =='
+if id akyldash-runner >/dev/null 2>&1; then
+  id akyldash-runner
+else
+  echo 'akyldash-runner user absent'
+fi
+if command -v act_runner >/dev/null 2>&1; then
+  act_runner --version
+else
+  echo 'act_runner binary absent from PATH'
+fi
+for path in /etc/akyldash-runner/config.yaml /var/lib/akyldash-runner/.runner /var/lib/akyldash-runner/docker-store.img; do
+  if [[ -e $path ]]; then
+    stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
+  else
+    printf '%s absent\n' "$path"
+  fi
+done
+systemctl status akyldash-act-runner.service --no-pager --full || true
+systemctl status akyldash-runner-healthcheck.timer --no-pager --full || true
+findmnt /var/lib/akyldash-runner/docker-data || true
+
+echo '== Firewall and listeners (read-only) =='
+ufw status verbose
+ufw status numbered
+ss -lntup

infra/bootstrap/bootstrap-ubuntu-24.04.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -gt 1 || ( $# -eq 1 && $1 != --recovery-target ) ]]; then
+  echo "Usage: $0 [--recovery-target]" >&2
+  exit 2
+fi
+recovery_target=false
+if [[ ${1:-} == --recovery-target ]]; then
+  recovery_target=true
+  . "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+  recovery_hostname=$(hostname -f 2>/dev/null || true)
+  recovery_ipv4=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  if [[ -z $recovery_hostname || -z $recovery_ipv4 ]]; then
+    echo "Cannot verify recovery host identity; refusing recovery bootstrap." >&2
+    exit 1
+  fi
+  if akyldash_is_production_host "$recovery_hostname" "$recovery_ipv4"; then
+    echo "Refusing recovery bootstrap on the current production host." >&2
+    exit 1
+  fi
+fi
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run this script as root." >&2
+  exit 1
+fi
+
+. /etc/os-release
+if [[ ${ID:-} != ubuntu || ${VERSION_ID:-} != 24.04 ]]; then
+  echo "Expected Ubuntu 24.04; found ${PRETTY_NAME:-unknown}." >&2
+  exit 1
+fi
+
+if ! id codex >/dev/null 2>&1; then
+  echo "Create the SSH-authorized codex account before running bootstrap." >&2
+  exit 1
+fi
+
+if [[ -e /etc/docker/daemon.json ]] && ! python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' /etc/docker/daemon.json; then
+  echo "/etc/docker/daemon.json exists with different log settings; review it manually." >&2
+  exit 1
+fi
+
+install -d -m 0755 /etc/docker
+if [[ ! -e /etc/docker/daemon.json ]]; then
+  cat > /etc/docker/daemon.json <<'EOF'
+{
+  "log-driver": "local",
+  "log-opts": {"max-size": "10m", "max-file": "3"}
+}
+EOF
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y ca-certificates curl git
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
+  "$(dpkg --print-architecture)" "$VERSION_CODENAME" > /etc/apt/sources.list.d/docker.list
+apt-get update
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+install -d -o root -g root -m 0755 /opt/akyldash
+install -d -o root -g root -m 0700 /etc/akyldash
+install -d -o root -g root -m 0750 /var/lib/akyldash/data
+install -d -o root -g root -m 0700 /var/backups/akyldash
+if [[ $recovery_target == true ]]; then
+  install -o root -g root -m 0600 /dev/null /etc/akyldash/recovery-target
+fi
+systemctl enable --now docker
+docker compose version
+echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
+if [[ $recovery_target == true ]]; then
+  echo 'Recovery target marker created at /etc/akyldash/recovery-target.'
+fi

infra/bootstrap/host-guard.sh

@@ -0,0 +1,11 @@
+akyldash_is_production_host() {
+  local hostname=${1:-}
+  local ipv4_addresses=${2:-}
+  if [[ -z $hostname ]]; then
+    hostname=$(hostname -f 2>/dev/null || true)
+  fi
+  if [[ -z $ipv4_addresses ]]; then
+    ipv4_addresses=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
+  fi
+  [[ $hostname == avprogs.hlab.kz ]] || grep -Fqx '206.223.249.118' <<<"$ipv4_addresses"
+}

infra/bootstrap/test-audit-host.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+source "$script_dir/../test/run-as-unprivileged.sh"
+
+output=$(mktemp)
+trap 'rm -f "$output"' EXIT
+if run_as_unprivileged bash "$script_dir/audit-host.sh" >"$output" 2>&1; then
+  echo 'Host audit unexpectedly succeeded as an unprivileged UID.' >&2
+  exit 1
+fi
+grep -qx 'Run this read-only audit as root.' "$output"

infra/bootstrap/test-host-guard.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
+
+akyldash_is_production_host avprogs.hlab.kz 127.0.0.1
+akyldash_is_production_host other.example $'10.0.0.2\n206.223.249.118'
+if akyldash_is_production_host recovery.example 10.0.0.2; then
+  echo 'FAIL unrelated recovery host identified as production' >&2
+  exit 1
+fi
+echo 'PASS recovery host identity guard'

infra/bootstrap/test-root-guard.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+guard=$(mktemp)
+output=$(mktemp)
+trap 'rm -f "$guard" "$output"' EXIT
+
+awk '
+  /^if \[\[ \$EUID -ne 0 \]\]; then$/ { capture = 1 }
+  capture { print }
+  capture && /^fi$/ { found = 1; exit }
+  END { if (!found) exit 1 }
+' "$script_dir/audit-host.sh" | sed 's/\$EUID/\$TEST_EUID/g' >"$guard" || {
+  echo 'Could not extract the host audit root guard.' >&2
+  exit 1
+}
+if ! grep -Fq 'if [[ $TEST_EUID -ne 0 ]]; then' "$guard"; then
+  echo 'Extracted guard does not use the simulated effective UID.' >&2
+  exit 1
+fi
+if TEST_EUID=0 bash "$guard" >"$output" 2>&1; then
+  [[ ! -s $output ]] || { cat "$output" >&2; exit 1; }
+else
+  echo 'Host audit root guard rejected simulated UID 0.' >&2
+  exit 1
+fi
+if TEST_EUID=65534 bash "$guard" >"$output" 2>&1; then
+  echo 'Host audit root guard accepted simulated non-root UID.' >&2
+  exit 1
+fi
+grep -qx 'Run this read-only audit as root.' "$output"
+echo 'PASS host audit root guard accepts UID 0 and rejects non-root UIDs'

infra/bootstrap/test-verify-helper.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if bash "$(dirname "${BASH_SOURCE[0]}")/verify-helper.sh" unexpected >/dev/null 2>&1; then
+  echo 'FAIL verifier helper accepted command arguments' >&2
+  exit 1
+fi
+echo 'PASS verifier helper rejects command arguments'

infra/bootstrap/verify-helper.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+checkout=/opt/akyldash/current
+if [[ $# -ne 0 || $EUID -ne 0 || ! -d $checkout || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
+  echo 'Refusing verification: the approved root-owned checkout is unavailable.' >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ -n $unsafe ]]; then
+  echo 'Refusing verification: the checkout contains a writable or non-root-owned path.' >&2
+  exit 1
+fi
+exec /bin/bash "$checkout/infra/bootstrap/verify.sh" --remote

infra/bootstrap/verify.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ${1:-} != --remote ]]; then
+  ssh_alias=${1:-akyldash_vps}
+  exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n /usr/local/sbin/akyldash-verify'
+fi
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run the remote check as root; it is read-only." >&2
+  exit 1
+fi
+
+failed=0
+for package in git docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do
+  if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx 'install ok installed'; then
+    printf 'PASS package %s\n' "$package"
+  else
+    printf 'FAIL package %s missing\n' "$package"
+    failed=1
+  fi
+done
+
+if systemctl is-active --quiet docker; then
+  echo 'PASS Docker service active'
+else
+  echo 'FAIL Docker service inactive'
+  failed=1
+fi
+if docker compose version >/dev/null 2>&1; then
+  echo 'PASS Docker Compose responds'
+else
+  echo 'FAIL Docker Compose unavailable'
+  failed=1
+fi
+if python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' 2>/dev/null; then
+  echo 'PASS Docker log rotation configuration'
+else
+  echo 'FAIL Docker log rotation configuration differs from repository target'
+  failed=1
+fi
+if id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo 'FAIL codex has sudo or Docker group access'
+  failed=1
+else
+  echo 'PASS codex is outside sudo and docker groups'
+fi
+
+for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
+  if [[ -d $path ]]; then
+    printf 'PASS directory %s\n' "$path"
+  else
+    printf 'FAIL directory %s missing\n' "$path"
+    failed=1
+  fi
+done
+for path in /etc/akyldash /var/backups/akyldash; do
+  if [[ -d $path && $(stat -c %a "$path") == 700 ]]; then
+    printf 'PASS permissions %s mode 0700\n' "$path"
+  elif [[ -d $path ]]; then
+    printf 'FAIL permissions %s mode %s, expected 0700\n' "$path" "$(stat -c %a "$path")"
+    failed=1
+  fi
+done
+for path in /etc/akyldash/production.env /etc/akyldash/review_secret; do
+  if [[ -f $path && $(stat -c %U:%a "$path") == root:600 ]]; then
+    printf 'PASS protected file %s mode 0600\n' "$path"
+  elif [[ -e $path ]]; then
+    printf 'FAIL protected file %s must be root-owned mode 0600\n' "$path"
+    failed=1
+  else
+    printf 'FAIL protected file %s missing\n' "$path"
+    failed=1
+  fi
+done
+
+checkout=/opt/akyldash/current
+compose_file=$checkout/deploy/production/compose.yaml
+if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Caddyfile deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
+  echo 'PASS production Compose, Caddyfile, and Dockerfiles match the checked-out Git revision'
+else
+  echo 'FAIL production Compose or Dockerfile has local drift'
+  failed=1
+fi
+if docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --quiet >/dev/null 2>&1; then
+  echo 'PASS rendered production Compose configuration'
+else
+  echo 'FAIL production Compose configuration cannot be rendered'
+  failed=1
+fi
+
+if ufw status | grep -q '^Status: active$'; then
+  echo 'PASS UFW active'
+  if ! ufw status verbose | grep -q '^Default: deny (incoming)'; then
+    echo 'FAIL UFW default incoming policy is not deny'
+    failed=1
+  fi
+  while read -r rule action direction; do
+    [[ $action == ALLOW && $direction == IN ]] || continue
+    ports=${rule//\/tcp/}
+    ports=${ports//\/udp/}
+    IFS=, read -ra allowed <<< "$ports"
+    for port in "${allowed[@]}"; do
+      case "$port" in
+        22|80|443) ;;
+        *) printf 'FAIL unexpected UFW allow rule %s\n' "$rule"; failed=1 ;;
+      esac
+    done
+  done < <(ufw status | awk 'NR > 3 && NF >= 3 {print $1, $2, $3}')
+else
+  echo 'FAIL UFW inactive or unavailable'
+  failed=1
+fi
+
+tcp_listeners=$(ss -H -lnt | awk '{print $4}')
+for endpoint in $tcp_listeners; do
+  address=${endpoint%:*}
+  port=${endpoint##*:}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  case "$port" in 22|80|443) ;; *) printf 'FAIL unexpected non-loopback TCP listener %s\n' "$endpoint"; failed=1 ;; esac
+done
+printf 'INFO non-loopback TCP listeners: %s\n' "${tcp_listeners//$'\n'/,}"
+
+udp_listeners=$(ss -H -lnu | awk '{print $5}')
+for endpoint in $udp_listeners; do
+  address=${endpoint%:*}
+  case "$address" in 127.*|::1|\[::1\]) continue ;; esac
+  printf 'FAIL unexpected non-loopback UDP listener %s\n' "$endpoint"
+  failed=1
+done
+
+expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
+if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \
+      -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
+      -x /usr/local/sbin/akyldash-compose && $(stat -c %U:%a /usr/local/sbin/akyldash-compose) == root:750 && \
+      -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
+      -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
+      $(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then
+  echo 'PASS read-only verifier, Compose status/config helper, rollback helper, and exact sudoers rules present'
+else
+  echo 'FAIL restricted helper or sudoers installation missing/different'
+  failed=1
+fi
+if [[ -f /opt/akyldash/deployed-image.txt ]]; then
+  expected=$(cat /opt/akyldash/deployed-image.txt)
+  container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
+  actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)
+  if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then
+    printf 'PASS running backend image ID matches %s\n' "$expected"
+  else
+    echo 'FAIL running backend image does not match recorded digest'
+    failed=1
+  fi
+else
+  echo 'FAIL deployed image digest record missing'
+  failed=1
+fi
+
+expected_proxy=$(docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --format json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["services"]["caddy"]["image"])' 2>/dev/null || true)
+proxy_container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=caddy | head -n1)
+actual_proxy=$(docker inspect --format '{{.Config.Image}}' "$proxy_container" 2>/dev/null || true)
+if [[ $expected_proxy == *@sha256:* && $actual_proxy == "$expected_proxy" ]]; then
+  printf 'PASS running Caddy image matches %s\n' "$expected_proxy"
+else
+  echo 'FAIL running Caddy image does not match the pinned Compose digest'
+  failed=1
+fi
+exit "$failed"

infra/deploy/compare_corpus_inventories.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Require non-empty completed normalized audits and compare every file row."""
+
+from __future__ import annotations
+
+import argparse
+import fcntl
+import itertools
+import json
+import os
+import sqlite3
+import stat
+import sys
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from typing import Any
+
+
+class ComparisonError(ValueError):
+    pass
+
+
+@contextmanager
+def shared_inventory_lock(result_path: Path, label: str):
+    lock_path = result_path.absolute().parent / ".inventory.lock"
+    if not hasattr(os, "O_NOFOLLOW"):
+        raise ComparisonError(f"{label}: safe inventory lock opens are unsupported")
+    try:
+        descriptor = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW)
+    except OSError as error:
+        raise ComparisonError(f"{label}: cannot open inventory lock: {error}") from error
+    try:
+        metadata = os.fstat(descriptor)
+        if (
+            not stat.S_ISREG(metadata.st_mode)
+            or metadata.st_nlink != 1
+            or stat.S_IMODE(metadata.st_mode) & 0o077
+        ):
+            raise ComparisonError(f"{label}: inventory lock must be a private regular file")
+        try:
+            fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
+        except BlockingIOError as error:
+            raise ComparisonError(f"{label}: inventory is active (lock busy)") from error
+        yield
+    finally:
+        os.close(descriptor)
+
+
+def read_audit(result_path: Path, label: str) -> tuple[sqlite3.Connection, dict[str, Any]]:
+    result_path = result_path.absolute()
+    try:
+        metadata = result_path.lstat()
+        if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
+            raise ComparisonError(f"{label}: result.json must be a regular file")
+        result = json.loads(result_path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise ComparisonError(f"{label}: cannot read result.json: {error}") from error
+
+    if not isinstance(result, dict):
+        raise ComparisonError(f"{label}: result.json must contain an object")
+    if result.get("status") != "complete" or result.get("verification_status") != "passed":
+        raise ComparisonError(f"{label}: inventory must be complete and passed")
+    verifier_sha256 = result.get("verifier_sha256")
+    if not isinstance(verifier_sha256, str) or len(verifier_sha256) != 64:
+        raise ComparisonError(f"{label}: verifier SHA-256 is missing")
+    normalized_root = result.get("roots", {}).get("normalized")
+    if not isinstance(normalized_root, str):
+        raise ComparisonError(f"{label}: normalized root is missing")
+    summary = result.get("manifest_summary", {}).get("normalized")
+    if not isinstance(summary, dict):
+        raise ComparisonError(f"{label}: normalized manifest summary is missing")
+    for key in ("documents", "fragments"):
+        if not isinstance(summary.get(key), int) or summary[key] <= 0:
+            raise ComparisonError(f"{label}: normalized {key} must be greater than zero")
+
+    database_path = result_path.with_name("inventory.sqlite3")
+    try:
+        db_metadata = database_path.lstat()
+        if (
+            stat.S_ISLNK(db_metadata.st_mode)
+            or not stat.S_ISREG(db_metadata.st_mode)
+            or db_metadata.st_nlink != 1
+        ):
+            raise ComparisonError(f"{label}: inventory.sqlite3 must be a regular unlinked file")
+        for suffix in ("-wal", "-shm", "-journal"):
+            if Path(f"{database_path}{suffix}").exists():
+                raise ComparisonError(f"{label}: SQLite sidecar files are present")
+        connection = sqlite3.connect(database_path.resolve().as_uri() + "?mode=ro&immutable=1", uri=True)
+        connection.row_factory = sqlite3.Row
+        if connection.execute("PRAGMA quick_check").fetchone()[0] != "ok":
+            raise ComparisonError(f"{label}: inventory database failed quick_check")
+        manifest_documents = connection.execute(
+            "SELECT count(*) FROM manifest_docs WHERE root_id='normalized'"
+        ).fetchone()[0]
+        if manifest_documents != summary["documents"]:
+            raise ComparisonError(f"{label}: normalized document count does not match inventory")
+        files, byte_count, unfinished = connection.execute(
+            "SELECT count(*),coalesce(sum(size),0),"
+            "sum(CASE WHEN status!='hashed' OR size IS NULL OR sha256 IS NULL THEN 1 ELSE 0 END) "
+            "FROM files WHERE root_id='normalized'"
+        ).fetchone()
+        progress = result.get("progress", {}).get("normalized", {})
+        if unfinished or files != progress.get("files") or byte_count != progress.get("bytes"):
+            raise ComparisonError(f"{label}: inventory rows do not match completed scan counters")
+        return connection, {
+            "root": normalized_root,
+            "documents": summary["documents"],
+            "fragments": summary["fragments"],
+            "files": files,
+            "bytes": byte_count,
+            "verifier_sha256": verifier_sha256,
+            "schema_version": result.get("schema_version"),
+        }
+    except sqlite3.Error as error:
+        raise ComparisonError(f"{label}: cannot validate inventory database: {error}") from error
+
+
+def compare_inventories(source_result: Path, staging_result: Path) -> dict[str, int]:
+    with ExitStack() as locks:
+        locks.enter_context(shared_inventory_lock(source_result, "source"))
+        locks.enter_context(shared_inventory_lock(staging_result, "staging"))
+        source_db, source = read_audit(source_result, "source")
+        try:
+            staging_db, staging = read_audit(staging_result, "staging")
+        except Exception:
+            source_db.close()
+            raise
+        try:
+            if source["root"] == staging["root"]:
+                raise ComparisonError("source and staging must be different normalized roots")
+            if source["verifier_sha256"] != staging["verifier_sha256"] or source["schema_version"] != staging["schema_version"]:
+                raise ComparisonError("source and staging audits must use the same verifier and schema")
+            if (source["documents"], source["fragments"]) != (staging["documents"], staging["fragments"]):
+                raise ComparisonError("source and staging document or fragment counts differ")
+
+            query = (
+                "SELECT relative_path,size,sha256 FROM files "
+                "WHERE root_id='normalized' ORDER BY relative_path"
+            )
+            left = iter(source_db.execute(query))
+            right = iter(staging_db.execute(query))
+            mismatch_count = 0
+            for source_row, staging_row in itertools.zip_longest(left, right):
+                if source_row != staging_row:
+                    mismatch_count += 1
+            if mismatch_count:
+                raise ComparisonError(f"source and staging differ in {mismatch_count} inventory rows")
+            return {"documents": source["documents"], "fragments": source["fragments"], "files": source["files"], "bytes": source["bytes"]}
+        finally:
+            source_db.close()
+            staging_db.close()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("source_result", type=Path)
+    parser.add_argument("staging_result", type=Path)
+    arguments = parser.parse_args()
+    try:
+        summary = compare_inventories(arguments.source_result, arguments.staging_result)
+    except (ComparisonError, OSError) as error:
+        print(f"FAIL: {error}", file=sys.stderr)
+        return 1
+    print(
+        "PASS: matched "
+        f"{summary['documents']} documents, {summary['fragments']} fragments, "
+        f"{summary['files']} files, {summary['bytes']} bytes"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infra/deploy/compose-helper.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 || ( $1 != config && $1 != status ) ]]; then
+  echo 'Usage: akyldash-compose {config|status}' >&2
+  exit 2
+fi
+if [[ $EUID -ne 0 ]]; then
+  echo 'Run this helper through its fixed sudoers command.' >&2
+  exit 1
+fi
+
+checkout=/opt/akyldash/current
+if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
+  echo 'Refusing Compose access: approved root-owned checkout is unavailable.' >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ -n $unsafe ]]; then
+  echo 'Refusing Compose access: checkout contains a writable or non-root-owned path.' >&2
+  exit 1
+fi
+
+compose=(docker compose --env-file /etc/akyldash/production.env -f "$checkout/deploy/production/compose.yaml")
+case $1 in
+  config) "${compose[@]}" config --quiet ;;
+  status) "${compose[@]}" ps ;;
+esac

infra/deploy/install-helper.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo "Run this installer as root." >&2
+  exit 1
+fi
+
+checkout=/opt/akyldash/current
+rollback_source=$checkout/infra/deploy/rollback.py
+backup_source=$checkout/infra/backup/archive.py
+verify_source=$checkout/infra/bootstrap/verify-helper.sh
+compose_source=$checkout/infra/deploy/compose-helper.sh
+if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then
+  echo "Install the approved repository checkout at $checkout first." >&2
+  exit 1
+fi
+if [[ -L $checkout ]]; then
+  echo "The checkout path must not be a symlink." >&2
+  exit 1
+fi
+for directory in /opt /opt/akyldash; do
+  mode=$(stat -c %a "$directory")
+  if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then
+    echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2
+    exit 1
+  fi
+done
+if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
+  echo "Create codex without sudo or Docker-group membership before installing the helper." >&2
+  exit 1
+fi
+if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then
+  echo "Existing codex sudoers rules need owner review before installing the helper." >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ $(stat -c %U "$checkout") != root || -n $unsafe ]]; then
+  echo "The checkout must be root-owned and not group/world-writable." >&2
+  exit 1
+fi
+if ! command -v visudo >/dev/null 2>&1; then
+  echo "visudo is required to validate the restricted sudoers rule." >&2
+  exit 1
+fi
+
+for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
+  if [[ -L $path ]]; then
+    echo "Refusing symlink at $path." >&2
+    exit 1
+  fi
+done
+
+install -d -o root -g root -m 0755 /usr/local/lib/akyldash
+install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
+install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify
+install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
+install -o root -g root -m 0750 "$compose_source" /usr/local/sbin/akyldash-compose
+
+sudoers_file=/etc/sudoers.d/akyldash-codex
+expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
+created=false
+if [[ -e $sudoers_file ]]; then
+  if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2
+    exit 1
+  fi
+else
+  temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX)
+  trap 'rm -f "$temp"' EXIT
+  printf '%s\n' "$expected" > "$temp"
+  chmod 0440 "$temp"
+  visudo -cf "$temp"
+  visudo -c -f /etc/sudoers
+  mv -n "$temp" "$sudoers_file"
+  if [[ $(cat "$sudoers_file") != "$expected" ]]; then
+    echo "Could not install the expected sudoers rule." >&2
+    exit 1
+  fi
+  created=true
+fi
+
+if ! visudo -c -f /etc/sudoers; then
+  if [[ $created == true ]]; then
+    rm -f "$sudoers_file"
+  fi
+  echo "Sudoers validation failed; the newly added rule was removed." >&2
+  exit 1
+fi
+echo "Installed fixed read-only verification/Compose commands and guarded rollback; codex receives no general sudo or Docker access."

infra/deploy/prepare-production-config.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo 'Run this configuration step as root.' >&2
+  exit 1
+fi
+
+checkout=/opt/akyldash/current
+example=$checkout/deploy/production/.env.example
+config_dir=/etc/akyldash
+env_file=$config_dir/production.env
+secret_file=$config_dir/review_secret
+if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root || ! -f $example ]]; then
+  echo 'Refusing configuration: approved root-owned checkout is unavailable.' >&2
+  exit 1
+fi
+unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
+if [[ -n $unsafe ]] || ! git -C "$checkout" diff --quiet HEAD -- deploy/production/.env.example; then
+  echo 'Refusing configuration: checkout is writable or .env.example differs from Git.' >&2
+  exit 1
+fi
+if [[ ! -d $config_dir || $(stat -c %U:%G:%a "$config_dir") != root:root:700 ]]; then
+  echo 'Expected /etc/akyldash to be root-owned mode 0700.' >&2
+  exit 1
+fi
+if [[ -e $env_file || -L $env_file || -e $secret_file || -L $secret_file ]]; then
+  echo 'Production env or review secret already exists; refusing to overwrite either file.' >&2
+  exit 1
+fi
+
+env_tmp=$(mktemp "$config_dir/.production.env.XXXXXX")
+secret_tmp=$(mktemp "$config_dir/.review_secret.XXXXXX")
+trap 'rm -f "$env_tmp" "$secret_tmp"' EXIT
+install -o root -g root -m 0600 "$example" "$env_tmp"
+openssl rand -hex 32 > "$secret_tmp"
+chown root:root "$secret_tmp"
+chmod 0600 "$secret_tmp"
+mv -n "$env_tmp" "$env_file"
+if [[ -e $env_tmp ]]; then
+  echo 'Production env already appeared; refusing to continue.' >&2
+  exit 1
+fi
+mv -n "$secret_tmp" "$secret_file"
+if [[ -e $secret_tmp ]]; then
+  rm -f "$env_file"
+  echo 'Review secret already appeared; removed the new env file and stopped.' >&2
+  exit 1
+fi
+echo 'Created root-owned production.env and review_secret with mode 0600. No services or volumes were changed.'

infra/deploy/rollback.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""Restore prior local application images from a verified cold backup."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path("/opt/akyldash/current")
+COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
+ENV_FILE = Path("/etc/akyldash/production.env")
+BACKUP_DIR = Path("/var/backups/akyldash")
+BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
+APP_SERVICES = {"backend", "opensearch"}
+IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*")
+IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}")
+
+
+def is_digest_reference(reference: str) -> bool:
+    return IMAGE_DIGEST_RE.fullmatch(reference) is not None
+
+
+def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
+    images = manifest.get("images")
+    if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
+        raise ValueError("backup has no previous backend and OpenSearch images")
+    plan = {}
+    for service in sorted(APP_SERVICES):
+        image = images[service]
+        if not isinstance(image, dict):
+            raise ValueError(f"invalid image record for {service}")
+        reference = image.get("configured")
+        image_id = image.get("image_id")
+        if reference != configured.get(service):
+            raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
+        if not isinstance(reference, str) or not (
+            IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference)
+        ):
+            raise ValueError(f"invalid image reference for {service}")
+        if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+            raise ValueError(f"invalid previous image ID for {service}")
+        plan[service] = (image_id, reference)
+    return plan
+
+
+def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None:
+    for image_id, reference in plan.values():
+        try:
+            actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
+        except subprocess.CalledProcessError as error:
+            raise ValueError(f"previous image is unavailable: {image_id}") from error
+        if actual != image_id:
+            raise ValueError(f"previous image is unavailable: {image_id}")
+        if is_digest_reference(reference):
+            try:
+                referenced_id = run(
+                    ["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True
+                )
+            except subprocess.CalledProcessError as error:
+                raise ValueError(f"previous digest reference is unavailable: {reference}") from error
+            if referenced_id != image_id:
+                raise ValueError(f"previous digest reference does not match its recorded image: {reference}")
+
+
+def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None:
+    if any(is_digest_reference(reference) for _image_id, reference in plan.values()):
+        raise ValueError(
+            "rollback of digest-pinned images is not supported safely; no services or image references were changed"
+        )
+
+
+def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None:
+    require_tag_rollback(plan)
+    for image_id, reference in plan.values():
+        run(["docker", "image", "tag", image_id, reference])
+
+
+def run(command: list[str], *, capture: bool = False) -> str:
+    result = subprocess.run(command, check=True, text=True, capture_output=capture)
+    return result.stdout.strip() if capture else ""
+
+
+def compose(*args: str, capture: bool = False) -> str:
+    return run(
+        ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
+        capture=capture,
+    )
+
+
+def record_backend_image() -> None:
+    container = compose("ps", "--quiet", "backend", capture=True)
+    if not container:
+        raise ValueError("backend container is missing after rollback")
+    image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
+    if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+        raise ValueError("backend returned an invalid image ID")
+    target = Path("/opt/akyldash/deployed-image.txt")
+    with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
+        temporary = Path(stream.name)
+        stream.write(f"{image_id}\n".encode("ascii"))
+    os.chmod(temporary, 0o600)
+    try:
+        os.replace(temporary, target)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def rollback(backup: Path, data_compatible: bool) -> None:
+    if not data_compatible:
+        raise ValueError("confirm data compatibility with --data-compatible before rollback")
+    if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
+        raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
+    if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback backup must be root-owned mode 0600")
+    checksum = Path(f"{backup}.sha256")
+    if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
+        raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
+
+    sys.path.insert(0, str(BACKUP_TOOL.parent))
+    from archive import digest, verify_archive
+
+    manifest = verify_archive(backup)
+    deployment_files = {
+        relative: digest(ROOT / relative)
+        for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
+    }
+    if deployment_files != manifest.get("deployment_files"):
+        raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
+    rendered = json.loads(compose("config", "--format", "json", capture=True))
+    configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
+    plan = image_restore_plan(manifest, configured)
+    verify_previous_images(plan)
+    require_tag_rollback(plan)
+
+    backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
+    backup_output = BACKUP_DIR / backup_name
+    compose("stop")
+    try:
+        run(
+            [
+                "python3",
+                str(BACKUP_TOOL),
+                "create",
+                "--compose-file",
+                str(COMPOSE_FILE),
+                "--env-file",
+                str(ENV_FILE),
+                "--output",
+                str(backup_output),
+            ]
+        )
+        run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
+    except Exception:
+        compose("up", "--detach")
+        raise
+
+    tag_previous_images(plan)
+    compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
+    run(
+        [
+            "curl",
+            "--fail",
+            "--silent",
+            "--show-error",
+            "--resolve",
+            "akyldash.kg:443:127.0.0.1",
+            "https://akyldash.kg/review",
+            "--output",
+            "/dev/null",
+        ]
+    )
+    record_backend_image()
+    print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("backup", type=Path)
+    parser.add_argument("--data-compatible", action="store_true")
+    args = parser.parse_args()
+    try:
+        if os.geteuid() != 0:
+            raise ValueError("rollback must run through the restricted root helper")
+        rollback(args.backup, args.data_compatible)
+    except Exception as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infra/deploy/test-compose-helper.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+helper=$(dirname "$0")/compose-helper.sh
+for action in deploy up down docker; do
+  if bash "$helper" "$action" >/dev/null 2>&1; then
+    printf 'FAIL Compose helper accepted %s\n' "$action" >&2
+    exit 1
+  fi
+done
+echo 'PASS Compose helper rejects operations outside its read-only allowlist'

infra/deploy/test-production-config-guard.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+source "$script_dir/../test/run-as-unprivileged.sh"
+
+output=$(mktemp)
+trap 'rm -f "$output"' EXIT
+if run_as_unprivileged bash "$script_dir/prepare-production-config.sh" >"$output" 2>&1; then
+  echo 'Production config setup unexpectedly succeeded as an unprivileged UID.' >&2
+  exit 1
+fi
+grep -qx 'Run this configuration step as root.' "$output"

infra/deploy/test_compare_corpus_inventories.py

@@ -0,0 +1,132 @@
+import fcntl
+import json
+import os
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+from compare_corpus_inventories import ComparisonError, compare_inventories
+
+
+class CompareCorpusInventoryTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name)
+        self.source = self._artifact("source", "/data/source")
+        self.staging = self._artifact("staging", "/data/staging")
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def _artifact(self, name, root, documents=1, fragments=2, rows=None):
+        directory = self.base / name
+        directory.mkdir(mode=0o700)
+        (directory / ".inventory.lock").touch(mode=0o600)
+        database = directory / "inventory.sqlite3"
+        connection = sqlite3.connect(database)
+        connection.execute(
+            "CREATE TABLE files (root_id TEXT, relative_path TEXT, size INTEGER, sha256 TEXT, status TEXT)"
+        )
+        connection.execute("CREATE TABLE manifest_docs (root_id TEXT, code TEXT)")
+        connection.executemany(
+            "INSERT INTO manifest_docs VALUES ('normalized',?)",
+            [(str(code),) for code in range(documents)],
+        )
+        if rows is None:
+            rows = [
+                ("manifest.sqlite3", 16, "a" * 64),
+                ("documents/1/document.json", 24, "b" * 64),
+            ]
+        connection.executemany(
+            "INSERT INTO files VALUES ('normalized',?,?,?,'hashed')", rows
+        )
+        connection.commit()
+        connection.close()
+        result = {
+            "status": "complete",
+            "verification_status": "passed",
+            "schema_version": 2,
+            "verifier_sha256": "c" * 64,
+            "roots": {"normalized": root},
+            "manifest_summary": {
+                "normalized": {"documents": documents, "fragments": fragments}
+            },
+            "progress": {
+                "normalized": {
+                    "files": len(rows),
+                    "bytes": sum(row[1] for row in rows),
+                }
+            },
+        }
+        result_path = directory / "result.json"
+        result_path.write_text(json.dumps(result), encoding="utf-8")
+        return result_path
+
+    def test_nonempty_source_and_staging_inventories_match(self):
+        self.assertEqual(
+            compare_inventories(self.source, self.staging),
+            {"documents": 1, "fragments": 2, "files": 2, "bytes": 40},
+        )
+
+    def test_empty_source_is_rejected(self):
+        empty = self._artifact("empty-source", "/data/empty-source", documents=0, fragments=0)
+        with self.assertRaisesRegex(ComparisonError, "source: normalized documents"):
+            compare_inventories(empty, self.staging)
+
+    def test_empty_staging_is_rejected(self):
+        empty = self._artifact("empty-staging", "/data/empty-staging", documents=0, fragments=0)
+        with self.assertRaisesRegex(ComparisonError, "staging: normalized documents"):
+            compare_inventories(self.source, empty)
+
+    def test_zero_fragments_are_rejected_even_with_documents(self):
+        empty = self._artifact("no-fragments", "/data/no-fragments", fragments=0)
+        with self.assertRaisesRegex(ComparisonError, "normalized fragments"):
+            compare_inventories(self.source, empty)
+
+    def test_different_file_sha_is_rejected(self):
+        changed = self._artifact(
+            "changed", "/data/changed",
+            rows=[("manifest.sqlite3", 16, "a" * 64), ("documents/1/document.json", 24, "d" * 64)],
+        )
+        with self.assertRaisesRegex(ComparisonError, "differ in 1 inventory rows"):
+            compare_inventories(self.source, changed)
+
+    def test_busy_source_lock_is_rejected(self):
+        self._assert_busy_lock(self.source, "source")
+
+    def test_busy_staging_lock_is_rejected(self):
+        self._assert_busy_lock(self.staging, "staging")
+
+    def _assert_busy_lock(self, result_path, label):
+        descriptor = os.open(result_path.parent / ".inventory.lock", os.O_RDWR)
+        try:
+            fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            with self.assertRaisesRegex(ComparisonError, rf"{label}: inventory is active"):
+                compare_inventories(self.source, self.staging)
+        finally:
+            os.close(descriptor)
+
+    def test_missing_lock_is_rejected_without_creating_it(self):
+        lock = self.source.parent / ".inventory.lock"
+        lock.unlink()
+        with self.assertRaisesRegex(ComparisonError, "source: cannot open inventory lock"):
+            compare_inventories(self.source, self.staging)
+        self.assertFalse(lock.exists())
+
+    def test_symlink_lock_is_rejected(self):
+        lock = self.source.parent / ".inventory.lock"
+        lock.unlink()
+        target = self.base / "external-lock"
+        target.touch(mode=0o600)
+        lock.symlink_to(target)
+        with self.assertRaisesRegex(ComparisonError, "source: cannot open inventory lock"):
+            compare_inventories(self.source, self.staging)
+
+    def test_read_only_lock_can_be_shared(self):
+        (self.source.parent / ".inventory.lock").chmod(0o400)
+        self.assertEqual(compare_inventories(self.source, self.staging)["documents"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/deploy/test_release_manifest.py

@@ -0,0 +1,131 @@
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from validate_release import validate_manifest
+
+ROOT = Path(__file__).resolve().parents[2]
+REPOSITORIES = {
+    "backend": ["registry.invalid/akyldash/backend"],
+    "opensearch": ["registry.invalid/akyldash/opensearch"],
+}
+
+
+class ReleaseManifestTests(unittest.TestCase):
+    def setUp(self):
+        self.policy = {"schema_version": 1, "repositories": REPOSITORIES}
+        self.manifest = {
+            "schema_version": 1,
+            "source_ref": "refs/heads/main",
+            "commit": "a" * 40,
+            "images": {
+                "backend": "registry.invalid/akyldash/backend@sha256:" + "b" * 64,
+                "opensearch": "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64,
+            },
+        }
+
+    def test_accepts_both_policy_pinned_images(self):
+        self.assertEqual(validate_manifest(self.manifest, self.policy), self.manifest)
+
+    def test_rejects_untrusted_ref(self):
+        self.manifest["source_ref"] = "refs/pull/12/head"
+        with self.assertRaisesRegex(ValueError, "source_ref"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_tag_instead_of_digest(self):
+        self.manifest["images"]["backend"] = "registry.invalid/akyldash/backend:latest"
+        with self.assertRaisesRegex(ValueError, "sha256"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_empty_image_reference(self):
+        self.manifest["images"]["backend"] = ""
+        with self.assertRaisesRegex(ValueError, "sha256"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_invalid_commit(self):
+        self.manifest["commit"] = "not-a-commit"
+        with self.assertRaisesRegex(ValueError, "commit"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_repository_outside_policy(self):
+        self.manifest["images"]["backend"] = "registry.invalid/other/backend@sha256:" + "b" * 64
+        with self.assertRaisesRegex(ValueError, "not allowed"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_missing_or_extra_images(self):
+        self.manifest["images"].pop("opensearch")
+        with self.assertRaisesRegex(ValueError, "exactly backend and opensearch"):
+            validate_manifest(self.manifest, self.policy)
+        self.manifest["images"]["opensearch"] = "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64
+        self.manifest["images"]["extra"] = "registry.invalid/akyldash/extra@sha256:" + "d" * 64
+        with self.assertRaisesRegex(ValueError, "exactly backend and opensearch"):
+            validate_manifest(self.manifest, self.policy)
+
+    def test_rejects_duplicate_json_keys(self):
+        with tempfile.TemporaryDirectory() as directory:
+            manifest_path = Path(directory) / "manifest.json"
+            policy_path = Path(directory) / "policy.json"
+            manifest_path.write_text(
+                '{"schema_version":1,"source_ref":"refs/heads/main","commit":"' + "a" * 40
+                + '","images":{"backend":"first","backend":"second","opensearch":"third"}}',
+                encoding="utf-8",
+            )
+            policy_path.write_text(json.dumps(self.policy), encoding="utf-8")
+            command = [
+                "python3", str(ROOT / "infra/deploy/validate_release.py"), str(manifest_path),
+                "--policy", str(policy_path),
+            ]
+            result = subprocess.run(command, text=True, capture_output=True)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("duplicate JSON key: backend", result.stderr)
+
+    def test_rejects_malformed_json(self):
+        with tempfile.TemporaryDirectory() as directory:
+            manifest_path = Path(directory) / "manifest.json"
+            policy_path = Path(directory) / "policy.json"
+            manifest_path.write_text('{"schema_version":', encoding="utf-8")
+            policy_path.write_text(json.dumps(self.policy), encoding="utf-8")
+            result = subprocess.run(
+                ["python3", str(ROOT / "infra/deploy/validate_release.py"), str(manifest_path), "--policy", str(policy_path)],
+                text=True,
+                capture_output=True,
+            )
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("cannot read valid JSON", result.stderr)
+
+    @unittest.skipUnless(shutil.which("docker"), "Docker CLI is unavailable")
+    def test_compose_override_renders_digest_images_and_preserves_build(self):
+        backend = "registry.invalid/akyldash/backend@sha256:" + "b" * 64
+        opensearch = "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64
+        with tempfile.TemporaryDirectory() as directory:
+            temp = Path(directory)
+            secret = temp / "review_secret"
+            secret.touch()
+            override = temp / "release-override.json"
+            override.write_text(json.dumps({"services": {
+                "backend": {"image": backend},
+                "opensearch": {"image": opensearch},
+            }}), encoding="utf-8")
+            command = [
+                "docker", "compose", "--env-file", str(ROOT / "deploy/production/.env.example"),
+                "-f", str(ROOT / "deploy/production/compose.yaml"), "-f", str(override),
+                "config", "--format", "json",
+            ]
+            result = subprocess.run(
+                command, text=True, capture_output=True,
+                env={**os.environ, "REVIEW_SECRET_FILE": str(secret)},
+            )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        services = json.loads(result.stdout)["services"]
+        self.assertEqual(services["backend"]["image"], backend)
+        self.assertEqual(services["opensearch"]["image"], opensearch)
+        self.assertIn("build", services["backend"])
+        self.assertIn("build", services["opensearch"])
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/deploy/test_rollback.py

@@ -0,0 +1,129 @@
+import json
+import subprocess
+import sys
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import rollback
+from rollback import image_restore_plan, require_tag_rollback, tag_previous_images, verify_previous_images
+
+
+class RollbackPlanTests(unittest.TestCase):
+    def setUp(self):
+        self.manifest = {
+            "images": {
+                "backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
+                "opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
+            }
+        }
+        self.configured = {
+            "backend": "akyldash/backend:local",
+            "opensearch": "akyldash/opensearch:local",
+        }
+
+    def test_plans_only_recorded_local_images(self):
+        plan = image_restore_plan(self.manifest, self.configured)
+        self.assertEqual(plan["backend"][1], "akyldash/backend:local")
+        self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local")
+
+    def test_refuses_changed_compose_image_reference(self):
+        with self.assertRaisesRegex(ValueError, "reference changed"):
+            image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"})
+
+    def test_refuses_digest_reference_that_cannot_be_retagged(self):
+        self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64
+        self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
+        plan = image_restore_plan(self.manifest, self.configured)
+        self.assertEqual(plan["backend"], ("sha256:" + "a" * 64, self.configured["backend"]))
+        with self.assertRaisesRegex(ValueError, "not supported safely"):
+            require_tag_rollback(plan)
+
+    def test_refuses_malformed_digest_reference(self):
+        self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:xyz"
+        self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
+        with self.assertRaisesRegex(ValueError, "invalid image reference"):
+            image_restore_plan(self.manifest, self.configured)
+
+    def test_refuses_invalid_recorded_image_id(self):
+        self.manifest["images"]["backend"]["image_id"] = "sha256:not-a-digest"
+        with self.assertRaisesRegex(ValueError, "invalid previous image ID"):
+            image_restore_plan(self.manifest, self.configured)
+
+    def test_digest_reference_must_resolve_to_recorded_image_id(self):
+        digest = "registry/backend@sha256:" + "c" * 64
+        self.manifest["images"]["backend"]["configured"] = digest
+        self.configured["backend"] = digest
+        plan = image_restore_plan(self.manifest, self.configured)
+        with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "a" * 64,
+                                                 "sha256:" + "b" * 64]):
+            verify_previous_images(plan)
+        with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "d" * 64]):
+            with self.assertRaisesRegex(ValueError, "does not match its recorded image"):
+                verify_previous_images(plan)
+
+    def test_missing_previous_image_fails_closed(self):
+        plan = image_restore_plan(self.manifest, self.configured)
+        with patch("rollback.run", side_effect=subprocess.CalledProcessError(1, "docker")):
+            with self.assertRaisesRegex(ValueError, "previous image is unavailable"):
+                verify_previous_images(plan)
+
+    def test_digest_reference_is_never_tagged(self):
+        digest = "registry/backend@sha256:" + "c" * 64
+        plan = {"backend": ("sha256:" + "a" * 64, digest)}
+        with patch("rollback.run") as run_mock:
+            with self.assertRaisesRegex(ValueError, "not supported safely"):
+                tag_previous_images(plan)
+        run_mock.assert_not_called()
+
+    def test_digest_rollback_fails_before_stopping_or_changing_references(self):
+        backend_id = "sha256:" + "a" * 64
+        opensearch_id = "sha256:" + "b" * 64
+        digest = "registry/backend@sha256:" + "c" * 64
+        manifest = {
+            "images": {
+                "backend": {"configured": digest, "image_id": backend_id},
+                "opensearch": self.manifest["images"]["opensearch"],
+            },
+            "deployment_files": {
+                "deploy/production/compose.yaml": "compose-hash",
+                "deploy/production/Caddyfile": "caddy-hash",
+            },
+        }
+        rendered = {
+            "services": {
+                "backend": {"image": digest},
+                "opensearch": {"image": self.configured["opensearch"]},
+            }
+        }
+        fake_archive = SimpleNamespace(
+            digest=lambda path: "compose-hash" if path.name == "compose.yaml" else "caddy-hash",
+            verify_archive=lambda _backup: manifest,
+        )
+        backup = rollback.BACKUP_DIR / "synthetic.tar.gz"
+        with (
+            patch("rollback.Path.is_file", return_value=True),
+            patch("rollback.Path.is_symlink", return_value=False),
+            patch("rollback.Path.stat", return_value=SimpleNamespace(st_uid=0, st_mode=0o100600)),
+            patch("rollback.compose", return_value=json.dumps(rendered)) as compose_mock,
+            patch("rollback.run", side_effect=[backend_id, backend_id, opensearch_id]) as run_mock,
+            patch.dict(sys.modules, {"archive": fake_archive}),
+        ):
+            with self.assertRaisesRegex(ValueError, "not supported safely"):
+                rollback.rollback(backup, data_compatible=True)
+        compose_mock.assert_called_once_with("config", "--format", "json", capture=True)
+        self.assertEqual(run_mock.call_count, 3)
+        self.assertTrue(all(call.args[0][:3] == ["docker", "image", "inspect"]
+                            for call in run_mock.call_args_list))
+
+    def test_tag_rollback_keeps_existing_image_tag_operation(self):
+        plan = image_restore_plan(self.manifest, self.configured)
+        with patch("rollback.run") as run_mock:
+            tag_previous_images(plan)
+        self.assertEqual(run_mock.call_count, 2)
+        run_mock.assert_any_call(["docker", "image", "tag", "sha256:" + "a" * 64,
+                                  "akyldash/backend:local"])
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/deploy/test_verify_corpus.py

@@ -0,0 +1,349 @@
+import hashlib
+import json
+import os
+import fcntl
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+from verify_corpus import (
+    InventoryError,
+    database,
+    insert_or_update_file,
+    prefix_upper_bound,
+    raw_file_rows,
+    combined_source_hash,
+    NORMALIZER_VERSION,
+    safe_relative_path,
+    validate_fragment_file,
+    verify_corpus,
+)
+
+
+class CorpusInventoryTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name)
+        self.raw = self.base / "raw"
+        self.raw.mkdir()
+        self._manifest()
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def _manifest(self):
+        connection = sqlite3.connect(self.raw / "manifest.sqlite3")
+        connection.executescript(
+            "CREATE TABLE documents (code INTEGER PRIMARY KEY, source_sha256 TEXT, fetched_at TEXT);"
+             "CREATE TABLE errors (code INTEGER PRIMARY KEY, message TEXT, failed_at TEXT);"
+        )
+        connection.close()
+
+    def _nonempty_corpus(self, normalizer_version=NORMALIZER_VERSION):
+        document = self.raw / "documents/1"
+        edition = document / "editions/2"
+        edition.mkdir(parents=True)
+        (document / "metadata.json").write_text('{"Code":"1"}\n', encoding="utf-8")
+        (edition / "metadata.json").write_text('{"Code":"2"}\n', encoding="utf-8")
+        html = edition / "ru.html"
+        html.write_text("<p>Фрагмент</p>\n", encoding="utf-8")
+        source_files = []
+        for path in sorted(document.rglob("*")):
+            if path.is_file():
+                digest = hashlib.sha256(path.read_bytes()).hexdigest()
+                source_files.append({"path": path.relative_to(self.raw).as_posix(), "sha256": digest})
+        source_sha = combined_source_hash(source_files)
+        raw_manifest = sqlite3.connect(self.raw / "manifest.sqlite3")
+        raw_manifest.execute("INSERT INTO documents VALUES (1, ?, 'now')", (source_sha,))
+        raw_manifest.commit()
+        raw_manifest.close()
+
+        normalized = self.base / "normalized"
+        language = normalized / "documents/1/editions/2/ru"
+        language.mkdir(parents=True)
+        fragment_text = "Фрагмент"
+        fragments = [{
+            "id": "document:1:edition:2:lang:ru:fragment:1",
+            "document_code": "1",
+            "edition_code": "2",
+            "language": "ru",
+            "position": 1,
+            "text": fragment_text,
+            "text_sha256": hashlib.sha256(fragment_text.encode("utf-8")).hexdigest(),
+            "source_path": "documents/1/editions/2/ru.html",
+            "source_sha256": next(item["sha256"] for item in source_files if item["path"].endswith("/ru.html")),
+        }]
+        (language / "fragments.json").write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
+        (language / "content.txt").write_text(fragment_text + "\n", encoding="utf-8")
+        (language / "content.html").write_text("<p>Фрагмент</p>\n", encoding="utf-8")
+        document_json = {
+            "schema_version": "1",
+            "source_code": "1",
+            "normalizer": {"version": normalizer_version},
+            "source": {"path": "documents/1", "files": source_files, "sha256": source_sha},
+            "editions": [{"source_code": "2"}],
+        }
+        (normalized / "documents/1/document.json").write_text(json.dumps(document_json), encoding="utf-8")
+        edition_json = {
+            "source_code": "2",
+            "available_languages": ["ru"],
+            "source": {
+                "path": "documents/1/editions/2",
+                "files": [item for item in source_files if item["path"].startswith("documents/1/editions/2/")],
+            },
+            "images": [],
+        }
+        (normalized / "documents/1/editions/2/edition.json").write_text(json.dumps(edition_json), encoding="utf-8")
+        normalized_manifest = sqlite3.connect(normalized / "manifest.sqlite3")
+        normalized_manifest.execute(
+            "CREATE TABLE documents (code TEXT PRIMARY KEY, source_sha256 TEXT, schema_version TEXT, normalizer_version TEXT, state TEXT)"
+        )
+        normalized_manifest.execute(
+            "INSERT INTO documents VALUES ('1', ?, '1', ?, 'success')", (source_sha, normalizer_version)
+        )
+        normalized_manifest.commit()
+        normalized_manifest.close()
+        return normalized, source_sha
+
+    def test_output_inside_source_is_rejected_without_creating_it(self):
+        output = self.raw / "artifacts"
+        before_mode = self.raw.stat().st_mode
+        with self.assertRaisesRegex(InventoryError, "separate"):
+            verify_corpus(self.raw, None, output)
+        self.assertFalse(output.exists())
+        self.assertEqual(self.raw.stat().st_mode, before_mode)
+        self.assertEqual({path.name for path in self.raw.iterdir()}, {"manifest.sqlite3"})
+
+    def test_symlink_output_is_rejected_without_changing_target(self):
+        target = self.base / "target"
+        target.mkdir(mode=0o755)
+        link = self.base / "output-link"
+        try:
+            link.symlink_to(target, target_is_directory=True)
+        except (NotImplementedError, OSError):
+            self.skipTest("directory symlinks are unavailable")
+        before_mode = target.stat().st_mode
+        with self.assertRaisesRegex(InventoryError, "link or file"):
+            verify_corpus(self.raw, None, link)
+        self.assertEqual(target.stat().st_mode, before_mode)
+        self.assertEqual(list(target.iterdir()), [])
+
+    def test_public_existing_output_is_rejected_without_chmod(self):
+        output = self.base / "public-output"
+        output.mkdir(mode=0o755)
+        output.chmod(0o755)
+        before_mode = output.stat().st_mode
+        with self.assertRaisesRegex(InventoryError, "must be private"):
+            verify_corpus(self.raw, None, output)
+        self.assertEqual(output.stat().st_mode, before_mode)
+        self.assertEqual(list(output.iterdir()), [])
+
+    def test_resume_rejects_symlink_inventory_without_touching_target(self):
+        output = self.base / "artifacts"
+        output.mkdir(mode=0o700)
+        target = self.base / "do-not-touch.sqlite3"
+        target.write_bytes(b"sentinel")
+        link = output / "inventory.sqlite3"
+        try:
+            link.symlink_to(target)
+        except (NotImplementedError, OSError):
+            self.skipTest("file symlinks are unavailable")
+        with self.assertRaisesRegex(InventoryError, "private regular file"):
+            verify_corpus(self.raw, None, output, resume=True)
+        self.assertEqual(target.read_bytes(), b"sentinel")
+
+    def test_rejects_unsafe_cross_platform_relative_paths(self):
+        for value in ("../outside", "C:\\outside", "folder\\file", "//server/share"):
+            with self.subTest(value=value), self.assertRaises(InventoryError):
+                safe_relative_path(value)
+
+    def test_interrupted_inventory_resumes_and_detects_changed_input(self):
+        payload = self.raw / "payload.txt"
+        payload.write_text("before", encoding="utf-8")
+        output = self.base / "artifacts"
+        stopped = verify_corpus(self.raw, None, output, stop_after_entries=1)
+        self.assertEqual(stopped["status"], "incomplete")
+
+        resumed = verify_corpus(self.raw, None, output, resume=True)
+        self.assertEqual(resumed["status"], "complete")
+        self.assertEqual(resumed["input_changes"]["raw"]["added"], 1)
+
+        payload.write_text("after", encoding="utf-8")
+        changed = verify_corpus(self.raw, None, output, resume=True)
+        self.assertEqual(changed["status"], "complete")
+        self.assertEqual(changed["input_changes"]["raw"]["changed"], 1)
+        rows = sqlite3.connect(output / "inventory.sqlite3").execute(
+            "SELECT sha256 FROM files WHERE relative_path='payload.txt'"
+        ).fetchone()
+        self.assertEqual(rows[0], hashlib.sha256(b"after").hexdigest())
+
+    def test_large_filesystem_identity_values_are_stored_without_integer_overflow(self):
+        connection = database(self.base / "identity.sqlite3")
+        insert_or_update_file(
+            connection, "run", "raw", "payload", 1, "a" * 64,
+            (2**100, 2**120, 1, 2**110, 2**115, 1), {"raw": {"added": 0, "changed": 0}},
+        )
+        device, inode, mtime_ns, ctime_ns = connection.execute(
+            "SELECT device,inode,mtime_ns,ctime_ns FROM files"
+        ).fetchone()
+        self.assertEqual((device, inode, mtime_ns, ctime_ns), tuple(map(str, (2**100, 2**120, 2**110, 2**115))))
+        connection.close()
+
+    def test_document_prefix_uses_primary_key_range_and_excludes_neighbor_codes(self):
+        connection = database(self.base / "paths.sqlite3")
+        prefix = "documents/1/"
+        upper = prefix_upper_bound(prefix)
+        plan = connection.execute(
+            "EXPLAIN QUERY PLAN SELECT relative_path FROM files WHERE root_id='raw' AND relative_path>=? AND relative_path<?",
+            (prefix, upper),
+        ).fetchone()[3]
+        self.assertIn("USING COVERING INDEX sqlite_autoindex_files_1", plan)
+        connection.executemany(
+            "INSERT INTO files(root_id,relative_path,status,last_seen_run) VALUES('raw',?,'hashed','run')",
+            [(f"documents/{code}/a",) for code in range(20000)]
+            + [("documents/1x/a",)],
+        )
+        self.assertEqual([item["path"] for item in raw_file_rows(connection, "1")], ["documents/1/a"])
+        connection.close()
+
+    def test_inventory_rejects_a_concurrent_writer_and_records_verifier_hash(self):
+        output = self.base / "artifacts"
+        output.mkdir(mode=0o700)
+        lock_path = output / ".inventory.lock"
+        descriptor = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600)
+        try:
+            fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            with self.assertRaisesRegex(InventoryError, "another inventory run"):
+                verify_corpus(self.raw, None, output)
+        finally:
+            os.close(descriptor)
+
+        result = verify_corpus(self.raw, None, output)
+        self.assertEqual(result["status"], "complete")
+        self.assertRegex(result["verifier_sha256"], r"^[a-f0-9]{64}$")
+
+    def test_removed_file_is_reported_on_resume(self):
+        payload = self.raw / "payload.txt"
+        payload.write_text("payload", encoding="utf-8")
+        output = self.base / "artifacts"
+        self.assertEqual(verify_corpus(self.raw, None, output)["status"], "complete")
+        payload.unlink()
+        result = verify_corpus(self.raw, None, output, resume=True)
+        self.assertEqual(result["input_changes"]["raw"]["removed"], 1)
+        self.assertIn("removed_file", {item["kind"] for item in result["finding_examples"]})
+
+    def test_added_file_is_reported_on_resume(self):
+        output = self.base / "artifacts"
+        self.assertEqual(verify_corpus(self.raw, None, output)["status"], "complete")
+        (self.raw / "extra.txt").write_text("extra", encoding="utf-8")
+        result = verify_corpus(self.raw, None, output, resume=True)
+        self.assertEqual(result["input_changes"]["raw"]["added"], 1)
+
+    def test_manifest_error_missing_document_and_extra_directory_are_reported(self):
+        connection = sqlite3.connect(self.raw / "manifest.sqlite3")
+        connection.execute("INSERT INTO documents VALUES (1, ?, 'now')", ("a" * 64,))
+        connection.execute("INSERT INTO errors VALUES (6, 'fetch failed', 'now')")
+        connection.commit()
+        connection.close()
+        (self.raw / "documents" / "999").mkdir(parents=True)
+
+        result = verify_corpus(self.raw, None, self.base / "artifacts")
+        kinds = {finding["kind"] for finding in result["finding_examples"]}
+        self.assertEqual(result["status"], "failed")
+        self.assertIn("raw_manifest_error", kinds)
+        self.assertIn("missing_raw_document_directory", kinds)
+        self.assertIn("extra_raw_document_directory", kinds)
+
+    def test_empty_raw_and_normalized_manifests_complete_successfully(self):
+        normalized = self.base / "normalized"
+        normalized.mkdir()
+        connection = sqlite3.connect(normalized / "manifest.sqlite3")
+        connection.execute(
+            "CREATE TABLE documents (code TEXT PRIMARY KEY, source_sha256 TEXT, schema_version TEXT, normalizer_version TEXT, state TEXT)"
+        )
+        connection.commit()
+        connection.close()
+        result = verify_corpus(self.raw, normalized, self.base / "artifacts")
+        self.assertEqual(result["status"], "complete", result["finding_examples"])
+        self.assertEqual(result["manifest_summary"]["raw"]["documents"], 0)
+        self.assertEqual(result["manifest_summary"]["normalized"]["documents"], 0)
+
+    def test_nonempty_raw_and_normalized_manifests_cross_validate_fragment_sources(self):
+        normalized, _ = self._nonempty_corpus()
+        result = verify_corpus(self.raw, normalized, self.base / "artifacts")
+        self.assertEqual(result["status"], "complete", result["finding_examples"])
+        self.assertEqual(result["manifest_summary"]["raw"]["documents"], 1)
+        self.assertEqual(result["manifest_summary"]["normalized"]["fragments"], 1)
+
+    def test_verifies_legacy_normalizer_1_0_0_manifest(self):
+        normalized, _ = self._nonempty_corpus(normalizer_version="1.0.0")
+        result = verify_corpus(self.raw, normalized, self.base / "artifacts")
+        self.assertEqual(result["status"], "complete", result["finding_examples"])
+        self.assertNotIn(
+            "normalizer_version_mismatch",
+            {item["kind"] for item in result["finding_examples"]},
+        )
+
+    def test_rejects_unsupported_normalizer_version(self):
+        normalized, _ = self._nonempty_corpus(normalizer_version="9.9.9")
+        result = verify_corpus(self.raw, normalized, self.base / "artifacts")
+        self.assertEqual(result["status"], "failed")
+        self.assertIn(
+            "normalizer_version_mismatch",
+            {item["kind"] for item in result["finding_examples"]},
+        )
+
+    def test_raw_normalized_source_mismatch_is_reported(self):
+        normalized, _ = self._nonempty_corpus()
+        manifest = sqlite3.connect(normalized / "manifest.sqlite3")
+        manifest.execute("UPDATE documents SET source_sha256=? WHERE code='1'", ("0" * 64,))
+        manifest.commit()
+        manifest.close()
+        result = verify_corpus(self.raw, normalized, self.base / "artifacts")
+        kinds = {item["kind"] for item in result["finding_examples"]}
+        self.assertEqual(result["status"], "failed")
+        self.assertIn("normalized_source_hash_mismatch", kinds)
+        self.assertIn("document_source_manifest_mismatch", kinds)
+
+    def test_fragment_text_hash_mismatch_is_reported(self):
+        normalized = self.base / "normalized"
+        fragment_rel = "documents/1/editions/2/ru/fragments.json"
+        fragment_path = normalized / fragment_rel
+        fragment_path.parent.mkdir(parents=True)
+        fragment = {
+            "id": "document:1:edition:2:lang:ru:fragment:1",
+            "document_code": "1",
+            "edition_code": "2",
+            "language": "ru",
+            "position": 1,
+            "text": "text",
+            "text_sha256": "0" * 64,
+            "source_path": "documents/1/source.txt",
+            "source_sha256": "1" * 64,
+        }
+        fragment_path.write_text(json.dumps([fragment]), encoding="utf-8")
+        content_path = fragment_path.with_name("content.txt")
+        content_path.write_text("text\n", encoding="utf-8")
+        connection = database(self.base / "test.sqlite3")
+        content_digest = hashlib.sha256(b"text\n").hexdigest()
+        connection.execute(
+            "INSERT INTO files(root_id,relative_path,sha256,status,last_seen_run) VALUES('normalized',?,?, 'hashed','run')",
+            (content_path.relative_to(normalized).as_posix(), content_digest),
+        )
+        validate_fragment_file(
+            connection,
+            "run",
+            normalized,
+            fragment_path,
+            fragment_rel,
+            hashlib.sha256(fragment_path.read_bytes()).hexdigest(),
+            {"documents/1/source.txt": "1" * 64},
+        )
+        kinds = {row[0] for row in connection.execute("SELECT kind FROM findings WHERE run_id='run'")}
+        connection.close()
+        self.assertIn("fragment_text_checksum_mismatch", kinds)
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/deploy/validate_release.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Validate the shape and repository allowlist of a prepared release manifest."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+SERVICES = {"backend", "opensearch"}
+COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z")
+REPOSITORY_RE = re.compile(
+    r"[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?"
+    r"(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*\Z"
+)
+IMAGE_RE = re.compile(r"(.+)@sha256:([0-9a-f]{64})\Z")
+
+
+def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f"duplicate JSON key: {key}")
+        result[key] = value
+    return result
+
+
+def read_json(path: Path) -> object:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=unique_object)
+    except (OSError, UnicodeError, json.JSONDecodeError) as error:
+        raise ValueError(f"cannot read valid JSON from {path}: {error}") from error
+
+
+def validate_policy(policy: object) -> dict[str, list[str]]:
+    if not isinstance(policy, dict) or set(policy) != {"schema_version", "repositories"}:
+        raise ValueError("policy must contain exactly schema_version and repositories")
+    if type(policy["schema_version"]) is not int or policy["schema_version"] != 1:
+        raise ValueError("unsupported policy schema_version")
+    repositories = policy["repositories"]
+    if not isinstance(repositories, dict) or set(repositories) != SERVICES:
+        raise ValueError("policy repositories must contain exactly backend and opensearch")
+    result: dict[str, list[str]] = {}
+    for service in sorted(SERVICES):
+        allowed = repositories[service]
+        if not isinstance(allowed, list) or not allowed:
+            raise ValueError(f"policy repository allowlist for {service} must be a non-empty list")
+        if any(not isinstance(repository, str) or not REPOSITORY_RE.fullmatch(repository) for repository in allowed):
+            raise ValueError(f"policy contains an invalid repository for {service}")
+        if len(set(allowed)) != len(allowed):
+            raise ValueError(f"policy repository allowlist for {service} contains duplicates")
+        result[service] = allowed
+    return result
+
+
+def validate_manifest(manifest: object, policy: object) -> dict[str, object]:
+    allowed_repositories = validate_policy(policy)
+    if not isinstance(manifest, dict) or set(manifest) != {
+        "schema_version", "source_ref", "commit", "images"
+    }:
+        raise ValueError("manifest must contain exactly schema_version, source_ref, commit, and images")
+    if type(manifest["schema_version"]) is not int or manifest["schema_version"] != 1:
+        raise ValueError("unsupported manifest schema_version")
+    if manifest["source_ref"] != "refs/heads/main":
+        raise ValueError("manifest source_ref must be refs/heads/main")
+    commit = manifest["commit"]
+    if not isinstance(commit, str) or not COMMIT_RE.fullmatch(commit):
+        raise ValueError("manifest commit must be a 40-character lowercase Git SHA-1")
+    images = manifest["images"]
+    if not isinstance(images, dict) or set(images) != SERVICES:
+        raise ValueError("manifest images must contain exactly backend and opensearch")
+    for service in sorted(SERVICES):
+        image = images[service]
+        match = IMAGE_RE.fullmatch(image) if isinstance(image, str) else None
+        if not match:
+            raise ValueError(f"{service} image must use repository@sha256:<64 lowercase hex>")
+        repository = match.group(1)
+        if not REPOSITORY_RE.fullmatch(repository):
+            raise ValueError(f"{service} image has an invalid OCI repository")
+        if repository not in allowed_repositories[service]:
+            raise ValueError(f"{service} image repository is not allowed by policy")
+    return manifest
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("manifest", type=Path)
+    parser.add_argument("--policy", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        manifest = validate_manifest(read_json(args.manifest), read_json(args.policy))
+    except ValueError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    images = manifest["images"]
+    print(
+        "PASS release manifest "
+        f"commit={manifest['commit']} backend={images['backend']} opensearch={images['opensearch']}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infra/inventory/production.yml

@@ -0,0 +1,134 @@
+project: akyldash
+environment: production
+ssh:
+  alias: akyldash_vps
+  host: 206.223.249.118
+  port: 22
+  user: codex
+  identity_file: ~/.ssh/akyldash_codex
+domain: akyldash.kg
+facts:
+  hostname: avprogs.hlab.kz
+  os: Ubuntu Server 24.04.5 LTS
+  kernel: 6.8.0-142-generic
+  cpu_vcpus: 4
+  memory_gib: 7.7
+  swap_gib: 3.8
+  disk_gib: 99
+  disk_free_gib: 92
+  network_interface: ens3
+  ipv4: 206.223.249.118/32
+  domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses
+  domain_http_status: Cloudflare 521 on HTTP and HTTPS from the VPS
+  user_groups: [codex, users]
+  sudo_noninteractive: false
+  installed:
+    - OpenSSH server
+    - systemd-resolved
+    - unattended-upgrades
+    - Docker CE and CLI 29.9.0
+    - Docker Compose plugin 5.6.0
+    - containerd 2.4.1
+    - runc 1.5.2
+    - Git
+  absent:
+    - ISPmanager package/service (unprivileged package and unit checks)
+    - root-owned application checkout
+    - restricted verifier and rollback helper
+    - non-loopback listeners on TCP 80, 443, 1500
+  access_audit_date: 2026-10-12
+  docker_service_active: true
+  docker_api_accessible_to_codex: false
+  docker_daemon_log_config:
+    driver: local
+    max_size: 10m
+    max_file: 3
+  bootstrap_paths:
+    /opt/akyldash: root:root 0755
+    /etc/akyldash: root:root 0700
+    /var/lib/akyldash/data: root:root 0750
+    /var/backups/akyldash: root:root 0700
+  unprivileged_nonloopback_tcp_listeners: [22]
+  owner_reported_ufw_allowed_tcp: [22, 80, 443, 1500]
+  port_1500_listener: false
+  rootless_docker_command_present: /usr/bin/dockerd-rootless.sh
+  runner_gitea_host_resolution: git.vakanaut.ru resolves to 195.209.48.28
+  runner_gitea_https_probe: /api/v1/version returned Gitea 1.25.5 (Actions protocol unverified)
+  cloudflare_edge_status:
+    http: 521
+    https: 521
+target:
+  compose_file_in_repository: deploy/production/compose.yaml
+  compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml
+  production_env_file: /etc/akyldash/production.env
+  secrets_directory: /etc/akyldash
+  data_directory: /var/lib/akyldash/data
+  backup_directory: /var/backups/akyldash
+  services:
+    caddy:
+      container: compose project service caddy
+      image: caddy:2.11.6-alpine pinned by digest in deploy/production/compose.yaml
+      host_ports: [80, 443]
+      persistent_data: named Compose volumes caddy-data and caddy-config
+    backend:
+      container: compose project service backend
+      image: built from deploy/production/Dockerfile.backend; digest deployment not configured
+      port: 8080 on private proxy and search Compose networks only
+      healthcheck: HTTP GET /review on container port 8080
+      persistent_data: /var/lib/akyldash/data (bind mount)
+    opensearch:
+      container: compose project service opensearch
+      image: opensearchproject/opensearch:3.7.0 plus analysis-icu plugin
+      port: 9200 on private Compose network only
+      persistent_data: named Compose volume opensearch-data
+  reverse_proxy:
+    target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080
+    status: no listener observed on 80/443; container state needs root Docker audit
+  external_ports_target: [22, 80, 443]
+  compose_networks:
+    proxy: caddy and backend
+    search: backend and OpenSearch; internal network
+  secrets:
+    - REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
+    - Gitea credentials from environment-backed gitea-api helper
+    - SSH key at ~/.ssh/akyldash_codex
+  restricted_helper:
+    path: /usr/local/sbin/akyldash-rollback
+    status: repository code and owner-run installer exist; not installed on VPS
+  compose_read_only_helper:
+    path: /usr/local/sbin/akyldash-compose
+    allowed_operations: [config, status]
+    status: repository code and owner-run installer exist; not installed on VPS
+  read_only_verifier:
+    path: /usr/local/sbin/akyldash-verify
+    status: repository code and owner-run installer exist; not installed on VPS
+  gitea_actions_runner:
+    repository: admin/akyldash
+    registration_scope: repository
+    implementation: official act_runner 0.2.13; pinned binary SHA-256 in infra/runner/install-runner.sh
+    label: akyldash-ci
+    capacity: 1
+    job_limits: {cpu: 2, memory_gib: 3, pids: 512}
+    account_slice_limits: {cpu: 2.5, memory_gib: 4, tasks: 2048}
+    docker_store_limit_gib: 15
+    docker_boundary: rootless dedicated daemon; no production socket; job socket injection disabled
+    service_status: configuration prepared, not installed or registered; Online unverified
+    owner_action: run the root installer from the reviewed checkout; register through repository runner settings
+  checks:
+    - infra/bootstrap/verify.sh akyldash_vps
+    - docker compose -f deploy/production/compose.yaml ps
+    - curl -fsS https://akyldash.kg/review
+    - python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
+  backup_path: /var/backups/akyldash (directory exists; archive script in infra/backup/archive.py is not installed or run)
+  deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment)
+versions:
+  application:
+    backend: 0.11.3
+    frontend: 0.1.2
+    telegram_bot: 0.2.2
+  infrastructure: unknown; no production install or release exists
+  caddy_image: 2.11.6-alpine pinned by manifest digest in Compose
+  docker_engine: 29.9.0 (daemon active; server API unverified as codex)
+  docker_compose: 5.6.0
+  containerd: 2.4.1
+  runc: 1.5.2

infra/runner/act-runner-healthcheck.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+systemctl is-active --quiet akyldash-act-runner.service
+runner_uid=$(id -u akyldash-runner)
+runuser -u akyldash-runner -- env \
+  HOME=/var/lib/akyldash-runner/.config/act-runner \
+  XDG_CONFIG_HOME=/var/lib/akyldash-runner/.config \
+  XDG_DATA_HOME=/var/lib/akyldash-runner/.config/act-runner/.local/share \
+  DOCKER_HOST="unix:///run/user/$runner_uid/docker.sock" \
+  /usr/bin/docker info --format '{{.ServerVersion}}' >/dev/null

infra/runner/akyldash-runner-healthcheck.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Check AkylDash Gitea Actions runner health
+OnFailure=akyldash-runner-restart.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/akyldash-runner-healthcheck

infra/runner/akyldash-runner-healthcheck.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodically check AkylDash Gitea Actions runner health
+
+[Timer]
+OnBootSec=2m
+OnUnitActiveSec=1m
+Unit=akyldash-runner-healthcheck.service
+
+[Install]
+WantedBy=timers.target

infra/runner/akyldash-runner-restart.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Restart AkylDash Gitea Actions runner after a failed health check
+
+[Service]
+Type=oneshot
+ExecStart=/bin/systemctl restart akyldash-act-runner.service