Compare commits
146
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d3b34f34b | ||
|
|
cc8cc1bce6 | ||
|
|
a68d3c315e | ||
|
|
ba669fad34 | ||
|
|
8f64131131 | ||
|
|
802bfd117e | ||
|
|
2281705c95 | ||
|
|
fc0473e7e2 | ||
|
|
e12a8a919c | ||
|
|
37050b1458 | ||
|
|
6bd4b05af2 | ||
|
|
0da2bcd82d | ||
|
|
0979b8f544 | ||
|
|
4d106903e1 | ||
|
|
4b2e5578cd | ||
|
|
0e276ae652 | ||
|
|
554a7ab5ff | ||
|
|
cf99f4532f | ||
|
|
5d72bb092d | ||
|
|
d96fc4d36b | ||
|
|
436806a834 | ||
|
|
3a89dc2196 | ||
|
|
69c62466ca | ||
|
|
36892aef77 | ||
|
|
ac611cc878 | ||
|
|
9ec03200fb | ||
|
|
03c5809d69 | ||
|
|
00922e6c09 | ||
|
|
e6b099cf31 | ||
|
|
d8a0646829 | ||
|
|
433c3059b9 | ||
|
|
4110aa54c9 | ||
|
|
db6ded4075 | ||
|
|
b68421bb5b | ||
|
|
223e06ee77 | ||
|
|
52027ecf81 | ||
|
|
e5a4dab5f3 | ||
|
|
e034fdb816 | ||
|
|
a34f516de8 | ||
|
|
3b41d1f20f | ||
|
|
752266d408 | ||
|
|
f6c9b9883a | ||
|
|
d6d029d98c | ||
|
|
9084fd4f16 | ||
|
|
02cf4e6e95 | ||
|
|
711700fcc9 | ||
|
|
6f809d3196 | ||
|
|
16f97d5f73 | ||
|
|
97803ca5de | ||
|
|
fb556228c7 | ||
|
|
e42c314552 | ||
|
|
7124c82bb6 | ||
|
|
892781522d | ||
|
|
5490b22275 | ||
|
|
5c78f7ebb8 | ||
|
|
d03da40eee | ||
|
|
fb1a67044c | ||
|
|
7ed85dae34 | ||
|
|
88fd84a515 | ||
|
|
6de2f47933 | ||
|
|
7912fdd8b6 | ||
|
|
075e58fa07 | ||
|
|
734631a12d | ||
|
|
0a30adb85f | ||
|
|
93e5fc5837 | ||
|
|
ac4380c8b3 | ||
|
|
2b5b79c6c2 | ||
|
|
9a80e08739 | ||
|
|
3b03331955 | ||
|
|
20d58963e7 | ||
|
|
a13343647a | ||
|
|
4e2e12f83a | ||
|
|
e1b7d502b2 | ||
|
|
2ffb02428b | ||
|
|
9c3cbee141 | ||
|
|
feb0ddde32 | ||
|
|
3afac00394 | ||
|
|
cf510f08f8 | ||
|
|
a3d82b45b5 | ||
|
|
6f8b668907 | ||
|
|
3b56445bb2 | ||
|
|
bc57cb82fe | ||
|
|
9832c0c795 | ||
|
|
5e029be5ee | ||
|
|
6d393d38bb | ||
|
|
2c43c7c9ad | ||
|
|
14e86ccdd8 | ||
|
|
47d51cd36b | ||
|
|
c2537cf445 | ||
|
|
dffe072ce2 | ||
|
|
06c275cc40 | ||
|
|
92051184f5 | ||
|
|
c048f20734 | ||
|
|
af6b0b8098 | ||
|
|
94ba04ff56 | ||
|
|
9fb266e8ac | ||
|
|
f045233a50 | ||
|
|
6a187760db | ||
|
|
90298fa6cc | ||
|
|
503cb1f074 | ||
|
|
4d276e0776 | ||
|
|
298191173e | ||
|
|
81b085dd9d | ||
|
|
b6ad392ab7 | ||
|
|
bb5edf3ae2 | ||
|
|
000cf467df | ||
|
|
f8f98dfe46 | ||
|
|
5f44981bd3 | ||
|
|
a1b4787930 | ||
|
|
b412315ac6 | ||
|
|
7e07f3ab8d | ||
|
|
70c70fb7dd | ||
|
|
e18d477852 | ||
|
|
062aaa0074 | ||
|
|
470745fce3 | ||
|
|
b2274fee6f | ||
|
|
abd244fefb | ||
|
|
8302c6b782 | ||
|
|
bbf8e7a9c8 | ||
|
|
bf0b6ff070 | ||
|
|
b91bc98611 | ||
|
|
4210c2493d | ||
|
|
711c9e525e | ||
|
|
808ac6c792 | ||
|
|
73eb938c03 | ||
|
|
cccae446ae | ||
|
|
24b3a2d422 | ||
|
|
f472a17897 | ||
|
|
182b87fff2 | ||
|
|
407dd2dc09 | ||
|
|
a594aec912 | ||
|
|
c5f651bdae | ||
|
|
f4cd453088 | ||
|
|
57e4645a11 | ||
|
|
9b28616cf4 | ||
|
|
e9da5a6ccc | ||
|
|
cb58e91d6b | ||
|
|
39896c74de | ||
|
|
f3f564ab08 | ||
|
|
ccbb2f0944 | ||
|
|
d468c869c0 | ||
|
|
47e03bbc4d | ||
|
|
eafc06f0bc | ||
|
|
9dcbcad9aa | ||
|
|
028062bd14 | ||
|
|
bb69ea54b7 |
No files matched your search
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
.env.*
|
||||
*.json
|
||||
!backend/
|
||||
!backend/search/
|
||||
!backend/search/*.json
|
||||
*.xlsx
|
||||
*.pdf
|
||||
*.jpg
|
||||
data/
|
||||
docs/
|
||||
tools/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1 @@
|
||||
FROM caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Validate Caddyfile
|
||||
description: Run the pinned Caddy validator against the checked-out Caddyfile
|
||||
inputs:
|
||||
workspace:
|
||||
description: Repository workspace path supplied by the workflow
|
||||
required: true
|
||||
runs:
|
||||
using: docker
|
||||
image: Dockerfile
|
||||
entrypoint: /usr/bin/caddy
|
||||
args:
|
||||
- validate
|
||||
- --config
|
||||
- ${{ inputs.workspace }}/deploy/production/Caddyfile
|
||||
- --adapter
|
||||
- caddyfile
|
||||
@@ -0,0 +1 @@
|
||||
FROM ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Build image with Kaniko in a rootless container
|
||||
description: Validate a Dockerfile by building it without publishing the image or cache
|
||||
inputs:
|
||||
workspace:
|
||||
description: Repository workspace path supplied by the workflow
|
||||
required: true
|
||||
dockerfile:
|
||||
description: Dockerfile path relative to the checked-out repository
|
||||
required: true
|
||||
destination:
|
||||
description: Image reference required by Kaniko; the image is not pushed
|
||||
required: true
|
||||
runs:
|
||||
using: docker
|
||||
image: Dockerfile
|
||||
entrypoint: /kaniko/executor
|
||||
args:
|
||||
- --context=dir://${{ inputs.workspace }}
|
||||
- --dockerfile=${{ inputs.workspace }}/${{ inputs.dockerfile }}
|
||||
- --destination=${{ inputs.destination }}
|
||||
- --no-push
|
||||
- --no-push-cache
|
||||
@@ -0,0 +1,77 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
tests-and-images:
|
||||
runs-on: akyldash-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Run backend tests
|
||||
run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
|
||||
- name: Run Telegram bot tests
|
||||
run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
|
||||
- name: Run infrastructure backup tests
|
||||
run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
|
||||
- name: Check root guard UID decisions
|
||||
run: bash infra/bootstrap/test-root-guard.sh
|
||||
- name: Check recovery host guard
|
||||
run: bash infra/bootstrap/test-host-guard.sh
|
||||
- name: Check verifier helper argument guard
|
||||
run: bash infra/bootstrap/test-verify-helper.sh
|
||||
- name: Check host audit root guard
|
||||
run: bash infra/bootstrap/test-audit-host.sh
|
||||
- name: Check Compose helper allowlist
|
||||
run: bash infra/deploy/test-compose-helper.sh
|
||||
- name: Check production config root guard
|
||||
run: bash infra/deploy/test-production-config-guard.sh
|
||||
- name: Check runner config and isolation guards
|
||||
run: |
|
||||
bash infra/runner/test-runner-config.sh
|
||||
bash infra/runner/test-install-runner-idempotency.sh
|
||||
bash infra/runner/test-install-runner-guard.sh
|
||||
- name: Check CI container operation isolation
|
||||
run: bash infra/runner/test-ci-container-builds.sh
|
||||
- name: Check infrastructure shell syntax
|
||||
run: |
|
||||
bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
|
||||
infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/bootstrap/audit-host.sh \
|
||||
infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \
|
||||
infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \
|
||||
infra/bootstrap/test-audit-host.sh infra/bootstrap/test-root-guard.sh \
|
||||
infra/bootstrap/test-verify-helper.sh infra/test/run-as-unprivileged.sh \
|
||||
infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \
|
||||
infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \
|
||||
infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh \
|
||||
infra/runner/test-ci-container-builds.sh
|
||||
python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())"
|
||||
python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/deploy/validate_release.py').read_text())"
|
||||
- name: Run deployment contract and rollback tests
|
||||
run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
|
||||
- name: Validate production Compose and Caddyfile
|
||||
run: |
|
||||
secret_file=$(mktemp)
|
||||
trap 'rm -f "$secret_file"' EXIT
|
||||
REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet
|
||||
- name: Validate Caddyfile in isolated container action
|
||||
uses: ./.gitea/actions/caddy-validate
|
||||
with:
|
||||
workspace: ${{ github.workspace }}
|
||||
- name: Build production images
|
||||
uses: ./.gitea/actions/kaniko-build
|
||||
with:
|
||||
workspace: ${{ github.workspace }}
|
||||
dockerfile: deploy/production/Dockerfile.backend
|
||||
destination: akyldash-backend:ci
|
||||
- name: Build OpenSearch production image
|
||||
uses: ./.gitea/actions/kaniko-build
|
||||
with:
|
||||
workspace: ${{ github.workspace }}
|
||||
dockerfile: deploy/production/Dockerfile.opensearch
|
||||
destination: akyldash-opensearch:ci
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Runner smoke test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
runner-smoke:
|
||||
runs-on: akyldash-ci
|
||||
steps:
|
||||
- name: Verify isolated runner boundary
|
||||
run: |
|
||||
host_uid=$(awk 'NR == 1 {print $2}' /proc/self/uid_map)
|
||||
test "$host_uid" -gt 0
|
||||
test ! -S /var/run/docker.sock
|
||||
test ! -e /run/user/1000/docker.sock
|
||||
test -z "${DOCKER_HOST:-}"
|
||||
test ! -e /etc/akyldash/production.env
|
||||
test ! -e /etc/akyldash/review_secret
|
||||
test ! -e /opt/akyldash/current
|
||||
printf 'Isolated runner: version=%s host=%s uid-map=%s\n' \
|
||||
"$GITEA_ACTIONS_RUNNER_VERSION" "$(hostname)" "$host_uid"
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"detector": {
|
||||
"ignoreRules": [],
|
||||
"ignoreFiles": [],
|
||||
"ignoreValues": [
|
||||
{
|
||||
"rule": "cream-palette",
|
||||
"value": "*",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:40:02.635Z",
|
||||
"reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette"
|
||||
},
|
||||
{
|
||||
"rule": "cramped-padding",
|
||||
"value": "*",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:41:28.434Z",
|
||||
"reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector"
|
||||
},
|
||||
{
|
||||
"rule": "design-system-font",
|
||||
"value": "sans",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:41:33.020Z",
|
||||
"reason": "Agent: DESIGN.md specifies Noto Sans with sans-serif fallback; this finding is the inherited generic fallback"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
# Акылдаш — инструкции для агента
|
||||
|
||||
Проект: юридическая информационно-аналитическая платформа; Search API на
|
||||
Python, SQLite и OpenSearch. Production пока не развернут.
|
||||
|
||||
- Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и
|
||||
[правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно.
|
||||
- Архитектура и процедуры: [инфраструктура](docs/operations/INFRASTRUCTURE.md),
|
||||
[деплой](docs/operations/DEPLOYMENT.md), [backup](docs/operations/BACKUP.md),
|
||||
[rollback](docs/operations/ROLLBACK.md), [восстановление](docs/operations/RECOVERY.md).
|
||||
- Работай в feature/fix/refactor-ветке. `main` защищена; изменения вноси через
|
||||
PR. DEV не одобряет и не сливает собственный PR. Условный merge выполняет
|
||||
только REVIEWER по правилам раздела «Автономный workflow PR» ниже. Сохраняй
|
||||
незакоммиченные изменения.
|
||||
- SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай
|
||||
`codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
|
||||
Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
|
||||
- Репозиторный Gitea Actions runner описан в
|
||||
[CI_RUNNER.md](docs/operations/CI_RUNNER.md). Регистрация выполняется
|
||||
владельцем через Settings → Actions → Runners для `admin/akyldash`; токен
|
||||
вводится интерактивно и не попадает в Git. Runner использует отдельный
|
||||
rootless Docker daemon; job-контейнеры не получают Docker socket или
|
||||
production secrets. Не включай production deployment из CI без отдельного
|
||||
проверенного deploy-helper.
|
||||
- Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
|
||||
`python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
|
||||
`python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и
|
||||
`python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`,
|
||||
`bash infra/bootstrap/test-host-guard.sh`.
|
||||
Production images собираются командами из `.gitea/workflows/ci.yaml`.
|
||||
- Перед изменениями production необходим проверенный backup. Формального
|
||||
механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
|
||||
SSH, firewall или production вручную без владельца и последующего PR.
|
||||
- Холодный backup требует согласованного окна простоя; восстановление допускается
|
||||
только в новый staging-каталог, пока владелец отдельно не проверил процедуру
|
||||
для нового хоста.
|
||||
- После каждого изменения инфраструктуры обновляй
|
||||
`docs/operations/CURRENT_STATE.md` по фактическим проверкам.
|
||||
|
||||
## Автономный workflow PR
|
||||
|
||||
- Маршруты ниже указаны из контекста оркестратора/root: оркестратор — thread
|
||||
`01a144a5-8b95-77c3-b311-9a9afd062ca8`, host `local`; DEV — thread
|
||||
`01a138f1-3b34-7c91-a9ed-4b43c3c85f1f`, host
|
||||
`remote-ssh-codex-managed:712b8d4e-24e1-4dd5-92af-85d1fbd26449`, Gitea
|
||||
`codex` (ID 6); REVIEWER — thread `01a144f9-5b51-7923-8398-61d4a6a98306`,
|
||||
host `remote-ssh-codex-managed:b61555d7-8d1a-4cfa-988b-c8da4791d426`, Gitea
|
||||
`agent` (ID 7). Если этот же чат отображает REVIEWER как `local` внутри его
|
||||
собственного app-server, это self-label и не является маршрутом из контекста
|
||||
оркестратора.
|
||||
- DEV принимает только явно зарегистрированную оркестратором человеческую
|
||||
задачу с полями `task`, `repo`, `branch`, `pr`, `head_sha`, `phase`. Репозиторий
|
||||
должен быть `admin/akyldash`; работай только в назначенной ветке и PR. Веди
|
||||
одну задачу/PR за раз и не подхватывай старые PR самостоятельно.
|
||||
- Допустимые фазы: `initial_review` — реализация и устранение review findings;
|
||||
`final_review_and_merge` — только после первоначального успешного review,
|
||||
changelog-коммита и отдельного назначения оркестратора. Не переходи к следующей
|
||||
фазе без назначения. Если задача приостановлена, останови любые действия;
|
||||
после resume перепроверь назначение, PR, head/base, checks и review с нуля.
|
||||
- Повторная доставка одного назначения идемпотентна по ключу
|
||||
`task`+`pr`+`head_sha`+`stage`: не дублируй commit, push, review-запрос или
|
||||
merge; сообщи оркестратору о уже выполненном результате. Новый head или stage
|
||||
образует новое назначение и требует повторной проверки.
|
||||
- Межчатовое чтение недоступно: оркестратор через central relay сам читает
|
||||
результаты и выдаёт назначения. Завершай каждый этап отчётом с полями `task`,
|
||||
`repo`, `branch`, `pr`, `head_sha`, `stage`, `checks`, `findings`,
|
||||
`next_action`. Не настраивай отдельные webhook, daemon или транспорт.
|
||||
- DEV исследует, изменяет, проверяет, коммитит и отправляет изменения; REVIEWER
|
||||
не меняет код и независимо проверяет полный diff от base после каждого push,
|
||||
включая предыдущие findings. DEV не одобряет и не сливает свой PR.
|
||||
- Только после первоначального успешного формального review DEV добавляет
|
||||
значимые изменения в существующий changelog, коммитит и отправляет их в тот же
|
||||
PR. Затем REVIEWER полностью проверяет итоговый diff. Для docs-only задачи не
|
||||
придумывай изменения кода; правила версии применяй только если они явно
|
||||
относятся к этой документационной правке.
|
||||
- Слияние может выполнить только REVIEWER для зарегистрированной человеческой
|
||||
задачи, в фазе `final_review_and_merge`, после явного назначения оркестратора.
|
||||
Непосредственно перед ним REVIEWER подтверждает `admin/akyldash`, PR, base
|
||||
`main`, ожидаемую ветку, автора PR `codex` (ID 6), неизменность base и head,
|
||||
причём head должен совпадать с зарегистрированным `head_sha`, включая
|
||||
`head_commit_id` для merge-запроса. Требуются все успешные required CI/test
|
||||
checks и gates, отсутствие pending/failed/unknown checks, открытых
|
||||
findings и активных `REQUEST_CHANGES`, плюс актуальный формальный `APPROVED`
|
||||
от REVIEWER для точного текущего commit с параметрами `commit_id == head_sha`,
|
||||
`stale == false`, `dismissed == false`. При изменении head или base review и
|
||||
проверки выполняются заново. Комментарий с approval не является формальным
|
||||
review.
|
||||
Используй только обычный server-side merge с `head_commit_id`; не используй
|
||||
`force_merge` и не обходи branch protection. Если любой существенный gate,
|
||||
статус или защита неизвестны либо не проверяются — не сливай PR и сообщи
|
||||
оркестратору, что именно осталось неизвестным.
|
||||
- Все Gitea API-вызовы выполняй только через `gitea-api` с личным окружением;
|
||||
Git используй с обычным credential helper. Не читай credentials из файлов
|
||||
репозитория. Запускай Git- и API-команды отдельно, без объединения с другими
|
||||
командами.
|
||||
@@ -2,8 +2,67 @@
|
||||
|
||||
## Не выпущено
|
||||
|
||||
- Rollback preflight проверяет digest-pinned image references по сохранённому
|
||||
image ID и останавливается до изменений deployment; переключение production
|
||||
на digest и полноценный digest rollback пока не реализованы.
|
||||
- Infrastructure: добавлен офлайн-контракт release manifest с точной схемой,
|
||||
digest-ссылками и repository allowlist; проверен Compose override с
|
||||
`--no-build` как требованием будущего deploy. Публикация, provenance и
|
||||
production-деплой не включены.
|
||||
- Добавлен read-only возобновляемый inventory корпуса с полным хешированием,
|
||||
потоковой сверкой, индексируемыми диапазонами путей и provenance SHA-256
|
||||
проверяющего; значения `stat` хранятся без ограничения SQLite INTEGER.
|
||||
Сравнение source/staging сверяет путь, размер и SHA-256, удерживает shared
|
||||
locks и отклоняет пустые корпуса без документов или фрагментов. Runbook и
|
||||
`CURRENT_STATE.md` отражают, что полный аудит корпуса ещё выполняется; это
|
||||
изменение не означает готовность корпуса или production.
|
||||
- CI actions теперь используют workspace, переданный Gitea, для Caddyfile и Kaniko build context; CI run 69 и smoke run 70 прошли на self-hosted runner.
|
||||
- Runner installer сохраняет `HOME` и XDG-пути в user-unit `docker.service`, перезапускает активный daemon и проверяет окружение его процесса.
|
||||
- Runner installer защищает image и mountpoint root-owned home-каталогом, оставляет регистрацию и workspace в runner-owned каталоге, безопасно форматирует образ по открытому FD и проверяет `nosuid,nodev`.
|
||||
- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, Caddy HTTPS proxy с разделёнными Compose-сетями и CI-проверка конфигурации; публикация образов и production-деплой остаются выключены до проверки runner, registry, backup/restore и ограниченного deploy-helper.
|
||||
- Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
|
||||
- Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
|
||||
- Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
|
||||
- Backend `0.11.0` и frontend `0.1.0`: лаборатория переоформлена по дизайн-системе «Түндүк»; добавлены учетные записи, обязательная смена временного пароля, личный кабинет и управление пользователями для `admin`.
|
||||
- Backend обновлён до `0.10.4`: многословный поиск получил низковесный OR fallback с требованием совпадения минимум двух терминов; fallback пропускает распознанные названия нормативных актов.
|
||||
- Backend обновлён до `0.10.3`: запрос о налоге на прибыль поднимает действующий
|
||||
Налоговый кодекс; точное совпадение фразы дополнительно усиливает результат.
|
||||
- Backend обновлён до `0.10.2`: действующие документы ранжируются выше, а
|
||||
утратившие силу — после остальных; из лаборатории удалён пустой блок фрагмента.
|
||||
- Backend обновлён до `0.10.1`: поисковая выдача по регистрации больше не
|
||||
ограничивается только тремя заранее выбранными нормативными актами.
|
||||
- Backend обновлён до `0.10.0`: поисковый API использует общий планировщик
|
||||
запросов, расширяет аббревиатуры ОсОО/ООО/ЖЧК и для запроса о регистрации
|
||||
приоритизирует положение КР 2026 года, закон об обществах и ГК КР.
|
||||
- Backend обновлён до `0.9.4`: из поисковых фрагментов убраны технические маркеры
|
||||
подсветки OpenSearch.
|
||||
- Backend обновлён до `0.9.3`: исправлен локальный индекс production Search API,
|
||||
ошибки поиска показываются в общем status-блоке, а панель сохранения и footer
|
||||
выстроены в обычном порядке документа.
|
||||
- Backend обновлён до `0.9.2`: внутренняя лаборатория получила визуальный
|
||||
polish-проход и короткий повторно запускаемый tutorial для новых проверяющих.
|
||||
- Backend обновлён до `0.9.1`: все статусы из нормализованного корпуса доступны
|
||||
как отдельные фильтры, варианты конституционного закона и указа сопоставляются
|
||||
с подтверждённой юридической силой, неизвестные типы документов сохраняются
|
||||
под общим фильтром без вывода юридической силы.
|
||||
- Уточнено, что внутренняя лаборатория оценивает выдачу собственного OpenSearch;
|
||||
ЦБД Минюста служит источником для подтверждения документов, а прежние Excel/PDF
|
||||
бланки удалены.
|
||||
- Исключены секреты и локальные данные из Docker build context; синхронизирована версия интерфейса.
|
||||
- Добавлен production deployment baseline для Search API и OpenSearch с
|
||||
постоянными хранилищами и healthcheck.
|
||||
- Добавлено восстановление незавершённой разметки из `localStorage` после перезагрузки страницы.
|
||||
- Добавлена внутренняя лаборатория проверки поисковой выдачи: просмотр документов,
|
||||
оценка релевантности 0–3, комментарии и SQLite-экспорт подписанных снимков.
|
||||
- Уточнены доступные состояния и адаптивное поведение внутреннего интерфейса
|
||||
оценки поисковой выдачи.
|
||||
- Добавлен план внутреннего интерфейса оценки поисковой выдачи юристами.
|
||||
- Завершён Search API v1: стабильные справочники, валидный OpenAPI, безопасная
|
||||
пагинация и проверка актуальных редакций в локальном OpenSearch.
|
||||
- Добавлен фильтр Search API v1 по укрупнённой юридической силе. Полная
|
||||
иерархия GeneralClassifiers экспортирована в отдельный черновой каталог;
|
||||
тематическая навигация остаётся вне API v1 до ручного утверждения рубрик и
|
||||
подписей.
|
||||
- Добавлено безопасное переключение alias на новую версию поискового индекса
|
||||
после полной загрузки; checkpoint защищает возобновление загрузки от смены
|
||||
alias.
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
---
|
||||
name: "Акылдаш · Түндүк"
|
||||
description: "Утверждённая дизайн-система правового портала Кыргызской Республики"
|
||||
colors:
|
||||
bg: "#F7F2E8"
|
||||
surface: "#FFFFFF"
|
||||
surface-muted: "#FBF8F4"
|
||||
text: "#38262A"
|
||||
text-secondary: "#786469"
|
||||
border: "#DED3CE"
|
||||
primary: "#9D2633"
|
||||
primary-hover: "#84202B"
|
||||
primary-active: "#701B24"
|
||||
on-primary: "#FFFFFF"
|
||||
primary-subtle: "#F8EBED"
|
||||
accent: "#A37B32"
|
||||
highlight: "#FFF0C7"
|
||||
focus: "#245C78"
|
||||
success: "#246843"
|
||||
success-bg: "#EAF4EC"
|
||||
warning: "#82520D"
|
||||
warning-bg: "#FFF3DA"
|
||||
error: "#9D2633"
|
||||
error-bg: "#FBEDEF"
|
||||
info: "#245C78"
|
||||
info-bg: "#EAF2F6"
|
||||
typography:
|
||||
document-title:
|
||||
fontFamily: "Noto Serif, serif"
|
||||
fontSize: "32px"
|
||||
fontWeight: 700
|
||||
lineHeight: 1.2
|
||||
section-title:
|
||||
fontFamily: "Noto Serif, serif"
|
||||
fontSize: "24px"
|
||||
fontWeight: 600
|
||||
lineHeight: 1.3
|
||||
document-body:
|
||||
fontFamily: "Noto Sans, sans-serif"
|
||||
fontSize: "18px"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.65
|
||||
control:
|
||||
fontFamily: "Noto Sans, sans-serif"
|
||||
fontSize: "16px"
|
||||
fontWeight: 500
|
||||
lineHeight: 1.5
|
||||
metadata:
|
||||
fontFamily: "Noto Sans, sans-serif"
|
||||
fontSize: "14px"
|
||||
fontWeight: 400
|
||||
lineHeight: 1.5
|
||||
rounded:
|
||||
default: "6px"
|
||||
spacing:
|
||||
xs: "4px"
|
||||
sm: "8px"
|
||||
compact: "12px"
|
||||
md: "16px"
|
||||
lg: "24px"
|
||||
xl: "32px"
|
||||
section: "48px"
|
||||
---
|
||||
|
||||
# Design System: Акылдаш · Түндүк
|
||||
|
||||
## Overview
|
||||
|
||||
**Статус: утверждено пользователем 10 октября 2026 года.** Это целевая
|
||||
дизайн-система портала и внутренней лаборатории проверки поисковой выдачи.
|
||||
Она ещё не внедрена в работающий интерфейс. Утверждение дизайна не означает
|
||||
готовность публичного сервиса или разрешение на deployment.
|
||||
|
||||
**Creative North Star: "Түндүк — спокойное пространство для работы с правом".**
|
||||
|
||||
Профессиональный правовой сервис с кыргызской идентичностью: светлый фон
|
||||
цвета войлока, белое поле документа, глубокий красный для действий,
|
||||
тёмный сливовый текст и небольшие золотистые геометрические детали.
|
||||
Главное — читаемость длинных документов и ясная иерархия.
|
||||
|
||||
Это единственная нормативная спецификация дизайна проекта. Требования
|
||||
исходного brief перенесены сюда; отдельный brief и ранние варианты удалены.
|
||||
Визуальный ориентир — утверждённый пользователем
|
||||
[концептуальный экран](docs/design/tunduk-desktop-concept.png).
|
||||
Точные токены и правила этой спецификации важнее случайных деталей
|
||||
генеративного изображения. Название продукта — **Акылдаш**; «УКУК» из
|
||||
ранних концепций не используется.
|
||||
|
||||
**Key Characteristics:**
|
||||
|
||||
- Спокойное чтение и компактные, понятные элементы управления.
|
||||
- Национальная идентичность через палитру, знак и несколько точных деталей.
|
||||
- Поиск и документ сохраняют общий контекст на широком экране.
|
||||
- Единая визуальная система публичного сервиса и внутренней лаборатории.
|
||||
- Содержимое и доступные функции определяются реальными данными и API.
|
||||
|
||||
## Colors
|
||||
|
||||
Нормативные значения находятся в YAML выше. При реализации ключи `colors`
|
||||
переносятся в CSS custom properties `--color-<ключ>` либо эквивалентные
|
||||
семантические токены существующей системы тем. HEX не размножаются
|
||||
по компонентам.
|
||||
|
||||
| Роль | Токен | Применение |
|
||||
|---|---|---|
|
||||
| Светлый войлок | `bg` | Общий фон и окружение рабочей области |
|
||||
| Белый лист | `surface` | Документ, поля и основные поверхности |
|
||||
| Мягкая поверхность | `surface-muted` | Вторичные панели |
|
||||
| Сливовые чернила | `text` | Основной текст и заголовки |
|
||||
| Вторичный текст | `text-secondary` | Метаданные и пояснения |
|
||||
| Разделители | `border` | Контуры и границы панелей |
|
||||
| Глубокий красный | `primary` | Главное действие, ссылки и активная навигация |
|
||||
| Состояния красного | `primary-hover`, `primary-active` | Hover и нажатие |
|
||||
| Текст на красном | `on-primary` | Главная кнопка и активный пункт меню |
|
||||
| Мягкое красное выделение | `primary-subtle` | Выбранный результат, вторичная навигация |
|
||||
| Золотистая геометрия | `accent` | Небольшие декоративные линии |
|
||||
| Найденный фрагмент | `highlight` | Подсветка совпадения с исходным тёмным текстом |
|
||||
| Фокус | `focus` | Контрастная обводка клавиатурного фокуса |
|
||||
| Успех | `success`, `success-bg` | Подтверждение действия, подтверждённый статус |
|
||||
| Предупреждение | `warning`, `warning-bg` | Важные ограничения с текстовым объяснением |
|
||||
| Ошибка | `error`, `error-bg` | Ошибка с доступным действием для восстановления |
|
||||
| Информация | `info`, `info-bg` | Информационные сообщения |
|
||||
|
||||
Статус и ошибка всегда имеют текстовую подпись: цвет не передаёт смысл
|
||||
самостоятельно. Не назначать юридический статус по визуальной эвристике;
|
||||
использовать данные backend. Золотистый не используется для мелкого текста.
|
||||
Контраст проверяется на фактических сочетаниях: обычный текст не менее
|
||||
4.5:1, крупный текст и необходимые графические элементы не менее 3:1.
|
||||
|
||||
## Typography
|
||||
|
||||
**Заголовки документа и разделов:** Noto Serif, веса 600 и 700.
|
||||
**Интерфейс и основной текст:** Noto Sans, веса 400, 500 и 600.
|
||||
Шрифты подключаются штатным способом, предпочтительно локальными WOFF2
|
||||
с `font-display: swap`. До поставки проверить лицензии и символы
|
||||
`Ө ө Ү ү Ң ң`. Файлы шрифтов пока не добавлены.
|
||||
|
||||
| Элемент | Размер | Межстрочный интервал |
|
||||
|---|---|---|
|
||||
| Заголовок документа / страницы документа | 30–32 px; на телефоне 24–26 px | 1.2 |
|
||||
| Заголовок раздела | 22–24 px | 1.3 |
|
||||
| Текст документа | 17–18 px | 1.6–1.7 |
|
||||
| Элементы управления | 14–16 px | По токену `control` |
|
||||
| Метаданные | 13–14 px | По токену `metadata` |
|
||||
|
||||
Мелкий текст не ниже 12 px. Ориентир ширины документа — 65–75 символов
|
||||
в строке. Длинные названия документов и кыргызские подписи переносятся,
|
||||
не обрезаются. Даты, суммы и числовые колонки используют `tabular-nums`,
|
||||
обычные данные не оформляются моноширинным шрифтом.
|
||||
|
||||
## Layout
|
||||
|
||||
Шкала отступов определяется `spacing`. Шапка на desktop — ориентировочно
|
||||
72–88 px. Навигация / фильтры — 220–240 px. Панель контекста, когда
|
||||
существуют соответствующие данные и функции, — 300–340 px. Центральная
|
||||
колонка занимает остаток; это адаптивные ориентиры, а не фиксированная
|
||||
ширина всей страницы.
|
||||
|
||||
Для поискового рабочего экрана: шапка с брендом, поиском и языком;
|
||||
фильтры слева, результаты в середине, открытый документ справа. Оглавление
|
||||
можно сворачивать в подписанную кнопку, чтобы не сужать область чтения.
|
||||
Разворачивание документа и поиск внутри него реализуются только вместе
|
||||
с работающим поведением, а не как декоративные кнопки.
|
||||
|
||||
Внутренняя лаборатория сохраняет свой двухпанельный сценарий: фактический
|
||||
порядок результатов и оценивание слева, текст выбранной редакции справа.
|
||||
Она не получает фильтры, аналитику или публичную навигацию исключительно
|
||||
для сходства с концепцией. Панель проверяющего и сохранения не перекрывает
|
||||
результаты и текст документа.
|
||||
|
||||
При недостатке места сначала сворачивается дополнительная панель контекста,
|
||||
затем навигация. На телефоне — одна колонка; связанные материалы и
|
||||
оглавление доступны через подписанные кнопки. В лаборатории сохраняется
|
||||
существующее мобильное открытие документа и возврат фокуса при закрытии.
|
||||
Поиск, действия и метаданные переносятся. Основной текст не требует
|
||||
горизонтальной прокрутки; широкие таблицы имеют отдельную обозначенную
|
||||
область прокрутки.
|
||||
|
||||
Проверочные размеры при реализации: **1440, 1024 и 390 px**. Это размеры
|
||||
проверки, а не заранее навязанные CSS breakpoints.
|
||||
|
||||
## Elevation & Depth
|
||||
|
||||
Основные поверхности плоские. Иерархия создаётся белым листом на тёплом
|
||||
фоне, отступами, типографикой и тонкими разделителями. Мягкая тень нужна
|
||||
главным образом выпадающим меню и временным панелям. Значения теней
|
||||
уточняются при реализации; декоративное размытие и обилие теней запрещены.
|
||||
|
||||
## Shapes
|
||||
|
||||
Основное скругление — `rounded.default` (6 px), контуры — 1 px.
|
||||
Небольшие статусные метки могут быть округлее. Кнопки не превращаются
|
||||
в капсулы. Границы интерактивных полей обозначаются явным контуром,
|
||||
а не только фоновым цветом или разделителем соседней панели.
|
||||
|
||||
Если утверждённый логотип уже существует, он сохраняется. При отсутствии
|
||||
логотипа допускается небольшой оригинальный геометрический знак,
|
||||
вдохновлённый круглым верхним элементом юрты — түндүком. Знак из
|
||||
генеративного макета является направлением, а не готовым SVG-ассетом.
|
||||
Государственная символика не копируется.
|
||||
|
||||
Тонкий симметричный орнамент по мотивам шырдака допустим на краях шапки
|
||||
и в пустой нижней части навигации. Он выполняется простым SVG,
|
||||
помечается `aria-hidden`, не находится под текстом и не мешает нажатию.
|
||||
На телефоне орнамент сокращается либо убирается.
|
||||
|
||||
## Components
|
||||
|
||||
Это требования к будущей реализации, а не перечень уже обновлённых компонентов.
|
||||
|
||||
### Кнопки и поля
|
||||
|
||||
- Главная кнопка: `primary` и `on-primary`, высота около 40 px.
|
||||
- Вторичная кнопка: светлая поверхность и явный контур.
|
||||
- На touch-экране область нажатия не менее 44×44 px.
|
||||
- Поля белые, с постоянной подписью; placeholder не заменяет label.
|
||||
- Видимый фокус: обводка `focus` около 2 px с отступом.
|
||||
- Hover, active, focus-visible, disabled, loading, empty и error
|
||||
реализуются последовательно, с понятными сообщениями.
|
||||
- Переходы 120–180 мс, без анимации текста документа;
|
||||
учитывается `prefers-reduced-motion`.
|
||||
|
||||
### Навигация, результаты и документ
|
||||
|
||||
- Активный пункт меню — красный фон и белая подпись; вторичная
|
||||
навигация — `primary-subtle` и тёмно-красный текст.
|
||||
- Активная вкладка обозначена подписью и нижней линией. Длинные подписи
|
||||
переносятся либо прокручиваются без обрезки.
|
||||
- Результаты — упорядоченные строки: название, тип, статус, дата при
|
||||
наличии и выдержка. Каждое поле не оборачивается в отдельную карточку.
|
||||
- Выбранный результат обозначается мягким красным выделением и текстом.
|
||||
- Документ — единый белый лист: заголовок, реальные метаданные,
|
||||
доступные действия, оглавление и текст. Абзацы не разбиваются на карточки.
|
||||
- Подсветка совпадения сохраняет исходный текст; число совпадений
|
||||
и переходы между ними показываются только при работающем поиске.
|
||||
- Источник и выбранная редакция доступны рядом с документом.
|
||||
Недоступный язык не подменяется выдуманным переводом.
|
||||
- Язык интерфейса и язык документа — отдельные состояния:
|
||||
смена интерфейса не должна самовольно менять текст документа.
|
||||
- Footer сохраняет версии frontend и backend. Отсутствующий frontend
|
||||
или макет явно обозначаются, версия не придумывается.
|
||||
|
||||
### Внутренняя оценка выдачи
|
||||
|
||||
Сохраняются шкала 0–3 с полными текстовыми подписями, комментарии,
|
||||
черновик, идентификация проверяющего, сохранение подписанного снимка,
|
||||
обучение и существующие ограничения доступа. Оценки не передаются
|
||||
только цветом. Служебные данные не становятся публичными из-за редизайна.
|
||||
|
||||
### Доступность и состояния
|
||||
|
||||
Семантические заголовки, labels, ссылки и кнопки; иконки без текста
|
||||
имеют доступное имя. Проверяются Tab, Enter, Escape, возврат фокуса
|
||||
и отсутствие клавиатурных ловушек. Статусы доступны скринридеру.
|
||||
Пустая выдача предлагает изменить запрос; ошибка предлагает повторное
|
||||
действие. Ошибка сохранения не уничтожает черновик.
|
||||
|
||||
## Do's and Don'ts
|
||||
|
||||
### Do:
|
||||
|
||||
- Сохранять название «Акылдаш», реальные данные, маршруты, API и права доступа.
|
||||
- Использовать общий слой семантических токенов, существующие компоненты
|
||||
и CSS Modules / BEM согласно правилам соответствующей части проекта.
|
||||
- Разделять официальный текст, сведения о происхождении и выводы системы.
|
||||
- Проверять чтение длинных RU/KY-документов и одноязычные редакции.
|
||||
- Использовать этот документ как нормативную спецификацию для новых экранов.
|
||||
|
||||
### Don't:
|
||||
|
||||
- Не переносить демонстрационные даты, судебные материалы, связи
|
||||
и юридические формулировки из изображения в продукт.
|
||||
- Не добавлять аналитику, сравнение редакций, регистрацию и другие функции
|
||||
только ради сходства с концепцией.
|
||||
- Не менять поисковое ранжирование или сохранение разметки ради оформления.
|
||||
- Не использовать ковровые текстуры, фотографии гор, иллюстрации юрт,
|
||||
крупные флаги, золотые градиенты и декоративное размытие.
|
||||
- Не делать золотистыми мелкие подписи и не передавать смысл одним цветом.
|
||||
- Не добавлять отдельную тёмную тему без задачи; существующую сохранять.
|
||||
- Не считать генеративный скриншот доказательством точных HEX,
|
||||
загрузки Noto или проверки доступности работающего интерфейса.
|
||||
- Не считать утверждение дизайна разрешением на публикацию сервиса.
|
||||
@@ -9,16 +9,18 @@ Telegram-бот — только часть рабочего окружения
|
||||
|
||||
## Текущее состояние
|
||||
|
||||
Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии
|
||||
`0.7.1`: исправления контракта Search API v1 и его ограничений OpenSearch.
|
||||
размеченном наборе запросов.
|
||||
Сейчас реализованы Telegram-бот-секретарь версии `0.2.2`, backend версии
|
||||
`0.11.3` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
|
||||
SQLite; первый пользователь `admin` создаётся интерактивной CLI-командой.
|
||||
Поиск расширяет аббревиатуры организационно-правовых форм и поднимает
|
||||
действующий Налоговый кодекс для запросов о налоге на прибыль.
|
||||
|
||||
| Компонент | Версия | Состояние |
|
||||
|---|---:|---|
|
||||
| Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
|
||||
| Backend | `0.7.1` | исправлен контракт Search API v1 |
|
||||
| Frontend | — | ещё не создан |
|
||||
| Сбор и обработка правовых данных | `0.7.1` | добавлены relevance set и baseline-метрики |
|
||||
| Backend | `0.11.3` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
|
||||
| Frontend | `0.1.2` | лаборатория поиска, личный кабинет и управление учетными записями |
|
||||
| Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи |
|
||||
| RAG и база знаний | — | ещё не созданы |
|
||||
|
||||
## Структура репозитория
|
||||
@@ -34,11 +36,12 @@ tools/
|
||||
backend/
|
||||
ingestion/ получение и обновление правовых источников
|
||||
normalization/ воспроизводимая нормализация исходного архива
|
||||
search/ Search API, авторизация и внутренний frontend
|
||||
```
|
||||
|
||||
Каталоги для загрузки и обработки источников, RAG, backend и frontend будут
|
||||
создаваться с первой реальной задачей в соответствующей области. Это позволит
|
||||
выбрать структуру по фактическим требованиям, а не поддерживать пустой каркас.
|
||||
Внутренний frontend отдаётся существующим Search API без отдельного
|
||||
Node-сервиса. Отдельный публичный frontend остаётся за readiness gate в
|
||||
`docs/product/frontend-design-readiness-plan.md`.
|
||||
|
||||
Начать знакомство с проектом: [документация](docs/README.md) и
|
||||
[обзор продукта](docs/product/project-overview.md).
|
||||
@@ -59,4 +62,4 @@ python3 -m unittest discover -s tools/telegram-bot -v
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
+30
-2
@@ -1,6 +1,6 @@
|
||||
# Backend Акылдаш
|
||||
|
||||
Версия: `0.7.1`
|
||||
Версия: `0.11.3`
|
||||
|
||||
Первая backend-область проекта — загрузка правовых документов из официального
|
||||
Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
|
||||
@@ -216,6 +216,34 @@ PYTHONPATH=backend python3 -m search.evaluate_relevance \
|
||||
Менять веса или анализаторы следует только после фиксации этого baseline и
|
||||
разбора ошибок выдачи.
|
||||
|
||||
## Внутренняя лаборатория релевантности
|
||||
|
||||
Создайте первую учетную запись администратора:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=backend python3 -m search.api --users-db data/search-users.sqlite3 --create-admin
|
||||
```
|
||||
|
||||
Команда запросит и подтвердит пароль `admin` в терминале. Затем запустите
|
||||
Search API на localhost и откройте `http://127.0.0.1:8080/review`:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=backend python3 -m search.api \\
|
||||
--reviews-db data/search-reviews.sqlite3
|
||||
```
|
||||
|
||||
Лаборатория показывает фактический порядок выдачи OpenSearch, позволяет открыть
|
||||
текст редакции, поставить результату оценку от 0 до 3 и сохранить снимок с
|
||||
комментариями. Поиск и API доступны только после входа. Пользователь `admin`
|
||||
управляет учетными записями; новые пользователи должны заменить временный
|
||||
пароль при первом входе. Хранилище пользователей `data/search-users.sqlite3`
|
||||
нужно резервировать отдельно от поискового индекса. Экспорт доступен
|
||||
авторизованным пользователям через `GET /search-reviews/export`.
|
||||
|
||||
При утере пароля `admin` используйте `PYTHONPATH=backend python3 -m search.api
|
||||
--users-db data/search-users.sqlite3 --reset-admin-password`; следующий вход
|
||||
потребует сменить временный пароль.
|
||||
|
||||
Для проверки текущей выдачи без будущего HTTP API используйте CLI:
|
||||
|
||||
```bash
|
||||
@@ -225,4 +253,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -21,7 +21,7 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterable
|
||||
|
||||
APP_VERSION = "0.7.1"
|
||||
APP_VERSION = "0.9.3"
|
||||
API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/"
|
||||
LANGUAGES = {"Rus": "ru", "Kyr": "ky"}
|
||||
IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"}
|
||||
@@ -364,7 +364,7 @@ def main() -> int:
|
||||
print(
|
||||
f"discovered={result.discovered} downloaded={result.downloaded} "
|
||||
f"skipped={result.skipped} failed={result.failed}\n"
|
||||
f"Akyldash Backend v{APP_VERSION} · Frontend — not created"
|
||||
f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2"
|
||||
)
|
||||
return int(result.failed > 0)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from pathlib import Path
|
||||
from typing import Callable
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
APP_VERSION = "0.7.1"
|
||||
APP_VERSION = "0.9.3"
|
||||
SCHEMA_VERSION = "1"
|
||||
NORMALIZER_VERSION = "1.0.0"
|
||||
LANGUAGES = ("ru", "ky")
|
||||
@@ -720,7 +720,7 @@ def main() -> int:
|
||||
print(
|
||||
f"discovered={result.discovered} normalized={result.normalized} "
|
||||
f"skipped={result.skipped} failed={result.failed}\n"
|
||||
f"Akyldash Backend v{APP_VERSION} · Frontend — not created"
|
||||
f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2"
|
||||
)
|
||||
return int(result.failed > 0)
|
||||
|
||||
|
||||
@@ -56,17 +56,19 @@ cp backend/search/relevance-set-v1.template.json data/search/relevance-set-v1.js
|
||||
|
||||
## Разметка одного запроса
|
||||
|
||||
1. До просмотра выдачи зафиксируйте информационную потребность и формулировку
|
||||
`query`.
|
||||
2. Найдите кандидатов в локальном OpenSearch и в официальной ЦБД Минюста.
|
||||
Проверьте исходный запрос, его короткий вариант и вариант с юридическим
|
||||
термином или известным номером акта.
|
||||
3. Просмотрите не только заголовки, но и текст, статус, дату и редакцию каждого
|
||||
кандидата.
|
||||
4. Запишите уникальные `document_code` всех документов, удовлетворяющих
|
||||
критерию релевантности.
|
||||
5. Повторите поиск по ключевым терминам найденных документов, чтобы обнаружить
|
||||
пропущенные альтернативные акты.
|
||||
1. Зафиксируйте исходную формулировку `query` и информационную потребность до
|
||||
оценки выдачи нашего поиска.
|
||||
2. Юрист устанавливает релевантные акты по содержанию и реквизитам документов.
|
||||
Используйте ЦБД Минюста как авторитетный источник для проверки текста,
|
||||
статуса и редакции акта. Порядок выдачи ЦБД не является объектом оценки и не
|
||||
переносится в эталон.
|
||||
3. Запишите уникальные `document_code` всех документов, которые прямо отвечают
|
||||
на запрос. Спорные документы передайте на независимую проверку.
|
||||
4. Зафиксируйте согласованный набор до просмотра результатов OpenSearch и
|
||||
сохраните его отдельно от рабочих данных.
|
||||
5. Проверьте запрос во внутренней лаборатории (`/review`): оцените фактические
|
||||
результаты OpenSearch в исходном порядке, сохраните снимок и комментарии.
|
||||
Лаборатория проверяет качество нашей поисковой системы, а не ЦБД Минюста.
|
||||
|
||||
Пример структуры (код условный):
|
||||
|
||||
|
||||
+227
-14
@@ -4,17 +4,24 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import getpass
|
||||
import http.cookies
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from search.auth import AuthError, AuthStore
|
||||
from search.minjust_opensearch import APP_VERSION, request_json
|
||||
from search.catalog import CATALOGS, labels
|
||||
from search.query import build_search_query
|
||||
from search.reviews import ReviewSnapshots, ReviewStore
|
||||
|
||||
|
||||
API_VERSION = "v1"
|
||||
SEARCH_ALGORITHM_VERSION = "search-6"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
CODE = re.compile(r"^[0-9]+$")
|
||||
MAX_PAGE_SIZE = 100
|
||||
@@ -57,11 +64,20 @@ def date(value: str | None, name: str) -> str | None:
|
||||
|
||||
|
||||
def openapi() -> dict:
|
||||
responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
|
||||
responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "401": {"description": "Authentication required"}, "403": {"description": "Additional access required"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
|
||||
return {
|
||||
"openapi": "3.0.3",
|
||||
"info": {"title": "Akyldash Search API", "version": API_VERSION},
|
||||
"components": {"securitySchemes": {"SessionCookie": {"type": "apiKey", "in": "cookie", "name": "akyldash_session"}}},
|
||||
"security": [{"SessionCookie": []}],
|
||||
"paths": {
|
||||
"/auth/login": {"post": {"security": [], "responses": {"200": {"description": "Signed in and session cookie issued"}, "401": {"description": "Invalid login or password"}}}},
|
||||
"/auth/logout": {"post": {"responses": {"200": {"description": "Signed out"}}}},
|
||||
"/auth/me": {"get": {"responses": {"200": {"description": "Current account"}, "401": {"description": "Authentication required"}}}},
|
||||
"/auth/change-password": {"post": {"description": "Requires the current password and a different new password", "responses": {"200": {"description": "Password changed"}, "400": {"description": "Current password is missing or invalid, or the new password is invalid or unchanged"}}}},
|
||||
"/admin/users": {"get": {"responses": {"200": {"description": "User list"}}}, "post": {"responses": {"201": {"description": "User created with a temporary password"}}}},
|
||||
"/admin/users/{username}": {"delete": {"responses": {"200": {"description": "User deleted"}}}},
|
||||
"/admin/users/{username}/reset-password": {"post": {"responses": {"200": {"description": "Temporary password set"}}}},
|
||||
"/search": {"get": {"responses": responses, "parameters": [
|
||||
{"name": "q", "in": "query", "required": True, "schema": {"type": "string"}},
|
||||
{"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}},
|
||||
@@ -70,11 +86,15 @@ def openapi() -> dict:
|
||||
{"name": "document_type", "in": "query", "schema": {"type": "string"}},
|
||||
{"name": "status", "in": "query", "schema": {"type": "string"}},
|
||||
{"name": "authority", "in": "query", "schema": {"type": "string"}},
|
||||
{"name": "legal_force", "in": "query", "schema": {"type": "string"}},
|
||||
{"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}},
|
||||
{"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}},
|
||||
{"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}},
|
||||
]}},
|
||||
"/search/filters": {"get": {"responses": responses}},
|
||||
"/search-reviews": {"post": {"responses": {"201": {"description": "Review saved"}, "400": {"description": "Invalid review"}}}},
|
||||
"/search-reviews/export": {"get": {"responses": responses}},
|
||||
"/review": {"get": {"responses": {"200": {"description": "Review interface"}}}},
|
||||
"/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
|
||||
"/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
|
||||
"/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
|
||||
@@ -83,10 +103,14 @@ def openapi() -> dict:
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, base_url: str, index: str, data_root: Path):
|
||||
def __init__(self, base_url: str, index: str, data_root: Path, reviews_db: Path | str = ":memory:", review_secret: bytes | None = None, users_db: Path | str = ":memory:", cookie_secure: bool = False):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.index = index
|
||||
self.data_root = data_root
|
||||
self.review_store = ReviewStore(reviews_db)
|
||||
self.review_snapshots = ReviewSnapshots(review_secret)
|
||||
self.auth_store = AuthStore(users_db)
|
||||
self.cookie_secure = cookie_secure
|
||||
|
||||
def search_url(self, suffix: str) -> str:
|
||||
return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}"
|
||||
@@ -114,7 +138,7 @@ class Api:
|
||||
if sort not in {"relevance", "date"}:
|
||||
raise ApiError(400, "sort must be relevance or date")
|
||||
filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}]
|
||||
fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes"}
|
||||
fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes", "legal_force": "legal_force_code"}
|
||||
for parameter, field in fields.items():
|
||||
value = one(query, parameter)
|
||||
if value:
|
||||
@@ -130,19 +154,29 @@ class Api:
|
||||
body = {
|
||||
"from": (page - 1) * page_size,
|
||||
"size": page_size + 1,
|
||||
"_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "date_adopted", "number"],
|
||||
"query": {"bool": {"filter": filters, "must": {"multi_match": {"query": text, "fields": [f"document_name_{language}", f"text_{language}"], "type": "cross_fields"}}}},
|
||||
"_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "legal_force_code", "date_adopted", "number"],
|
||||
"query": {"bool": {"filter": filters, "must": build_search_query(language, text)}},
|
||||
"collapse": {"field": "document_code"},
|
||||
"highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}},
|
||||
}
|
||||
body["sort"] = [{"_script": {
|
||||
"type": "number",
|
||||
"order": "asc",
|
||||
"script": {"lang": "painless", "source": "doc['status_code'].size() == 0 ? 1 : (doc['status_code'].value == 'active' ? 0 : (doc['status_code'].value == 'repealed' ? 2 : 1))"},
|
||||
}}]
|
||||
if sort == "date":
|
||||
body["sort"] = [{"date_adopted": "desc"}, {"_score": "desc"}]
|
||||
body["sort"].append({"date_adopted": "desc"})
|
||||
body["sort"].append({"_score": "desc"})
|
||||
response = self.query_opensearch(body)
|
||||
try:
|
||||
hits = response["hits"]["hits"]
|
||||
except (KeyError, TypeError) as error:
|
||||
raise ApiError(502, "search backend returned an incomplete response") from error
|
||||
return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": [self.search_hit(hit, language) for hit in hits[:page_size]]}
|
||||
results = [self.search_hit(hit, language) for hit in hits[:page_size]]
|
||||
snapshot_results = [{"rank": rank, **result} for rank, result in enumerate(results, 1)]
|
||||
concrete_indexes = {hit.get("_index") for hit in hits[:page_size] if hit.get("_index")}
|
||||
index_name = next(iter(concrete_indexes)) if len(concrete_indexes) == 1 else self.index
|
||||
return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": results, "review_token": self.review_snapshots.create(text, language, index_name, snapshot_results, SEARCH_ALGORITHM_VERSION)}
|
||||
|
||||
@staticmethod
|
||||
def search_hit(hit: dict, language: str) -> dict:
|
||||
@@ -150,13 +184,15 @@ class Api:
|
||||
if not isinstance(source, dict) or not source.get("document_code"):
|
||||
raise ApiError(502, "search backend returned an incomplete result")
|
||||
highlight = hit.get("highlight", {}).get(f"text_{language}", [])
|
||||
return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": highlight[0] if highlight else None}
|
||||
legal_force = source.get("legal_force_code")
|
||||
snippet = highlight[0].replace("<em>", "").replace("</em>", "") if highlight else None
|
||||
return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "legal_force": labels("legal_force", legal_force)[language] if legal_force else None, "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": snippet}
|
||||
|
||||
def filters(self, query: dict[str, list[str]]) -> dict:
|
||||
language = one(query, "language") or "ru"
|
||||
if language not in LANGUAGES:
|
||||
raise ApiError(400, "language must be ru or ky")
|
||||
fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes")}
|
||||
fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes"), "legal_forces": ("legal_force", "legal_force_code")}
|
||||
body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
|
||||
response = self.query_opensearch(body)
|
||||
try:
|
||||
@@ -221,12 +257,99 @@ class Api:
|
||||
raise ApiError(404, "edition language not found")
|
||||
return {"api_version": API_VERSION, "edition": metadata, "content": content}
|
||||
|
||||
def handle(self, method: str, path: str) -> tuple[int, dict]:
|
||||
if method != "GET":
|
||||
raise ApiError(405, "method not allowed")
|
||||
def save_review(self, body: dict, username: str) -> dict:
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "request body must be an object")
|
||||
try:
|
||||
snapshot = self.review_snapshots.verify(body["review_token"])
|
||||
reviewer = username
|
||||
overall_comment = body.get("overall_comment", "").strip()
|
||||
submitted = body["results"]
|
||||
except (KeyError, AttributeError, TypeError, ValueError) as error:
|
||||
raise ApiError(400, "review_token, reviewer and results are required") from error
|
||||
if not reviewer or len(reviewer) > 120:
|
||||
raise ApiError(400, "reviewer must be between 1 and 120 characters")
|
||||
if len(overall_comment) > 4000:
|
||||
raise ApiError(400, "overall_comment is too long")
|
||||
if not isinstance(submitted, list):
|
||||
raise ApiError(400, "results must be an array")
|
||||
by_rank = {item["rank"]: item for item in snapshot["results"]}
|
||||
if len(submitted) != len(by_rank) or {item.get("rank") for item in submitted if isinstance(item, dict)} != set(by_rank):
|
||||
raise ApiError(400, "all search results must be reviewed exactly once")
|
||||
results = []
|
||||
for item in submitted:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("rank"), int) or item["rank"] not in by_rank:
|
||||
raise ApiError(400, "review result rank is invalid")
|
||||
source = by_rank[item["rank"]]
|
||||
if item.get("code") != source["code"]:
|
||||
raise ApiError(400, "review result document does not match the search snapshot")
|
||||
rating = item.get("rating")
|
||||
if rating is not None and (isinstance(rating, bool) or not isinstance(rating, int) or not 0 <= rating <= 3):
|
||||
raise ApiError(400, "rating must be an integer from 0 to 3")
|
||||
comment = item.get("comment", "")
|
||||
if not isinstance(comment, str) or len(comment) > 4000:
|
||||
raise ApiError(400, "result comment is too long")
|
||||
results.append({**source, "rating": rating, "comment": comment.strip()})
|
||||
if not results:
|
||||
raise ApiError(400, "at least one result must be reviewed")
|
||||
review = {"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "reviewer": reviewer, "query": snapshot["query"], "language": snapshot["language"], "index_name": snapshot["index_name"], "algorithm_version": snapshot["algorithm_version"], "top_result_code": snapshot["results"][0]["code"] if snapshot["results"] else None, "results": results, "overall_comment": overall_comment}
|
||||
review_id = self.review_store.save(review)
|
||||
return {"api_version": API_VERSION, "id": review_id, "created_at": review["created_at"]}
|
||||
|
||||
@staticmethod
|
||||
def review_page() -> str:
|
||||
try:
|
||||
return Path(__file__).with_name("review.html").read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeError) as error:
|
||||
raise ApiError(500, "review interface is unavailable") from error
|
||||
|
||||
def handle(self, method: str, path: str, body: dict | None = None, user: dict | None = None) -> tuple[int, dict]:
|
||||
parsed = urllib.parse.urlsplit(path)
|
||||
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
||||
parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
|
||||
if method == "POST" and parts == ["auth", "login"]:
|
||||
try:
|
||||
token, account = self.auth_store.login(body.get("username"), body.get("password"))
|
||||
return 200, {"account": account, "session_token": token}
|
||||
except AuthError as error:
|
||||
raise ApiError(401, str(error)) from error
|
||||
if method == "POST" and parts == ["auth", "logout"]:
|
||||
return 200, {"ok": True}
|
||||
if method == "GET" and parts == ["auth", "me"]:
|
||||
return 200, {"account": user}
|
||||
if method == "POST" and parts == ["auth", "change-password"]:
|
||||
try:
|
||||
self.auth_store.change_password(user["username"], body.get("current_password"), body.get("new_password"))
|
||||
token, account = self.auth_store.login(user["username"], body.get("new_password"))
|
||||
return 200, {"account": account, "session_token": token}
|
||||
except AuthError as error:
|
||||
raise ApiError(400, str(error)) from error
|
||||
if user and user["username"].casefold() == "admin" and method == "GET" and parts == ["admin", "users"]:
|
||||
return 200, {"users": self.auth_store.list_users()}
|
||||
if user and user["username"].casefold() == "admin" and method == "POST" and parts == ["admin", "users"]:
|
||||
try:
|
||||
self.auth_store.create_user(body.get("username"), body.get("password"))
|
||||
return 201, {"ok": True}
|
||||
except AuthError as error:
|
||||
raise ApiError(400, str(error)) from error
|
||||
if user and user["username"].casefold() == "admin" and method == "DELETE" and len(parts) == 3 and parts[:2] == ["admin", "users"]:
|
||||
try:
|
||||
self.auth_store.delete_user(parts[2])
|
||||
return 200, {"ok": True}
|
||||
except AuthError as error:
|
||||
raise ApiError(400, str(error)) from error
|
||||
if user and user["username"].casefold() == "admin" and method == "POST" and len(parts) == 4 and parts[:2] == ["admin", "users"] and parts[3] == "reset-password":
|
||||
try:
|
||||
self.auth_store.reset_password(parts[2], body.get("password"))
|
||||
return 200, {"ok": True}
|
||||
except AuthError as error:
|
||||
raise ApiError(400, str(error)) from error
|
||||
if method == "POST" and parts == ["search-reviews"]:
|
||||
return 201, self.save_review(body, user["username"] if user else body.get("reviewer", ""))
|
||||
if method == "GET" and parts == ["search-reviews", "export"]:
|
||||
return 200, {"api_version": API_VERSION, "reviews": self.review_store.export()}
|
||||
if method != "GET":
|
||||
raise ApiError(405, "method not allowed")
|
||||
if parts == ["openapi.json"]:
|
||||
return 200, openapi()
|
||||
if parts == ["search"]:
|
||||
@@ -246,12 +369,61 @@ def handler(api: Api):
|
||||
class RequestHandler(BaseHTTPRequestHandler):
|
||||
def respond(self, method: str):
|
||||
try:
|
||||
status, payload = api.handle(method, self.path)
|
||||
route = urllib.parse.urlsplit(self.path).path
|
||||
cookies = http.cookies.SimpleCookie(self.headers.get("Cookie", ""))
|
||||
session = cookies.get("akyldash_session")
|
||||
token = session.value if session else None
|
||||
user = api.auth_store.user_for_token(token)
|
||||
if method == "GET" and route in {"/", "/review"}:
|
||||
body = api.review_page().encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
if not (method == "POST" and route in {"/auth/login", "/auth/logout"}):
|
||||
if not user:
|
||||
raise ApiError(401, "authentication required")
|
||||
if user["must_change_password"] and not (method == "POST" and route == "/auth/change-password") and not (method == "GET" and route == "/auth/me"):
|
||||
raise ApiError(403, "change password before continuing")
|
||||
if route.startswith("/admin/users") and user["username"].casefold() != "admin":
|
||||
raise ApiError(403, "admin account required")
|
||||
body = None
|
||||
if method in {"POST", "DELETE"}:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
if length > 1_000_000:
|
||||
raise ApiError(413, "request body is too large")
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "request body must be a JSON object")
|
||||
if method == "POST" and route == "/auth/logout":
|
||||
api.auth_store.logout(token)
|
||||
status, payload = api.handle(method, self.path, body, user)
|
||||
except json.JSONDecodeError:
|
||||
status, payload = 400, {"api_version": API_VERSION, "error": "request body must be valid JSON"}
|
||||
except ApiError as error:
|
||||
status, payload = error.status, {"api_version": API_VERSION, "error": error.message}
|
||||
except (ValueError, http.cookies.CookieError):
|
||||
status, payload = 400, {"api_version": API_VERSION, "error": "invalid request"}
|
||||
session_token = payload.pop("session_token", None) if isinstance(payload, dict) else None
|
||||
body = json.dumps(payload, ensure_ascii=False).encode()
|
||||
self.send_response(status)
|
||||
if session_token:
|
||||
secure = "; Secure" if api.cookie_secure else ""
|
||||
self.send_header("Set-Cookie", f"akyldash_session={session_token}; Path=/; HttpOnly; SameSite=Strict; Max-Age=43200{secure}")
|
||||
elif method == "POST" and urllib.parse.urlsplit(self.path).path == "/auth/logout":
|
||||
secure = "; Secure" if api.cookie_secure else ""
|
||||
self.send_header("Set-Cookie", f"akyldash_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0{secure}")
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
@@ -262,6 +434,9 @@ def handler(api: Api):
|
||||
def do_POST(self):
|
||||
self.respond("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self.respond("DELETE")
|
||||
|
||||
def log_message(self, format: str, *args):
|
||||
return
|
||||
|
||||
@@ -273,11 +448,49 @@ def main() -> int:
|
||||
parser.add_argument("--url", default="http://127.0.0.1:9200")
|
||||
parser.add_argument("--index", default="akyldash-fragments-current")
|
||||
parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized"))
|
||||
parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
|
||||
parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
|
||||
parser.add_argument("--review-secret", default=None)
|
||||
parser.add_argument("--review-secret-file", type=Path)
|
||||
parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
|
||||
parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
|
||||
parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
parser.add_argument("--version", action="version", version=APP_VERSION)
|
||||
arguments = parser.parse_args()
|
||||
ThreadingHTTPServer((arguments.host, arguments.port), handler(Api(arguments.url, arguments.index, arguments.data))).serve_forever()
|
||||
if arguments.review_secret and arguments.review_secret_file:
|
||||
parser.error("choose either --review-secret or --review-secret-file")
|
||||
try:
|
||||
secret = arguments.review_secret.encode() if arguments.review_secret else arguments.review_secret_file.read_bytes().strip() if arguments.review_secret_file else None
|
||||
except OSError as error:
|
||||
parser.error(f"cannot read review secret file: {error}")
|
||||
if arguments.review_secret_file and not secret:
|
||||
parser.error("review secret file is empty")
|
||||
api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
|
||||
if arguments.create_admin or arguments.reset_admin_password:
|
||||
users = api.auth_store.list_users()
|
||||
admin_exists = any(user["username"].casefold() == "admin" for user in users)
|
||||
if arguments.create_admin and users:
|
||||
parser.error("accounts already exist")
|
||||
if arguments.reset_admin_password and not admin_exists:
|
||||
parser.error("admin account does not exist")
|
||||
if arguments.create_admin and arguments.reset_admin_password:
|
||||
parser.error("choose either --create-admin or --reset-admin-password")
|
||||
password = getpass.getpass("Password for admin (12+ characters): ")
|
||||
confirm = getpass.getpass("Confirm password: ")
|
||||
if password != confirm:
|
||||
parser.error("passwords do not match")
|
||||
try:
|
||||
if arguments.create_admin:
|
||||
api.auth_store.create_admin(password)
|
||||
else:
|
||||
api.auth_store.reset_password("admin", password)
|
||||
except AuthError as error:
|
||||
parser.error(str(error))
|
||||
print("Initial admin account created." if arguments.create_admin else "Admin password reset. Change it at next sign-in.")
|
||||
return 0
|
||||
ThreadingHTTPServer((arguments.host, arguments.port), handler(api)).serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Local user accounts and cookie sessions for the internal search workspace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
USERNAME = re.compile(r"^[A-Za-z0-9._-]{3,64}$")
|
||||
PASSWORD_ITERATIONS = 310_000
|
||||
SESSION_TTL = 60 * 60 * 12
|
||||
|
||||
|
||||
class AuthError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class AuthStore:
|
||||
def __init__(self, path: Path | str):
|
||||
if path != ":memory:":
|
||||
database = Path(path)
|
||||
database.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
descriptor = os.open(database, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
|
||||
os.close(descriptor)
|
||||
except FileExistsError:
|
||||
pass
|
||||
self.connection = sqlite3.connect(path, check_same_thread=False)
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
self._lock = threading.RLock()
|
||||
self.connection.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
username TEXT PRIMARY KEY COLLATE NOCASE,
|
||||
password_hash BLOB NOT NULL,
|
||||
password_salt BLOB NOT NULL,
|
||||
must_change_password INTEGER NOT NULL DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
first_login_at INTEGER,
|
||||
last_login_at INTEGER,
|
||||
password_reset_at INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash BLOB PRIMARY KEY,
|
||||
username TEXT NOT NULL REFERENCES users(username) ON DELETE CASCADE,
|
||||
expires_at INTEGER NOT NULL
|
||||
);
|
||||
""")
|
||||
user_columns = {row["name"] for row in self.connection.execute("PRAGMA table_info(users)")}
|
||||
for column in ("first_login_at", "last_login_at", "password_reset_at"):
|
||||
if column not in user_columns:
|
||||
self.connection.execute(f"ALTER TABLE users ADD COLUMN {column} INTEGER")
|
||||
self.connection.execute("PRAGMA foreign_keys = ON")
|
||||
self.connection.commit()
|
||||
|
||||
@staticmethod
|
||||
def _validate(username: str, password: str) -> tuple[str, str]:
|
||||
if not isinstance(username, str) or not USERNAME.fullmatch(username):
|
||||
raise AuthError("login must be 3–64 letters, numbers, dots, underscores or hyphens")
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 256:
|
||||
raise AuthError("password must be between 12 and 256 characters")
|
||||
return username, password
|
||||
|
||||
@staticmethod
|
||||
def _hash(password: str, salt: bytes) -> bytes:
|
||||
return hashlib.pbkdf2_hmac("sha256", password.encode(), salt, PASSWORD_ITERATIONS)
|
||||
|
||||
def create_admin(self, password: str) -> None:
|
||||
self._validate("admin", password)
|
||||
salt = secrets.token_bytes(16)
|
||||
with self._lock:
|
||||
if self.connection.execute("SELECT 1 FROM users LIMIT 1").fetchone():
|
||||
raise AuthError("accounts already exist")
|
||||
self.connection.execute("INSERT INTO users(username,password_hash,password_salt,must_change_password,created_at) VALUES('admin',?,?,0,?)", (self._hash(password, salt), salt, int(time.time())))
|
||||
self.connection.commit()
|
||||
|
||||
# shortcut: login throttling belongs to the HTTPS reverse proxy; add local limits before direct public exposure.
|
||||
def login(self, username: str, password: str) -> tuple[str, dict]:
|
||||
if not isinstance(username, str) or not isinstance(password, str):
|
||||
raise AuthError("invalid login or password")
|
||||
with self._lock:
|
||||
user = self.connection.execute("SELECT * FROM users WHERE username = ? COLLATE NOCASE", (username,)).fetchone()
|
||||
salt = user["password_salt"] if user else b"\0" * 16
|
||||
expected_hash = user["password_hash"] if user else b"\0" * 32
|
||||
valid = hmac.compare_digest(self._hash(password, salt), expected_hash)
|
||||
if not user or not valid:
|
||||
raise AuthError("invalid login or password")
|
||||
with self._lock:
|
||||
current = self.connection.execute("SELECT * FROM users WHERE username=?", (user["username"],)).fetchone()
|
||||
if not current or not hmac.compare_digest(current["password_hash"], expected_hash):
|
||||
raise AuthError("invalid login or password")
|
||||
token = secrets.token_urlsafe(32)
|
||||
now = int(time.time())
|
||||
self.connection.execute("DELETE FROM sessions WHERE expires_at <= ?", (now,))
|
||||
self.connection.execute("INSERT INTO sessions(token_hash,username,expires_at) VALUES(?,?,?)", (hashlib.sha256(token.encode()).digest(), user["username"], now + SESSION_TTL))
|
||||
self.connection.execute("UPDATE users SET first_login_at=COALESCE(first_login_at,?),last_login_at=? WHERE username=?", (now, now, user["username"]))
|
||||
self.connection.commit()
|
||||
current = self.connection.execute("SELECT * FROM users WHERE username=?", (user["username"],)).fetchone()
|
||||
return token, self._public(current)
|
||||
|
||||
@staticmethod
|
||||
def _public(user: sqlite3.Row) -> dict:
|
||||
return {"username": user["username"], "role": "Администратор", "must_change_password": bool(user["must_change_password"]), "created_at": user["created_at"]}
|
||||
|
||||
def user_for_token(self, token: str | None) -> dict | None:
|
||||
if not token:
|
||||
return None
|
||||
with self._lock:
|
||||
user = self.connection.execute("SELECT users.* FROM sessions JOIN users USING(username) WHERE token_hash = ? AND expires_at > ?", (hashlib.sha256(token.encode()).digest(), int(time.time()))).fetchone()
|
||||
return self._public(user) if user else None
|
||||
|
||||
def logout(self, token: str | None) -> None:
|
||||
if token:
|
||||
with self._lock:
|
||||
self.connection.execute("DELETE FROM sessions WHERE token_hash = ?", (hashlib.sha256(token.encode()).digest(),))
|
||||
self.connection.commit()
|
||||
|
||||
def change_password(self, username: str, old_password: str, new_password: str) -> None:
|
||||
if not isinstance(old_password, str):
|
||||
raise AuthError("current password is required")
|
||||
self._validate(username, new_password)
|
||||
with self._lock:
|
||||
user = self.connection.execute("SELECT * FROM users WHERE username = ?", (username,)).fetchone()
|
||||
if not user:
|
||||
raise AuthError("account not found")
|
||||
if not hmac.compare_digest(self._hash(old_password, user["password_salt"]), user["password_hash"]):
|
||||
raise AuthError("current password is incorrect")
|
||||
if hmac.compare_digest(self._hash(new_password, user["password_salt"]), user["password_hash"]):
|
||||
raise AuthError("new password must be different")
|
||||
salt = secrets.token_bytes(16)
|
||||
password_hash = self._hash(new_password, salt)
|
||||
self.connection.execute("UPDATE users SET password_hash=?,password_salt=?,must_change_password=0 WHERE username=?", (password_hash, salt, username))
|
||||
self.connection.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||||
self.connection.commit()
|
||||
|
||||
def create_user(self, username: str, password: str) -> None:
|
||||
self._validate(username, password)
|
||||
salt = secrets.token_bytes(16)
|
||||
password_hash = self._hash(password, salt)
|
||||
try:
|
||||
with self._lock:
|
||||
self.connection.execute("INSERT INTO users(username,password_hash,password_salt,must_change_password,created_at) VALUES(?,?,?,1,?)", (username, password_hash, salt, int(time.time())))
|
||||
self.connection.commit()
|
||||
except sqlite3.IntegrityError as error:
|
||||
raise AuthError("login already exists") from error
|
||||
|
||||
def delete_user(self, username: str) -> None:
|
||||
if username.casefold() == "admin":
|
||||
raise AuthError("admin account cannot be deleted")
|
||||
with self._lock:
|
||||
self.connection.execute("DELETE FROM users WHERE username=?", (username,))
|
||||
self.connection.commit()
|
||||
|
||||
def reset_password(self, username: str, password: str) -> None:
|
||||
self._validate(username, password)
|
||||
salt = secrets.token_bytes(16)
|
||||
password_hash = self._hash(password, salt)
|
||||
with self._lock:
|
||||
cursor = self.connection.execute("UPDATE users SET password_hash=?,password_salt=?,must_change_password=1,password_reset_at=? WHERE username=?", (password_hash, salt, int(time.time()), username))
|
||||
if not cursor.rowcount:
|
||||
raise AuthError("account not found")
|
||||
self.connection.execute("DELETE FROM sessions WHERE username=?", (username,))
|
||||
self.connection.commit()
|
||||
|
||||
def list_users(self) -> list[dict]:
|
||||
with self._lock:
|
||||
users = []
|
||||
for row in self.connection.execute("SELECT * FROM users ORDER BY username COLLATE NOCASE"):
|
||||
account = self._public(row)
|
||||
account.update(first_login_at=row["first_login_at"], last_login_at=row["last_login_at"], password_reset_at=row["password_reset_at"])
|
||||
users.append(account)
|
||||
return users
|
||||
@@ -1,11 +1,30 @@
|
||||
"""Immutable v1 search catalogs."""
|
||||
|
||||
DOCUMENT_TYPES = {
|
||||
"constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"),
|
||||
"constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "other": ("Прочие документы", "Башка документтер"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"),
|
||||
}
|
||||
STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")}
|
||||
STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "not_yet_effective": ("Не вступил в силу", "Күчүнө кире элек"), "canceled": ("Отменено", "Жокко чыгарылды"), "inactive": ("Не действует", "Күчүндө эмес же Колдонулбайт"), "terminated": ("Прекратило действие", "Күчүн токтотту же Колдонуу токтотулган"), "suspended": ("Действие приостановлено", "Күчүнө кирүүсү токтотулган"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")}
|
||||
AUTHORITIES = {"president": ("Президент", "Президент"), "parliament": ("Органы законодательной власти", "Мыйзам чыгаруу бийлик органдары"), "cabinet": ("Правительство и Кабинет Министров", "Өкмөт жана Министрлер Кабинети"), "ministries_and_committees": ("Министерства и государственные комитеты", "Министрликтер жана мамлекеттик комитеттер"), "administrative_agencies": ("Административные ведомства", "Административдик ведомстволор"), "national_bank": ("Национальный банк", "Улуттук банк"), "other_state_bodies": ("Иные государственные органы", "Башка мамлекеттик органдар"), "local_representative_bodies": ("Представительные органы местного самоуправления", "Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары"), "other": ("Прочие органы", "Башка органдар")}
|
||||
CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES}
|
||||
LEGAL_FORCE_LEVELS = {
|
||||
"constitutional": ("Конституционный уровень", "Конституциялык деңгээл"),
|
||||
"legislative": ("Законодательный уровень", "Мыйзам деңгээли"),
|
||||
"subordinate": ("Подзаконный уровень", "Мыйзам алдындагы деңгээл"),
|
||||
}
|
||||
LEGAL_FORCE_BY_DOCUMENT_TYPE = {
|
||||
"constitution": "constitutional",
|
||||
"constitutional_law": "constitutional",
|
||||
"code": "legislative",
|
||||
"law": "legislative",
|
||||
"decree": "subordinate",
|
||||
"resolution": "subordinate",
|
||||
}
|
||||
CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES, "legal_force": LEGAL_FORCE_LEVELS}
|
||||
DOCUMENT_TYPE_ALIASES = {
|
||||
("Положение", "Положение"): "provision",
|
||||
("Конституционный закон", "Конституционный закон"): "constitutional_law",
|
||||
("Указ", "Жарлыгы"): "decree",
|
||||
("устав", None): "charter",
|
||||
}
|
||||
|
||||
|
||||
def labels(category: str, code: str) -> dict[str, str]:
|
||||
@@ -20,9 +39,14 @@ def source_code(category: str, value: dict | None) -> str:
|
||||
pair = ((value or {}).get("ru"), (value or {}).get("ky"))
|
||||
if pair == (None, None):
|
||||
return "unspecified"
|
||||
if category == "document_type" and pair in DOCUMENT_TYPE_ALIASES:
|
||||
return DOCUMENT_TYPE_ALIASES[pair]
|
||||
for code, expected in CATALOGS[category].items():
|
||||
if pair == expected or category == "document_type" and code == "provision" and pair == ("Положение", "Положение"):
|
||||
if pair == expected:
|
||||
return code
|
||||
# ponytail: uncurated source types share one filter; add reviewed codes when separate filtering is needed.
|
||||
if category == "document_type":
|
||||
return "other"
|
||||
raise ValueError(f"Unmapped {category} catalog value: {pair!r}")
|
||||
|
||||
|
||||
@@ -49,3 +73,7 @@ def authority_codes(paths: list[dict]) -> list[str]:
|
||||
else:
|
||||
codes.add("other")
|
||||
return sorted(codes) or ["other"]
|
||||
|
||||
|
||||
def legal_force_code(document_type: dict | None) -> str | None:
|
||||
return LEGAL_FORCE_BY_DOCUMENT_TYPE.get(source_code("document_type", document_type))
|
||||
@@ -82,7 +82,7 @@ def main() -> int:
|
||||
raise SystemExit("--top-k must be greater than zero")
|
||||
result = evaluate(load_queries(arguments.relevance_set), arguments.url, arguments.index, arguments.top_k)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
print(f"Akyldash Backend v{APP_VERSION} · Frontend — not created", file=sys.stderr)
|
||||
print(f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"document_type_ru": { "type": "keyword" },
|
||||
"document_type_ky": { "type": "keyword" },
|
||||
"document_type_code": { "type": "keyword" },
|
||||
"legal_force_code": { "type": "keyword" },
|
||||
"status_ru": { "type": "keyword" },
|
||||
"status_ky": { "type": "keyword" },
|
||||
"status_code": { "type": "keyword" },
|
||||
|
||||
@@ -15,9 +15,9 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
from search.catalog import authority_codes, source_code
|
||||
from search.catalog import authority_codes, legal_force_code, source_code
|
||||
|
||||
APP_VERSION = "0.7.1"
|
||||
APP_VERSION = "0.11.3"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
|
||||
|
||||
@@ -76,6 +76,7 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st
|
||||
"document_type_ru": localized(document.get("type"), "ru"),
|
||||
"document_type_ky": localized(document.get("type"), "ky"),
|
||||
"document_type_code": source_code("document_type", document.get("type")),
|
||||
"legal_force_code": legal_force_code(document.get("type")),
|
||||
"status_ru": localized(document.get("status"), "ru"),
|
||||
"status_ky": localized(document.get("status"), "ky"),
|
||||
"status_code": source_code("status", document.get("status")),
|
||||
@@ -459,7 +460,7 @@ def main() -> int:
|
||||
else:
|
||||
documents, fragments = export_bulk(arguments.input, arguments.output, arguments.index, arguments.limit)
|
||||
destination = arguments.output
|
||||
print(f"documents={documents} fragments={fragments} destination={destination}\nAkyldash Backend v{APP_VERSION} · Frontend — not created")
|
||||
print(f"documents={documents} fragments={fragments} destination={destination}\nAkyldash · Backend v{APP_VERSION} · Frontend v0.1.2")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
+153
-27
@@ -10,23 +10,76 @@ import urllib.parse
|
||||
from search.minjust_opensearch import APP_VERSION, request_json
|
||||
|
||||
|
||||
def company_registration_clauses(language: str, query: str) -> list[dict]:
|
||||
patterns = {
|
||||
"ru": (r"\bкак\s+откры\w*\s+осоо\b", r"\b(порядок|процедура)\s+откры\w*\s+осоо\b", r"\bкак\s+зарегистр\w*\s+осоо\b"),
|
||||
"ky": (r"\bжчк\s+ач\w*\s+тартиби\b", r"\bжчк\s+кантип\s+ач\w*\b"),
|
||||
}[language]
|
||||
if not any(re.search(pattern, query.casefold()) for pattern in patterns):
|
||||
return []
|
||||
LEGAL_FORM_ALIASES = {
|
||||
"ru": {
|
||||
"осоо": ("ооо", "общество с ограниченной ответственностью"),
|
||||
"ооо": ("осоо", "общество с ограниченной ответственностью"),
|
||||
"общество с ограниченной ответственностью": ("осоо", "ооо"),
|
||||
"общества с ограниченной ответственностью": ("осоо", "ооо"),
|
||||
},
|
||||
"ky": {
|
||||
"жчк": ("жоопкерчилиги чектелген коом",),
|
||||
"жоопкерчилиги чектелген коом": ("жчк",),
|
||||
},
|
||||
}
|
||||
COMPANY_REGISTRATION_DOCUMENTS = (("230044970", 3000), ("667", 2000), ("4", 1000))
|
||||
TAX_PROFIT_DOCUMENTS = (("112340", 3000),)
|
||||
BOILERPLATE_PREFIXES = {
|
||||
"ru": (
|
||||
"как регулируется", "правовое регулирование", "судебная практика", "сроки и ответственность",
|
||||
"действующая редакция", "последние изменения", "основные положения", "комментарий к акту",
|
||||
),
|
||||
"ky": (),
|
||||
}
|
||||
ACT_TITLE_PREFIXES = {
|
||||
"ru": (
|
||||
r"(?:гражданский\s+)?процессуальный\s+кодекс(?:\s+(?:кыргызской\s+республики|кр))?",
|
||||
r"(?:гражданск\w*|уголовн\w*|налогов\w*|трудов\w*|семейн\w*)\s+кодекс\w*(?:\s+(?:кыргызской\s+республики|кр))?",
|
||||
r"кодекс\s+о\s+правонарушениях(?:\s+(?:кыргызской\s+республики|кр))?",
|
||||
r"конституци\w*(?:\s+(?:кыргызской\s+республики|кр))?",
|
||||
r"(?:гк|ук|нк|коап|гпк|упк|уик)(?:\s+(?:кыргызской\s+республики|кр))?",
|
||||
r"закон(?:\s+(?:кыргызской\s+республики|кр))?\s+о\s+(?:защит\w*\s+прав\w*\s+потребител\w*|государственн\w+\s+регистраци\w+\s+юридическ\w+\s+лиц|труд\w*)",
|
||||
),
|
||||
"ky": (
|
||||
r"кыргыз\w*\s+республикасынын\s+конституци\w*",
|
||||
r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жарандык|кылмыш\w*|салык|эмгек|үй-бүлө|административдик|жарандык\s+процесстик)\s+кодекс\w*",
|
||||
r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жк|үк)(?:\s+кр)?",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def relaxed_query_text(language: str, query: str) -> str:
|
||||
value = query.strip()
|
||||
for prefix in BOILERPLATE_PREFIXES[language]:
|
||||
if value.casefold().startswith(prefix):
|
||||
value = value[len(prefix):].strip()
|
||||
break
|
||||
for pattern in ACT_TITLE_PREFIXES[language]:
|
||||
match = re.match(rf"\s*{pattern}(?=\s|$|[.,;:!?])", value, re.IGNORECASE)
|
||||
if match:
|
||||
value = value[match.end():].lstrip(" \t,;:—-")
|
||||
break
|
||||
return value
|
||||
|
||||
|
||||
def query_variants(language: str, query: str) -> list[str]:
|
||||
variants = [query]
|
||||
for alias, expansions in LEGAL_FORM_ALIASES[language].items():
|
||||
pattern = re.compile(rf"(?<!\w){re.escape(alias)}(?!\w)", re.IGNORECASE)
|
||||
if pattern.search(query):
|
||||
variants.extend(pattern.sub(expansion, query) for expansion in expansions)
|
||||
return list(dict.fromkeys(variants))
|
||||
|
||||
|
||||
def document_priority_clauses(language: str, documents: tuple[tuple[str, int], ...]) -> list[dict]:
|
||||
status = {"ru": "Действует", "ky": "Күчүндө"}[language]
|
||||
# ponytail: curated legal mapping; replace with a reviewed intent catalog when coverage expands.
|
||||
def clause(document_code: str, boost: int) -> dict:
|
||||
return {
|
||||
return [
|
||||
{
|
||||
"constant_score": {
|
||||
"filter": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{"term": {"document_code": document_code}},
|
||||
{"term": {"document_code": code}},
|
||||
{"term": {f"status_{language}": status}},
|
||||
]
|
||||
}
|
||||
@@ -34,29 +87,102 @@ def company_registration_clauses(language: str, query: str) -> list[dict]:
|
||||
"boost": boost,
|
||||
}
|
||||
}
|
||||
for code, boost in documents
|
||||
]
|
||||
|
||||
return [clause("230044970", 2000), clause("667", 1000)]
|
||||
|
||||
def company_registration_clauses(language: str, query: str) -> list[dict]:
|
||||
non_registration_context = {
|
||||
"ru": (r"\bсч[её]т\w*", r"\bбанк\w*", r"\bфилиал\w*", r"\bпредставительств\w*"),
|
||||
"ky": (r"\bбанк\w*", r"\bэсеп\w*", r"\bфилиал\w*", r"\bөкүлчүлүк\w*"),
|
||||
}[language]
|
||||
normalized_query = query.casefold()
|
||||
if any(re.search(pattern, normalized_query) for pattern in non_registration_context):
|
||||
return []
|
||||
|
||||
forms, patterns = {
|
||||
"ru": (
|
||||
r"(?:осоо|ооо|обществ\w*\s+с\s+ограниченной\s+ответственностью)",
|
||||
(
|
||||
r"\bкак\s+(?:откры\w*|созда\w*|зарегистр\w*)\s+{form}\b",
|
||||
r"\b(?:порядок|процедура)\s+(?:открыти\w*|создани\w*|регистраци\w*)\s+{form}\b",
|
||||
r"\b(?:государственн\w+\s+)?регистраци\w*\s+{form}\b",
|
||||
r"\bсоздани\w*\s+{form}\b",
|
||||
),
|
||||
),
|
||||
"ky": (
|
||||
r"(?:жчк|жоопкерчилиги\s+чектелген\s+коом)",
|
||||
(
|
||||
r"\bжчк\s+ач\w*\s+тартиби\b",
|
||||
r"\bжчк\s+кантип\s+ач\w*\b",
|
||||
r"\b{form}\s+(?:ачуу|ачуунун|каттоо|каттоонун)\b",
|
||||
r"\b(?:ачуу|каттоо)\s+{form}\b",
|
||||
),
|
||||
),
|
||||
}[language]
|
||||
if not any(re.search(pattern.format(form=forms), normalized_query) for pattern in patterns):
|
||||
return []
|
||||
|
||||
return document_priority_clauses(language, COMPANY_REGISTRATION_DOCUMENTS)
|
||||
|
||||
|
||||
def tax_profit_clauses(language: str, query: str) -> list[dict]:
|
||||
if language != "ru" or not re.search(r"\bналог\w*\s+(?:на\s+)?прибыл\w*\b", query.casefold()):
|
||||
return []
|
||||
# ponytail: exact topic anchor; expand only with verified tax-code sources.
|
||||
return document_priority_clauses(language, TAX_PROFIT_DOCUMENTS)
|
||||
|
||||
|
||||
def build_search_query(language: str, query: str) -> dict:
|
||||
registration_clauses = company_registration_clauses(language, query)
|
||||
intent_clauses = [*registration_clauses, *tax_profit_clauses(language, query)]
|
||||
variants = query_variants(language, query)
|
||||
full_text = [
|
||||
{
|
||||
"multi_match": {
|
||||
"query": variant,
|
||||
"fields": [f"document_name_{language}^4", f"text_{language}"],
|
||||
"type": "cross_fields",
|
||||
"operator": "OR" if registration_clauses else "AND",
|
||||
}
|
||||
}
|
||||
for variant in variants
|
||||
]
|
||||
relaxed_text = []
|
||||
if not registration_clauses:
|
||||
seen_relaxed = set()
|
||||
for variant in variants:
|
||||
value = relaxed_query_text(language, variant)
|
||||
words = value.split()
|
||||
if value in seen_relaxed or len(words) < 2 or (value == variant.strip() and len(words) < 3):
|
||||
continue
|
||||
if len(words) < 3 and not any(len(word) >= 5 for word in words):
|
||||
continue
|
||||
seen_relaxed.add(value)
|
||||
relaxed_text.append({
|
||||
"multi_match": {
|
||||
"query": value,
|
||||
"fields": [f"document_name_{language}^4", f"text_{language}"],
|
||||
"type": "cross_fields",
|
||||
"operator": "OR",
|
||||
"minimum_should_match": 2,
|
||||
"boost": 0.1,
|
||||
}
|
||||
})
|
||||
phrase_matches = [
|
||||
{"match_phrase": {f"text_{language}": {"query": variant, "boost": 3}}}
|
||||
for variant in query_variants(language, query)
|
||||
]
|
||||
planned = {"should": [*full_text, *phrase_matches, *relaxed_text, *intent_clauses], "minimum_should_match": 1}
|
||||
return {"bool": planned}
|
||||
|
||||
|
||||
def build_search_body(language: str, query: str, top_k: int) -> bytes:
|
||||
full_text = {
|
||||
"multi_match": {
|
||||
"query": query,
|
||||
"fields": [f"document_name_{language}", f"text_{language}"],
|
||||
"type": "cross_fields",
|
||||
}
|
||||
}
|
||||
clauses = company_registration_clauses(language, query)
|
||||
bool_query = {"filter": {"term": {"language": language}}}
|
||||
if clauses:
|
||||
bool_query.update({"should": [full_text, *clauses], "minimum_should_match": 1})
|
||||
else:
|
||||
bool_query["must"] = full_text
|
||||
return json.dumps({
|
||||
"size": top_k,
|
||||
"track_total_hits": False,
|
||||
"_source": ["document_code"],
|
||||
"query": {"bool": bool_query},
|
||||
"query": {"bool": {"filter": [{"term": {"language": language}}, {"term": {"is_current_edition": True}}], "must": build_search_query(language, query)}},
|
||||
"collapse": {"field": "document_code"},
|
||||
}, ensure_ascii=False).encode()
|
||||
|
||||
@@ -82,7 +208,7 @@ def main() -> int:
|
||||
if arguments.top_k <= 0:
|
||||
raise SystemExit("--top-k must be greater than zero")
|
||||
print(json.dumps(search_documents(arguments.url, arguments.index, arguments.language, arguments.query, arguments.top_k), ensure_ascii=False))
|
||||
print(f"Akyldash Backend v{APP_VERSION} · Frontend — not created")
|
||||
print(f"Akyldash · Backend v{APP_VERSION} · Frontend v0.1.2")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Оценка поисковой выдачи · Акылдаш</title>
|
||||
<style>
|
||||
:root { color-scheme: light; font: 16px/1.5 "Noto Sans", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", serif; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; color: var(--ink); background: #F7F2E8; }
|
||||
.review__skip { position: absolute; inset-block-start: 0; inset-inline-start: -10000px; padding: .5rem 1rem; background: var(--surface); }
|
||||
.review__skip:focus { inset-inline-start: 1rem; z-index: 2; }
|
||||
.review__header, .review__main { max-width: 1320px; margin: auto; padding-inline: clamp(1rem, 3vw, 3rem); }
|
||||
.review__header { padding-block: 0 1.5rem; background: var(--surface); border-block-end: 1px solid var(--line); }
|
||||
.review__header h1 { max-width: 24ch; margin: 1.5rem 0 .4rem; font: 700 clamp(1.75rem, 3vw, 2rem)/1.2 var(--serif); }
|
||||
.review__header p { max-width: 65ch; margin: 0; color: var(--muted); }
|
||||
.review__header-top { display: flex; justify-content: space-between; gap: 1rem; align-items: start; }
|
||||
.review__eyebrow { display: inline-flex; align-items: center; gap: .5rem; color: var(--ink); font: 600 1.1rem/1.25 var(--serif); }
|
||||
.review__eyebrow::before { content: ''; width: 1.5rem; height: 1.5rem; border: 2px solid var(--brand); border-radius: 50%; background: linear-gradient(90deg, transparent 45%, var(--brand) 45% 55%, transparent 55%), linear-gradient(0deg, transparent 45%, var(--brand) 45% 55%, transparent 55%); }
|
||||
.review__main { padding-block-end: 2rem; }
|
||||
.review__search { display: grid; grid-template-columns: minmax(18rem, 1fr) 9rem 9rem auto; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
|
||||
.review__field { display: grid; gap: .35rem; min-width: 0; }
|
||||
.review__field span, .review__actions label { color: var(--muted); font-size: .875rem; font-weight: 700; }
|
||||
.review__field--small { min-width: 0; }
|
||||
input, select, textarea, button { font: inherit; }
|
||||
input, select, textarea { width: 100%; min-height: 44px; border: 1px solid var(--line); border-radius: 6px; padding: .7rem .8rem; background: var(--surface); color: var(--ink); }
|
||||
input::placeholder, textarea::placeholder { color: var(--muted); }
|
||||
input:focus-visible, select:focus-visible, textarea:focus-visible, button:focus-visible { outline: 3px solid var(--focus); outline-offset: 2px; }
|
||||
button { min-height: 44px; cursor: pointer; border: 1px solid var(--line); border-radius: 6px; padding: .55rem .9rem; background: var(--surface); color: var(--ink); font-weight: 600; transition-property: background-color, border-color, color, scale; transition-duration: 150ms; transition-timing-function: cubic-bezier(.2, 0, 0, 1); }
|
||||
button:active { scale: .96; }
|
||||
button:disabled { cursor: wait; opacity: .55; }
|
||||
.review__button--primary { background: var(--brand); color: #fff; border-color: var(--brand); }
|
||||
.review__status { min-height: 1.7rem; margin-block: 1rem; color: var(--muted); }
|
||||
.review__status--error { color: #9D2633; }
|
||||
.review__workspace { display: grid; grid-template-columns: minmax(24rem, .9fr) minmax(28rem, 1.1fr); gap: 1rem; align-items: start; }
|
||||
.review__results, .review__document { min-width: 0; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; padding: clamp(1rem, 2vw, 1.5rem); }
|
||||
.review__panel-heading { display: flex; justify-content: space-between; gap: 1rem; align-items: baseline; margin-block-end: 1rem; }
|
||||
.review__panel-heading h2 { margin: 0; font: 600 1.5rem/1.3 var(--serif); }
|
||||
.review__panel-hint { color: var(--muted); font-size: .875rem; }
|
||||
.review__results-list { display: grid; gap: .75rem; margin: 0; padding: 0; list-style: none; }
|
||||
.review__result { padding: 1rem 0; border: 0; border-block-end: 1px solid var(--line); border-radius: 0; background: var(--surface); transition-property: background-color, border-color; transition-duration: 150ms; transition-timing-function: cubic-bezier(.2, 0, 0, 1); }
|
||||
.review__result--selected { background: var(--subtle); box-shadow: none; }
|
||||
.review__result:first-child { border-block-start: 1px solid var(--line); }
|
||||
.review__result-title { display: flex; gap: .65rem; align-items: baseline; width: 100%; min-height: 44px; text-align: start; font-weight: 700; border: 0; padding: 0; background: none; color: var(--brand-dark); }
|
||||
.review__rank { flex: 0 0 auto; color: var(--brand); font-variant-numeric: tabular-nums; }
|
||||
.review__meta { margin-block: .5rem; color: var(--muted); font-size: .875rem; }
|
||||
.review__rating { display: grid; grid-template-columns: repeat(4, 1fr); gap: .4rem; margin-block: 1rem .75rem; padding: 0; border: 0; }
|
||||
.review__rating legend { width: 100%; margin-block-end: .35rem; font-size: .875rem; font-weight: 700; color: var(--muted); grid-column: 1 / -1; }
|
||||
.review__rating label { display: grid; place-items: center; min-width: 0; min-height: 44px; padding: .45rem .25rem; border: 1px solid var(--line); border-radius: 6px; color: var(--muted); text-align: center; font-size: .875rem; cursor: pointer; }
|
||||
.review__rating label:has(input:checked) { border-color: var(--brand); background: var(--brand); color: #fff; }
|
||||
.review__rating label:has(input:focus-visible) { outline: 3px solid var(--focus); outline-offset: 2px; }
|
||||
.review__rating input { position: absolute; opacity: 0; inline-size: 1px; min-height: 1px; block-size: 1px; }
|
||||
.review__comment { min-height: 4rem; resize: vertical; background: var(--surface); }
|
||||
.review__document { position: sticky; inset-block-start: 1rem; min-height: 24rem; }
|
||||
.review__document-meta { margin-block-end: 1rem; padding-block-end: 1rem; border-block-end: 1px solid var(--line); color: var(--muted); }
|
||||
.review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
|
||||
.review__document-body img { max-width: 100%; height: auto; }
|
||||
.review__actions { max-width: 1320px; margin: 1.5rem auto 0; padding-inline: clamp(1rem, 3vw, 3rem); }
|
||||
.review__actions-inner { display: flex; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
|
||||
.review__actions label { display: grid; gap: .35rem; flex: 1; color: var(--muted); font-size: .875rem; font-weight: 700; }
|
||||
.review__actions label:last-of-type { flex: 2; }
|
||||
dialog { max-width: min(38rem, calc(100% - 2rem)); max-height: calc(100% - 2rem); border: 1px solid var(--line); border-radius: 6px; padding: 1.5rem; box-shadow: 0 1rem 2.5rem rgb(10 20 30 / .18); }
|
||||
dialog::backdrop { background: rgb(10 20 30 / .5); }
|
||||
.review__dialog-close { float: inline-end; }
|
||||
.review__empty { padding: 3rem 1rem; color: var(--muted); text-align: center; }
|
||||
.review__empty strong { display: block; margin-block-end: .35rem; color: var(--ink); }
|
||||
.review__tour-spotlight { position: fixed; z-index: 1; pointer-events: none; border: 4px solid var(--accent); border-radius: 6px; box-shadow: 0 0 0 100vmax rgb(56 38 42 / .58), 0 0 0 8px rgb(163 123 50 / .25); }
|
||||
.review__tour { width: min(34rem, calc(100% - 2rem)); padding: .5rem; color: var(--ink); }
|
||||
.review__tour::backdrop { background: transparent; }
|
||||
.review__tour-content { padding: clamp(1.5rem, 4vw, 2rem); }
|
||||
.review__tour h2 { margin: 0 0 .65rem; font-size: clamp(1.5rem, 3vw, 2rem); line-height: 1.2; letter-spacing: -.02em; }
|
||||
.review__tour p { margin: 0; color: var(--muted); }
|
||||
.review__tour-footer { display: flex; justify-content: space-between; gap: .75rem; align-items: center; margin-block-start: 1.5rem; }
|
||||
.review__tour-progress { color: var(--muted); font-size: .8125rem; }
|
||||
.review__tour-actions { display: flex; gap: .5rem; }
|
||||
.review__nav { display: flex; align-items: center; justify-content: space-between; gap: 1rem; max-width: 1440px; margin: auto; padding: .75rem clamp(1rem, 3vw, 3rem); border-block-end: 1px solid var(--line); }
|
||||
.review__nav-links, .review__nav-account { display: flex; align-items: center; gap: .5rem; }
|
||||
.review__nav button, .review__nav a { min-height: 44px; }
|
||||
.review__nav .is-active { color: var(--brand); border-color: var(--brand); background: var(--subtle); }
|
||||
.review__page { max-width: 38rem; }
|
||||
.review__page h2 { font: 600 1.5rem/1.3 var(--serif); }
|
||||
.review__page label { display: grid; gap: .35rem; margin-block: .9rem; color: var(--muted); font-size: .9rem; font-weight: 600; }
|
||||
.review__page-actions { display: flex; flex-wrap: wrap; gap: .5rem; margin-block-start: 1rem; }
|
||||
.review__notice { padding: .75rem; background: #FFF3DA; color: #82520D; border-radius: 6px; }
|
||||
.review__user-list { padding: 0; list-style: none; }
|
||||
.review__user-row { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: .75rem; padding-block: .75rem; border-block-end: 1px solid var(--line); }
|
||||
.review__user-meta { display: grid; gap: .25rem; color: var(--muted); white-space: pre-line; }
|
||||
.review__user-reset { display: flex; flex-wrap: wrap; gap: .5rem; align-items: end; }
|
||||
.review__user-reset label { display: grid; gap: .25rem; margin: 0; }
|
||||
.review__user-reset input { width: min(15rem, 100%); }
|
||||
.review__header-top { padding-block: .75rem; align-items: center; }
|
||||
.review__header-content { max-width: 1440px; margin: auto; padding-inline: clamp(1rem, 3vw, 3rem); }
|
||||
body.review__locked .review__header, body.review__locked .review__main, body.review__locked .review__actions, body.review__locked footer, body.review__locked .search { visibility: hidden; }
|
||||
.search { max-width: 1600px; margin: auto; }
|
||||
.search__query { padding: 1rem clamp(1rem, 3vw, 3rem); background: var(--surface); border-block-end: 1px solid var(--line); }
|
||||
.search__query > label, .search__filters > label { display: grid; gap: .35rem; color: var(--muted); font-size: .85rem; font-weight: 600; }
|
||||
.search__query-row { display: grid; grid-template-columns: minmax(15rem, 1fr) auto 9rem; gap: .75rem; align-items: end; }
|
||||
.search__language { display: grid; gap: .25rem; color: var(--muted); font-size: .8125rem; }
|
||||
.search__status { margin-inline: clamp(1rem, 3vw, 3rem); }
|
||||
.search__layout { display: grid; grid-template-columns: 230px minmax(320px, .9fr) minmax(420px, 1.2fr); min-height: 65vh; border-block: 1px solid var(--line); background: var(--surface); }
|
||||
.search__filters, .search__results, .search__document { min-width: 0; padding: 1rem; }
|
||||
.search__filters, .search__results { border-inline-end: 1px solid var(--line); }
|
||||
.search__filters h2, .search__results h2, .search__document h2 { margin: 0; font: 600 1.5rem/1.3 var(--serif); }
|
||||
.search__filters .review__panel-heading, .search__results .review__panel-heading { align-items: center; }
|
||||
.search__filter { margin-block: 1.25rem; }
|
||||
.search__filter h3 { margin: 0 0 .4rem; font-size: 1rem; }
|
||||
.search__filter label { display: flex; align-items: center; gap: .5rem; min-height: 44px; }
|
||||
.search__filter input { width: 24px; min-height: 24px; height: 24px; accent-color: var(--brand); }
|
||||
.search__filters > label { margin-block: .75rem; }
|
||||
.search__result-list { margin: 0; padding: 0; list-style: none; }
|
||||
.search__result { padding: 1rem .5rem; border-block-end: 1px solid var(--line); }
|
||||
.search__result--selected { background: var(--subtle); }
|
||||
.search__result-title { min-height: 44px; border: 0; padding: 0; text-align: start; background: none; color: var(--brand); font-weight: 700; }
|
||||
.search__pagination { display: flex; justify-content: space-between; align-items: center; gap: .5rem; padding-block-start: 1rem; }
|
||||
.search__document { min-height: 60vh; }
|
||||
.search__document-controls { display: flex; flex-wrap: wrap; gap: .75rem; }
|
||||
.search__document-controls label { display: grid; gap: .25rem; min-width: 11rem; color: var(--muted); font-size: .875rem; }
|
||||
.search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
|
||||
.search__document-body h1, .search__document-body h2, .search__document-body h3 { font-family: var(--serif); }
|
||||
.search__document-body img { max-width: 100%; height: auto; }
|
||||
.search__sr-only { position: absolute; inline-size: 1px; block-size: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
|
||||
@media (max-width: 969px) { .search__layout { grid-template-columns: 1fr; } }
|
||||
@media (max-width: 800px) {
|
||||
.search__filters, .search__results { border-inline-end: 0; border-block-end: 1px solid var(--line); }
|
||||
.search__document { min-height: 24rem; }
|
||||
.review__search { grid-template-columns: 1fr 1fr; }
|
||||
.review__search .review__field:first-child { grid-column: 1 / -1; }
|
||||
.review__workspace { grid-template-columns: 1fr; }
|
||||
.review__main { padding-block-end: 2rem; }
|
||||
.review__document { display: none; }
|
||||
.review__document--tour-visible { display: block; }
|
||||
.review__search button { grid-column: 1 / -1; }
|
||||
.review__actions { padding-inline: 1rem; }
|
||||
.review__actions-inner { flex-wrap: wrap; }
|
||||
.review__actions label { flex: 1 1 14rem !important; }
|
||||
.review__actions button { flex: 1 1 100%; }
|
||||
}
|
||||
@media (max-width: 480px) { .review__search { grid-template-columns: 1fr; } .review__search .review__field:first-child, .review__search button { grid-column: auto; } .review__rating { grid-template-columns: repeat(2, minmax(0, 1fr)); } .search__query-row { grid-template-columns: 1fr 1fr; } .search__query-row > input { grid-column: 1 / -1; } .review__nav { align-items: start; flex-wrap: wrap; } .review__nav-links { order: 3; width: 100%; flex-wrap: wrap; } .review__nav-account { margin-inline-start: auto; } }
|
||||
@media (hover: hover) { button:hover { background: var(--surface-muted); border-color: var(--muted); } .review__button--primary:hover { background: var(--brand-dark); border-color: var(--brand-dark); } .review__result:hover { background: var(--subtle); } .review__result-title:hover, .search__result-title:hover { background: none; color: var(--brand); } .review__rating label:hover { border-color: var(--brand); } }
|
||||
@media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; transition: none !important; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<a class="review__skip" href="#results">Перейти к результатам</a>
|
||||
<header class="review__header">
|
||||
<div class="review__nav"><div class="review__eyebrow">Акылдаш</div><nav class="review__nav-links" aria-label="Основная навигация"><a id="search-nav" href="/">Поиск</a><a id="review-nav" href="/review">Лаборатория</a><button id="account-open" type="button">Личный кабинет</button><button id="users-open" type="button" hidden>Пользователи</button></nav><div class="review__nav-account"><span id="signed-in-user"></span><button id="logout" type="button">Выйти</button></div></div>
|
||||
<div class="review__header-content"><div class="review__header-top"><button id="tutorial-open" type="button">Как это работает</button></div>
|
||||
<h1>Проверка качества поиска</h1><p>Сопоставляйте запрос с текстом документов и сохраняйте оценку выдачи.</p></div>
|
||||
</header>
|
||||
<main class="review__main" id="review">
|
||||
<form class="review__search" id="search-form">
|
||||
<label class="review__field">Юридический запрос<input id="query" name="q" maxlength="500" required autocomplete="off" placeholder="Например, как открыть ОсОО?"></label>
|
||||
<label class="review__field review__field--small">Язык запроса<select id="language" name="language"><option value="ru">Русский</option><option value="ky">Кыргызский</option></select></label>
|
||||
<label class="review__field review__field--small">Результатов<select id="page-size" name="page_size"><option>10</option><option>20</option></select></label>
|
||||
<button class="review__button--primary" type="submit">Показать результаты</button>
|
||||
</form>
|
||||
<div class="review__status" id="status" role="status" aria-live="polite"></div>
|
||||
<div class="review__workspace">
|
||||
<section class="review__results" aria-labelledby="results-heading"><div class="review__panel-heading"><h2 id="results-heading">Результаты поиска</h2><span class="review__panel-hint">Оцените каждый документ</span></div><ol class="review__results-list" id="results"><li class="review__empty"><strong>Начните с запроса</strong>Введите юридический вопрос выше, чтобы увидеть результаты.</li></ol></section>
|
||||
<section class="review__document" id="document-panel" aria-labelledby="document-heading"><div class="review__panel-heading"><h2 id="document-heading">Просмотр документа</h2><span class="review__panel-hint">Откройте результат</span></div><div class="review__document-meta" id="document-meta">Выберите название в списке, чтобы прочитать документ.</div><article class="review__document-body" id="document-body"></article></section>
|
||||
</div>
|
||||
</main>
|
||||
<main class="search" id="search-page" hidden>
|
||||
<h1 class="search__sr-only">Поиск по правовой базе</h1>
|
||||
<form class="search__query" id="public-search-form"><label for="public-query">Поиск по правовой базе</label><div class="search__query-row"><input id="public-query" maxlength="500" autocomplete="off" placeholder="Введите правовой вопрос или название документа"><button class="review__button--primary" type="submit">Найти</button><label class="search__language">Язык<select id="public-language"><option value="ru">Русский</option><option value="ky">Кыргызский</option></select></label></div></form>
|
||||
<p class="review__status" id="public-status" role="status" aria-live="polite"></p>
|
||||
<div class="search__layout"><aside class="search__filters" aria-labelledby="filters-heading"><div class="review__panel-heading"><h2 id="filters-heading">Фильтры</h2><button id="clear-filters" type="button">Сбросить</button></div><div id="filter-fields"></div><label>Дата принятия от<input id="date-from" type="date"></label><label>Дата принятия до<input id="date-to" type="date"></label></aside>
|
||||
<section class="search__results" aria-labelledby="search-results-heading"><div class="review__panel-heading"><div><h2 id="search-results-heading">Результаты</h2><span class="review__panel-hint" id="result-count">Введите запрос</span></div><label>Сортировка<select id="sort-order"><option value="relevance">По релевантности</option><option value="date">По дате</option></select></label></div><ol id="public-results" class="search__result-list"><li class="review__empty"><strong>Начните с поиска</strong>Введите запрос, чтобы найти документы.</li></ol><div class="search__pagination"><button id="page-prev" type="button" disabled>Назад</button><span id="page-label">Страница 1</span><button id="page-next" type="button" disabled>Далее</button></div></section>
|
||||
<section class="search__document" aria-labelledby="public-document-heading"><div class="review__panel-heading"><h2 id="public-document-heading">Документ</h2></div><div class="search__document-controls"><label>Редакция<select id="document-edition" disabled></select></label><label>Язык документа<select id="document-language" disabled></select></label></div><p id="public-document-meta" class="review__document-meta">Выберите результат, чтобы прочитать доступную редакцию.</p><article id="public-document-body" class="review__document-body"></article></section>
|
||||
</div>
|
||||
</main>
|
||||
<div class="review__actions"><div class="review__actions-inner"><label>Проверяющий<input id="reviewer" readonly aria-label="Проверяющий"></label><label>Комментарий к выдаче <input id="overall-comment" maxlength="4000" placeholder="Что важно учесть? (необязательно)"></label><button class="review__button--primary" id="save" type="button">Сохранить оценку</button></div></div>
|
||||
<dialog id="login-dialog" aria-labelledby="login-heading"><form class="review__page" id="login-form"><h2 id="login-heading">Вход в Акылдаш</h2><p>Введите учетные данные, выданные администратором.</p><label>Логин<input id="login-username" name="username" autocomplete="username" required minlength="3" maxlength="64"></label><label>Пароль<input id="login-password" name="password" type="password" autocomplete="current-password" required></label><p id="login-error" class="review__status review__status--error" role="alert"></p><button class="review__button--primary" type="submit">Войти</button></form></dialog>
|
||||
<dialog id="password-dialog" aria-labelledby="password-heading"><form class="review__page" id="password-form"><h2 id="password-heading">Создайте новый пароль</h2><p class="review__notice">Для продолжения замените временный пароль. Используйте не менее 12 символов.</p><label>Текущий пароль<input id="current-password" type="password" autocomplete="current-password" required></label><label>Новый пароль<input id="new-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><label>Повторите новый пароль<input id="confirm-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p id="password-error" class="review__status review__status--error" role="alert"></p><button class="review__button--primary" type="submit">Сохранить пароль</button></form></dialog>
|
||||
<dialog id="account-dialog" aria-labelledby="account-heading"><section class="review__page"><h2 id="account-heading">Личный кабинет</h2><dl><dt>Логин</dt><dd id="profile-username"></dd><dt>Роль</dt><dd id="profile-role">Администратор</dd><dt>Дата создания</dt><dd id="profile-created"></dd></dl><form id="profile-password-form"><h3>Изменить пароль</h3><p>Новый пароль должен содержать не менее 12 символов.</p><label>Текущий пароль<input id="profile-current-password" type="password" autocomplete="current-password" required></label><label>Новый пароль<input id="profile-new-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p id="profile-error" role="alert" class="review__status review__status--error"></p><div class="review__page-actions"><button class="review__button--primary" type="submit">Изменить пароль</button><button type="button" data-close="account-dialog">Закрыть</button></div></form></section></dialog>
|
||||
<dialog id="users-dialog" aria-labelledby="users-heading"><section class="review__page"><h2 id="users-heading">Управление пользователями</h2><form id="create-user-form"><label>Логин<input id="new-username" autocomplete="off" minlength="3" maxlength="64" required></label><label>Временный пароль<input id="temporary-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p>Пароль — от 12 символов. Новый пользователь сменит его при первом входе. Передайте временный пароль отдельно.</p><p id="users-error" role="alert" class="review__status review__status--error"></p><button class="review__button--primary" type="submit">Создать пользователя</button></form><ul id="user-list" class="review__user-list"></ul><button type="button" data-close="users-dialog">Закрыть</button></section></dialog>
|
||||
<dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
|
||||
<div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
|
||||
<footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.3 · Внутренняя лаборатория релевантности</div></footer>
|
||||
<script>
|
||||
document.body.classList.add('review__locked');
|
||||
const DRAFT_KEY = 'akyldash-search-review-draft';
|
||||
const draftStorageKey = () => `${DRAFT_KEY}:${account?.username || 'anonymous'}`;
|
||||
const TOUR_KEY = 'akyldash-search-review-tour-seen';
|
||||
const TOUR_STEPS = [
|
||||
{ target: 'query', title: 'Начните с юридического вопроса', text: 'Введите практический запрос, например «как открыть ОсОО?», выберите язык и нажмите «Показать результаты».', label: 'Шаг 1 из 4' },
|
||||
{ target: 'results', title: 'Смотрите порядок выдачи', text: 'Документы расположены в том порядке, в котором их нашёл поиск. Номер слева — позиция результата.', label: 'Шаг 2 из 4' },
|
||||
{ target: 'document-panel', title: 'Читайте документ перед оценкой', text: 'Нажмите на название результата. Справа откроется текст документа и ссылка на официальный источник.', label: 'Шаг 3 из 4' },
|
||||
{ target: 'save', title: 'Сохраните свою разметку', text: 'Поставьте оценку каждому просмотренному документу, добавьте комментарии и сохраните результат внизу страницы.', label: 'Шаг 4 из 4' }
|
||||
];
|
||||
const state = { results: [], token: '', selected: null, query: '' };
|
||||
const $ = (id) => document.getElementById(id);
|
||||
let account = null;
|
||||
const reviewRoute = location.pathname === '/review';
|
||||
$('review').hidden = !reviewRoute; $('search-page').hidden = reviewRoute; $('review-nav').classList.toggle('is-active', reviewRoute); $('search-nav').classList.toggle('is-active', !reviewRoute);
|
||||
if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.3'; }
|
||||
async function apiFetch(path, options = {}) { const response = await fetch(path, {credentials: 'same-origin', ...options}); if (response.status === 401 && path !== '/auth/login') { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); } return response; }
|
||||
const showLogin = () => { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); };
|
||||
const enterWorkspace = (user) => { account = user; $('signed-in-user').textContent = `${user.username} · ${user.role}`; $('reviewer').value = user.username; $('profile-username').textContent = user.username; $('profile-role').textContent = user.role; $('profile-created').textContent = new Date(user.created_at * 1000).toLocaleDateString('ru-RU'); $('users-open').hidden = user.username.toLowerCase() !== 'admin'; document.body.classList.remove('review__locked'); if (!reviewRoute) initSearchPage(); else if (!hasSeenTour()) setTimeout(openTutorial, 350); };
|
||||
const filterFields = [['document_types', 'Вид документа', 'document_type'], ['statuses', 'Статус', 'status'], ['authorities', 'Орган принятия', 'authority'], ['legal_forces', 'Юридическая сила', 'legal_force']];
|
||||
let filterCatalogs = {};
|
||||
let publicPage = 1;
|
||||
let selectedPublicDocument = null;
|
||||
const selectedFilters = () => Object.fromEntries(filterFields.map(([key,, parameter]) => [parameter, document.querySelector(`input[name="filter-${parameter}"]:checked`)?.value || '']));
|
||||
function renderFilterCatalogs(payload) { filterCatalogs = payload; const root = $('filter-fields'); root.replaceChildren(); filterFields.forEach(([key, title, parameter]) => { const items = payload[key] || []; if (!items.length) return; const group = document.createElement('section'); group.className = 'search__filter'; const heading = document.createElement('h3'); heading.textContent = title; group.append(heading); items.forEach((item) => { const label = document.createElement('label'); const input = document.createElement('input'); input.type = 'radio'; input.name = `filter-${parameter}`; input.value = item.code; input.addEventListener('change', () => runSearch(1)); const text = document.createElement('span'); text.textContent = `${item.labels.ru || item.labels.ky || item.code} (${item.count})`; label.append(input, text); group.append(label); }); root.append(group); }); }
|
||||
function renderPublicResults(results) { const list = $('public-results'); list.replaceChildren(); if (!results.length) { list.innerHTML = '<li class="review__empty"><strong>Документы не найдены</strong>Измените запрос или сбросьте фильтры.</li>'; return; } results.forEach((result, index) => { const row = document.createElement('li'); row.className = 'search__result'; const title = document.createElement('button'); title.type = 'button'; title.className = 'search__result-title'; title.textContent = result.name || 'Название не указано'; title.addEventListener('click', () => openPublicDocument(result, row)); const meta = document.createElement('p'); meta.className = 'review__meta'; meta.textContent = [result.type, result.status, result.date_adopted, result.number, result.legal_force].filter(Boolean).join(' · '); const snippet = document.createElement('p'); snippet.textContent = result.snippet || 'Для документа нет выдержки из текста.'; row.append(title, meta, snippet); list.append(row); }); }
|
||||
async function runSearch(page = 1) { const query = $('public-query').value.trim(); if (!query) { $('public-status').textContent = 'Введите запрос, чтобы начать поиск.'; return; } publicPage = page; $('public-status').textContent = 'Выполняется поиск…'; const params = new URLSearchParams({q: query, language: $('public-language').value, page: String(page), page_size: '20', sort: $('sort-order').value}); for (const [key, value] of Object.entries(selectedFilters())) if (value) params.set(key, value); if ($('date-from').value) params.set('date_from', $('date-from').value); if ($('date-to').value) params.set('date_to', $('date-to').value); history.replaceState(null, '', `/?${params}`); try { const response = await apiFetch(`/search?${params}`); const payload = await response.json(); if (!response.ok) throw new Error(payload.error || 'Не удалось выполнить поиск.'); renderPublicResults(payload.results); $('result-count').textContent = `${payload.results.length} документов`; $('public-status').textContent = payload.results.length ? `Результаты для запроса «${query}».` : `По запросу «${query}» документов нет. Измените запрос или фильтры.`; $('page-label').textContent = `Страница ${page}`; $('page-prev').disabled = page <= 1; $('page-next').disabled = !payload.has_next; } catch (error) { $('public-status').textContent = error.message || 'Не удалось выполнить поиск. Повторите попытку.'; $('result-count').textContent = 'Поиск недоступен'; } }
|
||||
async function loadPublicEdition() {
|
||||
if (!selectedPublicDocument) return;
|
||||
const {code} = selectedPublicDocument;
|
||||
const edition = $('document-edition').value;
|
||||
const editionInfo = selectedPublicDocument.editions.find((item) => item.source_code === edition);
|
||||
const languages = editionInfo?.available_languages || [];
|
||||
const languageSelect = $('document-language');
|
||||
[...languageSelect.options].forEach((option) => { option.disabled = !languages.includes(option.value); });
|
||||
if (!languages.length) { $('public-document-meta').textContent = 'В этой редакции нет доступного текста.'; $('public-document-body').replaceChildren(); return; }
|
||||
if (!languages.includes(languageSelect.value)) languageSelect.value = languages[0];
|
||||
const language = languageSelect.value;
|
||||
$('public-document-meta').textContent = 'Загрузка редакции…';
|
||||
try {
|
||||
const response = await apiFetch(`/documents/${encodeURIComponent(code)}/editions/${encodeURIComponent(edition)}?language=${encodeURIComponent(language)}`);
|
||||
const payload = await response.json();
|
||||
if (!response.ok) throw new Error(payload.error || 'Редакция недоступна.');
|
||||
const content = payload.content[language];
|
||||
const meta = $('public-document-meta'); meta.replaceChildren();
|
||||
const details = document.createElement('span'); details.textContent = [selectedPublicDocument.type, selectedPublicDocument.status, selectedPublicDocument.date_adopted].filter(Boolean).join(' · ');
|
||||
const source = document.createElement('a'); source.href = `https://cbd.minjust.gov.kg/${encodeURIComponent(code)}/edition/${encodeURIComponent(edition)}/${encodeURIComponent(language)}`; source.target = '_blank'; source.rel = 'noreferrer'; source.textContent = 'Официальный источник';
|
||||
meta.append(details, document.createTextNode(' · '), source);
|
||||
$('public-document-body').innerHTML = content.html;
|
||||
} catch (error) { $('public-document-meta').textContent = error.message || 'Не удалось загрузить документ.'; $('public-document-body').replaceChildren(); }
|
||||
}
|
||||
async function openPublicDocument(result, row) {
|
||||
document.querySelectorAll('.search__result--selected').forEach((item) => item.classList.remove('search__result--selected'));
|
||||
row.classList.add('search__result--selected'); selectedPublicDocument = result;
|
||||
$('public-document-heading').textContent = result.name || 'Документ'; $('public-document-body').replaceChildren();
|
||||
try {
|
||||
const response = await apiFetch(`/documents/${encodeURIComponent(result.code)}/editions`);
|
||||
const payload = await response.json();
|
||||
if (!response.ok) throw new Error(payload.error || 'Не удалось получить список редакций.');
|
||||
selectedPublicDocument.editions = payload.editions || [];
|
||||
const editionSelect = $('document-edition'); editionSelect.replaceChildren();
|
||||
(payload.editions || []).forEach((edition) => { const option = document.createElement('option'); option.value = edition.source_code; const name = edition.name?.[$('public-language').value] || edition.name?.ru || edition.name?.ky; option.textContent = [name, edition.source_code].filter(Boolean).join(' · '); editionSelect.append(option); });
|
||||
const matchedEdition = [...editionSelect.options].find((option) => option.value === String(result.edition));
|
||||
if (matchedEdition) editionSelect.value = matchedEdition.value;
|
||||
editionSelect.disabled = !editionSelect.options.length;
|
||||
const languages = payload.available_languages || [];
|
||||
const languageSelect = $('document-language'); languageSelect.replaceChildren();
|
||||
languages.forEach((language) => { const option = document.createElement('option'); option.value = language; option.textContent = language === 'ru' ? 'Русский' : 'Кыргызский'; languageSelect.append(option); });
|
||||
languageSelect.disabled = !languages.length;
|
||||
if (!languages.length || !editionSelect.options.length) throw new Error('Для документа нет доступных редакций с текстом.');
|
||||
languageSelect.value = languages.includes($('public-language').value) ? $('public-language').value : languages[0];
|
||||
await loadPublicEdition();
|
||||
} catch (error) { $('public-document-meta').textContent = error.message || 'Документ недоступен.'; }
|
||||
}
|
||||
$('document-edition').addEventListener('change', loadPublicEdition); $('document-language').addEventListener('change', loadPublicEdition);
|
||||
async function initSearchPage() { if (reviewRoute || filterCatalogs.length) return; const params = new URLSearchParams(location.search); $('public-query').value = params.get('q') || ''; $('public-language').value = params.get('language') || 'ru'; $('sort-order').value = params.get('sort') || 'relevance'; $('date-from').value = params.get('date_from') || ''; $('date-to').value = params.get('date_to') || ''; const response = await apiFetch(`/search/filters?language=${encodeURIComponent($('public-language').value)}`); if (response.ok) renderFilterCatalogs(await response.json()); for (const [key,, parameter] of filterFields) { const value = params.get(parameter); const input = value && document.querySelector(`input[name="filter-${parameter}"][value="${CSS.escape(value)}"]`); if (input) input.checked = true; } const query = params.get('q'); if (query) runSearch(Number(params.get('page') || 1)); }
|
||||
$('public-search-form').addEventListener('submit', (event) => { event.preventDefault(); runSearch(1); });
|
||||
$('sort-order').addEventListener('change', () => runSearch(1)); $('date-from').addEventListener('change', () => runSearch(1)); $('date-to').addEventListener('change', () => runSearch(1)); $('public-language').addEventListener('change', async () => { const response = await apiFetch(`/search/filters?language=${encodeURIComponent($('public-language').value)}`); if (response.ok) renderFilterCatalogs(await response.json()); if ($('public-query').value.trim()) runSearch(1); });
|
||||
$('page-prev').addEventListener('click', () => runSearch(publicPage - 1)); $('page-next').addEventListener('click', () => runSearch(publicPage + 1));
|
||||
$('clear-filters').addEventListener('click', () => { document.querySelectorAll('#filter-fields input:checked').forEach((input) => { input.checked = false; }); $('date-from').value = ''; $('date-to').value = ''; $('sort-order').value = 'relevance'; runSearch(1); });
|
||||
async function loadUsers() {
|
||||
$('users-error').textContent = '';
|
||||
const response = await apiFetch('/admin/users');
|
||||
if (!response.ok) { $('users-error').textContent = 'Не удалось загрузить список пользователей.'; return; }
|
||||
const {users} = await response.json();
|
||||
const formatUserDate = (timestamp) => timestamp ? new Intl.DateTimeFormat('ru-RU', {dateStyle: 'medium', timeStyle: 'short'}).format(new Date(timestamp * 1000)) : '—';
|
||||
const list = $('user-list'); list.replaceChildren();
|
||||
users.forEach((user) => {
|
||||
const row = document.createElement('li'); row.className = 'review__user-row';
|
||||
const label = document.createElement('span'); label.textContent = `${user.username} · Администратор${user.must_change_password ? ' · требуется сменить пароль' : ''}`; row.append(label);
|
||||
const dates = document.createElement('small'); dates.className = 'review__user-meta'; dates.textContent = `Первый вход: ${formatUserDate(user.first_login_at)}\nПоследний вход: ${formatUserDate(user.last_login_at)}\nСброс пароля: ${formatUserDate(user.password_reset_at)}`; row.append(dates);
|
||||
if (user.username.toLowerCase() !== 'admin') {
|
||||
const resetForm = document.createElement('form'); resetForm.className = 'review__user-reset';
|
||||
const password = document.createElement('input'); password.type = 'password'; password.autocomplete = 'new-password'; password.minLength = 12; password.maxLength = 256; password.required = true;
|
||||
const passwordLabel = document.createElement('label'); passwordLabel.append(document.createTextNode('Новый временный пароль (от 12 символов)'), password);
|
||||
const reset = document.createElement('button'); reset.type = 'submit'; reset.textContent = 'Сбросить пароль'; resetForm.append(passwordLabel, reset);
|
||||
resetForm.addEventListener('submit', async (event) => { event.preventDefault(); const result = await apiFetch(`/admin/users/${encodeURIComponent(user.username)}/reset-password`, {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({password: password.value})}); if (!result.ok) { $('users-error').textContent = (await result.json()).error || 'Не удалось сбросить пароль.'; return; } await loadUsers(); });
|
||||
const remove = document.createElement('button'); remove.type = 'button'; remove.textContent = 'Удалить'; remove.addEventListener('click', async () => { if (!confirm(`Удалить учетную запись ${user.username}? Сохраненная разметка останется в истории.`)) return; const result = await apiFetch(`/admin/users/${encodeURIComponent(user.username)}`, {method: 'DELETE'}); if (!result.ok) { $('users-error').textContent = 'Не удалось удалить учетную запись.'; return; } await loadUsers(); });
|
||||
row.append(resetForm, remove);
|
||||
}
|
||||
list.append(row);
|
||||
});
|
||||
}
|
||||
$('login-form').addEventListener('submit', async (event) => { event.preventDefault(); $('login-error').textContent = ''; const response = await fetch('/auth/login', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({username: $('login-username').value, password: $('login-password').value})}); const payload = await response.json(); if (!response.ok) { $('login-error').textContent = payload.error || 'Не удалось войти. Проверьте логин и пароль.'; $('login-password').focus(); return; } $('login-dialog').close(); $('login-password').value = ''; if (payload.account.must_change_password) { $('current-password').value = sessionTemporaryPassword; $('password-dialog').showModal(); $('new-password').focus(); } else { enterWorkspace(payload.account); } });
|
||||
let sessionTemporaryPassword = '';
|
||||
$('login-form').addEventListener('submit', () => { sessionTemporaryPassword = $('login-password').value; }, true);
|
||||
$('password-form').addEventListener('submit', async (event) => { event.preventDefault(); $('password-error').textContent = ''; if ($('new-password').value !== $('confirm-password').value) { $('password-error').textContent = 'Новые пароли не совпадают.'; $('confirm-password').focus(); return; } const response = await fetch('/auth/change-password', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({current_password: $('current-password').value, new_password: $('new-password').value})}); const payload = await response.json(); if (!response.ok) { $('password-error').textContent = payload.error || 'Не удалось изменить пароль.'; return; } $('password-dialog').close(); sessionTemporaryPassword = ''; $('password-form').reset(); enterWorkspace(payload.account); });
|
||||
$('profile-password-form').addEventListener('submit', async (event) => { event.preventDefault(); $('profile-error').textContent = ''; const response = await apiFetch('/auth/change-password', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({current_password: $('profile-current-password').value, new_password: $('profile-new-password').value})}); const payload = await response.json(); if (!response.ok) { $('profile-error').textContent = payload.error || 'Не удалось изменить пароль.'; return; } $('account-dialog').close(); $('profile-password-form').reset(); enterWorkspace(payload.account); });
|
||||
$('create-user-form').addEventListener('submit', async (event) => { event.preventDefault(); $('users-error').textContent = ''; const response = await apiFetch('/admin/users', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({username: $('new-username').value, password: $('temporary-password').value})}); const payload = await response.json(); if (!response.ok) { $('users-error').textContent = payload.error || 'Не удалось создать учетную запись.'; return; } $('create-user-form').reset(); await loadUsers(); });
|
||||
$('account-open').addEventListener('click', () => $('account-dialog').showModal());
|
||||
$('users-open').addEventListener('click', async () => { await loadUsers(); $('users-dialog').showModal(); });
|
||||
document.querySelectorAll('[data-close]').forEach((button) => button.addEventListener('click', () => $(button.dataset.close).close()));
|
||||
$('logout').addEventListener('click', async () => { await fetch('/auth/logout', {method: 'POST', credentials: 'same-origin', headers: {'Content-Type': 'application/json'}, body: '{}'}); account = null; document.body.classList.add('review__locked'); showLogin(); });
|
||||
(async () => { const response = await fetch('/auth/me', {credentials: 'same-origin'}); if (!response.ok) { showLogin(); return; } const payload = await response.json(); if (payload.account.must_change_password) { $('password-dialog').showModal(); $('current-password').focus(); return; } enterWorkspace(payload.account); })();
|
||||
const setStatus = (text, error = false) => { $('status').textContent = text; $('status').classList.toggle('review__status--error', error); $('status').setAttribute('role', error ? 'alert' : 'status'); $('status').setAttribute('aria-live', error ? 'assertive' : 'polite'); };
|
||||
const setError = (text) => setStatus(text, true);
|
||||
const escapeText = (value) => value == null ? '' : String(value);
|
||||
const readDraft = () => { try { return JSON.parse(localStorage.getItem(draftStorageKey()) || 'null'); } catch (_) { return null; } };
|
||||
const hasSeenTour = () => { try { return localStorage.getItem(TOUR_KEY) === '1'; } catch (_) { return false; } };
|
||||
const markTourSeen = () => { try { localStorage.setItem(TOUR_KEY, '1'); } catch (_) {} };
|
||||
const saveDraft = () => { if (!state.token) return; const results = [...$('results').children].map((item, index) => ({rank: index + 1, code: item.dataset.code, rating: item.querySelector('input:checked')?.value ?? null, comment: item.querySelector('textarea').value})); try { localStorage.setItem(draftStorageKey(), JSON.stringify({query: state.query, language: $('language').value, pageSize: $('page-size').value, overallComment: $('overall-comment').value, results})); } catch (_) {} };
|
||||
const applyDraft = () => { const draft = readDraft(); if (!draft || draft.query !== state.query || draft.language !== $('language').value || draft.pageSize !== $('page-size').value) return; $('reviewer').value = account?.username || ''; $('overall-comment').value = draft.overallComment || ''; (draft.results || []).forEach((saved) => { const item = [...$('results').children].find((candidate) => candidate.dataset.rank === String(saved.rank) && candidate.dataset.code === saved.code); if (!item) return; if (saved.rating != null) { const input = item.querySelector(`input[value="${CSS.escape(String(saved.rating))}"]`); if (input) input.checked = true; } item.querySelector('textarea').value = saved.comment || ''; }); };
|
||||
function renderResults() {
|
||||
$('results').replaceChildren();
|
||||
if (!state.results.length) { $('results').innerHTML = '<li class="review__empty"><strong>Результатов пока нет</strong>Введите другой запрос и попробуйте снова.</li>'; return; }
|
||||
state.results.forEach((result, index) => {
|
||||
const item = document.createElement('li'); item.className = 'review__result'; item.dataset.rank = index + 1; item.dataset.code = result.code;
|
||||
const title = document.createElement('button'); title.type = 'button'; title.className = 'review__result-title'; title.innerHTML = `<span class="review__rank">#${index + 1}</span><span></span>`; title.lastElementChild.textContent = escapeText(result.name || 'Название не указано'); title.addEventListener('click', () => openDocument(index, title));
|
||||
const meta = document.createElement('div'); meta.className = 'review__meta'; meta.textContent = [result.type, result.status, result.date_adopted, result.number].filter(Boolean).join(' · ');
|
||||
const rating = document.createElement('fieldset'); rating.className = 'review__rating'; rating.innerHTML = `<legend>Оценка результата</legend>`;
|
||||
[['0', 'нерелевантен'], ['1', 'косвенно полезен'], ['2', 'частично полезен'], ['3', 'прямо отвечает']].forEach(([value, label]) => { const id = `rating-${index}-${value}`; const wrapper = document.createElement('label'); wrapper.htmlFor = id; wrapper.textContent = `${value} — ${label}`; const input = document.createElement('input'); input.type = 'radio'; input.name = `rating-${index}`; input.id = id; input.value = value; wrapper.prepend(input); rating.append(wrapper); });
|
||||
const comment = document.createElement('textarea'); comment.className = 'review__comment'; comment.maxLength = 4000; comment.placeholder = 'Комментарий к результату (необязательно)'; comment.setAttribute('aria-label', `Комментарий к результату #${index + 1}`);
|
||||
item.append(title, meta, rating, comment); $('results').append(item);
|
||||
});
|
||||
applyDraft();
|
||||
}
|
||||
async function openDocument(index, trigger) {
|
||||
const result = state.results[index]; state.selected = trigger; [...$('results').children].forEach((item) => item.classList.remove('review__result--selected')); trigger.closest('.review__result')?.classList.add('review__result--selected'); setStatus('Загрузка документа…');
|
||||
try { const response = await apiFetch(`/documents/${encodeURIComponent(result.code)}/editions/${encodeURIComponent(result.edition)}?language=${encodeURIComponent($('language').value)}`); if (!response.ok) throw new Error('Документ недоступен'); const payload = await response.json(); const content = payload.content[$('language').value]; const meta = $('document-meta'); meta.textContent = [result.name, result.status, result.date_adopted].filter(Boolean).join(' · '); const source = document.createElement('a'); source.href = `https://cbd.minjust.gov.kg/${encodeURIComponent(result.code)}/edition/${encodeURIComponent(result.edition)}/${encodeURIComponent($('language').value)}`; source.target = '_blank'; source.rel = 'noreferrer'; source.textContent = ' Официальный источник'; meta.append(source); $('document-body').innerHTML = content.html; $('dialog-heading').textContent = result.name || 'Документ'; $('dialog-body').innerHTML = content.html; if (matchMedia('(max-width: 800px)').matches) $('document-dialog').showModal(); setStatus('Документ загружен.'); } catch (error) { setError('Не удалось загрузить документ. Повторите попытку.'); }
|
||||
}
|
||||
$('dialog-close').addEventListener('click', () => { $('document-dialog').close(); if (state.selected) state.selected.focus(); });
|
||||
let tutorialStep = 0; let tutorialReturnFocus = null;
|
||||
function clearTourTarget() { $('tutorial-spotlight').hidden = true; $('document-panel').classList.remove('review__document--tour-visible'); }
|
||||
function renderTutorial() { const step = TOUR_STEPS[tutorialStep]; const target = $(step.target); if (step.target === 'document-panel' && getComputedStyle(target).display === 'none') target.classList.add('review__document--tour-visible'); target.scrollIntoView({block: 'nearest', inline: 'nearest'}); const bounds = target.getBoundingClientRect(); $('tutorial-spotlight').style.cssText = `inset-block-start: ${Math.max(8, bounds.top - 4)}px; inset-inline-start: ${Math.max(8, bounds.left - 4)}px; inline-size: ${bounds.width + 8}px; block-size: ${bounds.height + 8}px;`; $('tutorial-spotlight').hidden = false; $('tutorial-heading').textContent = step.title; $('tutorial-text').textContent = step.text; $('tutorial-progress').textContent = step.label; $('tutorial-next').textContent = tutorialStep === TOUR_STEPS.length - 1 ? 'Понятно' : 'Далее'; }
|
||||
function closeTutorial() { clearTourTarget(); $('tutorial-dialog').close(); markTourSeen(); tutorialReturnFocus?.focus(); }
|
||||
function openTutorial() { tutorialReturnFocus = document.activeElement; tutorialStep = 0; renderTutorial(); $('tutorial-dialog').showModal(); $('tutorial-next').focus(); }
|
||||
$('tutorial-open').addEventListener('click', openTutorial); $('tutorial-next').addEventListener('click', () => { if (tutorialStep === TOUR_STEPS.length - 1) closeTutorial(); else { tutorialStep += 1; renderTutorial(); } }); $('tutorial-skip').addEventListener('click', closeTutorial); $('tutorial-dialog').addEventListener('cancel', (event) => { event.preventDefault(); closeTutorial(); });
|
||||
$('search-form').addEventListener('submit', async (event) => { event.preventDefault(); const query = $('query').value.trim(); if (!query) return; state.query = query; setStatus('Поиск выполняется…'); $('save').disabled = true; try { const params = new URLSearchParams({q: query, language: $('language').value, page_size: $('page-size').value}); const response = await apiFetch(`/search?${params}`); if (!response.ok) throw new Error(); const payload = await response.json(); state.results = payload.results; state.token = payload.review_token; renderResults(); setStatus(state.results.length ? `Найдено результатов: ${state.results.length}.` : `По запросу «${query}» ничего не найдено. Измените запрос.`); } catch (error) { state.results = []; state.token = ''; renderResults(); setError('Не удалось выполнить поиск. Повторите поиск.'); } finally { $('save').disabled = false; } });
|
||||
document.addEventListener('input', saveDraft); document.addEventListener('change', saveDraft);
|
||||
$('save').addEventListener('click', async () => { if (!state.token) { setError('Сначала выполните поиск.'); return; } const reviewer = $('reviewer').value.trim(); const results = [...$('results').children].map((item, index) => ({rank: index + 1, code: item.dataset.code, rating: item.querySelector('input:checked') ? Number(item.querySelector('input:checked').value) : null, comment: item.querySelector('textarea').value})); $('save').disabled = true; setStatus('Сохранение выполняется…'); try { const response = await apiFetch('/search-reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({review_token: state.token, reviewer, overall_comment: $('overall-comment').value, results})}); if (!response.ok) throw new Error(); const payload = await response.json(); localStorage.removeItem(draftStorageKey()); setStatus(`Оценка сохранена · № ${payload.id}.`); } catch (error) { setError('Не удалось сохранить. Проверьте подключение и повторите.'); } finally { $('save').disabled = false; } });
|
||||
const draft = readDraft(); if (draft) { $('query').value = draft.query || ''; $('language').value = draft.language || 'ru'; $('page-size').value = draft.pageSize || '20'; $('reviewer').value = account?.username || ''; $('overall-comment').value = draft.overallComment || ''; }
|
||||
if (reviewRoute) $('tutorial-open').hidden = false;
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Persistence and signed snapshots for search relevance reviews."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_COMMENT = 4000
|
||||
MAX_REVIEWER = 120
|
||||
|
||||
|
||||
class ReviewStore:
|
||||
def __init__(self, path: Path | str = ":memory:"):
|
||||
if path != ":memory:":
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
self.connection = sqlite3.connect(path, check_same_thread=False)
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
# ponytail: one SQLite lock; split connections only if review throughput matters.
|
||||
self._lock = threading.Lock()
|
||||
self.connection.execute("""
|
||||
CREATE TABLE IF NOT EXISTS search_reviews (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL,
|
||||
reviewer TEXT NOT NULL,
|
||||
query TEXT NOT NULL,
|
||||
language TEXT NOT NULL,
|
||||
index_name TEXT NOT NULL,
|
||||
algorithm_version TEXT NOT NULL,
|
||||
top_result_code TEXT,
|
||||
results_json TEXT NOT NULL,
|
||||
overall_comment TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
self.connection.commit()
|
||||
|
||||
def save(self, review: dict) -> int:
|
||||
with self._lock:
|
||||
cursor = self.connection.execute(
|
||||
"INSERT INTO search_reviews(created_at, reviewer, query, language, index_name, algorithm_version, top_result_code, results_json, overall_comment) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(review["created_at"], review["reviewer"], review["query"], review["language"], review["index_name"], review["algorithm_version"], review["top_result_code"], json.dumps(review["results"], ensure_ascii=False), review["overall_comment"]),
|
||||
)
|
||||
self.connection.commit()
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
def export(self) -> list[dict]:
|
||||
with self._lock:
|
||||
return [
|
||||
{**dict(row), "results": json.loads(row["results_json"])}
|
||||
for row in self.connection.execute("SELECT * FROM search_reviews ORDER BY id")
|
||||
]
|
||||
|
||||
|
||||
class ReviewSnapshots:
|
||||
def __init__(self, secret: bytes | None = None, ttl: int = 3600):
|
||||
self.secret = secret or secrets.token_bytes(32)
|
||||
self.ttl = ttl
|
||||
|
||||
def create(self, query: str, language: str, index: str, results: list[dict], algorithm_version: str) -> str:
|
||||
payload = {"query": query, "language": language, "index_name": index, "algorithm_version": algorithm_version, "results": results, "expires_at": int(time.time()) + self.ttl}
|
||||
encoded = base64.urlsafe_b64encode(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).decode().rstrip("=")
|
||||
signature = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest()
|
||||
return f"{encoded}.{signature}"
|
||||
|
||||
def verify(self, token: str) -> dict:
|
||||
try:
|
||||
encoded, signature = token.split(".", 1)
|
||||
expected = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(signature, expected):
|
||||
raise ValueError
|
||||
payload = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)))
|
||||
if payload["expires_at"] < int(time.time()):
|
||||
raise ValueError
|
||||
return payload
|
||||
except (ValueError, KeyError, TypeError, json.JSONDecodeError, UnicodeError) as error:
|
||||
raise ValueError("invalid or expired search snapshot") from error
|
||||
@@ -9,10 +9,22 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from search.catalog import legal_force_code, labels, source_code
|
||||
from search.minjust_opensearch import bulk_batches, document_codes, export_bulk, load_bulk, request_json, switch_alias
|
||||
|
||||
|
||||
class MinjustOpenSearchTest(unittest.TestCase):
|
||||
def test_unknown_document_type_keeps_source_label_and_has_no_inferred_force(self):
|
||||
value = {"ru": "Решение", "ky": "Решение"}
|
||||
|
||||
self.assertEqual(source_code("document_type", value), "other")
|
||||
self.assertEqual(labels("document_type", "other"), {"ru": "Прочие документы", "ky": "Башка документтер"})
|
||||
self.assertIsNone(legal_force_code(value))
|
||||
self.assertEqual(source_code("status", {"ru": "Не вступил в силу", "ky": "Күчүнө кире элек"}), "not_yet_effective")
|
||||
self.assertEqual(legal_force_code({"ru": "Конституционный закон", "ky": "Конституционный закон"}), "constitutional")
|
||||
self.assertEqual(legal_force_code({"ru": "Указ", "ky": "Жарлыгы"}), "subordinate")
|
||||
self.assertEqual(source_code("document_type", {"ru": "устав"}), "charter")
|
||||
|
||||
def test_switches_alias_atomically_after_successful_load(self):
|
||||
with patch("search.minjust_opensearch.request_json", return_value={"acknowledged": True}) as request:
|
||||
switch_alias("http://127.0.0.1:9200/", "akyldash-fragments-v2", "akyldash-fragments-current")
|
||||
@@ -93,6 +105,7 @@ class MinjustOpenSearchTest(unittest.TestCase):
|
||||
self.assertIn("text_ky", lines[1])
|
||||
self.assertTrue(lines[1]["is_current_edition"])
|
||||
self.assertNotIn("text_ru", lines[1])
|
||||
self.assertEqual(lines[1]["legal_force_code"], "legislative")
|
||||
self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока")
|
||||
self.assertEqual(
|
||||
list(bulk_batches(iter((("7", b"a\nb\n"), ("8", b"c\nd\n"))), 4)),
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import contextlib
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -29,13 +31,72 @@ class SearchApiTest(unittest.TestCase):
|
||||
with patch("search.api.request_json", return_value=response) as request:
|
||||
status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=active")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(payload["results"][0]["snippet"], "<em>Закон</em>")
|
||||
self.assertEqual(payload["results"][0]["snippet"], "Закон")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
self.assertEqual((body["from"], body["size"]), (5, 6))
|
||||
self.assertIn({"term": {"status_code": "active"}}, body["query"]["bool"]["filter"])
|
||||
with self.assertRaisesRegex(ApiError, "within 10000 results"):
|
||||
api.handle("GET", "/search?q=x&page=100&page_size=100")
|
||||
|
||||
def test_review_secret_file_is_read_without_command_line_value(self):
|
||||
from search.api import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
secret_file = Path(temporary) / "secret"
|
||||
secret_file.write_text("secret-value\n", encoding="utf-8")
|
||||
with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), patch("search.api.Api") as api, patch("search.api.ThreadingHTTPServer"):
|
||||
main()
|
||||
self.assertEqual(api.call_args.args[4], b"secret-value")
|
||||
|
||||
def test_review_secret_file_rejects_empty_content(self):
|
||||
from search.api import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
secret_file = Path(temporary) / "secret"
|
||||
secret_file.touch()
|
||||
with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), contextlib.redirect_stderr(io.StringIO()):
|
||||
with self.assertRaises(SystemExit) as result:
|
||||
main()
|
||||
self.assertEqual(result.exception.code, 2)
|
||||
|
||||
def test_production_api_uses_registration_intent_boosts(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
response = {"hits": {"hits": []}}
|
||||
with patch("search.api.request_json", return_value=response) as request:
|
||||
api.handle("GET", "/search?q=%D0%BA%D0%B0%D0%BA+%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D1%8C+%D0%9E%D1%81%D0%9E%D0%9E&language=ru")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
planned = body["query"]["bool"]["must"]["bool"]
|
||||
self.assertEqual(planned["minimum_should_match"], 1)
|
||||
self.assertNotIn("filter", planned)
|
||||
self.assertTrue(any("constant_score" in clause for clause in planned["should"]))
|
||||
self.assertTrue(any("общество с ограниченной ответственностью" in clause.get("multi_match", {}).get("query", "") for clause in planned["should"]))
|
||||
|
||||
def test_production_api_boosts_tax_code_for_profit_tax_query(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
with patch("search.api.request_json", return_value={"hits": {"hits": []}}) as request:
|
||||
api.handle("GET", "/search?q=%D0%BD%D0%B0%D0%BB%D0%BE%D0%B3+%D0%BD%D0%B0+%D0%BF%D1%80%D0%B8%D0%B1%D1%8B%D0%BB%D1%8C&language=ru")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
planned = body["query"]["bool"]["must"]["bool"]
|
||||
anchors = [clause["constant_score"]["filter"]["bool"]["filter"][0] for clause in planned["should"] if "constant_score" in clause]
|
||||
self.assertIn({"term": {"document_code": "112340"}}, anchors)
|
||||
|
||||
def test_search_sorts_repealed_documents_after_other_statuses(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
with patch("search.api.request_json", return_value={"hits": {"hits": []}}) as request:
|
||||
api.handle("GET", "/search?q=НДС+на+импорт&language=ru")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
source = body["sort"][0]["_script"]["script"]["source"]
|
||||
self.assertIn("'active' ? 0", source)
|
||||
self.assertIn("'repealed' ? 2", source)
|
||||
self.assertEqual(body["sort"][1], {"_score": "desc"})
|
||||
|
||||
api.handle("GET", "/search?q=НДС+на+импорт&language=ru&sort=date")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
self.assertEqual(body["sort"][1], {"date_adopted": "desc"})
|
||||
|
||||
def test_document_editions_openapi_and_validation(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
@@ -55,12 +116,15 @@ class SearchApiTest(unittest.TestCase):
|
||||
def test_filters_count_documents_and_return_bilingual_labels(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
response = {"aggregations": {name: {"buckets": [{"key": "law" if name == "document_types" else "active" if name == "statuses" else "parliament", "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}}
|
||||
keys = {"document_types": "law", "statuses": "active", "authorities": "parliament", "legal_forces": "legislative"}
|
||||
response = {"aggregations": {name: {"buckets": [{"key": keys[name], "documents": {"value": 3}}]} for name in keys}}
|
||||
with patch("search.api.request_json", return_value=response) as request:
|
||||
payload = api.handle("GET", "/search/filters?language=ky")[1]
|
||||
self.assertEqual(payload["document_types"][0], {"code": "law", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
|
||||
self.assertEqual(payload["legal_forces"][0]["labels"]["ky"], "Мыйзам деңгээли")
|
||||
body = json.loads(request.call_args.args[2])
|
||||
self.assertIn("terms", body["aggs"]["document_types"])
|
||||
self.assertIn("legal_force_code", body["aggs"]["legal_forces"]["terms"]["field"])
|
||||
self.assertEqual(body["query"], {"term": {"is_current_edition": True}})
|
||||
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ class SearchApiOpenSearchTest(unittest.TestCase):
|
||||
request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json")
|
||||
try:
|
||||
documents = [
|
||||
{"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "status_code": "repealed", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
|
||||
{"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
|
||||
{"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "document_type_code": "decree", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"], "authority_codes": ["president"]},
|
||||
{"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "legal_force_code": "legislative", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "status_code": "repealed", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
|
||||
{"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "legal_force_code": "legislative", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
|
||||
{"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "document_type_code": "decree", "legal_force_code": "subordinate", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"], "authority_codes": ["president"]},
|
||||
]
|
||||
for number, document in enumerate(documents):
|
||||
request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json")
|
||||
@@ -27,8 +27,12 @@ class SearchApiOpenSearchTest(unittest.TestCase):
|
||||
self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], [])
|
||||
filtered = api.handle("GET", "/search?q=needle&document_type=law")[1]
|
||||
self.assertEqual([result["code"] for result in filtered["results"]], ["1"])
|
||||
force_filtered = api.handle("GET", "/search?q=needle&legal_force=subordinate")[1]
|
||||
self.assertEqual([result["code"] for result in force_filtered["results"]], ["2"])
|
||||
self.assertEqual(force_filtered["results"][0]["legal_force"], "Подзаконный уровень")
|
||||
filters = api.handle("GET", "/search/filters")[1]
|
||||
self.assertEqual({item["count"] for item in filters["statuses"]}, {2})
|
||||
self.assertEqual({item["code"] for item in filters["legal_forces"]}, {"legislative", "subordinate"})
|
||||
finally:
|
||||
request_json(f"{base_url}/{index}", "DELETE", None, "application/json")
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from search.evaluate_relevance import evaluate, load_queries
|
||||
from search.query import build_search_body
|
||||
from search.query import COMPANY_REGISTRATION_DOCUMENTS, TAX_PROFIT_DOCUMENTS, build_search_body, query_variants
|
||||
|
||||
|
||||
class SearchRelevanceTest(unittest.TestCase):
|
||||
@@ -37,7 +37,83 @@ class SearchRelevanceTest(unittest.TestCase):
|
||||
body = json.loads(build_search_body("ru", "трудовой договор", 10))
|
||||
self.assertFalse(body["track_total_hits"])
|
||||
self.assertEqual(body["collapse"], {"field": "document_code"})
|
||||
self.assertEqual(body["query"]["bool"]["must"]["multi_match"]["type"], "cross_fields")
|
||||
self.assertIn({"term": {"is_current_edition": True}}, body["query"]["bool"]["filter"])
|
||||
lexical_query = body["query"]["bool"]["must"]["bool"]["should"][0]["multi_match"]
|
||||
self.assertEqual(lexical_query["type"], "cross_fields")
|
||||
self.assertEqual(lexical_query["fields"], ["document_name_ru^4", "text_ru"])
|
||||
self.assertEqual(lexical_query["operator"], "AND")
|
||||
self.assertTrue(any("match_phrase" in clause for clause in body["query"]["bool"]["must"]["bool"]["should"]))
|
||||
|
||||
def test_legal_form_abbreviations_expand_in_both_directions(self):
|
||||
self.assertIn("как открыть общество с ограниченной ответственностью", query_variants("ru", "как открыть ОсОО"))
|
||||
self.assertIn("регистрация осоо", [variant.casefold() for variant in query_variants("ru", "регистрация общества с ограниченной ответственностью")])
|
||||
self.assertIn("жчк ачуу тартиби", [variant.casefold() for variant in query_variants("ky", "жоопкерчилиги чектелген коом ачуу тартиби")])
|
||||
|
||||
def test_relaxed_match_drops_ru_template_prefix_and_stays_low_weight(self):
|
||||
body = json.loads(build_search_body("ru", "правовое регулирование расторжение договора аренды", 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
strict = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("operator") == "AND"]
|
||||
relaxed = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("boost") == 0.1]
|
||||
self.assertEqual(strict[0]["query"], "правовое регулирование расторжение договора аренды")
|
||||
self.assertEqual(relaxed[0]["query"], "расторжение договора аренды")
|
||||
self.assertEqual(relaxed[0]["operator"], "OR")
|
||||
self.assertEqual(relaxed[0]["minimum_should_match"], 2)
|
||||
self.assertNotIn("boost", strict[0])
|
||||
|
||||
def test_relaxed_match_supports_kyrgyz_and_avoids_short_named_acts(self):
|
||||
ky = json.loads(build_search_body("ky", "эмгек келишимин бузуу шарттары", 10))
|
||||
ky_clauses = ky["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertTrue(any(item.get("multi_match", {}).get("operator") == "OR" and item["multi_match"].get("boost") == 0.1 for item in ky_clauses))
|
||||
|
||||
named_act = json.loads(build_search_body("ru", "Гражданский кодекс", 10))
|
||||
act_clauses = named_act["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in act_clauses))
|
||||
|
||||
long_named_act = json.loads(build_search_body("ru", "последние изменения Гражданского кодекса Кыргызской Республики", 10))
|
||||
long_act_clauses = long_named_act["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in long_act_clauses))
|
||||
|
||||
short_template = json.loads(build_search_body("ru", "как регулируется взыскание алиментов", 10))
|
||||
short_clauses = short_template["query"]["bool"]["must"]["bool"]["should"]
|
||||
short_fallback = next(item["multi_match"] for item in short_clauses if item.get("multi_match", {}).get("boost") == 0.1)
|
||||
self.assertEqual(short_fallback["query"], "взыскание алиментов")
|
||||
self.assertEqual(short_fallback["minimum_should_match"], 2)
|
||||
|
||||
stopword_query = json.loads(build_search_body("ru", "правовое регулирование права и обязанности", 10))
|
||||
stopword_clauses = stopword_query["query"]["bool"]["must"]["bool"]["should"]
|
||||
stopword_fallback = next(item["multi_match"] for item in stopword_clauses if item.get("multi_match", {}).get("boost") == 0.1)
|
||||
self.assertEqual(stopword_fallback["minimum_should_match"], 2)
|
||||
|
||||
for language, query in (("ru", "последние изменения ГК КР"), ("ky", "Кыргыз Республикасынын Конституциясы")):
|
||||
act_body = json.loads(build_search_body(language, query, 10))
|
||||
act_clauses = act_body["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in act_clauses))
|
||||
|
||||
for language, query, expected in (
|
||||
("ru", "Налоговый кодекс ответственность за неуплату налога", "ответственность за неуплату налога"),
|
||||
("ru", "Налоговый кодекс налоговая ставка", "налоговая ставка"),
|
||||
("ru", "Налоговый кодекс: налоговая ставка", "налоговая ставка"),
|
||||
("ru", "закон о труде порядок выплаты зарплаты", "порядок выплаты зарплаты"),
|
||||
("ky", "Кыргыз Республикасынын Конституциясы укуктарды чектөө тартиби", "укуктарды чектөө тартиби"),
|
||||
):
|
||||
body = json.loads(build_search_body(language, query, 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
fallback = next(item["multi_match"] for item in clauses if item.get("multi_match", {}).get("boost") == 0.1)
|
||||
self.assertEqual(fallback["query"], expected)
|
||||
|
||||
for query in (
|
||||
"Закон о защите прав потребителей",
|
||||
"Закон Кыргызской Республики о защите прав потребителей",
|
||||
"Закон Кыргызской Республики о государственной регистрации юридических лиц",
|
||||
):
|
||||
body = json.loads(build_search_body("ru", query, 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in clauses))
|
||||
|
||||
consumer_rights = json.loads(build_search_body("ru", "Закон о защите прав потребителей возврат денег", 10))
|
||||
consumer_clauses = consumer_rights["query"]["bool"]["must"]["bool"]["should"]
|
||||
consumer_fallback = next(item["multi_match"] for item in consumer_clauses if item.get("multi_match", {}).get("boost") == 0.1)
|
||||
self.assertEqual(consumer_fallback["query"], "возврат денег")
|
||||
|
||||
def test_company_registration_intent_boosts_current_documents(self):
|
||||
for language, query, status in (
|
||||
@@ -45,21 +121,40 @@ class SearchRelevanceTest(unittest.TestCase):
|
||||
("ky", "ЖЧК ачуу тартиби", "Күчүндө"),
|
||||
):
|
||||
body = json.loads(build_search_body(language, query, 10))
|
||||
search_query = body["query"]["bool"]
|
||||
search_query = body["query"]["bool"]["must"]["bool"]
|
||||
self.assertEqual(search_query["minimum_should_match"], 1)
|
||||
boosts = [clause["constant_score"] for clause in search_query["should"][1:]]
|
||||
self.assertEqual([item["boost"] for item in boosts], [2000, 1000])
|
||||
self.assertNotIn("filter", search_query)
|
||||
lexical = [clause["multi_match"] for clause in search_query["should"] if "multi_match" in clause]
|
||||
self.assertTrue(all(clause["type"] == "cross_fields" and clause["operator"] == "OR" for clause in lexical))
|
||||
boosts = [clause["constant_score"] for clause in search_query["should"] if "constant_score" in clause]
|
||||
self.assertEqual([item["boost"] for item in boosts], [boost for _, boost in COMPANY_REGISTRATION_DOCUMENTS])
|
||||
self.assertEqual(
|
||||
[item["filter"]["bool"]["filter"][0]["term"]["document_code"] for item in boosts],
|
||||
["230044970", "667"],
|
||||
[code for code, _ in COMPANY_REGISTRATION_DOCUMENTS],
|
||||
)
|
||||
self.assertTrue(all(item["filter"]["bool"]["filter"][1] == {"term": {f"status_{language}": status}} for item in boosts))
|
||||
|
||||
def test_profit_tax_intent_prioritizes_the_current_tax_code(self):
|
||||
for query in ("налог на прибыль", "налога на прибыль организаций", "налогообложение прибыли"):
|
||||
body = json.loads(build_search_body("ru", query, 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
boosts = [clause["constant_score"] for clause in clauses if "constant_score" in clause]
|
||||
self.assertEqual([item["boost"] for item in boosts], [boost for _, boost in TAX_PROFIT_DOCUMENTS])
|
||||
self.assertEqual([item["filter"]["bool"]["filter"][0]["term"]["document_code"] for item in boosts], ["112340"])
|
||||
self.assertTrue(all(item["filter"]["bool"]["filter"][1] == {"term": {"status_ru": "Действует"}} for item in boosts))
|
||||
|
||||
for query in ("прибыль организации", "налог на доходы", "налог на имущество"):
|
||||
body = json.loads(build_search_body("ru", query, 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any("constant_score" in clause for clause in clauses))
|
||||
|
||||
def test_company_registration_intent_ignores_non_procedural_queries(self):
|
||||
for language, query in (
|
||||
("ru", "ОсОО зарегистрирован?"),
|
||||
("ru", "кто зарегистрировал ОсОО"),
|
||||
("ru", "как открыть счет ОсОО"),
|
||||
("ru", "как открыть ОсОО банковский счет"),
|
||||
("ru", "как открыть ОсОО банковский счёт"),
|
||||
("ru", "как открыть филиал ОсОО"),
|
||||
("ru", "как создать договор для ОсОО"),
|
||||
("ru", "порядок создания логотипа ОсОО"),
|
||||
@@ -69,7 +164,8 @@ class SearchRelevanceTest(unittest.TestCase):
|
||||
("ky", "ЖЧК кантип логотип түзүү"),
|
||||
):
|
||||
body = json.loads(build_search_body(language, query, 10))
|
||||
self.assertNotIn("should", body["query"]["bool"])
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any("constant_score" in clause for clause in clauses))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from search.api import Api, ApiError
|
||||
|
||||
|
||||
class SearchReviewTest(unittest.TestCase):
|
||||
def test_review_must_cover_each_snapshot_rank_once(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret")
|
||||
response = {"hits": {"hits": [{"_index": "search-20260827", "_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}, {"_index": "search-20260827", "_source": {"document_code": "8", "edition_code": "11", "document_name_ru": "Кодекс"}}]}}
|
||||
with patch("search.api.request_json", return_value=response):
|
||||
result = api.handle("GET", "/search?q=test&language=ru&page_size=2")[1]
|
||||
with self.assertRaisesRegex(ApiError, "exactly once"):
|
||||
api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3}]})
|
||||
|
||||
def test_saves_signed_search_snapshot_and_rejects_tampering(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret")
|
||||
response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}]}}
|
||||
with patch("search.api.request_json", return_value=response):
|
||||
result = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru")[1]
|
||||
saved = api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3, "comment": "Прямой ответ"}]})
|
||||
self.assertEqual(saved[0], 201)
|
||||
exported = api.handle("GET", "/search-reviews/export")[1]["reviews"]
|
||||
self.assertEqual(exported[0]["top_result_code"], "7")
|
||||
self.assertEqual(exported[0]["results"][0]["rating"], 3)
|
||||
with self.assertRaisesRegex(ApiError, "does not match"):
|
||||
api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "8", "rating": 3}]})
|
||||
|
||||
def test_snapshot_and_review_page_are_available(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = Api("http://opensearch:9200", "current", Path(temporary))
|
||||
page = api.review_page()
|
||||
self.assertIn("Оценка поисковой выдачи", page)
|
||||
self.assertIn("akyldash-search-review-draft", page)
|
||||
self.assertIn("tutorial-dialog", page)
|
||||
self.assertIn("Как это работает", page)
|
||||
self.assertNotIn('id="error"', page)
|
||||
self.assertIn("localStorage", page)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from tools.build_search_topic_taxonomy import build, group_for
|
||||
|
||||
|
||||
class SearchTopicTaxonomyTest(unittest.TestCase):
|
||||
def test_ambiguous_roots_are_left_for_manual_review(self):
|
||||
self.assertEqual(group_for("Иностранные инвестиции"), "review_required")
|
||||
|
||||
def test_preserves_source_tree_counts_and_marks_unclassified_roots(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
(root / "documents/1").mkdir(parents=True)
|
||||
(root / "documents/2").mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.executemany("INSERT INTO documents VALUES (?, 'success')", [("1",), ("2",)])
|
||||
db.commit()
|
||||
classifier = {
|
||||
"Code": None,
|
||||
"Name": {"Rus": "ЗАКОНОДАТЕЛЬСТВО О ТРУДЕ", "Kyr": None},
|
||||
"GeneralClassifiers": [{
|
||||
"Code": "child-1",
|
||||
"Name": {"Rus": "Рабочее время", "Kyr": "Иш убактысы"},
|
||||
"GeneralClassifiers": [],
|
||||
}],
|
||||
}
|
||||
unknown = {
|
||||
"Code": None,
|
||||
"Name": {"Rus": "НЕИЗВЕСТНАЯ РУБРИКА", "Kyr": None},
|
||||
"GeneralClassifiers": [],
|
||||
}
|
||||
(root / "documents/1/document.json").write_text(
|
||||
json.dumps({"general_classifiers": [classifier]}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "documents/2/document.json").write_text(
|
||||
json.dumps({"general_classifiers": [classifier, unknown]}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
catalog = build(root)
|
||||
groups = {group["code"]: group for group in catalog["groups"]}
|
||||
labor_root = groups["labor_social"]["children"][0]
|
||||
|
||||
self.assertEqual((catalog["source_document_count"], catalog["source_node_count"], catalog["source_root_count"]), (2, 3, 2))
|
||||
self.assertEqual(labor_root["document_count"], 2)
|
||||
self.assertEqual(labor_root["children"][0]["document_count"], 2)
|
||||
self.assertEqual(groups["review_required"]["children"][0]["grouping_status"], "manual_review")
|
||||
self.assertEqual(catalog["nodes_missing_kyrgyz_label"], 2)
|
||||
|
||||
def test_rebuild_fails_instead_of_omitting_successful_manifest_document(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
(root / "documents/1").mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||||
db.commit()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
|
||||
build(root)
|
||||
|
||||
def test_rebuild_rejects_normalized_document_without_classifier_list(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
document = root / "documents/1"
|
||||
document.mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||||
db.commit()
|
||||
(document / "document.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
|
||||
build(root)
|
||||
|
||||
def test_rebuild_rejects_invalid_nested_classifier_branch(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
document = root / "documents/1"
|
||||
document.mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||||
db.commit()
|
||||
(document / "document.json").write_text(
|
||||
json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
|
||||
build(root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,9 @@
|
||||
# Absolute path on the production host containing minjust-normalized/.
|
||||
DATA_ROOT=/var/lib/akyldash/data
|
||||
REVIEW_SECRET_FILE=/etc/akyldash/review_secret
|
||||
BACKEND_IMAGE=akyldash/backend:local
|
||||
OPENSEARCH_IMAGE=akyldash/opensearch:local
|
||||
SEARCH_INDEX=akyldash-fragments-current
|
||||
OPENSEARCH_MEM_LIMIT=4g
|
||||
OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
|
||||
COOKIE_SECURE=true
|
||||
@@ -0,0 +1,4 @@
|
||||
akyldash.kg {
|
||||
encode zstd gzip
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY backend /app/backend
|
||||
|
||||
ENV PYTHONPATH=/app/backend
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["python", "-m", "search.api", "--host", "0.0.0.0", "--port", "8080", "--url", "http://opensearch:9200", "--index", "akyldash-fragments-current", "--data", "/app/data/minjust-normalized", "--reviews-db", "/app/data/search-reviews.sqlite3"]
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/review', timeout=3)"]
|
||||
@@ -0,0 +1,3 @@
|
||||
FROM opensearchproject/opensearch:3.7.0
|
||||
|
||||
RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu
|
||||
@@ -0,0 +1,112 @@
|
||||
# Production deployment
|
||||
|
||||
Compose runs Caddy, the Search API, and a private OpenSearch node. Only Caddy
|
||||
publishes ports `80` and `443`; backend and OpenSearch have no host ports. The
|
||||
OpenSearch security plugin is disabled to match the current API client, so keep
|
||||
it on the private network.
|
||||
|
||||
## Host access and configuration
|
||||
|
||||
The production environment file is `/etc/akyldash/production.env`; the review
|
||||
secret is `/etc/akyldash/review_secret`. Both must stay outside the checkout,
|
||||
root-owned, and mode `0600`. `DATA_ROOT` must be an absolute path. The `codex`
|
||||
account is intentionally outside the `sudo` and `docker` groups because access
|
||||
to the Docker socket grants root-equivalent control. Run host operations with
|
||||
the narrowly required `sudo` command; do not use `docker` as `codex` directly.
|
||||
|
||||
After reviewing the host audit and confirming these files do not already exist,
|
||||
the owner can create them from the approved checkout with
|
||||
`sudo bash infra/deploy/prepare-production-config.sh`. The script refuses to
|
||||
overwrite existing configuration, installs the tracked env example with mode
|
||||
`0600`, and generates a random review secret with mode `0600`. It does not
|
||||
start Compose or create volumes. Review and set `DATA_ROOT` in the protected
|
||||
env file before deployment.
|
||||
|
||||
Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus
|
||||
is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in
|
||||
[`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the
|
||||
root bootstrap installs packages and creates directories, but does not deploy
|
||||
the application or alter SSH, firewall, user groups, or sudoers. The restricted
|
||||
`akyldash-compose` helper allows only root-owned Compose `config` and `status`;
|
||||
service start/stop remains an owner-run root operation.
|
||||
|
||||
The repository Gitea Actions runner is an isolated CI worker described in
|
||||
[`docs/operations/CI_RUNNER.md`](../../docs/operations/CI_RUNNER.md). It has no
|
||||
production Docker socket, production env files, or application checkout. CI
|
||||
does not deploy or change production; all deployment remains owner-run.
|
||||
|
||||
## First deployment and upgrades
|
||||
|
||||
Run commands from the root-owned checkout at `/opt/akyldash/current`. Render
|
||||
and review Compose before changing services:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml config --quiet
|
||||
```
|
||||
|
||||
On a clean first deployment, `infra/backup/archive.py create` reports that no
|
||||
backup is required when all persistent mounts are empty and no Compose
|
||||
containers exist. Continue without a backup in that case. If any persistent
|
||||
data exists, create and verify a backup before changing schema, images, or
|
||||
starting a new application version. This also applies when containers are
|
||||
absent, including an initial installation where the normalized corpus was
|
||||
preloaded under `DATA_ROOT`; create the archive before the first `up`. When
|
||||
named volumes do not exist, pass `--initial-deployment`; this is accepted only
|
||||
without Compose containers and when `DATA_ROOT` contains only
|
||||
`minjust-normalized/`. During upgrades, missing volumes are an error because
|
||||
they can indicate lost production data. The archive tool also refuses
|
||||
ambiguous storage state, running containers, and incomplete Compose service
|
||||
containers. See
|
||||
[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
|
||||
commands. Do not remove an older usable backup to make room; retain the
|
||||
pre-change archive until the new deployment and a newer verified backup are
|
||||
confirmed.
|
||||
|
||||
For an initial clean installation, start the stack only after config review:
|
||||
|
||||
```bash
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d --build
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml ps
|
||||
curl -fsS https://akyldash.kg/review >/dev/null
|
||||
```
|
||||
|
||||
Before this step, confirm DNS and have the owner review firewall rules for TCP
|
||||
`80` and `443`. Caddy manages the origin certificate; confirm HTTPS before
|
||||
setting Cloudflare to Full (strict). On the initial start, create the `admin`
|
||||
account interactively:
|
||||
|
||||
```bash
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --create-admin
|
||||
```
|
||||
|
||||
The account database is `$DATA_ROOT/search-users.sqlite3` and is included in
|
||||
the `data` backup. Admin password reset is also interactive:
|
||||
|
||||
```bash
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
|
||||
```
|
||||
|
||||
After the initial OpenSearch index is loaded and validated, the `admin` account
|
||||
is created, and `/review` works over HTTPS, create and verify the required
|
||||
post-setup baseline backup before treating deployment as complete. It captures
|
||||
the initialized index, account database, and Caddy certificate data. Follow
|
||||
the exact stop/create/verify/start sequence in
|
||||
[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md#initial-post-setup-baseline).
|
||||
|
||||
## Backup, restore, and rollback
|
||||
|
||||
Backups are cold and require downtime. Stop services, create the archive, then
|
||||
verify both its checksum and contents before proceeding. Do not run an upgrade
|
||||
or schema change until verification succeeds. Keep archives root-only under
|
||||
`/var/backups/akyldash` and copy them to separately managed encrypted storage.
|
||||
|
||||
Restore is a staged operation on a marked replacement host; it does not
|
||||
overwrite production. For application-only rollback, use the guarded helper
|
||||
with a verified pre-release backup and explicit data-compatibility review.
|
||||
Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md),
|
||||
[`ROLLBACK.md`](../../docs/operations/ROLLBACK.md), and
|
||||
[`RECOVERY.md`](../../docs/operations/RECOVERY.md).
|
||||
|
||||
Automated production deployment is not enabled. Deployments remain owner-run;
|
||||
record the release and verify service health before considering it complete.
|
||||
@@ -0,0 +1,124 @@
|
||||
name: akyldash
|
||||
|
||||
services:
|
||||
caddy:
|
||||
image: caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- caddy-data:/data
|
||||
- caddy-config:/config
|
||||
networks:
|
||||
- proxy
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
opensearch:
|
||||
image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: deploy/production/Dockerfile.opensearch
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
bootstrap.memory_lock: "true"
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: ${OPENSEARCH_JAVA_OPTS:--Xms2g -Xmx2g}
|
||||
mem_limit: ${OPENSEARCH_MEM_LIMIT:-4g}
|
||||
expose:
|
||||
- "9200"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
networks:
|
||||
- search
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9200/_cluster/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
backend:
|
||||
image: ${BACKEND_IMAGE:-akyldash/backend:local}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: deploy/production/Dockerfile.backend
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
environment:
|
||||
COOKIE_SECURE: ${COOKIE_SECURE:-true}
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- search.api
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
- --port
|
||||
- "8080"
|
||||
- --url
|
||||
- http://opensearch:9200
|
||||
- --index
|
||||
- ${SEARCH_INDEX:-akyldash-fragments-current}
|
||||
- --data
|
||||
- /app/data/minjust-normalized
|
||||
- --reviews-db
|
||||
- /app/data/search-reviews.sqlite3
|
||||
- --users-db
|
||||
- /app/data/search-users.sqlite3
|
||||
- --review-secret-file
|
||||
- /run/secrets/review_secret
|
||||
secrets:
|
||||
- review_secret
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ${DATA_ROOT:?set DATA_ROOT in .env}:/app/data
|
||||
networks:
|
||||
- proxy
|
||||
- search
|
||||
|
||||
volumes:
|
||||
caddy-data:
|
||||
caddy-config:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
search:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
review_secret:
|
||||
file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}
|
||||
+11
-1
@@ -2,12 +2,18 @@
|
||||
|
||||
## Продукт
|
||||
|
||||
- [Дизайн-система «Түндүк»](../DESIGN.md) — утверждённые палитра,
|
||||
типографика, компоненты и правила адаптивности портала и лаборатории.
|
||||
- [Обзор проекта](product/project-overview.md) — назначение, основные области и
|
||||
правила работы с данными.
|
||||
- [План frontend поисковой СПС](product/frontend-search-sps-plan.md) — границы
|
||||
MVP, зависимости и спринты.
|
||||
- [Готовность к проектированию frontend](product/frontend-design-readiness-plan.md) —
|
||||
обязательные работы и критерии перехода к frontend.
|
||||
- [План интерфейса оценки поисковой выдачи](product/search-relevance-review-interface-plan.md) —
|
||||
внутренняя лаборатория сбора оценок юристов для настройки OpenSearch.
|
||||
- [Разметка relevance set](../backend/search/RELEVANCE_ANNOTATION.md) — подготовка
|
||||
эталонных документов и воспроизводимая оценка собственного поиска.
|
||||
- [Задание по нормализации документов](product/minjust-document-normalization-agent-task.md) —
|
||||
требования и критерии приёмки нормализатора ЦБД Минюста КР.
|
||||
|
||||
@@ -20,6 +26,10 @@
|
||||
|
||||
- [Текущий статус](operations/project-status.md) — выполненные шаги, риски и
|
||||
ближайшие действия.
|
||||
- [Состояние инфраструктуры](operations/CURRENT_STATE.md) — подтверждённое
|
||||
состояние production и оставшиеся проверки.
|
||||
- [Инфраструктура и эксплуатация](operations/INFRASTRUCTURE.md) — схема,
|
||||
доступы, deployment, backup, rollback и recovery.
|
||||
- [План Telegram-пространства](operations/telegram-workspace-plan.md) — темы,
|
||||
роли и этапы развития командного окружения.
|
||||
|
||||
@@ -35,4 +45,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.5 MiB |
@@ -0,0 +1,84 @@
|
||||
# Infrastructure access
|
||||
|
||||
## Discover existing access first
|
||||
|
||||
In a new Codex session, inspect the SSH alias and key file metadata without
|
||||
printing key contents, then check the connection:
|
||||
|
||||
```bash
|
||||
ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }'
|
||||
test -r ~/.ssh/akyldash_codex && echo 'SSH key is available'
|
||||
ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps id
|
||||
ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps hostname -f
|
||||
gitea-api GET /version
|
||||
gitea-api GET /repos/admin/akyldash
|
||||
```
|
||||
|
||||
Do not ask for the host, login, password, token, or key again before these
|
||||
checks. Never print a private key, token, or complete environment file.
|
||||
|
||||
## VPS
|
||||
|
||||
- SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`.
|
||||
- SSH configuration: `~/.ssh/config`.
|
||||
- Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents).
|
||||
- `codex` can run unprivileged read-only diagnostics. It has no sudo or Docker
|
||||
group membership; Docker CLI access to the daemon is denied. Do not add it to
|
||||
either group.
|
||||
- Bootstrap is complete. Docker and Compose are active, but no root-owned
|
||||
repository checkout or restricted helper is installed yet. The owner can run
|
||||
`infra/bootstrap/audit-host.sh` from the reviewed root-owned checkout to
|
||||
report Docker containers/volumes, protected-file metadata, UFW rules, ports,
|
||||
and ISPmanager status without showing secret contents or changing state.
|
||||
- Host package/service configuration, firewall changes, protected env/secret
|
||||
creation, backup creation, recovery apply, and production deployment require
|
||||
a reviewed owner/root command. `infra/deploy/install-helper.sh` grants the
|
||||
fixed verifier, read-only Compose `config`/`status`, and guarded rollback
|
||||
commands. It does not grant general sudo or Docker access; it is not
|
||||
installed on the VPS.
|
||||
- `infra/deploy/prepare-production-config.sh` is an owner-run root operation.
|
||||
It refuses to overwrite existing files, copies only the tracked env example,
|
||||
generates a random review secret, and writes both files with mode `0600`.
|
||||
It does not start Compose or create volumes.
|
||||
- Never connect to other VPS hosts or Synology as part of this deployment.
|
||||
|
||||
This execution environment is WSL2. The repository is on the Windows-mounted
|
||||
`/mnt/d` filesystem, while SSH config and the identity file are in the Linux
|
||||
home directory. Do not assume Windows SSH credentials are available or copy
|
||||
keys between environments. Plain `ssh akyldash_vps` failed because the system
|
||||
SSH config include had unsafe permissions. Use `ssh -F ~/.ssh/config ...`; in a
|
||||
network-restricted session this read-only command may also require approved
|
||||
execution. If the key is missing in a future environment, report that
|
||||
limitation and do not retrieve it elsewhere.
|
||||
|
||||
## Gitea
|
||||
|
||||
- Repository: `https://git.vakanaut.ru/admin/akyldash`.
|
||||
- Use the installed `gitea-api` helper; it reads credentials from the
|
||||
environment. Never inspect local credential files, print environment values,
|
||||
or construct HTTP authorization headers.
|
||||
- The audited API token can read/write this repository and create issues, but
|
||||
does not have `read:user` or `read:package`. Listing repository Actions
|
||||
runners returns `403` because the API principal is not the repository owner.
|
||||
An API `403` caused by missing scope/ownership is not evidence that no runner
|
||||
exists.
|
||||
- Protected `main` requires one approval. Do not push directly to `main`, merge
|
||||
automatically, or start a review. Work on a feature branch and use a PR.
|
||||
- Gitea Actions and package features are enabled for the repository. The Actions
|
||||
task API currently returns no runs, but runner availability and package
|
||||
publishing permissions have not been confirmed. The repository-level runner
|
||||
is prepared by `infra/runner/install-runner.sh`, but is not installed or
|
||||
registered yet. Only a repository owner may obtain its registration token in
|
||||
`admin/akyldash` Settings → Actions → Runners; do not use organization or
|
||||
instance tokens and never put the token in Git, a command argument, or a
|
||||
report. See [CI_RUNNER.md](CI_RUNNER.md).
|
||||
|
||||
## Who can do what
|
||||
|
||||
| Role | Operations |
|
||||
|---|---|
|
||||
| `codex` over SSH | Unprivileged diagnostics; Docker daemon, UFW rules, and protected paths require the root audit script |
|
||||
| Restricted helpers | Read-only drift check and Compose `config`/`status`, plus guarded rollback; not installed on the audited VPS |
|
||||
| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
|
||||
| VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
|
||||
| Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
|
||||
@@ -0,0 +1,131 @@
|
||||
# Backup and restore
|
||||
|
||||
## Current status
|
||||
|
||||
No production data or backup existed during the 2026-10-11 audit. The repository
|
||||
now contains `infra/backup/archive.py`; it is not installed or run on the VPS,
|
||||
and `/var/backups/akyldash` remains a planned root-only location.
|
||||
|
||||
## Required release policy
|
||||
|
||||
- Create a backup before every production change, including any future schema
|
||||
or index migration.
|
||||
- Include the normalized corpus, `search-users.sqlite3`,
|
||||
`search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
|
||||
Caddy's certificate data, and required secret file in the protected backup
|
||||
set. Store secrets and Caddy private keys with equivalent access controls.
|
||||
- Verify archive readability, checksums, and SQLite integrity before relying
|
||||
on it. Never overwrite or delete the only usable backup.
|
||||
- Keep one current pre-release backup. If deployment fails, retain the backup
|
||||
required to restore the previous state.
|
||||
- Test restore into an isolated target before relying on the backup. Do not
|
||||
restore over production as a test.
|
||||
|
||||
## Clean first deployment
|
||||
|
||||
No backup is needed only when Compose has no containers and all four
|
||||
persistent mounts are confirmed empty. For a first install with a preloaded
|
||||
corpus and not-yet-created named volumes, use `--initial-deployment` to create
|
||||
the pre-start backup. This flag is accepted only with no Compose containers and
|
||||
when `DATA_ROOT` contains exactly `minjust-normalized/`; it archives absent
|
||||
named volumes as empty components. In every normal backup, an absent volume is
|
||||
an error because it could mean that a previous installation lost data.
|
||||
Missing mount definitions, inaccessible Docker state, or partial service
|
||||
containers are also errors and must stop deployment. Do not create placeholder
|
||||
containers to make backup pass.
|
||||
|
||||
For the corpus-only initial case, run this before the first `up` from the
|
||||
approved checkout. Replace the timestamp with one real UTC timestamp:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--initial-deployment \
|
||||
--output /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
```
|
||||
|
||||
## Cold backup procedure
|
||||
|
||||
The script requires every Compose container to be stopped. Schedule downtime,
|
||||
then run from the approved checkout on the VPS:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--output /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
```
|
||||
|
||||
Replace the timestamp placeholder with one real UTC timestamp before running.
|
||||
On a clean first deployment, `create` exits with a clear message that backup
|
||||
can be skipped. With existing data and all expected volumes present, it
|
||||
archives the storage even when no Compose containers exist. A preloaded corpus
|
||||
with no named volumes requires the explicit `--initial-deployment` mode;
|
||||
otherwise missing volumes are rejected. Partial service containers or running
|
||||
services are rejected as unsafe. The create command archives the application data,
|
||||
OpenSearch volume, Caddy data/config volumes, production environment file, and
|
||||
review secret. It records the checked-out Git revision, deployment-file hashes,
|
||||
and image references/IDs for each stopped Compose service, writes a SHA-256
|
||||
sidecar, and refuses to overwrite an existing archive. It also runs SQLite
|
||||
`quick_check`. Start the stack again only after verification succeeds:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
|
||||
```
|
||||
|
||||
## Initial post-setup baseline
|
||||
|
||||
After the first index has been loaded and validated, the initial `admin` account
|
||||
has been created, and `/review` responds over HTTPS, create a second backup.
|
||||
This one captures the initialized index, account database, and newly created
|
||||
Caddy/OpenSearch volumes. Stop all services, create and verify the archive,
|
||||
then start the stack again:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--output /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
|
||||
```
|
||||
|
||||
Use one real UTC timestamp in both archive paths. Do not treat initial
|
||||
production setup as complete until this baseline archive verifies. Keep the
|
||||
pre-start corpus archive as well until the baseline is verified and copied to
|
||||
separately managed encrypted storage.
|
||||
|
||||
Keep the local archive root-only, then copy it to separately managed encrypted
|
||||
storage. A local archive alone does not protect against loss of the VPS disk.
|
||||
Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
|
||||
`/var/tmp` and validates the checksum and SQLite databases. Applying the staged
|
||||
contents requires the separate recovery-target tool; it refuses the inventoried
|
||||
production host, existing data/volumes, and an unmarked host. Example isolated
|
||||
restore:
|
||||
|
||||
```bash
|
||||
sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
|
||||
--target-dir /var/tmp/akyldash-restore-test
|
||||
```
|
||||
|
||||
This validates and stages the archive. On the replacement host only, after
|
||||
running bootstrap with `--recovery-target` and checking out the matching Git
|
||||
revision, apply the staged data:
|
||||
|
||||
```bash
|
||||
sudo python3 infra/backup/apply_restore.py \
|
||||
--staging-dir /var/tmp/akyldash-restore-test
|
||||
```
|
||||
|
||||
The apply command restores data and named volumes but leaves services stopped.
|
||||
Test the full procedure on a separate recovery host before relying on it. Do
|
||||
not run an unreviewed restore over existing data.
|
||||
@@ -0,0 +1,215 @@
|
||||
# AkylDash Gitea Actions runner
|
||||
|
||||
## Scope and isolation
|
||||
|
||||
The deployment follows the official [Gitea 1.25 act_runner guide](https://docs.gitea.com/1.25/usage/actions/act-runner/)
|
||||
for repository-scoped registration. Gitea warns that access to a Docker daemon
|
||||
is root-equivalent; the dedicated rootless daemon is used instead of the
|
||||
production daemon. The pinned binary and checksum are published in the
|
||||
[official Gitea download directory](https://dl.gitea.com/gitea-runner/0.2.13/).
|
||||
|
||||
The runner is reserved to repository `admin/akyldash`, using the token under
|
||||
`https://git.vakanaut.ru/admin/akyldash/settings/actions/runners`. Do not use
|
||||
organization or instance runner settings. Pin the official `act_runner`
|
||||
`0.2.13` x86_64 binary from `dl.gitea.com`; the installer verifies its SHA-256
|
||||
before placing it on the host.
|
||||
|
||||
CI jobs run in `docker.gitea.com/runner-images:ubuntu-24.04` containers with
|
||||
capacity 1, per-job limits of 2 CPU, 3 GiB RAM and 512 processes, and a 15 GiB
|
||||
dedicated ext4 loop-backed Docker data filesystem. The runner account's home
|
||||
directory is root-owned mode `0711`, so CI cannot rename the image or mountpoint
|
||||
entries. Docker config remains under `.config`; registration state and the
|
||||
runner workspace live in `.config/act-runner`, which is runner-owned. The
|
||||
rootless Docker setup and user service use `.config/act-runner` as writable
|
||||
`HOME`, `.config` as `XDG_CONFIG_HOME`, and
|
||||
`.config/act-runner/.local/share` as `XDG_DATA_HOME`. This keeps Docker config
|
||||
and its systemd user unit in the existing `.config` tree without granting
|
||||
write access to the protected home.
|
||||
rootless Docker daemon
|
||||
and runner user are separate from production Docker; `codex` is not added to
|
||||
sudo or Docker groups. Jobs do not receive the rootless Docker socket, the
|
||||
production Docker socket, production environment/secrets, or arbitrary host
|
||||
bind mounts. The runner account has no access to `/etc/akyldash` or the
|
||||
root-owned application checkout. These are host permission boundaries, not a
|
||||
claim that untrusted project code is safe; review workflow changes and actions
|
||||
before merging them.
|
||||
|
||||
The runner service restarts on failure and boot. Systemd caps the runner
|
||||
account's shared slice to 2.5 CPU, 4 GiB RAM and 2048 tasks; the rootless
|
||||
daemon is separately capped to 2 CPU, 4 GiB and 1024 tasks, while each job has
|
||||
the limits above. Runner logs go to a rotated file (10 MiB, 7 copies). A
|
||||
systemd timer checks the runner service and the rootless Docker API each minute;
|
||||
on failure it restarts the fixed runner service. No UFW changes or public
|
||||
metrics ports are needed. Runner cache is disabled.
|
||||
|
||||
The CI workflow and `runner-smoke.yaml` target the repository runner label
|
||||
`akyldash-ci`. CI keeps the backend, Telegram, backup/restore, deployment,
|
||||
bootstrap, runner, Compose, and Caddy checks. Caddy validation runs in a pinned
|
||||
container action with explicit `/usr/bin/caddy` entrypoint. The workflow passes
|
||||
Gitea's `${{ github.workspace }}` value into each container action; Caddy and
|
||||
Kaniko use that supplied path for repository files instead of assuming a
|
||||
container workspace location. Both production
|
||||
Dockerfiles are built in a separate pinned Kaniko container action with explicit
|
||||
`/kaniko/executor` entrypoint using the runner's rootless container engine; the
|
||||
job container receives no Docker socket. Kaniko uses `--no-push` and
|
||||
`--no-push-cache`, so CI validates the full backend and OpenSearch builds but
|
||||
does not publish images or export build output. The two builds run sequentially
|
||||
within one job. CI run 69 completed both builds on the runner. Actual per-action
|
||||
resource-limit enforcement and peak memory or disk usage were not independently
|
||||
measured or verified. No registry credentials,
|
||||
production secrets, elevated container options, or arbitrary host mounts are configured.
|
||||
|
||||
The Kaniko action uses the `linux/amd64` v1.28.5-debug manifest
|
||||
`sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412`,
|
||||
listed in the [official GitHub Packages entry](https://github.com/osscontainertools/kaniko/pkgs/container/kaniko/1271366984?tag=v1.28.5-debug).
|
||||
The pinned Caddy image digest remains the same as the former `docker run`
|
||||
validation. act_runner starts these container actions; they do not expose its
|
||||
rootless Docker socket to workflow steps.
|
||||
Production deployment is not automated.
|
||||
|
||||
## Installation and registration
|
||||
|
||||
Review the root script and host audit output before running it. It installs
|
||||
`uidmap`, `dbus-user-session`, `curl`, `logrotate`, and Docker's rootless extras;
|
||||
creates the dedicated system user and selects the first free, shared 65536-ID
|
||||
range from `/etc/subuid` and `/etc/subgid`; reserves 15 GiB
|
||||
for runner images/workspaces; and configures systemd services. It starts only
|
||||
the separate rootless Docker daemon; the runner and health timer stay disabled
|
||||
until repository registration. It does not change
|
||||
SSH/UFW, production Docker configuration, application data, or production
|
||||
secrets. It can be rerun after partial setup: it validates and reuses an
|
||||
existing runner account, subordinate ID mappings, image, and mount unit. An
|
||||
existing image must be exactly 15 GiB and contain ext4; a mismatch stops the
|
||||
installer without resizing or formatting it. The runner account must retain
|
||||
a non-root system UID below `UID_MIN`, its expected home, primary group, and
|
||||
sole supplementary group. After validating the account, the installer locks
|
||||
its home directory to root ownership and mode `0711`; it refuses to rerun while
|
||||
the runner service is active, so it cannot interrupt a job. The mount
|
||||
is started only when it is not already active, and an active mount must be the
|
||||
expected ext4 loop device backed by the runner image with both `nosuid` and
|
||||
`nodev` active. Runner-home directories and user configuration files are
|
||||
created in runner-owned subdirectories. Root creates only the first `.config`
|
||||
entry beneath the already locked home; subsequent directory and config writes
|
||||
run as `akyldash-runner`. The installer rejects static symlinks and validates
|
||||
ownership and mode after setup. This also normalizes the `0755` intermediate
|
||||
directories left by the first installer version. Conflicting files or state
|
||||
stop the installer for manual inspection. A new image requires at least 25 GiB
|
||||
free on `/`. Image creation opens each parent component without following
|
||||
symlinks, creates the image exclusively relative to the opened directory, and
|
||||
passes the open file descriptor to `mkfs.ext4`; validation checks the same inode
|
||||
and rejects path replacement. Systemd mounts the image under the root-owned home
|
||||
entry, which the runner cannot replace after validation.
|
||||
|
||||
If the reviewed checkout is not present at the inventoried path, first verify
|
||||
that the target is absent and clone the reviewed `main` there. Do not overwrite
|
||||
an existing checkout.
|
||||
|
||||
```bash
|
||||
sudo test ! -e /opt/akyldash/current
|
||||
sudo git clone --branch main \
|
||||
--single-branch https://git.vakanaut.ru/admin/akyldash.git \
|
||||
/opt/akyldash/current
|
||||
```
|
||||
|
||||
From that root-owned reviewed checkout on the VPS:
|
||||
|
||||
```bash
|
||||
sudo bash /opt/akyldash/current/infra/runner/install-runner.sh
|
||||
```
|
||||
|
||||
Then, as a repository owner, open repository **Settings → Actions → Runners**,
|
||||
create a repository-level runner, and copy its registration token locally. Do
|
||||
not send the token in chat or save it in the checkout. Register interactively
|
||||
in a terminal so it is not placed in shell history or process arguments:
|
||||
|
||||
```bash
|
||||
sudo -u akyldash-runner env HOME=/var/lib/akyldash-runner/.config/act-runner \
|
||||
/usr/local/bin/act_runner \
|
||||
--config /etc/akyldash-runner/config.yaml register
|
||||
```
|
||||
|
||||
Enter `https://git.vakanaut.ru/`, the token, runner name `akyldash-ci-vps-01`,
|
||||
and label `akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04`
|
||||
when prompted. The config sets the label and capacity; verify them in the
|
||||
repository runner settings. The token is saved only in the mode-0700 runner
|
||||
runtime directory, outside Git. Start/enable the runner after registration:
|
||||
|
||||
```bash
|
||||
sudo systemctl enable --now akyldash-act-runner.service
|
||||
sudo systemctl enable --now akyldash-runner-healthcheck.timer
|
||||
sudo systemctl status akyldash-act-runner.service
|
||||
sudo systemctl status akyldash-runner-healthcheck.timer
|
||||
```
|
||||
|
||||
The interactive command's terminal behavior should be confirmed before pasting
|
||||
the token. If it echoes input, stop and use the repository settings to revoke
|
||||
that token; do not put it in command-line arguments or environment variables.
|
||||
|
||||
## Validation
|
||||
|
||||
1. Confirm the runner is **Online** in the repository runner settings, with the
|
||||
expected name and `akyldash-ci` label.
|
||||
2. Confirm **Runner smoke test** runs for the open PR (or dispatch it manually).
|
||||
It checks that production
|
||||
sockets, files, environment secrets, and writable application paths are not
|
||||
present in the job container.
|
||||
3. Confirm the run reports the expected runner, succeeds, and the runner
|
||||
service and health timer remain active.
|
||||
4. Inspect the `CI` workflow run and confirm Caddy plus both Kaniko actions pass
|
||||
on `akyldash-ci`. The Kaniko jobs are intentionally build-only: publishing
|
||||
images requires a separately reviewed registry workflow and credentials.
|
||||
Do not add production deploy workflows or secrets to this runner.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update `runner_version` and its expected official SHA-256 together in
|
||||
`infra/runner/install-runner.sh`, review upstream release compatibility with
|
||||
the installed Gitea, and open a PR. Do not replace the binary while a job is
|
||||
active. Stop the runner, install/verify the new binary, then start it and
|
||||
confirm Online plus a smoke run. Preserve
|
||||
`/var/lib/akyldash-runner/.config/act-runner/.runner`.
|
||||
|
||||
## Recovery and rollback
|
||||
|
||||
The runner is disposable: production data and secrets are not on its filesystem.
|
||||
For a runner process failure, inspect `systemctl status` and
|
||||
`journalctl -u akyldash-act-runner.service`; restart the fixed runner service.
|
||||
If installation stops after creating or enabling the mount unit, inspect the
|
||||
image and mount with `stat`, `blkid`, `findmnt`, and `losetup`, then rerun the
|
||||
reviewed installer only when the image is 15 GiB ext4 and the active mount is
|
||||
backed by it. The installer recognizes that state and does not format or
|
||||
restart the mounted filesystem. If size, filesystem, unit contents, or backing
|
||||
device differ, stop and resolve the state manually; never format the existing
|
||||
image as a retry.
|
||||
For lost/corrupt registration state, stop the service and obtain a fresh
|
||||
repository-level token from Settings → Actions → Runners. Preserve the 15 GiB
|
||||
data image and state for diagnosis; re-register only after the repository
|
||||
owner confirms/revokes the old runner in Gitea. The images/cache can be rebuilt
|
||||
from the workflow. Never format or remove an existing runner store as an
|
||||
automatic recovery step.
|
||||
|
||||
## Status
|
||||
|
||||
The owner reports that the repository-scoped runner is registered, appears
|
||||
Idle in Gitea, and its rootless Docker daemon works; the report date was not
|
||||
recorded. Gitea status checked 2026-10-18 showed smoke run 62 successful and CI
|
||||
run 61 pending on head `69c62466`. That workflow requested `ubuntu-latest`,
|
||||
while the smoke workflow used `akyldash-ci`; the label difference is a
|
||||
plausible explanation for the pending status, but the runner inventory API
|
||||
returned 403 and the scheduler's reason was not independently confirmed.
|
||||
|
||||
On head `3a89dc21`, CI run 63 failed at the host audit root guard and smoke run
|
||||
64 succeeded. On head `436806a8`, CI run 65 failed in the Caddy container action
|
||||
with `exec: "validate": executable file not found`; Kaniko was skipped and smoke
|
||||
run 66 succeeded. On head `d96fc4d3`, CI run 67 failed because the Caddy action
|
||||
looked for `/github/workspace/deploy/production/Caddyfile`, which was not
|
||||
mounted at that fixed path; Kaniko was skipped and smoke run 68 succeeded. On
|
||||
head `5d72bb09`, CI run 69 and smoke run 70 succeeded. CI validated the Caddyfile
|
||||
and built the backend and OpenSearch images using `${{ github.workspace }}`.
|
||||
Actual per-action resource-limit enforcement and peak usage were not
|
||||
independently measured or verified. Image publishing, host-wide resource
|
||||
metrics, and production deployment remain unverified. Root-guard tests
|
||||
use `setpriv` for negative cases when jobs start as UID 0 and separately test
|
||||
the host-audit guard with simulated UID 0 and UID 65534; the real host audit,
|
||||
production config setup, and runner installer are never executed as root by
|
||||
those tests. No VPS changes are part of this PR.
|
||||
@@ -0,0 +1,131 @@
|
||||
# Corpus inventory and production handoff
|
||||
|
||||
The repository contains a read-only verifier for the ignored local corpus at
|
||||
`data/minjust-cbd/` and `data/minjust-normalized/`. It does not download,
|
||||
normalize, copy, index, or modify source files. Keep inventory artifacts outside
|
||||
the repository and both source trees.
|
||||
|
||||
## Run the inventory
|
||||
|
||||
From the repository root, create a private artifact directory and run:
|
||||
|
||||
```bash
|
||||
artifact_dir=$(mktemp -d /tmp/akyldash-corpus-audit.XXXXXXXX)
|
||||
python3 infra/deploy/verify_corpus.py \
|
||||
--raw-root data/minjust-cbd \
|
||||
--normalized-root data/minjust-normalized \
|
||||
--output-dir "$artifact_dir"
|
||||
```
|
||||
|
||||
The tool writes `result.json`, `inventory.sqlite3`, and a persistent lock file
|
||||
into that directory. Keep the directory private: the inventory contains source
|
||||
paths and checksums. A second run against the same artifact directory is
|
||||
rejected while a scan is active. Exit
|
||||
status `0` means the scan and structural checks completed without errors; `1`
|
||||
means findings were recorded or the run is incomplete; `2` means the arguments
|
||||
or artifact path were invalid. Read `result.json` to distinguish those states.
|
||||
Warnings, including documents with zero text fragments, are reported
|
||||
separately and do not turn the scan into a failure.
|
||||
|
||||
For an interrupted run, use the same artifact directory and add `--resume`.
|
||||
Resume rehashes every source file instead of trusting cached timestamps, then
|
||||
reports added, changed, and removed paths relative to entries already stored
|
||||
in the inventory. After resuming an interrupted first pass, those counters are
|
||||
relative to the partial saved inventory; the full rehash and final checks still
|
||||
cover every path. A successful result therefore describes the latest complete
|
||||
pass. Run against a quiescent corpus; do not normalize, fetch, or copy files
|
||||
into either source tree during the scan. Do not point `--output-dir` inside
|
||||
either source tree or through a symlink. The verifier rejects such paths before
|
||||
creating or changing them; it also flags symlinks, reparse points, hard links,
|
||||
unsupported file types, and files that change while being read.
|
||||
|
||||
The inventory database schema is versioned. Resume requires the same root
|
||||
metadata and verifier SHA-256 recorded in `result.json`; start a new artifact
|
||||
directory after changing the verifier. `entries` counts files plus directories;
|
||||
`files` and `directories` are separate counts. Document-prefix checks use
|
||||
indexed path ranges, and stale-path reporting streams rows from SQLite.
|
||||
|
||||
The audit checks SQLite integrity, manifest/document directory agreement,
|
||||
normalized schema and normalizer versions, source inventory checksums,
|
||||
fragment identities and text hashes, required output files, and text assembled
|
||||
from fragments. It does not establish the legal authority, completeness, or
|
||||
semantic correctness of MinJust content.
|
||||
|
||||
## Compare source and staging before transfer
|
||||
|
||||
Run a complete inventory of the approved normalized source and the copied
|
||||
staging tree into separate private artifact directories, using
|
||||
`--normalized-root` for each. Require `status: complete` and
|
||||
`verification_status: passed` in both results. The comparison command also
|
||||
requires each manifest to contain at least one document and each normalized
|
||||
tree to contain at least one fragment; an empty source or staging tree is a
|
||||
failure.
|
||||
|
||||
```bash
|
||||
python3 infra/deploy/compare_corpus_inventories.py \
|
||||
"$source_artifacts/result.json" "$staging_artifacts/result.json"
|
||||
```
|
||||
|
||||
The checker streams every relative path, size, and SHA-256 from both SQLite
|
||||
inventories, and requires the same verifier/schema version and different
|
||||
normalized roots. It opens both existing private `.inventory.lock` files
|
||||
read-only and holds shared nonblocking locks through comparison; a missing,
|
||||
unsafe, or busy lock fails closed. A zero exit status confirms a non-empty
|
||||
staging tree matches the source. Keep both trees quiescent during their scans
|
||||
and comparison. This is an equality check only; the owner still reviews the
|
||||
code `6` disposition and approves the transfer destination.
|
||||
|
||||
## Current data disposition
|
||||
|
||||
As measured on 2026-10-18, both local manifests contain 209,958 documents and
|
||||
pass SQLite `quick_check`. All normalized manifest entries are `success` at
|
||||
schema `1` and normalizer `1.0.0`. The raw manifest records one fetch error for
|
||||
code `6`, which is absent from its `documents` table; this does not prove that
|
||||
the source document is absent. On 2026-10-18, bounded calls through the
|
||||
repository's rate-limited MinJust client received HTTP `404` for
|
||||
`GetDocumentListByQuery`, `GetDocument(6)`, control `GetDocument(1096)`, and
|
||||
`CheckAvailable`. The listing call failed before returning a code list, and
|
||||
both document calls failed identically. The repeated `404` is consistent with
|
||||
the OpenData endpoint being unavailable or its route having changed, but does
|
||||
not identify which. It is not evidence that code `6` is absent. The [public
|
||||
document listing page](https://cbd.minjust.gov.kg/list-docs/kg) was reachable,
|
||||
but it does not confirm this numeric API code. Do not describe the corpus as
|
||||
fully verified or silently waive this error. Recheck the official listing and
|
||||
API when the endpoint responds, then record an explicit owner disposition.
|
||||
From the VPS, a metadata-only HTTPS check to `CheckAvailable` failed at the
|
||||
HTTP/2 transport layer (`curl` exit 92, `INTERNAL_ERROR`, HTTP status `000`,
|
||||
TLS verification `0`). An HTTP/1.1 retry was not completed.
|
||||
|
||||
The first full pass (run `b4a4556e30cc464d90bc8d75ef1673e9`) stopped before
|
||||
validation with `Python int too large to convert to SQLite INTEGER` after
|
||||
2,120,420 raw entries and 397,856 normalized entries. This was an inventory
|
||||
metadata storage failure, not a corpus finding. The verifier now stores file
|
||||
identity values as text and records its own SHA-256. A complete rehash with the
|
||||
updated verifier is required; do not treat the failed pass or progress output
|
||||
as a completed audit.
|
||||
|
||||
A fresh full rehash is now running in `/tmp/akyldash-corpus-audit-final.gtsD87vc`,
|
||||
run `f4dd5e53020f442bbbecee2d0e4986a6`, verifier SHA-256
|
||||
`83dac230099fc91972cd14e86e9f8ec72d13bca08f53d402f13a169e096d98f5`. Check
|
||||
its private `result.json` for final counters, findings, and manifest comparison.
|
||||
|
||||
## Production transfer and indexing gates
|
||||
|
||||
Production deployment is not enabled. Before any transfer, the owner must
|
||||
review the root-only host audit, confirm the target data location is empty or
|
||||
backed up, resolve the raw code `6` finding, and approve the exact normalized
|
||||
corpus to use. Transfer only `minjust-normalized/`; keep the raw source archive
|
||||
offline. Do not preserve the source's Windows/WSL file permission bits. Copy
|
||||
to a new staging location with owner-approved permissions, verify the copied
|
||||
manifest and files there, and only then consider placing it under `DATA_ROOT`.
|
||||
Never overwrite an existing corpus or OpenSearch volume.
|
||||
|
||||
The current bulk importer checks individual Bulk API item results, but it can
|
||||
switch an alias after a zero-fragment input and does not enforce a refresh,
|
||||
count comparison, or representative-query check before that switch. Until a
|
||||
separate change adds those gates, do not pass `--alias` during production
|
||||
loading. Load into a new versioned index only after the corpus transfer is
|
||||
verified; require a nonzero fragment count, successful item results, an
|
||||
explicit refresh, a matching OpenSearch document count, and representative
|
||||
Russian/Kyrgyz queries before an owner performs an alias change. Take and
|
||||
verify the required backup before any production index or service change.
|
||||
@@ -0,0 +1,217 @@
|
||||
# Infrastructure current state
|
||||
|
||||
Read-only host recheck: 2026-10-18. Source: SSH as `codex`, local repository
|
||||
inspection, and Gitea API. Root-only Docker state, UFW rule details, and
|
||||
protected file metadata remain unverified.
|
||||
|
||||
## VPS facts
|
||||
|
||||
- Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`.
|
||||
- OS: Ubuntu Server `24.04.5 LTS`; kernel `6.8.0-142-generic` from the prior
|
||||
inventory audit.
|
||||
- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB root filesystem with
|
||||
84 GiB free on 2026-10-18. Runner storage is a separate 15 GiB ext4 loop
|
||||
mount with 12 GiB free at that check.
|
||||
- Network: interface `ens3`, IPv4 `206.223.249.118/32`.
|
||||
- `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires
|
||||
a password. Docker CLI cannot access `/var/run/docker.sock` as `codex`.
|
||||
- SSH service is running. The inspected sshd configuration has public-key and
|
||||
password authentication enabled and `PermitRootLogin yes`; no SSH changes
|
||||
were made.
|
||||
- Bootstrap completed. Docker CLI `29.9.0` and Compose `5.6.0` were rechecked
|
||||
over SSH on 2026-10-18; containerd `2.4.1` and runc `1.5.2` are from the
|
||||
prior package audit. The Docker daemon API remains root-only.
|
||||
- `/etc/docker/daemon.json` has the expected `local` log driver and `10m` / `3`
|
||||
rotation limits. Bootstrap paths exist with root ownership: `/opt/akyldash`
|
||||
mode `0755`, `/etc/akyldash` and `/var/backups/akyldash` mode `0700`, and
|
||||
`/var/lib/akyldash/data` mode `0750`.
|
||||
- `/opt/akyldash/current`, verifier/rollback binaries, and the sudoers file are
|
||||
absent. `codex` has not received sudo or Docker-group access. Metadata for
|
||||
`/etc/akyldash/production.env` and `/etc/akyldash/review_secret` could not be
|
||||
checked because access is denied. The target
|
||||
`/var/lib/akyldash/data/minjust-normalized` and its manifest metadata were
|
||||
also denied; their presence, ownership, and contents are unknown. No sudo was
|
||||
used.
|
||||
- TCP listeners observed: SSH `22` and loopback DNS only; no listener on `1500`,
|
||||
`80`, or `443`. No ISPmanager package or service unit was found through the
|
||||
unprivileged checks; root can confirm with the audit script below.
|
||||
- Owner reports UFW is active and allows TCP `22`, `80`, `443`, and `1500`.
|
||||
Rule details were not independently read; no firewall changes were made.
|
||||
- Additional read-only runner checks on 2026-10-12 confirmed `/` is ext4 with
|
||||
about 92 GiB free at the prior audit, `/usr/bin/dockerd-rootless.sh` exists,
|
||||
and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS,
|
||||
`https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner
|
||||
registration and Gitea Actions protocol connectivity are not confirmed.
|
||||
- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS,
|
||||
HTTPS returned Cloudflare `521` with TLS verification result `0` on
|
||||
2026-10-18. TCP listeners were SSH `22` and local DNS only, so no origin
|
||||
listener on `80`/`443` was observed. Origin TLS and Cloudflare Full (strict)
|
||||
are not verified.
|
||||
|
||||
## Application and Gitea facts
|
||||
|
||||
- Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`.
|
||||
- The repository API reports push permission, Actions and packages enabled, and
|
||||
PRs enabled. `main` is protected and requires one approval; the current API
|
||||
principal cannot push or merge directly to `main`.
|
||||
- The repository runner-list API returns `403` because this API principal is
|
||||
not the repository owner. The latest verified CI run 75 and smoke run 76 both
|
||||
succeeded on `main` head `0e276ae6`. The Actions run-list endpoint returned
|
||||
`403` for this API principal during this recheck, so no newer run state was
|
||||
independently retrieved. The owner reports the runner is registered and
|
||||
Idle; repository runner inventory and package publishing permissions remain
|
||||
unverified.
|
||||
- No public application listener is observed on TCP `80`/`443`. Docker
|
||||
container/volume state still requires the root audit. Repository Compose describes
|
||||
Caddy `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite
|
||||
account/review data, and OpenSearch `3.7.0` with `analysis-icu`.
|
||||
- The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a
|
||||
proxy network; OpenSearch and backend share a separate internal network, and
|
||||
backend has no host port. Caddy certificate storage, OpenSearch, and
|
||||
application data use persistent volumes/bind mounts.
|
||||
- The root-owned repository checkout, production env/secret files, Compose
|
||||
volumes/containers, backup, verifier/rollback helpers, and application are
|
||||
not confirmed installed. TLS and registry digest deployment for
|
||||
backend/OpenSearch are not configured.
|
||||
- The repository has a cold-backup archive/verification tool and a one-shot
|
||||
restore apply tool that requires a marked replacement host. Backup/restore
|
||||
tests use synthetic data; neither tool has run against production. The
|
||||
archive preflight permits a clean first deployment to skip backup, requires
|
||||
backup for existing persistent data even without containers, allows absent
|
||||
named volumes only in explicit corpus-only initial-deployment mode, and
|
||||
rejects inaccessible/ambiguous mounts and incomplete/running service
|
||||
containers. Initial deployment docs require a second verified backup after
|
||||
index and admin initialization.
|
||||
- Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
|
||||
inventoried production hostname/IP; its host-identity guard has shell tests.
|
||||
The mode has not been run on a VPS.
|
||||
- The repository defines a root-owned read-only verifier, Compose `config` and
|
||||
`status` helper, guarded rollback helper, and installer with exact sudoers
|
||||
command rules. None is installed on the VPS. CI contains no production
|
||||
deployment job; starting/stopping services remains owner-run.
|
||||
- Rollback preflight validates digest-pinned image references against archived
|
||||
image IDs and local availability, then refuses before stopping services or
|
||||
creating a backup because the current flow cannot safely switch immutable
|
||||
digest references. This behavior is covered by synthetic local tests; it has
|
||||
not been exercised on the VPS.
|
||||
- This branch adds an offline release-manifest validator and tests for the
|
||||
repository allowlist intended for a future root-owned host integration. The
|
||||
manifest format does not prove source provenance. No registry, host policy,
|
||||
publish credential, or deployment integration is configured or installed.
|
||||
- A repository-scoped official `act_runner` `0.2.13` installation is
|
||||
configured in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB
|
||||
ext4 loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU,
|
||||
3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048
|
||||
tasks), rotated file logs, and a systemd health timer. Jobs target the
|
||||
`akyldash-ci` label and do not receive either Docker socket. On 2026-10-18,
|
||||
SSH showed `akyldash-act-runner.service` active under `akyldash-runner`, with
|
||||
zero restarts and about 115 MiB current memory. Its rootless Docker store is
|
||||
mounted from `/dev/loop0` as ext4 with `rw,nosuid,nodev`; the 15 GiB mount had
|
||||
12 GiB free. The separate `akyldash-runner-restart.service` remains failed
|
||||
with exit status 1 since 2026-10-14; its journal had no entries, so the cause
|
||||
is unknown. The latest verified main CI run 75 and smoke run 76 succeeded on
|
||||
head `0e276ae6`. The Actions API returned 403 during this recheck, so no newer
|
||||
run state was independently retrieved. On
|
||||
head `3a89dc21`, CI run 63 failed at the host audit root guard and smoke run 64
|
||||
succeeded. On head `436806a8`, CI run 65 failed in the Caddy container action
|
||||
with `exec: "validate": executable file not found`; Kaniko was skipped and
|
||||
smoke run 66 succeeded. On head `d96fc4d3`, CI run 67 failed because the
|
||||
Caddy action's fixed `/github/workspace/...` path did not exist; Kaniko was
|
||||
skipped and smoke run 68 succeeded. On head `5d72bb09`, CI run 69 and smoke
|
||||
run 70 succeeded. CI validated the Caddyfile and built the backend and
|
||||
OpenSearch images using `${{ github.workspace }}`. Actual per-action
|
||||
resource-limit enforcement and peak usage were not independently measured or
|
||||
verified. Image publishing, host-wide resource metrics, and production
|
||||
deployment remain unverified. No VPS changes are made.
|
||||
- An isolated local Compose smoke test built backend/OpenSearch images,
|
||||
reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
|
||||
200 for `/review` through Caddy. Its containers, networks, and volumes were
|
||||
removed afterward. This does not verify VPS startup, DNS, or public TLS.
|
||||
- The app has no migration framework or SMTP setup. Current app versions:
|
||||
backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
|
||||
deployed on this VPS).
|
||||
|
||||
## Corpus readiness (2026-10-18)
|
||||
|
||||
- The local raw archive and normalized corpus are ignored under `data/`; no
|
||||
corpus files were changed. Their read-only SQLite manifests both pass
|
||||
`quick_check` and each report 209,958 documents. The raw manifest is
|
||||
28,295,168 bytes with SHA-256
|
||||
`f8e3dd2fdec607583ca09cfdb5b76171bb7bfa1e41a0e07ee5aca8481e18e17e`; it has
|
||||
one fetch error, code `6`, absent from `documents`. The normalized manifest
|
||||
is 31,571,968 bytes with SHA-256
|
||||
`1679c20e6d0f3e5a9b50737c006c4417a28ebcaf7467d194d3bf8243538f83bf`; all
|
||||
entries are `success`, schema `1`, normalizer `1.0.0`.
|
||||
- The read-only full-file inventory in `/tmp/akyldash-corpus-audit.Mjr6aa`
|
||||
stopped before validation in run `b4a4556e30cc464d90bc8d75ef1673e9` with
|
||||
`Python int too large to convert to SQLite INTEGER`. At stop it had processed
|
||||
2,120,420 raw entries (1,081,428 files, 1,038,992 directories) and 397,856
|
||||
normalized entries (246,321 files, 151,535 directories). This is a verifier
|
||||
metadata storage failure, not a corpus finding. A full rehash with the
|
||||
corrected, versioned verifier is required before readiness can be stated.
|
||||
- A fresh full rehash is running in `/tmp/akyldash-corpus-audit-final.gtsD87vc`,
|
||||
run `f4dd5e53020f442bbbecee2d0e4986a6`, with verifier SHA-256
|
||||
`83dac230099fc91972cd14e86e9f8ec72d13bca08f53d402f13a169e096d98f5`.
|
||||
`result.json` currently reports `scan_status: running`; its live counters
|
||||
must be recorded after completion. Sources remain read-only.
|
||||
- Through the repository client, bounded MinJust OpenData requests for
|
||||
`GetDocumentListByQuery`, `GetDocument(6)`, control `GetDocument(1096)`, and
|
||||
`CheckAvailable` each returned HTTP `404`. The list API returned no code list;
|
||||
the direct requests do not establish whether code `6` exists. The public
|
||||
listing page is reachable, but it does not map the numeric API code. A
|
||||
metadata-only `CheckAvailable` request from the VPS failed in HTTP/2 with
|
||||
`curl` exit 92 (`INTERNAL_ERROR`), HTTP status `000`, and TLS verification 0;
|
||||
the HTTP/1.1 retry was not completed. Treat the code `6` disposition and
|
||||
full corpus readiness as unresolved.
|
||||
- On the VPS, read-only `stat` of `/var/lib/akyldash/data/minjust-normalized`
|
||||
and its manifest was denied. Production corpus presence and metadata remain
|
||||
unknown. No corpus was transferred and no production service, Docker socket,
|
||||
runner policy, or VPS configuration was changed.
|
||||
|
||||
## Checks not completed
|
||||
|
||||
- Docker Engine version from the server API, container and volume inventory,
|
||||
protected env/secret metadata, sudoers, and detailed UFW rules require root.
|
||||
`infra/bootstrap/audit-host.sh` is provided for an owner-run read-only audit.
|
||||
- The backup archive/restore tool was tested with synthetic local data; a backup
|
||||
and restore of real production data remain unverified because production is
|
||||
not deployed.
|
||||
- Cloudflare edge response was checked; origin DNS target, origin certificate,
|
||||
Full (strict), registry publishing, host-wide runner metrics, and production
|
||||
deployment have not been verified. The
|
||||
owner-reported Idle state has not been independently checked through runner
|
||||
inventory because the API returned 403.
|
||||
- This audit made no host changes. No package or service was installed by this
|
||||
audit; the owner performed the initial bootstrap beforehand.
|
||||
|
||||
## Next owner actions
|
||||
|
||||
1. Install the approved root-owned checkout after its PR is reviewed, then run
|
||||
`sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh` and share
|
||||
the output. The script prints no secret values and does not mutate the host.
|
||||
2. After reviewing the audit output and confirming the target files are absent,
|
||||
run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`;
|
||||
it refuses to overwrite existing files. Set the approved `DATA_ROOT` and
|
||||
place the approved corpus/index and Cloudflare origin configuration. Do not
|
||||
deploy until data/service state and the required backup path are reviewed.
|
||||
3. Run `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh` after
|
||||
reviewing the checkout; it grants only fixed verification, Compose
|
||||
`config`/`status`, and guarded rollback commands.
|
||||
4. After merging the reviewed PR, confirm a subsequent CI and smoke run on
|
||||
`akyldash-ci`. Image publishing and production deployment remain manual and
|
||||
are not part of CI.
|
||||
5. Test backup/restore and rollback only with synthetic data in an isolated
|
||||
environment before any production deployment.
|
||||
|
||||
## Change in this PR
|
||||
|
||||
Adds a read-only, resumable full-file corpus inventory and normalized-data
|
||||
cross-check, a source/staging comparator requiring nonzero documents and
|
||||
fragments plus held shared artifact locks, synthetic regression tests, and an
|
||||
operator handoff runbook. CI's
|
||||
existing `infra/deploy` unittest discovery includes the new tests. No workflow,
|
||||
runner, VPS, production data, or Docker settings are changed. The first real
|
||||
corpus scan stopped on an integer-storage bug; full validation with the
|
||||
corrected verifier is still required. The source API's code `6` outcome remains
|
||||
unconfirmed.
|
||||
@@ -0,0 +1,10 @@
|
||||
# Infrastructure decisions
|
||||
|
||||
| Date | Decision | Reason and alternatives | Status / limits |
|
||||
|---|---|---|---|
|
||||
| 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
|
||||
| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only a fixed read-only verifier and root-owned rollback helper after checkout ownership and sudoers validation | Helpers are defined in Git; not installed on VPS |
|
||||
| 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
|
||||
| 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
|
||||
| 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
|
||||
| 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |
|
||||
@@ -0,0 +1,126 @@
|
||||
# Deployment
|
||||
|
||||
## Current status
|
||||
|
||||
Production deployment is not enabled or tested on the VPS. The owner completed
|
||||
the root bootstrap: Docker/Compose are active and target directories exist.
|
||||
The `codex` account cannot access the Docker socket. No root-owned checkout or
|
||||
restricted helpers are installed; protected env/secret metadata and Docker
|
||||
container/volume state remain unverified because they require root.
|
||||
The unprivileged audit sees no listener on `80`, `443`, or `1500`; Cloudflare
|
||||
still returns `521`. Do not deploy until the owner has reviewed root-only host
|
||||
state, supplied protected configuration and the approved corpus/index, and
|
||||
backup/restore have been tested.
|
||||
|
||||
## First host preparation
|
||||
|
||||
1. The initial root bootstrap is complete. After reviewing and merging the
|
||||
infrastructure PR, the owner must install that approved `main` checkout at
|
||||
`/opt/akyldash/current` (the path was absent during the unprivileged audit):
|
||||
|
||||
```bash
|
||||
sudo git clone --branch main --single-branch https://git.vakanaut.ru/admin/akyldash.git /opt/akyldash/current
|
||||
```
|
||||
|
||||
The checkout is root-owned by default. Then run
|
||||
`sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh`. This
|
||||
read-only audit reports Docker state, protected-file metadata, UFW rules,
|
||||
listeners, and ISPmanager without printing secret contents. Resolve any
|
||||
existing containers, volumes, or data before proceeding; do not overwrite
|
||||
or delete them.
|
||||
2. After confirming the existing env and secret do not exist and reviewing the
|
||||
tracked defaults, run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`.
|
||||
It creates `/etc/akyldash/production.env` from the tracked example and
|
||||
generates `/etc/akyldash/review_secret`; both are root-owned mode `0600`.
|
||||
It refuses to overwrite either file and does not start services or create
|
||||
volumes. Review `DATA_ROOT` and follow the
|
||||
[corpus inventory and handoff gates](CORPUS_HANDOFF.md) before placing the
|
||||
approved normalized corpus there.
|
||||
3. After reviewing the root-owned checkout and audit output, run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh`. It installs
|
||||
the backup reader, read-only verifier, `akyldash-compose config|status`, and
|
||||
rollback helper with exact sudoers command rules. The Compose helper has no
|
||||
start/stop/build operation and grants no direct Docker access. Actual
|
||||
deployments remain owner-run through the documented Compose commands.
|
||||
4. Confirm DNS points through Cloudflare to this origin. The owner reports UFW
|
||||
already allows TCP `80` and `443`; read and document the full rules before
|
||||
any firewall decision. TCP `1500` is allowed by owner report but has no
|
||||
observed listener and no ISPmanager package/service was found; the owner
|
||||
must identify its purpose. Do not change UFW as part of this procedure. The
|
||||
Compose Caddy service obtains and renews HTTPS certificates automatically;
|
||||
Cloudflare Full (strict) still needs end-to-end verification.
|
||||
5. For a clean first deployment, skip backup only when Compose confirms there
|
||||
are no containers and all persistent mounts are empty. If any data exists,
|
||||
create and verify a backup before starting a new version. For a preloaded
|
||||
corpus with absent named volumes, use `--initial-deployment`; it is limited
|
||||
to a container-free host whose `DATA_ROOT` contains only
|
||||
`minjust-normalized/`. Normal backups reject absent volumes. An uncertain
|
||||
or partial state stops deployment. Then start services, load and validate
|
||||
the OpenSearch index, create the initial admin interactively, verify
|
||||
`/review` over HTTPS, and create the required post-setup baseline backup
|
||||
described in `BACKUP.md`. Record the backend image ID for drift checks:
|
||||
|
||||
```bash
|
||||
sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
|
||||
sudo chmod 0600 /opt/akyldash/deployed-image.txt
|
||||
```
|
||||
|
||||
Application deployment remains owner-run. The repository-scoped `act_runner`
|
||||
`0.2.13` service is active under `akyldash-runner` and uses rootless Docker on a
|
||||
separate loop-backed filesystem. CI run 75 and smoke run 76 succeeded on main
|
||||
head `0e276ae6`. Runner job containers have neither Docker socket nor production
|
||||
secrets. CI has no image publishing or production deployment step. The
|
||||
repository defines sudo helpers for fixed verification, read-only Compose
|
||||
`config`/`status`, and guarded rollback, but none is installed on the VPS. They
|
||||
do not start or stop services.
|
||||
|
||||
## Release image contract (preparation only)
|
||||
|
||||
`infra/deploy/validate_release.py` checks an offline JSON release manifest
|
||||
against an explicitly supplied repository allowlist. Schema version 1 requires
|
||||
exactly `schema_version`, `source_ref`, `commit`, and `images`; the source ref
|
||||
must be `refs/heads/main`, the commit must be a 40-character lowercase Git
|
||||
SHA-1, and the images must contain exactly `backend` and `opensearch` as
|
||||
`repository@sha256:<64 lowercase hex>` references. The policy has schema
|
||||
version 1 and one non-empty repository allowlist for each service. Unknown or
|
||||
duplicate JSON keys are rejected.
|
||||
|
||||
The validator takes the manifest path and an explicit policy path:
|
||||
|
||||
```bash
|
||||
python3 infra/deploy/validate_release.py release.json --policy /etc/akyldash/release-policy.json
|
||||
```
|
||||
|
||||
The policy shape is `{"schema_version":1,"repositories":{"backend":["registry.invalid/akyldash/backend"],"opensearch":["registry.invalid/akyldash/opensearch"]}}`.
|
||||
These `registry.invalid` names are format examples only; this change does not
|
||||
create a policy file or approve a registry.
|
||||
|
||||
The repository name in the policy must be approved for the registry. No real
|
||||
registry namespace or root-owned host policy is configured by this change; do
|
||||
not use the synthetic `registry.invalid` names in tests as production values.
|
||||
The host integration must load policy only from its fixed root-owned path.
|
||||
This validator checks format and allowlist only: a manifest can claim a main
|
||||
ref and commit, but is not proof of source, build provenance, signature, or
|
||||
trusted publication. CI still has no publish credentials, image publication,
|
||||
host helper, or deploy trigger.
|
||||
|
||||
Compose merges an image override with the existing service `build` entries.
|
||||
The regression test renders fake digest references and confirms both entries
|
||||
remain; any future deploy command must use `docker compose up --no-build` so
|
||||
the host cannot silently build a different image. This does not make digest
|
||||
rollback available: the current rollback helper expects local tags. A separate
|
||||
host-side deploy/rollback design, owner-approved registry permissions and
|
||||
retention, and manual backup/health/rollback end-to-end validation are still
|
||||
required before publishing or enabling CD.
|
||||
|
||||
## CI
|
||||
|
||||
`.gitea/workflows/ci.yaml` runs the Python unit suites and builds both
|
||||
production images on pull requests and pushes to `main`. It does not publish an
|
||||
image or deploy production. `.gitea/workflows/runner-smoke.yaml` exercises the
|
||||
`akyldash-ci` runner on pull requests, pushes to `main`, or manual dispatch.
|
||||
Both workflows passed on main head `0e276ae6` in runs 75 and 76 respectively.
|
||||
The repository runner-list endpoint is not available to the current API
|
||||
principal, so its registration details cannot be independently read through
|
||||
Gitea API.
|
||||
@@ -0,0 +1,59 @@
|
||||
# Infrastructure
|
||||
|
||||
## Intended layout
|
||||
|
||||
Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
|
||||
`deploy/production/compose.yaml` defines Caddy, a Python Search API, and private
|
||||
OpenSearch 3.7. Caddy terminates HTTPS for `akyldash.kg` and proxies to the API.
|
||||
The API is reachable only on private Compose networks; OpenSearch is reachable
|
||||
only by the backend. Cloudflare should use Full (strict) only after the origin
|
||||
certificate and DNS path are verified.
|
||||
|
||||
The API stores its account and review SQLite files beside the normalized corpus
|
||||
under the persistent data directory. OpenSearch and Caddy use named volumes.
|
||||
The API's review signing key is mounted from a host file; it is not passed in
|
||||
process arguments or committed to Git. Docker logs are bounded to 10 MiB per
|
||||
file and three files per service.
|
||||
|
||||
## Actual state
|
||||
|
||||
The 2026-10-11 read-only audit found a fresh Ubuntu 24.04.5 host with SSH only.
|
||||
Docker, Git, a reverse proxy, application directories, Compose services, TLS,
|
||||
and backups are not installed/configured. The `codex` account has no
|
||||
non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
|
||||
[production inventory](../../infra/inventory/production.yml).
|
||||
|
||||
The Compose file is a repository target, not proof of deployment. The Caddy
|
||||
image is pinned by digest, while backend and OpenSearch still build locally;
|
||||
immutable registry images and automated deploy are not configured. No migration
|
||||
framework exists. The application uses SQLite and OpenSearch; SMTP
|
||||
registration/recovery is not implemented.
|
||||
|
||||
## Repository paths and target host paths
|
||||
|
||||
- Compose and Dockerfiles: `deploy/production/`.
|
||||
- Host checkout target: `/opt/akyldash/current`.
|
||||
- Environment configuration: `/etc/akyldash/production.env`.
|
||||
- Secret file: `/etc/akyldash/review_secret`, mode `0600`.
|
||||
- Application data: `/var/lib/akyldash/data`.
|
||||
- Intended backups: `/var/backups/akyldash`.
|
||||
|
||||
All host paths above are targets; none existed at audit time. Root bootstrap is
|
||||
prepared at [bootstrap-ubuntu-24.04.sh](../../infra/bootstrap/bootstrap-ubuntu-24.04.sh).
|
||||
It does not edit SSH or firewall configuration, create credentials, deploy the
|
||||
application, or grant `codex` Docker/root access.
|
||||
|
||||
## Network and operations
|
||||
|
||||
Target public TCP listeners are SSH `22` and Caddy `80`/`443`. Backend `8080`
|
||||
and OpenSearch `9200` must remain private to Compose. The
|
||||
firewall state could not be independently verified because the SSH account has
|
||||
no sudo; do not change it without owner approval. The domain resolves to
|
||||
Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
|
||||
unreachable. Cloudflare origin mode, certificate validity, and production
|
||||
health have not been verified.
|
||||
|
||||
Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
|
||||
[ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
|
||||
limits and procedures. Configuration changes must go through Git and review;
|
||||
do not reconcile a Git/server difference by overwriting the VPS automatically.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Recovery on a clean VPS
|
||||
|
||||
## Scope and safety
|
||||
|
||||
The bootstrap below prepares Ubuntu packages and directories only. It is not a
|
||||
complete recovery: production secrets, data, reverse proxy, TLS, registry
|
||||
credentials, backup restore, and deploy-helper still require owner-controlled
|
||||
setup. Run only on the new recovery host after verifying its identity. Never
|
||||
run recovery commands against the existing production by accident.
|
||||
|
||||
## Procedure
|
||||
|
||||
1. Provision a clean Ubuntu Server 24.04 host and confirm hostname, IP, console
|
||||
access, and a tested owner recovery path.
|
||||
2. Create the SSH-authorized `codex` account without sudo or Docker group access.
|
||||
Preserve the host's existing root SSH and firewall policy until an owner has
|
||||
reviewed the change.
|
||||
3. Check out the approved Git revision, root-owned and not group/world-writable,
|
||||
at `/opt/akyldash/current`, then run
|
||||
`sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
|
||||
recovery mode refuses the inventoried production hostname/IP and writes a
|
||||
root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
|
||||
Engine, Compose, Git, Docker log limits, and target directories.
|
||||
4. Install the limited read-only verifier and rollback helper with
|
||||
`sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
|
||||
checkout and gives `codex` access only to those fixed commands.
|
||||
5. Configure DNS to point through Cloudflare to the recovery host and allow the
|
||||
owner to review/open TCP `80` and `443`. The production Compose Caddy service
|
||||
obtains its origin certificate automatically. Verify Cloudflare Full
|
||||
(strict) end to end before cutover.
|
||||
6. Transfer the protected backup to the new host through the owner's approved
|
||||
channel. The archive contains the production environment and review secret;
|
||||
keep it root-only and never paste secret values into chat.
|
||||
7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging
|
||||
directory with `infra/backup/archive.py`, then run
|
||||
`sudo python3 infra/backup/apply_restore.py --staging-dir <staging-dir>`.
|
||||
Apply requires the recovery marker, matching clean Git revision, empty data
|
||||
path, absent Compose volumes/config files, and no running containers. It
|
||||
refuses the inventoried production host and leaves services stopped.
|
||||
8. Build the backend and OpenSearch images from the matching checked-out Git
|
||||
revision and explicitly start Compose with `--wait`. Verify HTTPS and
|
||||
container health, record the backend image ID as described in
|
||||
[DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in
|
||||
[CURRENT_STATE.md](CURRENT_STATE.md).
|
||||
|
||||
Do not repoint DNS or retire the old host until the owner verifies recovery and
|
||||
approves the cutover. The restore and cutover have not been practiced yet.
|
||||
@@ -0,0 +1,41 @@
|
||||
# Rollback
|
||||
|
||||
## Current status
|
||||
|
||||
There is no production release, backup, or installed rollback helper. The
|
||||
repository contains a root-owned rollback helper and a separate installer, but
|
||||
neither has been run on the VPS.
|
||||
|
||||
After a verified cold backup is created before a release, an owner must check
|
||||
that the data format is compatible with the prior backend/OpenSearch images.
|
||||
Then run the limited helper with that archive:
|
||||
|
||||
```bash
|
||||
sudo -n /usr/local/sbin/akyldash-rollback \
|
||||
/var/backups/akyldash/<pre-release-backup>.tar.gz --data-compatible
|
||||
```
|
||||
|
||||
The helper verifies archive integrity, Compose image references, deployment
|
||||
file hashes, and the availability of both prior local images. It makes and
|
||||
verifies a second cold backup, restores only the prior backend/OpenSearch image
|
||||
tags, waits for Compose healthchecks, and checks HTTPS locally. Digest-pinned
|
||||
references are validated against their recorded local image IDs, but rollback
|
||||
fails before stopping services or making a backup when either image reference
|
||||
uses `@sha256:`. The helper cannot safely switch immutable digest references
|
||||
with the current release-recording flow. Do not retag a digest manually; digest
|
||||
rollback needs a separately reviewed atomic release-reference mechanism.
|
||||
|
||||
It refuses to restore if the current Compose/Caddyfile differs from the saved
|
||||
release. This does not restore data or automatically undo a release with an
|
||||
unknown schema change. The application has no formal migration framework;
|
||||
compatibility must be checked for every change. If it is uncertain, stop and ask
|
||||
the owner.
|
||||
|
||||
The initial owner-run deployment must record its running backend image ID in
|
||||
`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates
|
||||
that record after a successful rollback, and `infra/bootstrap/verify.sh` checks
|
||||
it against the running container.
|
||||
|
||||
Any data restore requires the owner, a verified pre-change backup, and a
|
||||
maintenance window. Preserve the failed release and both backups until recovery
|
||||
is confirmed.
|
||||
@@ -1,19 +1,37 @@
|
||||
# Статус проекта
|
||||
|
||||
Последняя проверка: 2026-08-14
|
||||
Последняя проверка: 2026-10-10
|
||||
Назначение документа: быстро восстановить контекст проекта для участников команды и будущих агентов.
|
||||
|
||||
- Telegram-бот: `0.2.2`
|
||||
- Telegram-бот на Synology: `0.2.1`
|
||||
- Backend: `0.7.1`
|
||||
- Frontend: не создан
|
||||
- Backend: `0.11.3`
|
||||
- Frontend: `0.1.2` — поиск, лаборатория релевантности, аккаунты
|
||||
|
||||
## Краткий итог
|
||||
|
||||
Репозиторий переориентирован с отдельного бота на весь проект юридической информационно-аналитической платформы. Telegram-бот выделен в инструмент рабочего окружения. Реализованы возобновляемая выгрузка документов из официального Open Data API ЦБД Минюста КР и их локальная воспроизводимая нормализация.
|
||||
|
||||
Ближайшая цель — оценить полный локальный индекс на 50–100 запросах RU/KY и
|
||||
настроить ранжирование до начала разработки поискового API.
|
||||
Поисковая система и внутренняя лаборатория оценки выдачи реализованы. Ближайшая
|
||||
цель — проверить собственную выдачу на практических RU/KY-запросах, разобрать
|
||||
оценки юристов и зафиксировать baseline. ЦБД Минюста служит источником для
|
||||
проверки документов и редакций, а не системой для сравнения поисковой выдачи.
|
||||
|
||||
### Точка остановки на 2026-09-18
|
||||
|
||||
- В PR #37 из ветки `feature/tax-profit-search-intent` тематический запрос
|
||||
«налог на прибыль» поднимает действующий НК КР; для обычных запросов сохранён
|
||||
`cross_fields`, добавлено усиление точной фразы. Нужна дальнейшая проверка на
|
||||
контрольном наборе запросов.
|
||||
- Независимое ревью поискового кода не выявило дефектов; замечание об устаревшем
|
||||
handoff исправлено. Backend-тесты: 33 пройдены, 1 пропущен;
|
||||
`git diff --check` чистый.
|
||||
- Текущий локальный backend-контейнер при последней проверке был `0.10.2`;
|
||||
изменения ветки (`0.10.3`) не развёрнуты. Не деплоить без отдельного запроса.
|
||||
- В рабочем дереве есть постороннее пользовательское изменение `.gitignore`;
|
||||
не включать его в коммиты и не удалять.
|
||||
- Следующий шаг: повторно проверить полный diff PR #37 и после успешных проверок
|
||||
передать его на слияние; затем продолжить оценку поиска на контрольном наборе.
|
||||
|
||||
## Уже сделано
|
||||
|
||||
@@ -77,6 +95,8 @@
|
||||
- Полный проход завершён: 209 958 документов нормализованы без ошибок.
|
||||
- Контрольная выборка RU/KY прошла проверки текста, фрагментов, ID и SHA-256.
|
||||
- Добавлены строгий mapping и атомарный Bulk NDJSON-экспорт для OpenSearch.
|
||||
- На предыдущей итерации использовались Excel/PDF-бланки юридической проверки;
|
||||
текущий процесс опирается на relevance set и внутреннюю лабораторию.
|
||||
|
||||
### Развёртывание
|
||||
|
||||
@@ -90,8 +110,9 @@
|
||||
|
||||
### Развёртывание и сопровождение
|
||||
|
||||
- Docker-конфигурация развёртывания не хранится в репозитории.
|
||||
- Не настроен CI/CD.
|
||||
- Production Compose, Ubuntu bootstrap и drift-проверка описаны в репозитории.
|
||||
- CI workflow добавлен в PR #42, но его Gitea run остаётся в очереди; наличие
|
||||
runner и автоматический production-деплой не подтверждены.
|
||||
|
||||
### Хранилище
|
||||
|
||||
@@ -112,6 +133,20 @@
|
||||
- Не настроены уведомления Gitea.
|
||||
- Не проведён полный пользовательский тест: сообщение → сохранение → экспорт → анализ в ChatGPT → возврат отчёта.
|
||||
|
||||
### Готовность поиска к публичному frontend
|
||||
|
||||
- Реализована внутренняя лаборатория для просмотра и оценки фактической
|
||||
выдачи OpenSearch; пилот с юристами ещё не проведён.
|
||||
- Не завершена независимая юридическая проверка всех 50 запросов relevance set
|
||||
v1; спорные строки нельзя включать в baseline.
|
||||
- Не зафиксированы неизменяемая копия подтверждённого relevance set и baseline
|
||||
`Recall@10`/`MRR@10`.
|
||||
- Публичный readiness gate остаётся открытым: нужны обновление корпуса,
|
||||
переиндексация, проверка выдачи и API-сценарии для RU, KY и одноязычных актов.
|
||||
- Создан внутренний авторизованный frontend поиска и лаборатории; первая
|
||||
учетная запись `admin` создаётся CLI-командой, новых пользователей добавляет
|
||||
только `admin`.
|
||||
|
||||
## Важные неизвестные
|
||||
|
||||
По мере реального использования нужно принять решения по следующим вопросам:
|
||||
@@ -122,19 +157,32 @@
|
||||
|
||||
## Следующий этап
|
||||
|
||||
### Фиксация MVP и проверка полного цикла
|
||||
### Проверка собственной поисковой выдачи
|
||||
|
||||
Рекомендуемый порядок:
|
||||
|
||||
1. Настроить резервное копирование `/volume1/docker/akyldash/data`.
|
||||
2. Зафиксировать состав MVP с командой и закрыть тему `MVP`.
|
||||
3. Провести одно реальное обсуждение с ответами, правками и вложением.
|
||||
4. Экспортировать обсуждение и вручную проанализировать Markdown в ChatGPT.
|
||||
5. Вернуть подтверждённый итог в тему `Решения` и зафиксировать его в Git/Gitea.
|
||||
6. Записать обнаруженные неудобства и только после этого выбирать следующую доработку.
|
||||
1. Подготовить согласованный RU/KY relevance set: фиксировать потребность
|
||||
запроса и подтверждённые `document_code`, не ориентируясь на порядок выдачи.
|
||||
2. Проверить практические запросы во внутренней лаборатории `/review`, сохранить
|
||||
снимки и оценки фактических результатов OpenSearch.
|
||||
3. Разобрать ошибки ранжирования, сохранить baseline и повторить ту же проверку
|
||||
после изменений.
|
||||
4. Затем пройти readiness gate issue #21 для корпуса, индекса и Search API перед
|
||||
подготовкой публичного frontend.
|
||||
|
||||
## История изменений статуса
|
||||
|
||||
### 2026-09-12
|
||||
|
||||
- Внутренняя лаборатория оценивает фактическую выдачу собственного OpenSearch;
|
||||
ЦБД Минюста используется только для проверки источников и редакций актов.
|
||||
- Backend `0.8.3`; закрытая внутренняя лаборатория готова к пилоту.
|
||||
|
||||
### 2026-09-06
|
||||
|
||||
- Подготовлены бланки независимой юридической проверки relevance set v1 для
|
||||
русского и кыргызского языков и инструкция для юриста.
|
||||
- Ближайшим этапом зафиксированы юридическая верификация, baseline качества
|
||||
поиска и финальный readiness gate перед frontend.
|
||||
|
||||
### 2026-08-18
|
||||
|
||||
- Оценщик поиска использует `cross_fields` для совместного сопоставления
|
||||
@@ -234,4 +282,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -1,19 +1,20 @@
|
||||
# Готовность к проектированию frontend
|
||||
# Готовность к публичному frontend
|
||||
|
||||
Этот документ определяет обязательный результат до начала проектирования и
|
||||
разработки пользовательского интерфейса. Он дополняет
|
||||
разработки публичной пользовательской системы. Он дополняет
|
||||
[план frontend поисковой СПС](frontend-search-sps-plan.md): тот описывает MVP и
|
||||
спринты frontend, этот — критерий перехода к ним.
|
||||
|
||||
## Правило перехода
|
||||
|
||||
Проектирование frontend начинается, когда выполнены все обязательные пункты
|
||||
этого плана и пройден финальный readiness gate. До этого не создаются
|
||||
`frontend/`, макеты, UI-компоненты или mock-данные, заменяющие неготовый
|
||||
backend-контракт.
|
||||
Публичный frontend и его выпуск начинаются, когда выполнены все обязательные
|
||||
пункты этого плана и пройден финальный readiness gate. Внутренняя
|
||||
авторизованная поисковая страница и лаборатория могут развиваться отдельно по
|
||||
явной задаче и используют только реальные существующие API.
|
||||
|
||||
Вне этого этапа остаются аккаунты, уведомления, RAG, судебная практика и
|
||||
внешние коммерческие сервисы.
|
||||
Персональные функции вне этого этапа: закладки, заметки, уведомления, RAG,
|
||||
судебная практика и внешние коммерческие сервисы. Внутренние учетные записи
|
||||
для лаборатории не включают публичную регистрацию.
|
||||
|
||||
## 1. Данные и поисковый индекс
|
||||
|
||||
@@ -52,13 +53,14 @@ alias без смешивания старого и нового корпуса.
|
||||
1. Заполнить рабочую копию из `backend/search/relevance-set-v1.template.json`
|
||||
естественными формулировками пользователей и подтверждёнными
|
||||
`document_code` релевантных действующих актов.
|
||||
2. Проверить каждый запрос по официальной ЦБД и локальному индексу; не
|
||||
включать запросы без установленного эталонного результата.
|
||||
2. Подтвердить документы по официальной ЦБД и проверить их наличие в локальном
|
||||
индексе по `document_code`, не оценивая порядок поисковой выдачи. Не включать
|
||||
запросы без установленного эталонного результата.
|
||||
3. Провести независимую вторую проверку языка, формулировки и полного списка
|
||||
кодов. Спорные результаты не включать до согласования.
|
||||
4. После проверки сохранить неизменяемую копию набора и baseline
|
||||
`Recall@10`/`MRR@10`; новые правила ранжирования оценивать только сравнением
|
||||
с этим baseline.
|
||||
4. После независимой проверки сохранить неизменяемую копию набора, затем
|
||||
оценить собственную выдачу и сохранить baseline `Recall@10`/`MRR@10`. Новые
|
||||
правила ранжирования проверять на том же наборе и сравнивать с baseline.
|
||||
5. Отдельно проверить распространённые сокращения, опечатки и запросы о
|
||||
практическом действии. Правило принимается, только если улучшает
|
||||
подтверждённые запросы и не создаёт ложных срабатываний в негативных
|
||||
|
||||
@@ -105,6 +105,11 @@ MVP не входят.
|
||||
|
||||
## Рекомендуемая основа frontend
|
||||
|
||||
Для внутреннего frontend лаборатории и авторизованного поиска используется
|
||||
одна HTML-страница, которую отдаёт существующий Search API, и браузерный
|
||||
JavaScript без сборщика и runtime-зависимостей. Это сохраняет текущую схему
|
||||
развёртывания. Публичный frontend после readiness gate можно оценить отдельно.
|
||||
|
||||
- Next.js App Router и TypeScript;
|
||||
- CSS Modules с BEM-именованием;
|
||||
- дизайн-токены для цветов, отступов, типографики и состояний;
|
||||
@@ -131,13 +136,13 @@ TypeScript в стандартной конфигурации. См.
|
||||
|
||||
Правила применения:
|
||||
|
||||
1. До разработки экранов выбрать в Refero не более трёх подходящих направлений
|
||||
и на их основе утвердить одно собственное направление Акылдаша.
|
||||
1. Визуальное направление «Түндүк» утверждено пользователем. Применять
|
||||
корневой [DESIGN.md](../../DESIGN.md); повторный выбор направления не нужен.
|
||||
2. Не копировать чужую дизайн-систему целиком. Цвета, типографика, плотность и
|
||||
компоненты должны учитывать длинные юридические тексты, два языка и
|
||||
доступность.
|
||||
3. Зафиксировать утверждённое направление в `frontend/DESIGN.md` и перенести
|
||||
значения в дизайн-токены проекта.
|
||||
3. Перенести значения из корневого `DESIGN.md` в дизайн-токены проекта.
|
||||
Не создавать отдельный `frontend/DESIGN.md` или конкурирующие спецификации.
|
||||
4. Дизайн-токены и компоненты Акылдаша являются источником истины. Внешние
|
||||
рекомендации не могут отменять BEM, доступность, требования безопасности и
|
||||
продуктовые ограничения проекта.
|
||||
@@ -156,9 +161,7 @@ runtime-зависимостями frontend. Регистрация в стор
|
||||
|
||||
- создать `frontend/`;
|
||||
- настроить Next.js, TypeScript, lint и сборку;
|
||||
- выбрать до трёх референсов в Refero Styles и утвердить одно визуальное
|
||||
направление;
|
||||
- создать `frontend/DESIGN.md` с правилами выбранного направления;
|
||||
- применить утверждённую дизайн-систему «Түндүк» из корневого `DESIGN.md`;
|
||||
- установить полный набор `jakubkrehel/skills` для проектных design review;
|
||||
- определить маршруты и типы API;
|
||||
- создать дизайн-токены;
|
||||
@@ -230,9 +233,10 @@ runtime-зависимостями frontend. Регистрация в стор
|
||||
|
||||
## Спринты после MVP
|
||||
|
||||
### Спринт 5 — персональный кабинет
|
||||
### Спринт 5 — персональные функции
|
||||
|
||||
Авторизация, закладки, заметки, подборки и сохранённые фильтры.
|
||||
Базовый вход и смена пароля для внутреннего frontend уже реализованы. Для
|
||||
публичного продукта остаются закладки, заметки, подборки и сохранённые фильтры.
|
||||
|
||||
### Спринт 6 — контроль изменений
|
||||
|
||||
@@ -258,4 +262,4 @@ runtime-зависимостями frontend. Регистрация в стор
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -11,8 +11,13 @@
|
||||
|
||||
- загрузчик находится в `backend/ingestion/minjust_cbd.py`;
|
||||
- сырой архив хранится в `data/minjust-cbd` и исключён из Git;
|
||||
- в манифесте 209 811 успешно загруженных документов;
|
||||
- 13 кодов остаются в таблице `errors` и отсутствуют в таблице `documents`;
|
||||
- в read-only проверенном манифесте 209 958 документов;
|
||||
- один код, `6`, остаётся в таблице `errors` и отсутствует в `documents`;
|
||||
это не подтверждает отсутствие документа в источнике: при последней
|
||||
проверке OpenData API возвращал HTTP `404` для списка, `GetDocument(6)`,
|
||||
контрольного `GetDocument(1096)` и `CheckAvailable`;
|
||||
- в нормализованном манифесте 209 958 успешных записей, schema `1`,
|
||||
normalizer `1.0.0`; полная потоковая проверка всех файлов ещё выполняется.
|
||||
- документ содержит `metadata.json` и каталог `editions`;
|
||||
- редакция содержит `metadata.json`, `ru.html` и/или `ky.html`, иногда
|
||||
изображения;
|
||||
|
||||
@@ -44,4 +44,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -1,6 +1,7 @@
|
||||
# Справочники Search API v1
|
||||
|
||||
Статус: утверждённый контракт для Search API v1.
|
||||
Статус: рабочий контракт Search API v1. Тематическая иерархия вынесена в
|
||||
отдельный черновой каталог и не входит в фильтры API v1.
|
||||
|
||||
`code` — неизменяемый идентификатор, независимый от языка. Клиент хранит и
|
||||
передаёт только `code`; русское и кыргызское названия являются подписями и могут
|
||||
@@ -37,6 +38,7 @@
|
||||
| `study` | Исследование | Исследование |
|
||||
| `report` | Доклад | Доклад |
|
||||
| `principles` | Основные принципы | Основные принципы |
|
||||
| `unspecified` | Не указан | Көрсөтүлгөн эмес |
|
||||
|
||||
## Статусы
|
||||
|
||||
@@ -46,6 +48,25 @@
|
||||
| `repealed` | Утратил силу | Күчүн жоготту |
|
||||
| `unspecified` | Не указан | Көрсөтүлгөн эмес |
|
||||
|
||||
## Юридическая сила
|
||||
|
||||
Это укрупнённая группировка для фильтра Search API, а не полная иерархия
|
||||
нормативных правовых актов из статьи 6 Закона КР «О нормативных правовых
|
||||
актах». В этой группировке Конституция и конституционные законы относятся к
|
||||
конституционному уровню, кодексы и законы — к законодательному, указы и
|
||||
постановления, перечисленные законом как виды НПА, — к подзаконному.
|
||||
|
||||
| Code | RU | KY |
|
||||
| --- | --- | --- |
|
||||
| `constitutional` | Конституционный уровень | Конституциялык деңгээл |
|
||||
| `legislative` | Законодательный уровень | Мыйзам деңгээли |
|
||||
| `subordinate` | Подзаконный уровень | Мыйзам алдындагы деңгээл |
|
||||
|
||||
Для прочих типов значение не назначается автоматически: одного названия типа
|
||||
недостаточно, чтобы установить юридическую силу утверждающего НПА. Основание:
|
||||
[статьи 4 и 6 Закона КР «О нормативных правовых актах»](https://cbd.minjust.gov.kg/4-3987/edition/53304/ru).
|
||||
Эта группировка реализована как фильтр `legal_force` Search API v1.
|
||||
|
||||
## Органы принятия
|
||||
|
||||
Орган содержит два уровня: стабильную группу и конкретный орган. В v1 группы
|
||||
@@ -70,6 +91,35 @@
|
||||
когда источник предоставит такие идентификаторы либо будет утверждён вручную
|
||||
поддерживаемый реестр.
|
||||
|
||||
## Темы
|
||||
|
||||
Полная исходная иерархия `GeneralClassifiers` сохранена отдельно в
|
||||
[`search-topic-taxonomy-v1.json`](search-topic-taxonomy-v1.json). Над исходными
|
||||
ветвями добавлены предложенные смысловые группы; исходные подписи и вложенность
|
||||
не переставляются. Поле `document_count` показывает число документов, где узел
|
||||
встречается. Предлагаются группы: конституционный строй и права; государство и
|
||||
публичное управление; право, суд и правопорядок; международные отношения;
|
||||
оборона и безопасность; экономика, финансы и предпринимательство; труд и
|
||||
социальная защита; здравоохранение; образование, наука и культура; земля,
|
||||
природные ресурсы и окружающая среда; сельское хозяйство; транспорт, связь,
|
||||
строительство и жильё; информация и СМИ; гражданские, семейные и жилищные
|
||||
отношения; общие и межотраслевые вопросы.
|
||||
|
||||
Каталог является черновиком: группа корневой рубрики предложена по русской
|
||||
подписи, рубрики с несколькими совпадениями правил или без совпадений помещены
|
||||
в `review_required` и отмечены `manual_review`; остальные помечены
|
||||
`provisional_lexical`. В корпусе 2 065 узлов и 687 вариантов корневых рубрик;
|
||||
328 корневых вариантов ожидают ручной классификации. Отсутствующие в источнике
|
||||
кыргызские подписи оставлены пустыми: они отсутствуют у 1 524 узлов, новые
|
||||
переводы не придуманы. Внутренние `source-*` ID являются
|
||||
временными хешами кода источника и цепочки подписей; при `Code: null` они
|
||||
зависят от подписей и не утверждаются как публичные стабильные коды.
|
||||
|
||||
Тематическая навигация и фильтр не входят в Search API v1. До включения в API
|
||||
нужно вручную утвердить смысловые группы, идентификаторы и недостающие KY-
|
||||
подписи. Источник — нормализованные данные ЦБД Минюста; команда для пересборки
|
||||
каталога находится в `tools/build_search_topic_taxonomy.py`.
|
||||
|
||||
## Правила совместимости
|
||||
|
||||
- Код из этой таблицы нельзя переиспользовать и нельзя менять его значение.
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# План внутреннего интерфейса оценки поисковой выдачи
|
||||
|
||||
## Цель
|
||||
|
||||
Создать закрытую лабораторию релевантности: юрист оценивает фактическую выдачу
|
||||
нашего OpenSearch по практическим запросам. Цель — улучшать собственное
|
||||
ранжирование, а не воспроизводить алгоритм сайта Минюста КР.
|
||||
|
||||
ЦБД Минюста используется как официальный источник текста, реквизитов, статуса
|
||||
и редакции акта. Порядок результатов и оценка их полезности определяются в
|
||||
нашем сервисе.
|
||||
|
||||
Это внутренний рабочий инструмент, а не публичный frontend MVP. Он не включает
|
||||
регистрацию, личные кабинеты, публичный дизайн, сложные фильтры или сравнение
|
||||
редакций.
|
||||
|
||||
## Сценарий юриста
|
||||
|
||||
1. Указать поисковый запрос и язык.
|
||||
2. Получить первые 10–20 результатов в точном порядке OpenSearch.
|
||||
3. Увидеть позицию каждого результата: `#1`, `#2` и далее.
|
||||
4. Открыть выбранный документ по клику на название в правой панели страницы.
|
||||
5. Поставить каждому просмотренному результату оценку и комментарий.
|
||||
6. Сохранить снимок выдачи и оценок.
|
||||
7. Передать накопленные записи на анализ ранжирования.
|
||||
|
||||
## Оценка результата
|
||||
|
||||
| Балл | Значение |
|
||||
| ---: | --- |
|
||||
| 0 | Нерелевантен: совпали слова, но акт не отвечает на вопрос. |
|
||||
| 1 | Косвенно полезен: относится к теме, но прямого ответа нет. |
|
||||
| 2 | Частично полезен: отвечает не полностью или требует другого акта. |
|
||||
| 3 | Прямо и достаточно отвечает на запрос. |
|
||||
|
||||
Оценка относится к отдельному документу, а не ко всей выдаче. Результат без
|
||||
оценки считается непросмотренным, а не нерелевантным.
|
||||
|
||||
## Интерфейс
|
||||
|
||||
Рекомендуемый экран — две панели.
|
||||
|
||||
- Вверху: поле запроса, переключатель RU/KY, выбор числа результатов и кнопка
|
||||
«Найти».
|
||||
- Слева: карточки результатов в порядке выдачи. Карточка содержит позицию,
|
||||
название, тип, статус, дату, номер и фрагмент текста.
|
||||
- Справа: заголовок, реквизиты, очищенный HTML выбранной редакции и ссылка на
|
||||
официальный источник.
|
||||
- В карточке: кнопки оценки `0`, `1`, `2`, `3` и раскрываемое поле
|
||||
комментария.
|
||||
- Внизу: имя или псевдоним проверяющего, общий комментарий и кнопка
|
||||
«Сохранить оценку».
|
||||
|
||||
Правая панель предпочтительнее popup: она не блокируется браузером, сохраняет
|
||||
контекст выдачи и работает на одном экране с оценкой.
|
||||
|
||||
### Доступность оценки и документа
|
||||
|
||||
Оценка реализуется нативной группой `radio` внутри `fieldset` с `legend`
|
||||
«Оценка результата». У каждого значения есть видимая подпись: «0 —
|
||||
нерелевантен», «1 — косвенно полезен», «2 — частично полезен», «3 — прямо
|
||||
отвечает». Нельзя передавать смысл оценки только цветом. Все элементы управления
|
||||
доступны с клавиатуры, имеют видимый `:focus-visible`; выбранный результат
|
||||
обозначается текстом и визуальным состоянием.
|
||||
|
||||
На узком экране список результатов занимает всю страницу. Кнопка «Открыть
|
||||
документ» открывает полноэкранный нативный `<dialog>` с явной кнопкой
|
||||
«Закрыть». При закрытии фокус возвращается на исходную кнопку «Открыть
|
||||
документ».
|
||||
|
||||
### Состояния
|
||||
|
||||
- Во время поиска и сохранения показывается состояние загрузки; повторная
|
||||
отправка на это время недоступна. Стабильная пустая область `role="status"`
|
||||
в DOM объявляет начало поиска, число результатов и успешное сохранение.
|
||||
- Пустая выдача сообщает: «По запросу „…“ ничего не найдено» и предлагает
|
||||
«Изменить запрос».
|
||||
- Ошибка поиска сообщает причину и предлагает «Повторить поиск»; текст ошибки
|
||||
выводится в `role="alert"`.
|
||||
- Ошибка сохранения сообщает: «Не удалось сохранить. Проверьте подключение и
|
||||
повторите». Черновик остаётся в браузере, текст ошибки выводится в
|
||||
`role="alert"`.
|
||||
- После сохранения выводится: «Оценка сохранена · № … · дата и время» в
|
||||
указанной стабильной области `role="status"`.
|
||||
|
||||
До сохранения черновик хранится в `localStorage`. После успешного сохранения
|
||||
интерфейс показывает ID записи и время сохранения.
|
||||
|
||||
## Backend и хранение
|
||||
|
||||
Использовать существующие endpoint:
|
||||
|
||||
- `GET /search` — получить ранжированный список результатов;
|
||||
- `GET /documents/{code}/editions/{edition}` — получить текст выбранной
|
||||
редакции.
|
||||
|
||||
Добавить два endpoint:
|
||||
|
||||
- `POST /search-reviews` — валидирует и сохраняет оценку;
|
||||
- `GET /search-reviews/export` — отдаёт накопленные записи в JSON.
|
||||
|
||||
Для первой версии достаточно отдельной SQLite-базы. Это стандартная библиотека
|
||||
Python, данные переживают перезапуск и легко выгружаются для анализа. Доступ к
|
||||
интерфейсу и всем endpoint `search-reviews`, включая экспорт, должен быть
|
||||
ограничен локальной сетью/VPN или аутентификацией reverse proxy.
|
||||
|
||||
Одна запись представляет один сохранённый поисковый сеанс:
|
||||
|
||||
| Поле | Назначение |
|
||||
| --- | --- |
|
||||
| `id`, `created_at` | Идентификатор и время сохранения. |
|
||||
| `reviewer` | Имя или псевдоним проверяющего. |
|
||||
| `query`, `language` | Исходный запрос и язык поиска. |
|
||||
| `index_name`, `algorithm_version` | Версия индекса и алгоритма на момент оценки. |
|
||||
| `top_result_code` | Код документа в позиции `#1`. |
|
||||
| `results_json` | Снимок результатов в исходном порядке с оценками и комментариями. |
|
||||
| `overall_comment` | Общий комментарий к выдаче. |
|
||||
|
||||
В `results_json` для каждого результата сохраняются: `rank`, `document_code`,
|
||||
`edition_code`, название, реквизиты, фрагмент, оценка и комментарий. Снимок
|
||||
выдачи обязателен: после изменения алгоритма можно будет восстановить именно
|
||||
тот результат, который видел юрист.
|
||||
|
||||
## Правила сохранения
|
||||
|
||||
- Запрос не пустой, не длиннее 500 символов.
|
||||
- Оценка может быть только целым числом от 0 до 3 либо отсутствовать у
|
||||
непросмотренного результата.
|
||||
- Комментарии имеют ограничение длины; пользовательские значения не вставляются
|
||||
в HTML.
|
||||
- `GET /search` возвращает краткоживущий HMAC-подписанный снимок выдачи:
|
||||
запрос, язык, индекс, результаты и их позиции. `POST /search-reviews`
|
||||
принимает этот снимок и только оценки с комментариями. Сервер проверяет
|
||||
подпись и срок, самостоятельно формирует `results_json` и отклоняет оценки
|
||||
для отсутствующих либо подменённых позиций и документов.
|
||||
- Нельзя передавать персональные данные или закрытые материалы в комментариях.
|
||||
- Кнопка «Сохранить оценку» остаётся доступной до отправки: отсутствующие
|
||||
обязательные поля проверяются после нажатия, ошибка показана рядом с полем и
|
||||
фокус переводится на первое некорректное поле.
|
||||
|
||||
## Анализ данных
|
||||
|
||||
Экспорт должен содержать исходный JSON-снимок, чтобы его можно было обработать
|
||||
скриптом или открыть в табличном инструменте. Первый отчёт строит:
|
||||
|
||||
- среднюю оценку для каждой позиции выдачи;
|
||||
- долю результатов с оценкой `3` в top-1, top-3 и top-10;
|
||||
- запросы, где нет результатов с оценкой `2` или `3`;
|
||||
- документы, которые часто получают низкую оценку в первых позициях;
|
||||
- комментарии для ручного разбора ошибок.
|
||||
|
||||
Сырые оценки не должны автоматически менять веса поиска. Сначала команда
|
||||
разбирает причины: анализатор, синонимы, статус, отсутствие документа,
|
||||
неправильная формулировка запроса или юридическая неоднозначность.
|
||||
|
||||
## Этапы реализации
|
||||
|
||||
1. Утвердить шкалу 0–3, обязательность имени проверяющего и правила доступа.
|
||||
2. Добавить SQLite-хранилище, валидацию, сохранение и JSON-экспорт.
|
||||
3. Добавить статическую внутреннюю страницу в существующий Python-сервер, без
|
||||
Next.js и отдельного публичного приложения.
|
||||
4. Подключить поиск, правую панель документа, черновик, адаптивный режим и
|
||||
сохранение в закреплённой панели действий на широком экране.
|
||||
5. Добавить минимальные backend-проверки сохранения, повторного запуска,
|
||||
экспорта и недопустимых оценок.
|
||||
6. Провести ручный прогон на десяти русскоязычных практических запросах с
|
||||
двумя юристами.
|
||||
7. На собранных записях настроить OpenSearch и повторить тот же набор запросов.
|
||||
|
||||
## Критерий готовности
|
||||
|
||||
Юрист вводит запрос, видит порядок выдачи, открывает документ, выставляет
|
||||
оценки и комментарии, сохраняет их. Экспорт содержит запрос, язык, документ
|
||||
на позиции `#1`, полный порядок результатов, оценки, комментарии и версию
|
||||
индекса. Данные можно сравнить до и после изменения алгоритма.
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install a verified staging restore onto an explicitly marked new VPS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from archive import COMPONENTS, check_sqlite, digest
|
||||
|
||||
ROOT = Path("/opt/akyldash/current")
|
||||
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
||||
ENV_DEST = Path("/etc/akyldash/production.env")
|
||||
SECRET_DEST = Path("/etc/akyldash/review_secret")
|
||||
DATA_DEST = Path("/var/lib/akyldash/data")
|
||||
MARKER = Path("/etc/akyldash/recovery-target")
|
||||
PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
VOLUME_TARGETS = {
|
||||
"/usr/share/opensearch/data": "opensearch-data",
|
||||
"/data": "caddy-data",
|
||||
"/config": "caddy-config",
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], *, capture: bool = False) -> str:
|
||||
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def require_empty_directory(path: Path) -> None:
|
||||
if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
|
||||
raise ValueError(f"restore destination must be an existing empty directory: {path}")
|
||||
|
||||
|
||||
def tree_size(root: Path) -> int:
|
||||
total = 0
|
||||
for directory, names, files in os.walk(root, followlinks=False):
|
||||
names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
|
||||
for name in files:
|
||||
path = Path(directory) / name
|
||||
if path.is_symlink():
|
||||
continue
|
||||
if path.is_file():
|
||||
total += path.stat().st_size
|
||||
return total
|
||||
|
||||
|
||||
def copy_contents(source: Path, destination: Path) -> None:
|
||||
for child in source.iterdir():
|
||||
target = destination / child.name
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
shutil.copytree(child, target, symlinks=True)
|
||||
else:
|
||||
shutil.copy2(child, target, follow_symlinks=False)
|
||||
|
||||
|
||||
def empty_directory(path: Path) -> None:
|
||||
for child in path.iterdir():
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
child.unlink()
|
||||
|
||||
|
||||
def validate_stage(staging: Path) -> dict:
|
||||
staging = staging.resolve(strict=True)
|
||||
if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
|
||||
raise ValueError("staging directory must be a direct child of /var/tmp")
|
||||
if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
|
||||
raise ValueError("staging directory must be root-owned mode 0700")
|
||||
manifest_path = staging / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
not isinstance(manifest, dict)
|
||||
or manifest.get("format") != 1
|
||||
or set(manifest.get("components", [])) != COMPONENTS
|
||||
):
|
||||
raise ValueError("staging manifest is invalid")
|
||||
for name in COMPONENTS - {"production-env", "review-secret"}:
|
||||
component = staging / name
|
||||
if not component.is_dir() or component.is_symlink():
|
||||
raise ValueError(f"staged component must be a directory: {name}")
|
||||
for name in ("production-env", "review-secret"):
|
||||
path = staging / name
|
||||
if not path.is_file() or path.is_symlink():
|
||||
raise ValueError(f"staged sensitive component must be a regular file: {name}")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_checkout(manifest: dict) -> None:
|
||||
if ROOT.is_symlink() or not (ROOT / ".git").exists():
|
||||
raise ValueError("approved Git checkout is missing or is a symlink")
|
||||
if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
|
||||
raise ValueError("recovery checkout must be clean")
|
||||
unsafe = subprocess.run(
|
||||
["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if unsafe:
|
||||
raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
|
||||
revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
|
||||
if revision != manifest.get("git_revision"):
|
||||
raise ValueError("checkout Git revision does not match the backup manifest")
|
||||
for relative, expected in manifest.get("deployment_files", {}).items():
|
||||
if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
|
||||
raise ValueError("Compose or Caddyfile does not match the backup manifest")
|
||||
if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
|
||||
raise ValueError("backup manifest is missing deployment file hashes")
|
||||
|
||||
|
||||
def validate_recovery_target() -> None:
|
||||
if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("root-only recovery-target marker is required")
|
||||
hostname = socket.getfqdn()
|
||||
addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
|
||||
if not hostname or not addresses:
|
||||
raise ValueError("cannot verify the recovery host identity")
|
||||
if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
|
||||
raise ValueError("refusing to apply recovery data to the inventoried production VPS")
|
||||
|
||||
|
||||
def compose_configuration(staging: Path) -> dict:
|
||||
rendered = run(
|
||||
[
|
||||
"docker", "compose", "--env-file", str(staging / "production-env"),
|
||||
"-f", str(COMPOSE_FILE), "config", "--format", "json",
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
config = json.loads(rendered)
|
||||
data_source = next(
|
||||
mount["source"]
|
||||
for mount in config["services"]["backend"]["volumes"]
|
||||
if mount["target"] == "/app/data"
|
||||
)
|
||||
secret_source = config["secrets"]["review_secret"]["file"]
|
||||
if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
|
||||
raise ValueError("backup production paths do not match the clean recovery host layout")
|
||||
return config
|
||||
|
||||
|
||||
def expected_volume_names(config: dict) -> set[str]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
names: set[str] = set()
|
||||
for target in VOLUME_TARGETS:
|
||||
mount = mounts.get(target)
|
||||
if not mount or mount["type"] != "volume":
|
||||
raise ValueError(f"expected persistent volume at {target}")
|
||||
names.add(config["volumes"][mount["source"]]["name"])
|
||||
return names
|
||||
|
||||
|
||||
def volume_destinations(config: dict) -> dict[str, Path]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
destinations = {}
|
||||
for target, component in VOLUME_TARGETS.items():
|
||||
mount = mounts.get(target)
|
||||
if not mount or mount["type"] != "volume":
|
||||
raise ValueError(f"expected persistent volume at {target}")
|
||||
name = config["volumes"][mount["source"]]["name"]
|
||||
destinations[component] = Path(
|
||||
run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
|
||||
)
|
||||
return destinations
|
||||
|
||||
|
||||
def apply_restore(staging: Path) -> None:
|
||||
if os.geteuid() != 0:
|
||||
raise ValueError("restore apply must run as root")
|
||||
validate_recovery_target()
|
||||
manifest = validate_stage(staging)
|
||||
validate_checkout(manifest)
|
||||
config = compose_configuration(staging)
|
||||
require_empty_directory(DATA_DEST)
|
||||
if ENV_DEST.exists() or SECRET_DEST.exists():
|
||||
raise ValueError("production environment and secret files must not already exist")
|
||||
if run(["docker", "ps", "--quiet"], capture=True):
|
||||
raise ValueError("recovery target must have no running Docker containers")
|
||||
|
||||
volume_names = expected_volume_names(config)
|
||||
for name in volume_names:
|
||||
result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
raise ValueError(f"recovery volume already exists: {name}")
|
||||
|
||||
created_volumes: list[str] = []
|
||||
created_files: list[Path] = []
|
||||
data_started = False
|
||||
try:
|
||||
for name in volume_names:
|
||||
run(["docker", "volume", "create", name])
|
||||
created_volumes.append(name)
|
||||
|
||||
destinations = volume_destinations(config)
|
||||
destinations["data"] = DATA_DEST
|
||||
|
||||
required: dict[int, int] = {}
|
||||
free: dict[int, int] = {}
|
||||
for component, destination in destinations.items():
|
||||
device = destination.stat().st_dev
|
||||
required[device] = required.get(device, 0) + tree_size(staging / component)
|
||||
free[device] = shutil.disk_usage(destination).free
|
||||
for device, size in required.items():
|
||||
if free[device] < size + 512 * 1024 * 1024:
|
||||
raise ValueError("recovery host lacks space for a safe data copy")
|
||||
|
||||
for component, destination in destinations.items():
|
||||
require_empty_directory(destination)
|
||||
if component == "data":
|
||||
data_started = True
|
||||
copy_contents(staging / component, destination)
|
||||
|
||||
for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
|
||||
os.chown(directory, 1000, 1000, follow_symlinks=False)
|
||||
for name in files:
|
||||
os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
|
||||
|
||||
for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
|
||||
shutil.copyfile(source, destination)
|
||||
created_files.append(destination)
|
||||
os.chown(destination, 0, 0)
|
||||
os.chmod(destination, 0o600)
|
||||
check_sqlite(DATA_DEST)
|
||||
except BaseException:
|
||||
for path in created_files:
|
||||
path.unlink(missing_ok=True)
|
||||
if data_started:
|
||||
empty_directory(DATA_DEST)
|
||||
for name in created_volumes:
|
||||
subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
|
||||
raise
|
||||
|
||||
MARKER.unlink()
|
||||
print("PASS staged recovery data applied to the marked new host; services remain stopped")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--staging-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
apply_restore(args.staging_dir)
|
||||
except Exception as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify cold, local Akyldash backups; restore only into staging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
COMPONENTS = {
|
||||
"data",
|
||||
"opensearch",
|
||||
"caddy-data",
|
||||
"caddy-config",
|
||||
"production-env",
|
||||
"review-secret",
|
||||
}
|
||||
NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
|
||||
members = archive.getmembers()
|
||||
names: set[str] = set()
|
||||
for member in members:
|
||||
name = PurePosixPath(member.name)
|
||||
if name.is_absolute() or ".." in name.parts or not name.parts:
|
||||
raise ValueError(f"unsafe archive path: {member.name}")
|
||||
if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
|
||||
raise ValueError(f"non-canonical archive path: {member.name}")
|
||||
if member.name in names:
|
||||
raise ValueError(f"duplicate archive path: {member.name}")
|
||||
names.add(member.name)
|
||||
if not (member.isdir() or member.isfile() or member.issym()):
|
||||
raise ValueError(f"unsupported archive entry: {member.name}")
|
||||
if member.issym():
|
||||
tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
|
||||
if "manifest.json" not in names:
|
||||
raise ValueError("backup manifest is missing")
|
||||
if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
|
||||
raise ValueError("backup is missing one or more required components")
|
||||
manifest_member = next(item for item in members if item.name == "manifest.json")
|
||||
if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
|
||||
raise ValueError("backup manifest is not a regular file")
|
||||
return members
|
||||
|
||||
|
||||
def read_manifest(archive: tarfile.TarFile) -> dict:
|
||||
checked_members(archive)
|
||||
stream = archive.extractfile("manifest.json")
|
||||
if stream is None:
|
||||
raise ValueError("backup manifest is unreadable")
|
||||
manifest = json.load(stream)
|
||||
components = manifest.get("components") if isinstance(manifest, dict) else None
|
||||
if (
|
||||
not isinstance(manifest, dict)
|
||||
or manifest.get("format") != 1
|
||||
or not isinstance(components, list)
|
||||
or not all(isinstance(item, str) for item in components)
|
||||
or set(components) != COMPONENTS
|
||||
):
|
||||
raise ValueError("backup manifest has an unsupported format or component set")
|
||||
if not isinstance(manifest.get("images"), dict):
|
||||
raise ValueError("backup manifest has no image records")
|
||||
if not isinstance(manifest.get("deployment_files"), dict):
|
||||
raise ValueError("backup manifest has no deployment file records")
|
||||
for member in archive.getmembers():
|
||||
if member.name == "manifest.json":
|
||||
continue
|
||||
if PurePosixPath(member.name).parts[0] not in COMPONENTS:
|
||||
raise ValueError(f"unexpected top-level archive path: {member.name}")
|
||||
return manifest
|
||||
|
||||
|
||||
def verify_archive(path: Path) -> dict:
|
||||
checksum_path = Path(f"{path}.sha256")
|
||||
expected = checksum_path.read_text(encoding="ascii").split()[0]
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
|
||||
raise ValueError("archive SHA-256 does not match its sidecar")
|
||||
with tarfile.open(path, "r:gz") as archive:
|
||||
return read_manifest(archive)
|
||||
|
||||
|
||||
def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
|
||||
sources: dict[str, Path] = {}
|
||||
for value in values:
|
||||
name, separator, raw_path = value.partition("=")
|
||||
if not separator or not NAME_RE.fullmatch(name) or name in sources:
|
||||
raise ValueError(f"invalid or duplicate source: {value}")
|
||||
source = Path(raw_path)
|
||||
if not source.is_absolute() or not source.exists() or source.is_symlink():
|
||||
raise ValueError(f"source must be an existing absolute path: {raw_path}")
|
||||
sources[name] = source.resolve(strict=True)
|
||||
if set(sources) != required:
|
||||
missing = ", ".join(sorted(required - sources.keys()))
|
||||
extra = ", ".join(sorted(sources.keys() - required))
|
||||
raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
|
||||
return sources
|
||||
|
||||
|
||||
def check_sqlite(root: Path) -> None:
|
||||
for path in root.rglob("*.sqlite3"):
|
||||
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
|
||||
try:
|
||||
result = connection.execute("PRAGMA quick_check").fetchone()
|
||||
if result != ("ok",):
|
||||
raise ValueError(f"SQLite integrity check failed: {path}")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def compose_storage(config: dict) -> dict[str, Path | None]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
volumes = subprocess.run(
|
||||
["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
|
||||
).stdout.split()
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, component in (
|
||||
("/app/data", "data"),
|
||||
("/usr/share/opensearch/data", "opensearch"),
|
||||
("/data", "caddy-data"),
|
||||
("/config", "caddy-config"),
|
||||
):
|
||||
mount = mounts.get(target)
|
||||
if not mount:
|
||||
raise ValueError(f"persistent Compose mount is missing: {target}")
|
||||
if mount["type"] == "bind":
|
||||
source = Path(mount["source"])
|
||||
elif mount["type"] == "volume":
|
||||
volume_name = config["volumes"][mount["source"]]["name"]
|
||||
if volume_name not in volumes:
|
||||
sources[component] = None
|
||||
continue
|
||||
result = subprocess.run(
|
||||
["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
source = Path(result.stdout.strip())
|
||||
else:
|
||||
raise ValueError(f"unsupported persistent mount type at {target}")
|
||||
if not source.is_dir() or source.is_symlink():
|
||||
raise ValueError(f"cannot determine persistent data state for {component}: {source}")
|
||||
sources[component] = source
|
||||
return sources
|
||||
|
||||
|
||||
def has_persistent_data(sources: dict[str, Path | None]) -> bool:
|
||||
return any(any(path.iterdir()) for path in sources.values() if path is not None)
|
||||
|
||||
|
||||
def is_preloaded_corpus_only(data: Path | None) -> bool:
|
||||
if data is None:
|
||||
return False
|
||||
entries = list(data.iterdir())
|
||||
return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
|
||||
|
||||
|
||||
def compose_sources(
|
||||
compose_file: Path, env_file: Path, initial_deployment: bool = False
|
||||
) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
|
||||
command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
|
||||
rendered = subprocess.run(
|
||||
[*command, "config", "--format", "json"], check=True, capture_output=True, text=True
|
||||
)
|
||||
config = json.loads(rendered.stdout)
|
||||
container_result = subprocess.run(
|
||||
[*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
|
||||
)
|
||||
containers = container_result.stdout.split()
|
||||
storage = compose_storage(config)
|
||||
persistent_data = has_persistent_data(storage)
|
||||
if not containers and not persistent_data:
|
||||
raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
|
||||
missing_volumes = any(path is None for path in storage.values())
|
||||
if missing_volumes and containers:
|
||||
raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
|
||||
if missing_volumes and not initial_deployment:
|
||||
raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
|
||||
if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
|
||||
raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
|
||||
images: dict[str, dict[str, str]] = {}
|
||||
for container in containers:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
'{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
|
||||
container,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
fields = result.stdout.strip().split("\t")
|
||||
if len(fields) != 4 or fields[0].lower() != "false":
|
||||
raise ValueError("stop all Compose services before taking a cold backup")
|
||||
images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
|
||||
if containers and set(images) != set(config["services"]):
|
||||
raise ValueError("not all configured Compose services have stopped containers")
|
||||
|
||||
secret_file = Path(config["secrets"]["review_secret"]["file"])
|
||||
sources = {
|
||||
**storage,
|
||||
"production-env": env_file,
|
||||
"review-secret": secret_file,
|
||||
}
|
||||
repository = compose_file.resolve().parents[2]
|
||||
changed = subprocess.run(
|
||||
["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if changed:
|
||||
raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
|
||||
revision = subprocess.run(
|
||||
["git", "-C", str(repository), "rev-parse", "HEAD"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
deployment_files = {
|
||||
relative: digest(repository / relative)
|
||||
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
}
|
||||
present = {name: path for name, path in sources.items() if path is not None}
|
||||
missing = {name for name, path in sources.items() if path is None}
|
||||
parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
|
||||
return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
|
||||
|
||||
|
||||
def create_backup(
|
||||
output: Path,
|
||||
sources: dict[str, Path | None],
|
||||
revision: str,
|
||||
images: dict | None = None,
|
||||
deployment_files: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
output = output.absolute()
|
||||
parent = output.parent
|
||||
if not parent.is_dir() or parent.stat().st_mode & 0o077:
|
||||
raise ValueError("backup directory must exist and have mode 0700 or stricter")
|
||||
if output.exists() or Path(f"{output}.sha256").exists():
|
||||
raise ValueError("refusing to overwrite an existing backup")
|
||||
for name, source in sources.items():
|
||||
if source is None:
|
||||
continue
|
||||
if source.is_symlink() or not (source.is_dir() or source.is_file()):
|
||||
raise ValueError(f"source must be a regular file or directory: {name}")
|
||||
if source.is_dir() and (output == source or source in output.parents):
|
||||
raise ValueError(f"backup output cannot be inside source {name}")
|
||||
for name in ("production-env", "review-secret"):
|
||||
if sources[name] is None or not sources[name].is_file():
|
||||
raise ValueError(f"sensitive component must be a regular file: {name}")
|
||||
if sources["data"] is not None and sources["data"].is_dir():
|
||||
check_sqlite(sources["data"])
|
||||
|
||||
manifest = {
|
||||
"format": 1,
|
||||
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_revision": revision,
|
||||
"images": images or {},
|
||||
"deployment_files": deployment_files or {},
|
||||
"components": sorted(sources),
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
|
||||
temporary = Path(temp.name)
|
||||
os.chmod(temporary, 0o600)
|
||||
checksum_temp = Path(f"{temporary}.sha256")
|
||||
try:
|
||||
with tarfile.open(temporary, "w:gz") as archive:
|
||||
encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(encoded)
|
||||
info.mode = 0o600
|
||||
archive.addfile(info, io.BytesIO(encoded))
|
||||
for name, source in sorted(sources.items()):
|
||||
if source is None:
|
||||
info = tarfile.TarInfo(f"{name}/")
|
||||
info.type = tarfile.DIRTYPE
|
||||
info.mode = 0o700
|
||||
archive.addfile(info)
|
||||
continue
|
||||
def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
|
||||
if name in {"production-env", "review-secret"}:
|
||||
member.mode = 0o600
|
||||
if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
|
||||
member.mode = 0o600
|
||||
return member
|
||||
|
||||
archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
|
||||
with tarfile.open(temporary, "r:gz") as archive:
|
||||
read_manifest(archive)
|
||||
checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
|
||||
stream.write(f"{digest(temporary)} {output.name}\n")
|
||||
os.link(temporary, output)
|
||||
try:
|
||||
os.link(checksum_temp, Path(f"{output}.sha256"))
|
||||
except Exception:
|
||||
output.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
checksum_temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def restore_to_staging(backup: Path, target: Path) -> None:
|
||||
if not target.name.startswith("akyldash-restore-"):
|
||||
raise ValueError("restore target must be a new akyldash-restore-* staging directory")
|
||||
parent = target.parent.resolve()
|
||||
if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
|
||||
raise ValueError("restore staging directory must be under /tmp or /var/tmp")
|
||||
if target.exists():
|
||||
raise ValueError("restore target already exists; refusing to overwrite it")
|
||||
target = parent / target.name
|
||||
verify_archive(backup)
|
||||
target.mkdir(mode=0o700)
|
||||
try:
|
||||
with tarfile.open(backup, "r:gz") as archive:
|
||||
members = checked_members(archive)
|
||||
required_bytes = sum(member.size for member in members if member.isfile())
|
||||
if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
|
||||
raise ValueError("insufficient free space for a validated restore staging copy")
|
||||
for member in members:
|
||||
archive.extract(member, target, filter="data")
|
||||
check_sqlite(target / "data")
|
||||
except Exception:
|
||||
shutil.rmtree(target)
|
||||
raise
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
create = subparsers.add_parser("create", help="create a cold backup archive")
|
||||
create.add_argument("--output", required=True, type=Path)
|
||||
create.add_argument("--compose-file", required=True, type=Path)
|
||||
create.add_argument("--env-file", required=True, type=Path)
|
||||
create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
|
||||
verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
|
||||
verify.add_argument("archive", type=Path)
|
||||
restore = subparsers.add_parser("restore", help="extract into a new staging directory")
|
||||
restore.add_argument("archive", type=Path)
|
||||
restore.add_argument("--target-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command in {"create", "restore"} and os.geteuid() != 0:
|
||||
raise ValueError("create and restore must run as root to protect production secrets and data")
|
||||
if args.command == "create":
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
args.compose_file, args.env_file, args.initial_deployment
|
||||
)
|
||||
create_backup(args.output, sources, revision, images, deployment_files)
|
||||
elif args.command == "verify":
|
||||
manifest = verify_archive(args.archive)
|
||||
print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
|
||||
else:
|
||||
restore_to_staging(args.archive, args.target_dir)
|
||||
print(f"PASS restored and validated staging data at {args.target_dir}")
|
||||
except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,90 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import apply_restore
|
||||
|
||||
|
||||
class ApplyRestoreTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.staging = self.root / "akyldash-restore-test"
|
||||
self.staging.mkdir()
|
||||
for component in apply_restore.COMPONENTS:
|
||||
path = self.staging / component
|
||||
if component in {"production-env", "review-secret"}:
|
||||
path.write_text("private", encoding="utf-8")
|
||||
else:
|
||||
path.mkdir()
|
||||
(path / "payload.txt").write_text(component, encoding="utf-8")
|
||||
self.data = self.root / "data"
|
||||
self.data.mkdir()
|
||||
self.volumes = {
|
||||
name: self.root / name
|
||||
for name in ("opensearch", "caddy-data", "caddy-config")
|
||||
}
|
||||
for path in self.volumes.values():
|
||||
path.mkdir()
|
||||
self.env = self.root / "production.env"
|
||||
self.secret = self.root / "review_secret"
|
||||
self.marker = self.root / "recovery-target"
|
||||
self.marker.touch(mode=0o600)
|
||||
self.config = {
|
||||
"volumes": {
|
||||
"opensearch-data": {"name": "akyldash_opensearch-data"},
|
||||
"caddy-data": {"name": "akyldash_caddy-data"},
|
||||
"caddy-config": {"name": "akyldash_caddy-config"},
|
||||
}
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def run_apply(self):
|
||||
def docker_run(command, **_kwargs):
|
||||
if command[:3] == ["docker", "ps", "--quiet"]:
|
||||
return ""
|
||||
if command[1:3] == ["volume", "create"]:
|
||||
return command[-1]
|
||||
return ""
|
||||
|
||||
with (
|
||||
patch("apply_restore.os.geteuid", return_value=0),
|
||||
patch("apply_restore.validate_recovery_target"),
|
||||
patch("apply_restore.validate_stage", return_value={"format": 1}),
|
||||
patch("apply_restore.validate_checkout"),
|
||||
patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}),
|
||||
patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])),
|
||||
patch("apply_restore.volume_destinations", return_value=self.volumes),
|
||||
patch("apply_restore.DATA_DEST", self.data),
|
||||
patch("apply_restore.ENV_DEST", self.env),
|
||||
patch("apply_restore.SECRET_DEST", self.secret),
|
||||
patch("apply_restore.MARKER", self.marker),
|
||||
patch("apply_restore.run", side_effect=docker_run),
|
||||
patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)),
|
||||
patch("apply_restore.os.chown"),
|
||||
):
|
||||
apply_restore.apply_restore(self.staging)
|
||||
|
||||
def test_applies_components_and_protects_secret_files(self):
|
||||
self.run_apply()
|
||||
self.assertEqual((self.data / "payload.txt").read_text(), "data")
|
||||
self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch")
|
||||
self.assertEqual(self.env.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600)
|
||||
self.assertFalse(self.marker.exists())
|
||||
|
||||
def test_refuses_nonempty_data_destination_before_writing_secrets(self):
|
||||
(self.data / "keep.txt").write_text("existing", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "empty directory"):
|
||||
self.run_apply()
|
||||
self.assertEqual((self.data / "keep.txt").read_text(), "existing")
|
||||
self.assertFalse(self.env.exists())
|
||||
self.assertTrue(self.marker.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,362 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from archive import COMPONENTS, compose_sources, create_backup, restore_to_staging, verify_archive
|
||||
|
||||
|
||||
class BackupArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.private = self.root / "private"
|
||||
self.private.mkdir(mode=0o700)
|
||||
self.sources = {}
|
||||
for name in COMPONENTS:
|
||||
source = self.root / name
|
||||
source.mkdir()
|
||||
self.sources[name] = source
|
||||
for name in ("production-env", "review-secret"):
|
||||
path = self.root / f"{name}.txt"
|
||||
path.write_text("private", encoding="utf-8")
|
||||
path.chmod(0o644)
|
||||
self.sources[name] = path
|
||||
connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
|
||||
connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
self.backup = self.private / "backup.tar.gz"
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_create_verify_and_restore_only_to_new_staging(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
manifest = verify_archive(self.backup)
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
self.assertEqual(manifest["images"], {})
|
||||
|
||||
target = self.root / "akyldash-restore-test"
|
||||
restore_to_staging(self.backup, target)
|
||||
self.assertTrue((target / "data/search-users.sqlite3").is_file())
|
||||
self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
|
||||
|
||||
def test_rejects_checksum_mismatch_without_creating_staging(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
|
||||
target = self.root / "akyldash-restore-tampered"
|
||||
with self.assertRaisesRegex(ValueError, "SHA-256"):
|
||||
restore_to_staging(self.backup, target)
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
def test_rejects_non_staging_and_existing_targets(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
with self.assertRaisesRegex(ValueError, "staging"):
|
||||
restore_to_staging(self.backup, self.root / "production")
|
||||
target = self.root / "akyldash-restore-existing"
|
||||
target.mkdir()
|
||||
with self.assertRaisesRegex(ValueError, "already exists"):
|
||||
restore_to_staging(self.backup, target)
|
||||
|
||||
def test_create_checks_sqlite_integrity_before_writing_archive(self):
|
||||
(self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
self.assertFalse(self.backup.exists())
|
||||
|
||||
def test_rejects_unsafe_archive_paths(self):
|
||||
archive_path = self.private / "unsafe.tar.gz"
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.addfile(tarfile.TarInfo("manifest.json"))
|
||||
archive.addfile(tarfile.TarInfo("../outside"))
|
||||
checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
|
||||
Path(f"{archive_path}.sha256").write_text(f"{checksum} unsafe.tar.gz\n")
|
||||
with self.assertRaisesRegex(ValueError, "unsafe archive path"):
|
||||
verify_archive(archive_path)
|
||||
|
||||
def test_cold_backup_refuses_a_running_compose_container(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="container-id\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
return SimpleNamespace(stdout="true\n")
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "stop all Compose services"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def compose_fixture(self):
|
||||
paths = {}
|
||||
mounts = {
|
||||
"backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
|
||||
"opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
|
||||
"caddy": [
|
||||
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
||||
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
||||
],
|
||||
}
|
||||
volumes = {}
|
||||
for volume in ("os-data", "caddy-data", "caddy-config"):
|
||||
name = f"akyldash_{volume}"
|
||||
path = self.root / name
|
||||
path.mkdir()
|
||||
paths[name] = path
|
||||
volumes[volume] = {"name": name}
|
||||
env_file = self.root / "production.env"
|
||||
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
||||
secret = self.root / "review-secret.txt"
|
||||
secret.write_text("secret", encoding="utf-8")
|
||||
config = {
|
||||
"services": {name: {"volumes": value} for name, value in mounts.items()},
|
||||
"volumes": volumes,
|
||||
"secrets": {"review_secret": {"file": str(secret)}},
|
||||
}
|
||||
return config, paths, env_file, secret
|
||||
|
||||
def test_clean_first_deployment_does_not_require_backup(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
|
||||
config, _volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_existing_data_without_containers_is_backup_eligible(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file
|
||||
)
|
||||
archive = self.private / "offline-backup.tar.gz"
|
||||
create_backup(archive, sources, revision, images, deployment_files)
|
||||
manifest = verify_archive(archive)
|
||||
self.assertEqual(set(sources), COMPONENTS)
|
||||
self.assertEqual(images, {})
|
||||
self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
|
||||
def test_missing_persistent_mount_state_is_rejected(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
volume_paths["akyldash_os-data"].rmdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_partial_compose_containers_are_rejected_with_existing_data(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="backend-id\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if command[1] == "inspect":
|
||||
return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
(self.root / "data/existing.json").write_text("data", encoding="utf-8")
|
||||
available = set(volume_paths) - {"akyldash_os-data"}
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
||||
if command[1] == "volume":
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
|
||||
compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
||||
)
|
||||
|
||||
def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
corpus = self.root / "data/minjust-normalized"
|
||||
corpus.mkdir()
|
||||
(corpus / "document.json").write_text("corpus", encoding="utf-8")
|
||||
available = set(volume_paths) - {"akyldash_os-data"}
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
||||
if command[1] == "volume":
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
||||
)
|
||||
self.assertIsNone(sources["opensearch"])
|
||||
archive = self.private / "initial-corpus-backup.tar.gz"
|
||||
create_backup(archive, sources, revision, images, deployment_files)
|
||||
manifest = verify_archive(archive)
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
with tarfile.open(archive, "r:gz") as saved:
|
||||
self.assertTrue(saved.getmember("opensearch/").isdir())
|
||||
self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
|
||||
|
||||
def test_compose_backup_records_sources_and_image_ids(self):
|
||||
data = self.root / "production-data"
|
||||
data.mkdir()
|
||||
secret = self.root / "secret"
|
||||
secret.write_text("private", encoding="utf-8")
|
||||
env_file = self.root / "production.env"
|
||||
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
||||
volumes = {}
|
||||
volume_paths = {}
|
||||
mounts = {
|
||||
"caddy": [
|
||||
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
||||
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
||||
],
|
||||
"backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
|
||||
"opensearch": [
|
||||
{"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
|
||||
],
|
||||
}
|
||||
for name in ("caddy-data", "caddy-config", "opensearch-data"):
|
||||
path = self.root / f"volume-{name}"
|
||||
path.mkdir()
|
||||
volumes[name] = {"name": f"akyldash_{name}"}
|
||||
volume_paths[f"akyldash_{name}"] = path
|
||||
config = {
|
||||
"services": {name: {"volumes": values} for name, values in mounts.items()},
|
||||
"volumes": volumes,
|
||||
"secrets": {"review_secret": {"file": str(secret)}},
|
||||
}
|
||||
service_names = ["caddy", "backend", "opensearch"]
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
|
||||
if command[1] == "inspect":
|
||||
name = command[-1].removesuffix("-id")
|
||||
return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
return SimpleNamespace(stdout="")
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file
|
||||
)
|
||||
self.assertEqual(revision, "")
|
||||
self.assertEqual(set(sources), COMPONENTS)
|
||||
self.assertEqual(images["backend"]["image_id"], "sha256:backend")
|
||||
self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run this read-only audit as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo '== Host =='
|
||||
hostname -f
|
||||
grep -E '^(PRETTY_NAME|VERSION_ID)=' /etc/os-release
|
||||
nproc
|
||||
free -h
|
||||
df -h /
|
||||
|
||||
echo '== Packages and services =='
|
||||
dpkg-query -W docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
docker version --format 'Docker Engine {{.Server.Version}}'
|
||||
docker compose version
|
||||
containerd --version
|
||||
runc --version
|
||||
systemctl is-active docker
|
||||
python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); print("Docker logs:", d.get("log-driver"), d.get("log-opts"))'
|
||||
|
||||
echo '== Account and paths =='
|
||||
id codex
|
||||
for path in /opt/akyldash /opt/akyldash/current /etc/akyldash /etc/akyldash/production.env /etc/akyldash/review_secret /var/lib/akyldash/data /var/backups/akyldash /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /etc/sudoers.d/akyldash-codex; do
|
||||
if [[ -e $path ]]; then
|
||||
stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
|
||||
else
|
||||
printf '%s absent\n' "$path"
|
||||
fi
|
||||
done
|
||||
|
||||
echo '== ISPmanager =='
|
||||
dpkg-query -W -f='${binary:Package} ${Version}\n' 'ispmanager*' 'ispmgr*' 2>/dev/null || true
|
||||
systemctl list-unit-files --type=service --no-legend | grep -iE 'ispmanager|ispmgr' || echo 'No ISPmanager service unit found.'
|
||||
|
||||
echo '== Docker state =='
|
||||
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
|
||||
docker volume ls
|
||||
|
||||
echo '== Gitea Actions runner (read-only) =='
|
||||
if id akyldash-runner >/dev/null 2>&1; then
|
||||
id akyldash-runner
|
||||
else
|
||||
echo 'akyldash-runner user absent'
|
||||
fi
|
||||
if command -v act_runner >/dev/null 2>&1; then
|
||||
act_runner --version
|
||||
else
|
||||
echo 'act_runner binary absent from PATH'
|
||||
fi
|
||||
for path in /etc/akyldash-runner/config.yaml /var/lib/akyldash-runner/.runner /var/lib/akyldash-runner/docker-store.img; do
|
||||
if [[ -e $path ]]; then
|
||||
stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
|
||||
else
|
||||
printf '%s absent\n' "$path"
|
||||
fi
|
||||
done
|
||||
systemctl status akyldash-act-runner.service --no-pager --full || true
|
||||
systemctl status akyldash-runner-healthcheck.timer --no-pager --full || true
|
||||
findmnt /var/lib/akyldash-runner/docker-data || true
|
||||
|
||||
echo '== Firewall and listeners (read-only) =='
|
||||
ufw status verbose
|
||||
ufw status numbered
|
||||
ss -lntup
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -gt 1 || ( $# -eq 1 && $1 != --recovery-target ) ]]; then
|
||||
echo "Usage: $0 [--recovery-target]" >&2
|
||||
exit 2
|
||||
fi
|
||||
recovery_target=false
|
||||
if [[ ${1:-} == --recovery-target ]]; then
|
||||
recovery_target=true
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
|
||||
recovery_hostname=$(hostname -f 2>/dev/null || true)
|
||||
recovery_ipv4=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
|
||||
if [[ -z $recovery_hostname || -z $recovery_ipv4 ]]; then
|
||||
echo "Cannot verify recovery host identity; refusing recovery bootstrap." >&2
|
||||
exit 1
|
||||
fi
|
||||
if akyldash_is_production_host "$recovery_hostname" "$recovery_ipv4"; then
|
||||
echo "Refusing recovery bootstrap on the current production host." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "Run this script as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
if [[ ${ID:-} != ubuntu || ${VERSION_ID:-} != 24.04 ]]; then
|
||||
echo "Expected Ubuntu 24.04; found ${PRETTY_NAME:-unknown}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! id codex >/dev/null 2>&1; then
|
||||
echo "Create the SSH-authorized codex account before running bootstrap." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -e /etc/docker/daemon.json ]] && ! python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' /etc/docker/daemon.json; then
|
||||
echo "/etc/docker/daemon.json exists with different log settings; review it manually." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d -m 0755 /etc/docker
|
||||
if [[ ! -e /etc/docker/daemon.json ]]; then
|
||||
cat > /etc/docker/daemon.json <<'EOF'
|
||||
{
|
||||
"log-driver": "local",
|
||||
"log-opts": {"max-size": "10m", "max-file": "3"}
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl git
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
|
||||
"$(dpkg --print-architecture)" "$VERSION_CODENAME" > /etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
|
||||
install -d -o root -g root -m 0755 /opt/akyldash
|
||||
install -d -o root -g root -m 0700 /etc/akyldash
|
||||
install -d -o root -g root -m 0750 /var/lib/akyldash/data
|
||||
install -d -o root -g root -m 0700 /var/backups/akyldash
|
||||
if [[ $recovery_target == true ]]; then
|
||||
install -o root -g root -m 0600 /dev/null /etc/akyldash/recovery-target
|
||||
fi
|
||||
systemctl enable --now docker
|
||||
docker compose version
|
||||
echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
|
||||
if [[ $recovery_target == true ]]; then
|
||||
echo 'Recovery target marker created at /etc/akyldash/recovery-target.'
|
||||
fi
|
||||
@@ -0,0 +1,11 @@
|
||||
akyldash_is_production_host() {
|
||||
local hostname=${1:-}
|
||||
local ipv4_addresses=${2:-}
|
||||
if [[ -z $hostname ]]; then
|
||||
hostname=$(hostname -f 2>/dev/null || true)
|
||||
fi
|
||||
if [[ -z $ipv4_addresses ]]; then
|
||||
ipv4_addresses=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
|
||||
fi
|
||||
[[ $hostname == avprogs.hlab.kz ]] || grep -Fqx '206.223.249.118' <<<"$ipv4_addresses"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
source "$script_dir/../test/run-as-unprivileged.sh"
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if run_as_unprivileged bash "$script_dir/audit-host.sh" >"$output" 2>&1; then
|
||||
echo 'Host audit unexpectedly succeeded as an unprivileged UID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'Run this read-only audit as root.' "$output"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
|
||||
|
||||
akyldash_is_production_host avprogs.hlab.kz 127.0.0.1
|
||||
akyldash_is_production_host other.example $'10.0.0.2\n206.223.249.118'
|
||||
if akyldash_is_production_host recovery.example 10.0.0.2; then
|
||||
echo 'FAIL unrelated recovery host identified as production' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'PASS recovery host identity guard'
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
guard=$(mktemp)
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$guard" "$output"' EXIT
|
||||
|
||||
awk '
|
||||
/^if \[\[ \$EUID -ne 0 \]\]; then$/ { capture = 1 }
|
||||
capture { print }
|
||||
capture && /^fi$/ { found = 1; exit }
|
||||
END { if (!found) exit 1 }
|
||||
' "$script_dir/audit-host.sh" | sed 's/\$EUID/\$TEST_EUID/g' >"$guard" || {
|
||||
echo 'Could not extract the host audit root guard.' >&2
|
||||
exit 1
|
||||
}
|
||||
if ! grep -Fq 'if [[ $TEST_EUID -ne 0 ]]; then' "$guard"; then
|
||||
echo 'Extracted guard does not use the simulated effective UID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if TEST_EUID=0 bash "$guard" >"$output" 2>&1; then
|
||||
[[ ! -s $output ]] || { cat "$output" >&2; exit 1; }
|
||||
else
|
||||
echo 'Host audit root guard rejected simulated UID 0.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if TEST_EUID=65534 bash "$guard" >"$output" 2>&1; then
|
||||
echo 'Host audit root guard accepted simulated non-root UID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'Run this read-only audit as root.' "$output"
|
||||
echo 'PASS host audit root guard accepts UID 0 and rejects non-root UIDs'
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if bash "$(dirname "${BASH_SOURCE[0]}")/verify-helper.sh" unexpected >/dev/null 2>&1; then
|
||||
echo 'FAIL verifier helper accepted command arguments' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'PASS verifier helper rejects command arguments'
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
if [[ $# -ne 0 || $EUID -ne 0 || ! -d $checkout || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
|
||||
echo 'Refusing verification: the approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]]; then
|
||||
echo 'Refusing verification: the checkout contains a writable or non-root-owned path.' >&2
|
||||
exit 1
|
||||
fi
|
||||
exec /bin/bash "$checkout/infra/bootstrap/verify.sh" --remote
|
||||
Executable
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${1:-} != --remote ]]; then
|
||||
ssh_alias=${1:-akyldash_vps}
|
||||
exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n /usr/local/sbin/akyldash-verify'
|
||||
fi
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "Run the remote check as root; it is read-only." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for package in git docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do
|
||||
if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx 'install ok installed'; then
|
||||
printf 'PASS package %s\n' "$package"
|
||||
else
|
||||
printf 'FAIL package %s missing\n' "$package"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if systemctl is-active --quiet docker; then
|
||||
echo 'PASS Docker service active'
|
||||
else
|
||||
echo 'FAIL Docker service inactive'
|
||||
failed=1
|
||||
fi
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
echo 'PASS Docker Compose responds'
|
||||
else
|
||||
echo 'FAIL Docker Compose unavailable'
|
||||
failed=1
|
||||
fi
|
||||
if python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' 2>/dev/null; then
|
||||
echo 'PASS Docker log rotation configuration'
|
||||
else
|
||||
echo 'FAIL Docker log rotation configuration differs from repository target'
|
||||
failed=1
|
||||
fi
|
||||
if id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
|
||||
echo 'FAIL codex has sudo or Docker group access'
|
||||
failed=1
|
||||
else
|
||||
echo 'PASS codex is outside sudo and docker groups'
|
||||
fi
|
||||
|
||||
for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
|
||||
if [[ -d $path ]]; then
|
||||
printf 'PASS directory %s\n' "$path"
|
||||
else
|
||||
printf 'FAIL directory %s missing\n' "$path"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
for path in /etc/akyldash /var/backups/akyldash; do
|
||||
if [[ -d $path && $(stat -c %a "$path") == 700 ]]; then
|
||||
printf 'PASS permissions %s mode 0700\n' "$path"
|
||||
elif [[ -d $path ]]; then
|
||||
printf 'FAIL permissions %s mode %s, expected 0700\n' "$path" "$(stat -c %a "$path")"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
for path in /etc/akyldash/production.env /etc/akyldash/review_secret; do
|
||||
if [[ -f $path && $(stat -c %U:%a "$path") == root:600 ]]; then
|
||||
printf 'PASS protected file %s mode 0600\n' "$path"
|
||||
elif [[ -e $path ]]; then
|
||||
printf 'FAIL protected file %s must be root-owned mode 0600\n' "$path"
|
||||
failed=1
|
||||
else
|
||||
printf 'FAIL protected file %s missing\n' "$path"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
compose_file=$checkout/deploy/production/compose.yaml
|
||||
if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Caddyfile deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
|
||||
echo 'PASS production Compose, Caddyfile, and Dockerfiles match the checked-out Git revision'
|
||||
else
|
||||
echo 'FAIL production Compose or Dockerfile has local drift'
|
||||
failed=1
|
||||
fi
|
||||
if docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --quiet >/dev/null 2>&1; then
|
||||
echo 'PASS rendered production Compose configuration'
|
||||
else
|
||||
echo 'FAIL production Compose configuration cannot be rendered'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
if ufw status | grep -q '^Status: active$'; then
|
||||
echo 'PASS UFW active'
|
||||
if ! ufw status verbose | grep -q '^Default: deny (incoming)'; then
|
||||
echo 'FAIL UFW default incoming policy is not deny'
|
||||
failed=1
|
||||
fi
|
||||
while read -r rule action direction; do
|
||||
[[ $action == ALLOW && $direction == IN ]] || continue
|
||||
ports=${rule//\/tcp/}
|
||||
ports=${ports//\/udp/}
|
||||
IFS=, read -ra allowed <<< "$ports"
|
||||
for port in "${allowed[@]}"; do
|
||||
case "$port" in
|
||||
22|80|443) ;;
|
||||
*) printf 'FAIL unexpected UFW allow rule %s\n' "$rule"; failed=1 ;;
|
||||
esac
|
||||
done
|
||||
done < <(ufw status | awk 'NR > 3 && NF >= 3 {print $1, $2, $3}')
|
||||
else
|
||||
echo 'FAIL UFW inactive or unavailable'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
tcp_listeners=$(ss -H -lnt | awk '{print $4}')
|
||||
for endpoint in $tcp_listeners; do
|
||||
address=${endpoint%:*}
|
||||
port=${endpoint##*:}
|
||||
case "$address" in 127.*|::1|\[::1\]) continue ;; esac
|
||||
case "$port" in 22|80|443) ;; *) printf 'FAIL unexpected non-loopback TCP listener %s\n' "$endpoint"; failed=1 ;; esac
|
||||
done
|
||||
printf 'INFO non-loopback TCP listeners: %s\n' "${tcp_listeners//$'\n'/,}"
|
||||
|
||||
udp_listeners=$(ss -H -lnu | awk '{print $5}')
|
||||
for endpoint in $udp_listeners; do
|
||||
address=${endpoint%:*}
|
||||
case "$address" in 127.*|::1|\[::1\]) continue ;; esac
|
||||
printf 'FAIL unexpected non-loopback UDP listener %s\n' "$endpoint"
|
||||
failed=1
|
||||
done
|
||||
|
||||
expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
|
||||
if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \
|
||||
-x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
|
||||
-x /usr/local/sbin/akyldash-compose && $(stat -c %U:%a /usr/local/sbin/akyldash-compose) == root:750 && \
|
||||
-f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
|
||||
-f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
|
||||
$(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then
|
||||
echo 'PASS read-only verifier, Compose status/config helper, rollback helper, and exact sudoers rules present'
|
||||
else
|
||||
echo 'FAIL restricted helper or sudoers installation missing/different'
|
||||
failed=1
|
||||
fi
|
||||
if [[ -f /opt/akyldash/deployed-image.txt ]]; then
|
||||
expected=$(cat /opt/akyldash/deployed-image.txt)
|
||||
container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
|
||||
actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)
|
||||
if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then
|
||||
printf 'PASS running backend image ID matches %s\n' "$expected"
|
||||
else
|
||||
echo 'FAIL running backend image does not match recorded digest'
|
||||
failed=1
|
||||
fi
|
||||
else
|
||||
echo 'FAIL deployed image digest record missing'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
expected_proxy=$(docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --format json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["services"]["caddy"]["image"])' 2>/dev/null || true)
|
||||
proxy_container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=caddy | head -n1)
|
||||
actual_proxy=$(docker inspect --format '{{.Config.Image}}' "$proxy_container" 2>/dev/null || true)
|
||||
if [[ $expected_proxy == *@sha256:* && $actual_proxy == "$expected_proxy" ]]; then
|
||||
printf 'PASS running Caddy image matches %s\n' "$expected_proxy"
|
||||
else
|
||||
echo 'FAIL running Caddy image does not match the pinned Compose digest'
|
||||
failed=1
|
||||
fi
|
||||
exit "$failed"
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Require non-empty completed normalized audits and compare every file row."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import stat
|
||||
import sys
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ComparisonError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def shared_inventory_lock(result_path: Path, label: str):
|
||||
lock_path = result_path.absolute().parent / ".inventory.lock"
|
||||
if not hasattr(os, "O_NOFOLLOW"):
|
||||
raise ComparisonError(f"{label}: safe inventory lock opens are unsupported")
|
||||
try:
|
||||
descriptor = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
except OSError as error:
|
||||
raise ComparisonError(f"{label}: cannot open inventory lock: {error}") from error
|
||||
try:
|
||||
metadata = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(metadata.st_mode)
|
||||
or metadata.st_nlink != 1
|
||||
or stat.S_IMODE(metadata.st_mode) & 0o077
|
||||
):
|
||||
raise ComparisonError(f"{label}: inventory lock must be a private regular file")
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
except BlockingIOError as error:
|
||||
raise ComparisonError(f"{label}: inventory is active (lock busy)") from error
|
||||
yield
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def read_audit(result_path: Path, label: str) -> tuple[sqlite3.Connection, dict[str, Any]]:
|
||||
result_path = result_path.absolute()
|
||||
try:
|
||||
metadata = result_path.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
||||
raise ComparisonError(f"{label}: result.json must be a regular file")
|
||||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
||||
raise ComparisonError(f"{label}: cannot read result.json: {error}") from error
|
||||
|
||||
if not isinstance(result, dict):
|
||||
raise ComparisonError(f"{label}: result.json must contain an object")
|
||||
if result.get("status") != "complete" or result.get("verification_status") != "passed":
|
||||
raise ComparisonError(f"{label}: inventory must be complete and passed")
|
||||
verifier_sha256 = result.get("verifier_sha256")
|
||||
if not isinstance(verifier_sha256, str) or len(verifier_sha256) != 64:
|
||||
raise ComparisonError(f"{label}: verifier SHA-256 is missing")
|
||||
normalized_root = result.get("roots", {}).get("normalized")
|
||||
if not isinstance(normalized_root, str):
|
||||
raise ComparisonError(f"{label}: normalized root is missing")
|
||||
summary = result.get("manifest_summary", {}).get("normalized")
|
||||
if not isinstance(summary, dict):
|
||||
raise ComparisonError(f"{label}: normalized manifest summary is missing")
|
||||
for key in ("documents", "fragments"):
|
||||
if not isinstance(summary.get(key), int) or summary[key] <= 0:
|
||||
raise ComparisonError(f"{label}: normalized {key} must be greater than zero")
|
||||
|
||||
database_path = result_path.with_name("inventory.sqlite3")
|
||||
try:
|
||||
db_metadata = database_path.lstat()
|
||||
if (
|
||||
stat.S_ISLNK(db_metadata.st_mode)
|
||||
or not stat.S_ISREG(db_metadata.st_mode)
|
||||
or db_metadata.st_nlink != 1
|
||||
):
|
||||
raise ComparisonError(f"{label}: inventory.sqlite3 must be a regular unlinked file")
|
||||
for suffix in ("-wal", "-shm", "-journal"):
|
||||
if Path(f"{database_path}{suffix}").exists():
|
||||
raise ComparisonError(f"{label}: SQLite sidecar files are present")
|
||||
connection = sqlite3.connect(database_path.resolve().as_uri() + "?mode=ro&immutable=1", uri=True)
|
||||
connection.row_factory = sqlite3.Row
|
||||
if connection.execute("PRAGMA quick_check").fetchone()[0] != "ok":
|
||||
raise ComparisonError(f"{label}: inventory database failed quick_check")
|
||||
manifest_documents = connection.execute(
|
||||
"SELECT count(*) FROM manifest_docs WHERE root_id='normalized'"
|
||||
).fetchone()[0]
|
||||
if manifest_documents != summary["documents"]:
|
||||
raise ComparisonError(f"{label}: normalized document count does not match inventory")
|
||||
files, byte_count, unfinished = connection.execute(
|
||||
"SELECT count(*),coalesce(sum(size),0),"
|
||||
"sum(CASE WHEN status!='hashed' OR size IS NULL OR sha256 IS NULL THEN 1 ELSE 0 END) "
|
||||
"FROM files WHERE root_id='normalized'"
|
||||
).fetchone()
|
||||
progress = result.get("progress", {}).get("normalized", {})
|
||||
if unfinished or files != progress.get("files") or byte_count != progress.get("bytes"):
|
||||
raise ComparisonError(f"{label}: inventory rows do not match completed scan counters")
|
||||
return connection, {
|
||||
"root": normalized_root,
|
||||
"documents": summary["documents"],
|
||||
"fragments": summary["fragments"],
|
||||
"files": files,
|
||||
"bytes": byte_count,
|
||||
"verifier_sha256": verifier_sha256,
|
||||
"schema_version": result.get("schema_version"),
|
||||
}
|
||||
except sqlite3.Error as error:
|
||||
raise ComparisonError(f"{label}: cannot validate inventory database: {error}") from error
|
||||
|
||||
|
||||
def compare_inventories(source_result: Path, staging_result: Path) -> dict[str, int]:
|
||||
with ExitStack() as locks:
|
||||
locks.enter_context(shared_inventory_lock(source_result, "source"))
|
||||
locks.enter_context(shared_inventory_lock(staging_result, "staging"))
|
||||
source_db, source = read_audit(source_result, "source")
|
||||
try:
|
||||
staging_db, staging = read_audit(staging_result, "staging")
|
||||
except Exception:
|
||||
source_db.close()
|
||||
raise
|
||||
try:
|
||||
if source["root"] == staging["root"]:
|
||||
raise ComparisonError("source and staging must be different normalized roots")
|
||||
if source["verifier_sha256"] != staging["verifier_sha256"] or source["schema_version"] != staging["schema_version"]:
|
||||
raise ComparisonError("source and staging audits must use the same verifier and schema")
|
||||
if (source["documents"], source["fragments"]) != (staging["documents"], staging["fragments"]):
|
||||
raise ComparisonError("source and staging document or fragment counts differ")
|
||||
|
||||
query = (
|
||||
"SELECT relative_path,size,sha256 FROM files "
|
||||
"WHERE root_id='normalized' ORDER BY relative_path"
|
||||
)
|
||||
left = iter(source_db.execute(query))
|
||||
right = iter(staging_db.execute(query))
|
||||
mismatch_count = 0
|
||||
for source_row, staging_row in itertools.zip_longest(left, right):
|
||||
if source_row != staging_row:
|
||||
mismatch_count += 1
|
||||
if mismatch_count:
|
||||
raise ComparisonError(f"source and staging differ in {mismatch_count} inventory rows")
|
||||
return {"documents": source["documents"], "fragments": source["fragments"], "files": source["files"], "bytes": source["bytes"]}
|
||||
finally:
|
||||
source_db.close()
|
||||
staging_db.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("source_result", type=Path)
|
||||
parser.add_argument("staging_result", type=Path)
|
||||
arguments = parser.parse_args()
|
||||
try:
|
||||
summary = compare_inventories(arguments.source_result, arguments.staging_result)
|
||||
except (ComparisonError, OSError) as error:
|
||||
print(f"FAIL: {error}", file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
"PASS: matched "
|
||||
f"{summary['documents']} documents, {summary['fragments']} fragments, "
|
||||
f"{summary['files']} files, {summary['bytes']} bytes"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -ne 1 || ( $1 != config && $1 != status ) ]]; then
|
||||
echo 'Usage: akyldash-compose {config|status}' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo 'Run this helper through its fixed sudoers command.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
|
||||
echo 'Refusing Compose access: approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]]; then
|
||||
echo 'Refusing Compose access: checkout contains a writable or non-root-owned path.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose=(docker compose --env-file /etc/akyldash/production.env -f "$checkout/deploy/production/compose.yaml")
|
||||
case $1 in
|
||||
config) "${compose[@]}" config --quiet ;;
|
||||
status) "${compose[@]}" ps ;;
|
||||
esac
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run this installer as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
rollback_source=$checkout/infra/deploy/rollback.py
|
||||
backup_source=$checkout/infra/backup/archive.py
|
||||
verify_source=$checkout/infra/bootstrap/verify-helper.sh
|
||||
compose_source=$checkout/infra/deploy/compose-helper.sh
|
||||
if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then
|
||||
echo "Install the approved repository checkout at $checkout first." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -L $checkout ]]; then
|
||||
echo "The checkout path must not be a symlink." >&2
|
||||
exit 1
|
||||
fi
|
||||
for directory in /opt /opt/akyldash; do
|
||||
mode=$(stat -c %a "$directory")
|
||||
if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then
|
||||
echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
|
||||
echo "Create codex without sudo or Docker-group membership before installing the helper." >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then
|
||||
echo "Existing codex sudoers rules need owner review before installing the helper." >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ $(stat -c %U "$checkout") != root || -n $unsafe ]]; then
|
||||
echo "The checkout must be root-owned and not group/world-writable." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v visudo >/dev/null 2>&1; then
|
||||
echo "visudo is required to validate the restricted sudoers rule." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
|
||||
if [[ -L $path ]]; then
|
||||
echo "Refusing symlink at $path." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
install -d -o root -g root -m 0755 /usr/local/lib/akyldash
|
||||
install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
|
||||
install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify
|
||||
install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
|
||||
install -o root -g root -m 0750 "$compose_source" /usr/local/sbin/akyldash-compose
|
||||
|
||||
sudoers_file=/etc/sudoers.d/akyldash-codex
|
||||
expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
|
||||
created=false
|
||||
if [[ -e $sudoers_file ]]; then
|
||||
if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
|
||||
echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX)
|
||||
trap 'rm -f "$temp"' EXIT
|
||||
printf '%s\n' "$expected" > "$temp"
|
||||
chmod 0440 "$temp"
|
||||
visudo -cf "$temp"
|
||||
visudo -c -f /etc/sudoers
|
||||
mv -n "$temp" "$sudoers_file"
|
||||
if [[ $(cat "$sudoers_file") != "$expected" ]]; then
|
||||
echo "Could not install the expected sudoers rule." >&2
|
||||
exit 1
|
||||
fi
|
||||
created=true
|
||||
fi
|
||||
|
||||
if ! visudo -c -f /etc/sudoers; then
|
||||
if [[ $created == true ]]; then
|
||||
rm -f "$sudoers_file"
|
||||
fi
|
||||
echo "Sudoers validation failed; the newly added rule was removed." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Installed fixed read-only verification/Compose commands and guarded rollback; codex receives no general sudo or Docker access."
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo 'Run this configuration step as root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
example=$checkout/deploy/production/.env.example
|
||||
config_dir=/etc/akyldash
|
||||
env_file=$config_dir/production.env
|
||||
secret_file=$config_dir/review_secret
|
||||
if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root || ! -f $example ]]; then
|
||||
echo 'Refusing configuration: approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]] || ! git -C "$checkout" diff --quiet HEAD -- deploy/production/.env.example; then
|
||||
echo 'Refusing configuration: checkout is writable or .env.example differs from Git.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d $config_dir || $(stat -c %U:%G:%a "$config_dir") != root:root:700 ]]; then
|
||||
echo 'Expected /etc/akyldash to be root-owned mode 0700.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -e $env_file || -L $env_file || -e $secret_file || -L $secret_file ]]; then
|
||||
echo 'Production env or review secret already exists; refusing to overwrite either file.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
env_tmp=$(mktemp "$config_dir/.production.env.XXXXXX")
|
||||
secret_tmp=$(mktemp "$config_dir/.review_secret.XXXXXX")
|
||||
trap 'rm -f "$env_tmp" "$secret_tmp"' EXIT
|
||||
install -o root -g root -m 0600 "$example" "$env_tmp"
|
||||
openssl rand -hex 32 > "$secret_tmp"
|
||||
chown root:root "$secret_tmp"
|
||||
chmod 0600 "$secret_tmp"
|
||||
mv -n "$env_tmp" "$env_file"
|
||||
if [[ -e $env_tmp ]]; then
|
||||
echo 'Production env already appeared; refusing to continue.' >&2
|
||||
exit 1
|
||||
fi
|
||||
mv -n "$secret_tmp" "$secret_file"
|
||||
if [[ -e $secret_tmp ]]; then
|
||||
rm -f "$env_file"
|
||||
echo 'Review secret already appeared; removed the new env file and stopped.' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'Created root-owned production.env and review_secret with mode 0600. No services or volumes were changed.'
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore prior local application images from a verified cold backup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path("/opt/akyldash/current")
|
||||
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
||||
ENV_FILE = Path("/etc/akyldash/production.env")
|
||||
BACKUP_DIR = Path("/var/backups/akyldash")
|
||||
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
|
||||
APP_SERVICES = {"backend", "opensearch"}
|
||||
IMAGE_TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*")
|
||||
IMAGE_DIGEST_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}")
|
||||
|
||||
|
||||
def is_digest_reference(reference: str) -> bool:
|
||||
return IMAGE_DIGEST_RE.fullmatch(reference) is not None
|
||||
|
||||
|
||||
def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
|
||||
images = manifest.get("images")
|
||||
if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
|
||||
raise ValueError("backup has no previous backend and OpenSearch images")
|
||||
plan = {}
|
||||
for service in sorted(APP_SERVICES):
|
||||
image = images[service]
|
||||
if not isinstance(image, dict):
|
||||
raise ValueError(f"invalid image record for {service}")
|
||||
reference = image.get("configured")
|
||||
image_id = image.get("image_id")
|
||||
if reference != configured.get(service):
|
||||
raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
|
||||
if not isinstance(reference, str) or not (
|
||||
IMAGE_TAG_RE.fullmatch(reference) or is_digest_reference(reference)
|
||||
):
|
||||
raise ValueError(f"invalid image reference for {service}")
|
||||
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
||||
raise ValueError(f"invalid previous image ID for {service}")
|
||||
plan[service] = (image_id, reference)
|
||||
return plan
|
||||
|
||||
|
||||
def verify_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
||||
for image_id, reference in plan.values():
|
||||
try:
|
||||
actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise ValueError(f"previous image is unavailable: {image_id}") from error
|
||||
if actual != image_id:
|
||||
raise ValueError(f"previous image is unavailable: {image_id}")
|
||||
if is_digest_reference(reference):
|
||||
try:
|
||||
referenced_id = run(
|
||||
["docker", "image", "inspect", "--format", "{{.Id}}", reference], capture=True
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise ValueError(f"previous digest reference is unavailable: {reference}") from error
|
||||
if referenced_id != image_id:
|
||||
raise ValueError(f"previous digest reference does not match its recorded image: {reference}")
|
||||
|
||||
|
||||
def require_tag_rollback(plan: dict[str, tuple[str, str]]) -> None:
|
||||
if any(is_digest_reference(reference) for _image_id, reference in plan.values()):
|
||||
raise ValueError(
|
||||
"rollback of digest-pinned images is not supported safely; no services or image references were changed"
|
||||
)
|
||||
|
||||
|
||||
def tag_previous_images(plan: dict[str, tuple[str, str]]) -> None:
|
||||
require_tag_rollback(plan)
|
||||
for image_id, reference in plan.values():
|
||||
run(["docker", "image", "tag", image_id, reference])
|
||||
|
||||
|
||||
def run(command: list[str], *, capture: bool = False) -> str:
|
||||
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def compose(*args: str, capture: bool = False) -> str:
|
||||
return run(
|
||||
["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
|
||||
def record_backend_image() -> None:
|
||||
container = compose("ps", "--quiet", "backend", capture=True)
|
||||
if not container:
|
||||
raise ValueError("backend container is missing after rollback")
|
||||
image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
|
||||
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
||||
raise ValueError("backend returned an invalid image ID")
|
||||
target = Path("/opt/akyldash/deployed-image.txt")
|
||||
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
stream.write(f"{image_id}\n".encode("ascii"))
|
||||
os.chmod(temporary, 0o600)
|
||||
try:
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def rollback(backup: Path, data_compatible: bool) -> None:
|
||||
if not data_compatible:
|
||||
raise ValueError("confirm data compatibility with --data-compatible before rollback")
|
||||
if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
|
||||
raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
|
||||
if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("rollback backup must be root-owned mode 0600")
|
||||
checksum = Path(f"{backup}.sha256")
|
||||
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
|
||||
|
||||
sys.path.insert(0, str(BACKUP_TOOL.parent))
|
||||
from archive import digest, verify_archive
|
||||
|
||||
manifest = verify_archive(backup)
|
||||
deployment_files = {
|
||||
relative: digest(ROOT / relative)
|
||||
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
}
|
||||
if deployment_files != manifest.get("deployment_files"):
|
||||
raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
|
||||
rendered = json.loads(compose("config", "--format", "json", capture=True))
|
||||
configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
|
||||
plan = image_restore_plan(manifest, configured)
|
||||
verify_previous_images(plan)
|
||||
require_tag_rollback(plan)
|
||||
|
||||
backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
|
||||
backup_output = BACKUP_DIR / backup_name
|
||||
compose("stop")
|
||||
try:
|
||||
run(
|
||||
[
|
||||
"python3",
|
||||
str(BACKUP_TOOL),
|
||||
"create",
|
||||
"--compose-file",
|
||||
str(COMPOSE_FILE),
|
||||
"--env-file",
|
||||
str(ENV_FILE),
|
||||
"--output",
|
||||
str(backup_output),
|
||||
]
|
||||
)
|
||||
run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
|
||||
except Exception:
|
||||
compose("up", "--detach")
|
||||
raise
|
||||
|
||||
tag_previous_images(plan)
|
||||
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
||||
run(
|
||||
[
|
||||
"curl",
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--resolve",
|
||||
"akyldash.kg:443:127.0.0.1",
|
||||
"https://akyldash.kg/review",
|
||||
"--output",
|
||||
"/dev/null",
|
||||
]
|
||||
)
|
||||
record_backend_image()
|
||||
print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("backup", type=Path)
|
||||
parser.add_argument("--data-compatible", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if os.geteuid() != 0:
|
||||
raise ValueError("rollback must run through the restricted root helper")
|
||||
rollback(args.backup, args.data_compatible)
|
||||
except Exception as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
helper=$(dirname "$0")/compose-helper.sh
|
||||
for action in deploy up down docker; do
|
||||
if bash "$helper" "$action" >/dev/null 2>&1; then
|
||||
printf 'FAIL Compose helper accepted %s\n' "$action" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo 'PASS Compose helper rejects operations outside its read-only allowlist'
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
source "$script_dir/../test/run-as-unprivileged.sh"
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if run_as_unprivileged bash "$script_dir/prepare-production-config.sh" >"$output" 2>&1; then
|
||||
echo 'Production config setup unexpectedly succeeded as an unprivileged UID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'Run this configuration step as root.' "$output"
|
||||
@@ -0,0 +1,132 @@
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from compare_corpus_inventories import ComparisonError, compare_inventories
|
||||
|
||||
|
||||
class CompareCorpusInventoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.base = Path(self.temp.name)
|
||||
self.source = self._artifact("source", "/data/source")
|
||||
self.staging = self._artifact("staging", "/data/staging")
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def _artifact(self, name, root, documents=1, fragments=2, rows=None):
|
||||
directory = self.base / name
|
||||
directory.mkdir(mode=0o700)
|
||||
(directory / ".inventory.lock").touch(mode=0o600)
|
||||
database = directory / "inventory.sqlite3"
|
||||
connection = sqlite3.connect(database)
|
||||
connection.execute(
|
||||
"CREATE TABLE files (root_id TEXT, relative_path TEXT, size INTEGER, sha256 TEXT, status TEXT)"
|
||||
)
|
||||
connection.execute("CREATE TABLE manifest_docs (root_id TEXT, code TEXT)")
|
||||
connection.executemany(
|
||||
"INSERT INTO manifest_docs VALUES ('normalized',?)",
|
||||
[(str(code),) for code in range(documents)],
|
||||
)
|
||||
if rows is None:
|
||||
rows = [
|
||||
("manifest.sqlite3", 16, "a" * 64),
|
||||
("documents/1/document.json", 24, "b" * 64),
|
||||
]
|
||||
connection.executemany(
|
||||
"INSERT INTO files VALUES ('normalized',?,?,?,'hashed')", rows
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
result = {
|
||||
"status": "complete",
|
||||
"verification_status": "passed",
|
||||
"schema_version": 2,
|
||||
"verifier_sha256": "c" * 64,
|
||||
"roots": {"normalized": root},
|
||||
"manifest_summary": {
|
||||
"normalized": {"documents": documents, "fragments": fragments}
|
||||
},
|
||||
"progress": {
|
||||
"normalized": {
|
||||
"files": len(rows),
|
||||
"bytes": sum(row[1] for row in rows),
|
||||
}
|
||||
},
|
||||
}
|
||||
result_path = directory / "result.json"
|
||||
result_path.write_text(json.dumps(result), encoding="utf-8")
|
||||
return result_path
|
||||
|
||||
def test_nonempty_source_and_staging_inventories_match(self):
|
||||
self.assertEqual(
|
||||
compare_inventories(self.source, self.staging),
|
||||
{"documents": 1, "fragments": 2, "files": 2, "bytes": 40},
|
||||
)
|
||||
|
||||
def test_empty_source_is_rejected(self):
|
||||
empty = self._artifact("empty-source", "/data/empty-source", documents=0, fragments=0)
|
||||
with self.assertRaisesRegex(ComparisonError, "source: normalized documents"):
|
||||
compare_inventories(empty, self.staging)
|
||||
|
||||
def test_empty_staging_is_rejected(self):
|
||||
empty = self._artifact("empty-staging", "/data/empty-staging", documents=0, fragments=0)
|
||||
with self.assertRaisesRegex(ComparisonError, "staging: normalized documents"):
|
||||
compare_inventories(self.source, empty)
|
||||
|
||||
def test_zero_fragments_are_rejected_even_with_documents(self):
|
||||
empty = self._artifact("no-fragments", "/data/no-fragments", fragments=0)
|
||||
with self.assertRaisesRegex(ComparisonError, "normalized fragments"):
|
||||
compare_inventories(self.source, empty)
|
||||
|
||||
def test_different_file_sha_is_rejected(self):
|
||||
changed = self._artifact(
|
||||
"changed", "/data/changed",
|
||||
rows=[("manifest.sqlite3", 16, "a" * 64), ("documents/1/document.json", 24, "d" * 64)],
|
||||
)
|
||||
with self.assertRaisesRegex(ComparisonError, "differ in 1 inventory rows"):
|
||||
compare_inventories(self.source, changed)
|
||||
|
||||
def test_busy_source_lock_is_rejected(self):
|
||||
self._assert_busy_lock(self.source, "source")
|
||||
|
||||
def test_busy_staging_lock_is_rejected(self):
|
||||
self._assert_busy_lock(self.staging, "staging")
|
||||
|
||||
def _assert_busy_lock(self, result_path, label):
|
||||
descriptor = os.open(result_path.parent / ".inventory.lock", os.O_RDWR)
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
with self.assertRaisesRegex(ComparisonError, rf"{label}: inventory is active"):
|
||||
compare_inventories(self.source, self.staging)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def test_missing_lock_is_rejected_without_creating_it(self):
|
||||
lock = self.source.parent / ".inventory.lock"
|
||||
lock.unlink()
|
||||
with self.assertRaisesRegex(ComparisonError, "source: cannot open inventory lock"):
|
||||
compare_inventories(self.source, self.staging)
|
||||
self.assertFalse(lock.exists())
|
||||
|
||||
def test_symlink_lock_is_rejected(self):
|
||||
lock = self.source.parent / ".inventory.lock"
|
||||
lock.unlink()
|
||||
target = self.base / "external-lock"
|
||||
target.touch(mode=0o600)
|
||||
lock.symlink_to(target)
|
||||
with self.assertRaisesRegex(ComparisonError, "source: cannot open inventory lock"):
|
||||
compare_inventories(self.source, self.staging)
|
||||
|
||||
def test_read_only_lock_can_be_shared(self):
|
||||
(self.source.parent / ".inventory.lock").chmod(0o400)
|
||||
self.assertEqual(compare_inventories(self.source, self.staging)["documents"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from validate_release import validate_manifest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
REPOSITORIES = {
|
||||
"backend": ["registry.invalid/akyldash/backend"],
|
||||
"opensearch": ["registry.invalid/akyldash/opensearch"],
|
||||
}
|
||||
|
||||
|
||||
class ReleaseManifestTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.policy = {"schema_version": 1, "repositories": REPOSITORIES}
|
||||
self.manifest = {
|
||||
"schema_version": 1,
|
||||
"source_ref": "refs/heads/main",
|
||||
"commit": "a" * 40,
|
||||
"images": {
|
||||
"backend": "registry.invalid/akyldash/backend@sha256:" + "b" * 64,
|
||||
"opensearch": "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64,
|
||||
},
|
||||
}
|
||||
|
||||
def test_accepts_both_policy_pinned_images(self):
|
||||
self.assertEqual(validate_manifest(self.manifest, self.policy), self.manifest)
|
||||
|
||||
def test_rejects_untrusted_ref(self):
|
||||
self.manifest["source_ref"] = "refs/pull/12/head"
|
||||
with self.assertRaisesRegex(ValueError, "source_ref"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_tag_instead_of_digest(self):
|
||||
self.manifest["images"]["backend"] = "registry.invalid/akyldash/backend:latest"
|
||||
with self.assertRaisesRegex(ValueError, "sha256"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_empty_image_reference(self):
|
||||
self.manifest["images"]["backend"] = ""
|
||||
with self.assertRaisesRegex(ValueError, "sha256"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_invalid_commit(self):
|
||||
self.manifest["commit"] = "not-a-commit"
|
||||
with self.assertRaisesRegex(ValueError, "commit"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_repository_outside_policy(self):
|
||||
self.manifest["images"]["backend"] = "registry.invalid/other/backend@sha256:" + "b" * 64
|
||||
with self.assertRaisesRegex(ValueError, "not allowed"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_missing_or_extra_images(self):
|
||||
self.manifest["images"].pop("opensearch")
|
||||
with self.assertRaisesRegex(ValueError, "exactly backend and opensearch"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
self.manifest["images"]["opensearch"] = "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64
|
||||
self.manifest["images"]["extra"] = "registry.invalid/akyldash/extra@sha256:" + "d" * 64
|
||||
with self.assertRaisesRegex(ValueError, "exactly backend and opensearch"):
|
||||
validate_manifest(self.manifest, self.policy)
|
||||
|
||||
def test_rejects_duplicate_json_keys(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
manifest_path = Path(directory) / "manifest.json"
|
||||
policy_path = Path(directory) / "policy.json"
|
||||
manifest_path.write_text(
|
||||
'{"schema_version":1,"source_ref":"refs/heads/main","commit":"' + "a" * 40
|
||||
+ '","images":{"backend":"first","backend":"second","opensearch":"third"}}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
policy_path.write_text(json.dumps(self.policy), encoding="utf-8")
|
||||
command = [
|
||||
"python3", str(ROOT / "infra/deploy/validate_release.py"), str(manifest_path),
|
||||
"--policy", str(policy_path),
|
||||
]
|
||||
result = subprocess.run(command, text=True, capture_output=True)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("duplicate JSON key: backend", result.stderr)
|
||||
|
||||
def test_rejects_malformed_json(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
manifest_path = Path(directory) / "manifest.json"
|
||||
policy_path = Path(directory) / "policy.json"
|
||||
manifest_path.write_text('{"schema_version":', encoding="utf-8")
|
||||
policy_path.write_text(json.dumps(self.policy), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["python3", str(ROOT / "infra/deploy/validate_release.py"), str(manifest_path), "--policy", str(policy_path)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn("cannot read valid JSON", result.stderr)
|
||||
|
||||
@unittest.skipUnless(shutil.which("docker"), "Docker CLI is unavailable")
|
||||
def test_compose_override_renders_digest_images_and_preserves_build(self):
|
||||
backend = "registry.invalid/akyldash/backend@sha256:" + "b" * 64
|
||||
opensearch = "registry.invalid/akyldash/opensearch@sha256:" + "c" * 64
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
temp = Path(directory)
|
||||
secret = temp / "review_secret"
|
||||
secret.touch()
|
||||
override = temp / "release-override.json"
|
||||
override.write_text(json.dumps({"services": {
|
||||
"backend": {"image": backend},
|
||||
"opensearch": {"image": opensearch},
|
||||
}}), encoding="utf-8")
|
||||
command = [
|
||||
"docker", "compose", "--env-file", str(ROOT / "deploy/production/.env.example"),
|
||||
"-f", str(ROOT / "deploy/production/compose.yaml"), "-f", str(override),
|
||||
"config", "--format", "json",
|
||||
]
|
||||
result = subprocess.run(
|
||||
command, text=True, capture_output=True,
|
||||
env={**os.environ, "REVIEW_SECRET_FILE": str(secret)},
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
services = json.loads(result.stdout)["services"]
|
||||
self.assertEqual(services["backend"]["image"], backend)
|
||||
self.assertEqual(services["opensearch"]["image"], opensearch)
|
||||
self.assertIn("build", services["backend"])
|
||||
self.assertIn("build", services["opensearch"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import rollback
|
||||
from rollback import image_restore_plan, require_tag_rollback, tag_previous_images, verify_previous_images
|
||||
|
||||
|
||||
class RollbackPlanTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.manifest = {
|
||||
"images": {
|
||||
"backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
|
||||
"opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
|
||||
}
|
||||
}
|
||||
self.configured = {
|
||||
"backend": "akyldash/backend:local",
|
||||
"opensearch": "akyldash/opensearch:local",
|
||||
}
|
||||
|
||||
def test_plans_only_recorded_local_images(self):
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
self.assertEqual(plan["backend"][1], "akyldash/backend:local")
|
||||
self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local")
|
||||
|
||||
def test_refuses_changed_compose_image_reference(self):
|
||||
with self.assertRaisesRegex(ValueError, "reference changed"):
|
||||
image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"})
|
||||
|
||||
def test_refuses_digest_reference_that_cannot_be_retagged(self):
|
||||
self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64
|
||||
self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
self.assertEqual(plan["backend"], ("sha256:" + "a" * 64, self.configured["backend"]))
|
||||
with self.assertRaisesRegex(ValueError, "not supported safely"):
|
||||
require_tag_rollback(plan)
|
||||
|
||||
def test_refuses_malformed_digest_reference(self):
|
||||
self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:xyz"
|
||||
self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
|
||||
with self.assertRaisesRegex(ValueError, "invalid image reference"):
|
||||
image_restore_plan(self.manifest, self.configured)
|
||||
|
||||
def test_refuses_invalid_recorded_image_id(self):
|
||||
self.manifest["images"]["backend"]["image_id"] = "sha256:not-a-digest"
|
||||
with self.assertRaisesRegex(ValueError, "invalid previous image ID"):
|
||||
image_restore_plan(self.manifest, self.configured)
|
||||
|
||||
def test_digest_reference_must_resolve_to_recorded_image_id(self):
|
||||
digest = "registry/backend@sha256:" + "c" * 64
|
||||
self.manifest["images"]["backend"]["configured"] = digest
|
||||
self.configured["backend"] = digest
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "a" * 64,
|
||||
"sha256:" + "b" * 64]):
|
||||
verify_previous_images(plan)
|
||||
with patch("rollback.run", side_effect=["sha256:" + "a" * 64, "sha256:" + "d" * 64]):
|
||||
with self.assertRaisesRegex(ValueError, "does not match its recorded image"):
|
||||
verify_previous_images(plan)
|
||||
|
||||
def test_missing_previous_image_fails_closed(self):
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
with patch("rollback.run", side_effect=subprocess.CalledProcessError(1, "docker")):
|
||||
with self.assertRaisesRegex(ValueError, "previous image is unavailable"):
|
||||
verify_previous_images(plan)
|
||||
|
||||
def test_digest_reference_is_never_tagged(self):
|
||||
digest = "registry/backend@sha256:" + "c" * 64
|
||||
plan = {"backend": ("sha256:" + "a" * 64, digest)}
|
||||
with patch("rollback.run") as run_mock:
|
||||
with self.assertRaisesRegex(ValueError, "not supported safely"):
|
||||
tag_previous_images(plan)
|
||||
run_mock.assert_not_called()
|
||||
|
||||
def test_digest_rollback_fails_before_stopping_or_changing_references(self):
|
||||
backend_id = "sha256:" + "a" * 64
|
||||
opensearch_id = "sha256:" + "b" * 64
|
||||
digest = "registry/backend@sha256:" + "c" * 64
|
||||
manifest = {
|
||||
"images": {
|
||||
"backend": {"configured": digest, "image_id": backend_id},
|
||||
"opensearch": self.manifest["images"]["opensearch"],
|
||||
},
|
||||
"deployment_files": {
|
||||
"deploy/production/compose.yaml": "compose-hash",
|
||||
"deploy/production/Caddyfile": "caddy-hash",
|
||||
},
|
||||
}
|
||||
rendered = {
|
||||
"services": {
|
||||
"backend": {"image": digest},
|
||||
"opensearch": {"image": self.configured["opensearch"]},
|
||||
}
|
||||
}
|
||||
fake_archive = SimpleNamespace(
|
||||
digest=lambda path: "compose-hash" if path.name == "compose.yaml" else "caddy-hash",
|
||||
verify_archive=lambda _backup: manifest,
|
||||
)
|
||||
backup = rollback.BACKUP_DIR / "synthetic.tar.gz"
|
||||
with (
|
||||
patch("rollback.Path.is_file", return_value=True),
|
||||
patch("rollback.Path.is_symlink", return_value=False),
|
||||
patch("rollback.Path.stat", return_value=SimpleNamespace(st_uid=0, st_mode=0o100600)),
|
||||
patch("rollback.compose", return_value=json.dumps(rendered)) as compose_mock,
|
||||
patch("rollback.run", side_effect=[backend_id, backend_id, opensearch_id]) as run_mock,
|
||||
patch.dict(sys.modules, {"archive": fake_archive}),
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "not supported safely"):
|
||||
rollback.rollback(backup, data_compatible=True)
|
||||
compose_mock.assert_called_once_with("config", "--format", "json", capture=True)
|
||||
self.assertEqual(run_mock.call_count, 3)
|
||||
self.assertTrue(all(call.args[0][:3] == ["docker", "image", "inspect"]
|
||||
for call in run_mock.call_args_list))
|
||||
|
||||
def test_tag_rollback_keeps_existing_image_tag_operation(self):
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
with patch("rollback.run") as run_mock:
|
||||
tag_previous_images(plan)
|
||||
self.assertEqual(run_mock.call_count, 2)
|
||||
run_mock.assert_any_call(["docker", "image", "tag", "sha256:" + "a" * 64,
|
||||
"akyldash/backend:local"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,328 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from verify_corpus import (
|
||||
InventoryError,
|
||||
database,
|
||||
insert_or_update_file,
|
||||
prefix_upper_bound,
|
||||
raw_file_rows,
|
||||
combined_source_hash,
|
||||
safe_relative_path,
|
||||
validate_fragment_file,
|
||||
verify_corpus,
|
||||
)
|
||||
|
||||
|
||||
class CorpusInventoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.base = Path(self.temp.name)
|
||||
self.raw = self.base / "raw"
|
||||
self.raw.mkdir()
|
||||
self._manifest()
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def _manifest(self):
|
||||
connection = sqlite3.connect(self.raw / "manifest.sqlite3")
|
||||
connection.executescript(
|
||||
"CREATE TABLE documents (code INTEGER PRIMARY KEY, source_sha256 TEXT, fetched_at TEXT);"
|
||||
"CREATE TABLE errors (code INTEGER PRIMARY KEY, message TEXT, failed_at TEXT);"
|
||||
)
|
||||
connection.close()
|
||||
|
||||
def _nonempty_corpus(self):
|
||||
document = self.raw / "documents/1"
|
||||
edition = document / "editions/2"
|
||||
edition.mkdir(parents=True)
|
||||
(document / "metadata.json").write_text('{"Code":"1"}\n', encoding="utf-8")
|
||||
(edition / "metadata.json").write_text('{"Code":"2"}\n', encoding="utf-8")
|
||||
html = edition / "ru.html"
|
||||
html.write_text("<p>Фрагмент</p>\n", encoding="utf-8")
|
||||
source_files = []
|
||||
for path in sorted(document.rglob("*")):
|
||||
if path.is_file():
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
source_files.append({"path": path.relative_to(self.raw).as_posix(), "sha256": digest})
|
||||
source_sha = combined_source_hash(source_files)
|
||||
raw_manifest = sqlite3.connect(self.raw / "manifest.sqlite3")
|
||||
raw_manifest.execute("INSERT INTO documents VALUES (1, ?, 'now')", (source_sha,))
|
||||
raw_manifest.commit()
|
||||
raw_manifest.close()
|
||||
|
||||
normalized = self.base / "normalized"
|
||||
language = normalized / "documents/1/editions/2/ru"
|
||||
language.mkdir(parents=True)
|
||||
fragment_text = "Фрагмент"
|
||||
fragments = [{
|
||||
"id": "document:1:edition:2:lang:ru:fragment:1",
|
||||
"document_code": "1",
|
||||
"edition_code": "2",
|
||||
"language": "ru",
|
||||
"position": 1,
|
||||
"text": fragment_text,
|
||||
"text_sha256": hashlib.sha256(fragment_text.encode("utf-8")).hexdigest(),
|
||||
"source_path": "documents/1/editions/2/ru.html",
|
||||
"source_sha256": next(item["sha256"] for item in source_files if item["path"].endswith("/ru.html")),
|
||||
}]
|
||||
(language / "fragments.json").write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
|
||||
(language / "content.txt").write_text(fragment_text + "\n", encoding="utf-8")
|
||||
(language / "content.html").write_text("<p>Фрагмент</p>\n", encoding="utf-8")
|
||||
document_json = {
|
||||
"schema_version": "1",
|
||||
"source_code": "1",
|
||||
"normalizer": {"version": "1.0.0"},
|
||||
"source": {"path": "documents/1", "files": source_files, "sha256": source_sha},
|
||||
"editions": [{"source_code": "2"}],
|
||||
}
|
||||
(normalized / "documents/1/document.json").write_text(json.dumps(document_json), encoding="utf-8")
|
||||
edition_json = {
|
||||
"source_code": "2",
|
||||
"available_languages": ["ru"],
|
||||
"source": {
|
||||
"path": "documents/1/editions/2",
|
||||
"files": [item for item in source_files if item["path"].startswith("documents/1/editions/2/")],
|
||||
},
|
||||
"images": [],
|
||||
}
|
||||
(normalized / "documents/1/editions/2/edition.json").write_text(json.dumps(edition_json), encoding="utf-8")
|
||||
normalized_manifest = sqlite3.connect(normalized / "manifest.sqlite3")
|
||||
normalized_manifest.execute(
|
||||
"CREATE TABLE documents (code TEXT PRIMARY KEY, source_sha256 TEXT, schema_version TEXT, normalizer_version TEXT, state TEXT)"
|
||||
)
|
||||
normalized_manifest.execute("INSERT INTO documents VALUES ('1', ?, '1', '1.0.0', 'success')", (source_sha,))
|
||||
normalized_manifest.commit()
|
||||
normalized_manifest.close()
|
||||
return normalized, source_sha
|
||||
|
||||
def test_output_inside_source_is_rejected_without_creating_it(self):
|
||||
output = self.raw / "artifacts"
|
||||
before_mode = self.raw.stat().st_mode
|
||||
with self.assertRaisesRegex(InventoryError, "separate"):
|
||||
verify_corpus(self.raw, None, output)
|
||||
self.assertFalse(output.exists())
|
||||
self.assertEqual(self.raw.stat().st_mode, before_mode)
|
||||
self.assertEqual({path.name for path in self.raw.iterdir()}, {"manifest.sqlite3"})
|
||||
|
||||
def test_symlink_output_is_rejected_without_changing_target(self):
|
||||
target = self.base / "target"
|
||||
target.mkdir(mode=0o755)
|
||||
link = self.base / "output-link"
|
||||
try:
|
||||
link.symlink_to(target, target_is_directory=True)
|
||||
except (NotImplementedError, OSError):
|
||||
self.skipTest("directory symlinks are unavailable")
|
||||
before_mode = target.stat().st_mode
|
||||
with self.assertRaisesRegex(InventoryError, "link or file"):
|
||||
verify_corpus(self.raw, None, link)
|
||||
self.assertEqual(target.stat().st_mode, before_mode)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
|
||||
def test_public_existing_output_is_rejected_without_chmod(self):
|
||||
output = self.base / "public-output"
|
||||
output.mkdir(mode=0o755)
|
||||
output.chmod(0o755)
|
||||
before_mode = output.stat().st_mode
|
||||
with self.assertRaisesRegex(InventoryError, "must be private"):
|
||||
verify_corpus(self.raw, None, output)
|
||||
self.assertEqual(output.stat().st_mode, before_mode)
|
||||
self.assertEqual(list(output.iterdir()), [])
|
||||
|
||||
def test_resume_rejects_symlink_inventory_without_touching_target(self):
|
||||
output = self.base / "artifacts"
|
||||
output.mkdir(mode=0o700)
|
||||
target = self.base / "do-not-touch.sqlite3"
|
||||
target.write_bytes(b"sentinel")
|
||||
link = output / "inventory.sqlite3"
|
||||
try:
|
||||
link.symlink_to(target)
|
||||
except (NotImplementedError, OSError):
|
||||
self.skipTest("file symlinks are unavailable")
|
||||
with self.assertRaisesRegex(InventoryError, "private regular file"):
|
||||
verify_corpus(self.raw, None, output, resume=True)
|
||||
self.assertEqual(target.read_bytes(), b"sentinel")
|
||||
|
||||
def test_rejects_unsafe_cross_platform_relative_paths(self):
|
||||
for value in ("../outside", "C:\\outside", "folder\\file", "//server/share"):
|
||||
with self.subTest(value=value), self.assertRaises(InventoryError):
|
||||
safe_relative_path(value)
|
||||
|
||||
def test_interrupted_inventory_resumes_and_detects_changed_input(self):
|
||||
payload = self.raw / "payload.txt"
|
||||
payload.write_text("before", encoding="utf-8")
|
||||
output = self.base / "artifacts"
|
||||
stopped = verify_corpus(self.raw, None, output, stop_after_entries=1)
|
||||
self.assertEqual(stopped["status"], "incomplete")
|
||||
|
||||
resumed = verify_corpus(self.raw, None, output, resume=True)
|
||||
self.assertEqual(resumed["status"], "complete")
|
||||
self.assertEqual(resumed["input_changes"]["raw"]["added"], 1)
|
||||
|
||||
payload.write_text("after", encoding="utf-8")
|
||||
changed = verify_corpus(self.raw, None, output, resume=True)
|
||||
self.assertEqual(changed["status"], "complete")
|
||||
self.assertEqual(changed["input_changes"]["raw"]["changed"], 1)
|
||||
rows = sqlite3.connect(output / "inventory.sqlite3").execute(
|
||||
"SELECT sha256 FROM files WHERE relative_path='payload.txt'"
|
||||
).fetchone()
|
||||
self.assertEqual(rows[0], hashlib.sha256(b"after").hexdigest())
|
||||
|
||||
def test_large_filesystem_identity_values_are_stored_without_integer_overflow(self):
|
||||
connection = database(self.base / "identity.sqlite3")
|
||||
insert_or_update_file(
|
||||
connection, "run", "raw", "payload", 1, "a" * 64,
|
||||
(2**100, 2**120, 1, 2**110, 2**115, 1), {"raw": {"added": 0, "changed": 0}},
|
||||
)
|
||||
device, inode, mtime_ns, ctime_ns = connection.execute(
|
||||
"SELECT device,inode,mtime_ns,ctime_ns FROM files"
|
||||
).fetchone()
|
||||
self.assertEqual((device, inode, mtime_ns, ctime_ns), tuple(map(str, (2**100, 2**120, 2**110, 2**115))))
|
||||
connection.close()
|
||||
|
||||
def test_document_prefix_uses_primary_key_range_and_excludes_neighbor_codes(self):
|
||||
connection = database(self.base / "paths.sqlite3")
|
||||
prefix = "documents/1/"
|
||||
upper = prefix_upper_bound(prefix)
|
||||
plan = connection.execute(
|
||||
"EXPLAIN QUERY PLAN SELECT relative_path FROM files WHERE root_id='raw' AND relative_path>=? AND relative_path<?",
|
||||
(prefix, upper),
|
||||
).fetchone()[3]
|
||||
self.assertIn("USING COVERING INDEX sqlite_autoindex_files_1", plan)
|
||||
connection.executemany(
|
||||
"INSERT INTO files(root_id,relative_path,status,last_seen_run) VALUES('raw',?,'hashed','run')",
|
||||
[(f"documents/{code}/a",) for code in range(20000)]
|
||||
+ [("documents/1x/a",)],
|
||||
)
|
||||
self.assertEqual([item["path"] for item in raw_file_rows(connection, "1")], ["documents/1/a"])
|
||||
connection.close()
|
||||
|
||||
def test_inventory_rejects_a_concurrent_writer_and_records_verifier_hash(self):
|
||||
output = self.base / "artifacts"
|
||||
output.mkdir(mode=0o700)
|
||||
lock_path = output / ".inventory.lock"
|
||||
descriptor = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
with self.assertRaisesRegex(InventoryError, "another inventory run"):
|
||||
verify_corpus(self.raw, None, output)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
result = verify_corpus(self.raw, None, output)
|
||||
self.assertEqual(result["status"], "complete")
|
||||
self.assertRegex(result["verifier_sha256"], r"^[a-f0-9]{64}$")
|
||||
|
||||
def test_removed_file_is_reported_on_resume(self):
|
||||
payload = self.raw / "payload.txt"
|
||||
payload.write_text("payload", encoding="utf-8")
|
||||
output = self.base / "artifacts"
|
||||
self.assertEqual(verify_corpus(self.raw, None, output)["status"], "complete")
|
||||
payload.unlink()
|
||||
result = verify_corpus(self.raw, None, output, resume=True)
|
||||
self.assertEqual(result["input_changes"]["raw"]["removed"], 1)
|
||||
self.assertIn("removed_file", {item["kind"] for item in result["finding_examples"]})
|
||||
|
||||
def test_added_file_is_reported_on_resume(self):
|
||||
output = self.base / "artifacts"
|
||||
self.assertEqual(verify_corpus(self.raw, None, output)["status"], "complete")
|
||||
(self.raw / "extra.txt").write_text("extra", encoding="utf-8")
|
||||
result = verify_corpus(self.raw, None, output, resume=True)
|
||||
self.assertEqual(result["input_changes"]["raw"]["added"], 1)
|
||||
|
||||
def test_manifest_error_missing_document_and_extra_directory_are_reported(self):
|
||||
connection = sqlite3.connect(self.raw / "manifest.sqlite3")
|
||||
connection.execute("INSERT INTO documents VALUES (1, ?, 'now')", ("a" * 64,))
|
||||
connection.execute("INSERT INTO errors VALUES (6, 'fetch failed', 'now')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
(self.raw / "documents" / "999").mkdir(parents=True)
|
||||
|
||||
result = verify_corpus(self.raw, None, self.base / "artifacts")
|
||||
kinds = {finding["kind"] for finding in result["finding_examples"]}
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("raw_manifest_error", kinds)
|
||||
self.assertIn("missing_raw_document_directory", kinds)
|
||||
self.assertIn("extra_raw_document_directory", kinds)
|
||||
|
||||
def test_empty_raw_and_normalized_manifests_complete_successfully(self):
|
||||
normalized = self.base / "normalized"
|
||||
normalized.mkdir()
|
||||
connection = sqlite3.connect(normalized / "manifest.sqlite3")
|
||||
connection.execute(
|
||||
"CREATE TABLE documents (code TEXT PRIMARY KEY, source_sha256 TEXT, schema_version TEXT, normalizer_version TEXT, state TEXT)"
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
result = verify_corpus(self.raw, normalized, self.base / "artifacts")
|
||||
self.assertEqual(result["status"], "complete", result["finding_examples"])
|
||||
self.assertEqual(result["manifest_summary"]["raw"]["documents"], 0)
|
||||
self.assertEqual(result["manifest_summary"]["normalized"]["documents"], 0)
|
||||
|
||||
def test_nonempty_raw_and_normalized_manifests_cross_validate_fragment_sources(self):
|
||||
normalized, _ = self._nonempty_corpus()
|
||||
result = verify_corpus(self.raw, normalized, self.base / "artifacts")
|
||||
self.assertEqual(result["status"], "complete", result["finding_examples"])
|
||||
self.assertEqual(result["manifest_summary"]["raw"]["documents"], 1)
|
||||
self.assertEqual(result["manifest_summary"]["normalized"]["fragments"], 1)
|
||||
|
||||
def test_raw_normalized_source_mismatch_is_reported(self):
|
||||
normalized, _ = self._nonempty_corpus()
|
||||
manifest = sqlite3.connect(normalized / "manifest.sqlite3")
|
||||
manifest.execute("UPDATE documents SET source_sha256=? WHERE code='1'", ("0" * 64,))
|
||||
manifest.commit()
|
||||
manifest.close()
|
||||
result = verify_corpus(self.raw, normalized, self.base / "artifacts")
|
||||
kinds = {item["kind"] for item in result["finding_examples"]}
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("normalized_source_hash_mismatch", kinds)
|
||||
self.assertIn("document_source_manifest_mismatch", kinds)
|
||||
|
||||
def test_fragment_text_hash_mismatch_is_reported(self):
|
||||
normalized = self.base / "normalized"
|
||||
fragment_rel = "documents/1/editions/2/ru/fragments.json"
|
||||
fragment_path = normalized / fragment_rel
|
||||
fragment_path.parent.mkdir(parents=True)
|
||||
fragment = {
|
||||
"id": "document:1:edition:2:lang:ru:fragment:1",
|
||||
"document_code": "1",
|
||||
"edition_code": "2",
|
||||
"language": "ru",
|
||||
"position": 1,
|
||||
"text": "text",
|
||||
"text_sha256": "0" * 64,
|
||||
"source_path": "documents/1/source.txt",
|
||||
"source_sha256": "1" * 64,
|
||||
}
|
||||
fragment_path.write_text(json.dumps([fragment]), encoding="utf-8")
|
||||
content_path = fragment_path.with_name("content.txt")
|
||||
content_path.write_text("text\n", encoding="utf-8")
|
||||
connection = database(self.base / "test.sqlite3")
|
||||
content_digest = hashlib.sha256(b"text\n").hexdigest()
|
||||
connection.execute(
|
||||
"INSERT INTO files(root_id,relative_path,sha256,status,last_seen_run) VALUES('normalized',?,?, 'hashed','run')",
|
||||
(content_path.relative_to(normalized).as_posix(), content_digest),
|
||||
)
|
||||
validate_fragment_file(
|
||||
connection,
|
||||
"run",
|
||||
normalized,
|
||||
fragment_path,
|
||||
fragment_rel,
|
||||
hashlib.sha256(fragment_path.read_bytes()).hexdigest(),
|
||||
{"documents/1/source.txt": "1" * 64},
|
||||
)
|
||||
kinds = {row[0] for row in connection.execute("SELECT kind FROM findings WHERE run_id='run'")}
|
||||
connection.close()
|
||||
self.assertIn("fragment_text_checksum_mismatch", kinds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the shape and repository allowlist of a prepared release manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SERVICES = {"backend", "opensearch"}
|
||||
COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z")
|
||||
REPOSITORY_RE = re.compile(
|
||||
r"[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?"
|
||||
r"(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*\Z"
|
||||
)
|
||||
IMAGE_RE = re.compile(r"(.+)@sha256:([0-9a-f]{64})\Z")
|
||||
|
||||
|
||||
def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
||||
result: dict[str, object] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON key: {key}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def read_json(path: Path) -> object:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=unique_object)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
||||
raise ValueError(f"cannot read valid JSON from {path}: {error}") from error
|
||||
|
||||
|
||||
def validate_policy(policy: object) -> dict[str, list[str]]:
|
||||
if not isinstance(policy, dict) or set(policy) != {"schema_version", "repositories"}:
|
||||
raise ValueError("policy must contain exactly schema_version and repositories")
|
||||
if type(policy["schema_version"]) is not int or policy["schema_version"] != 1:
|
||||
raise ValueError("unsupported policy schema_version")
|
||||
repositories = policy["repositories"]
|
||||
if not isinstance(repositories, dict) or set(repositories) != SERVICES:
|
||||
raise ValueError("policy repositories must contain exactly backend and opensearch")
|
||||
result: dict[str, list[str]] = {}
|
||||
for service in sorted(SERVICES):
|
||||
allowed = repositories[service]
|
||||
if not isinstance(allowed, list) or not allowed:
|
||||
raise ValueError(f"policy repository allowlist for {service} must be a non-empty list")
|
||||
if any(not isinstance(repository, str) or not REPOSITORY_RE.fullmatch(repository) for repository in allowed):
|
||||
raise ValueError(f"policy contains an invalid repository for {service}")
|
||||
if len(set(allowed)) != len(allowed):
|
||||
raise ValueError(f"policy repository allowlist for {service} contains duplicates")
|
||||
result[service] = allowed
|
||||
return result
|
||||
|
||||
|
||||
def validate_manifest(manifest: object, policy: object) -> dict[str, object]:
|
||||
allowed_repositories = validate_policy(policy)
|
||||
if not isinstance(manifest, dict) or set(manifest) != {
|
||||
"schema_version", "source_ref", "commit", "images"
|
||||
}:
|
||||
raise ValueError("manifest must contain exactly schema_version, source_ref, commit, and images")
|
||||
if type(manifest["schema_version"]) is not int or manifest["schema_version"] != 1:
|
||||
raise ValueError("unsupported manifest schema_version")
|
||||
if manifest["source_ref"] != "refs/heads/main":
|
||||
raise ValueError("manifest source_ref must be refs/heads/main")
|
||||
commit = manifest["commit"]
|
||||
if not isinstance(commit, str) or not COMMIT_RE.fullmatch(commit):
|
||||
raise ValueError("manifest commit must be a 40-character lowercase Git SHA-1")
|
||||
images = manifest["images"]
|
||||
if not isinstance(images, dict) or set(images) != SERVICES:
|
||||
raise ValueError("manifest images must contain exactly backend and opensearch")
|
||||
for service in sorted(SERVICES):
|
||||
image = images[service]
|
||||
match = IMAGE_RE.fullmatch(image) if isinstance(image, str) else None
|
||||
if not match:
|
||||
raise ValueError(f"{service} image must use repository@sha256:<64 lowercase hex>")
|
||||
repository = match.group(1)
|
||||
if not REPOSITORY_RE.fullmatch(repository):
|
||||
raise ValueError(f"{service} image has an invalid OCI repository")
|
||||
if repository not in allowed_repositories[service]:
|
||||
raise ValueError(f"{service} image repository is not allowed by policy")
|
||||
return manifest
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("manifest", type=Path)
|
||||
parser.add_argument("--policy", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
manifest = validate_manifest(read_json(args.manifest), read_json(args.policy))
|
||||
except ValueError as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
images = manifest["images"]
|
||||
print(
|
||||
"PASS release manifest "
|
||||
f"commit={manifest['commit']} backend={images['backend']} opensearch={images['opensearch']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,134 @@
|
||||
project: akyldash
|
||||
environment: production
|
||||
ssh:
|
||||
alias: akyldash_vps
|
||||
host: 206.223.249.118
|
||||
port: 22
|
||||
user: codex
|
||||
identity_file: ~/.ssh/akyldash_codex
|
||||
domain: akyldash.kg
|
||||
facts:
|
||||
hostname: avprogs.hlab.kz
|
||||
os: Ubuntu Server 24.04.5 LTS
|
||||
kernel: 6.8.0-142-generic
|
||||
cpu_vcpus: 4
|
||||
memory_gib: 7.7
|
||||
swap_gib: 3.8
|
||||
disk_gib: 99
|
||||
disk_free_gib: 92
|
||||
network_interface: ens3
|
||||
ipv4: 206.223.249.118/32
|
||||
domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses
|
||||
domain_http_status: Cloudflare 521 on HTTP and HTTPS from the VPS
|
||||
user_groups: [codex, users]
|
||||
sudo_noninteractive: false
|
||||
installed:
|
||||
- OpenSSH server
|
||||
- systemd-resolved
|
||||
- unattended-upgrades
|
||||
- Docker CE and CLI 29.9.0
|
||||
- Docker Compose plugin 5.6.0
|
||||
- containerd 2.4.1
|
||||
- runc 1.5.2
|
||||
- Git
|
||||
absent:
|
||||
- ISPmanager package/service (unprivileged package and unit checks)
|
||||
- root-owned application checkout
|
||||
- restricted verifier and rollback helper
|
||||
- non-loopback listeners on TCP 80, 443, 1500
|
||||
access_audit_date: 2026-10-12
|
||||
docker_service_active: true
|
||||
docker_api_accessible_to_codex: false
|
||||
docker_daemon_log_config:
|
||||
driver: local
|
||||
max_size: 10m
|
||||
max_file: 3
|
||||
bootstrap_paths:
|
||||
/opt/akyldash: root:root 0755
|
||||
/etc/akyldash: root:root 0700
|
||||
/var/lib/akyldash/data: root:root 0750
|
||||
/var/backups/akyldash: root:root 0700
|
||||
unprivileged_nonloopback_tcp_listeners: [22]
|
||||
owner_reported_ufw_allowed_tcp: [22, 80, 443, 1500]
|
||||
port_1500_listener: false
|
||||
rootless_docker_command_present: /usr/bin/dockerd-rootless.sh
|
||||
runner_gitea_host_resolution: git.vakanaut.ru resolves to 195.209.48.28
|
||||
runner_gitea_https_probe: /api/v1/version returned Gitea 1.25.5 (Actions protocol unverified)
|
||||
cloudflare_edge_status:
|
||||
http: 521
|
||||
https: 521
|
||||
target:
|
||||
compose_file_in_repository: deploy/production/compose.yaml
|
||||
compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml
|
||||
production_env_file: /etc/akyldash/production.env
|
||||
secrets_directory: /etc/akyldash
|
||||
data_directory: /var/lib/akyldash/data
|
||||
backup_directory: /var/backups/akyldash
|
||||
services:
|
||||
caddy:
|
||||
container: compose project service caddy
|
||||
image: caddy:2.11.6-alpine pinned by digest in deploy/production/compose.yaml
|
||||
host_ports: [80, 443]
|
||||
persistent_data: named Compose volumes caddy-data and caddy-config
|
||||
backend:
|
||||
container: compose project service backend
|
||||
image: built from deploy/production/Dockerfile.backend; digest deployment not configured
|
||||
port: 8080 on private proxy and search Compose networks only
|
||||
healthcheck: HTTP GET /review on container port 8080
|
||||
persistent_data: /var/lib/akyldash/data (bind mount)
|
||||
opensearch:
|
||||
container: compose project service opensearch
|
||||
image: opensearchproject/opensearch:3.7.0 plus analysis-icu plugin
|
||||
port: 9200 on private Compose network only
|
||||
persistent_data: named Compose volume opensearch-data
|
||||
reverse_proxy:
|
||||
target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080
|
||||
status: no listener observed on 80/443; container state needs root Docker audit
|
||||
external_ports_target: [22, 80, 443]
|
||||
compose_networks:
|
||||
proxy: caddy and backend
|
||||
search: backend and OpenSearch; internal network
|
||||
secrets:
|
||||
- REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
|
||||
- Gitea credentials from environment-backed gitea-api helper
|
||||
- SSH key at ~/.ssh/akyldash_codex
|
||||
restricted_helper:
|
||||
path: /usr/local/sbin/akyldash-rollback
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
compose_read_only_helper:
|
||||
path: /usr/local/sbin/akyldash-compose
|
||||
allowed_operations: [config, status]
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
read_only_verifier:
|
||||
path: /usr/local/sbin/akyldash-verify
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
gitea_actions_runner:
|
||||
repository: admin/akyldash
|
||||
registration_scope: repository
|
||||
implementation: official act_runner 0.2.13; pinned binary SHA-256 in infra/runner/install-runner.sh
|
||||
label: akyldash-ci
|
||||
capacity: 1
|
||||
job_limits: {cpu: 2, memory_gib: 3, pids: 512}
|
||||
account_slice_limits: {cpu: 2.5, memory_gib: 4, tasks: 2048}
|
||||
docker_store_limit_gib: 15
|
||||
docker_boundary: rootless dedicated daemon; no production socket; job socket injection disabled
|
||||
service_status: configuration prepared, not installed or registered; Online unverified
|
||||
owner_action: run the root installer from the reviewed checkout; register through repository runner settings
|
||||
checks:
|
||||
- infra/bootstrap/verify.sh akyldash_vps
|
||||
- docker compose -f deploy/production/compose.yaml ps
|
||||
- curl -fsS https://akyldash.kg/review
|
||||
- python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
|
||||
backup_path: /var/backups/akyldash (directory exists; archive script in infra/backup/archive.py is not installed or run)
|
||||
deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment)
|
||||
versions:
|
||||
application:
|
||||
backend: 0.11.3
|
||||
frontend: 0.1.2
|
||||
telegram_bot: 0.2.2
|
||||
infrastructure: unknown; no production install or release exists
|
||||
caddy_image: 2.11.6-alpine pinned by manifest digest in Compose
|
||||
docker_engine: 29.9.0 (daemon active; server API unverified as codex)
|
||||
docker_compose: 5.6.0
|
||||
containerd: 2.4.1
|
||||
runc: 1.5.2
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
systemctl is-active --quiet akyldash-act-runner.service
|
||||
runner_uid=$(id -u akyldash-runner)
|
||||
runuser -u akyldash-runner -- env \
|
||||
HOME=/var/lib/akyldash-runner/.config/act-runner \
|
||||
XDG_CONFIG_HOME=/var/lib/akyldash-runner/.config \
|
||||
XDG_DATA_HOME=/var/lib/akyldash-runner/.config/act-runner/.local/share \
|
||||
DOCKER_HOST="unix:///run/user/$runner_uid/docker.sock" \
|
||||
/usr/bin/docker info --format '{{.ServerVersion}}' >/dev/null
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Check AkylDash Gitea Actions runner health
|
||||
OnFailure=akyldash-runner-restart.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/akyldash-runner-healthcheck
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Periodically check AkylDash Gitea Actions runner health
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2m
|
||||
OnUnitActiveSec=1m
|
||||
Unit=akyldash-runner-healthcheck.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=Restart AkylDash Gitea Actions runner after a failed health check
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl restart akyldash-act-runner.service
|
||||
@@ -0,0 +1,15 @@
|
||||
log:
|
||||
level: info
|
||||
runner:
|
||||
file: /var/lib/akyldash-runner/.config/act-runner/.runner
|
||||
capacity: 1
|
||||
labels:
|
||||
- "akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04"
|
||||
container:
|
||||
privileged: false
|
||||
options: --cpus=2 --memory=3g --pids-limit=512
|
||||
valid_volumes: []
|
||||
docker_host: "-"
|
||||
force_pull: true
|
||||
cache:
|
||||
enabled: false
|
||||
@@ -0,0 +1,22 @@
|
||||
BEGIN { FS = ":" }
|
||||
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
|
||||
NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1 { invalid = 1; next }
|
||||
{
|
||||
start[FILENAME, ++count[FILENAME]] = $2
|
||||
end[FILENAME, count[FILENAME]] = $2 + $3 - 1
|
||||
files[FILENAME] = 1
|
||||
}
|
||||
END {
|
||||
if (invalid) exit 2
|
||||
for (base = 100000; base + 65535 <= 4294901760; base += 65536) {
|
||||
overlap = 0
|
||||
for (file in files)
|
||||
for (i = 1; i <= count[file]; i++)
|
||||
if (start[file, i] <= base + 65535 && end[file, i] >= base) overlap = 1
|
||||
if (!overlap) {
|
||||
print base "-" base + 65535
|
||||
exit
|
||||
}
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
runner_die() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
|
||||
|
||||
runner_ensure_account() {
|
||||
local user=$1 home=$2 entry uid passwd_home shell groups uid_min
|
||||
if ! entry=$(getent passwd "$user"); then
|
||||
[[ ! -e $home && ! -L $home ]] || { runner_die "runner home already exists without its account: $home"; return 1; }
|
||||
useradd --system --create-home --home-dir "$home" --shell /usr/sbin/nologin "$user" || return 1
|
||||
return
|
||||
fi
|
||||
IFS=: read -r _ _ uid _ _ passwd_home shell <<<"$entry"
|
||||
uid_min=$(awk '$1 == "UID_MIN" && $2 ~ /^[0-9]+$/ {print $2; exit}' /etc/login.defs)
|
||||
[[ $uid =~ ^[0-9]+$ && $uid -gt 0 && $uid_min =~ ^[0-9]+$ && $uid -lt $uid_min && $passwd_home == "$home" && $shell == /usr/sbin/nologin ]] || { runner_die "existing $user account has an unexpected UID, home, or shell"; return 1; }
|
||||
[[ $(id -gn "$user") == "$user" ]] || { runner_die "existing $user account has an unexpected primary group"; return 1; }
|
||||
groups=$(id -nG "$user")
|
||||
[[ $groups == "$user" ]] || { runner_die "existing $user account has unexpected supplementary groups: $groups"; return 1; }
|
||||
[[ -d $home && ! -L $home ]] || { runner_die "existing $user home is missing or has unexpected type"; return 1; }
|
||||
case $(stat -c %U "$home") in
|
||||
"$user") ;;
|
||||
root) [[ $(stat -c '%G:%a' "$home") == root:711 ]] || { runner_die "root-owned $user home must be mode 0711"; return 1; } ;;
|
||||
*) runner_die "existing $user home has unexpected ownership"; return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
runner_lock_home() {
|
||||
local home=$1 parent_mode
|
||||
[[ -d $home && ! -L $home && $(stat -c %U /var/lib) == root ]] || { runner_die "runner home or its parent is unsafe: $home"; return 1; }
|
||||
parent_mode=$(stat -c %a /var/lib)
|
||||
(( (8#$parent_mode & 022) == 0 )) || { runner_die '/var/lib must not be group/world-writable'; return 1; }
|
||||
chown root:root "$home" && chmod 0711 "$home" || return 1
|
||||
[[ $(stat -c '%U:%G:%a' "$home") == root:root:711 ]] || { runner_die 'runner home could not be locked from runner writes'; return 1; }
|
||||
}
|
||||
|
||||
runner_ensure_image() {
|
||||
local image=$1 bytes=$2 parent_uid=$3 parent_mode=$4
|
||||
python3 "$(dirname "${BASH_SOURCE[0]}")/secure-runner-image.py" "$image" "$bytes" "$parent_uid" "$parent_mode" || { runner_die "runner image validation or initialization failed: $image"; return 1; }
|
||||
}
|
||||
|
||||
runner_install_file() {
|
||||
local source=$1 destination=$2 owner=$3 group=$4 mode=${5#0}
|
||||
if [[ -e $destination || -L $destination ]]; then
|
||||
[[ -f $destination && ! -L $destination ]] && cmp -s "$source" "$destination" || { runner_die "existing runner file differs from reviewed configuration: $destination"; return 1; }
|
||||
[[ $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { runner_die "existing runner file has unexpected ownership or permissions: $destination"; return 1; }
|
||||
return
|
||||
fi
|
||||
install -o "$owner" -g "$group" -m "0$mode" "$source" "$destination"
|
||||
}
|
||||
|
||||
runner_install_content() {
|
||||
local destination=$1 owner=$2 group=$3 mode=$4 temporary
|
||||
temporary=$(mktemp)
|
||||
cat >"$temporary"
|
||||
if ! runner_install_file "$temporary" "$destination" "$owner" "$group" "$mode"; then
|
||||
rm -f "$temporary"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$temporary"
|
||||
}
|
||||
|
||||
runner_ensure_private_directory() {
|
||||
local directory=$1 owner=$2 group=$3 current= component index
|
||||
local -a components
|
||||
[[ $directory == /* ]] || { runner_die "runner directory path is not absolute: $directory"; return 1; }
|
||||
IFS=/ read -r -a components <<<"${directory#/}"
|
||||
for index in "${!components[@]}"; do
|
||||
component=${components[index]}
|
||||
[[ -n $component && $component != . && $component != .. ]] || { runner_die "runner directory path has an unsafe component: $directory"; return 1; }
|
||||
current+="/$component"
|
||||
[[ ! -L $current ]] || { runner_die "runner directory path contains a symbolic link: $current"; return 1; }
|
||||
if [[ ! -e $current ]]; then
|
||||
((index == ${#components[@]} - 1)) || { runner_die "runner directory parent is missing: $current"; return 1; }
|
||||
break
|
||||
fi
|
||||
[[ -d $current ]] || { runner_die "runner directory path component is not a directory: $current"; return 1; }
|
||||
if [[ $current == /var/lib/akyldash-runner ]]; then
|
||||
[[ $(stat -c '%U:%G:%a' "$current") == root:root:711 ]] || { runner_die 'runner home must be root-owned mode 0711'; return 1; }
|
||||
elif [[ $current == /var/lib/akyldash-runner/* ]]; then
|
||||
[[ $(stat -c '%U:%G' "$current") == "$owner:$group" ]] || { runner_die "runner directory ancestor has unexpected ownership: $current"; return 1; }
|
||||
fi
|
||||
done
|
||||
# These paths are runner-writable; make changes as that user so a path swap
|
||||
# cannot turn a root operation into a write outside its home.
|
||||
if [[ $directory == /var/lib/akyldash-runner/.config ]]; then
|
||||
install -d -o "$owner" -g "$group" -m 0700 "$directory" || return 1
|
||||
else
|
||||
runuser -u "$owner" -- install -d -m 0700 "$directory" || return 1
|
||||
fi
|
||||
[[ -d $directory && ! -L $directory && $(stat -c '%U:%G:%a' "$directory") == "$owner:$group:700" ]] || { runner_die "runner directory failed validation: $directory"; return 1; }
|
||||
}
|
||||
|
||||
runner_install_user_content() {
|
||||
local destination=$1 owner=$2 group=$3 mode=${4#0} temporary
|
||||
temporary=$(mktemp)
|
||||
cat >"$temporary"
|
||||
if [[ -e $destination || -L $destination ]]; then
|
||||
[[ -f $destination && ! -L $destination ]] && cmp -s "$temporary" "$destination" || { rm -f "$temporary"; runner_die "existing runner file differs from reviewed configuration: $destination"; return 1; }
|
||||
[[ $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { rm -f "$temporary"; runner_die "existing runner file has unexpected ownership or permissions: $destination"; return 1; }
|
||||
rm -f "$temporary"
|
||||
return
|
||||
fi
|
||||
chown "$owner:$group" "$temporary" && chmod "0$mode" "$temporary" || { rm -f "$temporary"; return 1; }
|
||||
local result=0
|
||||
runuser -u "$owner" -- install -m "0$mode" "$temporary" "$destination" || result=$?
|
||||
rm -f "$temporary"
|
||||
((result == 0)) || return 1
|
||||
[[ -f $destination && ! -L $destination && $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { runner_die "runner file failed validation: $destination"; return 1; }
|
||||
}
|
||||
|
||||
runner_validate_mount() {
|
||||
local image=$1 store=$2 target filesystem source options backing mount_info
|
||||
mount_info=$(findmnt --noheadings --output TARGET,FSTYPE,SOURCE,OPTIONS --mountpoint "$store") || { runner_die 'runner store is not mounted'; return 1; }
|
||||
read -r target filesystem source options <<<"$mount_info" || { runner_die 'runner store mount information is empty'; return 1; }
|
||||
[[ $target == "$store" && $filesystem == ext4 && $source == /dev/loop* ]] || { runner_die 'runner store is mounted from an unexpected target, filesystem, or device'; return 1; }
|
||||
case ",$options," in *,nosuid,*nodev,*|*,nodev,*nosuid,*) ;; *) runner_die 'runner store mount is missing nosuid or nodev'; return 1 ;; esac
|
||||
backing=$(losetup --noheadings --output BACK-FILE "$source") || { runner_die 'cannot identify the runner loop backing file'; return 1; }
|
||||
[[ $(realpath -e "$backing") == "$(realpath -e "$image")" ]] || { runner_die 'runner store is backed by a different image'; return 1; }
|
||||
}
|
||||
|
||||
runner_ensure_mount() {
|
||||
local image=$1 store=$2 unit_name=$3 unit_file=$4 unit_tmp mounted=0
|
||||
[[ -d $store && ! -L $store ]] || { runner_die "runner mount point is missing or is not a directory: $store"; return 1; }
|
||||
if mountpoint -q "$store"; then
|
||||
runner_validate_mount "$image" "$store" || return 1
|
||||
mounted=1
|
||||
fi
|
||||
unit_tmp=$(mktemp)
|
||||
if ! cat >"$unit_tmp" <<EOF
|
||||
[Unit]
|
||||
Description=Bounded storage for the AkylDash CI runner
|
||||
Before=local-fs.target
|
||||
|
||||
[Mount]
|
||||
What=$image
|
||||
Where=$store
|
||||
Type=ext4
|
||||
Options=loop,nosuid,nodev
|
||||
|
||||
[Install]
|
||||
WantedBy=local-fs.target
|
||||
EOF
|
||||
then
|
||||
rm -f "$unit_tmp"
|
||||
runner_die 'failed to prepare runner mount unit'
|
||||
return 1
|
||||
fi
|
||||
if [[ -e $unit_file || -L $unit_file ]]; then
|
||||
if [[ ! -f $unit_file || -L $unit_file ]] || ! cmp -s "$unit_tmp" "$unit_file"; then
|
||||
rm -f "$unit_tmp"
|
||||
runner_die "existing runner mount unit differs from the reviewed configuration: $unit_file"
|
||||
return 1
|
||||
fi
|
||||
[[ $(stat -c '%u:%g:%a' "$unit_file") == "$(id -u):$(id -g):644" ]] || { rm -f "$unit_tmp"; runner_die 'existing runner mount unit has unexpected ownership or permissions'; return 1; }
|
||||
else
|
||||
install -m 0644 "$unit_tmp" "$unit_file" || { rm -f "$unit_tmp"; return 1; }
|
||||
fi
|
||||
rm -f "$unit_tmp"
|
||||
|
||||
systemctl daemon-reload || return 1
|
||||
if ((mounted)); then
|
||||
systemctl enable "$unit_name" || return 1
|
||||
else
|
||||
systemctl enable "$unit_name" || return 1
|
||||
systemctl start "$unit_name" || return 1
|
||||
runner_validate_mount "$image" "$store" || return 1
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly runner_user=akyldash-runner
|
||||
readonly runner_home=/var/lib/akyldash-runner
|
||||
readonly runner_runtime_home="$runner_home/.config/act-runner"
|
||||
readonly runner_xdg_config_home="$runner_home/.config"
|
||||
readonly runner_xdg_data_home="$runner_runtime_home/.local/share"
|
||||
readonly runner_store="$runner_home/docker-data"
|
||||
readonly runner_image="$runner_home/docker-store.img"
|
||||
readonly runner_mount_unit_name="$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)"
|
||||
readonly runner_mount_unit="/etc/systemd/system/$runner_mount_unit_name"
|
||||
readonly runner_image_bytes=16106127360
|
||||
readonly runner_version=0.2.13
|
||||
readonly runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2
|
||||
readonly runner_url="https://dl.gitea.com/gitea-runner/$runner_version/act_runner-$runner_version-linux-amd64"
|
||||
readonly checkout=/opt/akyldash/current
|
||||
|
||||
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die 'run this script as root'
|
||||
[[ -r /etc/os-release ]] || die 'cannot identify operating system'
|
||||
. /etc/os-release
|
||||
[[ ${ID:-} == ubuntu && ${VERSION_ID:-} == 24.04 ]] || die 'Ubuntu 24.04 is required'
|
||||
[[ $(uname -m) == x86_64 ]] || die 'this pinned binary is for x86_64 only'
|
||||
[[ $(stat -fc %T /sys/fs/cgroup) == cgroup2fs ]] || die 'rootless Docker requires cgroup v2'
|
||||
[[ $(findmnt -n -o FSTYPE /) == ext4 ]] || die 'the bounded runner store currently requires ext4 on /'
|
||||
[[ -d $checkout/.git && ! -L $checkout && $(stat -c %U "$checkout") == root ]] || die 'install the reviewed root-owned checkout at /opt/akyldash/current first'
|
||||
for directory in /opt /opt/akyldash; do
|
||||
[[ $(stat -c %U "$directory") == root ]] || die "$directory must be owned by root"
|
||||
mode=$(stat -c %a "$directory")
|
||||
(( (8#$mode & 022) == 0 )) || die "$directory must not be group/world-writable"
|
||||
done
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
[[ -z $unsafe ]] || die 'the repository checkout contains files writable by non-root users'
|
||||
git -C "$checkout" diff --quiet HEAD -- infra/runner || die 'runner files differ from the reviewed Git revision'
|
||||
source "$checkout/infra/runner/install-runner-lib.sh"
|
||||
if systemctl is-active --quiet akyldash-act-runner.service; then
|
||||
die 'stop akyldash-act-runner.service after all jobs finish before rerunning this installer'
|
||||
fi
|
||||
[[ -r /etc/subuid && -r /etc/subgid ]] || die 'subordinate ID allocation files are unavailable'
|
||||
if getent passwd "$runner_user" >/dev/null; then
|
||||
runner_ensure_account "$runner_user" "$runner_home" || die 'runner account validation failed'
|
||||
runner_lock_home "$runner_home" || die 'runner home protection failed'
|
||||
else
|
||||
[[ ! -e $runner_home && ! -L $runner_home ]] || die 'runner home exists without its account; inspect it before continuing'
|
||||
fi
|
||||
if [[ -e $runner_image || -L $runner_image ]]; then
|
||||
runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711 || die 'runner image validation failed'
|
||||
else
|
||||
[[ $(df -Pk / | awk 'NR==2 {print $4}') -ge 25000000 ]] || die 'need at least 25 GiB free before reserving 15 GiB for runner data'
|
||||
fi
|
||||
|
||||
for file in /etc/subuid /etc/subgid; do
|
||||
awk -F: '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ && (NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1) { bad=1 } END { exit bad }' "$file" || die "$file contains an invalid subordinate ID entry"
|
||||
done
|
||||
subuid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subuid) || die 'runner has multiple subuid allocations; inspect them before continuing'
|
||||
subgid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subgid) || die 'runner has multiple subgid allocations; inspect them before continuing'
|
||||
for pair in "$subuid_pair" "$subgid_pair"; do
|
||||
[[ -z $pair || $pair =~ ^[0-9]+:65536$ ]] || die 'existing runner subordinate ID allocation is not a single 65536-ID range'
|
||||
done
|
||||
if [[ -n $subuid_pair && -n $subgid_pair && $subuid_pair != "$subgid_pair" ]]; then
|
||||
die 'runner subuid and subgid allocations differ; inspect them before continuing'
|
||||
fi
|
||||
existing_pair=${subuid_pair:-$subgid_pair}
|
||||
if [[ -n $existing_pair ]]; then
|
||||
subid_start=${existing_pair%%:*}
|
||||
[[ $subid_start =~ ^[1-9][0-9]*$ && $subid_start -ge 100000 && $subid_start -le 4294901760 ]] || die 'existing runner subordinate ID range is outside the supported range'
|
||||
subid_end=$((subid_start + 65535))
|
||||
subid_range="$subid_start-$subid_end"
|
||||
else
|
||||
subid_range=$(awk -f "$(dirname "$0")/find-free-subid-range.awk" /etc/subuid /etc/subgid) || die 'could not find a free 65536-ID range shared by /etc/subuid and /etc/subgid'
|
||||
subid_start=${subid_range%-*}
|
||||
subid_end=${subid_range#*-}
|
||||
fi
|
||||
for file in /etc/subuid /etc/subgid; do
|
||||
awk -F: -v user="$runner_user" -v start="$subid_start" -v end="$subid_end" '$1 != user && $2 <= end && $2 + $3 - 1 >= start { conflict=1 } END { exit conflict }' "$file" || die "runner subordinate ID range conflicts with another entry in $file"
|
||||
done
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate docker-ce-rootless-extras
|
||||
|
||||
runner_ensure_account "$runner_user" "$runner_home" || die 'runner account creation failed'
|
||||
runner_lock_home "$runner_home" || die 'runner home protection failed'
|
||||
subid_args=()
|
||||
[[ -n $subuid_pair ]] || subid_args+=(--add-subuids "$subid_range")
|
||||
[[ -n $subgid_pair ]] || subid_args+=(--add-subgids "$subid_range")
|
||||
if ((${#subid_args[@]})); then
|
||||
usermod "${subid_args[@]}" "$runner_user"
|
||||
fi
|
||||
runner_uid=$(id -u "$runner_user")
|
||||
|
||||
runner_ensure_private_directory "$runner_home/.config" "$runner_user" "$runner_user" || die 'runner config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/docker" "$runner_user" "$runner_user" || die 'runner Docker config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd" "$runner_user" "$runner_user" || die 'runner systemd config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd/user" "$runner_user" "$runner_user" || die 'runner user systemd directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd/user/docker.service.d" "$runner_user" "$runner_user" || die 'runner Docker override directory failed validation'
|
||||
runner_ensure_private_directory "$runner_runtime_home" "$runner_user" "$runner_user" || die 'runner runtime home failed directory validation'
|
||||
runner_ensure_private_directory "$runner_runtime_home/.local" "$runner_user" "$runner_user" || die 'runner XDG data parent failed directory validation'
|
||||
runner_ensure_private_directory "$runner_xdg_data_home" "$runner_user" "$runner_user" || die 'runner XDG data directory failed directory validation'
|
||||
if [[ -e $runner_home/.runner || -L $runner_home/.runner ]]; then
|
||||
[[ -f $runner_home/.runner && ! -L $runner_home/.runner ]] || die 'legacy runner registration state is not a regular file'
|
||||
if [[ ! -e $runner_runtime_home/.runner && ! -L $runner_runtime_home/.runner ]]; then
|
||||
runner_install_user_content "$runner_runtime_home/.runner" "$runner_user" "$runner_user" 0600 <"$runner_home/.runner" || die 'legacy runner registration state could not be preserved'
|
||||
fi
|
||||
fi
|
||||
if [[ ! -e $runner_store && ! -L $runner_store ]]; then
|
||||
install -d -o root -g root -m 0755 "$runner_store"
|
||||
fi
|
||||
[[ -d $runner_store && ! -L $runner_store ]] || die 'runner mount path is not a directory'
|
||||
install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner
|
||||
install -d -o "$runner_user" -g "$runner_user" -m 0750 /var/log/akyldash-runner
|
||||
|
||||
runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711 || die 'runner image initialization failed'
|
||||
runner_ensure_mount "$runner_image" "$runner_store" "$runner_mount_unit_name" "$runner_mount_unit" || die 'runner mount activation failed'
|
||||
chown "$runner_user:$runner_user" "$runner_store"
|
||||
chmod 0700 "$runner_store"
|
||||
|
||||
runner_install_file "$(dirname "$0")/act-runner-healthcheck.sh" /usr/local/sbin/akyldash-runner-healthcheck root root 0755 || die 'healthcheck command differs from reviewed configuration'
|
||||
runner_install_file "$(dirname "$0")/config.yaml" /etc/akyldash-runner/config.yaml root "$runner_user" 0640 || die 'runner config differs from reviewed configuration'
|
||||
if [[ -e /usr/local/bin/act_runner || -L /usr/local/bin/act_runner ]]; then
|
||||
[[ -f /usr/local/bin/act_runner && ! -L /usr/local/bin/act_runner ]] || die 'existing act_runner path is not a regular file'
|
||||
printf '%s %s\n' "$runner_sha256" /usr/local/bin/act_runner | sha256sum --check --status || die 'existing act_runner binary does not match the pinned release'
|
||||
else
|
||||
tmp_binary=$(mktemp)
|
||||
trap 'rm -f "$tmp_binary"' EXIT
|
||||
curl --fail --location --silent --show-error "$runner_url" -o "$tmp_binary"
|
||||
printf '%s %s\n' "$runner_sha256" "$tmp_binary" | sha256sum --check --status || die 'official act_runner checksum mismatch'
|
||||
install -o root -g root -m 0755 "$tmp_binary" /usr/local/bin/act_runner
|
||||
rm -f "$tmp_binary"
|
||||
trap - EXIT
|
||||
fi
|
||||
|
||||
runner_install_user_content "$runner_home/.config/docker/daemon.json" "$runner_user" "$runner_user" 0600 <<EOF
|
||||
{
|
||||
"data-root": "$runner_store",
|
||||
"log-driver": "local",
|
||||
"log-opts": {"max-size": "10m", "max-file": "2"}
|
||||
}
|
||||
EOF
|
||||
runner_install_user_content "$runner_home/.config/systemd/user/docker.service.d/limits.conf" "$runner_user" "$runner_user" 0644 <<'EOF'
|
||||
[Service]
|
||||
Delegate=yes
|
||||
CPUQuota=200%
|
||||
MemoryMax=4G
|
||||
TasksMax=1024
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
EOF
|
||||
runner_install_user_content "$runner_home/.config/systemd/user/docker.service.d/environment.conf" "$runner_user" "$runner_user" 0644 <<EOF
|
||||
[Service]
|
||||
Environment=HOME=$runner_runtime_home
|
||||
Environment=XDG_CONFIG_HOME=$runner_xdg_config_home
|
||||
Environment=XDG_DATA_HOME=$runner_xdg_data_home
|
||||
EOF
|
||||
runner_install_content /etc/systemd/system/akyldash-act-runner.service root root 0644 <<EOF
|
||||
[Unit]
|
||||
Description=AkylDash repository Gitea Actions runner
|
||||
After=network-online.target local-fs.target
|
||||
Wants=network-online.target
|
||||
RequiresMountsFor=$runner_store
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Slice=user-$runner_uid.slice
|
||||
User=$runner_user
|
||||
Group=$runner_user
|
||||
Environment=HOME=$runner_runtime_home
|
||||
Environment=XDG_CONFIG_HOME=$runner_xdg_config_home
|
||||
Environment=XDG_DATA_HOME=$runner_xdg_data_home
|
||||
WorkingDirectory=$runner_runtime_home
|
||||
Environment=XDG_RUNTIME_DIR=/run/user/$runner_uid
|
||||
Environment=DOCKER_HOST=unix:///run/user/$runner_uid/docker.sock
|
||||
ExecStartPre=/bin/sh -c 'for i in \$(seq 1 60); do /usr/bin/docker info >/dev/null 2>&1 && exit 0; sleep 2; done; exit 1'
|
||||
ExecStart=/usr/local/bin/act_runner --config /etc/akyldash-runner/config.yaml daemon
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
CPUQuota=50%
|
||||
MemoryMax=768M
|
||||
TasksMax=256
|
||||
NoNewPrivileges=yes
|
||||
PrivateTmp=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
RestrictSUIDSGID=yes
|
||||
ReadWritePaths=$runner_runtime_home $runner_xdg_config_home /var/log/akyldash-runner /run/user/$runner_uid
|
||||
StandardOutput=append:/var/log/akyldash-runner/act_runner.log
|
||||
StandardError=append:/var/log/akyldash-runner/act_runner.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
install -d -o root -g root -m 0755 "/etc/systemd/system/user-$runner_uid.slice.d"
|
||||
runner_install_content "/etc/systemd/system/user-$runner_uid.slice.d/limits.conf" root root 0644 <<EOF
|
||||
[Slice]
|
||||
CPUQuota=250%
|
||||
MemoryMax=4G
|
||||
TasksMax=2048
|
||||
EOF
|
||||
runner_install_content /etc/logrotate.d/akyldash-runner root root 0644 <<'EOF'
|
||||
/var/log/akyldash-runner/act_runner.log {
|
||||
daily
|
||||
rotate 7
|
||||
size 10M
|
||||
compress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
}
|
||||
EOF
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.service" /etc/systemd/system/akyldash-runner-healthcheck.service root root 0644
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.timer" /etc/systemd/system/akyldash-runner-healthcheck.timer root root 0644
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-restart.service" /etc/systemd/system/akyldash-runner-restart.service root root 0644
|
||||
|
||||
loginctl enable-linger "$runner_user"
|
||||
systemctl daemon-reload
|
||||
systemctl start "user@$runner_uid.service"
|
||||
rootless_docker_unit="$runner_xdg_config_home/systemd/user/docker.service"
|
||||
rootless_env=(HOME="$runner_runtime_home" XDG_CONFIG_HOME="$runner_xdg_config_home" XDG_DATA_HOME="$runner_xdg_data_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus")
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" sh -c 'test -w "$HOME" && test -w "$XDG_CONFIG_HOME" && test -w "$XDG_DATA_HOME"' || die 'rootless Docker HOME and XDG directories must be writable by the runner'
|
||||
if [[ -e $rootless_docker_unit || -L $rootless_docker_unit ]]; then
|
||||
[[ -f $rootless_docker_unit && ! -L $rootless_docker_unit && $(stat -c %U "$rootless_docker_unit") == "$runner_user" ]] || die 'existing rootless Docker unit has an unexpected type or owner'
|
||||
grep -Fq 'dockerd-rootless.sh' "$rootless_docker_unit" || die 'existing rootless Docker unit differs from the expected service'
|
||||
else
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" dockerd-rootless-setuptool.sh install
|
||||
fi
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user daemon-reload
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user enable docker.service
|
||||
if runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user is-active --quiet docker.service; then
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user restart docker.service
|
||||
else
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user start docker.service
|
||||
fi
|
||||
docker_pid=$(runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user show docker.service --property=MainPID --value)
|
||||
[[ $docker_pid =~ ^[1-9][0-9]*$ && -r /proc/$docker_pid/environ ]] || die 'rootless Docker process environment is unavailable'
|
||||
docker_environment=$(tr '\0' '\n' <"/proc/$docker_pid/environ") || die 'could not read rootless Docker process environment'
|
||||
for expected in "HOME=$runner_runtime_home" "XDG_CONFIG_HOME=$runner_xdg_config_home" "XDG_DATA_HOME=$runner_xdg_data_home"; do
|
||||
grep -Fqx -- "$expected" <<<"$docker_environment" || die "running rootless Docker process is missing $expected"
|
||||
done
|
||||
|
||||
printf '\nRunner runtime installed. Register it interactively, then enable its services as described in docs/operations/CI_RUNNER.md.\n'
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create or validate the runner image without following replaceable paths."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def open_directory(path: str) -> int:
|
||||
current = os.open("/", os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
for component in path.strip("/").split("/"):
|
||||
if not component:
|
||||
continue
|
||||
next_fd = os.open(
|
||||
component,
|
||||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
||||
dir_fd=current,
|
||||
)
|
||||
os.close(current)
|
||||
current = next_fd
|
||||
return current
|
||||
except BaseException:
|
||||
os.close(current)
|
||||
raise
|
||||
|
||||
|
||||
def filesystem_type(fd: int) -> str:
|
||||
result = subprocess.run(
|
||||
["blkid", "-p", "-s", "TYPE", "-o", "value", f"/proc/self/fd/{fd}"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
pass_fds=(fd,),
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
image, bytes_text, parent_uid_text, parent_mode_text = sys.argv[1:]
|
||||
image = os.path.abspath(image)
|
||||
parent_path, name = os.path.split(image)
|
||||
if not name or name in {".", ".."}:
|
||||
raise ValueError("invalid image filename")
|
||||
|
||||
parent_fd = open_directory(parent_path)
|
||||
image_fd = -1
|
||||
try:
|
||||
parent_stat = os.fstat(parent_fd)
|
||||
if parent_stat.st_uid != int(parent_uid_text) or stat.S_IMODE(parent_stat.st_mode) != int(parent_mode_text, 8):
|
||||
raise RuntimeError("image parent has unexpected owner or permissions")
|
||||
|
||||
created = False
|
||||
try:
|
||||
image_fd = os.open(
|
||||
name,
|
||||
os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
image_fd = os.open(
|
||||
name,
|
||||
os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
created = True
|
||||
|
||||
image_stat = os.fstat(image_fd)
|
||||
if not stat.S_ISREG(image_stat.st_mode) or image_stat.st_uid != os.geteuid() or image_stat.st_nlink != 1:
|
||||
raise RuntimeError("image must be a singly-linked regular file owned by the installer")
|
||||
if not created and image_stat.st_mode & 0o022:
|
||||
raise RuntimeError("existing image is writable by group or other users")
|
||||
|
||||
if created:
|
||||
os.posix_fallocate(image_fd, 0, int(bytes_text))
|
||||
subprocess.run(
|
||||
["mkfs.ext4", "-F", "-q", f"/proc/self/fd/{image_fd}"],
|
||||
check=True,
|
||||
pass_fds=(image_fd,),
|
||||
)
|
||||
image_stat = os.fstat(image_fd)
|
||||
|
||||
if image_stat.st_size != int(bytes_text):
|
||||
raise RuntimeError(f"image size is {image_stat.st_size} bytes, expected {bytes_text}; refusing to resize or format")
|
||||
if filesystem_type(image_fd) != "ext4":
|
||||
raise RuntimeError("image does not contain ext4; refusing to format")
|
||||
path_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
||||
if not stat.S_ISREG(path_stat.st_mode) or (path_stat.st_dev, path_stat.st_ino) != (image_stat.st_dev, image_stat.st_ino):
|
||||
raise RuntimeError("image path changed during validation")
|
||||
return 0
|
||||
finally:
|
||||
if image_fd >= 0:
|
||||
os.close(image_fd)
|
||||
os.close(parent_fd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
workflow=.gitea/workflows/ci.yaml
|
||||
kaniko_action=.gitea/actions/kaniko-build/action.yml
|
||||
caddy_action=.gitea/actions/caddy-validate/action.yml
|
||||
|
||||
grep -Fq 'runs-on: akyldash-ci' "$workflow"
|
||||
grep -Fq 'uses: ./.gitea/actions/caddy-validate' "$workflow"
|
||||
[[ $(grep -Fc 'uses: ./.gitea/actions/kaniko-build' "$workflow") -eq 2 ]]
|
||||
check_workspace_wiring() {
|
||||
local workflow_file=$1 caddy_file=${2:-$caddy_action} kaniko_file=${3:-$kaniko_action}
|
||||
[[ $(grep -Fc 'workspace: ${{ github.workspace }}' "$workflow_file") -eq 3 ]] &&
|
||||
grep -Fxq ' - ${{ inputs.workspace }}/deploy/production/Caddyfile' "$caddy_file" &&
|
||||
grep -Fxq ' - --context=dir://${{ inputs.workspace }}' "$kaniko_file" &&
|
||||
grep -Fxq ' - --dockerfile=${{ inputs.workspace }}/${{ inputs.dockerfile }}' "$kaniko_file" &&
|
||||
! grep -Fq '/github/workspace' "$caddy_file" "$kaniko_file"
|
||||
}
|
||||
if ! check_workspace_wiring "$workflow"; then
|
||||
echo 'Container actions must receive github.workspace and use it for repository paths.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'dockerfile: deploy/production/Dockerfile.backend' "$workflow"
|
||||
grep -Fq 'dockerfile: deploy/production/Dockerfile.opensearch' "$workflow"
|
||||
grep -Fq 'docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet' "$workflow"
|
||||
grep -Fxq ' entrypoint: /usr/bin/caddy' "$caddy_action"
|
||||
grep -Fxq ' - validate' "$caddy_action"
|
||||
grep -Fxq ' - --config' "$caddy_action"
|
||||
grep -Fxq ' - ${{ inputs.workspace }}/deploy/production/Caddyfile' "$caddy_action"
|
||||
grep -Fxq ' - --adapter' "$caddy_action"
|
||||
grep -Fxq ' - caddyfile' "$caddy_action"
|
||||
if grep -Eq 'docker (build|run)([[:space:]]|$)' "$workflow"; then
|
||||
echo 'workflow must not invoke docker build/run in a job container' >&2
|
||||
exit 1
|
||||
fi
|
||||
check_kaniko_action() {
|
||||
grep -Fxq ' entrypoint: /kaniko/executor' "$1" &&
|
||||
grep -Fxq ' - --context=dir://${{ inputs.workspace }}' "$1" &&
|
||||
grep -Fxq ' - --dockerfile=${{ inputs.workspace }}/${{ inputs.dockerfile }}' "$1" &&
|
||||
grep -Fxq ' - --destination=${{ inputs.destination }}' "$1" &&
|
||||
grep -Fxq -- ' - --no-push' "$1" &&
|
||||
grep -Fxq -- ' - --no-push-cache' "$1"
|
||||
}
|
||||
if ! check_kaniko_action "$kaniko_action"; then
|
||||
echo 'Kaniko action must define its entrypoint, build arguments, and no-push flags.' >&2
|
||||
exit 1
|
||||
fi
|
||||
without_entrypoint=$(mktemp)
|
||||
without_no_push=$(mktemp)
|
||||
without_workspace=$(mktemp)
|
||||
wrong_workspace=$(mktemp)
|
||||
trap 'rm -f "$without_entrypoint" "$without_no_push" "$without_workspace" "$wrong_workspace"' EXIT
|
||||
sed '/^ entrypoint: \/kaniko\/executor$/d' "$kaniko_action" >"$without_entrypoint"
|
||||
if check_kaniko_action "$without_entrypoint"; then
|
||||
echo 'CI isolation test accepted a Kaniko action without its executor entrypoint.' >&2
|
||||
exit 1
|
||||
fi
|
||||
sed '/^ - --no-push$/d' "$kaniko_action" >"$without_no_push"
|
||||
if check_kaniko_action "$without_no_push"; then
|
||||
echo 'CI isolation test accepted --no-push-cache without standalone --no-push.' >&2
|
||||
exit 1
|
||||
fi
|
||||
sed '/workspace: \${{ github.workspace }}/d' "$workflow" >"$without_workspace"
|
||||
if check_workspace_wiring "$without_workspace"; then
|
||||
echo 'CI isolation test accepted container actions without workflow workspace inputs.' >&2
|
||||
exit 1
|
||||
fi
|
||||
sed 's#${{ inputs.workspace }}#/github/workspace#g' "$kaniko_action" >"$wrong_workspace"
|
||||
if check_workspace_wiring "$workflow" "$caddy_action" "$wrong_workspace"; then
|
||||
echo 'CI isolation test accepted a fixed Kaniko workspace root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'FROM ghcr.io/osscontainertools/kaniko:v1.28.5-debug@sha256:c73ca67a3b494bce79a8fb6090c4f73c18997ab272d80bffc460a62343aeb412' .gitea/actions/kaniko-build/Dockerfile
|
||||
grep -Fq 'FROM caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028' .gitea/actions/caddy-validate/Dockerfile
|
||||
if grep -REn 'docker.sock|privileged:|container.options:|^[[:space:]]*volumes:' "$workflow" .gitea/actions/kaniko-build .gitea/actions/caddy-validate; then
|
||||
echo 'workflow and container actions must not request sockets, privileges, or additional host mounts' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo 'PASS CI container actions use the supplied workspace and remain isolated without socket or publish access'
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
script_dir=$(dirname "${BASH_SOURCE[0]}")
|
||||
source "$script_dir/../test/run-as-unprivileged.sh"
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if run_as_unprivileged bash "$script_dir/install-runner.sh" >"$output" 2>&1; then
|
||||
echo 'Runner installer unexpectedly succeeded as an unprivileged UID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'ERROR: run this script as root' "$output"
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
runner_dir=$(cd "$(dirname "$0")" && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
mkdir -p "$tmp/bin" "$tmp/store" "$tmp/partial-store"
|
||||
log=$tmp/commands.log
|
||||
export RUNNER_TEST_LOG=$log
|
||||
|
||||
cat >"$tmp/bin/mkfs.ext4" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf 'mkfs\n' >>"$RUNNER_TEST_LOG"
|
||||
if [[ -n ${RUNNER_TEST_REPLACE_IMAGE:-} ]]; then
|
||||
rm -f "$RUNNER_TEST_IMAGE"
|
||||
ln -s "$RUNNER_TEST_REPLACE_IMAGE" "$RUNNER_TEST_IMAGE"
|
||||
fi
|
||||
EOF
|
||||
cat >"$tmp/bin/blkid" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "${RUNNER_TEST_FS:-ext4}"
|
||||
EOF
|
||||
cat >"$tmp/bin/systemctl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$RUNNER_TEST_LOG"
|
||||
if [[ $1 == start ]]; then
|
||||
[[ ${RUNNER_TEST_FAIL_START:-0} == 0 ]] || exit 1
|
||||
: >"$RUNNER_TEST_MOUNTED"
|
||||
fi
|
||||
EOF
|
||||
cat >"$tmp/bin/mountpoint" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
[[ -e $RUNNER_TEST_MOUNTED ]]
|
||||
EOF
|
||||
cat >"$tmp/bin/findmnt" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s ext4 /dev/loop0 %s\n' "$RUNNER_TEST_STORE" "${RUNNER_TEST_OPTIONS:-rw,nosuid,nodev,relatime}"
|
||||
EOF
|
||||
cat >"$tmp/bin/losetup" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$RUNNER_TEST_BACKING"
|
||||
EOF
|
||||
chmod +x "$tmp/bin/"*
|
||||
PATH="$tmp/bin:$PATH"
|
||||
export PATH
|
||||
source "$runner_dir/install-runner-lib.sh"
|
||||
runuser() {
|
||||
shift 2
|
||||
[[ $1 == -- ]] || return 2
|
||||
shift
|
||||
"$@"
|
||||
}
|
||||
getent() {
|
||||
[[ $1 == passwd ]] || return 2
|
||||
if [[ -n ${RUNNER_TEST_PASSWD:-} ]]; then
|
||||
printf '%s\n' "$RUNNER_TEST_PASSWD"
|
||||
elif [[ -r ${RUNNER_TEST_ACCOUNT_STATE:-} ]]; then
|
||||
cat "$RUNNER_TEST_ACCOUNT_STATE"
|
||||
else
|
||||
return 2
|
||||
fi
|
||||
}
|
||||
id() {
|
||||
case $1 in
|
||||
-gn) printf '%s\n' "${RUNNER_TEST_PRIMARY_GROUP:-akyldash-runner}" ;;
|
||||
-nG) [[ ${TEST_EXTRA_GROUP:-0} == 1 ]] && printf 'akyldash-runner docker\n' || printf 'akyldash-runner\n' ;;
|
||||
*) command id "$@" ;;
|
||||
esac
|
||||
}
|
||||
stat() {
|
||||
if [[ $1 == -c && $2 == %U && $3 == ${RUNNER_TEST_HOME:-} ]]; then
|
||||
printf '%s\n' "${RUNNER_TEST_HOME_OWNER:-akyldash-runner}"
|
||||
elif [[ $1 == -c && $2 == %G:%a && $3 == ${RUNNER_TEST_HOME:-} ]]; then
|
||||
printf '%s:%s\n' "${RUNNER_TEST_HOME_GROUP:-root}" "${RUNNER_TEST_HOME_MODE:-711}"
|
||||
else
|
||||
command stat "$@"
|
||||
fi
|
||||
}
|
||||
useradd() {
|
||||
printf 'useradd\n' >>"$RUNNER_TEST_LOG"
|
||||
local user=${@: -1} home=
|
||||
while (($#)); do
|
||||
if [[ $1 == --home-dir ]]; then home=$2; shift 2; else shift; fi
|
||||
done
|
||||
mkdir -p "$home"
|
||||
printf '%s:x:990:990::%s:/usr/sbin/nologin\n' "$user" "$home" >"$RUNNER_TEST_ACCOUNT_STATE"
|
||||
}
|
||||
|
||||
export RUNNER_TEST_HOME="$tmp/account-home"
|
||||
export RUNNER_TEST_ACCOUNT_STATE="$tmp/account.passwd"
|
||||
mkdir -p "$RUNNER_TEST_HOME"
|
||||
export RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
RUNNER_TEST_HOME_OWNER=root runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
if RUNNER_TEST_HOME_OWNER=root RUNNER_TEST_HOME_MODE=755 runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Writable root-owned runner home was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
TEST_EXTRA_GROUP=1
|
||||
if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with extra group membership was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset TEST_EXTRA_GROUP
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${tmp}/wrong-home:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with a different home was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:0:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with UID 0 was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:1000:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with a regular-user UID was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
RUNNER_TEST_PRIMARY_GROUP=unexpected
|
||||
if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with an unexpected primary group was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset RUNNER_TEST_PRIMARY_GROUP
|
||||
unset RUNNER_TEST_PASSWD
|
||||
export RUNNER_TEST_HOME="$tmp/new-account-home"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
[[ $(grep -c '^useradd$' "$log") == 1 ]]
|
||||
|
||||
private_root=$tmp/private
|
||||
mkdir -m 0700 "$private_root"
|
||||
runner_owner=$(command id -un)
|
||||
runner_group=$(command id -gn)
|
||||
runner_ensure_private_directory "$private_root/config" "$runner_owner" "$runner_group"
|
||||
runner_ensure_private_directory "$private_root/config" "$runner_owner" "$runner_group"
|
||||
mkdir -m 0700 "$tmp/outside"
|
||||
ln -s "$tmp/outside" "$private_root/linked"
|
||||
if runner_ensure_private_directory "$private_root/linked/docker" "$runner_owner" "$runner_group" 2>/dev/null; then
|
||||
echo 'Runner config symlink was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ ! -e $tmp/outside/docker ]]
|
||||
|
||||
legacy_root=$tmp/legacy
|
||||
mkdir -p "$legacy_root/.config/systemd/user/docker.service.d"
|
||||
chmod 0755 "$legacy_root" "$legacy_root/.config" "$legacy_root/.config/systemd" "$legacy_root/.config/systemd/user" "$legacy_root/.config/systemd/user/docker.service.d"
|
||||
for legacy_dir in "$legacy_root" "$legacy_root/.config" "$legacy_root/.config/systemd" "$legacy_root/.config/systemd/user" "$legacy_root/.config/systemd/user/docker.service.d"; do
|
||||
runner_ensure_private_directory "$legacy_dir" "$runner_owner" "$runner_group"
|
||||
[[ $(stat -c %a "$legacy_dir") == 700 ]]
|
||||
done
|
||||
user_config=$private_root/daemon.json
|
||||
runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF'
|
||||
{"data-root":"/private/runner"}
|
||||
EOF
|
||||
runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF'
|
||||
{"data-root":"/private/runner"}
|
||||
EOF
|
||||
if runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF' 2>/dev/null
|
||||
{"data-root":"/unexpected"}
|
||||
EOF
|
||||
then
|
||||
echo 'Conflicting runner user config was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
image=$tmp/docker-store.img
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
|
||||
truncate -s 1048575 "$image"
|
||||
if runner_ensure_image "$image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Mismatched image size was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
truncate -s 1048576 "$image"
|
||||
if RUNNER_TEST_FS=ext3 runner_ensure_image "$image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Existing non-ext4 image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
external=$tmp/external-file
|
||||
printf 'preserve this file\n' >"$external"
|
||||
race_image=$tmp/race.img
|
||||
ln -s "$external" "$race_image"
|
||||
if runner_ensure_image "$race_image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Symlink runner image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(cat "$external") == 'preserve this file' ]]
|
||||
rm -f "$race_image"
|
||||
race_image=$tmp/race-during-format.img
|
||||
export RUNNER_TEST_IMAGE="$race_image" RUNNER_TEST_REPLACE_IMAGE="$external"
|
||||
if runner_ensure_image "$race_image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Image path replacement during formatting was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(cat "$external") == 'preserve this file' ]]
|
||||
rm -f "$race_image"
|
||||
unset RUNNER_TEST_IMAGE RUNNER_TEST_REPLACE_IMAGE
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
|
||||
unit_name='var-lib-akyldash\x2drunner-docker\x2ddata.mount'
|
||||
unit_file=$tmp/systemd/$unit_name
|
||||
mkdir -p "$(dirname "$unit_file")"
|
||||
export RUNNER_TEST_STORE="$tmp/store" RUNNER_TEST_BACKING="$image"
|
||||
export RUNNER_TEST_MOUNTED="$tmp/mounted"
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
grep -Fq "enable $unit_name" "$log"
|
||||
grep -Fq "start $unit_name" "$log"
|
||||
|
||||
# A second run with the image mounted must reuse it and avoid another start.
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
chmod 0664 "$unit_file"
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
|
||||
echo 'Writable systemd mount unit was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
chmod 0644 "$unit_file"
|
||||
export RUNNER_TEST_BACKING="$tmp/unrelated.img"
|
||||
touch "$RUNNER_TEST_BACKING"
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
|
||||
echo 'Active mount backed by a different image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
export RUNNER_TEST_BACKING="$image"
|
||||
|
||||
# Simulate interruption after unit installation/enable, before mount activation.
|
||||
export RUNNER_TEST_STORE="$tmp/partial-store" RUNNER_TEST_MOUNTED="$tmp/partial-mounted"
|
||||
export RUNNER_TEST_FAIL_START=1
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount" 2>/dev/null; then
|
||||
echo 'Injected mount interruption was not observed' >&2
|
||||
exit 1
|
||||
fi
|
||||
export RUNNER_TEST_FAIL_START=0
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount"
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
[[ $(grep -c '^start ' "$log") == 3 ]]
|
||||
export RUNNER_TEST_STORE="$tmp/store" RUNNER_TEST_MOUNTED="$tmp/mounted" RUNNER_TEST_OPTIONS=rw,relatime
|
||||
if runner_validate_mount "$image" "$RUNNER_TEST_STORE" 2>/dev/null; then
|
||||
echo 'Runner mount missing nosuid/nodev was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset RUNNER_TEST_OPTIONS
|
||||
echo 'runner installer idempotency checks passed'
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
config=infra/runner/config.yaml
|
||||
installer=infra/runner/install-runner.sh
|
||||
assert_absent() {
|
||||
local pattern=$1 file=$2
|
||||
if grep -Eq "$pattern" "$file"; then
|
||||
echo "Forbidden content matched in $file" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
grep -Eq '^ capacity: 1$' "$config"
|
||||
grep -Fq 'akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04' "$config"
|
||||
grep -Fq 'docker_host: "-"' "$config"
|
||||
grep -Eq '^ privileged: false$' "$config"
|
||||
grep -Eq '^ valid_volumes: \[\]$' "$config"
|
||||
grep -Fq -- '--cpus=2 --memory=3g --pids-limit=512' "$config"
|
||||
assert_absent 'var/run/docker.sock|metrics:|health_check:|production.env|review_secret' "$config"
|
||||
grep -Fq 'docker-ce-rootless-extras' "$installer"
|
||||
grep -Fq 'runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711' "$installer"
|
||||
grep -Fq 'runner_lock_home "$runner_home"' "$installer"
|
||||
grep -Fq 'test -w "$HOME" && test -w "$XDG_CONFIG_HOME" && test -w "$XDG_DATA_HOME"' "$installer"
|
||||
grep -Fq 'rootless_env=(HOME="$runner_runtime_home" XDG_CONFIG_HOME="$runner_xdg_config_home" XDG_DATA_HOME="$runner_xdg_data_home"' "$installer"
|
||||
grep -Fq 'Environment=XDG_CONFIG_HOME=$runner_xdg_config_home' "$installer"
|
||||
grep -Fq 'Environment=XDG_DATA_HOME=$runner_xdg_data_home' "$installer"
|
||||
grep -Fq 'runner_home/.config/systemd/user/docker.service.d/environment.conf' "$installer"
|
||||
grep -Fq 'Environment=HOME=$runner_runtime_home' "$installer"
|
||||
grep -Fq 'systemctl --user restart docker.service' "$installer"
|
||||
grep -Fq 'systemctl --user show docker.service --property=MainPID --value' "$installer"
|
||||
grep -Fq 'tr '\''\0'\'' '\''\n'\'' <"/proc/$docker_pid/environ"' "$installer"
|
||||
grep -Fq 'grep -Fqx -- "$expected"' "$installer"
|
||||
if grep -Fq 'systemctl --user set-environment' "$installer"; then
|
||||
echo 'Rootless Docker environment must persist in its user unit drop-in' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'HOME=/var/lib/akyldash-runner/.config/act-runner' infra/runner/act-runner-healthcheck.sh
|
||||
grep -Fq 'XDG_CONFIG_HOME=/var/lib/akyldash-runner/.config' infra/runner/act-runner-healthcheck.sh
|
||||
grep -Fq 'XDG_DATA_HOME=/var/lib/akyldash-runner/.config/act-runner/.local/share' infra/runner/act-runner-healthcheck.sh
|
||||
if grep -Fq 'env HOME="$runner_home"' "$installer"; then
|
||||
echo 'Protected runner home must not be passed as rootless Docker HOME' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'die '\''stop akyldash-act-runner.service after all jobs finish before rerunning this installer' "$installer"
|
||||
grep -Fq 'ReadWritePaths=$runner_runtime_home $runner_xdg_config_home /var/log/akyldash-runner /run/user/$runner_uid' "$installer"
|
||||
grep -Fq 'os.O_NOFOLLOW | os.O_CLOEXEC' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'os.posix_fallocate(image_fd, 0, int(bytes_text))' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'pass_fds=(image_fd,)' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'systemctl enable "$unit_name"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'systemctl start "$unit_name"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'runuser -u "$owner" -- install -d -m 0700 "$directory"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'runner_install_user_content "$runner_home/.config/docker/daemon.json"' "$installer"
|
||||
assert_absent 'basename.*runner_mount_unit|enable --now.*runner_mount_unit' "$installer"
|
||||
grep -Fq 'groups=$(id -nG "$user")' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'id -gn "$user"' infra/runner/install-runner-lib.sh
|
||||
assert_absent 'dockerd-rootless-setuptool.sh install --force' "$installer"
|
||||
grep -Fq 'runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2' "$installer"
|
||||
grep -Fq 'find-free-subid-range.awk' "$installer"
|
||||
grep -Fq 'usermod "${subid_args[@]}" "$runner_user"' "$installer"
|
||||
grep -Fq 'subid_args+=(--add-subuids "$subid_range")' "$installer"
|
||||
assert_absent 'usermod.*(sudo|docker)|/var/run/docker.sock|/etc/sudoers' "$installer"
|
||||
grep -Fq 'runs-on: akyldash-ci' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -S /var/run/docker.sock' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -e /etc/akyldash/production.env' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -e /etc/akyldash/review_secret' .gitea/workflows/runner-smoke.yaml
|
||||
assert_absent '(^|[[:space:]])(docker|docker-compose|deploy)([[:space:]]|$)|production.env=|review_secret=' .gitea/workflows/runner-smoke.yaml
|
||||
|
||||
fixture=$(mktemp)
|
||||
fixture_g=$(mktemp)
|
||||
trap 'rm -f "$fixture" "$fixture_g"' EXIT
|
||||
printf 'production.env\n' >"$fixture"
|
||||
if assert_absent 'deploy|production.env|review_secret|docker.sock' "$fixture" 2>/dev/null; then
|
||||
echo 'Negative guard regression: prohibited fixture was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'codex:100000:65536\n' >"$fixture"
|
||||
printf 'codex:165000:1000\n' >"$fixture_g"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 231072-296607 ]]
|
||||
: >"$fixture_g"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 165536-231071 ]]
|
||||
: >"$fixture"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 100000-165535 ]]
|
||||
printf 'broken\n' >"$fixture_g"
|
||||
if awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g" >/dev/null 2>&1; then
|
||||
echo 'Malformed subordinate ID allocation was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'runner configuration guards passed'
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
run_as_unprivileged() {
|
||||
if [[ $EUID -eq 0 ]]; then
|
||||
if ! command -v setpriv >/dev/null 2>&1; then
|
||||
echo 'setpriv is required to run guard tests as an unprivileged UID.' >&2
|
||||
return 127
|
||||
fi
|
||||
setpriv --reuid=65534 --regid=65534 --clear-groups -- "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
# Вторая волна проверки поиска Akyldash
|
||||
|
||||
Период сбора: 25–26 сентября 2026 года. Результаты получены независимым
|
||||
повторным прогоном по 250 ситуациям. Предыдущий отчет сохранен отдельно и не
|
||||
использован как источник запросов, результатов или выводов.
|
||||
|
||||
## Методика
|
||||
|
||||
- Составлены 250 разных бытовых ситуаций: по 25 в десяти областях права.
|
||||
- Каждая строка из `queries.json` отправлялась без изменения и в Akyldash, и в
|
||||
[поиск Consultant.ru](https://www.consultant.ru/search/). Для Akyldash вызван
|
||||
read-only `GET /search` с `page_size=10`; сохранены первые десять карточек.
|
||||
Для Consultant использован штатный поток страницы: запрос поиска, затем
|
||||
AJAX-поллинг до готового ответа; также сохранены первые десять карточек.
|
||||
- Исходный runtime Akyldash подтвердил backend `0.10.4`, алгоритм `search-6`
|
||||
и индекс `akyldash-fragments-v2`. Изменение затем проверено в отдельном
|
||||
контейнере с backend `0.10.5 / search-7` на том же индексе; production не
|
||||
перезапускался. Consultant отдавал страницу с версией `2026.1.1.9b09de08`.
|
||||
- Получено 250/250 пар, ID и формулировки уникальны; пропусков, повторов и
|
||||
ошибок локального API нет. Consultant вернул 40 готовых пустых выдач и 210
|
||||
непустых. Пустой готовый ответ учитывался как ноль; ответы `ready=false`
|
||||
повторялись и не выдавались за пустые. История повторов находится в
|
||||
`capture/failed-attempts.jsonl`.
|
||||
- Системы работают с разными корпусами: Akyldash — документы Кыргызской
|
||||
Республики, Consultant — документы Российской Федерации. Сравнивается
|
||||
попадание в один и тот же жизненный интент, а не совпадение законов или
|
||||
юридическая правильность правовых последствий.
|
||||
|
||||
## Итоговые метрики
|
||||
|
||||
| Метрика | Akyldash | Consultant.ru |
|
||||
|---|---:|---:|
|
||||
| Успешные запросы | 250/250 | 250/250 |
|
||||
| Пустая выдача | 0 (0%) | 40 (16%) |
|
||||
| Среднее число карточек | 10,0 | 7,0 |
|
||||
| Медиана задержки | 25,2 мс | 2,54 с |
|
||||
| P95 задержки | 54,1 мс | 6,40 с |
|
||||
| Лексическое совпадение темы в top-1 | 52,8% | 67,2% |
|
||||
| Лексическое совпадение темы в top-3 | 72,8% | 76,0% |
|
||||
| Лексическое совпадение темы в top-10 | 84,4% | 78,8% |
|
||||
| Лексический маркер отрасли в top-10 | 76,0% | 77,6% |
|
||||
| Покрытие значимых слов запроса в top-1 | 17,5% | 30,5% |
|
||||
| Покрытие значимых слов запроса в top-10 | 36,4% | 50,0% |
|
||||
|
||||
`topic_hit` проверяет совпадение по префиксу одного из первых четырех
|
||||
содержательных слов запроса; `domain_anchor` ищет лексический маркер отрасли;
|
||||
покрытие слов считает долю префиксов значимых слов запроса в карточках.
|
||||
Это автоматические диагностические признаки, не экспертная оценка
|
||||
релевантности. Consultant показывает текст и выдержки, тогда как у большинства
|
||||
локальных карточек `snippet` отсутствует, поэтому сравнение покрытия текста
|
||||
несимметрично.
|
||||
|
||||
Полные метрики и отраслевые разрезы сохранены в
|
||||
[`capture/summary.json`](capture/summary.json), все карточки и измерения — в
|
||||
[`capture/results.jsonl`](capture/results.jsonl).
|
||||
|
||||
## Проверка настройки после review
|
||||
|
||||
Кандидат с `minimum_should_match=1` на low-weight OR fallback проверялся в
|
||||
изолированном контейнере `0.10.5 / search-7` на корпусе
|
||||
`akyldash-fragments-v2`. Первая проба `2<50%` оказалась слишком строгой:
|
||||
topic hit@10 составил 50,4%, пустыми стали 20 ответов из 250; она сохранена в
|
||||
`search7/summary.json` и отвергнута. По исходным агрегатам вариант `1` выглядел
|
||||
лучше:
|
||||
|
||||
| Метрика Akyldash | До | После |
|
||||
|---|---:|---:|
|
||||
| Успешные запросы | 250/250 | 250/250 |
|
||||
| Пустая выдача | 0 | 0 |
|
||||
| Topic hit@1 | 52,8% | 54,0% |
|
||||
| Topic hit@3 | 72,8% | 72,8% |
|
||||
| Topic hit@10 | 84,4% | 85,2% |
|
||||
| Маркер отрасли в top-10 | 76,0% | 75,2% |
|
||||
| Покрытие слов запроса в top-1 | 17,46% | 17,31% |
|
||||
| Покрытие слов запроса в top-3 | 26,73% | 26,77% |
|
||||
| Покрытие слов запроса в top-10 | 36,38% | 37,09% |
|
||||
| Медиана локального ответа | 25,2 мс | 65,4 мс |
|
||||
| P95 локального ответа | 54,1 мс | 273,6 мс |
|
||||
|
||||
Эти лексические агрегаты скрыли регрессии ранжирования. Официальный review PR
|
||||
сопоставил все 250 top-1 и выявил 17 перестановок. Среди них: для `criminal-11`
|
||||
профильный порядок действий следователя (`94181`) уступил распоряжению об
|
||||
установке телефонов в автомашинах (`43480`); для `administrative-14` правила
|
||||
митингов (`23570`) уступили проекту постановления о доработке законов о
|
||||
собраниях (`8978`). Поэтому вариант `1` не принят, несмотря на прирост
|
||||
topic hit@10 на 0,8 п.п.
|
||||
|
||||
Проверен компромисс: оставить исходную ветку с порогом `2` и добавить однословную
|
||||
ветку с boost `0.01`. Она вернула оба указанных top-1, но все top-10 позиции и
|
||||
метрики полностью совпали с baseline. Дополнительная ветка не дала результата,
|
||||
поэтому ее не оставили. Другие пороги `1<30%`, `1<40%`, `2<40%` и изменение
|
||||
веса названия также не прошли проверку: они вызвали новые нерелевантные первые
|
||||
результаты, пустые выдачи или падение topic hit@10. Итоговый вывод: безопасная
|
||||
настройка по этой лексической выборке не найдена; код поиска возвращен к
|
||||
`0.10.4 / search-6` и исходному порогу `2`. Для следующего этапа нужна
|
||||
экспертная разметка документов по интенту.
|
||||
|
||||
Основной кандидат `1` и двухветочный вариант сохранены отдельно в
|
||||
[`search7-mm1/summary.json`](search7-mm1/summary.json),
|
||||
[`search7-mm1/results.jsonl`](search7-mm1/results.jsonl),
|
||||
[`search7-dual/summary.json`](search7-dual/summary.json) и
|
||||
[`search7-dual/results.jsonl`](search7-dual/results.jsonl). Метрики остаются
|
||||
диагностическими и не заменяют экспертную оценку релевантности.
|
||||
|
||||
### Контроль текущего production
|
||||
|
||||
26 сентября 2026 года live backend `0.10.4` повторно проверен read-only по тем
|
||||
же 250 запросам. Качественные метрики полностью совпали с исходным baseline:
|
||||
250/250 успешных ответов, 0 пустых, topic hit@1/@3/@10 — 52,8% / 72,8% /
|
||||
84,4%, покрытие слов top-10 — 36,38%. Медиана составила 19,6 мс, P95 — 52,9 мс;
|
||||
задержка зависит от состояния сервиса. Consultant повторно не опрашивался — в
|
||||
контрольном сравнении использованы сохраненные исходные ответы. Карточки
|
||||
production и сводка доступны в
|
||||
[`production-before-rollout/results.jsonl`](production-before-rollout/results.jsonl)
|
||||
и [`production-before-rollout/summary.json`](production-before-rollout/summary.json).
|
||||
|
||||
## Отраслевые разрезы
|
||||
|
||||
Значения `topic_hit@1 / topic_hit@10`; каждая строка содержит 25 ситуаций.
|
||||
В этой таблице Akyldash показан на production baseline (`0.10.4 / search-6`),
|
||||
а Consultant.ru — из исходного сбора.
|
||||
|
||||
| Область | Akyldash, baseline | Consultant, исходный сбор | Пусто у Consultant |
|
||||
|---|---:|---:|---:|
|
||||
| Трудовое право | 56% / 92% | 100% / 100% | 0 |
|
||||
| Жилищные споры | 40% / 84% | 44% / 56% | 8 |
|
||||
| Права потребителей | 36% / 64% | 40% / 68% | 6 |
|
||||
| Семейные споры | 48% / 84% | 60% / 80% | 4 |
|
||||
| Наследование | 44% / 88% | 64% / 84% | 2 |
|
||||
| Договоры и имущество | 56% / 80% | 76% / 84% | 4 |
|
||||
| Уголовные ситуации | 52% / 84% | 64% / 68% | 6 |
|
||||
| Административные споры | 52% / 92% | 72% / 72% | 5 |
|
||||
| Налоги и бизнес | 84% / 96% | 88% / 96% | 1 |
|
||||
| Суд и госуслуги | 60% / 80% | 64% / 80% | 4 |
|
||||
|
||||
## Что видно в карточках
|
||||
|
||||
1. **Akyldash обеспечивает покрытие, но часто не поднимает лучший результат
|
||||
вверх.** Все 250 запросов вернули по десять карточек. Совпадение темы в
|
||||
top-10 выше Consultant на 5,6 п.п., но в top-1 ниже на 14,4 п.п., а покрытие
|
||||
слов запроса в top-1 ниже на 13,1 п.п. Это указывает прежде всего на
|
||||
ранжирование и слишком широкое расширение.
|
||||
2. **Выдачи с высокой лексической похожестью могут относиться к другому
|
||||
смыслу.** Для вопроса о возврате залога при аренде локальный top-1 — типовой
|
||||
договор охраны квартир; для штрафа на уже проданный автомобиль — строительные
|
||||
нормы о стоянках. Формальные совпадения слов не подтверждают попадание в
|
||||
интент.
|
||||
3. **Сильные локальные результаты тоже есть.** На вопрос о травме на работе
|
||||
Akyldash ставит действующее Положение о расследовании несчастных случаев на
|
||||
производстве. Consultant начинает с судебного акта об оспаривании вывода
|
||||
трудового инспектора. Лексическая метрика недооценивает локальный результат,
|
||||
потому что название использует юридическую формулировку «несчастный случай
|
||||
на производстве», а запрос — «получил травму».
|
||||
4. **Пустые результаты Consultant надо отличать от недоступности.** В исходном
|
||||
сборе 40 поисков не успели стать готовыми за первое окно ожидания. После
|
||||
повторного прохода все они вернули готовый ответ без карточек; в итоговой
|
||||
таблице это нули, а не ошибки сети.
|
||||
5. **Наиболее заметный разрыв — в бытовых формулировках, не в скорости.**
|
||||
Consultant чаще показывает сразу описание спора или судебного материала.
|
||||
Akyldash отвечает быстрее и полнее по числу карточек, но его title-only
|
||||
выдача чаще ставит рядом документы с общими словами «выплата», «ущерб»,
|
||||
«работа» или «договор».
|
||||
|
||||
Примеры подтверждаются сырыми парами в `capture/results.jsonl`: запросы
|
||||
`labour-04` (сверхурочная работа), `administrative-02` (штраф после продажи
|
||||
машины), `civil-02` (заем без расписки), `consumer-15` (оплаченные, но
|
||||
прекращенные занятия), `inheritance-01` (пропущенный срок наследства) и
|
||||
`public_court-02` (восстановление срока апелляции).
|
||||
|
||||
## Ограничения
|
||||
|
||||
- Все формулировки русскоязычные; кыргызский поиск этой волной не оценивался.
|
||||
- Не создан экспертно размеченный набор `document_code`; метрики выше нельзя
|
||||
называть Recall@10/MRR или считать доказательством юридической релевантности.
|
||||
- Выдержки Consultant богаче локальных карточек без `snippet`; это повышает
|
||||
измеряемое покрытие запроса в Consultant.
|
||||
- Consultant индексирует российское право и используется как ориентир
|
||||
обработки похожего пользовательского интента. Его документы не служат
|
||||
эталоном для права КР.
|
||||
- Вторая волна не заменяет проверку поисковых решений юристом на официальных
|
||||
источниках Минюста КР.
|
||||
@@ -0,0 +1,48 @@
|
||||
# План настройки поиска после второй волны
|
||||
|
||||
## Цель
|
||||
|
||||
Улучшить ранжирование 250 русскоязычных бытовых запросов, сохраняя покрытие
|
||||
полезных документов. Baseline: production `0.10.4 / search-6`, индекс
|
||||
`akyldash-fragments-v2`, результаты в `capture/results.jsonl`.
|
||||
|
||||
## Вывод по первому кандидату
|
||||
|
||||
Снижение `minimum_should_match` для low-weight OR fallback с `2` до `1` дало
|
||||
небольшой рост topic hit@10 (`84,4% → 85,2%`) и coverage@10
|
||||
(`36,38% → 37,09%`), но независимое review сравнило top-1 всех 250 запросов и
|
||||
нашло 17 перестановок. В частности, появились явно нерелевантные первые
|
||||
результаты для изъятого телефона при обыске и задержания на митинге. Кандидат
|
||||
отклонен.
|
||||
|
||||
Проверенные компромиссы:
|
||||
|
||||
- `1<30%` вернул два примера review к baseline, но оставил 14 других перестановок,
|
||||
включая замену точной карточки об установлении отцовства нерелевантным
|
||||
докладом; coverage@10 не вырос.
|
||||
- `1<40%` и `2<40%` дали 1 пустую выдачу, topic hit@10 упал до 76%, top-1
|
||||
изменился в 77 запросах.
|
||||
- Ослабление веса названия документа резко ухудшило метрики.
|
||||
- Однословный fallback с boost `0.01` вместе с исходной веткой `minimum=2`
|
||||
сохранил весь baseline top-10 и метрики без изменений; дополнительная ветка
|
||||
не дала полезного эффекта.
|
||||
|
||||
## Решение
|
||||
|
||||
Поиск возвращен к исходному `minimum_should_match=2`; версия остается
|
||||
`0.10.4 / search-6`. В production никаких изменений не вносилось. На текущей
|
||||
выборке невозможно обосновать новую конфигурацию без риска ухудшить отдельные
|
||||
первые результаты. Следующее изменение ранжирования делать после экспертной
|
||||
разметки релевантных документов для изменяющихся запросов, а не по одному
|
||||
лексическому hit@10.
|
||||
|
||||
## Статус
|
||||
|
||||
- [x] Создано 250 уникальных ситуаций в 10 правовых областях.
|
||||
- [x] Сравнены выдачи Akyldash и Consultant.ru; пустые готовые ответы Consultant
|
||||
отделены от timeout.
|
||||
- [x] Повторно подтвержден production baseline на всех 250 запросах.
|
||||
- [x] Проведен независимый review; blocking finding о top-1 регрессиях
|
||||
подтвержден и учтен.
|
||||
- [x] Непрошедшие настройки отвергнуты; поисковый код возвращен к baseline.
|
||||
- [ ] Подготовить экспертную разметку и возобновить настройку после ее проверки.
|
||||
Loaded 100 of 116 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user