Compare commits
32
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9ec03200fb | ||
|
|
03c5809d69 | ||
|
|
00922e6c09 | ||
|
|
e6b099cf31 | ||
|
|
d8a0646829 | ||
|
|
433c3059b9 | ||
|
|
4110aa54c9 | ||
|
|
db6ded4075 | ||
|
|
b68421bb5b | ||
|
|
223e06ee77 | ||
|
|
52027ecf81 | ||
|
|
e5a4dab5f3 | ||
|
|
e034fdb816 | ||
|
|
a34f516de8 | ||
|
|
3b41d1f20f | ||
|
|
752266d408 | ||
|
|
f6c9b9883a | ||
|
|
d6d029d98c | ||
|
|
9084fd4f16 | ||
|
|
02cf4e6e95 | ||
|
|
711700fcc9 | ||
|
|
6f809d3196 | ||
|
|
16f97d5f73 | ||
|
|
97803ca5de | ||
|
|
fb556228c7 | ||
|
|
e42c314552 | ||
|
|
7124c82bb6 | ||
|
|
892781522d | ||
|
|
5490b22275 | ||
|
|
5c78f7ebb8 | ||
|
|
d03da40eee | ||
|
|
fb1a67044c |
No files matched your search
@@ -0,0 +1,59 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
tests-and-images:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Run backend tests
|
||||
run: PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v
|
||||
- name: Run Telegram bot tests
|
||||
run: python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v
|
||||
- name: Run infrastructure backup tests
|
||||
run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v
|
||||
- name: Check recovery host guard
|
||||
run: bash infra/bootstrap/test-host-guard.sh
|
||||
- name: Check verifier helper argument guard
|
||||
run: bash infra/bootstrap/test-verify-helper.sh
|
||||
- name: Check host audit root guard
|
||||
run: bash infra/bootstrap/test-audit-host.sh
|
||||
- name: Check Compose helper allowlist
|
||||
run: bash infra/deploy/test-compose-helper.sh
|
||||
- name: Check production config root guard
|
||||
run: bash infra/deploy/test-production-config-guard.sh
|
||||
- name: Check runner config and isolation guards
|
||||
run: |
|
||||
bash infra/runner/test-runner-config.sh
|
||||
bash infra/runner/test-install-runner-idempotency.sh
|
||||
bash infra/runner/test-install-runner-guard.sh
|
||||
- name: Check infrastructure shell syntax
|
||||
run: |
|
||||
bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \
|
||||
infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/bootstrap/audit-host.sh \
|
||||
infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \
|
||||
infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \
|
||||
infra/bootstrap/test-audit-host.sh infra/bootstrap/test-verify-helper.sh \
|
||||
infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \
|
||||
infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \
|
||||
infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh
|
||||
python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())"
|
||||
- name: Run rollback helper tests
|
||||
run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
|
||||
- name: Validate production Compose and Caddyfile
|
||||
run: |
|
||||
secret_file=$(mktemp)
|
||||
trap 'rm -f "$secret_file"' EXIT
|
||||
REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet
|
||||
docker run --rm --entrypoint caddy -v "$PWD/deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028 validate --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
- name: Build production images
|
||||
run: |
|
||||
docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci .
|
||||
docker build -f deploy/production/Dockerfile.opensearch -t akyldash-opensearch:ci .
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Runner smoke test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
runner-smoke:
|
||||
runs-on: akyldash-ci
|
||||
steps:
|
||||
- name: Verify isolated runner boundary
|
||||
run: |
|
||||
host_uid=$(awk 'NR == 1 {print $2}' /proc/self/uid_map)
|
||||
test "$host_uid" -gt 0
|
||||
test ! -S /var/run/docker.sock
|
||||
test ! -e /run/user/1000/docker.sock
|
||||
test -z "${DOCKER_HOST:-}"
|
||||
test ! -e /etc/akyldash/production.env
|
||||
test ! -e /etc/akyldash/review_secret
|
||||
test ! -e /opt/akyldash/current
|
||||
printf 'Isolated runner: version=%s host=%s uid-map=%s\n' \
|
||||
"$GITEA_ACTIONS_RUNNER_VERSION" "$(hostname)" "$host_uid"
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"detector": {
|
||||
"ignoreRules": [],
|
||||
"ignoreFiles": [],
|
||||
"ignoreValues": [
|
||||
{
|
||||
"rule": "cream-palette",
|
||||
"value": "*",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:40:02.635Z",
|
||||
"reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette"
|
||||
},
|
||||
{
|
||||
"rule": "cramped-padding",
|
||||
"value": "*",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:41:28.434Z",
|
||||
"reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector"
|
||||
},
|
||||
{
|
||||
"rule": "design-system-font",
|
||||
"value": "sans",
|
||||
"files": [
|
||||
"backend/search/review.html"
|
||||
],
|
||||
"createdAt": "2026-10-11T15:41:33.020Z",
|
||||
"reason": "Agent: DESIGN.md specifies Noto Sans with sans-serif fallback; this finding is the inherited generic fallback"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Акылдаш — инструкции для агента
|
||||
|
||||
Проект: юридическая информационно-аналитическая платформа; Search API на
|
||||
Python, SQLite и OpenSearch. Production пока не развернут.
|
||||
|
||||
- Начинай с [инфраструктурного состояния](docs/operations/CURRENT_STATE.md) и
|
||||
[правил доступа](docs/operations/ACCESS.md). Не запрашивай доступы повторно.
|
||||
- Архитектура и процедуры: [инфраструктура](docs/operations/INFRASTRUCTURE.md),
|
||||
[деплой](docs/operations/DEPLOYMENT.md), [backup](docs/operations/BACKUP.md),
|
||||
[rollback](docs/operations/ROLLBACK.md), [восстановление](docs/operations/RECOVERY.md).
|
||||
- Работай в feature/fix/refactor-ветке. `main` защищена; изменения вноси через
|
||||
PR, не сливай его автоматически. Сохраняй незакоммиченные изменения.
|
||||
- SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай
|
||||
`codex` общий sudo или доступ к Docker socket. Root-действия требует владелец.
|
||||
Для Gitea используй `gitea-api`, не читай файлы с учётными данными.
|
||||
- Репозиторный Gitea Actions runner описан в
|
||||
[CI_RUNNER.md](docs/operations/CI_RUNNER.md). Регистрация выполняется
|
||||
владельцем через Settings → Actions → Runners для `admin/akyldash`; токен
|
||||
вводится интерактивно и не попадает в Git. Runner использует отдельный
|
||||
rootless Docker daemon; job-контейнеры не получают Docker socket или
|
||||
production secrets. Не включай production deployment из CI без отдельного
|
||||
проверенного deploy-helper.
|
||||
- Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и
|
||||
`python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`,
|
||||
`python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и
|
||||
`python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`,
|
||||
`bash infra/bootstrap/test-host-guard.sh`.
|
||||
Production images собираются командами из `.gitea/workflows/ci.yaml`.
|
||||
- Перед изменениями production необходим проверенный backup. Формального
|
||||
механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй
|
||||
SSH, firewall или production вручную без владельца и последующего PR.
|
||||
- Холодный backup требует согласованного окна простоя; восстановление допускается
|
||||
только в новый staging-каталог, пока владелец отдельно не проверил процедуру
|
||||
для нового хоста.
|
||||
- После каждого изменения инфраструктуры обновляй
|
||||
`docs/operations/CURRENT_STATE.md` по фактическим проверкам.
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
## Не выпущено
|
||||
|
||||
- Runner installer сохраняет `HOME` и XDG-пути в user-unit `docker.service`, перезапускает активный daemon и проверяет окружение его процесса.
|
||||
- Runner installer защищает image и mountpoint root-owned home-каталогом, оставляет регистрацию и workspace в runner-owned каталоге, безопасно форматирует образ по открытому FD и проверяет `nosuid,nodev`.
|
||||
- Backend `0.11.3`: ключ подписи поисковых снимков читается из Docker secret-файла, а не из аргументов процесса. Добавлена базовая инфраструктура VPS, Caddy HTTPS proxy с разделёнными Compose-сетями и CI-проверка конфигурации; публикация образов и production-деплой остаются выключены до проверки runner, registry, backup/restore и ограниченного deploy-helper.
|
||||
- Frontend `0.1.2`: на планшетной ширине экран поиска переходит в одну колонку, чтобы фильтры, результаты и документ помещались в окно.
|
||||
- Backend `0.11.2`: смена пароля требует текущий пароль и запрещает оставить прежний, включая временный пароль при первом входе.
|
||||
- Backend `0.11.1` и frontend `0.1.1`: в управлении пользователями показываются даты первого и последнего входа, а также последнего сброса пароля.
|
||||
|
||||
@@ -10,7 +10,7 @@ Telegram-бот — только часть рабочего окружения
|
||||
## Текущее состояние
|
||||
|
||||
Сейчас реализованы Telegram-бот-секретарь версии `0.2.2`, backend версии
|
||||
`0.11.2` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
|
||||
`0.11.3` и внутренний frontend лаборатории `0.1.2`. Авторизация хранится в
|
||||
SQLite; первый пользователь `admin` создаётся интерактивной CLI-командой.
|
||||
Поиск расширяет аббревиатуры организационно-правовых форм и поднимает
|
||||
действующий Налоговый кодекс для запросов о налоге на прибыль.
|
||||
@@ -18,7 +18,7 @@ SQLite; первый пользователь `admin` создаётся инт
|
||||
| Компонент | Версия | Состояние |
|
||||
|---|---:|---|
|
||||
| Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
|
||||
| Backend | `0.11.2` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
|
||||
| Backend | `0.11.3` | авторизация и поиск с фильтрами; тематический поиск ОсОО/ЖЧК |
|
||||
| Frontend | `0.1.2` | лаборатория поиска, личный кабинет и управление учетными записями |
|
||||
| Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи |
|
||||
| RAG и база знаний | — | ещё не созданы |
|
||||
@@ -62,4 +62,4 @@ python3 -m unittest discover -s tools/telegram-bot -v
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Backend Акылдаш
|
||||
|
||||
Версия: `0.11.2`
|
||||
Версия: `0.11.3`
|
||||
|
||||
Первая backend-область проекта — загрузка правовых документов из официального
|
||||
Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
|
||||
@@ -253,4 +253,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Backend v0.11.2 · Frontend v0.1.2
|
||||
Акылдаш · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -451,6 +451,7 @@ def main() -> int:
|
||||
parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
|
||||
parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
|
||||
parser.add_argument("--review-secret", default=None)
|
||||
parser.add_argument("--review-secret-file", type=Path)
|
||||
parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
|
||||
parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
|
||||
parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
|
||||
@@ -458,7 +459,14 @@ def main() -> int:
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
parser.add_argument("--version", action="version", version=APP_VERSION)
|
||||
arguments = parser.parse_args()
|
||||
secret = arguments.review_secret.encode() if arguments.review_secret else None
|
||||
if arguments.review_secret and arguments.review_secret_file:
|
||||
parser.error("choose either --review-secret or --review-secret-file")
|
||||
try:
|
||||
secret = arguments.review_secret.encode() if arguments.review_secret else arguments.review_secret_file.read_bytes().strip() if arguments.review_secret_file else None
|
||||
except OSError as error:
|
||||
parser.error(f"cannot read review secret file: {error}")
|
||||
if arguments.review_secret_file and not secret:
|
||||
parser.error("review secret file is empty")
|
||||
api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
|
||||
if arguments.create_admin or arguments.reset_admin_password:
|
||||
users = api.auth_store.list_users()
|
||||
|
||||
@@ -17,7 +17,7 @@ from typing import Iterator
|
||||
|
||||
from search.catalog import authority_codes, legal_force_code, source_code
|
||||
|
||||
APP_VERSION = "0.11.2"
|
||||
APP_VERSION = "0.11.3"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Оценка поисковой выдачи · Акылдаш</title>
|
||||
<style>
|
||||
:root { color-scheme: light; font: 16px/1.5 "Noto Sans", "Segoe UI", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", Georgia, serif; }
|
||||
:root { color-scheme: light; font: 16px/1.5 "Noto Sans", sans-serif; color: #38262A; background: #F7F2E8; --ink: #38262A; --muted: #786469; --line: #DED3CE; --surface: #fff; --surface-muted: #FBF8F4; --brand: #9D2633; --brand-dark: #84202B; --accent: #A37B32; --focus: #245C78; --subtle: #F8EBED; --serif: "Noto Serif", serif; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; color: var(--ink); background: #F7F2E8; }
|
||||
.review__skip { position: absolute; inset-block-start: 0; inset-inline-start: -10000px; padding: .5rem 1rem; background: var(--surface); }
|
||||
@@ -53,7 +53,7 @@
|
||||
.review__comment { min-height: 4rem; resize: vertical; background: var(--surface); }
|
||||
.review__document { position: sticky; inset-block-start: 1rem; min-height: 24rem; }
|
||||
.review__document-meta { margin-block-end: 1rem; padding-block-end: 1rem; border-block-end: 1px solid var(--line); color: var(--muted); }
|
||||
.review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
|
||||
.review__document-body { max-width: 75ch; overflow-wrap: anywhere; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
|
||||
.review__document-body img { max-width: 100%; height: auto; }
|
||||
.review__actions { max-width: 1320px; margin: 1.5rem auto 0; padding-inline: clamp(1rem, 3vw, 3rem); }
|
||||
.review__actions-inner { display: flex; align-items: end; gap: .75rem; padding: 1rem; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; }
|
||||
@@ -115,7 +115,7 @@
|
||||
.search__document { min-height: 60vh; }
|
||||
.search__document-controls { display: flex; flex-wrap: wrap; gap: .75rem; }
|
||||
.search__document-controls label { display: grid; gap: .25rem; min-width: 11rem; color: var(--muted); font-size: .875rem; }
|
||||
.search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", "Segoe UI", sans-serif; }
|
||||
.search__document-body { max-width: 75ch; margin: auto; font: 400 1.125rem/1.65 "Noto Sans", sans-serif; }
|
||||
.search__document-body h1, .search__document-body h2, .search__document-body h3 { font-family: var(--serif); }
|
||||
.search__document-body img { max-width: 100%; height: auto; }
|
||||
.search__sr-only { position: absolute; inline-size: 1px; block-size: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
|
||||
@@ -176,7 +176,7 @@
|
||||
<dialog id="users-dialog" aria-labelledby="users-heading"><section class="review__page"><h2 id="users-heading">Управление пользователями</h2><form id="create-user-form"><label>Логин<input id="new-username" autocomplete="off" minlength="3" maxlength="64" required></label><label>Временный пароль<input id="temporary-password" type="password" autocomplete="new-password" minlength="12" maxlength="256" required></label><p>Пароль — от 12 символов. Новый пользователь сменит его при первом входе. Передайте временный пароль отдельно.</p><p id="users-error" role="alert" class="review__status review__status--error"></p><button class="review__button--primary" type="submit">Создать пользователя</button></form><ul id="user-list" class="review__user-list"></ul><button type="button" data-close="users-dialog">Закрыть</button></section></dialog>
|
||||
<dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
|
||||
<div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
|
||||
<footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.2 · Внутренняя лаборатория релевантности</div></footer>
|
||||
<footer class="review__header"><div class="review__header-content">Акылдаш · Frontend v0.1.2 · Backend v0.11.3 · Внутренняя лаборатория релевантности</div></footer>
|
||||
<script>
|
||||
document.body.classList.add('review__locked');
|
||||
const DRAFT_KEY = 'akyldash-search-review-draft';
|
||||
@@ -193,7 +193,7 @@
|
||||
let account = null;
|
||||
const reviewRoute = location.pathname === '/review';
|
||||
$('review').hidden = !reviewRoute; $('search-page').hidden = reviewRoute; $('review-nav').classList.toggle('is-active', reviewRoute); $('search-nav').classList.toggle('is-active', !reviewRoute);
|
||||
if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.2'; }
|
||||
if (!reviewRoute) { document.title = 'Поиск по правовой базе · Акылдаш'; document.querySelector('.review__header-content').hidden = true; $('tutorial-open').hidden = true; document.querySelector('.review__skip').href = '#public-results'; document.querySelector('footer .review__header-content').textContent = 'Акылдаш · Frontend v0.1.2 · Backend v0.11.3'; }
|
||||
async function apiFetch(path, options = {}) { const response = await fetch(path, {credentials: 'same-origin', ...options}); if (response.status === 401 && path !== '/auth/login') { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); } return response; }
|
||||
const showLogin = () => { document.body.classList.add('review__locked'); if (!$('login-dialog').open) $('login-dialog').showModal(); $('login-username').focus(); };
|
||||
const enterWorkspace = (user) => { account = user; $('signed-in-user').textContent = `${user.username} · ${user.role}`; $('reviewer').value = user.username; $('profile-username').textContent = user.username; $('profile-role').textContent = user.role; $('profile-created').textContent = new Date(user.created_at * 1000).toLocaleDateString('ru-RU'); $('users-open').hidden = user.username.toLowerCase() !== 'admin'; document.body.classList.remove('review__locked'); if (!reviewRoute) initSearchPage(); else if (!hasSeenTour()) setTimeout(openTutorial, 350); };
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import contextlib
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -36,6 +38,27 @@ class SearchApiTest(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ApiError, "within 10000 results"):
|
||||
api.handle("GET", "/search?q=x&page=100&page_size=100")
|
||||
|
||||
def test_review_secret_file_is_read_without_command_line_value(self):
|
||||
from search.api import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
secret_file = Path(temporary) / "secret"
|
||||
secret_file.write_text("secret-value\n", encoding="utf-8")
|
||||
with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), patch("search.api.Api") as api, patch("search.api.ThreadingHTTPServer"):
|
||||
main()
|
||||
self.assertEqual(api.call_args.args[4], b"secret-value")
|
||||
|
||||
def test_review_secret_file_rejects_empty_content(self):
|
||||
from search.api import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
secret_file = Path(temporary) / "secret"
|
||||
secret_file.touch()
|
||||
with patch("sys.argv", ["search.api", "--review-secret-file", str(secret_file)]), contextlib.redirect_stderr(io.StringIO()):
|
||||
with self.assertRaises(SystemExit) as result:
|
||||
main()
|
||||
self.assertEqual(result.exception.code, 2)
|
||||
|
||||
def test_production_api_uses_registration_intent_boosts(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
api = self.make_api(Path(temporary))
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# Absolute path on the production host containing minjust-normalized/.
|
||||
DATA_ROOT=/volume1/docker/akyldash/data
|
||||
BACKEND_PORT=8080
|
||||
DATA_ROOT=/var/lib/akyldash/data
|
||||
REVIEW_SECRET_FILE=/etc/akyldash/review_secret
|
||||
BACKEND_IMAGE=akyldash/backend:local
|
||||
OPENSEARCH_IMAGE=akyldash/opensearch:local
|
||||
SEARCH_INDEX=akyldash-fragments-current
|
||||
OPENSEARCH_MEM_LIMIT=4g
|
||||
OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
|
||||
# Generate with: openssl rand -hex 32
|
||||
REVIEW_SECRET=replace-with-a-random-secret
|
||||
COOKIE_SECURE=true
|
||||
@@ -0,0 +1,4 @@
|
||||
akyldash.kg {
|
||||
encode zstd gzip
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
+93
-43
@@ -1,62 +1,112 @@
|
||||
# Production deployment
|
||||
|
||||
This compose project runs the Search API and its private OpenSearch node. It
|
||||
binds the API only to `127.0.0.1`; publish it through an authenticated reverse
|
||||
proxy or VPN. OpenSearch is not published outside the compose network.
|
||||
Compose runs Caddy, the Search API, and a private OpenSearch node. Only Caddy
|
||||
publishes ports `80` and `443`; backend and OpenSearch have no host ports. The
|
||||
OpenSearch security plugin is disabled to match the current API client, so keep
|
||||
it on the private network.
|
||||
|
||||
The current compose intentionally disables the OpenSearch security plugin to
|
||||
match the existing API client. Keep both services on a private host/network.
|
||||
The application provides cookie-based accounts for its search API and review
|
||||
workspace; it does not replace host-level network controls or TLS.
|
||||
## Host access and configuration
|
||||
|
||||
## First deployment
|
||||
The production environment file is `/etc/akyldash/production.env`; the review
|
||||
secret is `/etc/akyldash/review_secret`. Both must stay outside the checkout,
|
||||
root-owned, and mode `0600`. `DATA_ROOT` must be an absolute path. The `codex`
|
||||
account is intentionally outside the `sudo` and `docker` groups because access
|
||||
to the Docker socket grants root-equivalent control. Run host operations with
|
||||
the narrowly required `sudo` command; do not use `docker` as `codex` directly.
|
||||
|
||||
1. Copy this directory to the host with the repository source.
|
||||
2. Copy `.env.example` to `.env`, set an absolute `DATA_ROOT`, and replace
|
||||
`REVIEW_SECRET` with a random value. Do not commit `.env`.
|
||||
3. Put the normalized dataset under `$DATA_ROOT/minjust-normalized/`.
|
||||
4. Check the rendered configuration:
|
||||
After reviewing the host audit and confirming these files do not already exist,
|
||||
the owner can create them from the approved checkout with
|
||||
`sudo bash infra/deploy/prepare-production-config.sh`. The script refuses to
|
||||
overwrite existing configuration, installs the tracked env example with mode
|
||||
`0600`, and generates a random review secret with mode `0600`. It does not
|
||||
start Compose or create volumes. Review and set `DATA_ROOT` in the protected
|
||||
env file before deployment.
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env -f compose.yaml config
|
||||
```
|
||||
Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus
|
||||
is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in
|
||||
[`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the
|
||||
root bootstrap installs packages and creates directories, but does not deploy
|
||||
the application or alter SSH, firewall, user groups, or sudoers. The restricted
|
||||
`akyldash-compose` helper allows only root-owned Compose `config` and `status`;
|
||||
service start/stop remains an owner-run root operation.
|
||||
|
||||
5. Start the services:
|
||||
The repository Gitea Actions runner is an isolated CI worker described in
|
||||
[`docs/operations/CI_RUNNER.md`](../../docs/operations/CI_RUNNER.md). It has no
|
||||
production Docker socket, production env files, or application checkout. CI
|
||||
does not deploy or change production; all deployment remains owner-run.
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env -f compose.yaml up -d --build
|
||||
docker compose --env-file .env -f compose.yaml ps
|
||||
curl -fsS http://127.0.0.1:${BACKEND_PORT:-8080}/review >/dev/null
|
||||
```
|
||||
## First deployment and upgrades
|
||||
|
||||
6. Load the versioned index and switch its alias only after the import and
|
||||
validation succeed. Back up `DATA_ROOT` and the `opensearch-data` volume
|
||||
before the first import.
|
||||
|
||||
## Initial administrator and accounts
|
||||
|
||||
After starting the backend for the first time, create the initial `admin`
|
||||
account with an interactive password prompt:
|
||||
Run commands from the root-owned checkout at `/opt/akyldash/current`. Render
|
||||
and review Compose before changing services:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env -f compose.yaml exec backend python -m search.api --create-admin
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml config --quiet
|
||||
```
|
||||
|
||||
The account database is stored at `DATA_ROOT/search-users.sqlite3`. Back it up
|
||||
alongside the review database. Sign in at `/review`; `admin` can create accounts
|
||||
with temporary passwords. Each new account must replace its temporary password
|
||||
at first sign-in. Only the `admin` login can create, reset, or delete accounts.
|
||||
On a clean first deployment, `infra/backup/archive.py create` reports that no
|
||||
backup is required when all persistent mounts are empty and no Compose
|
||||
containers exist. Continue without a backup in that case. If any persistent
|
||||
data exists, create and verify a backup before changing schema, images, or
|
||||
starting a new application version. This also applies when containers are
|
||||
absent, including an initial installation where the normalized corpus was
|
||||
preloaded under `DATA_ROOT`; create the archive before the first `up`. When
|
||||
named volumes do not exist, pass `--initial-deployment`; this is accepted only
|
||||
without Compose containers and when `DATA_ROOT` contains only
|
||||
`minjust-normalized/`. During upgrades, missing volumes are an error because
|
||||
they can indicate lost production data. The archive tool also refuses
|
||||
ambiguous storage state, running containers, and incomplete Compose service
|
||||
containers. See
|
||||
[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
|
||||
commands. Do not remove an older usable backup to make room; retain the
|
||||
pre-change archive until the new deployment and a newer verified backup are
|
||||
confirmed.
|
||||
|
||||
If the `admin` password is lost, reset it in an interactive terminal with:
|
||||
For an initial clean installation, start the stack only after config review:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env -f compose.yaml exec backend python -m search.api --reset-admin-password
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d --build
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml ps
|
||||
curl -fsS https://akyldash.kg/review >/dev/null
|
||||
```
|
||||
|
||||
The next sign-in requires another password change. Serve the backend behind
|
||||
HTTPS, rate-limit login requests at the reverse proxy, and keep
|
||||
`COOKIE_SECURE=true` in production.
|
||||
Before this step, confirm DNS and have the owner review firewall rules for TCP
|
||||
`80` and `443`. Caddy manages the origin certificate; confirm HTTPS before
|
||||
setting Cloudflare to Full (strict). On the initial start, create the `admin`
|
||||
account interactively:
|
||||
|
||||
This is a deployment baseline, not a public internet exposure recipe. TLS,
|
||||
backups, monitoring, and a production OpenSearch security configuration must
|
||||
be provided by the host reverse proxy/operations setup.
|
||||
```bash
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --create-admin
|
||||
```
|
||||
|
||||
The account database is `$DATA_ROOT/search-users.sqlite3` and is included in
|
||||
the `data` backup. Admin password reset is also interactive:
|
||||
|
||||
```bash
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml exec backend python -m search.api --reset-admin-password
|
||||
```
|
||||
|
||||
After the initial OpenSearch index is loaded and validated, the `admin` account
|
||||
is created, and `/review` works over HTTPS, create and verify the required
|
||||
post-setup baseline backup before treating deployment as complete. It captures
|
||||
the initialized index, account database, and Caddy certificate data. Follow
|
||||
the exact stop/create/verify/start sequence in
|
||||
[`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md#initial-post-setup-baseline).
|
||||
|
||||
## Backup, restore, and rollback
|
||||
|
||||
Backups are cold and require downtime. Stop services, create the archive, then
|
||||
verify both its checksum and contents before proceeding. Do not run an upgrade
|
||||
or schema change until verification succeeds. Keep archives root-only under
|
||||
`/var/backups/akyldash` and copy them to separately managed encrypted storage.
|
||||
|
||||
Restore is a staged operation on a marked replacement host; it does not
|
||||
overwrite production. For application-only rollback, use the guarded helper
|
||||
with a verified pre-release backup and explicit data-compatibility review.
|
||||
Follow the procedures in [`BACKUP.md`](../../docs/operations/BACKUP.md),
|
||||
[`ROLLBACK.md`](../../docs/operations/ROLLBACK.md), and
|
||||
[`RECOVERY.md`](../../docs/operations/RECOVERY.md).
|
||||
|
||||
Automated production deployment is not enabled. Deployments remain owner-run;
|
||||
record the release and verify service health before considering it complete.
|
||||
@@ -1,9 +1,43 @@
|
||||
name: akyldash
|
||||
|
||||
services:
|
||||
caddy:
|
||||
image: caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- caddy-data:/data
|
||||
- caddy-config:/config
|
||||
networks:
|
||||
- proxy
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
opensearch:
|
||||
image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: deploy/production/Dockerfile.opensearch
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
bootstrap.memory_lock: "true"
|
||||
@@ -21,6 +55,8 @@ services:
|
||||
hard: 65536
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
networks:
|
||||
- search
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9200/_cluster/health || exit 1"]
|
||||
interval: 30s
|
||||
@@ -28,12 +64,17 @@ services:
|
||||
retries: 10
|
||||
|
||||
backend:
|
||||
image: ${BACKEND_IMAGE:-akyldash/backend:local}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: deploy/production/Dockerfile.backend
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: local
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "3"
|
||||
environment:
|
||||
REVIEW_SECRET: ${REVIEW_SECRET:?set REVIEW_SECRET in .env}
|
||||
COOKIE_SECURE: ${COOKIE_SECURE:-true}
|
||||
command:
|
||||
- python
|
||||
@@ -53,15 +94,31 @@ services:
|
||||
- /app/data/search-reviews.sqlite3
|
||||
- --users-db
|
||||
- /app/data/search-users.sqlite3
|
||||
- --review-secret
|
||||
- ${REVIEW_SECRET}
|
||||
ports:
|
||||
- "127.0.0.1:${BACKEND_PORT:-8080}:8080"
|
||||
- --review-secret-file
|
||||
- /run/secrets/review_secret
|
||||
secrets:
|
||||
- review_secret
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ${DATA_ROOT:?set DATA_ROOT in .env}:/app/data
|
||||
networks:
|
||||
- proxy
|
||||
- search
|
||||
|
||||
volumes:
|
||||
caddy-data:
|
||||
caddy-config:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
search:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
review_secret:
|
||||
file: ${REVIEW_SECRET_FILE:?set REVIEW_SECRET_FILE in .env}
|
||||
+5
-1
@@ -26,6 +26,10 @@
|
||||
|
||||
- [Текущий статус](operations/project-status.md) — выполненные шаги, риски и
|
||||
ближайшие действия.
|
||||
- [Состояние инфраструктуры](operations/CURRENT_STATE.md) — подтверждённое
|
||||
состояние production и оставшиеся проверки.
|
||||
- [Инфраструктура и эксплуатация](operations/INFRASTRUCTURE.md) — схема,
|
||||
доступы, deployment, backup, rollback и recovery.
|
||||
- [План Telegram-пространства](operations/telegram-workspace-plan.md) — темы,
|
||||
роли и этапы развития командного окружения.
|
||||
|
||||
@@ -41,4 +45,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -0,0 +1,84 @@
|
||||
# Infrastructure access
|
||||
|
||||
## Discover existing access first
|
||||
|
||||
In a new Codex session, inspect the SSH alias and key file metadata without
|
||||
printing key contents, then check the connection:
|
||||
|
||||
```bash
|
||||
ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }'
|
||||
test -r ~/.ssh/akyldash_codex && echo 'SSH key is available'
|
||||
ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps id
|
||||
ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps hostname -f
|
||||
gitea-api GET /version
|
||||
gitea-api GET /repos/admin/akyldash
|
||||
```
|
||||
|
||||
Do not ask for the host, login, password, token, or key again before these
|
||||
checks. Never print a private key, token, or complete environment file.
|
||||
|
||||
## VPS
|
||||
|
||||
- SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`.
|
||||
- SSH configuration: `~/.ssh/config`.
|
||||
- Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents).
|
||||
- `codex` can run unprivileged read-only diagnostics. It has no sudo or Docker
|
||||
group membership; Docker CLI access to the daemon is denied. Do not add it to
|
||||
either group.
|
||||
- Bootstrap is complete. Docker and Compose are active, but no root-owned
|
||||
repository checkout or restricted helper is installed yet. The owner can run
|
||||
`infra/bootstrap/audit-host.sh` from the reviewed root-owned checkout to
|
||||
report Docker containers/volumes, protected-file metadata, UFW rules, ports,
|
||||
and ISPmanager status without showing secret contents or changing state.
|
||||
- Host package/service configuration, firewall changes, protected env/secret
|
||||
creation, backup creation, recovery apply, and production deployment require
|
||||
a reviewed owner/root command. `infra/deploy/install-helper.sh` grants the
|
||||
fixed verifier, read-only Compose `config`/`status`, and guarded rollback
|
||||
commands. It does not grant general sudo or Docker access; it is not
|
||||
installed on the VPS.
|
||||
- `infra/deploy/prepare-production-config.sh` is an owner-run root operation.
|
||||
It refuses to overwrite existing files, copies only the tracked env example,
|
||||
generates a random review secret, and writes both files with mode `0600`.
|
||||
It does not start Compose or create volumes.
|
||||
- Never connect to other VPS hosts or Synology as part of this deployment.
|
||||
|
||||
This execution environment is WSL2. The repository is on the Windows-mounted
|
||||
`/mnt/d` filesystem, while SSH config and the identity file are in the Linux
|
||||
home directory. Do not assume Windows SSH credentials are available or copy
|
||||
keys between environments. Plain `ssh akyldash_vps` failed because the system
|
||||
SSH config include had unsafe permissions. Use `ssh -F ~/.ssh/config ...`; in a
|
||||
network-restricted session this read-only command may also require approved
|
||||
execution. If the key is missing in a future environment, report that
|
||||
limitation and do not retrieve it elsewhere.
|
||||
|
||||
## Gitea
|
||||
|
||||
- Repository: `https://git.vakanaut.ru/admin/akyldash`.
|
||||
- Use the installed `gitea-api` helper; it reads credentials from the
|
||||
environment. Never inspect local credential files, print environment values,
|
||||
or construct HTTP authorization headers.
|
||||
- The audited API token can read/write this repository and create issues, but
|
||||
does not have `read:user` or `read:package`. Listing repository Actions
|
||||
runners returns `403` because the API principal is not the repository owner.
|
||||
An API `403` caused by missing scope/ownership is not evidence that no runner
|
||||
exists.
|
||||
- Protected `main` requires one approval. Do not push directly to `main`, merge
|
||||
automatically, or start a review. Work on a feature branch and use a PR.
|
||||
- Gitea Actions and package features are enabled for the repository. The Actions
|
||||
task API currently returns no runs, but runner availability and package
|
||||
publishing permissions have not been confirmed. The repository-level runner
|
||||
is prepared by `infra/runner/install-runner.sh`, but is not installed or
|
||||
registered yet. Only a repository owner may obtain its registration token in
|
||||
`admin/akyldash` Settings → Actions → Runners; do not use organization or
|
||||
instance tokens and never put the token in Git, a command argument, or a
|
||||
report. See [CI_RUNNER.md](CI_RUNNER.md).
|
||||
|
||||
## Who can do what
|
||||
|
||||
| Role | Operations |
|
||||
|---|---|
|
||||
| `codex` over SSH | Unprivileged diagnostics; Docker daemon, UFW rules, and protected paths require the root audit script |
|
||||
| Restricted helpers | Read-only drift check and Compose `config`/`status`, plus guarded rollback; not installed on the audited VPS |
|
||||
| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS |
|
||||
| VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes |
|
||||
| Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval |
|
||||
@@ -0,0 +1,131 @@
|
||||
# Backup and restore
|
||||
|
||||
## Current status
|
||||
|
||||
No production data or backup existed during the 2026-10-11 audit. The repository
|
||||
now contains `infra/backup/archive.py`; it is not installed or run on the VPS,
|
||||
and `/var/backups/akyldash` remains a planned root-only location.
|
||||
|
||||
## Required release policy
|
||||
|
||||
- Create a backup before every production change, including any future schema
|
||||
or index migration.
|
||||
- Include the normalized corpus, `search-users.sqlite3`,
|
||||
`search-reviews.sqlite3`, the OpenSearch index/snapshot, deployment digest,
|
||||
Caddy's certificate data, and required secret file in the protected backup
|
||||
set. Store secrets and Caddy private keys with equivalent access controls.
|
||||
- Verify archive readability, checksums, and SQLite integrity before relying
|
||||
on it. Never overwrite or delete the only usable backup.
|
||||
- Keep one current pre-release backup. If deployment fails, retain the backup
|
||||
required to restore the previous state.
|
||||
- Test restore into an isolated target before relying on the backup. Do not
|
||||
restore over production as a test.
|
||||
|
||||
## Clean first deployment
|
||||
|
||||
No backup is needed only when Compose has no containers and all four
|
||||
persistent mounts are confirmed empty. For a first install with a preloaded
|
||||
corpus and not-yet-created named volumes, use `--initial-deployment` to create
|
||||
the pre-start backup. This flag is accepted only with no Compose containers and
|
||||
when `DATA_ROOT` contains exactly `minjust-normalized/`; it archives absent
|
||||
named volumes as empty components. In every normal backup, an absent volume is
|
||||
an error because it could mean that a previous installation lost data.
|
||||
Missing mount definitions, inaccessible Docker state, or partial service
|
||||
containers are also errors and must stop deployment. Do not create placeholder
|
||||
containers to make backup pass.
|
||||
|
||||
For the corpus-only initial case, run this before the first `up` from the
|
||||
approved checkout. Replace the timestamp with one real UTC timestamp:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--initial-deployment \
|
||||
--output /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-corpus-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
```
|
||||
|
||||
## Cold backup procedure
|
||||
|
||||
The script requires every Compose container to be stopped. Schedule downtime,
|
||||
then run from the approved checkout on the VPS:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--output /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
```
|
||||
|
||||
Replace the timestamp placeholder with one real UTC timestamp before running.
|
||||
On a clean first deployment, `create` exits with a clear message that backup
|
||||
can be skipped. With existing data and all expected volumes present, it
|
||||
archives the storage even when no Compose containers exist. A preloaded corpus
|
||||
with no named volumes requires the explicit `--initial-deployment` mode;
|
||||
otherwise missing volumes are rejected. Partial service containers or running
|
||||
services are rejected as unsafe. The create command archives the application data,
|
||||
OpenSearch volume, Caddy data/config volumes, production environment file, and
|
||||
review secret. It records the checked-out Git revision, deployment-file hashes,
|
||||
and image references/IDs for each stopped Compose service, writes a SHA-256
|
||||
sidecar, and refuses to overwrite an existing archive. It also runs SQLite
|
||||
`quick_check`. Start the stack again only after verification succeeds:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
|
||||
```
|
||||
|
||||
## Initial post-setup baseline
|
||||
|
||||
After the first index has been loaded and validated, the initial `admin` account
|
||||
has been created, and `/review` responds over HTTPS, create a second backup.
|
||||
This one captures the initialized index, account database, and newly created
|
||||
Caddy/OpenSearch volumes. Stop all services, create and verify the archive,
|
||||
then start the stack again:
|
||||
|
||||
```bash
|
||||
cd /opt/akyldash/current
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml stop
|
||||
sudo python3 infra/backup/archive.py create \
|
||||
--compose-file deploy/production/compose.yaml \
|
||||
--env-file /etc/akyldash/production.env \
|
||||
--output /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-initial-baseline-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production/compose.yaml up -d
|
||||
```
|
||||
|
||||
Use one real UTC timestamp in both archive paths. Do not treat initial
|
||||
production setup as complete until this baseline archive verifies. Keep the
|
||||
pre-start corpus archive as well until the baseline is verified and copied to
|
||||
separately managed encrypted storage.
|
||||
|
||||
Keep the local archive root-only, then copy it to separately managed encrypted
|
||||
storage. A local archive alone does not protect against loss of the VPS disk.
|
||||
Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or
|
||||
`/var/tmp` and validates the checksum and SQLite databases. Applying the staged
|
||||
contents requires the separate recovery-target tool; it refuses the inventoried
|
||||
production host, existing data/volumes, and an unmarked host. Example isolated
|
||||
restore:
|
||||
|
||||
```bash
|
||||
sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \
|
||||
--target-dir /var/tmp/akyldash-restore-test
|
||||
```
|
||||
|
||||
This validates and stages the archive. On the replacement host only, after
|
||||
running bootstrap with `--recovery-target` and checking out the matching Git
|
||||
revision, apply the staged data:
|
||||
|
||||
```bash
|
||||
sudo python3 infra/backup/apply_restore.py \
|
||||
--staging-dir /var/tmp/akyldash-restore-test
|
||||
```
|
||||
|
||||
The apply command restores data and named volumes but leaves services stopped.
|
||||
Test the full procedure on a separate recovery host before relying on it. Do
|
||||
not run an unreviewed restore over existing data.
|
||||
@@ -0,0 +1,178 @@
|
||||
# AkylDash Gitea Actions runner
|
||||
|
||||
## Scope and isolation
|
||||
|
||||
The deployment follows the official [Gitea 1.25 act_runner guide](https://docs.gitea.com/1.25/usage/actions/act-runner/)
|
||||
for repository-scoped registration. Gitea warns that access to a Docker daemon
|
||||
is root-equivalent; the dedicated rootless daemon is used instead of the
|
||||
production daemon. The pinned binary and checksum are published in the
|
||||
[official Gitea download directory](https://dl.gitea.com/gitea-runner/0.2.13/).
|
||||
|
||||
The runner is reserved to repository `admin/akyldash`, using the token under
|
||||
`https://git.vakanaut.ru/admin/akyldash/settings/actions/runners`. Do not use
|
||||
organization or instance runner settings. Pin the official `act_runner`
|
||||
`0.2.13` x86_64 binary from `dl.gitea.com`; the installer verifies its SHA-256
|
||||
before placing it on the host.
|
||||
|
||||
CI jobs run in `docker.gitea.com/runner-images:ubuntu-24.04` containers with
|
||||
capacity 1, per-job limits of 2 CPU, 3 GiB RAM and 512 processes, and a 15 GiB
|
||||
dedicated ext4 loop-backed Docker data filesystem. The runner account's home
|
||||
directory is root-owned mode `0711`, so CI cannot rename the image or mountpoint
|
||||
entries. Docker config remains under `.config`; registration state and the
|
||||
runner workspace live in `.config/act-runner`, which is runner-owned. The
|
||||
rootless Docker setup and user service use `.config/act-runner` as writable
|
||||
`HOME`, `.config` as `XDG_CONFIG_HOME`, and
|
||||
`.config/act-runner/.local/share` as `XDG_DATA_HOME`. This keeps Docker config
|
||||
and its systemd user unit in the existing `.config` tree without granting
|
||||
write access to the protected home.
|
||||
rootless Docker daemon
|
||||
and runner user are separate from production Docker; `codex` is not added to
|
||||
sudo or Docker groups. Jobs do not receive the rootless Docker socket, the
|
||||
production Docker socket, production environment/secrets, or arbitrary host
|
||||
bind mounts. The runner account has no access to `/etc/akyldash` or the
|
||||
root-owned application checkout. These are host permission boundaries, not a
|
||||
claim that untrusted project code is safe; review workflow changes and actions
|
||||
before merging them.
|
||||
|
||||
The runner service restarts on failure and boot. Systemd caps the runner
|
||||
account's shared slice to 2.5 CPU, 4 GiB RAM and 2048 tasks; the rootless
|
||||
daemon is separately capped to 2 CPU, 4 GiB and 1024 tasks, while each job has
|
||||
the limits above. Runner logs go to a rotated file (10 MiB, 7 copies). A
|
||||
systemd timer checks the runner service and the rootless Docker API each minute;
|
||||
on failure it restarts the fixed runner service. No UFW changes or public
|
||||
metrics ports are needed. Runner cache is disabled.
|
||||
|
||||
The label `akyldash-ci` is used by `runner-smoke.yaml` on pull requests, pushes
|
||||
to `main`, and manual dispatch. Existing CI image-build work remains on its current
|
||||
`ubuntu-latest` target until it has been tested separately against the
|
||||
isolated runner.
|
||||
Production deployment is not automated.
|
||||
|
||||
## Installation and registration
|
||||
|
||||
Review the root script and host audit output before running it. It installs
|
||||
`uidmap`, `dbus-user-session`, `curl`, `logrotate`, and Docker's rootless extras;
|
||||
creates the dedicated system user and selects the first free, shared 65536-ID
|
||||
range from `/etc/subuid` and `/etc/subgid`; reserves 15 GiB
|
||||
for runner images/workspaces; and configures systemd services. It starts only
|
||||
the separate rootless Docker daemon; the runner and health timer stay disabled
|
||||
until repository registration. It does not change
|
||||
SSH/UFW, production Docker configuration, application data, or production
|
||||
secrets. It can be rerun after partial setup: it validates and reuses an
|
||||
existing runner account, subordinate ID mappings, image, and mount unit. An
|
||||
existing image must be exactly 15 GiB and contain ext4; a mismatch stops the
|
||||
installer without resizing or formatting it. The runner account must retain
|
||||
a non-root system UID below `UID_MIN`, its expected home, primary group, and
|
||||
sole supplementary group. After validating the account, the installer locks
|
||||
its home directory to root ownership and mode `0711`; it refuses to rerun while
|
||||
the runner service is active, so it cannot interrupt a job. The mount
|
||||
is started only when it is not already active, and an active mount must be the
|
||||
expected ext4 loop device backed by the runner image with both `nosuid` and
|
||||
`nodev` active. Runner-home directories and user configuration files are
|
||||
created in runner-owned subdirectories. Root creates only the first `.config`
|
||||
entry beneath the already locked home; subsequent directory and config writes
|
||||
run as `akyldash-runner`. The installer rejects static symlinks and validates
|
||||
ownership and mode after setup. This also normalizes the `0755` intermediate
|
||||
directories left by the first installer version. Conflicting files or state
|
||||
stop the installer for manual inspection. A new image requires at least 25 GiB
|
||||
free on `/`. Image creation opens each parent component without following
|
||||
symlinks, creates the image exclusively relative to the opened directory, and
|
||||
passes the open file descriptor to `mkfs.ext4`; validation checks the same inode
|
||||
and rejects path replacement. Systemd mounts the image under the root-owned home
|
||||
entry, which the runner cannot replace after validation.
|
||||
|
||||
If the reviewed checkout is not present at the inventoried path, first verify
|
||||
that the target is absent and clone the reviewed `main` there. Do not overwrite
|
||||
an existing checkout.
|
||||
|
||||
```bash
|
||||
sudo test ! -e /opt/akyldash/current
|
||||
sudo git clone --branch main \
|
||||
--single-branch https://git.vakanaut.ru/admin/akyldash.git \
|
||||
/opt/akyldash/current
|
||||
```
|
||||
|
||||
From that root-owned reviewed checkout on the VPS:
|
||||
|
||||
```bash
|
||||
sudo bash /opt/akyldash/current/infra/runner/install-runner.sh
|
||||
```
|
||||
|
||||
Then, as a repository owner, open repository **Settings → Actions → Runners**,
|
||||
create a repository-level runner, and copy its registration token locally. Do
|
||||
not send the token in chat or save it in the checkout. Register interactively
|
||||
in a terminal so it is not placed in shell history or process arguments:
|
||||
|
||||
```bash
|
||||
sudo -u akyldash-runner env HOME=/var/lib/akyldash-runner/.config/act-runner \
|
||||
/usr/local/bin/act_runner \
|
||||
--config /etc/akyldash-runner/config.yaml register
|
||||
```
|
||||
|
||||
Enter `https://git.vakanaut.ru/`, the token, runner name `akyldash-ci-vps-01`,
|
||||
and label `akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04`
|
||||
when prompted. The config sets the label and capacity; verify them in the
|
||||
repository runner settings. The token is saved only in the mode-0700 runner
|
||||
runtime directory, outside Git. Start/enable the runner after registration:
|
||||
|
||||
```bash
|
||||
sudo systemctl enable --now akyldash-act-runner.service
|
||||
sudo systemctl enable --now akyldash-runner-healthcheck.timer
|
||||
sudo systemctl status akyldash-act-runner.service
|
||||
sudo systemctl status akyldash-runner-healthcheck.timer
|
||||
```
|
||||
|
||||
The interactive command's terminal behavior should be confirmed before pasting
|
||||
the token. If it echoes input, stop and use the repository settings to revoke
|
||||
that token; do not put it in command-line arguments or environment variables.
|
||||
|
||||
## Validation
|
||||
|
||||
1. Confirm the runner is **Online** in the repository runner settings, with the
|
||||
expected name and `akyldash-ci` label.
|
||||
2. Confirm **Runner smoke test** runs for the open PR (or dispatch it manually).
|
||||
It checks that production
|
||||
sockets, files, environment secrets, and writable application paths are not
|
||||
present in the job container.
|
||||
3. Confirm the run reports the expected runner, succeeds, and the runner
|
||||
service and health timer remain active.
|
||||
4. Only after the repository owner reviews this boundary should CI jobs be moved
|
||||
to `akyldash-ci`. Do not add a production deploy workflow or secrets to this
|
||||
runner until a separate restricted deploy mechanism is reviewed.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Update `runner_version` and its expected official SHA-256 together in
|
||||
`infra/runner/install-runner.sh`, review upstream release compatibility with
|
||||
the installed Gitea, and open a PR. Do not replace the binary while a job is
|
||||
active. Stop the runner, install/verify the new binary, then start it and
|
||||
confirm Online plus a smoke run. Preserve
|
||||
`/var/lib/akyldash-runner/.config/act-runner/.runner`.
|
||||
|
||||
## Recovery and rollback
|
||||
|
||||
The runner is disposable: production data and secrets are not on its filesystem.
|
||||
For a runner process failure, inspect `systemctl status` and
|
||||
`journalctl -u akyldash-act-runner.service`; restart the fixed runner service.
|
||||
If installation stops after creating or enabling the mount unit, inspect the
|
||||
image and mount with `stat`, `blkid`, `findmnt`, and `losetup`, then rerun the
|
||||
reviewed installer only when the image is 15 GiB ext4 and the active mount is
|
||||
backed by it. The installer recognizes that state and does not format or
|
||||
restart the mounted filesystem. If size, filesystem, unit contents, or backing
|
||||
device differ, stop and resolve the state manually; never format the existing
|
||||
image as a retry.
|
||||
For lost/corrupt registration state, stop the service and obtain a fresh
|
||||
repository-level token from Settings → Actions → Runners. Preserve the 15 GiB
|
||||
data image and state for diagnosis; re-register only after the repository
|
||||
owner confirms/revokes the old runner in Gitea. The images/cache can be rebuilt
|
||||
from the workflow. Never format or remove an existing runner store as an
|
||||
automatic recovery step.
|
||||
|
||||
## Status
|
||||
|
||||
The root installer and registration are incomplete. The owner reports that the
|
||||
15 GiB ext4 loop-backed mount is active on `/dev/loop0` after manually running
|
||||
`systemctl daemon-reload`, `enable`, and `start` when the first installer run
|
||||
stopped at mount activation. This PR has not changed the VPS and does not claim
|
||||
that remaining runner components are installed, a repository token is
|
||||
available, the runner is Online, or a workflow has run.
|
||||
@@ -0,0 +1,152 @@
|
||||
# Infrastructure current state
|
||||
|
||||
Last audit: 2026-10-12. Source: read-only SSH session as `codex`, owner-reported
|
||||
UFW state, local repository inspection, and Gitea API. Root-only Docker state,
|
||||
UFW rule details, and protected file metadata remain unverified.
|
||||
|
||||
## VPS facts
|
||||
|
||||
- Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`.
|
||||
- OS: Ubuntu Server `24.04.5 LTS`; kernel `6.8.0-142-generic` from the prior
|
||||
inventory audit.
|
||||
- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 92 GiB free
|
||||
at this audit.
|
||||
- Network: interface `ens3`, IPv4 `206.223.249.118/32`.
|
||||
- `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires
|
||||
a password. Docker CLI cannot access `/var/run/docker.sock` as `codex`.
|
||||
- SSH service is running. The inspected sshd configuration has public-key and
|
||||
password authentication enabled and `PermitRootLogin yes`; no SSH changes
|
||||
were made.
|
||||
- Bootstrap completed. Docker CE/CLI `29.9.0`, Compose `5.6.0`, containerd
|
||||
`2.4.1`, and runc `1.5.2` are installed; package versions and active Docker
|
||||
service were independently checked. The Docker daemon API remains root-only.
|
||||
- `/etc/docker/daemon.json` has the expected `local` log driver and `10m` / `3`
|
||||
rotation limits. Bootstrap paths exist with root ownership: `/opt/akyldash`
|
||||
mode `0755`, `/etc/akyldash` and `/var/backups/akyldash` mode `0700`, and
|
||||
`/var/lib/akyldash/data` mode `0750`.
|
||||
- `/opt/akyldash/current`, verifier/rollback binaries, and the sudoers file are
|
||||
absent. `codex` has not received sudo or Docker-group access. Metadata for
|
||||
`/etc/akyldash/production.env` and `/etc/akyldash/review_secret` could not be
|
||||
checked because `/etc/akyldash` is root-only.
|
||||
- TCP listeners observed: SSH `22` and loopback DNS only; no listener on `1500`,
|
||||
`80`, or `443`. No ISPmanager package or service unit was found through the
|
||||
unprivileged checks; root can confirm with the audit script below.
|
||||
- Owner reports UFW is active and allows TCP `22`, `80`, `443`, and `1500`.
|
||||
Rule details were not independently read; no firewall changes were made.
|
||||
- Additional read-only runner checks on 2026-10-12 confirmed `/` is ext4 with
|
||||
about 92 GiB free at the prior audit, `/usr/bin/dockerd-rootless.sh` exists,
|
||||
and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS,
|
||||
`https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner
|
||||
registration and Gitea Actions protocol connectivity are not confirmed.
|
||||
- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS,
|
||||
HTTP, HTTPS, and `/review` still return Cloudflare `521`; origin TLS and
|
||||
Cloudflare Full (strict) are not verified.
|
||||
|
||||
## Application and Gitea facts
|
||||
|
||||
- Repository: `admin/akyldash`, default branch `main`; Gitea version `1.25.5`.
|
||||
- The repository API reports push permission, Actions and packages enabled, and
|
||||
PRs enabled. `main` is protected and requires one approval; the current API
|
||||
principal cannot push or merge directly to `main`.
|
||||
- The repository runner-list API returns `403` because this API principal is
|
||||
not the repository owner. The Actions task API currently reports no workflow
|
||||
runs; this does not establish runner availability. Runner inventory and
|
||||
package publishing permissions remain unverified.
|
||||
- No public application listener is observed on TCP `80`/`443`. Docker
|
||||
container/volume state still requires the root audit. Repository Compose describes
|
||||
Caddy `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite
|
||||
account/review data, and OpenSearch `3.7.0` with `analysis-icu`.
|
||||
- The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a
|
||||
proxy network; OpenSearch and backend share a separate internal network, and
|
||||
backend has no host port. Caddy certificate storage, OpenSearch, and
|
||||
application data use persistent volumes/bind mounts.
|
||||
- The root-owned repository checkout, production env/secret files, Compose
|
||||
volumes/containers, backup, verifier/rollback helpers, and application are
|
||||
not confirmed installed. TLS and registry digest deployment for
|
||||
backend/OpenSearch are not configured.
|
||||
- The repository has a cold-backup archive/verification tool and a one-shot
|
||||
restore apply tool that requires a marked replacement host. Backup/restore
|
||||
tests use synthetic data; neither tool has run against production. The
|
||||
archive preflight permits a clean first deployment to skip backup, requires
|
||||
backup for existing persistent data even without containers, allows absent
|
||||
named volumes only in explicit corpus-only initial-deployment mode, and
|
||||
rejects inaccessible/ambiguous mounts and incomplete/running service
|
||||
containers. Initial deployment docs require a second verified backup after
|
||||
index and admin initialization.
|
||||
- Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
|
||||
inventoried production hostname/IP; its host-identity guard has shell tests.
|
||||
The mode has not been run on a VPS.
|
||||
- The repository defines a root-owned read-only verifier, Compose `config` and
|
||||
`status` helper, guarded rollback helper, and installer with exact sudoers
|
||||
command rules. None is installed on the VPS. CI contains no production
|
||||
deployment job; starting/stopping services remains owner-run.
|
||||
- A repository-scoped official `act_runner` `0.2.13` installation is prepared
|
||||
in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB ext4
|
||||
loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU,
|
||||
3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048
|
||||
tasks), rotated file logs, and a systemd health timer. Jobs use the
|
||||
`akyldash-ci` label and do not receive the rootless or production Docker
|
||||
socket. The owner reports a partial runner installation: the 15 GiB ext4
|
||||
loop-backed store is active on `/dev/loop0` after manually enabling and
|
||||
starting its mount unit when the installer stopped. Remaining components,
|
||||
repository registration, and Online state are unverified; this PR makes no
|
||||
VPS changes. The reviewed installer locks the runner home to root ownership
|
||||
(`0711`) so CI cannot replace the image or mountpoint, while writable config
|
||||
and runtime state remain in runner-owned subdirectories. It reuses legacy
|
||||
`0755` config directories, creates the image through a pinned file descriptor,
|
||||
rejects path replacement, and requires mount options `nosuid,nodev`.
|
||||
The reported installation blocker is that rootless setup receives this
|
||||
protected directory as `HOME`; the installer assigns writable runner-owned
|
||||
HOME/XDG paths, persists them in the `docker.service` user-unit drop-in, and
|
||||
restarts an active daemon and checks the running process environment. Local
|
||||
runner guards pass;
|
||||
the change has not been run on the VPS.
|
||||
- Gitea repository runner inventory could not be read by the current API
|
||||
principal (403). The owner has not supplied runner state, and no runner
|
||||
Online status or runner-hosted workflow is claimed. The manual
|
||||
`runner-smoke.yaml` workflow is prepared but has not run.
|
||||
- An isolated local Compose smoke test built backend/OpenSearch images,
|
||||
reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP
|
||||
200 for `/review` through Caddy. Its containers, networks, and volumes were
|
||||
removed afterward. This does not verify VPS startup, DNS, or public TLS.
|
||||
- The app has no migration framework or SMTP setup. Current app versions:
|
||||
backend `0.11.3`, frontend `0.1.2`, Telegram bot `0.2.2` (the bot is not
|
||||
deployed on this VPS).
|
||||
|
||||
## Checks not completed
|
||||
|
||||
- Docker Engine version from the server API, container and volume inventory,
|
||||
protected env/secret metadata, sudoers, and detailed UFW rules require root.
|
||||
`infra/bootstrap/audit-host.sh` is provided for an owner-run read-only audit.
|
||||
- The backup archive/restore tool was tested with synthetic local data; a backup
|
||||
and restore of real production data remain unverified because production is
|
||||
not deployed.
|
||||
- Cloudflare edge response was checked; origin DNS target, origin certificate,
|
||||
Full (strict), runner registration/Online state, registry publishing, and production
|
||||
deployment have not been verified.
|
||||
- This audit made no host changes. No package or service was installed by this
|
||||
audit; the owner performed the initial bootstrap beforehand.
|
||||
|
||||
## Next owner actions
|
||||
|
||||
1. Install the approved root-owned checkout after its PR is reviewed, then run
|
||||
`sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh` and share
|
||||
the output. The script prints no secret values and does not mutate the host.
|
||||
2. After reviewing the audit output and confirming the target files are absent,
|
||||
run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`;
|
||||
it refuses to overwrite existing files. Set the approved `DATA_ROOT` and
|
||||
place the approved corpus/index and Cloudflare origin configuration. Do not
|
||||
deploy until data/service state and the required backup path are reviewed.
|
||||
3. Run `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh` after
|
||||
reviewing the checkout; it grants only fixed verification, Compose
|
||||
`config`/`status`, and guarded rollback commands.
|
||||
4. After PR review and root audit, run
|
||||
`sudo bash /opt/akyldash/current/infra/runner/install-runner.sh`. In the
|
||||
repository Settings → Actions → Runners create a repository-level runner;
|
||||
enter the token only at the interactive registration prompt as documented
|
||||
in [CI_RUNNER.md](CI_RUNNER.md). Then enable the runner and health timer,
|
||||
confirm **Online**, and dispatch the smoke workflow. Production deployment
|
||||
remains manual and has no CI job.
|
||||
5. Test backup/restore and rollback only with synthetic data in an isolated
|
||||
environment before any production deployment.
|
||||
@@ -0,0 +1,10 @@
|
||||
# Infrastructure decisions
|
||||
|
||||
| Date | Decision | Reason and alternatives | Status / limits |
|
||||
|---|---|---|---|
|
||||
| 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed |
|
||||
| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only a fixed read-only verifier and root-owned rollback helper after checkout ownership and sudoers validation | Helpers are defined in Git; not installed on VPS |
|
||||
| 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed |
|
||||
| 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) |
|
||||
| 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed |
|
||||
| 2026-10-11 | Do not enable production deployment from Gitea Actions yet | Runner, package publishing scope, deploy-helper, backup/restore, and production health are not verified | Revisit after controlled end-to-end test |
|
||||
@@ -0,0 +1,83 @@
|
||||
# Deployment
|
||||
|
||||
## Current status
|
||||
|
||||
Production deployment is not enabled or tested on the VPS. The owner completed
|
||||
the root bootstrap: Docker/Compose are active and target directories exist.
|
||||
The `codex` account cannot access the Docker socket. No root-owned checkout or
|
||||
restricted helpers are installed; protected env/secret metadata and Docker
|
||||
container/volume state remain unverified because they require root.
|
||||
The unprivileged audit sees no listener on `80`, `443`, or `1500`; Cloudflare
|
||||
still returns `521`. Do not deploy until the owner has reviewed root-only host
|
||||
state, supplied protected configuration and the approved corpus/index, and
|
||||
backup/restore have been tested.
|
||||
|
||||
## First host preparation
|
||||
|
||||
1. The initial root bootstrap is complete. After reviewing and merging the
|
||||
infrastructure PR, the owner must install that approved `main` checkout at
|
||||
`/opt/akyldash/current` (the path was absent during the unprivileged audit):
|
||||
|
||||
```bash
|
||||
sudo git clone --branch main --single-branch https://git.vakanaut.ru/admin/akyldash.git /opt/akyldash/current
|
||||
```
|
||||
|
||||
The checkout is root-owned by default. Then run
|
||||
`sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh`. This
|
||||
read-only audit reports Docker state, protected-file metadata, UFW rules,
|
||||
listeners, and ISPmanager without printing secret contents. Resolve any
|
||||
existing containers, volumes, or data before proceeding; do not overwrite
|
||||
or delete them.
|
||||
2. After confirming the existing env and secret do not exist and reviewing the
|
||||
tracked defaults, run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`.
|
||||
It creates `/etc/akyldash/production.env` from the tracked example and
|
||||
generates `/etc/akyldash/review_secret`; both are root-owned mode `0600`.
|
||||
It refuses to overwrite either file and does not start services or create
|
||||
volumes. Review `DATA_ROOT` and place the approved normalized corpus there
|
||||
before deployment.
|
||||
3. After reviewing the root-owned checkout and audit output, run
|
||||
`sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh`. It installs
|
||||
the backup reader, read-only verifier, `akyldash-compose config|status`, and
|
||||
rollback helper with exact sudoers command rules. The Compose helper has no
|
||||
start/stop/build operation and grants no direct Docker access. Actual
|
||||
deployments remain owner-run through the documented Compose commands.
|
||||
4. Confirm DNS points through Cloudflare to this origin. The owner reports UFW
|
||||
already allows TCP `80` and `443`; read and document the full rules before
|
||||
any firewall decision. TCP `1500` is allowed by owner report but has no
|
||||
observed listener and no ISPmanager package/service was found; the owner
|
||||
must identify its purpose. Do not change UFW as part of this procedure. The
|
||||
Compose Caddy service obtains and renews HTTPS certificates automatically;
|
||||
Cloudflare Full (strict) still needs end-to-end verification.
|
||||
5. For a clean first deployment, skip backup only when Compose confirms there
|
||||
are no containers and all persistent mounts are empty. If any data exists,
|
||||
create and verify a backup before starting a new version. For a preloaded
|
||||
corpus with absent named volumes, use `--initial-deployment`; it is limited
|
||||
to a container-free host whose `DATA_ROOT` contains only
|
||||
`minjust-normalized/`. Normal backups reject absent volumes. An uncertain
|
||||
or partial state stops deployment. Then start services, load and validate
|
||||
the OpenSearch index, create the initial admin interactively, verify
|
||||
`/review` over HTTPS, and create the required post-setup baseline backup
|
||||
described in `BACKUP.md`. Record the backend image ID for drift checks:
|
||||
|
||||
```bash
|
||||
sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt'
|
||||
sudo chmod 0600 /opt/akyldash/deployed-image.txt
|
||||
```
|
||||
|
||||
Application deployment remains owner-run. A repository-level Actions runner
|
||||
installation is prepared in [CI_RUNNER.md](CI_RUNNER.md), but it is not yet
|
||||
installed or registered; its Online state and smoke workflow are unverified.
|
||||
The runner's job containers have no Docker socket or production secrets. CI has
|
||||
no image publishing or production deployment step. The installed sudo helpers
|
||||
support fixed verification, read-only Compose `config`/`status`, and guarded
|
||||
rollback. They do not start or stop services.
|
||||
|
||||
## CI
|
||||
|
||||
`.gitea/workflows/ci.yaml` runs the Python unit suites and builds both
|
||||
production images on pull requests and pushes to `main`. It does not publish an
|
||||
image or deploy production. `.gitea/workflows/runner-smoke.yaml` exercises the
|
||||
`akyldash-ci` runner on pull requests, pushes to `main`, or manual dispatch.
|
||||
Runner registration is incomplete, so neither workflow execution on that
|
||||
runner nor current `ubuntu-latest` runner availability has been confirmed.
|
||||
@@ -0,0 +1,59 @@
|
||||
# Infrastructure
|
||||
|
||||
## Intended layout
|
||||
|
||||
Production is planned for the dedicated Ubuntu VPS at `206.223.249.118`.
|
||||
`deploy/production/compose.yaml` defines Caddy, a Python Search API, and private
|
||||
OpenSearch 3.7. Caddy terminates HTTPS for `akyldash.kg` and proxies to the API.
|
||||
The API is reachable only on private Compose networks; OpenSearch is reachable
|
||||
only by the backend. Cloudflare should use Full (strict) only after the origin
|
||||
certificate and DNS path are verified.
|
||||
|
||||
The API stores its account and review SQLite files beside the normalized corpus
|
||||
under the persistent data directory. OpenSearch and Caddy use named volumes.
|
||||
The API's review signing key is mounted from a host file; it is not passed in
|
||||
process arguments or committed to Git. Docker logs are bounded to 10 MiB per
|
||||
file and three files per service.
|
||||
|
||||
## Actual state
|
||||
|
||||
The 2026-10-11 read-only audit found a fresh Ubuntu 24.04.5 host with SSH only.
|
||||
Docker, Git, a reverse proxy, application directories, Compose services, TLS,
|
||||
and backups are not installed/configured. The `codex` account has no
|
||||
non-interactive sudo. See [CURRENT_STATE.md](CURRENT_STATE.md) and the
|
||||
[production inventory](../../infra/inventory/production.yml).
|
||||
|
||||
The Compose file is a repository target, not proof of deployment. The Caddy
|
||||
image is pinned by digest, while backend and OpenSearch still build locally;
|
||||
immutable registry images and automated deploy are not configured. No migration
|
||||
framework exists. The application uses SQLite and OpenSearch; SMTP
|
||||
registration/recovery is not implemented.
|
||||
|
||||
## Repository paths and target host paths
|
||||
|
||||
- Compose and Dockerfiles: `deploy/production/`.
|
||||
- Host checkout target: `/opt/akyldash/current`.
|
||||
- Environment configuration: `/etc/akyldash/production.env`.
|
||||
- Secret file: `/etc/akyldash/review_secret`, mode `0600`.
|
||||
- Application data: `/var/lib/akyldash/data`.
|
||||
- Intended backups: `/var/backups/akyldash`.
|
||||
|
||||
All host paths above are targets; none existed at audit time. Root bootstrap is
|
||||
prepared at [bootstrap-ubuntu-24.04.sh](../../infra/bootstrap/bootstrap-ubuntu-24.04.sh).
|
||||
It does not edit SSH or firewall configuration, create credentials, deploy the
|
||||
application, or grant `codex` Docker/root access.
|
||||
|
||||
## Network and operations
|
||||
|
||||
Target public TCP listeners are SSH `22` and Caddy `80`/`443`. Backend `8080`
|
||||
and OpenSearch `9200` must remain private to Compose. The
|
||||
firewall state could not be independently verified because the SSH account has
|
||||
no sudo; do not change it without owner approval. The domain resolves to
|
||||
Cloudflare edge addresses, but HTTP and HTTPS return `521`, so the origin is
|
||||
unreachable. Cloudflare origin mode, certificate validity, and production
|
||||
health have not been verified.
|
||||
|
||||
Use [DEPLOYMENT.md](DEPLOYMENT.md), [BACKUP.md](BACKUP.md),
|
||||
[ROLLBACK.md](ROLLBACK.md), and [RECOVERY.md](RECOVERY.md) for operational
|
||||
limits and procedures. Configuration changes must go through Git and review;
|
||||
do not reconcile a Git/server difference by overwriting the VPS automatically.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Recovery on a clean VPS
|
||||
|
||||
## Scope and safety
|
||||
|
||||
The bootstrap below prepares Ubuntu packages and directories only. It is not a
|
||||
complete recovery: production secrets, data, reverse proxy, TLS, registry
|
||||
credentials, backup restore, and deploy-helper still require owner-controlled
|
||||
setup. Run only on the new recovery host after verifying its identity. Never
|
||||
run recovery commands against the existing production by accident.
|
||||
|
||||
## Procedure
|
||||
|
||||
1. Provision a clean Ubuntu Server 24.04 host and confirm hostname, IP, console
|
||||
access, and a tested owner recovery path.
|
||||
2. Create the SSH-authorized `codex` account without sudo or Docker group access.
|
||||
Preserve the host's existing root SSH and firewall policy until an owner has
|
||||
reviewed the change.
|
||||
3. Check out the approved Git revision, root-owned and not group/world-writable,
|
||||
at `/opt/akyldash/current`, then run
|
||||
`sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The
|
||||
recovery mode refuses the inventoried production hostname/IP and writes a
|
||||
root-only marker at `/etc/akyldash/recovery-target`. It installs Docker
|
||||
Engine, Compose, Git, Docker log limits, and target directories.
|
||||
4. Install the limited read-only verifier and rollback helper with
|
||||
`sudo bash infra/deploy/install-helper.sh`. It validates the root-owned
|
||||
checkout and gives `codex` access only to those fixed commands.
|
||||
5. Configure DNS to point through Cloudflare to the recovery host and allow the
|
||||
owner to review/open TCP `80` and `443`. The production Compose Caddy service
|
||||
obtains its origin certificate automatically. Verify Cloudflare Full
|
||||
(strict) end to end before cutover.
|
||||
6. Transfer the protected backup to the new host through the owner's approved
|
||||
channel. The archive contains the production environment and review secret;
|
||||
keep it root-only and never paste secret values into chat.
|
||||
7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging
|
||||
directory with `infra/backup/archive.py`, then run
|
||||
`sudo python3 infra/backup/apply_restore.py --staging-dir <staging-dir>`.
|
||||
Apply requires the recovery marker, matching clean Git revision, empty data
|
||||
path, absent Compose volumes/config files, and no running containers. It
|
||||
refuses the inventoried production host and leaves services stopped.
|
||||
8. Build the backend and OpenSearch images from the matching checked-out Git
|
||||
revision and explicitly start Compose with `--wait`. Verify HTTPS and
|
||||
container health, record the backend image ID as described in
|
||||
[DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in
|
||||
[CURRENT_STATE.md](CURRENT_STATE.md).
|
||||
|
||||
Do not repoint DNS or retire the old host until the owner verifies recovery and
|
||||
approves the cutover. The restore and cutover have not been practiced yet.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Rollback
|
||||
|
||||
## Current status
|
||||
|
||||
There is no production release, backup, or installed rollback helper. The
|
||||
repository contains a root-owned rollback helper and a separate installer, but
|
||||
neither has been run on the VPS.
|
||||
|
||||
After a verified cold backup is created before a release, an owner must check
|
||||
that the data format is compatible with the prior backend/OpenSearch images.
|
||||
Then run the limited helper with that archive:
|
||||
|
||||
```bash
|
||||
sudo -n /usr/local/sbin/akyldash-rollback \
|
||||
/var/backups/akyldash/<pre-release-backup>.tar.gz --data-compatible
|
||||
```
|
||||
|
||||
The helper verifies archive integrity, Compose image references, deployment
|
||||
file hashes, and the availability of both prior local images. It makes and
|
||||
verifies a second cold backup, restores only the prior backend/OpenSearch image
|
||||
tags, waits for Compose healthchecks, and checks HTTPS locally. It refuses to
|
||||
restore if the current Compose/Caddyfile differs from the saved release. This
|
||||
does not restore data or automatically undo a release with an unknown schema
|
||||
change. The application has no formal migration framework; compatibility must
|
||||
be checked for every change. If it is uncertain, stop and ask the owner.
|
||||
|
||||
The initial owner-run deployment must record its running backend image ID in
|
||||
`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates
|
||||
that record after a successful rollback, and `infra/bootstrap/verify.sh` checks
|
||||
it against the running container.
|
||||
|
||||
Any data restore requires the owner, a verified pre-change backup, and a
|
||||
maintenance window. Preserve the failed release and both backups until recovery
|
||||
is confirmed.
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
- Telegram-бот: `0.2.2`
|
||||
- Telegram-бот на Synology: `0.2.1`
|
||||
- Backend: `0.11.2`
|
||||
- Backend: `0.11.3`
|
||||
- Frontend: `0.1.2` — поиск, лаборатория релевантности, аккаунты
|
||||
|
||||
## Краткий итог
|
||||
@@ -110,8 +110,9 @@
|
||||
|
||||
### Развёртывание и сопровождение
|
||||
|
||||
- Docker-конфигурация развёртывания не хранится в репозитории.
|
||||
- Не настроен CI/CD.
|
||||
- Production Compose, Ubuntu bootstrap и drift-проверка описаны в репозитории.
|
||||
- CI workflow добавлен в PR #42, но его Gitea run остаётся в очереди; наличие
|
||||
runner и автоматический production-деплой не подтверждены.
|
||||
|
||||
### Хранилище
|
||||
|
||||
@@ -281,4 +282,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -262,4 +262,4 @@ runtime-зависимостями frontend. Регистрация в стор
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.1
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -44,4 +44,4 @@
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.2 · Frontend v0.1.2
|
||||
Акылдаш · Telegram-бот v0.2.2 · Backend v0.11.3 · Frontend v0.1.2
|
||||
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install a verified staging restore onto an explicitly marked new VPS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from archive import COMPONENTS, check_sqlite, digest
|
||||
|
||||
ROOT = Path("/opt/akyldash/current")
|
||||
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
||||
ENV_DEST = Path("/etc/akyldash/production.env")
|
||||
SECRET_DEST = Path("/etc/akyldash/review_secret")
|
||||
DATA_DEST = Path("/var/lib/akyldash/data")
|
||||
MARKER = Path("/etc/akyldash/recovery-target")
|
||||
PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
VOLUME_TARGETS = {
|
||||
"/usr/share/opensearch/data": "opensearch-data",
|
||||
"/data": "caddy-data",
|
||||
"/config": "caddy-config",
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], *, capture: bool = False) -> str:
|
||||
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def require_empty_directory(path: Path) -> None:
|
||||
if not path.is_dir() or path.is_symlink() or any(path.iterdir()):
|
||||
raise ValueError(f"restore destination must be an existing empty directory: {path}")
|
||||
|
||||
|
||||
def tree_size(root: Path) -> int:
|
||||
total = 0
|
||||
for directory, names, files in os.walk(root, followlinks=False):
|
||||
names[:] = [name for name in names if not (Path(directory) / name).is_symlink()]
|
||||
for name in files:
|
||||
path = Path(directory) / name
|
||||
if path.is_symlink():
|
||||
continue
|
||||
if path.is_file():
|
||||
total += path.stat().st_size
|
||||
return total
|
||||
|
||||
|
||||
def copy_contents(source: Path, destination: Path) -> None:
|
||||
for child in source.iterdir():
|
||||
target = destination / child.name
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
shutil.copytree(child, target, symlinks=True)
|
||||
else:
|
||||
shutil.copy2(child, target, follow_symlinks=False)
|
||||
|
||||
|
||||
def empty_directory(path: Path) -> None:
|
||||
for child in path.iterdir():
|
||||
if child.is_dir() and not child.is_symlink():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
child.unlink()
|
||||
|
||||
|
||||
def validate_stage(staging: Path) -> dict:
|
||||
staging = staging.resolve(strict=True)
|
||||
if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"):
|
||||
raise ValueError("staging directory must be a direct child of /var/tmp")
|
||||
if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700:
|
||||
raise ValueError("staging directory must be root-owned mode 0700")
|
||||
manifest_path = staging / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
not isinstance(manifest, dict)
|
||||
or manifest.get("format") != 1
|
||||
or set(manifest.get("components", [])) != COMPONENTS
|
||||
):
|
||||
raise ValueError("staging manifest is invalid")
|
||||
for name in COMPONENTS - {"production-env", "review-secret"}:
|
||||
component = staging / name
|
||||
if not component.is_dir() or component.is_symlink():
|
||||
raise ValueError(f"staged component must be a directory: {name}")
|
||||
for name in ("production-env", "review-secret"):
|
||||
path = staging / name
|
||||
if not path.is_file() or path.is_symlink():
|
||||
raise ValueError(f"staged sensitive component must be a regular file: {name}")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_checkout(manifest: dict) -> None:
|
||||
if ROOT.is_symlink() or not (ROOT / ".git").exists():
|
||||
raise ValueError("approved Git checkout is missing or is a symlink")
|
||||
if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip():
|
||||
raise ValueError("recovery checkout must be clean")
|
||||
unsafe = subprocess.run(
|
||||
["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if unsafe:
|
||||
raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}")
|
||||
revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True)
|
||||
if revision != manifest.get("git_revision"):
|
||||
raise ValueError("checkout Git revision does not match the backup manifest")
|
||||
for relative, expected in manifest.get("deployment_files", {}).items():
|
||||
if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected:
|
||||
raise ValueError("Compose or Caddyfile does not match the backup manifest")
|
||||
if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES):
|
||||
raise ValueError("backup manifest is missing deployment file hashes")
|
||||
|
||||
|
||||
def validate_recovery_target() -> None:
|
||||
if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("root-only recovery-target marker is required")
|
||||
hostname = socket.getfqdn()
|
||||
addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True)
|
||||
if not hostname or not addresses:
|
||||
raise ValueError("cannot verify the recovery host identity")
|
||||
if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses:
|
||||
raise ValueError("refusing to apply recovery data to the inventoried production VPS")
|
||||
|
||||
|
||||
def compose_configuration(staging: Path) -> dict:
|
||||
rendered = run(
|
||||
[
|
||||
"docker", "compose", "--env-file", str(staging / "production-env"),
|
||||
"-f", str(COMPOSE_FILE), "config", "--format", "json",
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
config = json.loads(rendered)
|
||||
data_source = next(
|
||||
mount["source"]
|
||||
for mount in config["services"]["backend"]["volumes"]
|
||||
if mount["target"] == "/app/data"
|
||||
)
|
||||
secret_source = config["secrets"]["review_secret"]["file"]
|
||||
if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST:
|
||||
raise ValueError("backup production paths do not match the clean recovery host layout")
|
||||
return config
|
||||
|
||||
|
||||
def expected_volume_names(config: dict) -> set[str]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
names: set[str] = set()
|
||||
for target in VOLUME_TARGETS:
|
||||
mount = mounts.get(target)
|
||||
if not mount or mount["type"] != "volume":
|
||||
raise ValueError(f"expected persistent volume at {target}")
|
||||
names.add(config["volumes"][mount["source"]]["name"])
|
||||
return names
|
||||
|
||||
|
||||
def volume_destinations(config: dict) -> dict[str, Path]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
destinations = {}
|
||||
for target, component in VOLUME_TARGETS.items():
|
||||
mount = mounts.get(target)
|
||||
if not mount or mount["type"] != "volume":
|
||||
raise ValueError(f"expected persistent volume at {target}")
|
||||
name = config["volumes"][mount["source"]]["name"]
|
||||
destinations[component] = Path(
|
||||
run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True)
|
||||
)
|
||||
return destinations
|
||||
|
||||
|
||||
def apply_restore(staging: Path) -> None:
|
||||
if os.geteuid() != 0:
|
||||
raise ValueError("restore apply must run as root")
|
||||
validate_recovery_target()
|
||||
manifest = validate_stage(staging)
|
||||
validate_checkout(manifest)
|
||||
config = compose_configuration(staging)
|
||||
require_empty_directory(DATA_DEST)
|
||||
if ENV_DEST.exists() or SECRET_DEST.exists():
|
||||
raise ValueError("production environment and secret files must not already exist")
|
||||
if run(["docker", "ps", "--quiet"], capture=True):
|
||||
raise ValueError("recovery target must have no running Docker containers")
|
||||
|
||||
volume_names = expected_volume_names(config)
|
||||
for name in volume_names:
|
||||
result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
raise ValueError(f"recovery volume already exists: {name}")
|
||||
|
||||
created_volumes: list[str] = []
|
||||
created_files: list[Path] = []
|
||||
data_started = False
|
||||
try:
|
||||
for name in volume_names:
|
||||
run(["docker", "volume", "create", name])
|
||||
created_volumes.append(name)
|
||||
|
||||
destinations = volume_destinations(config)
|
||||
destinations["data"] = DATA_DEST
|
||||
|
||||
required: dict[int, int] = {}
|
||||
free: dict[int, int] = {}
|
||||
for component, destination in destinations.items():
|
||||
device = destination.stat().st_dev
|
||||
required[device] = required.get(device, 0) + tree_size(staging / component)
|
||||
free[device] = shutil.disk_usage(destination).free
|
||||
for device, size in required.items():
|
||||
if free[device] < size + 512 * 1024 * 1024:
|
||||
raise ValueError("recovery host lacks space for a safe data copy")
|
||||
|
||||
for component, destination in destinations.items():
|
||||
require_empty_directory(destination)
|
||||
if component == "data":
|
||||
data_started = True
|
||||
copy_contents(staging / component, destination)
|
||||
|
||||
for directory, names, files in os.walk(destinations["opensearch"], followlinks=False):
|
||||
os.chown(directory, 1000, 1000, follow_symlinks=False)
|
||||
for name in files:
|
||||
os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False)
|
||||
|
||||
for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)):
|
||||
shutil.copyfile(source, destination)
|
||||
created_files.append(destination)
|
||||
os.chown(destination, 0, 0)
|
||||
os.chmod(destination, 0o600)
|
||||
check_sqlite(DATA_DEST)
|
||||
except BaseException:
|
||||
for path in created_files:
|
||||
path.unlink(missing_ok=True)
|
||||
if data_started:
|
||||
empty_directory(DATA_DEST)
|
||||
for name in created_volumes:
|
||||
subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True)
|
||||
raise
|
||||
|
||||
MARKER.unlink()
|
||||
print("PASS staged recovery data applied to the marked new host; services remain stopped")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--staging-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
apply_restore(args.staging_dir)
|
||||
except Exception as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create and verify cold, local Akyldash backups; restore only into staging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
COMPONENTS = {
|
||||
"data",
|
||||
"opensearch",
|
||||
"caddy-data",
|
||||
"caddy-config",
|
||||
"production-env",
|
||||
"review-secret",
|
||||
}
|
||||
NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def checked_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
|
||||
members = archive.getmembers()
|
||||
names: set[str] = set()
|
||||
for member in members:
|
||||
name = PurePosixPath(member.name)
|
||||
if name.is_absolute() or ".." in name.parts or not name.parts:
|
||||
raise ValueError(f"unsafe archive path: {member.name}")
|
||||
if member.name not in {name.as_posix(), f"{name.as_posix()}/"}:
|
||||
raise ValueError(f"non-canonical archive path: {member.name}")
|
||||
if member.name in names:
|
||||
raise ValueError(f"duplicate archive path: {member.name}")
|
||||
names.add(member.name)
|
||||
if not (member.isdir() or member.isfile() or member.issym()):
|
||||
raise ValueError(f"unsupported archive entry: {member.name}")
|
||||
if member.issym():
|
||||
tarfile.data_filter(member, Path("/tmp/akyldash-restore-check"))
|
||||
if "manifest.json" not in names:
|
||||
raise ValueError("backup manifest is missing")
|
||||
if {PurePosixPath(name).parts[0] for name in names if name != "manifest.json"} != COMPONENTS:
|
||||
raise ValueError("backup is missing one or more required components")
|
||||
manifest_member = next(item for item in members if item.name == "manifest.json")
|
||||
if not manifest_member.isfile() or manifest_member.size > 1024 * 1024:
|
||||
raise ValueError("backup manifest is not a regular file")
|
||||
return members
|
||||
|
||||
|
||||
def read_manifest(archive: tarfile.TarFile) -> dict:
|
||||
checked_members(archive)
|
||||
stream = archive.extractfile("manifest.json")
|
||||
if stream is None:
|
||||
raise ValueError("backup manifest is unreadable")
|
||||
manifest = json.load(stream)
|
||||
components = manifest.get("components") if isinstance(manifest, dict) else None
|
||||
if (
|
||||
not isinstance(manifest, dict)
|
||||
or manifest.get("format") != 1
|
||||
or not isinstance(components, list)
|
||||
or not all(isinstance(item, str) for item in components)
|
||||
or set(components) != COMPONENTS
|
||||
):
|
||||
raise ValueError("backup manifest has an unsupported format or component set")
|
||||
if not isinstance(manifest.get("images"), dict):
|
||||
raise ValueError("backup manifest has no image records")
|
||||
if not isinstance(manifest.get("deployment_files"), dict):
|
||||
raise ValueError("backup manifest has no deployment file records")
|
||||
for member in archive.getmembers():
|
||||
if member.name == "manifest.json":
|
||||
continue
|
||||
if PurePosixPath(member.name).parts[0] not in COMPONENTS:
|
||||
raise ValueError(f"unexpected top-level archive path: {member.name}")
|
||||
return manifest
|
||||
|
||||
|
||||
def verify_archive(path: Path) -> dict:
|
||||
checksum_path = Path(f"{path}.sha256")
|
||||
expected = checksum_path.read_text(encoding="ascii").split()[0]
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", expected) or digest(path) != expected:
|
||||
raise ValueError("archive SHA-256 does not match its sidecar")
|
||||
with tarfile.open(path, "r:gz") as archive:
|
||||
return read_manifest(archive)
|
||||
|
||||
|
||||
def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
|
||||
sources: dict[str, Path] = {}
|
||||
for value in values:
|
||||
name, separator, raw_path = value.partition("=")
|
||||
if not separator or not NAME_RE.fullmatch(name) or name in sources:
|
||||
raise ValueError(f"invalid or duplicate source: {value}")
|
||||
source = Path(raw_path)
|
||||
if not source.is_absolute() or not source.exists() or source.is_symlink():
|
||||
raise ValueError(f"source must be an existing absolute path: {raw_path}")
|
||||
sources[name] = source.resolve(strict=True)
|
||||
if set(sources) != required:
|
||||
missing = ", ".join(sorted(required - sources.keys()))
|
||||
extra = ", ".join(sorted(sources.keys() - required))
|
||||
raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
|
||||
return sources
|
||||
|
||||
|
||||
def check_sqlite(root: Path) -> None:
|
||||
for path in root.rglob("*.sqlite3"):
|
||||
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
|
||||
try:
|
||||
result = connection.execute("PRAGMA quick_check").fetchone()
|
||||
if result != ("ok",):
|
||||
raise ValueError(f"SQLite integrity check failed: {path}")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def compose_storage(config: dict) -> dict[str, Path | None]:
|
||||
mounts = {
|
||||
mount["target"]: mount
|
||||
for service in config["services"].values()
|
||||
for mount in service.get("volumes", [])
|
||||
}
|
||||
volumes = subprocess.run(
|
||||
["docker", "volume", "ls", "--quiet"], check=True, capture_output=True, text=True
|
||||
).stdout.split()
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, component in (
|
||||
("/app/data", "data"),
|
||||
("/usr/share/opensearch/data", "opensearch"),
|
||||
("/data", "caddy-data"),
|
||||
("/config", "caddy-config"),
|
||||
):
|
||||
mount = mounts.get(target)
|
||||
if not mount:
|
||||
raise ValueError(f"persistent Compose mount is missing: {target}")
|
||||
if mount["type"] == "bind":
|
||||
source = Path(mount["source"])
|
||||
elif mount["type"] == "volume":
|
||||
volume_name = config["volumes"][mount["source"]]["name"]
|
||||
if volume_name not in volumes:
|
||||
sources[component] = None
|
||||
continue
|
||||
result = subprocess.run(
|
||||
["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", volume_name],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
source = Path(result.stdout.strip())
|
||||
else:
|
||||
raise ValueError(f"unsupported persistent mount type at {target}")
|
||||
if not source.is_dir() or source.is_symlink():
|
||||
raise ValueError(f"cannot determine persistent data state for {component}: {source}")
|
||||
sources[component] = source
|
||||
return sources
|
||||
|
||||
|
||||
def has_persistent_data(sources: dict[str, Path | None]) -> bool:
|
||||
return any(any(path.iterdir()) for path in sources.values() if path is not None)
|
||||
|
||||
|
||||
def is_preloaded_corpus_only(data: Path | None) -> bool:
|
||||
if data is None:
|
||||
return False
|
||||
entries = list(data.iterdir())
|
||||
return len(entries) == 1 and entries[0].name == "minjust-normalized" and entries[0].is_dir() and not entries[0].is_symlink()
|
||||
|
||||
|
||||
def compose_sources(
|
||||
compose_file: Path, env_file: Path, initial_deployment: bool = False
|
||||
) -> tuple[dict[str, Path | None], str, dict[str, dict[str, str]], dict[str, str]]:
|
||||
command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)]
|
||||
rendered = subprocess.run(
|
||||
[*command, "config", "--format", "json"], check=True, capture_output=True, text=True
|
||||
)
|
||||
config = json.loads(rendered.stdout)
|
||||
container_result = subprocess.run(
|
||||
[*command, "ps", "--all", "--quiet"], check=True, capture_output=True, text=True
|
||||
)
|
||||
containers = container_result.stdout.split()
|
||||
storage = compose_storage(config)
|
||||
persistent_data = has_persistent_data(storage)
|
||||
if not containers and not persistent_data:
|
||||
raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
|
||||
missing_volumes = any(path is None for path in storage.values())
|
||||
if missing_volumes and containers:
|
||||
raise ValueError("persistent Compose volumes are missing while containers exist; refusing an incomplete backup")
|
||||
if missing_volumes and not initial_deployment:
|
||||
raise ValueError("persistent Compose volumes are missing; refusing an incomplete backup (use --initial-deployment only for a new host with a preloaded corpus)")
|
||||
if missing_volumes and not is_preloaded_corpus_only(storage["data"]):
|
||||
raise ValueError("initial deployment with missing volumes requires DATA_ROOT to contain only minjust-normalized/; refusing an incomplete backup")
|
||||
images: dict[str, dict[str, str]] = {}
|
||||
for container in containers:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
'{{.State.Running}}\t{{index .Config.Labels "com.docker.compose.service"}}\t{{.Config.Image}}\t{{.Image}}',
|
||||
container,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
fields = result.stdout.strip().split("\t")
|
||||
if len(fields) != 4 or fields[0].lower() != "false":
|
||||
raise ValueError("stop all Compose services before taking a cold backup")
|
||||
images[fields[1]] = {"configured": fields[2], "image_id": fields[3]}
|
||||
if containers and set(images) != set(config["services"]):
|
||||
raise ValueError("not all configured Compose services have stopped containers")
|
||||
|
||||
secret_file = Path(config["secrets"]["review_secret"]["file"])
|
||||
sources = {
|
||||
**storage,
|
||||
"production-env": env_file,
|
||||
"review-secret": secret_file,
|
||||
}
|
||||
repository = compose_file.resolve().parents[2]
|
||||
changed = subprocess.run(
|
||||
["git", "-C", str(repository), "status", "--porcelain", "--", "deploy/production/compose.yaml", "deploy/production/Caddyfile", "deploy/production/Dockerfile.backend", "deploy/production/Dockerfile.opensearch"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if changed:
|
||||
raise ValueError("production deployment files have local Git changes; refusing an untracked backup revision")
|
||||
revision = subprocess.run(
|
||||
["git", "-C", str(repository), "rev-parse", "HEAD"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
deployment_files = {
|
||||
relative: digest(repository / relative)
|
||||
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
}
|
||||
present = {name: path for name, path in sources.items() if path is not None}
|
||||
missing = {name for name, path in sources.items() if path is None}
|
||||
parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
|
||||
return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
|
||||
|
||||
|
||||
def create_backup(
|
||||
output: Path,
|
||||
sources: dict[str, Path | None],
|
||||
revision: str,
|
||||
images: dict | None = None,
|
||||
deployment_files: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
output = output.absolute()
|
||||
parent = output.parent
|
||||
if not parent.is_dir() or parent.stat().st_mode & 0o077:
|
||||
raise ValueError("backup directory must exist and have mode 0700 or stricter")
|
||||
if output.exists() or Path(f"{output}.sha256").exists():
|
||||
raise ValueError("refusing to overwrite an existing backup")
|
||||
for name, source in sources.items():
|
||||
if source is None:
|
||||
continue
|
||||
if source.is_symlink() or not (source.is_dir() or source.is_file()):
|
||||
raise ValueError(f"source must be a regular file or directory: {name}")
|
||||
if source.is_dir() and (output == source or source in output.parents):
|
||||
raise ValueError(f"backup output cannot be inside source {name}")
|
||||
for name in ("production-env", "review-secret"):
|
||||
if sources[name] is None or not sources[name].is_file():
|
||||
raise ValueError(f"sensitive component must be a regular file: {name}")
|
||||
if sources["data"] is not None and sources["data"].is_dir():
|
||||
check_sqlite(sources["data"])
|
||||
|
||||
manifest = {
|
||||
"format": 1,
|
||||
"created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_revision": revision,
|
||||
"images": images or {},
|
||||
"deployment_files": deployment_files or {},
|
||||
"components": sorted(sources),
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp:
|
||||
temporary = Path(temp.name)
|
||||
os.chmod(temporary, 0o600)
|
||||
checksum_temp = Path(f"{temporary}.sha256")
|
||||
try:
|
||||
with tarfile.open(temporary, "w:gz") as archive:
|
||||
encoded = json.dumps(manifest, sort_keys=True).encode("utf-8")
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(encoded)
|
||||
info.mode = 0o600
|
||||
archive.addfile(info, io.BytesIO(encoded))
|
||||
for name, source in sorted(sources.items()):
|
||||
if source is None:
|
||||
info = tarfile.TarInfo(f"{name}/")
|
||||
info.type = tarfile.DIRTYPE
|
||||
info.mode = 0o700
|
||||
archive.addfile(info)
|
||||
continue
|
||||
def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
|
||||
if name in {"production-env", "review-secret"}:
|
||||
member.mode = 0o600
|
||||
if member.name.endswith((".sqlite3", ".sqlite3-wal", ".sqlite3-shm")):
|
||||
member.mode = 0o600
|
||||
return member
|
||||
|
||||
archive.add(source, arcname=name, recursive=True, filter=protect_secrets)
|
||||
with tarfile.open(temporary, "r:gz") as archive:
|
||||
read_manifest(archive)
|
||||
checksum_fd = os.open(checksum_temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(checksum_fd, "w", encoding="ascii") as stream:
|
||||
stream.write(f"{digest(temporary)} {output.name}\n")
|
||||
os.link(temporary, output)
|
||||
try:
|
||||
os.link(checksum_temp, Path(f"{output}.sha256"))
|
||||
except Exception:
|
||||
output.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
checksum_temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def restore_to_staging(backup: Path, target: Path) -> None:
|
||||
if not target.name.startswith("akyldash-restore-"):
|
||||
raise ValueError("restore target must be a new akyldash-restore-* staging directory")
|
||||
parent = target.parent.resolve()
|
||||
if not any(parent == root or root in parent.parents for root in (Path("/tmp"), Path("/var/tmp"))):
|
||||
raise ValueError("restore staging directory must be under /tmp or /var/tmp")
|
||||
if target.exists():
|
||||
raise ValueError("restore target already exists; refusing to overwrite it")
|
||||
target = parent / target.name
|
||||
verify_archive(backup)
|
||||
target.mkdir(mode=0o700)
|
||||
try:
|
||||
with tarfile.open(backup, "r:gz") as archive:
|
||||
members = checked_members(archive)
|
||||
required_bytes = sum(member.size for member in members if member.isfile())
|
||||
if shutil.disk_usage(parent).free < required_bytes + 512 * 1024 * 1024:
|
||||
raise ValueError("insufficient free space for a validated restore staging copy")
|
||||
for member in members:
|
||||
archive.extract(member, target, filter="data")
|
||||
check_sqlite(target / "data")
|
||||
except Exception:
|
||||
shutil.rmtree(target)
|
||||
raise
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
create = subparsers.add_parser("create", help="create a cold backup archive")
|
||||
create.add_argument("--output", required=True, type=Path)
|
||||
create.add_argument("--compose-file", required=True, type=Path)
|
||||
create.add_argument("--env-file", required=True, type=Path)
|
||||
create.add_argument("--initial-deployment", action="store_true", help="allow absent named volumes only for a new host containing only the preloaded corpus")
|
||||
verify = subparsers.add_parser("verify", help="verify archive checksum and contents")
|
||||
verify.add_argument("archive", type=Path)
|
||||
restore = subparsers.add_parser("restore", help="extract into a new staging directory")
|
||||
restore.add_argument("archive", type=Path)
|
||||
restore.add_argument("--target-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command in {"create", "restore"} and os.geteuid() != 0:
|
||||
raise ValueError("create and restore must run as root to protect production secrets and data")
|
||||
if args.command == "create":
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
args.compose_file, args.env_file, args.initial_deployment
|
||||
)
|
||||
create_backup(args.output, sources, revision, images, deployment_files)
|
||||
elif args.command == "verify":
|
||||
manifest = verify_archive(args.archive)
|
||||
print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)")
|
||||
else:
|
||||
restore_to_staging(args.archive, args.target_dir)
|
||||
print(f"PASS restored and validated staging data at {args.target_dir}")
|
||||
except (OSError, ValueError, tarfile.TarError, json.JSONDecodeError, sqlite3.Error, IndexError, subprocess.SubprocessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,90 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import apply_restore
|
||||
|
||||
|
||||
class ApplyRestoreTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.staging = self.root / "akyldash-restore-test"
|
||||
self.staging.mkdir()
|
||||
for component in apply_restore.COMPONENTS:
|
||||
path = self.staging / component
|
||||
if component in {"production-env", "review-secret"}:
|
||||
path.write_text("private", encoding="utf-8")
|
||||
else:
|
||||
path.mkdir()
|
||||
(path / "payload.txt").write_text(component, encoding="utf-8")
|
||||
self.data = self.root / "data"
|
||||
self.data.mkdir()
|
||||
self.volumes = {
|
||||
name: self.root / name
|
||||
for name in ("opensearch", "caddy-data", "caddy-config")
|
||||
}
|
||||
for path in self.volumes.values():
|
||||
path.mkdir()
|
||||
self.env = self.root / "production.env"
|
||||
self.secret = self.root / "review_secret"
|
||||
self.marker = self.root / "recovery-target"
|
||||
self.marker.touch(mode=0o600)
|
||||
self.config = {
|
||||
"volumes": {
|
||||
"opensearch-data": {"name": "akyldash_opensearch-data"},
|
||||
"caddy-data": {"name": "akyldash_caddy-data"},
|
||||
"caddy-config": {"name": "akyldash_caddy-config"},
|
||||
}
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def run_apply(self):
|
||||
def docker_run(command, **_kwargs):
|
||||
if command[:3] == ["docker", "ps", "--quiet"]:
|
||||
return ""
|
||||
if command[1:3] == ["volume", "create"]:
|
||||
return command[-1]
|
||||
return ""
|
||||
|
||||
with (
|
||||
patch("apply_restore.os.geteuid", return_value=0),
|
||||
patch("apply_restore.validate_recovery_target"),
|
||||
patch("apply_restore.validate_stage", return_value={"format": 1}),
|
||||
patch("apply_restore.validate_checkout"),
|
||||
patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}),
|
||||
patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])),
|
||||
patch("apply_restore.volume_destinations", return_value=self.volumes),
|
||||
patch("apply_restore.DATA_DEST", self.data),
|
||||
patch("apply_restore.ENV_DEST", self.env),
|
||||
patch("apply_restore.SECRET_DEST", self.secret),
|
||||
patch("apply_restore.MARKER", self.marker),
|
||||
patch("apply_restore.run", side_effect=docker_run),
|
||||
patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)),
|
||||
patch("apply_restore.os.chown"),
|
||||
):
|
||||
apply_restore.apply_restore(self.staging)
|
||||
|
||||
def test_applies_components_and_protects_secret_files(self):
|
||||
self.run_apply()
|
||||
self.assertEqual((self.data / "payload.txt").read_text(), "data")
|
||||
self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch")
|
||||
self.assertEqual(self.env.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600)
|
||||
self.assertFalse(self.marker.exists())
|
||||
|
||||
def test_refuses_nonempty_data_destination_before_writing_secrets(self):
|
||||
(self.data / "keep.txt").write_text("existing", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "empty directory"):
|
||||
self.run_apply()
|
||||
self.assertEqual((self.data / "keep.txt").read_text(), "existing")
|
||||
self.assertFalse(self.env.exists())
|
||||
self.assertTrue(self.marker.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,362 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from archive import COMPONENTS, compose_sources, create_backup, restore_to_staging, verify_archive
|
||||
|
||||
|
||||
class BackupArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name)
|
||||
self.private = self.root / "private"
|
||||
self.private.mkdir(mode=0o700)
|
||||
self.sources = {}
|
||||
for name in COMPONENTS:
|
||||
source = self.root / name
|
||||
source.mkdir()
|
||||
self.sources[name] = source
|
||||
for name in ("production-env", "review-secret"):
|
||||
path = self.root / f"{name}.txt"
|
||||
path.write_text("private", encoding="utf-8")
|
||||
path.chmod(0o644)
|
||||
self.sources[name] = path
|
||||
connection = sqlite3.connect(self.sources["data"] / "search-users.sqlite3")
|
||||
connection.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
self.backup = self.private / "backup.tar.gz"
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_create_verify_and_restore_only_to_new_staging(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
manifest = verify_archive(self.backup)
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
self.assertEqual(manifest["images"], {})
|
||||
|
||||
target = self.root / "akyldash-restore-test"
|
||||
restore_to_staging(self.backup, target)
|
||||
self.assertTrue((target / "data/search-users.sqlite3").is_file())
|
||||
self.assertEqual((target / "data/search-users.sqlite3").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual((target / "production-env").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual((target / "review-secret").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(verify_archive(self.backup)["components"], sorted(COMPONENTS))
|
||||
|
||||
def test_rejects_checksum_mismatch_without_creating_staging(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
self.backup.write_bytes(self.backup.read_bytes() + b"tamper")
|
||||
target = self.root / "akyldash-restore-tampered"
|
||||
with self.assertRaisesRegex(ValueError, "SHA-256"):
|
||||
restore_to_staging(self.backup, target)
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
def test_rejects_non_staging_and_existing_targets(self):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
with self.assertRaisesRegex(ValueError, "staging"):
|
||||
restore_to_staging(self.backup, self.root / "production")
|
||||
target = self.root / "akyldash-restore-existing"
|
||||
target.mkdir()
|
||||
with self.assertRaisesRegex(ValueError, "already exists"):
|
||||
restore_to_staging(self.backup, target)
|
||||
|
||||
def test_create_checks_sqlite_integrity_before_writing_archive(self):
|
||||
(self.sources["data"] / "search-reviews.sqlite3").write_bytes(b"not a database")
|
||||
with self.assertRaises(sqlite3.DatabaseError):
|
||||
create_backup(self.backup, self.sources, "abc123")
|
||||
self.assertFalse(self.backup.exists())
|
||||
|
||||
def test_rejects_unsafe_archive_paths(self):
|
||||
archive_path = self.private / "unsafe.tar.gz"
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.addfile(tarfile.TarInfo("manifest.json"))
|
||||
archive.addfile(tarfile.TarInfo("../outside"))
|
||||
checksum = hashlib.sha256(archive_path.read_bytes()).hexdigest()
|
||||
Path(f"{archive_path}.sha256").write_text(f"{checksum} unsafe.tar.gz\n")
|
||||
with self.assertRaisesRegex(ValueError, "unsafe archive path"):
|
||||
verify_archive(archive_path)
|
||||
|
||||
def test_cold_backup_refuses_a_running_compose_container(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="container-id\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
return SimpleNamespace(stdout="true\n")
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "stop all Compose services"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def compose_fixture(self):
|
||||
paths = {}
|
||||
mounts = {
|
||||
"backend": [{"type": "bind", "source": str(self.root / "data"), "target": "/app/data"}],
|
||||
"opensearch": [{"type": "volume", "source": "os-data", "target": "/usr/share/opensearch/data"}],
|
||||
"caddy": [
|
||||
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
||||
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
||||
],
|
||||
}
|
||||
volumes = {}
|
||||
for volume in ("os-data", "caddy-data", "caddy-config"):
|
||||
name = f"akyldash_{volume}"
|
||||
path = self.root / name
|
||||
path.mkdir()
|
||||
paths[name] = path
|
||||
volumes[volume] = {"name": name}
|
||||
env_file = self.root / "production.env"
|
||||
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
||||
secret = self.root / "review-secret.txt"
|
||||
secret.write_text("secret", encoding="utf-8")
|
||||
config = {
|
||||
"services": {name: {"volumes": value} for name, value in mounts.items()},
|
||||
"volumes": volumes,
|
||||
"secrets": {"review_secret": {"file": str(secret)}},
|
||||
}
|
||||
return config, paths, env_file, secret
|
||||
|
||||
def test_clean_first_deployment_does_not_require_backup(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_first_deployment_with_uncreated_named_volumes_is_clean(self):
|
||||
config, _volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "clean first deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_existing_data_without_containers_is_backup_eligible(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file
|
||||
)
|
||||
archive = self.private / "offline-backup.tar.gz"
|
||||
create_backup(archive, sources, revision, images, deployment_files)
|
||||
manifest = verify_archive(archive)
|
||||
self.assertEqual(set(sources), COMPONENTS)
|
||||
self.assertEqual(images, {})
|
||||
self.assertEqual(sources["opensearch"], volume_paths["akyldash_os-data"])
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
|
||||
def test_missing_persistent_mount_state_is_rejected(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
volume_paths["akyldash_os-data"].rmdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "cannot determine persistent data state"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_partial_compose_containers_are_rejected_with_existing_data(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(volume_paths["akyldash_os-data"] / "node-data").mkdir()
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="backend-id\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if command[1] == "inspect":
|
||||
return SimpleNamespace(stdout="false\tbackend\tbackend:local\tsha256:backend\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
|
||||
def test_missing_named_volume_with_existing_data_requires_explicit_initial_mode(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
(self.root / "data/existing.json").write_text("data", encoding="utf-8")
|
||||
available = set(volume_paths) - {"akyldash_os-data"}
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
||||
if command[1] == "volume":
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
with self.assertRaisesRegex(ValueError, "use --initial-deployment"):
|
||||
compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
|
||||
with self.assertRaisesRegex(ValueError, "contain only minjust-normalized"):
|
||||
compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
||||
)
|
||||
|
||||
def test_initial_mode_backs_up_only_preloaded_corpus_when_volumes_are_absent(self):
|
||||
config, volume_paths, env_file, _secret = self.compose_fixture()
|
||||
(self.root / "data/search-users.sqlite3").unlink()
|
||||
corpus = self.root / "data/minjust-normalized"
|
||||
corpus.mkdir()
|
||||
(corpus / "document.json").write_text("corpus", encoding="utf-8")
|
||||
available = set(volume_paths) - {"akyldash_os-data"}
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if command[1] == "volume" and "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(available) + "\n")
|
||||
if command[1] == "volume":
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
if "status" in command:
|
||||
return SimpleNamespace(stdout="")
|
||||
if "rev-parse" in command:
|
||||
return SimpleNamespace(stdout="abc123\n")
|
||||
raise AssertionError(command)
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file, initial_deployment=True
|
||||
)
|
||||
self.assertIsNone(sources["opensearch"])
|
||||
archive = self.private / "initial-corpus-backup.tar.gz"
|
||||
create_backup(archive, sources, revision, images, deployment_files)
|
||||
manifest = verify_archive(archive)
|
||||
self.assertEqual(manifest["git_revision"], "abc123")
|
||||
with tarfile.open(archive, "r:gz") as saved:
|
||||
self.assertTrue(saved.getmember("opensearch/").isdir())
|
||||
self.assertTrue(saved.getmember("data/minjust-normalized/document.json").isfile())
|
||||
|
||||
def test_compose_backup_records_sources_and_image_ids(self):
|
||||
data = self.root / "production-data"
|
||||
data.mkdir()
|
||||
secret = self.root / "secret"
|
||||
secret.write_text("private", encoding="utf-8")
|
||||
env_file = self.root / "production.env"
|
||||
env_file.write_text("DATA_ROOT=/var/lib/akyldash/data\n", encoding="utf-8")
|
||||
volumes = {}
|
||||
volume_paths = {}
|
||||
mounts = {
|
||||
"caddy": [
|
||||
{"type": "volume", "source": "caddy-data", "target": "/data"},
|
||||
{"type": "volume", "source": "caddy-config", "target": "/config"},
|
||||
],
|
||||
"backend": [{"type": "bind", "source": str(data), "target": "/app/data"}],
|
||||
"opensearch": [
|
||||
{"type": "volume", "source": "opensearch-data", "target": "/usr/share/opensearch/data"}
|
||||
],
|
||||
}
|
||||
for name in ("caddy-data", "caddy-config", "opensearch-data"):
|
||||
path = self.root / f"volume-{name}"
|
||||
path.mkdir()
|
||||
volumes[name] = {"name": f"akyldash_{name}"}
|
||||
volume_paths[f"akyldash_{name}"] = path
|
||||
config = {
|
||||
"services": {name: {"volumes": values} for name, values in mounts.items()},
|
||||
"volumes": volumes,
|
||||
"secrets": {"review_secret": {"file": str(secret)}},
|
||||
}
|
||||
service_names = ["caddy", "backend", "opensearch"]
|
||||
|
||||
def run(command, **_kwargs):
|
||||
if "config" in command:
|
||||
return SimpleNamespace(stdout=json.dumps(config))
|
||||
if "ps" in command:
|
||||
return SimpleNamespace(stdout="\n".join(f"{name}-id" for name in service_names))
|
||||
if command[1] == "inspect":
|
||||
name = command[-1].removesuffix("-id")
|
||||
return SimpleNamespace(stdout=f"false\t{name}\t{name}:local\tsha256:{name}\n")
|
||||
if command[1] == "volume":
|
||||
if "ls" in command:
|
||||
return SimpleNamespace(stdout="\n".join(volume_paths) + "\n")
|
||||
return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
|
||||
return SimpleNamespace(stdout="")
|
||||
|
||||
with patch("archive.subprocess.run", side_effect=run):
|
||||
sources, revision, images, deployment_files = compose_sources(
|
||||
Path.cwd() / "deploy/production/compose.yaml", env_file
|
||||
)
|
||||
self.assertEqual(revision, "")
|
||||
self.assertEqual(set(sources), COMPONENTS)
|
||||
self.assertEqual(images["backend"]["image_id"], "sha256:backend")
|
||||
self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run this read-only audit as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo '== Host =='
|
||||
hostname -f
|
||||
grep -E '^(PRETTY_NAME|VERSION_ID)=' /etc/os-release
|
||||
nproc
|
||||
free -h
|
||||
df -h /
|
||||
|
||||
echo '== Packages and services =='
|
||||
dpkg-query -W docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
docker version --format 'Docker Engine {{.Server.Version}}'
|
||||
docker compose version
|
||||
containerd --version
|
||||
runc --version
|
||||
systemctl is-active docker
|
||||
python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); print("Docker logs:", d.get("log-driver"), d.get("log-opts"))'
|
||||
|
||||
echo '== Account and paths =='
|
||||
id codex
|
||||
for path in /opt/akyldash /opt/akyldash/current /etc/akyldash /etc/akyldash/production.env /etc/akyldash/review_secret /var/lib/akyldash/data /var/backups/akyldash /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /etc/sudoers.d/akyldash-codex; do
|
||||
if [[ -e $path ]]; then
|
||||
stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
|
||||
else
|
||||
printf '%s absent\n' "$path"
|
||||
fi
|
||||
done
|
||||
|
||||
echo '== ISPmanager =='
|
||||
dpkg-query -W -f='${binary:Package} ${Version}\n' 'ispmanager*' 'ispmgr*' 2>/dev/null || true
|
||||
systemctl list-unit-files --type=service --no-legend | grep -iE 'ispmanager|ispmgr' || echo 'No ISPmanager service unit found.'
|
||||
|
||||
echo '== Docker state =='
|
||||
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
|
||||
docker volume ls
|
||||
|
||||
echo '== Gitea Actions runner (read-only) =='
|
||||
if id akyldash-runner >/dev/null 2>&1; then
|
||||
id akyldash-runner
|
||||
else
|
||||
echo 'akyldash-runner user absent'
|
||||
fi
|
||||
if command -v act_runner >/dev/null 2>&1; then
|
||||
act_runner --version
|
||||
else
|
||||
echo 'act_runner binary absent from PATH'
|
||||
fi
|
||||
for path in /etc/akyldash-runner/config.yaml /var/lib/akyldash-runner/.runner /var/lib/akyldash-runner/docker-store.img; do
|
||||
if [[ -e $path ]]; then
|
||||
stat -c '%n owner=%U:%G mode=%a type=%F' "$path"
|
||||
else
|
||||
printf '%s absent\n' "$path"
|
||||
fi
|
||||
done
|
||||
systemctl status akyldash-act-runner.service --no-pager --full || true
|
||||
systemctl status akyldash-runner-healthcheck.timer --no-pager --full || true
|
||||
findmnt /var/lib/akyldash-runner/docker-data || true
|
||||
|
||||
echo '== Firewall and listeners (read-only) =='
|
||||
ufw status verbose
|
||||
ufw status numbered
|
||||
ss -lntup
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -gt 1 || ( $# -eq 1 && $1 != --recovery-target ) ]]; then
|
||||
echo "Usage: $0 [--recovery-target]" >&2
|
||||
exit 2
|
||||
fi
|
||||
recovery_target=false
|
||||
if [[ ${1:-} == --recovery-target ]]; then
|
||||
recovery_target=true
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
|
||||
recovery_hostname=$(hostname -f 2>/dev/null || true)
|
||||
recovery_ipv4=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
|
||||
if [[ -z $recovery_hostname || -z $recovery_ipv4 ]]; then
|
||||
echo "Cannot verify recovery host identity; refusing recovery bootstrap." >&2
|
||||
exit 1
|
||||
fi
|
||||
if akyldash_is_production_host "$recovery_hostname" "$recovery_ipv4"; then
|
||||
echo "Refusing recovery bootstrap on the current production host." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "Run this script as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
if [[ ${ID:-} != ubuntu || ${VERSION_ID:-} != 24.04 ]]; then
|
||||
echo "Expected Ubuntu 24.04; found ${PRETTY_NAME:-unknown}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! id codex >/dev/null 2>&1; then
|
||||
echo "Create the SSH-authorized codex account before running bootstrap." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -e /etc/docker/daemon.json ]] && ! python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' /etc/docker/daemon.json; then
|
||||
echo "/etc/docker/daemon.json exists with different log settings; review it manually." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d -m 0755 /etc/docker
|
||||
if [[ ! -e /etc/docker/daemon.json ]]; then
|
||||
cat > /etc/docker/daemon.json <<'EOF'
|
||||
{
|
||||
"log-driver": "local",
|
||||
"log-opts": {"max-size": "10m", "max-file": "3"}
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates curl git
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu %s stable\n' \
|
||||
"$(dpkg --print-architecture)" "$VERSION_CODENAME" > /etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
|
||||
install -d -o root -g root -m 0755 /opt/akyldash
|
||||
install -d -o root -g root -m 0700 /etc/akyldash
|
||||
install -d -o root -g root -m 0750 /var/lib/akyldash/data
|
||||
install -d -o root -g root -m 0700 /var/backups/akyldash
|
||||
if [[ $recovery_target == true ]]; then
|
||||
install -o root -g root -m 0600 /dev/null /etc/akyldash/recovery-target
|
||||
fi
|
||||
systemctl enable --now docker
|
||||
docker compose version
|
||||
echo "Bootstrap complete. codex was not added to sudo or the docker group; SSH and firewall settings were not changed."
|
||||
if [[ $recovery_target == true ]]; then
|
||||
echo 'Recovery target marker created at /etc/akyldash/recovery-target.'
|
||||
fi
|
||||
@@ -0,0 +1,11 @@
|
||||
akyldash_is_production_host() {
|
||||
local hostname=${1:-}
|
||||
local ipv4_addresses=${2:-}
|
||||
if [[ -z $hostname ]]; then
|
||||
hostname=$(hostname -f 2>/dev/null || true)
|
||||
fi
|
||||
if [[ -z $ipv4_addresses ]]; then
|
||||
ipv4_addresses=$(ip -o -4 addr show 2>/dev/null | awk '{split($4, address, "/"); print address[1]}')
|
||||
fi
|
||||
[[ $hostname == avprogs.hlab.kz ]] || grep -Fqx '206.223.249.118' <<<"$ipv4_addresses"
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if bash "$(dirname "$0")/audit-host.sh" >"$output" 2>&1; then
|
||||
echo 'Host audit unexpectedly succeeded without root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'Run this read-only audit as root.' "$output"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/host-guard.sh"
|
||||
|
||||
akyldash_is_production_host avprogs.hlab.kz 127.0.0.1
|
||||
akyldash_is_production_host other.example $'10.0.0.2\n206.223.249.118'
|
||||
if akyldash_is_production_host recovery.example 10.0.0.2; then
|
||||
echo 'FAIL unrelated recovery host identified as production' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'PASS recovery host identity guard'
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if bash "$(dirname "${BASH_SOURCE[0]}")/verify-helper.sh" unexpected >/dev/null 2>&1; then
|
||||
echo 'FAIL verifier helper accepted command arguments' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'PASS verifier helper rejects command arguments'
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
if [[ $# -ne 0 || $EUID -ne 0 || ! -d $checkout || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
|
||||
echo 'Refusing verification: the approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]]; then
|
||||
echo 'Refusing verification: the checkout contains a writable or non-root-owned path.' >&2
|
||||
exit 1
|
||||
fi
|
||||
exec /bin/bash "$checkout/infra/bootstrap/verify.sh" --remote
|
||||
Executable
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${1:-} != --remote ]]; then
|
||||
ssh_alias=${1:-akyldash_vps}
|
||||
exec ssh -F "${HOME}/.ssh/config" -o BatchMode=yes "$ssh_alias" 'sudo -n /usr/local/sbin/akyldash-verify'
|
||||
fi
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "Run the remote check as root; it is read-only." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for package in git docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do
|
||||
if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -qx 'install ok installed'; then
|
||||
printf 'PASS package %s\n' "$package"
|
||||
else
|
||||
printf 'FAIL package %s missing\n' "$package"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if systemctl is-active --quiet docker; then
|
||||
echo 'PASS Docker service active'
|
||||
else
|
||||
echo 'FAIL Docker service inactive'
|
||||
failed=1
|
||||
fi
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
echo 'PASS Docker Compose responds'
|
||||
else
|
||||
echo 'FAIL Docker Compose unavailable'
|
||||
failed=1
|
||||
fi
|
||||
if python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); assert d.get("log-driver") == "local" and d.get("log-opts",{}).get("max-size") == "10m" and d.get("log-opts",{}).get("max-file") == "3"' 2>/dev/null; then
|
||||
echo 'PASS Docker log rotation configuration'
|
||||
else
|
||||
echo 'FAIL Docker log rotation configuration differs from repository target'
|
||||
failed=1
|
||||
fi
|
||||
if id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
|
||||
echo 'FAIL codex has sudo or Docker group access'
|
||||
failed=1
|
||||
else
|
||||
echo 'PASS codex is outside sudo and docker groups'
|
||||
fi
|
||||
|
||||
for path in /opt/akyldash /etc/akyldash /var/lib/akyldash/data /var/backups/akyldash; do
|
||||
if [[ -d $path ]]; then
|
||||
printf 'PASS directory %s\n' "$path"
|
||||
else
|
||||
printf 'FAIL directory %s missing\n' "$path"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
for path in /etc/akyldash /var/backups/akyldash; do
|
||||
if [[ -d $path && $(stat -c %a "$path") == 700 ]]; then
|
||||
printf 'PASS permissions %s mode 0700\n' "$path"
|
||||
elif [[ -d $path ]]; then
|
||||
printf 'FAIL permissions %s mode %s, expected 0700\n' "$path" "$(stat -c %a "$path")"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
for path in /etc/akyldash/production.env /etc/akyldash/review_secret; do
|
||||
if [[ -f $path && $(stat -c %U:%a "$path") == root:600 ]]; then
|
||||
printf 'PASS protected file %s mode 0600\n' "$path"
|
||||
elif [[ -e $path ]]; then
|
||||
printf 'FAIL protected file %s must be root-owned mode 0600\n' "$path"
|
||||
failed=1
|
||||
else
|
||||
printf 'FAIL protected file %s missing\n' "$path"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
compose_file=$checkout/deploy/production/compose.yaml
|
||||
if git -C "$checkout" diff --quiet HEAD -- deploy/production/compose.yaml deploy/production/Caddyfile deploy/production/Dockerfile.backend deploy/production/Dockerfile.opensearch; then
|
||||
echo 'PASS production Compose, Caddyfile, and Dockerfiles match the checked-out Git revision'
|
||||
else
|
||||
echo 'FAIL production Compose or Dockerfile has local drift'
|
||||
failed=1
|
||||
fi
|
||||
if docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --quiet >/dev/null 2>&1; then
|
||||
echo 'PASS rendered production Compose configuration'
|
||||
else
|
||||
echo 'FAIL production Compose configuration cannot be rendered'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
if ufw status | grep -q '^Status: active$'; then
|
||||
echo 'PASS UFW active'
|
||||
if ! ufw status verbose | grep -q '^Default: deny (incoming)'; then
|
||||
echo 'FAIL UFW default incoming policy is not deny'
|
||||
failed=1
|
||||
fi
|
||||
while read -r rule action direction; do
|
||||
[[ $action == ALLOW && $direction == IN ]] || continue
|
||||
ports=${rule//\/tcp/}
|
||||
ports=${ports//\/udp/}
|
||||
IFS=, read -ra allowed <<< "$ports"
|
||||
for port in "${allowed[@]}"; do
|
||||
case "$port" in
|
||||
22|80|443) ;;
|
||||
*) printf 'FAIL unexpected UFW allow rule %s\n' "$rule"; failed=1 ;;
|
||||
esac
|
||||
done
|
||||
done < <(ufw status | awk 'NR > 3 && NF >= 3 {print $1, $2, $3}')
|
||||
else
|
||||
echo 'FAIL UFW inactive or unavailable'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
tcp_listeners=$(ss -H -lnt | awk '{print $4}')
|
||||
for endpoint in $tcp_listeners; do
|
||||
address=${endpoint%:*}
|
||||
port=${endpoint##*:}
|
||||
case "$address" in 127.*|::1|\[::1\]) continue ;; esac
|
||||
case "$port" in 22|80|443) ;; *) printf 'FAIL unexpected non-loopback TCP listener %s\n' "$endpoint"; failed=1 ;; esac
|
||||
done
|
||||
printf 'INFO non-loopback TCP listeners: %s\n' "${tcp_listeners//$'\n'/,}"
|
||||
|
||||
udp_listeners=$(ss -H -lnu | awk '{print $5}')
|
||||
for endpoint in $udp_listeners; do
|
||||
address=${endpoint%:*}
|
||||
case "$address" in 127.*|::1|\[::1\]) continue ;; esac
|
||||
printf 'FAIL unexpected non-loopback UDP listener %s\n' "$endpoint"
|
||||
failed=1
|
||||
done
|
||||
|
||||
expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
|
||||
if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \
|
||||
-x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \
|
||||
-x /usr/local/sbin/akyldash-compose && $(stat -c %U:%a /usr/local/sbin/akyldash-compose) == root:750 && \
|
||||
-f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \
|
||||
-f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \
|
||||
$(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then
|
||||
echo 'PASS read-only verifier, Compose status/config helper, rollback helper, and exact sudoers rules present'
|
||||
else
|
||||
echo 'FAIL restricted helper or sudoers installation missing/different'
|
||||
failed=1
|
||||
fi
|
||||
if [[ -f /opt/akyldash/deployed-image.txt ]]; then
|
||||
expected=$(cat /opt/akyldash/deployed-image.txt)
|
||||
container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1)
|
||||
actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)
|
||||
if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then
|
||||
printf 'PASS running backend image ID matches %s\n' "$expected"
|
||||
else
|
||||
echo 'FAIL running backend image does not match recorded digest'
|
||||
failed=1
|
||||
fi
|
||||
else
|
||||
echo 'FAIL deployed image digest record missing'
|
||||
failed=1
|
||||
fi
|
||||
|
||||
expected_proxy=$(docker compose --env-file /etc/akyldash/production.env -f "$compose_file" config --format json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["services"]["caddy"]["image"])' 2>/dev/null || true)
|
||||
proxy_container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=caddy | head -n1)
|
||||
actual_proxy=$(docker inspect --format '{{.Config.Image}}' "$proxy_container" 2>/dev/null || true)
|
||||
if [[ $expected_proxy == *@sha256:* && $actual_proxy == "$expected_proxy" ]]; then
|
||||
printf 'PASS running Caddy image matches %s\n' "$expected_proxy"
|
||||
else
|
||||
echo 'FAIL running Caddy image does not match the pinned Compose digest'
|
||||
failed=1
|
||||
fi
|
||||
exit "$failed"
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -ne 1 || ( $1 != config && $1 != status ) ]]; then
|
||||
echo 'Usage: akyldash-compose {config|status}' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo 'Run this helper through its fixed sudoers command.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root ]]; then
|
||||
echo 'Refusing Compose access: approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]]; then
|
||||
echo 'Refusing Compose access: checkout contains a writable or non-root-owned path.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose=(docker compose --env-file /etc/akyldash/production.env -f "$checkout/deploy/production/compose.yaml")
|
||||
case $1 in
|
||||
config) "${compose[@]}" config --quiet ;;
|
||||
status) "${compose[@]}" ps ;;
|
||||
esac
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run this installer as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
rollback_source=$checkout/infra/deploy/rollback.py
|
||||
backup_source=$checkout/infra/backup/archive.py
|
||||
verify_source=$checkout/infra/bootstrap/verify-helper.sh
|
||||
compose_source=$checkout/infra/deploy/compose-helper.sh
|
||||
if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then
|
||||
echo "Install the approved repository checkout at $checkout first." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -L $checkout ]]; then
|
||||
echo "The checkout path must not be a symlink." >&2
|
||||
exit 1
|
||||
fi
|
||||
for directory in /opt /opt/akyldash; do
|
||||
mode=$(stat -c %a "$directory")
|
||||
if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then
|
||||
echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then
|
||||
echo "Create codex without sudo or Docker-group membership before installing the helper." >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then
|
||||
echo "Existing codex sudoers rules need owner review before installing the helper." >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ $(stat -c %U "$checkout") != root || -n $unsafe ]]; then
|
||||
echo "The checkout must be root-owned and not group/world-writable." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v visudo >/dev/null 2>&1; then
|
||||
echo "visudo is required to validate the restricted sudoers rule." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do
|
||||
if [[ -L $path ]]; then
|
||||
echo "Refusing symlink at $path." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
install -d -o root -g root -m 0755 /usr/local/lib/akyldash
|
||||
install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py
|
||||
install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify
|
||||
install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback
|
||||
install -o root -g root -m 0750 "$compose_source" /usr/local/sbin/akyldash-compose
|
||||
|
||||
sudoers_file=/etc/sudoers.d/akyldash-codex
|
||||
expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status'
|
||||
created=false
|
||||
if [[ -e $sudoers_file ]]; then
|
||||
if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then
|
||||
echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX)
|
||||
trap 'rm -f "$temp"' EXIT
|
||||
printf '%s\n' "$expected" > "$temp"
|
||||
chmod 0440 "$temp"
|
||||
visudo -cf "$temp"
|
||||
visudo -c -f /etc/sudoers
|
||||
mv -n "$temp" "$sudoers_file"
|
||||
if [[ $(cat "$sudoers_file") != "$expected" ]]; then
|
||||
echo "Could not install the expected sudoers rule." >&2
|
||||
exit 1
|
||||
fi
|
||||
created=true
|
||||
fi
|
||||
|
||||
if ! visudo -c -f /etc/sudoers; then
|
||||
if [[ $created == true ]]; then
|
||||
rm -f "$sudoers_file"
|
||||
fi
|
||||
echo "Sudoers validation failed; the newly added rule was removed." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Installed fixed read-only verification/Compose commands and guarded rollback; codex receives no general sudo or Docker access."
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo 'Run this configuration step as root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checkout=/opt/akyldash/current
|
||||
example=$checkout/deploy/production/.env.example
|
||||
config_dir=/etc/akyldash
|
||||
env_file=$config_dir/production.env
|
||||
secret_file=$config_dir/review_secret
|
||||
if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root || ! -f $example ]]; then
|
||||
echo 'Refusing configuration: approved root-owned checkout is unavailable.' >&2
|
||||
exit 1
|
||||
fi
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
if [[ -n $unsafe ]] || ! git -C "$checkout" diff --quiet HEAD -- deploy/production/.env.example; then
|
||||
echo 'Refusing configuration: checkout is writable or .env.example differs from Git.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d $config_dir || $(stat -c %U:%G:%a "$config_dir") != root:root:700 ]]; then
|
||||
echo 'Expected /etc/akyldash to be root-owned mode 0700.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -e $env_file || -L $env_file || -e $secret_file || -L $secret_file ]]; then
|
||||
echo 'Production env or review secret already exists; refusing to overwrite either file.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
env_tmp=$(mktemp "$config_dir/.production.env.XXXXXX")
|
||||
secret_tmp=$(mktemp "$config_dir/.review_secret.XXXXXX")
|
||||
trap 'rm -f "$env_tmp" "$secret_tmp"' EXIT
|
||||
install -o root -g root -m 0600 "$example" "$env_tmp"
|
||||
openssl rand -hex 32 > "$secret_tmp"
|
||||
chown root:root "$secret_tmp"
|
||||
chmod 0600 "$secret_tmp"
|
||||
mv -n "$env_tmp" "$env_file"
|
||||
if [[ -e $env_tmp ]]; then
|
||||
echo 'Production env already appeared; refusing to continue.' >&2
|
||||
exit 1
|
||||
fi
|
||||
mv -n "$secret_tmp" "$secret_file"
|
||||
if [[ -e $secret_tmp ]]; then
|
||||
rm -f "$env_file"
|
||||
echo 'Review secret already appeared; removed the new env file and stopped.' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'Created root-owned production.env and review_secret with mode 0600. No services or volumes were changed.'
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore prior local application images from a verified cold backup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path("/opt/akyldash/current")
|
||||
COMPOSE_FILE = ROOT / "deploy/production/compose.yaml"
|
||||
ENV_FILE = Path("/etc/akyldash/production.env")
|
||||
BACKUP_DIR = Path("/var/backups/akyldash")
|
||||
BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py")
|
||||
APP_SERVICES = {"backend", "opensearch"}
|
||||
|
||||
|
||||
def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]:
|
||||
images = manifest.get("images")
|
||||
if not isinstance(images, dict) or not APP_SERVICES <= images.keys():
|
||||
raise ValueError("backup has no previous backend and OpenSearch images")
|
||||
plan = {}
|
||||
for service in sorted(APP_SERVICES):
|
||||
image = images[service]
|
||||
if not isinstance(image, dict):
|
||||
raise ValueError(f"invalid image record for {service}")
|
||||
reference = image.get("configured")
|
||||
image_id = image.get("image_id")
|
||||
if reference != configured.get(service):
|
||||
raise ValueError(f"Compose image reference changed for {service}; refusing rollback")
|
||||
if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*", reference):
|
||||
raise ValueError(f"{service} image reference is not a local tag")
|
||||
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
||||
raise ValueError(f"invalid previous image ID for {service}")
|
||||
plan[service] = (image_id, reference)
|
||||
return plan
|
||||
|
||||
|
||||
def run(command: list[str], *, capture: bool = False) -> str:
|
||||
result = subprocess.run(command, check=True, text=True, capture_output=capture)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def compose(*args: str, capture: bool = False) -> str:
|
||||
return run(
|
||||
["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args],
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
|
||||
def record_backend_image() -> None:
|
||||
container = compose("ps", "--quiet", "backend", capture=True)
|
||||
if not container:
|
||||
raise ValueError("backend container is missing after rollback")
|
||||
image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True)
|
||||
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
||||
raise ValueError("backend returned an invalid image ID")
|
||||
target = Path("/opt/akyldash/deployed-image.txt")
|
||||
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
stream.write(f"{image_id}\n".encode("ascii"))
|
||||
os.chmod(temporary, 0o600)
|
||||
try:
|
||||
os.replace(temporary, target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def rollback(backup: Path, data_compatible: bool) -> None:
|
||||
if not data_compatible:
|
||||
raise ValueError("confirm data compatibility with --data-compatible before rollback")
|
||||
if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve():
|
||||
raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}")
|
||||
if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("rollback backup must be root-owned mode 0600")
|
||||
checksum = Path(f"{backup}.sha256")
|
||||
if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600:
|
||||
raise ValueError("rollback checksum sidecar must be root-owned mode 0600")
|
||||
|
||||
sys.path.insert(0, str(BACKUP_TOOL.parent))
|
||||
from archive import digest, verify_archive
|
||||
|
||||
manifest = verify_archive(backup)
|
||||
deployment_files = {
|
||||
relative: digest(ROOT / relative)
|
||||
for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
|
||||
}
|
||||
if deployment_files != manifest.get("deployment_files"):
|
||||
raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first")
|
||||
rendered = json.loads(compose("config", "--format", "json", capture=True))
|
||||
configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES}
|
||||
plan = image_restore_plan(manifest, configured)
|
||||
for image_id, _reference in plan.values():
|
||||
actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True)
|
||||
if actual != image_id:
|
||||
raise ValueError(f"previous image is unavailable: {image_id}")
|
||||
|
||||
backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz"
|
||||
backup_output = BACKUP_DIR / backup_name
|
||||
compose("stop")
|
||||
try:
|
||||
run(
|
||||
[
|
||||
"python3",
|
||||
str(BACKUP_TOOL),
|
||||
"create",
|
||||
"--compose-file",
|
||||
str(COMPOSE_FILE),
|
||||
"--env-file",
|
||||
str(ENV_FILE),
|
||||
"--output",
|
||||
str(backup_output),
|
||||
]
|
||||
)
|
||||
run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)])
|
||||
except Exception:
|
||||
compose("up", "--detach")
|
||||
raise
|
||||
|
||||
for image_id, reference in plan.values():
|
||||
run(["docker", "image", "tag", image_id, reference])
|
||||
compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300")
|
||||
run(
|
||||
[
|
||||
"curl",
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--resolve",
|
||||
"akyldash.kg:443:127.0.0.1",
|
||||
"https://akyldash.kg/review",
|
||||
"--output",
|
||||
"/dev/null",
|
||||
]
|
||||
)
|
||||
record_backend_image()
|
||||
print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("backup", type=Path)
|
||||
parser.add_argument("--data-compatible", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if os.geteuid() != 0:
|
||||
raise ValueError("rollback must run through the restricted root helper")
|
||||
rollback(args.backup, args.data_compatible)
|
||||
except Exception as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
helper=$(dirname "$0")/compose-helper.sh
|
||||
for action in deploy up down docker; do
|
||||
if bash "$helper" "$action" >/dev/null 2>&1; then
|
||||
printf 'FAIL Compose helper accepted %s\n' "$action" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo 'PASS Compose helper rejects operations outside its read-only allowlist'
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if bash "$(dirname "$0")/prepare-production-config.sh" >"$output" 2>&1; then
|
||||
echo 'Production config setup unexpectedly succeeded without root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'Run this configuration step as root.' "$output"
|
||||
@@ -0,0 +1,36 @@
|
||||
import unittest
|
||||
|
||||
from rollback import image_restore_plan
|
||||
|
||||
|
||||
class RollbackPlanTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.manifest = {
|
||||
"images": {
|
||||
"backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64},
|
||||
"opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64},
|
||||
}
|
||||
}
|
||||
self.configured = {
|
||||
"backend": "akyldash/backend:local",
|
||||
"opensearch": "akyldash/opensearch:local",
|
||||
}
|
||||
|
||||
def test_plans_only_recorded_local_images(self):
|
||||
plan = image_restore_plan(self.manifest, self.configured)
|
||||
self.assertEqual(plan["backend"][1], "akyldash/backend:local")
|
||||
self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local")
|
||||
|
||||
def test_refuses_changed_compose_image_reference(self):
|
||||
with self.assertRaisesRegex(ValueError, "reference changed"):
|
||||
image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"})
|
||||
|
||||
def test_refuses_digest_reference_that_cannot_be_retagged(self):
|
||||
self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64
|
||||
self.configured["backend"] = self.manifest["images"]["backend"]["configured"]
|
||||
with self.assertRaisesRegex(ValueError, "local tag"):
|
||||
image_restore_plan(self.manifest, self.configured)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
project: akyldash
|
||||
environment: production
|
||||
ssh:
|
||||
alias: akyldash_vps
|
||||
host: 206.223.249.118
|
||||
port: 22
|
||||
user: codex
|
||||
identity_file: ~/.ssh/akyldash_codex
|
||||
domain: akyldash.kg
|
||||
facts:
|
||||
hostname: avprogs.hlab.kz
|
||||
os: Ubuntu Server 24.04.5 LTS
|
||||
kernel: 6.8.0-142-generic
|
||||
cpu_vcpus: 4
|
||||
memory_gib: 7.7
|
||||
swap_gib: 3.8
|
||||
disk_gib: 99
|
||||
disk_free_gib: 92
|
||||
network_interface: ens3
|
||||
ipv4: 206.223.249.118/32
|
||||
domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses
|
||||
domain_http_status: Cloudflare 521 on HTTP and HTTPS from the VPS
|
||||
user_groups: [codex, users]
|
||||
sudo_noninteractive: false
|
||||
installed:
|
||||
- OpenSSH server
|
||||
- systemd-resolved
|
||||
- unattended-upgrades
|
||||
- Docker CE and CLI 29.9.0
|
||||
- Docker Compose plugin 5.6.0
|
||||
- containerd 2.4.1
|
||||
- runc 1.5.2
|
||||
- Git
|
||||
absent:
|
||||
- ISPmanager package/service (unprivileged package and unit checks)
|
||||
- root-owned application checkout
|
||||
- restricted verifier and rollback helper
|
||||
- non-loopback listeners on TCP 80, 443, 1500
|
||||
access_audit_date: 2026-10-12
|
||||
docker_service_active: true
|
||||
docker_api_accessible_to_codex: false
|
||||
docker_daemon_log_config:
|
||||
driver: local
|
||||
max_size: 10m
|
||||
max_file: 3
|
||||
bootstrap_paths:
|
||||
/opt/akyldash: root:root 0755
|
||||
/etc/akyldash: root:root 0700
|
||||
/var/lib/akyldash/data: root:root 0750
|
||||
/var/backups/akyldash: root:root 0700
|
||||
unprivileged_nonloopback_tcp_listeners: [22]
|
||||
owner_reported_ufw_allowed_tcp: [22, 80, 443, 1500]
|
||||
port_1500_listener: false
|
||||
rootless_docker_command_present: /usr/bin/dockerd-rootless.sh
|
||||
runner_gitea_host_resolution: git.vakanaut.ru resolves to 195.209.48.28
|
||||
runner_gitea_https_probe: /api/v1/version returned Gitea 1.25.5 (Actions protocol unverified)
|
||||
cloudflare_edge_status:
|
||||
http: 521
|
||||
https: 521
|
||||
target:
|
||||
compose_file_in_repository: deploy/production/compose.yaml
|
||||
compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml
|
||||
production_env_file: /etc/akyldash/production.env
|
||||
secrets_directory: /etc/akyldash
|
||||
data_directory: /var/lib/akyldash/data
|
||||
backup_directory: /var/backups/akyldash
|
||||
services:
|
||||
caddy:
|
||||
container: compose project service caddy
|
||||
image: caddy:2.11.6-alpine pinned by digest in deploy/production/compose.yaml
|
||||
host_ports: [80, 443]
|
||||
persistent_data: named Compose volumes caddy-data and caddy-config
|
||||
backend:
|
||||
container: compose project service backend
|
||||
image: built from deploy/production/Dockerfile.backend; digest deployment not configured
|
||||
port: 8080 on private proxy and search Compose networks only
|
||||
healthcheck: HTTP GET /review on container port 8080
|
||||
persistent_data: /var/lib/akyldash/data (bind mount)
|
||||
opensearch:
|
||||
container: compose project service opensearch
|
||||
image: opensearchproject/opensearch:3.7.0 plus analysis-icu plugin
|
||||
port: 9200 on private Compose network only
|
||||
persistent_data: named Compose volume opensearch-data
|
||||
reverse_proxy:
|
||||
target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080
|
||||
status: no listener observed on 80/443; container state needs root Docker audit
|
||||
external_ports_target: [22, 80, 443]
|
||||
compose_networks:
|
||||
proxy: caddy and backend
|
||||
search: backend and OpenSearch; internal network
|
||||
secrets:
|
||||
- REVIEW_SECRET (host file mounted as /run/secrets/review_secret)
|
||||
- Gitea credentials from environment-backed gitea-api helper
|
||||
- SSH key at ~/.ssh/akyldash_codex
|
||||
restricted_helper:
|
||||
path: /usr/local/sbin/akyldash-rollback
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
compose_read_only_helper:
|
||||
path: /usr/local/sbin/akyldash-compose
|
||||
allowed_operations: [config, status]
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
read_only_verifier:
|
||||
path: /usr/local/sbin/akyldash-verify
|
||||
status: repository code and owner-run installer exist; not installed on VPS
|
||||
gitea_actions_runner:
|
||||
repository: admin/akyldash
|
||||
registration_scope: repository
|
||||
implementation: official act_runner 0.2.13; pinned binary SHA-256 in infra/runner/install-runner.sh
|
||||
label: akyldash-ci
|
||||
capacity: 1
|
||||
job_limits: {cpu: 2, memory_gib: 3, pids: 512}
|
||||
account_slice_limits: {cpu: 2.5, memory_gib: 4, tasks: 2048}
|
||||
docker_store_limit_gib: 15
|
||||
docker_boundary: rootless dedicated daemon; no production socket; job socket injection disabled
|
||||
service_status: configuration prepared, not installed or registered; Online unverified
|
||||
owner_action: run the root installer from the reviewed checkout; register through repository runner settings
|
||||
checks:
|
||||
- infra/bootstrap/verify.sh akyldash_vps
|
||||
- docker compose -f deploy/production/compose.yaml ps
|
||||
- curl -fsS https://akyldash.kg/review
|
||||
- python3 infra/backup/archive.py verify /var/backups/akyldash/<archive>.tar.gz
|
||||
backup_path: /var/backups/akyldash (directory exists; archive script in infra/backup/archive.py is not installed or run)
|
||||
deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment)
|
||||
versions:
|
||||
application:
|
||||
backend: 0.11.3
|
||||
frontend: 0.1.2
|
||||
telegram_bot: 0.2.2
|
||||
infrastructure: unknown; no production install or release exists
|
||||
caddy_image: 2.11.6-alpine pinned by manifest digest in Compose
|
||||
docker_engine: 29.9.0 (daemon active; server API unverified as codex)
|
||||
docker_compose: 5.6.0
|
||||
containerd: 2.4.1
|
||||
runc: 1.5.2
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
systemctl is-active --quiet akyldash-act-runner.service
|
||||
runner_uid=$(id -u akyldash-runner)
|
||||
runuser -u akyldash-runner -- env \
|
||||
HOME=/var/lib/akyldash-runner/.config/act-runner \
|
||||
XDG_CONFIG_HOME=/var/lib/akyldash-runner/.config \
|
||||
XDG_DATA_HOME=/var/lib/akyldash-runner/.config/act-runner/.local/share \
|
||||
DOCKER_HOST="unix:///run/user/$runner_uid/docker.sock" \
|
||||
/usr/bin/docker info --format '{{.ServerVersion}}' >/dev/null
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Check AkylDash Gitea Actions runner health
|
||||
OnFailure=akyldash-runner-restart.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/akyldash-runner-healthcheck
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Periodically check AkylDash Gitea Actions runner health
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2m
|
||||
OnUnitActiveSec=1m
|
||||
Unit=akyldash-runner-healthcheck.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=Restart AkylDash Gitea Actions runner after a failed health check
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl restart akyldash-act-runner.service
|
||||
@@ -0,0 +1,15 @@
|
||||
log:
|
||||
level: info
|
||||
runner:
|
||||
file: /var/lib/akyldash-runner/.config/act-runner/.runner
|
||||
capacity: 1
|
||||
labels:
|
||||
- "akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04"
|
||||
container:
|
||||
privileged: false
|
||||
options: --cpus=2 --memory=3g --pids-limit=512
|
||||
valid_volumes: []
|
||||
docker_host: "-"
|
||||
force_pull: true
|
||||
cache:
|
||||
enabled: false
|
||||
@@ -0,0 +1,22 @@
|
||||
BEGIN { FS = ":" }
|
||||
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
|
||||
NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1 { invalid = 1; next }
|
||||
{
|
||||
start[FILENAME, ++count[FILENAME]] = $2
|
||||
end[FILENAME, count[FILENAME]] = $2 + $3 - 1
|
||||
files[FILENAME] = 1
|
||||
}
|
||||
END {
|
||||
if (invalid) exit 2
|
||||
for (base = 100000; base + 65535 <= 4294901760; base += 65536) {
|
||||
overlap = 0
|
||||
for (file in files)
|
||||
for (i = 1; i <= count[file]; i++)
|
||||
if (start[file, i] <= base + 65535 && end[file, i] >= base) overlap = 1
|
||||
if (!overlap) {
|
||||
print base "-" base + 65535
|
||||
exit
|
||||
}
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
runner_die() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
|
||||
|
||||
runner_ensure_account() {
|
||||
local user=$1 home=$2 entry uid passwd_home shell groups uid_min
|
||||
if ! entry=$(getent passwd "$user"); then
|
||||
[[ ! -e $home && ! -L $home ]] || { runner_die "runner home already exists without its account: $home"; return 1; }
|
||||
useradd --system --create-home --home-dir "$home" --shell /usr/sbin/nologin "$user" || return 1
|
||||
return
|
||||
fi
|
||||
IFS=: read -r _ _ uid _ _ passwd_home shell <<<"$entry"
|
||||
uid_min=$(awk '$1 == "UID_MIN" && $2 ~ /^[0-9]+$/ {print $2; exit}' /etc/login.defs)
|
||||
[[ $uid =~ ^[0-9]+$ && $uid -gt 0 && $uid_min =~ ^[0-9]+$ && $uid -lt $uid_min && $passwd_home == "$home" && $shell == /usr/sbin/nologin ]] || { runner_die "existing $user account has an unexpected UID, home, or shell"; return 1; }
|
||||
[[ $(id -gn "$user") == "$user" ]] || { runner_die "existing $user account has an unexpected primary group"; return 1; }
|
||||
groups=$(id -nG "$user")
|
||||
[[ $groups == "$user" ]] || { runner_die "existing $user account has unexpected supplementary groups: $groups"; return 1; }
|
||||
[[ -d $home && ! -L $home ]] || { runner_die "existing $user home is missing or has unexpected type"; return 1; }
|
||||
case $(stat -c %U "$home") in
|
||||
"$user") ;;
|
||||
root) [[ $(stat -c '%G:%a' "$home") == root:711 ]] || { runner_die "root-owned $user home must be mode 0711"; return 1; } ;;
|
||||
*) runner_die "existing $user home has unexpected ownership"; return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
runner_lock_home() {
|
||||
local home=$1 parent_mode
|
||||
[[ -d $home && ! -L $home && $(stat -c %U /var/lib) == root ]] || { runner_die "runner home or its parent is unsafe: $home"; return 1; }
|
||||
parent_mode=$(stat -c %a /var/lib)
|
||||
(( (8#$parent_mode & 022) == 0 )) || { runner_die '/var/lib must not be group/world-writable'; return 1; }
|
||||
chown root:root "$home" && chmod 0711 "$home" || return 1
|
||||
[[ $(stat -c '%U:%G:%a' "$home") == root:root:711 ]] || { runner_die 'runner home could not be locked from runner writes'; return 1; }
|
||||
}
|
||||
|
||||
runner_ensure_image() {
|
||||
local image=$1 bytes=$2 parent_uid=$3 parent_mode=$4
|
||||
python3 "$(dirname "${BASH_SOURCE[0]}")/secure-runner-image.py" "$image" "$bytes" "$parent_uid" "$parent_mode" || { runner_die "runner image validation or initialization failed: $image"; return 1; }
|
||||
}
|
||||
|
||||
runner_install_file() {
|
||||
local source=$1 destination=$2 owner=$3 group=$4 mode=${5#0}
|
||||
if [[ -e $destination || -L $destination ]]; then
|
||||
[[ -f $destination && ! -L $destination ]] && cmp -s "$source" "$destination" || { runner_die "existing runner file differs from reviewed configuration: $destination"; return 1; }
|
||||
[[ $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { runner_die "existing runner file has unexpected ownership or permissions: $destination"; return 1; }
|
||||
return
|
||||
fi
|
||||
install -o "$owner" -g "$group" -m "0$mode" "$source" "$destination"
|
||||
}
|
||||
|
||||
runner_install_content() {
|
||||
local destination=$1 owner=$2 group=$3 mode=$4 temporary
|
||||
temporary=$(mktemp)
|
||||
cat >"$temporary"
|
||||
if ! runner_install_file "$temporary" "$destination" "$owner" "$group" "$mode"; then
|
||||
rm -f "$temporary"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$temporary"
|
||||
}
|
||||
|
||||
runner_ensure_private_directory() {
|
||||
local directory=$1 owner=$2 group=$3 current= component index
|
||||
local -a components
|
||||
[[ $directory == /* ]] || { runner_die "runner directory path is not absolute: $directory"; return 1; }
|
||||
IFS=/ read -r -a components <<<"${directory#/}"
|
||||
for index in "${!components[@]}"; do
|
||||
component=${components[index]}
|
||||
[[ -n $component && $component != . && $component != .. ]] || { runner_die "runner directory path has an unsafe component: $directory"; return 1; }
|
||||
current+="/$component"
|
||||
[[ ! -L $current ]] || { runner_die "runner directory path contains a symbolic link: $current"; return 1; }
|
||||
if [[ ! -e $current ]]; then
|
||||
((index == ${#components[@]} - 1)) || { runner_die "runner directory parent is missing: $current"; return 1; }
|
||||
break
|
||||
fi
|
||||
[[ -d $current ]] || { runner_die "runner directory path component is not a directory: $current"; return 1; }
|
||||
if [[ $current == /var/lib/akyldash-runner ]]; then
|
||||
[[ $(stat -c '%U:%G:%a' "$current") == root:root:711 ]] || { runner_die 'runner home must be root-owned mode 0711'; return 1; }
|
||||
elif [[ $current == /var/lib/akyldash-runner/* ]]; then
|
||||
[[ $(stat -c '%U:%G' "$current") == "$owner:$group" ]] || { runner_die "runner directory ancestor has unexpected ownership: $current"; return 1; }
|
||||
fi
|
||||
done
|
||||
# These paths are runner-writable; make changes as that user so a path swap
|
||||
# cannot turn a root operation into a write outside its home.
|
||||
if [[ $directory == /var/lib/akyldash-runner/.config ]]; then
|
||||
install -d -o "$owner" -g "$group" -m 0700 "$directory" || return 1
|
||||
else
|
||||
runuser -u "$owner" -- install -d -m 0700 "$directory" || return 1
|
||||
fi
|
||||
[[ -d $directory && ! -L $directory && $(stat -c '%U:%G:%a' "$directory") == "$owner:$group:700" ]] || { runner_die "runner directory failed validation: $directory"; return 1; }
|
||||
}
|
||||
|
||||
runner_install_user_content() {
|
||||
local destination=$1 owner=$2 group=$3 mode=${4#0} temporary
|
||||
temporary=$(mktemp)
|
||||
cat >"$temporary"
|
||||
if [[ -e $destination || -L $destination ]]; then
|
||||
[[ -f $destination && ! -L $destination ]] && cmp -s "$temporary" "$destination" || { rm -f "$temporary"; runner_die "existing runner file differs from reviewed configuration: $destination"; return 1; }
|
||||
[[ $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { rm -f "$temporary"; runner_die "existing runner file has unexpected ownership or permissions: $destination"; return 1; }
|
||||
rm -f "$temporary"
|
||||
return
|
||||
fi
|
||||
chown "$owner:$group" "$temporary" && chmod "0$mode" "$temporary" || { rm -f "$temporary"; return 1; }
|
||||
local result=0
|
||||
runuser -u "$owner" -- install -m "0$mode" "$temporary" "$destination" || result=$?
|
||||
rm -f "$temporary"
|
||||
((result == 0)) || return 1
|
||||
[[ -f $destination && ! -L $destination && $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { runner_die "runner file failed validation: $destination"; return 1; }
|
||||
}
|
||||
|
||||
runner_validate_mount() {
|
||||
local image=$1 store=$2 target filesystem source options backing mount_info
|
||||
mount_info=$(findmnt --noheadings --output TARGET,FSTYPE,SOURCE,OPTIONS --mountpoint "$store") || { runner_die 'runner store is not mounted'; return 1; }
|
||||
read -r target filesystem source options <<<"$mount_info" || { runner_die 'runner store mount information is empty'; return 1; }
|
||||
[[ $target == "$store" && $filesystem == ext4 && $source == /dev/loop* ]] || { runner_die 'runner store is mounted from an unexpected target, filesystem, or device'; return 1; }
|
||||
case ",$options," in *,nosuid,*nodev,*|*,nodev,*nosuid,*) ;; *) runner_die 'runner store mount is missing nosuid or nodev'; return 1 ;; esac
|
||||
backing=$(losetup --noheadings --output BACK-FILE "$source") || { runner_die 'cannot identify the runner loop backing file'; return 1; }
|
||||
[[ $(realpath -e "$backing") == "$(realpath -e "$image")" ]] || { runner_die 'runner store is backed by a different image'; return 1; }
|
||||
}
|
||||
|
||||
runner_ensure_mount() {
|
||||
local image=$1 store=$2 unit_name=$3 unit_file=$4 unit_tmp mounted=0
|
||||
[[ -d $store && ! -L $store ]] || { runner_die "runner mount point is missing or is not a directory: $store"; return 1; }
|
||||
if mountpoint -q "$store"; then
|
||||
runner_validate_mount "$image" "$store" || return 1
|
||||
mounted=1
|
||||
fi
|
||||
unit_tmp=$(mktemp)
|
||||
if ! cat >"$unit_tmp" <<EOF
|
||||
[Unit]
|
||||
Description=Bounded storage for the AkylDash CI runner
|
||||
Before=local-fs.target
|
||||
|
||||
[Mount]
|
||||
What=$image
|
||||
Where=$store
|
||||
Type=ext4
|
||||
Options=loop,nosuid,nodev
|
||||
|
||||
[Install]
|
||||
WantedBy=local-fs.target
|
||||
EOF
|
||||
then
|
||||
rm -f "$unit_tmp"
|
||||
runner_die 'failed to prepare runner mount unit'
|
||||
return 1
|
||||
fi
|
||||
if [[ -e $unit_file || -L $unit_file ]]; then
|
||||
if [[ ! -f $unit_file || -L $unit_file ]] || ! cmp -s "$unit_tmp" "$unit_file"; then
|
||||
rm -f "$unit_tmp"
|
||||
runner_die "existing runner mount unit differs from the reviewed configuration: $unit_file"
|
||||
return 1
|
||||
fi
|
||||
[[ $(stat -c '%u:%g:%a' "$unit_file") == "$(id -u):$(id -g):644" ]] || { rm -f "$unit_tmp"; runner_die 'existing runner mount unit has unexpected ownership or permissions'; return 1; }
|
||||
else
|
||||
install -m 0644 "$unit_tmp" "$unit_file" || { rm -f "$unit_tmp"; return 1; }
|
||||
fi
|
||||
rm -f "$unit_tmp"
|
||||
|
||||
systemctl daemon-reload || return 1
|
||||
if ((mounted)); then
|
||||
systemctl enable "$unit_name" || return 1
|
||||
else
|
||||
systemctl enable "$unit_name" || return 1
|
||||
systemctl start "$unit_name" || return 1
|
||||
runner_validate_mount "$image" "$store" || return 1
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly runner_user=akyldash-runner
|
||||
readonly runner_home=/var/lib/akyldash-runner
|
||||
readonly runner_runtime_home="$runner_home/.config/act-runner"
|
||||
readonly runner_xdg_config_home="$runner_home/.config"
|
||||
readonly runner_xdg_data_home="$runner_runtime_home/.local/share"
|
||||
readonly runner_store="$runner_home/docker-data"
|
||||
readonly runner_image="$runner_home/docker-store.img"
|
||||
readonly runner_mount_unit_name="$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)"
|
||||
readonly runner_mount_unit="/etc/systemd/system/$runner_mount_unit_name"
|
||||
readonly runner_image_bytes=16106127360
|
||||
readonly runner_version=0.2.13
|
||||
readonly runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2
|
||||
readonly runner_url="https://dl.gitea.com/gitea-runner/$runner_version/act_runner-$runner_version-linux-amd64"
|
||||
readonly checkout=/opt/akyldash/current
|
||||
|
||||
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
[[ $EUID -eq 0 ]] || die 'run this script as root'
|
||||
[[ -r /etc/os-release ]] || die 'cannot identify operating system'
|
||||
. /etc/os-release
|
||||
[[ ${ID:-} == ubuntu && ${VERSION_ID:-} == 24.04 ]] || die 'Ubuntu 24.04 is required'
|
||||
[[ $(uname -m) == x86_64 ]] || die 'this pinned binary is for x86_64 only'
|
||||
[[ $(stat -fc %T /sys/fs/cgroup) == cgroup2fs ]] || die 'rootless Docker requires cgroup v2'
|
||||
[[ $(findmnt -n -o FSTYPE /) == ext4 ]] || die 'the bounded runner store currently requires ext4 on /'
|
||||
[[ -d $checkout/.git && ! -L $checkout && $(stat -c %U "$checkout") == root ]] || die 'install the reviewed root-owned checkout at /opt/akyldash/current first'
|
||||
for directory in /opt /opt/akyldash; do
|
||||
[[ $(stat -c %U "$directory") == root ]] || die "$directory must be owned by root"
|
||||
mode=$(stat -c %a "$directory")
|
||||
(( (8#$mode & 022) == 0 )) || die "$directory must not be group/world-writable"
|
||||
done
|
||||
unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
|
||||
[[ -z $unsafe ]] || die 'the repository checkout contains files writable by non-root users'
|
||||
git -C "$checkout" diff --quiet HEAD -- infra/runner || die 'runner files differ from the reviewed Git revision'
|
||||
source "$checkout/infra/runner/install-runner-lib.sh"
|
||||
if systemctl is-active --quiet akyldash-act-runner.service; then
|
||||
die 'stop akyldash-act-runner.service after all jobs finish before rerunning this installer'
|
||||
fi
|
||||
[[ -r /etc/subuid && -r /etc/subgid ]] || die 'subordinate ID allocation files are unavailable'
|
||||
if getent passwd "$runner_user" >/dev/null; then
|
||||
runner_ensure_account "$runner_user" "$runner_home" || die 'runner account validation failed'
|
||||
runner_lock_home "$runner_home" || die 'runner home protection failed'
|
||||
else
|
||||
[[ ! -e $runner_home && ! -L $runner_home ]] || die 'runner home exists without its account; inspect it before continuing'
|
||||
fi
|
||||
if [[ -e $runner_image || -L $runner_image ]]; then
|
||||
runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711 || die 'runner image validation failed'
|
||||
else
|
||||
[[ $(df -Pk / | awk 'NR==2 {print $4}') -ge 25000000 ]] || die 'need at least 25 GiB free before reserving 15 GiB for runner data'
|
||||
fi
|
||||
|
||||
for file in /etc/subuid /etc/subgid; do
|
||||
awk -F: '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ && (NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1) { bad=1 } END { exit bad }' "$file" || die "$file contains an invalid subordinate ID entry"
|
||||
done
|
||||
subuid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subuid) || die 'runner has multiple subuid allocations; inspect them before continuing'
|
||||
subgid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subgid) || die 'runner has multiple subgid allocations; inspect them before continuing'
|
||||
for pair in "$subuid_pair" "$subgid_pair"; do
|
||||
[[ -z $pair || $pair =~ ^[0-9]+:65536$ ]] || die 'existing runner subordinate ID allocation is not a single 65536-ID range'
|
||||
done
|
||||
if [[ -n $subuid_pair && -n $subgid_pair && $subuid_pair != "$subgid_pair" ]]; then
|
||||
die 'runner subuid and subgid allocations differ; inspect them before continuing'
|
||||
fi
|
||||
existing_pair=${subuid_pair:-$subgid_pair}
|
||||
if [[ -n $existing_pair ]]; then
|
||||
subid_start=${existing_pair%%:*}
|
||||
[[ $subid_start =~ ^[1-9][0-9]*$ && $subid_start -ge 100000 && $subid_start -le 4294901760 ]] || die 'existing runner subordinate ID range is outside the supported range'
|
||||
subid_end=$((subid_start + 65535))
|
||||
subid_range="$subid_start-$subid_end"
|
||||
else
|
||||
subid_range=$(awk -f "$(dirname "$0")/find-free-subid-range.awk" /etc/subuid /etc/subgid) || die 'could not find a free 65536-ID range shared by /etc/subuid and /etc/subgid'
|
||||
subid_start=${subid_range%-*}
|
||||
subid_end=${subid_range#*-}
|
||||
fi
|
||||
for file in /etc/subuid /etc/subgid; do
|
||||
awk -F: -v user="$runner_user" -v start="$subid_start" -v end="$subid_end" '$1 != user && $2 <= end && $2 + $3 - 1 >= start { conflict=1 } END { exit conflict }' "$file" || die "runner subordinate ID range conflicts with another entry in $file"
|
||||
done
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate docker-ce-rootless-extras
|
||||
|
||||
runner_ensure_account "$runner_user" "$runner_home" || die 'runner account creation failed'
|
||||
runner_lock_home "$runner_home" || die 'runner home protection failed'
|
||||
subid_args=()
|
||||
[[ -n $subuid_pair ]] || subid_args+=(--add-subuids "$subid_range")
|
||||
[[ -n $subgid_pair ]] || subid_args+=(--add-subgids "$subid_range")
|
||||
if ((${#subid_args[@]})); then
|
||||
usermod "${subid_args[@]}" "$runner_user"
|
||||
fi
|
||||
runner_uid=$(id -u "$runner_user")
|
||||
|
||||
runner_ensure_private_directory "$runner_home/.config" "$runner_user" "$runner_user" || die 'runner config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/docker" "$runner_user" "$runner_user" || die 'runner Docker config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd" "$runner_user" "$runner_user" || die 'runner systemd config directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd/user" "$runner_user" "$runner_user" || die 'runner user systemd directory failed validation'
|
||||
runner_ensure_private_directory "$runner_home/.config/systemd/user/docker.service.d" "$runner_user" "$runner_user" || die 'runner Docker override directory failed validation'
|
||||
runner_ensure_private_directory "$runner_runtime_home" "$runner_user" "$runner_user" || die 'runner runtime home failed directory validation'
|
||||
runner_ensure_private_directory "$runner_runtime_home/.local" "$runner_user" "$runner_user" || die 'runner XDG data parent failed directory validation'
|
||||
runner_ensure_private_directory "$runner_xdg_data_home" "$runner_user" "$runner_user" || die 'runner XDG data directory failed directory validation'
|
||||
if [[ -e $runner_home/.runner || -L $runner_home/.runner ]]; then
|
||||
[[ -f $runner_home/.runner && ! -L $runner_home/.runner ]] || die 'legacy runner registration state is not a regular file'
|
||||
if [[ ! -e $runner_runtime_home/.runner && ! -L $runner_runtime_home/.runner ]]; then
|
||||
runner_install_user_content "$runner_runtime_home/.runner" "$runner_user" "$runner_user" 0600 <"$runner_home/.runner" || die 'legacy runner registration state could not be preserved'
|
||||
fi
|
||||
fi
|
||||
if [[ ! -e $runner_store && ! -L $runner_store ]]; then
|
||||
install -d -o root -g root -m 0755 "$runner_store"
|
||||
fi
|
||||
[[ -d $runner_store && ! -L $runner_store ]] || die 'runner mount path is not a directory'
|
||||
install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner
|
||||
install -d -o "$runner_user" -g "$runner_user" -m 0750 /var/log/akyldash-runner
|
||||
|
||||
runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711 || die 'runner image initialization failed'
|
||||
runner_ensure_mount "$runner_image" "$runner_store" "$runner_mount_unit_name" "$runner_mount_unit" || die 'runner mount activation failed'
|
||||
chown "$runner_user:$runner_user" "$runner_store"
|
||||
chmod 0700 "$runner_store"
|
||||
|
||||
runner_install_file "$(dirname "$0")/act-runner-healthcheck.sh" /usr/local/sbin/akyldash-runner-healthcheck root root 0755 || die 'healthcheck command differs from reviewed configuration'
|
||||
runner_install_file "$(dirname "$0")/config.yaml" /etc/akyldash-runner/config.yaml root "$runner_user" 0640 || die 'runner config differs from reviewed configuration'
|
||||
if [[ -e /usr/local/bin/act_runner || -L /usr/local/bin/act_runner ]]; then
|
||||
[[ -f /usr/local/bin/act_runner && ! -L /usr/local/bin/act_runner ]] || die 'existing act_runner path is not a regular file'
|
||||
printf '%s %s\n' "$runner_sha256" /usr/local/bin/act_runner | sha256sum --check --status || die 'existing act_runner binary does not match the pinned release'
|
||||
else
|
||||
tmp_binary=$(mktemp)
|
||||
trap 'rm -f "$tmp_binary"' EXIT
|
||||
curl --fail --location --silent --show-error "$runner_url" -o "$tmp_binary"
|
||||
printf '%s %s\n' "$runner_sha256" "$tmp_binary" | sha256sum --check --status || die 'official act_runner checksum mismatch'
|
||||
install -o root -g root -m 0755 "$tmp_binary" /usr/local/bin/act_runner
|
||||
rm -f "$tmp_binary"
|
||||
trap - EXIT
|
||||
fi
|
||||
|
||||
runner_install_user_content "$runner_home/.config/docker/daemon.json" "$runner_user" "$runner_user" 0600 <<EOF
|
||||
{
|
||||
"data-root": "$runner_store",
|
||||
"log-driver": "local",
|
||||
"log-opts": {"max-size": "10m", "max-file": "2"}
|
||||
}
|
||||
EOF
|
||||
runner_install_user_content "$runner_home/.config/systemd/user/docker.service.d/limits.conf" "$runner_user" "$runner_user" 0644 <<'EOF'
|
||||
[Service]
|
||||
Delegate=yes
|
||||
CPUQuota=200%
|
||||
MemoryMax=4G
|
||||
TasksMax=1024
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
EOF
|
||||
runner_install_user_content "$runner_home/.config/systemd/user/docker.service.d/environment.conf" "$runner_user" "$runner_user" 0644 <<EOF
|
||||
[Service]
|
||||
Environment=HOME=$runner_runtime_home
|
||||
Environment=XDG_CONFIG_HOME=$runner_xdg_config_home
|
||||
Environment=XDG_DATA_HOME=$runner_xdg_data_home
|
||||
EOF
|
||||
runner_install_content /etc/systemd/system/akyldash-act-runner.service root root 0644 <<EOF
|
||||
[Unit]
|
||||
Description=AkylDash repository Gitea Actions runner
|
||||
After=network-online.target local-fs.target
|
||||
Wants=network-online.target
|
||||
RequiresMountsFor=$runner_store
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Slice=user-$runner_uid.slice
|
||||
User=$runner_user
|
||||
Group=$runner_user
|
||||
Environment=HOME=$runner_runtime_home
|
||||
Environment=XDG_CONFIG_HOME=$runner_xdg_config_home
|
||||
Environment=XDG_DATA_HOME=$runner_xdg_data_home
|
||||
WorkingDirectory=$runner_runtime_home
|
||||
Environment=XDG_RUNTIME_DIR=/run/user/$runner_uid
|
||||
Environment=DOCKER_HOST=unix:///run/user/$runner_uid/docker.sock
|
||||
ExecStartPre=/bin/sh -c 'for i in \$(seq 1 60); do /usr/bin/docker info >/dev/null 2>&1 && exit 0; sleep 2; done; exit 1'
|
||||
ExecStart=/usr/local/bin/act_runner --config /etc/akyldash-runner/config.yaml daemon
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
CPUQuota=50%
|
||||
MemoryMax=768M
|
||||
TasksMax=256
|
||||
NoNewPrivileges=yes
|
||||
PrivateTmp=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
RestrictSUIDSGID=yes
|
||||
ReadWritePaths=$runner_runtime_home $runner_xdg_config_home /var/log/akyldash-runner /run/user/$runner_uid
|
||||
StandardOutput=append:/var/log/akyldash-runner/act_runner.log
|
||||
StandardError=append:/var/log/akyldash-runner/act_runner.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
install -d -o root -g root -m 0755 "/etc/systemd/system/user-$runner_uid.slice.d"
|
||||
runner_install_content "/etc/systemd/system/user-$runner_uid.slice.d/limits.conf" root root 0644 <<EOF
|
||||
[Slice]
|
||||
CPUQuota=250%
|
||||
MemoryMax=4G
|
||||
TasksMax=2048
|
||||
EOF
|
||||
runner_install_content /etc/logrotate.d/akyldash-runner root root 0644 <<'EOF'
|
||||
/var/log/akyldash-runner/act_runner.log {
|
||||
daily
|
||||
rotate 7
|
||||
size 10M
|
||||
compress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
}
|
||||
EOF
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.service" /etc/systemd/system/akyldash-runner-healthcheck.service root root 0644
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.timer" /etc/systemd/system/akyldash-runner-healthcheck.timer root root 0644
|
||||
runner_install_file "$(dirname "$0")/akyldash-runner-restart.service" /etc/systemd/system/akyldash-runner-restart.service root root 0644
|
||||
|
||||
loginctl enable-linger "$runner_user"
|
||||
systemctl daemon-reload
|
||||
systemctl start "user@$runner_uid.service"
|
||||
rootless_docker_unit="$runner_xdg_config_home/systemd/user/docker.service"
|
||||
rootless_env=(HOME="$runner_runtime_home" XDG_CONFIG_HOME="$runner_xdg_config_home" XDG_DATA_HOME="$runner_xdg_data_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus")
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" sh -c 'test -w "$HOME" && test -w "$XDG_CONFIG_HOME" && test -w "$XDG_DATA_HOME"' || die 'rootless Docker HOME and XDG directories must be writable by the runner'
|
||||
if [[ -e $rootless_docker_unit || -L $rootless_docker_unit ]]; then
|
||||
[[ -f $rootless_docker_unit && ! -L $rootless_docker_unit && $(stat -c %U "$rootless_docker_unit") == "$runner_user" ]] || die 'existing rootless Docker unit has an unexpected type or owner'
|
||||
grep -Fq 'dockerd-rootless.sh' "$rootless_docker_unit" || die 'existing rootless Docker unit differs from the expected service'
|
||||
else
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" dockerd-rootless-setuptool.sh install
|
||||
fi
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user daemon-reload
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user enable docker.service
|
||||
if runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user is-active --quiet docker.service; then
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user restart docker.service
|
||||
else
|
||||
runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user start docker.service
|
||||
fi
|
||||
docker_pid=$(runuser -u "$runner_user" -- env "${rootless_env[@]}" systemctl --user show docker.service --property=MainPID --value)
|
||||
[[ $docker_pid =~ ^[1-9][0-9]*$ && -r /proc/$docker_pid/environ ]] || die 'rootless Docker process environment is unavailable'
|
||||
docker_environment=$(tr '\0' '\n' <"/proc/$docker_pid/environ") || die 'could not read rootless Docker process environment'
|
||||
for expected in "HOME=$runner_runtime_home" "XDG_CONFIG_HOME=$runner_xdg_config_home" "XDG_DATA_HOME=$runner_xdg_data_home"; do
|
||||
grep -Fqx -- "$expected" <<<"$docker_environment" || die "running rootless Docker process is missing $expected"
|
||||
done
|
||||
|
||||
printf '\nRunner runtime installed. Register it interactively, then enable its services as described in docs/operations/CI_RUNNER.md.\n'
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create or validate the runner image without following replaceable paths."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def open_directory(path: str) -> int:
|
||||
current = os.open("/", os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
for component in path.strip("/").split("/"):
|
||||
if not component:
|
||||
continue
|
||||
next_fd = os.open(
|
||||
component,
|
||||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
||||
dir_fd=current,
|
||||
)
|
||||
os.close(current)
|
||||
current = next_fd
|
||||
return current
|
||||
except BaseException:
|
||||
os.close(current)
|
||||
raise
|
||||
|
||||
|
||||
def filesystem_type(fd: int) -> str:
|
||||
result = subprocess.run(
|
||||
["blkid", "-p", "-s", "TYPE", "-o", "value", f"/proc/self/fd/{fd}"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
pass_fds=(fd,),
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
image, bytes_text, parent_uid_text, parent_mode_text = sys.argv[1:]
|
||||
image = os.path.abspath(image)
|
||||
parent_path, name = os.path.split(image)
|
||||
if not name or name in {".", ".."}:
|
||||
raise ValueError("invalid image filename")
|
||||
|
||||
parent_fd = open_directory(parent_path)
|
||||
image_fd = -1
|
||||
try:
|
||||
parent_stat = os.fstat(parent_fd)
|
||||
if parent_stat.st_uid != int(parent_uid_text) or stat.S_IMODE(parent_stat.st_mode) != int(parent_mode_text, 8):
|
||||
raise RuntimeError("image parent has unexpected owner or permissions")
|
||||
|
||||
created = False
|
||||
try:
|
||||
image_fd = os.open(
|
||||
name,
|
||||
os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
image_fd = os.open(
|
||||
name,
|
||||
os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
created = True
|
||||
|
||||
image_stat = os.fstat(image_fd)
|
||||
if not stat.S_ISREG(image_stat.st_mode) or image_stat.st_uid != os.geteuid() or image_stat.st_nlink != 1:
|
||||
raise RuntimeError("image must be a singly-linked regular file owned by the installer")
|
||||
if not created and image_stat.st_mode & 0o022:
|
||||
raise RuntimeError("existing image is writable by group or other users")
|
||||
|
||||
if created:
|
||||
os.posix_fallocate(image_fd, 0, int(bytes_text))
|
||||
subprocess.run(
|
||||
["mkfs.ext4", "-F", "-q", f"/proc/self/fd/{image_fd}"],
|
||||
check=True,
|
||||
pass_fds=(image_fd,),
|
||||
)
|
||||
image_stat = os.fstat(image_fd)
|
||||
|
||||
if image_stat.st_size != int(bytes_text):
|
||||
raise RuntimeError(f"image size is {image_stat.st_size} bytes, expected {bytes_text}; refusing to resize or format")
|
||||
if filesystem_type(image_fd) != "ext4":
|
||||
raise RuntimeError("image does not contain ext4; refusing to format")
|
||||
path_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
||||
if not stat.S_ISREG(path_stat.st_mode) or (path_stat.st_dev, path_stat.st_ino) != (image_stat.st_dev, image_stat.st_ino):
|
||||
raise RuntimeError("image path changed during validation")
|
||||
return 0
|
||||
finally:
|
||||
if image_fd >= 0:
|
||||
os.close(image_fd)
|
||||
os.close(parent_fd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
output=$(mktemp)
|
||||
trap 'rm -f "$output"' EXIT
|
||||
if bash "$(dirname "$0")/install-runner.sh" >"$output" 2>&1; then
|
||||
echo 'Runner installer unexpectedly succeeded without root.' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qx 'ERROR: run this script as root' "$output"
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
runner_dir=$(cd "$(dirname "$0")" && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
mkdir -p "$tmp/bin" "$tmp/store" "$tmp/partial-store"
|
||||
log=$tmp/commands.log
|
||||
export RUNNER_TEST_LOG=$log
|
||||
|
||||
cat >"$tmp/bin/mkfs.ext4" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf 'mkfs\n' >>"$RUNNER_TEST_LOG"
|
||||
if [[ -n ${RUNNER_TEST_REPLACE_IMAGE:-} ]]; then
|
||||
rm -f "$RUNNER_TEST_IMAGE"
|
||||
ln -s "$RUNNER_TEST_REPLACE_IMAGE" "$RUNNER_TEST_IMAGE"
|
||||
fi
|
||||
EOF
|
||||
cat >"$tmp/bin/blkid" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "${RUNNER_TEST_FS:-ext4}"
|
||||
EOF
|
||||
cat >"$tmp/bin/systemctl" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"$RUNNER_TEST_LOG"
|
||||
if [[ $1 == start ]]; then
|
||||
[[ ${RUNNER_TEST_FAIL_START:-0} == 0 ]] || exit 1
|
||||
: >"$RUNNER_TEST_MOUNTED"
|
||||
fi
|
||||
EOF
|
||||
cat >"$tmp/bin/mountpoint" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
[[ -e $RUNNER_TEST_MOUNTED ]]
|
||||
EOF
|
||||
cat >"$tmp/bin/findmnt" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s ext4 /dev/loop0 %s\n' "$RUNNER_TEST_STORE" "${RUNNER_TEST_OPTIONS:-rw,nosuid,nodev,relatime}"
|
||||
EOF
|
||||
cat >"$tmp/bin/losetup" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$RUNNER_TEST_BACKING"
|
||||
EOF
|
||||
chmod +x "$tmp/bin/"*
|
||||
PATH="$tmp/bin:$PATH"
|
||||
export PATH
|
||||
source "$runner_dir/install-runner-lib.sh"
|
||||
runuser() {
|
||||
shift 2
|
||||
[[ $1 == -- ]] || return 2
|
||||
shift
|
||||
"$@"
|
||||
}
|
||||
getent() {
|
||||
[[ $1 == passwd ]] || return 2
|
||||
if [[ -n ${RUNNER_TEST_PASSWD:-} ]]; then
|
||||
printf '%s\n' "$RUNNER_TEST_PASSWD"
|
||||
elif [[ -r ${RUNNER_TEST_ACCOUNT_STATE:-} ]]; then
|
||||
cat "$RUNNER_TEST_ACCOUNT_STATE"
|
||||
else
|
||||
return 2
|
||||
fi
|
||||
}
|
||||
id() {
|
||||
case $1 in
|
||||
-gn) printf '%s\n' "${RUNNER_TEST_PRIMARY_GROUP:-akyldash-runner}" ;;
|
||||
-nG) [[ ${TEST_EXTRA_GROUP:-0} == 1 ]] && printf 'akyldash-runner docker\n' || printf 'akyldash-runner\n' ;;
|
||||
*) command id "$@" ;;
|
||||
esac
|
||||
}
|
||||
stat() {
|
||||
if [[ $1 == -c && $2 == %U && $3 == ${RUNNER_TEST_HOME:-} ]]; then
|
||||
printf '%s\n' "${RUNNER_TEST_HOME_OWNER:-akyldash-runner}"
|
||||
elif [[ $1 == -c && $2 == %G:%a && $3 == ${RUNNER_TEST_HOME:-} ]]; then
|
||||
printf '%s:%s\n' "${RUNNER_TEST_HOME_GROUP:-root}" "${RUNNER_TEST_HOME_MODE:-711}"
|
||||
else
|
||||
command stat "$@"
|
||||
fi
|
||||
}
|
||||
useradd() {
|
||||
printf 'useradd\n' >>"$RUNNER_TEST_LOG"
|
||||
local user=${@: -1} home=
|
||||
while (($#)); do
|
||||
if [[ $1 == --home-dir ]]; then home=$2; shift 2; else shift; fi
|
||||
done
|
||||
mkdir -p "$home"
|
||||
printf '%s:x:990:990::%s:/usr/sbin/nologin\n' "$user" "$home" >"$RUNNER_TEST_ACCOUNT_STATE"
|
||||
}
|
||||
|
||||
export RUNNER_TEST_HOME="$tmp/account-home"
|
||||
export RUNNER_TEST_ACCOUNT_STATE="$tmp/account.passwd"
|
||||
mkdir -p "$RUNNER_TEST_HOME"
|
||||
export RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
RUNNER_TEST_HOME_OWNER=root runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
if RUNNER_TEST_HOME_OWNER=root RUNNER_TEST_HOME_MODE=755 runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Writable root-owned runner home was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
TEST_EXTRA_GROUP=1
|
||||
if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with extra group membership was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset TEST_EXTRA_GROUP
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${tmp}/wrong-home:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with a different home was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:0:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with UID 0 was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
if RUNNER_TEST_PASSWD="akyldash-runner:x:1000:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with a regular-user UID was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
RUNNER_TEST_PRIMARY_GROUP=unexpected
|
||||
if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
|
||||
echo 'Runner account with an unexpected primary group was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset RUNNER_TEST_PRIMARY_GROUP
|
||||
unset RUNNER_TEST_PASSWD
|
||||
export RUNNER_TEST_HOME="$tmp/new-account-home"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
|
||||
[[ $(grep -c '^useradd$' "$log") == 1 ]]
|
||||
|
||||
private_root=$tmp/private
|
||||
mkdir -m 0700 "$private_root"
|
||||
runner_owner=$(command id -un)
|
||||
runner_group=$(command id -gn)
|
||||
runner_ensure_private_directory "$private_root/config" "$runner_owner" "$runner_group"
|
||||
runner_ensure_private_directory "$private_root/config" "$runner_owner" "$runner_group"
|
||||
mkdir -m 0700 "$tmp/outside"
|
||||
ln -s "$tmp/outside" "$private_root/linked"
|
||||
if runner_ensure_private_directory "$private_root/linked/docker" "$runner_owner" "$runner_group" 2>/dev/null; then
|
||||
echo 'Runner config symlink was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ ! -e $tmp/outside/docker ]]
|
||||
|
||||
legacy_root=$tmp/legacy
|
||||
mkdir -p "$legacy_root/.config/systemd/user/docker.service.d"
|
||||
chmod 0755 "$legacy_root" "$legacy_root/.config" "$legacy_root/.config/systemd" "$legacy_root/.config/systemd/user" "$legacy_root/.config/systemd/user/docker.service.d"
|
||||
for legacy_dir in "$legacy_root" "$legacy_root/.config" "$legacy_root/.config/systemd" "$legacy_root/.config/systemd/user" "$legacy_root/.config/systemd/user/docker.service.d"; do
|
||||
runner_ensure_private_directory "$legacy_dir" "$runner_owner" "$runner_group"
|
||||
[[ $(stat -c %a "$legacy_dir") == 700 ]]
|
||||
done
|
||||
user_config=$private_root/daemon.json
|
||||
runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF'
|
||||
{"data-root":"/private/runner"}
|
||||
EOF
|
||||
runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF'
|
||||
{"data-root":"/private/runner"}
|
||||
EOF
|
||||
if runner_install_user_content "$user_config" "$runner_owner" "$runner_group" 0600 <<'EOF' 2>/dev/null
|
||||
{"data-root":"/unexpected"}
|
||||
EOF
|
||||
then
|
||||
echo 'Conflicting runner user config was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
image=$tmp/docker-store.img
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
|
||||
truncate -s 1048575 "$image"
|
||||
if runner_ensure_image "$image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Mismatched image size was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
truncate -s 1048576 "$image"
|
||||
if RUNNER_TEST_FS=ext3 runner_ensure_image "$image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Existing non-ext4 image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^mkfs$' "$log") == 1 ]]
|
||||
external=$tmp/external-file
|
||||
printf 'preserve this file\n' >"$external"
|
||||
race_image=$tmp/race.img
|
||||
ln -s "$external" "$race_image"
|
||||
if runner_ensure_image "$race_image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Symlink runner image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(cat "$external") == 'preserve this file' ]]
|
||||
rm -f "$race_image"
|
||||
race_image=$tmp/race-during-format.img
|
||||
export RUNNER_TEST_IMAGE="$race_image" RUNNER_TEST_REPLACE_IMAGE="$external"
|
||||
if runner_ensure_image "$race_image" 1048576 "$(command id -u)" 700 2>/dev/null; then
|
||||
echo 'Image path replacement during formatting was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(cat "$external") == 'preserve this file' ]]
|
||||
rm -f "$race_image"
|
||||
unset RUNNER_TEST_IMAGE RUNNER_TEST_REPLACE_IMAGE
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
|
||||
unit_name='var-lib-akyldash\x2drunner-docker\x2ddata.mount'
|
||||
unit_file=$tmp/systemd/$unit_name
|
||||
mkdir -p "$(dirname "$unit_file")"
|
||||
export RUNNER_TEST_STORE="$tmp/store" RUNNER_TEST_BACKING="$image"
|
||||
export RUNNER_TEST_MOUNTED="$tmp/mounted"
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
grep -Fq "enable $unit_name" "$log"
|
||||
grep -Fq "start $unit_name" "$log"
|
||||
|
||||
# A second run with the image mounted must reuse it and avoid another start.
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
chmod 0664 "$unit_file"
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
|
||||
echo 'Writable systemd mount unit was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
chmod 0644 "$unit_file"
|
||||
export RUNNER_TEST_BACKING="$tmp/unrelated.img"
|
||||
touch "$RUNNER_TEST_BACKING"
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
|
||||
echo 'Active mount backed by a different image was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(grep -c '^start ' "$log") == 1 ]]
|
||||
export RUNNER_TEST_BACKING="$image"
|
||||
|
||||
# Simulate interruption after unit installation/enable, before mount activation.
|
||||
export RUNNER_TEST_STORE="$tmp/partial-store" RUNNER_TEST_MOUNTED="$tmp/partial-mounted"
|
||||
export RUNNER_TEST_FAIL_START=1
|
||||
if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount" 2>/dev/null; then
|
||||
echo 'Injected mount interruption was not observed' >&2
|
||||
exit 1
|
||||
fi
|
||||
export RUNNER_TEST_FAIL_START=0
|
||||
runner_ensure_image "$image" 1048576 "$(command id -u)" 700
|
||||
runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount"
|
||||
[[ $(grep -c '^mkfs$' "$log") == 2 ]]
|
||||
[[ $(grep -c '^start ' "$log") == 3 ]]
|
||||
export RUNNER_TEST_STORE="$tmp/store" RUNNER_TEST_MOUNTED="$tmp/mounted" RUNNER_TEST_OPTIONS=rw,relatime
|
||||
if runner_validate_mount "$image" "$RUNNER_TEST_STORE" 2>/dev/null; then
|
||||
echo 'Runner mount missing nosuid/nodev was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
unset RUNNER_TEST_OPTIONS
|
||||
echo 'runner installer idempotency checks passed'
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
config=infra/runner/config.yaml
|
||||
installer=infra/runner/install-runner.sh
|
||||
assert_absent() {
|
||||
local pattern=$1 file=$2
|
||||
if grep -Eq "$pattern" "$file"; then
|
||||
echo "Forbidden content matched in $file" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
grep -Eq '^ capacity: 1$' "$config"
|
||||
grep -Fq 'akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04' "$config"
|
||||
grep -Fq 'docker_host: "-"' "$config"
|
||||
grep -Eq '^ privileged: false$' "$config"
|
||||
grep -Eq '^ valid_volumes: \[\]$' "$config"
|
||||
grep -Fq -- '--cpus=2 --memory=3g --pids-limit=512' "$config"
|
||||
assert_absent 'var/run/docker.sock|metrics:|health_check:|production.env|review_secret' "$config"
|
||||
grep -Fq 'docker-ce-rootless-extras' "$installer"
|
||||
grep -Fq 'runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711' "$installer"
|
||||
grep -Fq 'runner_lock_home "$runner_home"' "$installer"
|
||||
grep -Fq 'test -w "$HOME" && test -w "$XDG_CONFIG_HOME" && test -w "$XDG_DATA_HOME"' "$installer"
|
||||
grep -Fq 'rootless_env=(HOME="$runner_runtime_home" XDG_CONFIG_HOME="$runner_xdg_config_home" XDG_DATA_HOME="$runner_xdg_data_home"' "$installer"
|
||||
grep -Fq 'Environment=XDG_CONFIG_HOME=$runner_xdg_config_home' "$installer"
|
||||
grep -Fq 'Environment=XDG_DATA_HOME=$runner_xdg_data_home' "$installer"
|
||||
grep -Fq 'runner_home/.config/systemd/user/docker.service.d/environment.conf' "$installer"
|
||||
grep -Fq 'Environment=HOME=$runner_runtime_home' "$installer"
|
||||
grep -Fq 'systemctl --user restart docker.service' "$installer"
|
||||
grep -Fq 'systemctl --user show docker.service --property=MainPID --value' "$installer"
|
||||
grep -Fq 'tr '\''\0'\'' '\''\n'\'' <"/proc/$docker_pid/environ"' "$installer"
|
||||
grep -Fq 'grep -Fqx -- "$expected"' "$installer"
|
||||
if grep -Fq 'systemctl --user set-environment' "$installer"; then
|
||||
echo 'Rootless Docker environment must persist in its user unit drop-in' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'HOME=/var/lib/akyldash-runner/.config/act-runner' infra/runner/act-runner-healthcheck.sh
|
||||
grep -Fq 'XDG_CONFIG_HOME=/var/lib/akyldash-runner/.config' infra/runner/act-runner-healthcheck.sh
|
||||
grep -Fq 'XDG_DATA_HOME=/var/lib/akyldash-runner/.config/act-runner/.local/share' infra/runner/act-runner-healthcheck.sh
|
||||
if grep -Fq 'env HOME="$runner_home"' "$installer"; then
|
||||
echo 'Protected runner home must not be passed as rootless Docker HOME' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'die '\''stop akyldash-act-runner.service after all jobs finish before rerunning this installer' "$installer"
|
||||
grep -Fq 'ReadWritePaths=$runner_runtime_home $runner_xdg_config_home /var/log/akyldash-runner /run/user/$runner_uid' "$installer"
|
||||
grep -Fq 'os.O_NOFOLLOW | os.O_CLOEXEC' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'os.posix_fallocate(image_fd, 0, int(bytes_text))' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'pass_fds=(image_fd,)' infra/runner/secure-runner-image.py
|
||||
grep -Fq 'systemctl enable "$unit_name"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'systemctl start "$unit_name"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'runuser -u "$owner" -- install -d -m 0700 "$directory"' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'runner_install_user_content "$runner_home/.config/docker/daemon.json"' "$installer"
|
||||
assert_absent 'basename.*runner_mount_unit|enable --now.*runner_mount_unit' "$installer"
|
||||
grep -Fq 'groups=$(id -nG "$user")' infra/runner/install-runner-lib.sh
|
||||
grep -Fq 'id -gn "$user"' infra/runner/install-runner-lib.sh
|
||||
assert_absent 'dockerd-rootless-setuptool.sh install --force' "$installer"
|
||||
grep -Fq 'runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2' "$installer"
|
||||
grep -Fq 'find-free-subid-range.awk' "$installer"
|
||||
grep -Fq 'usermod "${subid_args[@]}" "$runner_user"' "$installer"
|
||||
grep -Fq 'subid_args+=(--add-subuids "$subid_range")' "$installer"
|
||||
assert_absent 'usermod.*(sudo|docker)|/var/run/docker.sock|/etc/sudoers' "$installer"
|
||||
grep -Fq 'runs-on: akyldash-ci' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -S /var/run/docker.sock' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -e /etc/akyldash/production.env' .gitea/workflows/runner-smoke.yaml
|
||||
grep -Fq 'test ! -e /etc/akyldash/review_secret' .gitea/workflows/runner-smoke.yaml
|
||||
assert_absent '(^|[[:space:]])(docker|docker-compose|deploy)([[:space:]]|$)|production.env=|review_secret=' .gitea/workflows/runner-smoke.yaml
|
||||
|
||||
fixture=$(mktemp)
|
||||
fixture_g=$(mktemp)
|
||||
trap 'rm -f "$fixture" "$fixture_g"' EXIT
|
||||
printf 'production.env\n' >"$fixture"
|
||||
if assert_absent 'deploy|production.env|review_secret|docker.sock' "$fixture" 2>/dev/null; then
|
||||
echo 'Negative guard regression: prohibited fixture was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'codex:100000:65536\n' >"$fixture"
|
||||
printf 'codex:165000:1000\n' >"$fixture_g"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 231072-296607 ]]
|
||||
: >"$fixture_g"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 165536-231071 ]]
|
||||
: >"$fixture"
|
||||
range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g")
|
||||
[[ $range == 100000-165535 ]]
|
||||
printf 'broken\n' >"$fixture_g"
|
||||
if awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g" >/dev/null 2>&1; then
|
||||
echo 'Malformed subordinate ID allocation was accepted' >&2
|
||||
exit 1
|
||||
fi
|
||||
echo 'runner configuration guards passed'
|
||||
Reference in new issue
Block a user