diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index e15185e..5352f72 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -23,11 +23,25 @@ jobs: run: bash infra/bootstrap/test-host-guard.sh - name: Check verifier helper argument guard run: bash infra/bootstrap/test-verify-helper.sh + - name: Check host audit root guard + run: bash infra/bootstrap/test-audit-host.sh + - name: Check Compose helper allowlist + run: bash infra/deploy/test-compose-helper.sh + - name: Check production config root guard + run: bash infra/deploy/test-production-config-guard.sh + - name: Check runner config and isolation guards + run: | + bash infra/runner/test-runner-config.sh + bash infra/runner/test-install-runner-guard.sh - name: Check infrastructure shell syntax run: | bash -n infra/bootstrap/bootstrap-ubuntu-24.04.sh infra/bootstrap/host-guard.sh \ - infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/deploy/install-helper.sh \ - infra/bootstrap/test-verify-helper.sh + infra/bootstrap/verify.sh infra/bootstrap/verify-helper.sh infra/bootstrap/audit-host.sh \ + infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \ + infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \ + infra/bootstrap/test-audit-host.sh infra/bootstrap/test-verify-helper.sh \ + infra/runner/install-runner.sh infra/runner/act-runner-healthcheck.sh \ + infra/runner/test-runner-config.sh infra/runner/test-install-runner-guard.sh - name: Run rollback helper tests run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v - name: Validate production Compose and Caddyfile diff --git a/.gitea/workflows/runner-smoke.yaml b/.gitea/workflows/runner-smoke.yaml new file mode 100644 index 0000000..e7ca9c1 --- /dev/null +++ b/.gitea/workflows/runner-smoke.yaml @@ -0,0 +1,24 @@ +name: Runner smoke test + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + runner-smoke: + runs-on: akyldash-ci + steps: + - name: Verify isolated runner boundary + run: | + host_uid=$(awk 'NR == 1 {print $2}' /proc/self/uid_map) + test "$host_uid" -gt 0 + test ! -S /var/run/docker.sock + test ! -e /run/user/1000/docker.sock + test -z "${DOCKER_HOST:-}" + test ! -e /etc/akyldash/production.env + test ! -e /etc/akyldash/review_secret + test ! -e /opt/akyldash/current + printf 'Isolated runner: version=%s host=%s uid-map=%s\n' \ + "$GITEA_ACTIONS_RUNNER_VERSION" "$(hostname)" "$host_uid" diff --git a/AGENTS.md b/AGENTS.md index 07077e7..6d5c131 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,13 @@ Python, SQLite и OpenSearch. Production пока не развернут. - SSH alias — `akyldash_vps`; используй только read-only диагностику. Не давай `codex` общий sudo или доступ к Docker socket. Root-действия требует владелец. Для Gitea используй `gitea-api`, не читай файлы с учётными данными. +- Репозиторный Gitea Actions runner описан в + [CI_RUNNER.md](docs/operations/CI_RUNNER.md). Регистрация выполняется + владельцем через Settings → Actions → Runners для `admin/akyldash`; токен + вводится интерактивно и не попадает в Git. Runner использует отдельный + rootless Docker daemon; job-контейнеры не получают Docker socket или + production secrets. Не включай production deployment из CI без отдельного + проверенного deploy-helper. - Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`, `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и diff --git a/deploy/production/README.md b/deploy/production/README.md index ec0b0d3..b6caa67 100644 --- a/deploy/production/README.md +++ b/deploy/production/README.md @@ -14,18 +14,26 @@ account is intentionally outside the `sudo` and `docker` groups because access to the Docker socket grants root-equivalent control. Run host operations with the narrowly required `sudo` command; do not use `docker` as `codex` directly. -```bash -sudo install -d -o root -g root -m 0700 /etc/akyldash -sudo install -o root -g root -m 0600 deploy/production/.env.example /etc/akyldash/production.env -sudo sh -c 'umask 077; openssl rand -hex 32 > /etc/akyldash/review_secret' -sudoedit /etc/akyldash/production.env -``` +After reviewing the host audit and confirming these files do not already exist, +the owner can create them from the approved checkout with +`sudo bash infra/deploy/prepare-production-config.sh`. The script refuses to +overwrite existing configuration, installs the tracked env example with mode +`0600`, and generates a random review secret with mode `0600`. It does not +start Compose or create volumes. Review and set `DATA_ROOT` in the protected +env file before deployment. Set `DATA_ROOT` to the approved data directory and ensure the normalized corpus is at `$DATA_ROOT/minjust-normalized/`. The first host setup is documented in [`docs/operations/DEPLOYMENT.md`](../../docs/operations/DEPLOYMENT.md); the root bootstrap installs packages and creates directories, but does not deploy -the application or alter SSH, firewall, user groups, or sudoers. +the application or alter SSH, firewall, user groups, or sudoers. The restricted +`akyldash-compose` helper allows only root-owned Compose `config` and `status`; +service start/stop remains an owner-run root operation. + +The repository Gitea Actions runner is an isolated CI worker described in +[`docs/operations/CI_RUNNER.md`](../../docs/operations/CI_RUNNER.md). It has no +production Docker socket, production env files, or application checkout. CI +does not deploy or change production; all deployment remains owner-run. ## First deployment and upgrades diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md index c33fd9b..156f89c 100644 --- a/docs/operations/ACCESS.md +++ b/docs/operations/ACCESS.md @@ -8,7 +8,8 @@ printing key contents, then check the connection: ```bash ssh -F ~/.ssh/config -G akyldash_vps | awk '$1 == "hostname" || $1 == "user" || $1 == "port" || $1 == "identityfile" { print }' test -r ~/.ssh/akyldash_codex && echo 'SSH key is available' -ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps 'id; hostname' +ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps id +ssh -F ~/.ssh/config -o BatchMode=yes akyldash_vps hostname -f gitea-api GET /version gitea-api GET /repos/admin/akyldash ``` @@ -21,14 +22,24 @@ checks. Never print a private key, token, or complete environment file. - SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`. - SSH configuration: `~/.ssh/config`. - Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents). -- `codex` can run unprivileged read-only diagnostics. The audited account has no - non-interactive sudo and must not be added to `sudo` or the `docker` group. -- Host package installation, service setup, firewall changes, secret creation, - backup creation, recovery apply, and deployment require owner approval and a - reviewed root command. After the owner installs the root-owned repository - checkout, `infra/deploy/install-helper.sh` can grant a fixed read-only - verifier and guarded rollback command through sudoers; neither helper is - installed on the audited VPS. +- `codex` can run unprivileged read-only diagnostics. It has no sudo or Docker + group membership; Docker CLI access to the daemon is denied. Do not add it to + either group. +- Bootstrap is complete. Docker and Compose are active, but no root-owned + repository checkout or restricted helper is installed yet. The owner can run + `infra/bootstrap/audit-host.sh` from the reviewed root-owned checkout to + report Docker containers/volumes, protected-file metadata, UFW rules, ports, + and ISPmanager status without showing secret contents or changing state. +- Host package/service configuration, firewall changes, protected env/secret + creation, backup creation, recovery apply, and production deployment require + a reviewed owner/root command. `infra/deploy/install-helper.sh` grants the + fixed verifier, read-only Compose `config`/`status`, and guarded rollback + commands. It does not grant general sudo or Docker access; it is not + installed on the VPS. +- `infra/deploy/prepare-production-config.sh` is an owner-run root operation. + It refuses to overwrite existing files, copies only the tracked env example, + generates a random review secret, and writes both files with mode `0600`. + It does not start Compose or create volumes. - Never connect to other VPS hosts or Synology as part of this deployment. This execution environment is WSL2. The repository is on the Windows-mounted @@ -47,19 +58,27 @@ limitation and do not retrieve it elsewhere. environment. Never inspect local credential files, print environment values, or construct HTTP authorization headers. - The audited API token can read/write this repository and create issues, but - does not have `read:user` or `read:package`. An API `403` caused by a missing - scope is not evidence that the account itself lacks the permission. + does not have `read:user` or `read:package`. Listing repository Actions + runners returns `403` because the API principal is not the repository owner. + An API `403` caused by missing scope/ownership is not evidence that no runner + exists. - Protected `main` requires one approval. Do not push directly to `main`, merge automatically, or start a review. Work on a feature branch and use a PR. -- Gitea Actions and package features are enabled for the repository, but runner - availability and package publishing permissions have not been confirmed. +- Gitea Actions and package features are enabled for the repository. The Actions + task API currently returns no runs, but runner availability and package + publishing permissions have not been confirmed. The repository-level runner + is prepared by `infra/runner/install-runner.sh`, but is not installed or + registered yet. Only a repository owner may obtain its registration token in + `admin/akyldash` Settings → Actions → Runners; do not use organization or + instance tokens and never put the token in Git, a command argument, or a + report. See [CI_RUNNER.md](CI_RUNNER.md). ## Who can do what | Role | Operations | |---|---| -| `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners | -| Restricted verifier | Runs the root-owned read-only drift check; not installed on the audited VPS | +| `codex` over SSH | Unprivileged diagnostics; Docker daemon, UFW rules, and protected paths require the root audit script | +| Restricted helpers | Read-only drift check and Compose `config`/`status`, plus guarded rollback; not installed on the audited VPS | | Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS | | VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes | | Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval | diff --git a/docs/operations/CI_RUNNER.md b/docs/operations/CI_RUNNER.md new file mode 100644 index 0000000..410335c --- /dev/null +++ b/docs/operations/CI_RUNNER.md @@ -0,0 +1,138 @@ +# AkylDash Gitea Actions runner + +## Scope and isolation + +The deployment follows the official [Gitea 1.25 act_runner guide](https://docs.gitea.com/1.25/usage/actions/act-runner/) +for repository-scoped registration. Gitea warns that access to a Docker daemon +is root-equivalent; the dedicated rootless daemon is used instead of the +production daemon. The pinned binary and checksum are published in the +[official Gitea download directory](https://dl.gitea.com/gitea-runner/0.2.13/). + +The runner is reserved to repository `admin/akyldash`, using the token under +`https://git.vakanaut.ru/admin/akyldash/settings/actions/runners`. Do not use +organization or instance runner settings. Pin the official `act_runner` +`0.2.13` x86_64 binary from `dl.gitea.com`; the installer verifies its SHA-256 +before placing it on the host. + +CI jobs run in `docker.gitea.com/runner-images:ubuntu-24.04` containers with +capacity 1, per-job limits of 2 CPU, 3 GiB RAM and 512 processes, and a 15 GiB +dedicated ext4 loop-backed Docker data filesystem. The rootless Docker daemon +and runner user are separate from production Docker; `codex` is not added to +sudo or Docker groups. Jobs do not receive the rootless Docker socket, the +production Docker socket, production environment/secrets, or arbitrary host +bind mounts. The runner account has no access to `/etc/akyldash` or the +root-owned application checkout. These are host permission boundaries, not a +claim that untrusted project code is safe; review workflow changes and actions +before merging them. + +The runner service restarts on failure and boot. Systemd caps the runner +account's shared slice to 2.5 CPU, 4 GiB RAM and 2048 tasks; the rootless +daemon is separately capped to 2 CPU, 4 GiB and 1024 tasks, while each job has +the limits above. Runner logs go to a rotated file (10 MiB, 7 copies). A +systemd timer checks the runner service and the rootless Docker API each minute; +on failure it restarts the fixed runner service. No UFW changes or public +metrics ports are needed. Runner cache is disabled. + +The label `akyldash-ci` is used by `runner-smoke.yaml` on pull requests, pushes +to `main`, and manual dispatch. Existing CI image-build work remains on its current +`ubuntu-latest` target until it has been tested separately against the +isolated runner. +Production deployment is not automated. + +## Installation and registration + +Review the root script and host audit output before running it. It installs +`uidmap`, `dbus-user-session`, `curl`, `logrotate`, and Docker's rootless extras; +creates the dedicated system user and selects the first free, shared 65536-ID +range from `/etc/subuid` and `/etc/subgid`; reserves 15 GiB +for runner images/workspaces; and configures systemd services. It starts only +the separate rootless Docker daemon; the runner and health timer stay disabled +until repository registration. It does not change +SSH/UFW, production Docker configuration, application data, or production +secrets. It refuses to overwrite existing runner paths and requires at least +25 GiB free on `/` before reserving storage. + +If the reviewed checkout is not present at the inventoried path, first verify +that the target is absent and clone the reviewed PR branch there. This path is +currently absent per the read-only audit; do not overwrite it if that changes. + +```bash +sudo test ! -e /opt/akyldash/current +sudo git clone --branch feature/infrastructure-deployment-phase-2 \ + --single-branch https://git.vakanaut.ru/admin/akyldash.git \ + /opt/akyldash/current +``` + +From that root-owned reviewed checkout on the VPS: + +```bash +sudo bash /opt/akyldash/current/infra/runner/install-runner.sh +``` + +Then, as a repository owner, open repository **Settings → Actions → Runners**, +create a repository-level runner, and copy its registration token locally. Do +not send the token in chat or save it in the checkout. Register interactively +in a terminal so it is not placed in shell history or process arguments: + +```bash +sudo -u akyldash-runner -H /usr/local/bin/act_runner \ + --config /etc/akyldash-runner/config.yaml register +``` + +Enter `https://git.vakanaut.ru/`, the token, runner name `akyldash-ci-vps-01`, +and label `akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04` +when prompted. The config sets the label and capacity; verify them in the +repository runner settings. The token is saved only in the mode-0700 runner +state directory, outside Git. Start/enable the runner after registration: + +```bash +sudo systemctl enable --now akyldash-act-runner.service +sudo systemctl enable --now akyldash-runner-healthcheck.timer +sudo systemctl status akyldash-act-runner.service +sudo systemctl status akyldash-runner-healthcheck.timer +``` + +The interactive command's terminal behavior should be confirmed before pasting +the token. If it echoes input, stop and use the repository settings to revoke +that token; do not put it in command-line arguments or environment variables. + +## Validation + +1. Confirm the runner is **Online** in the repository runner settings, with the + expected name and `akyldash-ci` label. +2. Confirm **Runner smoke test** runs for the open PR (or dispatch it manually). + It checks that production + sockets, files, environment secrets, and writable application paths are not + present in the job container. +3. Confirm the run reports the expected runner, succeeds, and the runner + service and health timer remain active. +4. Only after the repository owner reviews this boundary should CI jobs be moved + to `akyldash-ci`. Do not add a production deploy workflow or secrets to this + runner until a separate restricted deploy mechanism is reviewed. + +## Upgrade + +Update `runner_version` and its expected official SHA-256 together in +`infra/runner/install-runner.sh`, review upstream release compatibility with +the installed Gitea, and open a PR. From the reviewed checkout, re-run the +installer only after resolving its existing-path guards. Do not replace the +binary while a job is active. Stop the runner, install/verify the new binary, +then start it and confirm Online plus a smoke run. Preserve `/var/lib/akyldash-runner/.runner`. + +## Recovery and rollback + +The runner is disposable: production data and secrets are not on its filesystem. +For a runner process failure, inspect `systemctl status` and +`journalctl -u akyldash-act-runner.service`; restart the fixed runner service. +For lost/corrupt registration state, stop the service and obtain a fresh +repository-level token from Settings → Actions → Runners. Preserve the 15 GiB +data image and state for diagnosis; re-register only after the repository +owner confirms/revokes the old runner in Gitea. The images/cache can be rebuilt +from the workflow. Never format or remove an existing runner store as an +automatic recovery step. + +## Status + +The configuration and scripts are prepared in Git. They have not been +installed on the VPS, a repository-level token is not available to this agent, +and no runner Online state or workflow execution is claimed yet. diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index 5aaf858..a3cbb8d 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -1,31 +1,46 @@ # Infrastructure current state -Last audit: 2026-10-11. Source: read-only SSH session as `codex`, local -repository inspection, and Gitea API. This document separates measured facts -from planned configuration. +Last audit: 2026-10-12. Source: read-only SSH session as `codex`, owner-reported +UFW state, local repository inspection, and Gitea API. Root-only Docker state, +UFW rule details, and protected file metadata remain unverified. ## VPS facts - Host: `avprogs.hlab.kz`, `206.223.249.118`, SSH port `22`, user `codex`. -- OS/kernel: Ubuntu Server `24.04.5 LTS`, `6.8.0-142-generic`. -- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 93 GiB free. +- OS: Ubuntu Server `24.04.5 LTS`; kernel `6.8.0-142-generic` from the prior + inventory audit. +- Resources: 4 vCPU, 7.7 GiB RAM, 3.8 GiB swap, 99 GiB disk with 92 GiB free + at this audit. - Network: interface `ens3`, IPv4 `206.223.249.118/32`. - `codex` is UID/GID 1000, groups `codex` and `users`; `sudo -n true` requires - a password. The current WSL2 session verified the SSH alias using the - explicit user config and approved read-only SSH execution. + a password. Docker CLI cannot access `/var/run/docker.sock` as `codex`. - SSH service is running. The inspected sshd configuration has public-key and password authentication enabled and `PermitRootLogin yes`; no SSH changes were made. -- Docker, Docker Compose, Git, Nginx, Caddy, Certbot, application paths, and - application services are absent. Only base OS services and SSH were running. -- Public listeners observed: TCP `22`; DNS listeners are limited to loopback - (`127.0.0.53` and `127.0.0.54`). -- UFW is reported active in the supplied VPS notes, but the audit account could - not independently read its status. Treat firewall state and rules as - unverified. -- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses from the VPS. - Both HTTP and HTTPS return Cloudflare `521`; the origin is currently - unreachable through the proxy. +- Bootstrap completed. Docker CE/CLI `29.9.0`, Compose `5.6.0`, containerd + `2.4.1`, and runc `1.5.2` are installed; package versions and active Docker + service were independently checked. The Docker daemon API remains root-only. +- `/etc/docker/daemon.json` has the expected `local` log driver and `10m` / `3` + rotation limits. Bootstrap paths exist with root ownership: `/opt/akyldash` + mode `0755`, `/etc/akyldash` and `/var/backups/akyldash` mode `0700`, and + `/var/lib/akyldash/data` mode `0750`. +- `/opt/akyldash/current`, verifier/rollback binaries, and the sudoers file are + absent. `codex` has not received sudo or Docker-group access. Metadata for + `/etc/akyldash/production.env` and `/etc/akyldash/review_secret` could not be + checked because `/etc/akyldash` is root-only. +- TCP listeners observed: SSH `22` and loopback DNS only; no listener on `1500`, + `80`, or `443`. No ISPmanager package or service unit was found through the + unprivileged checks; root can confirm with the audit script below. +- Owner reports UFW is active and allows TCP `22`, `80`, `443`, and `1500`. + Rule details were not independently read; no firewall changes were made. +- Additional read-only runner checks on 2026-10-12 confirmed `/` is ext4 with + about 92 GiB free at the prior audit, `/usr/bin/dockerd-rootless.sh` exists, + and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS, + `https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner + registration and Gitea Actions protocol connectivity are not confirmed. +- `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS, + HTTP, HTTPS, and `/review` still return Cloudflare `521`; origin TLS and + Cloudflare Full (strict) are not verified. ## Application and Gitea facts @@ -33,35 +48,51 @@ from planned configuration. - The repository API reports push permission, Actions and packages enabled, and PRs enabled. `main` is protected and requires one approval; the current API principal cannot push or merge directly to `main`. -- The Gitea token lacks `read:user` and `read:package`; runner status and package - publishing permissions could not be checked. This change adds - `.gitea/workflows/ci.yaml`; PR #42's run remains `Waiting to run` at the last - status check, so a compatible runner is not confirmed. -- Nothing is running on the VPS. Repository Compose now describes Caddy - `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite account/review - data, and OpenSearch `3.7.0` with `analysis-icu`. +- The repository runner-list API returns `403` because this API principal is + not the repository owner. The Actions task API currently reports no workflow + runs; this does not establish runner availability. Runner inventory and + package publishing permissions remain unverified. +- No public application listener is observed on TCP `80`/`443`. Docker + container/volume state still requires the root audit. Repository Compose describes + Caddy `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite + account/review data, and OpenSearch `3.7.0` with `analysis-icu`. - The target publishes only Caddy on TCP `80`/`443`. Backend and Caddy share a proxy network; OpenSearch and backend share a separate internal network, and - the backend has no host port. Caddy certificate storage, OpenSearch, and - application data use persistent volumes/bind mounts. TLS, registry digest - deployment for backend/OpenSearch, backup, rollback helper, and systemd - deployment are not configured on the host. + backend has no host port. Caddy certificate storage, OpenSearch, and + application data use persistent volumes/bind mounts. +- The root-owned repository checkout, production env/secret files, Compose + volumes/containers, backup, verifier/rollback helpers, and application are + not confirmed installed. TLS and registry digest deployment for + backend/OpenSearch are not configured. - The repository has a cold-backup archive/verification tool and a one-shot restore apply tool that requires a marked replacement host. Backup/restore tests use synthetic data; neither tool has run against production. The - archive preflight now permits a clean first deployment to skip backup, - requires backup for existing persistent data even without containers, - allows absent named volumes only in explicit corpus-only initial-deployment - mode, and rejects inaccessible/ambiguous mounts and incomplete/running - service containers. Initial deployment docs require a second verified backup - after index and admin initialization. + archive preflight permits a clean first deployment to skip backup, requires + backup for existing persistent data even without containers, allows absent + named volumes only in explicit corpus-only initial-deployment mode, and + rejects inaccessible/ambiguous mounts and incomplete/running service + containers. Initial deployment docs require a second verified backup after + index and admin initialization. - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the inventoried production hostname/IP; its host-identity guard has shell tests. The mode has not been run on a VPS. -- The repository defines a root-owned read-only verifier, rollback helper, and - owner-run installer with exact sudoers command rules. None is installed on - the VPS. -- An isolated local Compose smoke test built the backend/OpenSearch images, +- The repository defines a root-owned read-only verifier, Compose `config` and + `status` helper, guarded rollback helper, and installer with exact sudoers + command rules. None is installed on the VPS. CI contains no production + deployment job; starting/stopping services remains owner-run. +- A repository-scoped official `act_runner` `0.2.13` installation is prepared + in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB ext4 + loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU, + 3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048 + tasks), rotated file logs, and a systemd health timer. Jobs use the + `akyldash-ci` label and do not receive the rootless or production Docker + socket. This is repository code only: no runner package, account, store, + service, token, or registration is installed on the VPS. +- Gitea repository runner inventory could not be read by the current API + principal (403). The owner has not supplied runner state, and no runner + Online status or runner-hosted workflow is claimed. The manual + `runner-smoke.yaml` workflow is prepared but has not run. +- An isolated local Compose smoke test built backend/OpenSearch images, reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP 200 for `/review` through Caddy. Its containers, networks, and volumes were removed afterward. This does not verify VPS startup, DNS, or public TLS. @@ -71,27 +102,38 @@ from planned configuration. ## Checks not completed -- SSH completed with the user config explicitly selected; plain SSH was blocked - by permissions on a system config include. The private key was not read. -- UFW rules require root and remain unverified. +- Docker Engine version from the server API, container and volume inventory, + protected env/secret metadata, sudoers, and detailed UFW rules require root. + `infra/bootstrap/audit-host.sh` is provided for an owner-run read-only audit. - The backup archive/restore tool was tested with synthetic local data; a backup and restore of real production data remain unverified because production is not deployed. -- Production instructions now use `/etc/akyldash/production.env` and root-run - Docker commands, consistent with the `codex` account's restricted access. - Initial deployment and upgrade backup ordering are documented; no production - host was changed for this PR. -- The Cloudflare edge response was checked, but origin DNS configuration, - Cloudflare Full (strict), certificate validity, Actions runner availability, - registry publishing, and production deployment have not been tested. -- No packages or services were installed or changed on the VPS. +- Cloudflare edge response was checked; origin DNS target, origin certificate, + Full (strict), runner registration/Online state, registry publishing, and production + deployment have not been verified. +- This audit made no host changes. No package or service was installed by this + audit; the owner performed the initial bootstrap beforehand. ## Next owner actions -1. Review and run the root bootstrap on the dedicated host. -2. Review SSH/password/root policy and firewall rules separately; this change - did not touch them. -3. Set up DNS, origin certificate, reverse proxy, protected secret files, and - initial corpus/index. -4. Confirm a Gitea Actions runner and package permissions, then test CI and - backup/restore before considering automated production deploy. +1. Install the approved root-owned checkout after its PR is reviewed, then run + `sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh` and share + the output. The script prints no secret values and does not mutate the host. +2. After reviewing the audit output and confirming the target files are absent, + run + `sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`; + it refuses to overwrite existing files. Set the approved `DATA_ROOT` and + place the approved corpus/index and Cloudflare origin configuration. Do not + deploy until data/service state and the required backup path are reviewed. +3. Run `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh` after + reviewing the checkout; it grants only fixed verification, Compose + `config`/`status`, and guarded rollback commands. +4. After PR review and root audit, run + `sudo bash /opt/akyldash/current/infra/runner/install-runner.sh`. In the + repository Settings → Actions → Runners create a repository-level runner; + enter the token only at the interactive registration prompt as documented + in [CI_RUNNER.md](CI_RUNNER.md). Then enable the runner and health timer, + confirm **Online**, and dispatch the smoke workflow. Production deployment + remains manual and has no CI job. +5. Test backup/restore and rollback only with synthetic data in an isolated + environment before any production deployment. diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md index 5ec86dd..167947c 100644 --- a/docs/operations/DEPLOYMENT.md +++ b/docs/operations/DEPLOYMENT.md @@ -2,37 +2,53 @@ ## Current status -Production deployment is not enabled or tested on the VPS. An isolated local -Compose smoke test passed; the VPS has no Docker, Git, -application data, proxy, registry pull configuration, backup process, or -restricted verification/rollback helpers. Do not deploy until the owner has -completed the root bootstrap, provided a protected secret file and a reviewed -proxy/firewall configuration, and a backup and restore have been tested. +Production deployment is not enabled or tested on the VPS. The owner completed +the root bootstrap: Docker/Compose are active and target directories exist. +The `codex` account cannot access the Docker socket. No root-owned checkout or +restricted helpers are installed; protected env/secret metadata and Docker +container/volume state remain unverified because they require root. +The unprivileged audit sees no listener on `80`, `443`, or `1500`; Cloudflare +still returns `521`. Do not deploy until the owner has reviewed root-only host +state, supplied protected configuration and the approved corpus/index, and +backup/restore have been tested. ## First host preparation -1. Review the current inventory and state, then have the owner run - `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh` from a verified copy - of this branch on the VPS. The script installs Docker Engine/Compose and Git, - sets Docker log rotation, and creates root-owned paths. It refuses other OS - versions and does not change SSH, UFW, sudoers, or user groups. Use - `--recovery-target` only on a replacement host; it refuses the known - production host identity. -2. Install the approved repository checkout at `/opt/akyldash/current`, - root-owned and not group/world-writable. Copy - `deploy/production/.env.example` to `/etc/akyldash/production.env`, set - `DATA_ROOT`, and configure - `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside - Git. The secret file must be root-readable only. -3. After reviewing the root-owned checkout, run - `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader, - read-only verifier, and rollback helper with exact sudoers command rules; it - does not grant general sudo or Docker access. -4. Confirm DNS points through Cloudflare to this origin, then have the owner - review firewall rules and open only TCP `80` and `443` in addition to the - existing SSH access. The Compose Caddy service obtains and renews its HTTPS - certificate automatically; Cloudflare Full (strict) still needs owner-side - verification. +1. The initial root bootstrap is complete. After reviewing and merging the + infrastructure PR, the owner must install that approved `main` checkout at + `/opt/akyldash/current` (the path was absent during the unprivileged audit): + + ```bash + sudo git clone --branch main --single-branch https://git.vakanaut.ru/admin/akyldash.git /opt/akyldash/current + ``` + + The checkout is root-owned by default. Then run + `sudo bash /opt/akyldash/current/infra/bootstrap/audit-host.sh`. This + read-only audit reports Docker state, protected-file metadata, UFW rules, + listeners, and ISPmanager without printing secret contents. Resolve any + existing containers, volumes, or data before proceeding; do not overwrite + or delete them. +2. After confirming the existing env and secret do not exist and reviewing the + tracked defaults, run + `sudo bash /opt/akyldash/current/infra/deploy/prepare-production-config.sh`. + It creates `/etc/akyldash/production.env` from the tracked example and + generates `/etc/akyldash/review_secret`; both are root-owned mode `0600`. + It refuses to overwrite either file and does not start services or create + volumes. Review `DATA_ROOT` and place the approved normalized corpus there + before deployment. +3. After reviewing the root-owned checkout and audit output, run + `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh`. It installs + the backup reader, read-only verifier, `akyldash-compose config|status`, and + rollback helper with exact sudoers command rules. The Compose helper has no + start/stop/build operation and grants no direct Docker access. Actual + deployments remain owner-run through the documented Compose commands. +4. Confirm DNS points through Cloudflare to this origin. The owner reports UFW + already allows TCP `80` and `443`; read and document the full rules before + any firewall decision. TCP `1500` is allowed by owner report but has no + observed listener and no ISPmanager package/service was found; the owner + must identify its purpose. Do not change UFW as part of this procedure. The + Compose Caddy service obtains and renews HTTPS certificates automatically; + Cloudflare Full (strict) still needs end-to-end verification. 5. For a clean first deployment, skip backup only when Compose confirms there are no containers and all persistent mounts are empty. If any data exists, create and verify a backup before starting a new version. For a preloaded @@ -49,14 +65,19 @@ proxy/firewall configuration, and a backup and restore have been tested. sudo chmod 0600 /opt/akyldash/deployed-image.txt ``` -Application deployment remains owner-run until registry permissions, a live -Gitea runner, and a controlled deploy procedure are tested. The helper only -supports a guarded application rollback. +Application deployment remains owner-run. A repository-level Actions runner +installation is prepared in [CI_RUNNER.md](CI_RUNNER.md), but it is not yet +installed or registered; its Online state and smoke workflow are unverified. +The runner's job containers have no Docker socket or production secrets. CI has +no image publishing or production deployment step. The installed sudo helpers +support fixed verification, read-only Compose `config`/`status`, and guarded +rollback. They do not start or stop services. ## CI `.gitea/workflows/ci.yaml` runs the Python unit suites and builds both production images on pull requests and pushes to `main`. It does not publish an -image or deploy production. Gitea Actions is enabled at repository level, but a -successful workflow run has not yet confirmed that a compatible runner is -online. +image or deploy production. `.gitea/workflows/runner-smoke.yaml` exercises the +`akyldash-ci` runner on pull requests, pushes to `main`, or manual dispatch. +Runner registration is incomplete, so neither workflow execution on that +runner nor current `ubuntu-latest` runner availability has been confirmed. diff --git a/infra/bootstrap/audit-host.sh b/infra/bootstrap/audit-host.sh new file mode 100644 index 0000000..f48593a --- /dev/null +++ b/infra/bootstrap/audit-host.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Run this read-only audit as root." >&2 + exit 1 +fi + +echo '== Host ==' +hostname -f +grep -E '^(PRETTY_NAME|VERSION_ID)=' /etc/os-release +nproc +free -h +df -h / + +echo '== Packages and services ==' +dpkg-query -W docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +docker version --format 'Docker Engine {{.Server.Version}}' +docker compose version +containerd --version +runc --version +systemctl is-active docker +python3 -c 'import json; d=json.load(open("/etc/docker/daemon.json")); print("Docker logs:", d.get("log-driver"), d.get("log-opts"))' + +echo '== Account and paths ==' +id codex +for path in /opt/akyldash /opt/akyldash/current /etc/akyldash /etc/akyldash/production.env /etc/akyldash/review_secret /var/lib/akyldash/data /var/backups/akyldash /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /etc/sudoers.d/akyldash-codex; do + if [[ -e $path ]]; then + stat -c '%n owner=%U:%G mode=%a type=%F' "$path" + else + printf '%s absent\n' "$path" + fi +done + +echo '== ISPmanager ==' +dpkg-query -W -f='${binary:Package} ${Version}\n' 'ispmanager*' 'ispmgr*' 2>/dev/null || true +systemctl list-unit-files --type=service --no-legend | grep -iE 'ispmanager|ispmgr' || echo 'No ISPmanager service unit found.' + +echo '== Docker state ==' +docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' +docker volume ls + +echo '== Gitea Actions runner (read-only) ==' +if id akyldash-runner >/dev/null 2>&1; then + id akyldash-runner +else + echo 'akyldash-runner user absent' +fi +if command -v act_runner >/dev/null 2>&1; then + act_runner --version +else + echo 'act_runner binary absent from PATH' +fi +for path in /etc/akyldash-runner/config.yaml /var/lib/akyldash-runner/.runner /var/lib/akyldash-runner/docker-store.img; do + if [[ -e $path ]]; then + stat -c '%n owner=%U:%G mode=%a type=%F' "$path" + else + printf '%s absent\n' "$path" + fi +done +systemctl status akyldash-act-runner.service --no-pager --full || true +systemctl status akyldash-runner-healthcheck.timer --no-pager --full || true +findmnt /var/lib/akyldash-runner/docker-data || true + +echo '== Firewall and listeners (read-only) ==' +ufw status verbose +ufw status numbered +ss -lntup diff --git a/infra/bootstrap/test-audit-host.sh b/infra/bootstrap/test-audit-host.sh new file mode 100644 index 0000000..63a4adf --- /dev/null +++ b/infra/bootstrap/test-audit-host.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +output=$(mktemp) +trap 'rm -f "$output"' EXIT +if bash "$(dirname "$0")/audit-host.sh" >"$output" 2>&1; then + echo 'Host audit unexpectedly succeeded without root.' >&2 + exit 1 +fi +grep -qx 'Run this read-only audit as root.' "$output" diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh index 115b367..ffa8f3b 100755 --- a/infra/bootstrap/verify.sh +++ b/infra/bootstrap/verify.sh @@ -129,13 +129,14 @@ for endpoint in $udp_listeners; do failed=1 done -expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *' +expected_sudoers=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status' if [[ -x /usr/local/sbin/akyldash-verify && $(stat -c %U:%a /usr/local/sbin/akyldash-verify) == root:750 && \ -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \ + -x /usr/local/sbin/akyldash-compose && $(stat -c %U:%a /usr/local/sbin/akyldash-compose) == root:750 && \ -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \ -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \ $(cat /etc/sudoers.d/akyldash-codex) == "$expected_sudoers" ]]; then - echo 'PASS read-only verifier, rollback helper, and exact sudoers rules present' + echo 'PASS read-only verifier, Compose status/config helper, rollback helper, and exact sudoers rules present' else echo 'FAIL restricted helper or sudoers installation missing/different' failed=1 diff --git a/infra/deploy/compose-helper.sh b/infra/deploy/compose-helper.sh new file mode 100644 index 0000000..3de785a --- /dev/null +++ b/infra/deploy/compose-helper.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 || ( $1 != config && $1 != status ) ]]; then + echo 'Usage: akyldash-compose {config|status}' >&2 + exit 2 +fi +if [[ $EUID -ne 0 ]]; then + echo 'Run this helper through its fixed sudoers command.' >&2 + exit 1 +fi + +checkout=/opt/akyldash/current +if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root ]]; then + echo 'Refusing Compose access: approved root-owned checkout is unavailable.' >&2 + exit 1 +fi +unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit) +if [[ -n $unsafe ]]; then + echo 'Refusing Compose access: checkout contains a writable or non-root-owned path.' >&2 + exit 1 +fi + +compose=(docker compose --env-file /etc/akyldash/production.env -f "$checkout/deploy/production/compose.yaml") +case $1 in + config) "${compose[@]}" config --quiet ;; + status) "${compose[@]}" ps ;; +esac diff --git a/infra/deploy/install-helper.sh b/infra/deploy/install-helper.sh index 258af93..728c134 100644 --- a/infra/deploy/install-helper.sh +++ b/infra/deploy/install-helper.sh @@ -10,7 +10,8 @@ checkout=/opt/akyldash/current rollback_source=$checkout/infra/deploy/rollback.py backup_source=$checkout/infra/backup/archive.py verify_source=$checkout/infra/bootstrap/verify-helper.sh -if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source ]]; then +compose_source=$checkout/infra/deploy/compose-helper.sh +if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source || ! -f $verify_source || ! -f $compose_source ]]; then echo "Install the approved repository checkout at $checkout first." >&2 exit 1 fi @@ -43,7 +44,7 @@ if ! command -v visudo >/dev/null 2>&1; then exit 1 fi -for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do +for path in /usr/local/sbin/akyldash-verify /usr/local/sbin/akyldash-rollback /usr/local/sbin/akyldash-compose /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do if [[ -L $path ]]; then echo "Refusing symlink at $path." >&2 exit 1 @@ -54,9 +55,10 @@ install -d -o root -g root -m 0755 /usr/local/lib/akyldash install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py install -o root -g root -m 0750 "$verify_source" /usr/local/sbin/akyldash-verify install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback +install -o root -g root -m 0750 "$compose_source" /usr/local/sbin/akyldash-compose sudoers_file=/etc/sudoers.d/akyldash-codex -expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *' +expected=$'codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-verify\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose config\ncodex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-compose status' created=false if [[ -e $sudoers_file ]]; then if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then @@ -85,4 +87,4 @@ if ! visudo -c -f /etc/sudoers; then echo "Sudoers validation failed; the newly added rule was removed." >&2 exit 1 fi -echo "Installed read-only verification and guarded rollback helpers; codex receives no general sudo or Docker access." +echo "Installed fixed read-only verification/Compose commands and guarded rollback; codex receives no general sudo or Docker access." diff --git a/infra/deploy/prepare-production-config.sh b/infra/deploy/prepare-production-config.sh new file mode 100644 index 0000000..dcfdd35 --- /dev/null +++ b/infra/deploy/prepare-production-config.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo 'Run this configuration step as root.' >&2 + exit 1 +fi + +checkout=/opt/akyldash/current +example=$checkout/deploy/production/.env.example +config_dir=/etc/akyldash +env_file=$config_dir/production.env +secret_file=$config_dir/review_secret +if [[ ! -d $checkout/.git || -L $checkout || $(stat -c %U "$checkout") != root || ! -f $example ]]; then + echo 'Refusing configuration: approved root-owned checkout is unavailable.' >&2 + exit 1 +fi +unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit) +if [[ -n $unsafe ]] || ! git -C "$checkout" diff --quiet HEAD -- deploy/production/.env.example; then + echo 'Refusing configuration: checkout is writable or .env.example differs from Git.' >&2 + exit 1 +fi +if [[ ! -d $config_dir || $(stat -c %U:%G:%a "$config_dir") != root:root:700 ]]; then + echo 'Expected /etc/akyldash to be root-owned mode 0700.' >&2 + exit 1 +fi +if [[ -e $env_file || -L $env_file || -e $secret_file || -L $secret_file ]]; then + echo 'Production env or review secret already exists; refusing to overwrite either file.' >&2 + exit 1 +fi + +env_tmp=$(mktemp "$config_dir/.production.env.XXXXXX") +secret_tmp=$(mktemp "$config_dir/.review_secret.XXXXXX") +trap 'rm -f "$env_tmp" "$secret_tmp"' EXIT +install -o root -g root -m 0600 "$example" "$env_tmp" +openssl rand -hex 32 > "$secret_tmp" +chown root:root "$secret_tmp" +chmod 0600 "$secret_tmp" +mv -n "$env_tmp" "$env_file" +if [[ -e $env_tmp ]]; then + echo 'Production env already appeared; refusing to continue.' >&2 + exit 1 +fi +mv -n "$secret_tmp" "$secret_file" +if [[ -e $secret_tmp ]]; then + rm -f "$env_file" + echo 'Review secret already appeared; removed the new env file and stopped.' >&2 + exit 1 +fi +echo 'Created root-owned production.env and review_secret with mode 0600. No services or volumes were changed.' diff --git a/infra/deploy/test-compose-helper.sh b/infra/deploy/test-compose-helper.sh new file mode 100644 index 0000000..4ca3a66 --- /dev/null +++ b/infra/deploy/test-compose-helper.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +helper=$(dirname "$0")/compose-helper.sh +for action in deploy up down docker; do + if bash "$helper" "$action" >/dev/null 2>&1; then + printf 'FAIL Compose helper accepted %s\n' "$action" >&2 + exit 1 + fi +done +echo 'PASS Compose helper rejects operations outside its read-only allowlist' diff --git a/infra/deploy/test-production-config-guard.sh b/infra/deploy/test-production-config-guard.sh new file mode 100644 index 0000000..cf7ed3e --- /dev/null +++ b/infra/deploy/test-production-config-guard.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +output=$(mktemp) +trap 'rm -f "$output"' EXIT +if bash "$(dirname "$0")/prepare-production-config.sh" >"$output" 2>&1; then + echo 'Production config setup unexpectedly succeeded without root.' >&2 + exit 1 +fi +grep -qx 'Run this configuration step as root.' "$output" diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml index 0c3917c..c728847 100644 --- a/infra/inventory/production.yml +++ b/infra/inventory/production.yml @@ -15,7 +15,7 @@ facts: memory_gib: 7.7 swap_gib: 3.8 disk_gib: 99 - disk_free_gib: 93 + disk_free_gib: 92 network_interface: ens3 ipv4: 206.223.249.118/32 domain_resolution: akyldash.kg resolves to Cloudflare IPv4 and IPv6 edge addresses @@ -26,12 +26,37 @@ facts: - OpenSSH server - systemd-resolved - unattended-upgrades - absent: - - Docker Engine - - Docker Compose plugin + - Docker CE and CLI 29.9.0 + - Docker Compose plugin 5.6.0 + - containerd 2.4.1 + - runc 1.5.2 - Git - - reverse proxy - access_audit_date: 2026-10-11 + absent: + - ISPmanager package/service (unprivileged package and unit checks) + - root-owned application checkout + - restricted verifier and rollback helper + - non-loopback listeners on TCP 80, 443, 1500 + access_audit_date: 2026-10-12 + docker_service_active: true + docker_api_accessible_to_codex: false + docker_daemon_log_config: + driver: local + max_size: 10m + max_file: 3 + bootstrap_paths: + /opt/akyldash: root:root 0755 + /etc/akyldash: root:root 0700 + /var/lib/akyldash/data: root:root 0750 + /var/backups/akyldash: root:root 0700 + unprivileged_nonloopback_tcp_listeners: [22] + owner_reported_ufw_allowed_tcp: [22, 80, 443, 1500] + port_1500_listener: false + rootless_docker_command_present: /usr/bin/dockerd-rootless.sh + runner_gitea_host_resolution: git.vakanaut.ru resolves to 195.209.48.28 + runner_gitea_https_probe: /api/v1/version returned Gitea 1.25.5 (Actions protocol unverified) + cloudflare_edge_status: + http: 521 + https: 521 target: compose_file_in_repository: deploy/production/compose.yaml compose_file_on_host: /opt/akyldash/current/deploy/production/compose.yaml @@ -58,7 +83,7 @@ target: persistent_data: named Compose volume opensearch-data reverse_proxy: target: Caddy in Compose; Caddyfile proxies akyldash.kg to backend:8080 - status: repository configuration exists; not installed or running on VPS + status: no listener observed on 80/443; container state needs root Docker audit external_ports_target: [22, 80, 443] compose_networks: proxy: caddy and backend @@ -70,15 +95,31 @@ target: restricted_helper: path: /usr/local/sbin/akyldash-rollback status: repository code and owner-run installer exist; not installed on VPS + compose_read_only_helper: + path: /usr/local/sbin/akyldash-compose + allowed_operations: [config, status] + status: repository code and owner-run installer exist; not installed on VPS read_only_verifier: path: /usr/local/sbin/akyldash-verify status: repository code and owner-run installer exist; not installed on VPS + gitea_actions_runner: + repository: admin/akyldash + registration_scope: repository + implementation: official act_runner 0.2.13; pinned binary SHA-256 in infra/runner/install-runner.sh + label: akyldash-ci + capacity: 1 + job_limits: {cpu: 2, memory_gib: 3, pids: 512} + account_slice_limits: {cpu: 2.5, memory_gib: 4, tasks: 2048} + docker_store_limit_gib: 15 + docker_boundary: rootless dedicated daemon; no production socket; job socket injection disabled + service_status: configuration prepared, not installed or registered; Online unverified + owner_action: run the root installer from the reviewed checkout; register through repository runner settings checks: - infra/bootstrap/verify.sh akyldash_vps - docker compose -f deploy/production/compose.yaml ps - curl -fsS https://akyldash.kg/review - python3 infra/backup/archive.py verify /var/backups/akyldash/.tar.gz - backup_path: /var/backups/akyldash (target; archive script in infra/backup/archive.py is not installed or run) + backup_path: /var/backups/akyldash (directory exists; archive script in infra/backup/archive.py is not installed or run) deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment) versions: application: @@ -87,4 +128,7 @@ versions: telegram_bot: 0.2.2 infrastructure: unknown; no production install or release exists caddy_image: 2.11.6-alpine pinned by manifest digest in Compose - docker_engine: not installed + docker_engine: 29.9.0 (daemon active; server API unverified as codex) + docker_compose: 5.6.0 + containerd: 2.4.1 + runc: 1.5.2 diff --git a/infra/runner/act-runner-healthcheck.sh b/infra/runner/act-runner-healthcheck.sh new file mode 100644 index 0000000..3c0b687 --- /dev/null +++ b/infra/runner/act-runner-healthcheck.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +systemctl is-active --quiet akyldash-act-runner.service +runner_uid=$(id -u akyldash-runner) +runuser -u akyldash-runner -- env HOME=/var/lib/akyldash-runner \ + DOCKER_HOST="unix:///run/user/$runner_uid/docker.sock" \ + /usr/bin/docker info --format '{{.ServerVersion}}' >/dev/null diff --git a/infra/runner/akyldash-runner-healthcheck.service b/infra/runner/akyldash-runner-healthcheck.service new file mode 100644 index 0000000..4f40792 --- /dev/null +++ b/infra/runner/akyldash-runner-healthcheck.service @@ -0,0 +1,7 @@ +[Unit] +Description=Check AkylDash Gitea Actions runner health +OnFailure=akyldash-runner-restart.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/akyldash-runner-healthcheck diff --git a/infra/runner/akyldash-runner-healthcheck.timer b/infra/runner/akyldash-runner-healthcheck.timer new file mode 100644 index 0000000..7d1170b --- /dev/null +++ b/infra/runner/akyldash-runner-healthcheck.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodically check AkylDash Gitea Actions runner health + +[Timer] +OnBootSec=2m +OnUnitActiveSec=1m +Unit=akyldash-runner-healthcheck.service + +[Install] +WantedBy=timers.target diff --git a/infra/runner/akyldash-runner-restart.service b/infra/runner/akyldash-runner-restart.service new file mode 100644 index 0000000..17cd9e7 --- /dev/null +++ b/infra/runner/akyldash-runner-restart.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart AkylDash Gitea Actions runner after a failed health check + +[Service] +Type=oneshot +ExecStart=/bin/systemctl restart akyldash-act-runner.service diff --git a/infra/runner/config.yaml b/infra/runner/config.yaml new file mode 100644 index 0000000..0f892ac --- /dev/null +++ b/infra/runner/config.yaml @@ -0,0 +1,15 @@ +log: + level: info +runner: + file: /var/lib/akyldash-runner/.runner + capacity: 1 + labels: + - "akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04" +container: + privileged: false + options: --cpus=2 --memory=3g --pids-limit=512 + valid_volumes: [] + docker_host: "-" + force_pull: true +cache: + enabled: false diff --git a/infra/runner/find-free-subid-range.awk b/infra/runner/find-free-subid-range.awk new file mode 100644 index 0000000..6c1542a --- /dev/null +++ b/infra/runner/find-free-subid-range.awk @@ -0,0 +1,22 @@ +BEGIN { FS = ":" } +/^[[:space:]]*$/ || /^[[:space:]]*#/ { next } +NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1 { invalid = 1; next } +{ + start[FILENAME, ++count[FILENAME]] = $2 + end[FILENAME, count[FILENAME]] = $2 + $3 - 1 + files[FILENAME] = 1 +} +END { + if (invalid) exit 2 + for (base = 100000; base + 65535 <= 4294901760; base += 65536) { + overlap = 0 + for (file in files) + for (i = 1; i <= count[file]; i++) + if (start[file, i] <= base + 65535 && end[file, i] >= base) overlap = 1 + if (!overlap) { + print base "-" base + 65535 + exit + } + } + exit 1 +} diff --git a/infra/runner/install-runner.sh b/infra/runner/install-runner.sh new file mode 100644 index 0000000..c3e7dc8 --- /dev/null +++ b/infra/runner/install-runner.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly runner_user=akyldash-runner +readonly runner_home=/var/lib/akyldash-runner +readonly runner_store="$runner_home/docker-data" +readonly runner_image="$runner_home/docker-store.img" +readonly runner_mount_unit="/etc/systemd/system/$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)" +readonly runner_version=0.2.13 +readonly runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2 +readonly runner_url="https://dl.gitea.com/gitea-runner/$runner_version/act_runner-$runner_version-linux-amd64" +readonly checkout=/opt/akyldash/current + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +[[ $EUID -eq 0 ]] || die 'run this script as root' +[[ -r /etc/os-release ]] || die 'cannot identify operating system' +. /etc/os-release +[[ ${ID:-} == ubuntu && ${VERSION_ID:-} == 24.04 ]] || die 'Ubuntu 24.04 is required' +[[ $(uname -m) == x86_64 ]] || die 'this pinned binary is for x86_64 only' +[[ $(stat -fc %T /sys/fs/cgroup) == cgroup2fs ]] || die 'rootless Docker requires cgroup v2' +[[ $(findmnt -n -o FSTYPE /) == ext4 ]] || die 'the bounded runner store currently requires ext4 on /' +[[ -d $checkout/.git && ! -L $checkout && $(stat -c %U "$checkout") == root ]] || die 'install the reviewed root-owned checkout at /opt/akyldash/current first' +for directory in /opt /opt/akyldash; do + [[ $(stat -c %U "$directory") == root ]] || die "$directory must be owned by root" + mode=$(stat -c %a "$directory") + (( (8#$mode & 022) == 0 )) || die "$directory must not be group/world-writable" +done +unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit) +[[ -z $unsafe ]] || die 'the repository checkout contains files writable by non-root users' +git -C "$checkout" diff --quiet HEAD -- infra/runner || die 'runner files differ from the reviewed Git revision' +[[ -r /etc/subuid && -r /etc/subgid ]] || die 'subordinate ID allocation files are unavailable' +subid_range=$(awk -f "$(dirname "$0")/find-free-subid-range.awk" /etc/subuid /etc/subgid) || die 'could not find a free 65536-ID range shared by /etc/subuid and /etc/subgid' + +for path in "$runner_home" "$runner_image" "$runner_store" /etc/akyldash-runner; do + [[ ! -e $path ]] || die "$path already exists; inspect it and follow the documented recovery procedure" +done +! id "$runner_user" >/dev/null 2>&1 || die "user $runner_user already exists; inspect it before continuing" +[[ ! -e /usr/local/bin/act_runner ]] || die '/usr/local/bin/act_runner already exists' +[[ ! -e /etc/systemd/system/akyldash-act-runner.service ]] || die 'runner service already exists' +[[ ! -e /etc/systemd/system/akyldash-runner-healthcheck.service ]] || die 'runner health monitor already exists' +[[ ! -e /etc/logrotate.d/akyldash-runner ]] || die 'runner logrotate policy already exists' +[[ ! -e $runner_mount_unit ]] || die 'runner data mount unit already exists' +[[ $(df -Pk / | awk 'NR==2 {print $4}') -ge 25000000 ]] || die 'need at least 25 GiB free before reserving 15 GiB for runner data' + +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate docker-ce-rootless-extras + +useradd --system --create-home --home-dir "$runner_home" --shell /usr/sbin/nologin "$runner_user" +usermod --add-subuids "$subid_range" --add-subgids "$subid_range" "$runner_user" +runner_uid=$(id -u "$runner_user") + +install -d -o "$runner_user" -g "$runner_user" -m 0700 "$runner_home" "$runner_store" "$runner_home/.config/docker" "$runner_home/.config/systemd/user/docker.service.d" +install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner +install -d -o "$runner_user" -g "$runner_user" -m 0750 /var/log/akyldash-runner + +fallocate -l 15G "$runner_image" +mkfs.ext4 -F -q "$runner_image" +cat >"$runner_mount_unit" <"$runner_home/.config/docker/daemon.json" <"$runner_home/.config/systemd/user/docker.service.d/limits.conf" <<'EOF' +[Service] +Delegate=yes +CPUQuota=200% +MemoryMax=4G +TasksMax=1024 +Restart=always +RestartSec=5 +EOF +chown -R "$runner_user:$runner_user" "$runner_home/.config" + +install -o root -g root -m 0644 /dev/stdin /etc/systemd/system/akyldash-act-runner.service </dev/null 2>&1 && exit 0; sleep 2; done; exit 1' +ExecStart=/usr/local/bin/act_runner --config /etc/akyldash-runner/config.yaml daemon +Restart=always +RestartSec=10 +CPUQuota=50% +MemoryMax=768M +TasksMax=256 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +RestrictSUIDSGID=yes +ReadWritePaths=$runner_home /var/log/akyldash-runner /run/user/$runner_uid +StandardOutput=append:/var/log/akyldash-runner/act_runner.log +StandardError=append:/var/log/akyldash-runner/act_runner.log + +[Install] +WantedBy=multi-user.target +EOF + +mkdir -p "/etc/systemd/system/user-$runner_uid.slice.d" +cat >"/etc/systemd/system/user-$runner_uid.slice.d/limits.conf" </etc/logrotate.d/akyldash-runner <<'EOF' +/var/log/akyldash-runner/act_runner.log { + daily + rotate 7 + size 10M + compress + missingok + notifempty + copytruncate +} +EOF +install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-healthcheck.service" /etc/systemd/system/akyldash-runner-healthcheck.service +install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-healthcheck.timer" /etc/systemd/system/akyldash-runner-healthcheck.timer +install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-restart.service" /etc/systemd/system/akyldash-runner-restart.service + +loginctl enable-linger "$runner_user" +systemctl daemon-reload +systemctl start "user@$runner_uid.service" +runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" dockerd-rootless-setuptool.sh install --force +runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" systemctl --user daemon-reload +runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" systemctl --user enable --now docker.service + +printf '\nRunner runtime installed. Register it interactively, then enable its services as described in docs/operations/CI_RUNNER.md.\n' diff --git a/infra/runner/test-install-runner-guard.sh b/infra/runner/test-install-runner-guard.sh new file mode 100644 index 0000000..4e52819 --- /dev/null +++ b/infra/runner/test-install-runner-guard.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +output=$(mktemp) +trap 'rm -f "$output"' EXIT +if bash "$(dirname "$0")/install-runner.sh" >"$output" 2>&1; then + echo 'Runner installer unexpectedly succeeded without root.' >&2 + exit 1 +fi +grep -qx 'ERROR: run this script as root' "$output" diff --git a/infra/runner/test-runner-config.sh b/infra/runner/test-runner-config.sh new file mode 100644 index 0000000..24f2cb8 --- /dev/null +++ b/infra/runner/test-runner-config.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +config=infra/runner/config.yaml +installer=infra/runner/install-runner.sh +assert_absent() { + local pattern=$1 file=$2 + if grep -Eq "$pattern" "$file"; then + echo "Forbidden content matched in $file" >&2 + return 1 + fi +} +grep -Eq '^ capacity: 1$' "$config" +grep -Fq 'akyldash-ci:docker://docker.gitea.com/runner-images:ubuntu-24.04' "$config" +grep -Fq 'docker_host: "-"' "$config" +grep -Eq '^ privileged: false$' "$config" +grep -Eq '^ valid_volumes: \[\]$' "$config" +grep -Fq -- '--cpus=2 --memory=3g --pids-limit=512' "$config" +assert_absent 'var/run/docker.sock|metrics:|health_check:|production.env|review_secret' "$config" +grep -Fq 'docker-ce-rootless-extras' "$installer" +grep -Fq 'fallocate -l 15G' "$installer" +grep -Fq 'runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2' "$installer" +grep -Fq 'find-free-subid-range.awk' "$installer" +grep -Fq 'usermod --add-subuids "$subid_range" --add-subgids "$subid_range"' "$installer" +assert_absent 'usermod.*(sudo|docker)|/var/run/docker.sock|/etc/sudoers' "$installer" +grep -Fq 'runs-on: akyldash-ci' .gitea/workflows/runner-smoke.yaml +grep -Fq 'test ! -S /var/run/docker.sock' .gitea/workflows/runner-smoke.yaml +grep -Fq 'test ! -e /etc/akyldash/production.env' .gitea/workflows/runner-smoke.yaml +grep -Fq 'test ! -e /etc/akyldash/review_secret' .gitea/workflows/runner-smoke.yaml +assert_absent '(^|[[:space:]])(docker|docker-compose|deploy)([[:space:]]|$)|production.env=|review_secret=' .gitea/workflows/runner-smoke.yaml + +fixture=$(mktemp) +fixture_g=$(mktemp) +trap 'rm -f "$fixture" "$fixture_g"' EXIT +printf 'production.env\n' >"$fixture" +if assert_absent 'deploy|production.env|review_secret|docker.sock' "$fixture" 2>/dev/null; then + echo 'Negative guard regression: prohibited fixture was accepted' >&2 + exit 1 +fi +printf 'codex:100000:65536\n' >"$fixture" +printf 'codex:165000:1000\n' >"$fixture_g" +range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g") +[[ $range == 231072-296607 ]] +: >"$fixture_g" +range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g") +[[ $range == 165536-231071 ]] +: >"$fixture" +range=$(awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g") +[[ $range == 100000-165535 ]] +printf 'broken\n' >"$fixture_g" +if awk -f infra/runner/find-free-subid-range.awk "$fixture" "$fixture_g" >/dev/null 2>&1; then + echo 'Malformed subordinate ID allocation was accepted' >&2 + exit 1 +fi +echo 'runner configuration guards passed'