diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index c188117..70c9b56 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -7,7 +7,7 @@ on: jobs: tests-and-images: - runs-on: ubuntu-latest + runs-on: akyldash-ci steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -23,6 +23,8 @@ jobs: run: bash infra/bootstrap/test-host-guard.sh - name: Check verifier helper argument guard run: bash infra/bootstrap/test-verify-helper.sh + - name: Check unprivileged guard test helper + run: bash infra/test/test-run-as-unprivileged.sh - name: Check host audit root guard run: bash infra/bootstrap/test-audit-host.sh - name: Check Compose helper allowlist @@ -32,6 +34,8 @@ jobs: - name: Check runner config and isolation guards run: | bash infra/runner/test-runner-config.sh + bash infra/runner/test-ci-container-operations.sh + bash infra/runner/test-runner-log-acl.sh bash infra/runner/test-install-runner-idempotency.sh bash infra/runner/test-install-runner-guard.sh - name: Check infrastructure shell syntax @@ -41,8 +45,11 @@ jobs: infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \ infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \ infra/bootstrap/test-audit-host.sh infra/bootstrap/test-verify-helper.sh \ + infra/test/run-as-unprivileged.sh infra/test/test-run-as-unprivileged.sh \ infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \ infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \ + infra/runner/test-ci-container-operations.sh \ + infra/runner/test-runner-log-acl.sh \ infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())" - name: Run rollback helper tests @@ -50,9 +57,15 @@ jobs: - name: Validate production Compose and Caddyfile run: | secret_file=$(mktemp) - trap 'rm -f "$secret_file"' EXIT + caddy_dir=$(mktemp -d) + trap 'rm -f "$secret_file"; rm -rf "$caddy_dir"' EXIT REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet - docker run --rm --entrypoint caddy -v "$PWD/deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028 validate --config /etc/caddy/Caddyfile --adapter caddyfile + test "$(uname -m)" = x86_64 + caddy_archive="$caddy_dir/caddy.tar.gz" + curl --fail --location --retry 3 --output "$caddy_archive" https://github.com/caddyserver/caddy/releases/download/v2.11.6/caddy_2.11.6_linux_amd64.tar.gz + echo '422771007d505ea97efd1177a4905b2c1a471cd426668f2ace3bcda3d8e30b11f9b1610bfb02c6ad60f2a795f56124f2f5eec6409c17d5a0dd4c21a11375fb94 '"$caddy_archive" | sha512sum --check + tar -xzf "$caddy_archive" -C "$caddy_dir" caddy + "$caddy_dir/caddy" validate --config deploy/production/Caddyfile --adapter caddyfile - name: Build production images run: | docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci . diff --git a/.gitea/workflows/runner-smoke.yaml b/.gitea/workflows/runner-smoke.yaml index e7ca9c1..4f6a846 100644 --- a/.gitea/workflows/runner-smoke.yaml +++ b/.gitea/workflows/runner-smoke.yaml @@ -15,7 +15,9 @@ jobs: host_uid=$(awk 'NR == 1 {print $2}' /proc/self/uid_map) test "$host_uid" -gt 0 test ! -S /var/run/docker.sock - test ! -e /run/user/1000/docker.sock + for socket in /run/user/*/docker.sock; do + test ! -S "$socket" + done test -z "${DOCKER_HOST:-}" test ! -e /etc/akyldash/production.env test ! -e /etc/akyldash/review_secret diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md index 156f89c..9b3b8b9 100644 --- a/docs/operations/ACCESS.md +++ b/docs/operations/ACCESS.md @@ -22,9 +22,12 @@ checks. Never print a private key, token, or complete environment file. - SSH alias: `akyldash_vps`, user `codex`, host `206.223.249.118`, port `22`. - SSH configuration: `~/.ssh/config`. - Identity file: `~/.ssh/akyldash_codex` (path only; never copy its contents). -- `codex` can run unprivileged read-only diagnostics. It has no sudo or Docker - group membership; Docker CLI access to the daemon is denied. Do not add it to - either group. +- `codex` can run unprivileged read-only diagnostics and read + `/var/log/akyldash-runner/act_runner.log` through a file ACL. It has no sudo + or Docker group membership; Docker CLI access to the daemon is denied. Do not + add it to either group. The runner installer in PR #46 narrows this to search + permission on the dedicated log directory and read permission on that one + file, preserved by systemd startup and logrotate hooks. - Bootstrap is complete. Docker and Compose are active, but no root-owned repository checkout or restricted helper is installed yet. The owner can run `infra/bootstrap/audit-host.sh` from the reviewed root-owned checkout to @@ -64,14 +67,12 @@ limitation and do not retrieve it elsewhere. exists. - Protected `main` requires one approval. Do not push directly to `main`, merge automatically, or start a review. Work on a feature branch and use a PR. -- Gitea Actions and package features are enabled for the repository. The Actions - task API currently returns no runs, but runner availability and package - publishing permissions have not been confirmed. The repository-level runner - is prepared by `infra/runner/install-runner.sh`, but is not installed or - registered yet. Only a repository owner may obtain its registration token in - `admin/akyldash` Settings → Actions → Runners; do not use organization or - instance tokens and never put the token in Git, a command argument, or a - report. See [CI_RUNNER.md](CI_RUNNER.md). +- Gitea Actions and package features are enabled for the repository. The owner + reports the repository-level runner is registered and Idle, and its smoke + workflow passes on `akyldash-ci`; the runner inventory endpoint returns 403 + to this non-owner API principal. The registration token remains owner-managed + in the runner's protected state file. Do not re-register or rotate it unless + the owner confirms that action. See [CI_RUNNER.md](CI_RUNNER.md). ## Who can do what diff --git a/docs/operations/CI_RUNNER.md b/docs/operations/CI_RUNNER.md index 38890e4..619c560 100644 --- a/docs/operations/CI_RUNNER.md +++ b/docs/operations/CI_RUNNER.md @@ -25,7 +25,12 @@ rootless Docker setup and user service use `.config/act-runner` as writable `.config/act-runner/.local/share` as `XDG_DATA_HOME`. This keeps Docker config and its systemd user unit in the existing `.config` tree without granting write access to the protected home. -rootless Docker daemon + +The runner system service uses `ProtectHome=read-only`. `ProtectHome=yes` +hides `/run/user`, which contains the rootless Docker socket needed by the runner +service; read-only protection keeps that path visible without making home paths +writable. The installer accepts only the known prior unit with this one-line +difference and refuses any other service-unit drift. The rootless Docker daemon and runner user are separate from production Docker; `codex` is not added to sudo or Docker groups. Jobs do not receive the rootless Docker socket, the production Docker socket, production environment/secrets, or arbitrary host @@ -42,10 +47,69 @@ systemd timer checks the runner service and the rootless Docker API each minute; on failure it restarts the fixed runner service. No UFW changes or public metrics ports are needed. Runner cache is disabled. -The label `akyldash-ci` is used by `runner-smoke.yaml` on pull requests, pushes -to `main`, and manual dispatch. Existing CI image-build work remains on its current -`ubuntu-latest` target until it has been tested separately against the -isolated runner. +The runner daemon log is `/var/log/akyldash-runner/act_runner.log`. The +installer keeps its directory root-owned, grants `codex` only search permission +on that directory, and limits file access to the runner owner plus a named +read-only ACL for `codex`; group and other users have no file access. Systemd +starts with `UMask=0077`, and its fixed `ExecStartPost` command reapplies the +file ACL. Logrotate uses `copytruncate` and reapplies the same ACL after each +rotation. This access is limited to the runner daemon log; it does not grant +access to runner registration state, job workspaces, Docker state, or Gitea +workflow logs. The daemon log records task dispatch metadata, not step output. + +The registered project label is `akyldash-ci`, mapped to the pinned +`docker.gitea.com/runner-images:ubuntu-24.04` image. Both `ci.yaml` and +`runner-smoke.yaml` target this exact project label; neither uses the generic +`ubuntu-latest` label. The runner inventory API is owner-only, so the current +API account cannot independently list the registered labels; the owner reports +that the runner is Idle and the smoke workflow succeeds. + +The job container receives neither the rootless Docker socket nor the system +Docker socket (`container.docker_host: "-"`). The CI operations have different +engine requirements: + +| Operation | Purpose | Engine requirement | +| --- | --- | --- | +| `docker compose ... config --quiet` | Interpolate and validate production Compose configuration | Compose client only; this command does not start containers or contact a daemon. | +| Caddy `validate` | Check the production Caddyfile syntax and directives | Caddy binary only; CI downloads the exact version used by the pinned image and checks its SHA-512. | +| `docker build` for backend and OpenSearch | Exercise Dockerfile instructions, base-image resolution, and the OpenSearch plugin install | A container builder is required; ordinary Python/shell tests cannot replace this check. | + +Do not run rootless BuildKit inside the current job container. The official +BuildKit rootless container recipe requires relaxing seccomp/AppArmor and +system path restrictions; its documented no-process-sandbox fallback lets +build steps signal processes in the BuildKit daemon and is explicitly +discouraged. This runner intentionally keeps its job security profile intact. + +The recommended image-build path is a separate, disposable BuildKit VM, with +the current runner acting only as a `buildctl` client. Run BuildKit as a +non-root account with its own subordinate UID/GID range and scratch disk; use +mutual TLS over a private network, no Docker socket, no production credentials, +no route to production, one build at a time, and delete the VM and cache after +the job. Keep the test job on `akyldash-ci`; place image builds in a dependent +job on a separate builder label. This keeps arbitrary PR Dockerfile commands +off the runner and production host kernel. Gitea 1.25.5 and act_runner 0.2.13 +need no special BuildKit integration: the workflow invokes pinned `buildctl` +against the remote BuildKit API. + +For a first sizing estimate, allocate the disposable builder 2 vCPU, 4 GiB RAM, +and 10 GiB temporary disk at concurrency 1. Disable persistent cache or cap it +to 6 GiB with BuildKit garbage collection; remove OCI output archives after +validation. These are starting limits, not measured requirements. Reduce them +only after recording peak memory and disk use for both Dockerfiles. The current +VPS has 7.7 GiB RAM and the runner already has 3 GiB job / 4 GiB account limits, +so colocating a builder there would contend with jobs and share the host kernel; +it is not the recommended deployment. + +Provisioning that disposable VM and private network is an owner-level bootstrap +step. It must pin BuildKit and its checksum, create the non-root account and +subuid/subgid mapping, enforce CPU/RAM/disk and single-job limits, expose only a +TLS-authenticated BuildKit endpoint to the runner, issue a short-lived +per-job client certificate, and destroy the VM after the build. Do not enable +`--privileged`, relax seccomp/AppArmor, set +`--oci-worker-no-process-sandbox`, mount either Docker socket, or provide +production secrets. Until this bootstrap is approved and available, keep both +image builds required and expect the full CI job to remain blocked at those +steps. Production deployment is not automated. ## Installation and registration @@ -136,9 +200,11 @@ that token; do not put it in command-line arguments or environment variables. present in the job container. 3. Confirm the run reports the expected runner, succeeds, and the runner service and health timer remain active. -4. Only after the repository owner reviews this boundary should CI jobs be moved - to `akyldash-ci`. Do not add a production deploy workflow or secrets to this - runner until a separate restricted deploy mechanism is reviewed. +4. Confirm both workflows target `akyldash-ci`, and verify the container has no + system or runner-account Docker socket, production secret, writable host + checkout, or privileged execution. Do not add a production deploy workflow + or secrets to this runner until a separate restricted deploy mechanism is + reviewed. ## Upgrade @@ -170,9 +236,29 @@ automatic recovery step. ## Status -The root installer and registration are incomplete. The owner reports that the -15 GiB ext4 loop-backed mount is active on `/dev/loop0` after manually running -`systemctl daemon-reload`, `enable`, and `start` when the first installer run -stopped at mount activation. This PR has not changed the VPS and does not claim -that remaining runner components are installed, a repository token is -available, the runner is Online, or a workflow has run. +The owner reports that the repository-level runner is registered and Idle, and +that `runner-smoke.yaml` succeeds. Read-only checks on 2026-10-14 confirmed the +runner service and health timer are active, the runner account has no +supplementary groups, and the active service uses the rootless Docker socket. +The live service has `ProtectHome=read-only`; the installer still needs the PR +update applied by an owner-run command after merge. Read-only SSH now confirms +that act_runner 0.2.13 declares `[akyldash-ci]` and accepts task 301; its log +does not include workflow step output. The Actions jobs API returns an empty +list for completed run 49, so step details are not available through the +current API account. The repository runner inventory API returns 403 to the +current non-owner API account. PR #46 pins the narrow log ACL and its rotation +hooks; this installer change has not been applied to the VPS. On the updated +PR head `7701e77`, CI run 51 also failed after task 303 was accepted, while +smoke run 52 passed. The daemon log still exposes no job or step detail and +the Actions jobs endpoint is empty for run 51, so its failure remains +unattributed. The owner later provided the full log for task 305 (run 53): +`Check host audit root guard` exited 1 before Docker builds began. The test had +assumed its caller was non-root, but Gitea job commands run as root in this +container. The next run confirmed this guard passed, then failed at +`Check production config root guard` for the same UID assumption; Compose +allowlist validation passed. The production-config and runner-installer tests +had the same flaw. These three tests now use the shared +`infra/test/run-as-unprivileged.sh` helper to test denial as UID 65534 when CI +itself runs as root. The helper has a workflow regression test and prints a +diagnostic if `setpriv` cannot switch user. All production guards remain +unchanged. diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index e65ef8e..6e434de 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -1,8 +1,9 @@ # Infrastructure current state -Last audit: 2026-10-12. Source: read-only SSH session as `codex`, owner-reported -UFW state, local repository inspection, and Gitea API. Root-only Docker state, -UFW rule details, and protected file metadata remain unverified. +Last audit: 2026-10-14. Source: read-only SSH session as `codex`, local +repository inspection, Gitea API, and owner-reported workflow results. +Root-only Docker state, UFW rule details, and protected file metadata remain +unverified. ## VPS facts @@ -18,8 +19,8 @@ UFW rule details, and protected file metadata remain unverified. password authentication enabled and `PermitRootLogin yes`; no SSH changes were made. - Bootstrap completed. Docker CE/CLI `29.9.0`, Compose `5.6.0`, containerd - `2.4.1`, and runc `1.5.2` are installed; package versions and active Docker - service were independently checked. The Docker daemon API remains root-only. + `2.4.1`, and runc `1.5.2` are installed; the system Docker daemon API remains + inaccessible to `codex`. - `/etc/docker/daemon.json` has the expected `local` log driver and `10m` / `3` rotation limits. Bootstrap paths exist with root ownership: `/opt/akyldash` mode `0755`, `/etc/akyldash` and `/var/backups/akyldash` mode `0700`, and @@ -33,11 +34,33 @@ UFW rule details, and protected file metadata remain unverified. unprivileged checks; root can confirm with the audit script below. - Owner reports UFW is active and allows TCP `22`, `80`, `443`, and `1500`. Rule details were not independently read; no firewall changes were made. -- Additional read-only runner checks on 2026-10-12 confirmed `/` is ext4 with - about 92 GiB free at the prior audit, `/usr/bin/dockerd-rootless.sh` exists, - and `git.vakanaut.ru` resolves to `195.209.48.28`. From the VPS, - `https://git.vakanaut.ru/api/v1/version` returned Gitea `1.25.5`; runner - registration and Gitea Actions protocol connectivity are not confirmed. +- Read-only runner checks on 2026-10-14 confirmed `akyldash-runner` is UID 999, + GID 988, with no supplementary groups; the runner service and health timer are + active. Its systemd environment points Docker CLI to + `unix:///run/user/999/docker.sock`; the service's `ExecStartPre` Docker + health check succeeded. The service has `ProtectHome=read-only`, + `ProtectSystem=strict`, `NoNewPrivileges=yes`, `PrivateTmp=yes`, kernel + protections, and `RestrictSUIDSGID=yes`. Its only writable paths are runner + runtime/config, logs, and `/run/user/999`. The temporary live `ProtectHome` + change is not yet in `main` and this PR pins it in the installer. +- `systemd-analyze verify` accepted the active runner unit. `systemd-analyze + security` reports exposure 7.4 (medium) because several optional sandbox + filters are unset; the live process itself runs as UID/GID `999:988`, has + effective and permitted capabilities `0`, and `NoNewPrivs=1`. No + supplementary groups are present. +- The owner granted `codex` ACL access to `/var/log/akyldash-runner/act_runner.log`. + Read-only SSH confirmed the directory is `akyldash-runner:akyldash-runner` + mode `0750` with `codex:r-x`, while the log is `root:root` mode `0644` with + `codex:r--` and `other::r--`. The inspected log contains runner task metadata + only, no workflow step output. The active log is therefore currently + world-readable; PR #46 changes IaC to a root-owned directory, runner-owned + private log, `codex` search/read ACLs only, and ACL reapplication after + service startup and log rotation. No VPS changes were made. +- The unprivileged check did not find `/var/run/docker.sock`; the runner user + is not in `sudo` or `docker` groups. Production secret file metadata remains + root-only and was not independently checked. The Gitea runner inventory API + returns `403` to this non-owner account. The owner reports the registered + runner is **Idle** and `runner-smoke.yaml` succeeds with label `akyldash-ci`. - `akyldash.kg` resolves to Cloudflare IPv4/IPv6 edge addresses. From the VPS, HTTP, HTTPS, and `/review` still return Cloudflare `521`; origin TLS and Cloudflare Full (strict) are not verified. @@ -48,10 +71,13 @@ UFW rule details, and protected file metadata remain unverified. - The repository API reports push permission, Actions and packages enabled, and PRs enabled. `main` is protected and requires one approval; the current API principal cannot push or merge directly to `main`. -- The repository runner-list API returns `403` because this API principal is - not the repository owner. The Actions task API currently reports no workflow - runs; this does not establish runner availability. Runner inventory and - package publishing permissions remain unverified. +- Both repository workflows are active in Gitea. `runner-smoke.yaml` targets + `akyldash-ci`; `ci.yaml` still targets `ubuntu-latest` in `main`, which has no + matching online runner. This PR changes it to the project label. +- The Gitea runner inventory API returns `403` because this API principal is + not the repository owner. The registered runner's label list cannot be read + directly; the owner-reported successful smoke run confirms the custom label + is accepted. Package publishing permissions remain unverified. - No public application listener is observed on TCP `80`/`443`. Docker container/volume state still requires the root audit. Repository Compose describes Caddy `2.11.6` pinned by digest, Python Search API `0.11.3`, SQLite @@ -80,17 +106,16 @@ UFW rule details, and protected file metadata remain unverified. `status` helper, guarded rollback helper, and installer with exact sudoers command rules. None is installed on the VPS. CI contains no production deployment job; starting/stopping services remains owner-run. -- A repository-scoped official `act_runner` `0.2.13` installation is prepared +- A repository-scoped official `act_runner` `0.2.13` installation is managed in `infra/runner/`: rootless Docker under `akyldash-runner`, 15 GiB ext4 loop-backed image/workspace storage, capacity 1, per-job limits (2 CPU, 3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048 tasks), rotated file logs, and a systemd health timer. Jobs use the `akyldash-ci` label and do not receive the rootless or production Docker socket. The owner reports a partial runner installation: the 15 GiB ext4 - loop-backed store is active on `/dev/loop0` after manually enabling and - starting its mount unit when the installer stopped. Remaining components, - repository registration, and Online state are unverified; this PR makes no - VPS changes. The reviewed installer locks the runner home to root ownership + loop-backed store is active on `/dev/loop0`. The owner reports the runner is + registered and Idle, and the smoke workflow succeeds. This PR makes no VPS + changes. The installer locks the runner home to root ownership (`0711`) so CI cannot replace the image or mountpoint, while writable config and runtime state remain in runner-owned subdirectories. It reuses legacy `0755` config directories, creates the image through a pinned file descriptor, @@ -98,13 +123,43 @@ UFW rule details, and protected file metadata remain unverified. The reported installation blocker is that rootless setup receives this protected directory as `HOME`; the installer assigns writable runner-owned HOME/XDG paths, persists them in the `docker.service` user-unit drop-in, and - restarts an active daemon and checks the running process environment. Local - runner guards pass; - the change has not been run on the VPS. -- Gitea repository runner inventory could not be read by the current API - principal (403). The owner has not supplied runner state, and no runner - Online status or runner-hosted workflow is claimed. The manual - `runner-smoke.yaml` workflow is prepared but has not run. + restarts an active daemon and checks the running process environment. The + runner service's temporary `ProtectHome=read-only` workaround is also + committed to the installer with an exact known-drift migration guard. +- Runner jobs do not receive either Docker socket (`container.docker_host: "-"`) + or production secrets. PR #46 changes Caddyfile validation to use the pinned + Caddy binary directly; `docker compose config --quiet` remains a client-only + static check. The two production image builds still require a builder. The + PR audit rejects nested rootless BuildKit in the job container because it + requires weaker sandbox settings, and recommends a disposable, separately + isolated BuildKit VM reached through a short-lived, TLS-authenticated + `buildctl` connection. No VPS changes were made; image builds remain + unverified until the owner provisions that builder. +- Runner log lines for run 49 show act_runner 0.2.13 with registered label + `[akyldash-ci]` accepted task 301 at 2026-10-15 01:16:11 +05. The main CI + task failed at 01:16:24 +05, and smoke task 302 started immediately after. + These daemon logs do not record the failed step. Gitea 1.25.5's Actions jobs + endpoint returns an empty list for completed run 49; the current API account + cannot use the owner-only jobs endpoint to obtain the log download ID. The + exact failing step is undetermined. +- After pushing PR #46 commit `7701e77`, CI run 51 accepted task 303 at + 2026-10-15 01:48:05 +05 and failed 13 seconds later; smoke run 52 accepted + task 304 and succeeded. The runner daemon log again contains only task + dispatch lines. Gitea returns no jobs for run 303, and the owner-only jobs + endpoint remains unavailable to the current API account, so run 51's failed + step is also undetermined. This repeated result does not establish a cause. +- The owner supplied the complete log for task 305 (run 53): the failure was + `Check host audit root guard`, exit 1. Earlier backend (37 tests, 1 skipped), + Telegram (6), backup/restore (16), recovery host identity guard, and verifier + helper guard passed; Docker image builds were not reached. Reproduction with + `EUID=0` and `bash -x` confirmed the test ran the audit as root and then failed + because the expected non-root denial message was absent. Run 55 then confirmed + this test passed after dropping to UID 65534, while the next step, + `Check production config root guard`, failed for the same reason. The + Compose allowlist test passed. Root-context reproductions confirmed both + production-config and runner-installer guard tests had the same latent issue. + All three now use the shared `infra/test/run-as-unprivileged.sh` helper, which + has its own UID regression test. The production security guards are unchanged. - An isolated local Compose smoke test built backend/OpenSearch images, reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP 200 for `/review` through Caddy. Its containers, networks, and volumes were @@ -122,8 +177,8 @@ UFW rule details, and protected file metadata remain unverified. and restore of real production data remain unverified because production is not deployed. - Cloudflare edge response was checked; origin DNS target, origin certificate, - Full (strict), runner registration/Online state, registry publishing, and production - deployment have not been verified. + Full (strict), direct runner-label inventory, registry publishing, and + production deployment have not been verified. - This audit made no host changes. No package or service was installed by this audit; the owner performed the initial bootstrap beforehand. @@ -141,12 +196,10 @@ UFW rule details, and protected file metadata remain unverified. 3. Run `sudo bash /opt/akyldash/current/infra/deploy/install-helper.sh` after reviewing the checkout; it grants only fixed verification, Compose `config`/`status`, and guarded rollback commands. -4. After PR review and root audit, run - `sudo bash /opt/akyldash/current/infra/runner/install-runner.sh`. In the - repository Settings → Actions → Runners create a repository-level runner; - enter the token only at the interactive registration prompt as documented - in [CI_RUNNER.md](CI_RUNNER.md). Then enable the runner and health timer, - confirm **Online**, and dispatch the smoke workflow. Production deployment - remains manual and has no CI job. +4. After PR review and merge, install the reviewed checkout and rerun + `sudo bash /opt/akyldash/current/infra/runner/install-runner.sh` to pin the + effective `ProtectHome=read-only` setting. Then verify both workflows on + `akyldash-ci`; resolve the documented Docker-build isolation gap before + treating image-build CI as complete. Do not re-register the existing runner. 5. Test backup/restore and rollback only with synthetic data in an isolated environment before any production deployment. diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md index 167947c..03b4e37 100644 --- a/docs/operations/DEPLOYMENT.md +++ b/docs/operations/DEPLOYMENT.md @@ -65,13 +65,14 @@ backup/restore have been tested. sudo chmod 0600 /opt/akyldash/deployed-image.txt ``` -Application deployment remains owner-run. A repository-level Actions runner -installation is prepared in [CI_RUNNER.md](CI_RUNNER.md), but it is not yet -installed or registered; its Online state and smoke workflow are unverified. -The runner's job containers have no Docker socket or production secrets. CI has -no image publishing or production deployment step. The installed sudo helpers -support fixed verification, read-only Compose `config`/`status`, and guarded -rollback. They do not start or stop services. +Application deployment remains owner-run. The repository-level Actions runner +is installed and registered; the owner reports it Idle and the smoke workflow +passes. CI jobs have no rootless or system Docker socket and no production +secrets. The main CI workflow is being moved to `akyldash-ci`; Docker-dependent +image builds still need a compatible isolated builder design. CI has no image +publishing or production deployment step. The installed sudo helpers support +fixed verification, read-only Compose `config`/`status`, and guarded rollback. +They do not start or stop services. ## CI @@ -79,5 +80,6 @@ rollback. They do not start or stop services. production images on pull requests and pushes to `main`. It does not publish an image or deploy production. `.gitea/workflows/runner-smoke.yaml` exercises the `akyldash-ci` runner on pull requests, pushes to `main`, or manual dispatch. -Runner registration is incomplete, so neither workflow execution on that -runner nor current `ubuntu-latest` runner availability has been confirmed. +The current `main` workflow uses `ubuntu-latest`, while the registered runner +uses the project label `akyldash-ci`; the main CI job therefore has no matching +runner. The PR corrects the label. diff --git a/infra/bootstrap/test-audit-host.sh b/infra/bootstrap/test-audit-host.sh index 63a4adf..190cb4f 100644 --- a/infra/bootstrap/test-audit-host.sh +++ b/infra/bootstrap/test-audit-host.sh @@ -3,8 +3,16 @@ set -euo pipefail output=$(mktemp) trap 'rm -f "$output"' EXIT -if bash "$(dirname "$0")/audit-host.sh" >"$output" 2>&1; then - echo 'Host audit unexpectedly succeeded without root.' >&2 +audit_script="$(dirname "$0")/audit-host.sh" +run_as_unprivileged="$(dirname "$0")/../test/run-as-unprivileged.sh" +if bash "$run_as_unprivileged" bash "$audit_script" >"$output" 2>&1; then + audit_status=0 +else + audit_status=$? +fi +if ! grep -Fqx 'Run this read-only audit as root.' "$output"; then + echo "Host audit root guard did not reject unprivileged execution (audit exit code: $audit_status). Output:" >&2 + cat "$output" >&2 exit 1 fi -grep -qx 'Run this read-only audit as root.' "$output" +echo 'PASS host audit root guard rejects unprivileged execution' diff --git a/infra/deploy/test-production-config-guard.sh b/infra/deploy/test-production-config-guard.sh index cf7ed3e..90f8d49 100644 --- a/infra/deploy/test-production-config-guard.sh +++ b/infra/deploy/test-production-config-guard.sh @@ -3,8 +3,16 @@ set -euo pipefail output=$(mktemp) trap 'rm -f "$output"' EXIT -if bash "$(dirname "$0")/prepare-production-config.sh" >"$output" 2>&1; then - echo 'Production config setup unexpectedly succeeded without root.' >&2 +config_script="$(dirname "$0")/prepare-production-config.sh" +run_as_unprivileged="$(dirname "$0")/../test/run-as-unprivileged.sh" +if bash "$run_as_unprivileged" bash "$config_script" >"$output" 2>&1; then + config_status=0 +else + config_status=$? +fi +if ! grep -Fqx 'Run this configuration step as root.' "$output"; then + echo "Production config root guard did not reject unprivileged execution (exit code: $config_status). Output:" >&2 + cat "$output" >&2 exit 1 fi -grep -qx 'Run this configuration step as root.' "$output" +echo 'PASS production config root guard rejects unprivileged execution' diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml index c728847..9423419 100644 --- a/infra/inventory/production.yml +++ b/infra/inventory/production.yml @@ -111,9 +111,14 @@ target: job_limits: {cpu: 2, memory_gib: 3, pids: 512} account_slice_limits: {cpu: 2.5, memory_gib: 4, tasks: 2048} docker_store_limit_gib: 15 - docker_boundary: rootless dedicated daemon; no production socket; job socket injection disabled - service_status: configuration prepared, not installed or registered; Online unverified - owner_action: run the root installer from the reviewed checkout; register through repository runner settings + docker_boundary: dedicated rootless daemon; no system Docker socket; job socket injection disabled; no production secret or sudo/docker group access observed + service_status: active; owner reports registered and Idle; owner reports smoke workflow passes on akyldash-ci + systemd_protect_home: read-only in the live unit as a temporary owner change; installer update is in PR + systemd_verification: verify passes; security exposure 7.4 medium; live runner process UID/GID 999:988, effective/permitted capabilities 0, NoNewPrivs 1, no supplementary groups + runner_log_observed: /var/log/akyldash-runner/act_runner.log; codex ACL read works; current file root:root 0644 is also other-readable; log contains task metadata but no job step output + runner_log_acl_iac: root-owned directory with codex search-only ACL; runner-owned private log with codex read-only ACL; systemd and logrotate reapply; not installed on VPS + ci_status: main workflow uses ubuntu-latest in main and has no matching runner; PR changes it to akyldash-ci + owner_action: review and merge the PR, then rerun the root installer to pin ProtectHome and runner log ACLs; inspect run 49 step logs because current non-owner API lacks access checks: - infra/bootstrap/verify.sh akyldash_vps - docker compose -f deploy/production/compose.yaml ps diff --git a/infra/runner/install-runner-lib.sh b/infra/runner/install-runner-lib.sh index 0e00248..286e07c 100644 --- a/infra/runner/install-runner-lib.sh +++ b/infra/runner/install-runner-lib.sh @@ -58,6 +58,38 @@ runner_install_content() { rm -f "$temporary" } +runner_install_content_allow_line_change() { + local destination=$1 owner=$2 group=$3 mode=$4 old_line=$5 new_line=$6 + local temporary normalized expected_mode result=0 + temporary=$(mktemp) + normalized=$(mktemp) + cat >"$temporary" + expected_mode=${mode#0} + if [[ -e $destination || -L $destination ]]; then + [[ -f $destination && ! -L $destination && $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$expected_mode" ]] || { + rm -f "$temporary" "$normalized" + runner_die "existing runner file has unexpected type, owner, or permissions: $destination" + return 1 + } + if cmp -s "$temporary" "$destination"; then + rm -f "$temporary" "$normalized" + return + fi + awk -v old="$old_line" -v new="$new_line" '{ if ($0 == old) { $0 = new; count++ } print } END { if (count != 1) exit 1 }' "$destination" >"$normalized" || result=$? + if ((result != 0)) || ! cmp -s "$temporary" "$normalized"; then + rm -f "$temporary" "$normalized" + runner_die "existing runner file differs beyond the reviewed line change: $destination" + return 1 + fi + install -o "$owner" -g "$group" -m "$mode" "$temporary" "$destination" || result=$? + else + install -o "$owner" -g "$group" -m "$mode" "$temporary" "$destination" || result=$? + fi + rm -f "$temporary" "$normalized" + ((result == 0)) || return 1 + [[ -f $destination && ! -L $destination && $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$expected_mode" ]] || { runner_die "runner file failed validation: $destination"; return 1; } +} + runner_ensure_private_directory() { local directory=$1 owner=$2 group=$3 current= component index local -a components diff --git a/infra/runner/install-runner.sh b/infra/runner/install-runner.sh index e8fe971..12d5e6e 100644 --- a/infra/runner/install-runner.sh +++ b/infra/runner/install-runner.sh @@ -8,6 +8,8 @@ readonly runner_xdg_config_home="$runner_home/.config" readonly runner_xdg_data_home="$runner_runtime_home/.local/share" readonly runner_store="$runner_home/docker-data" readonly runner_image="$runner_home/docker-store.img" +readonly runner_log_dir=/var/log/akyldash-runner +readonly runner_log="$runner_log_dir/act_runner.log" readonly runner_mount_unit_name="$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)" readonly runner_mount_unit="/etc/systemd/system/$runner_mount_unit_name" readonly runner_image_bytes=16106127360 @@ -76,7 +78,7 @@ for file in /etc/subuid /etc/subgid; do awk -F: -v user="$runner_user" -v start="$subid_start" -v end="$subid_end" '$1 != user && $2 <= end && $2 + $3 - 1 >= start { conflict=1 } END { exit conflict }' "$file" || die "runner subordinate ID range conflicts with another entry in $file" done apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate docker-ce-rootless-extras +DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate acl docker-ce-rootless-extras runner_ensure_account "$runner_user" "$runner_home" || die 'runner account creation failed' runner_lock_home "$runner_home" || die 'runner home protection failed' @@ -106,8 +108,25 @@ if [[ ! -e $runner_store && ! -L $runner_store ]]; then install -d -o root -g root -m 0755 "$runner_store" fi [[ -d $runner_store && ! -L $runner_store ]] || die 'runner mount path is not a directory' -install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner -install -d -o "$runner_user" -g "$runner_user" -m 0750 /var/log/akyldash-runner +install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner "$runner_log_dir" +[[ -d $runner_log_dir && ! -L $runner_log_dir ]] || die 'runner log directory must be a real directory' +[[ ! -L $runner_log ]] || die 'runner log path must not be a symbolic link' +if [[ -e $runner_log ]]; then + [[ -f $runner_log ]] || die 'runner log path must be a regular file' +else + install -o "$runner_user" -g "$runner_user" -m 0600 /dev/null "$runner_log" +fi +chown "$runner_user:$runner_user" "$runner_log" +chmod 0600 "$runner_log" +setfacl -b "$runner_log_dir" +setfacl -k "$runner_log_dir" +setfacl -b "$runner_log" +setfacl -m u:codex:--x "$runner_log_dir" +setfacl -m u:codex:r--,g::---,m::r-- "$runner_log" +getfacl -cp "$runner_log_dir" | grep -Fqx 'user:codex:--x' || die 'runner log directory ACL is not restrictive' +getfacl -cp "$runner_log" | grep -Fqx 'user:codex:r--' || die 'runner log read ACL is missing' +getfacl -cp "$runner_log" | grep -Fqx 'group::---' || die 'runner log is readable by its group' +getfacl -cp "$runner_log" | grep -Fqx 'other::---' || die 'runner log is readable by other users' runner_ensure_image "$runner_image" "$runner_image_bytes" 0 711 || die 'runner image initialization failed' runner_ensure_mount "$runner_image" "$runner_store" "$runner_mount_unit_name" "$runner_mount_unit" || die 'runner mount activation failed' @@ -151,7 +170,7 @@ Environment=HOME=$runner_runtime_home Environment=XDG_CONFIG_HOME=$runner_xdg_config_home Environment=XDG_DATA_HOME=$runner_xdg_data_home EOF -runner_install_content /etc/systemd/system/akyldash-act-runner.service root root 0644 <&2 + exit 1 +fi +test "$(grep -Ec 'docker build -f deploy/production/Dockerfile\.(backend|opensearch)' "$workflow")" -eq 2 + +echo 'CI container operation checks passed' diff --git a/infra/runner/test-install-runner-guard.sh b/infra/runner/test-install-runner-guard.sh index 4e52819..e9f77b8 100644 --- a/infra/runner/test-install-runner-guard.sh +++ b/infra/runner/test-install-runner-guard.sh @@ -3,8 +3,16 @@ set -Eeuo pipefail output=$(mktemp) trap 'rm -f "$output"' EXIT -if bash "$(dirname "$0")/install-runner.sh" >"$output" 2>&1; then - echo 'Runner installer unexpectedly succeeded without root.' >&2 +installer="$(dirname "$0")/install-runner.sh" +run_as_unprivileged="$(dirname "$0")/../test/run-as-unprivileged.sh" +if bash "$run_as_unprivileged" bash "$installer" >"$output" 2>&1; then + installer_status=0 +else + installer_status=$? +fi +if ! grep -Fqx 'ERROR: run this script as root' "$output"; then + echo "Runner installer root guard did not reject unprivileged execution (exit code: $installer_status). Output:" >&2 + cat "$output" >&2 exit 1 fi -grep -qx 'ERROR: run this script as root' "$output" +echo 'PASS runner installer root guard rejects unprivileged execution' diff --git a/infra/runner/test-install-runner-idempotency.sh b/infra/runner/test-install-runner-idempotency.sh index f62f792..c5df0af 100644 --- a/infra/runner/test-install-runner-idempotency.sh +++ b/infra/runner/test-install-runner-idempotency.sh @@ -162,6 +162,31 @@ then exit 1 fi +migrated_unit=$tmp/akyldash-act-runner.service +printf '[Service]\nProtectHome=yes\nProtectSystem=strict\n' >"$migrated_unit" +chmod 0644 "$migrated_unit" +runner_install_content_allow_line_change "$migrated_unit" "$runner_owner" "$runner_group" 0644 'ProtectHome=yes' 'ProtectHome=read-only' <<'EOF' +[Service] +ProtectHome=read-only +ProtectSystem=strict +EOF +grep -Fqx 'ProtectHome=read-only' "$migrated_unit" +runner_install_content_allow_line_change "$migrated_unit" "$runner_owner" "$runner_group" 0644 'ProtectHome=yes' 'ProtectHome=read-only' <<'EOF' +[Service] +ProtectHome=read-only +ProtectSystem=strict +EOF +printf '[Service]\nProtectHome=read-only\nProtectSystem=no\n' >"$migrated_unit" +if runner_install_content_allow_line_change "$migrated_unit" "$runner_owner" "$runner_group" 0644 'ProtectHome=yes' 'ProtectHome=read-only' <<'EOF' 2>/dev/null +[Service] +ProtectHome=read-only +ProtectSystem=strict +EOF +then + echo 'Systemd unit with unrelated drift was accepted' >&2 + exit 1 +fi + image=$tmp/docker-store.img runner_ensure_image "$image" 1048576 "$(command id -u)" 700 runner_ensure_image "$image" 1048576 "$(command id -u)" 700 diff --git a/infra/runner/test-runner-config.sh b/infra/runner/test-runner-config.sh index b1267aa..51b9569 100644 --- a/infra/runner/test-runner-config.sh +++ b/infra/runner/test-runner-config.sh @@ -42,6 +42,19 @@ if grep -Fq 'env HOME="$runner_home"' "$installer"; then fi grep -Fq 'die '\''stop akyldash-act-runner.service after all jobs finish before rerunning this installer' "$installer" grep -Fq 'ReadWritePaths=$runner_runtime_home $runner_xdg_config_home /var/log/akyldash-runner /run/user/$runner_uid' "$installer" +grep -Fq 'ProtectHome=read-only' "$installer" +grep -Fq "runner_install_content_allow_line_change /etc/systemd/system/akyldash-act-runner.service root root 0644 'ProtectHome=yes' 'ProtectHome=read-only'" "$installer" +grep -Fq 'ProtectSystem=strict' "$installer" +grep -Fq 'NoNewPrivileges=yes' "$installer" +grep -Fq 'PrivateTmp=yes' "$installer" +grep -Fq 'ProtectKernelTunables=yes' "$installer" +grep -Fq 'ProtectKernelModules=yes' "$installer" +grep -Fq 'ProtectKernelLogs=yes' "$installer" +grep -Fq 'RestrictSUIDSGID=yes' "$installer" +if grep -Fxq 'ProtectHome=yes' "$installer"; then + echo 'Runner service must keep /run/user reachable with read-only home protection' >&2 + exit 1 +fi grep -Fq 'os.O_NOFOLLOW | os.O_CLOEXEC' infra/runner/secure-runner-image.py grep -Fq 'os.posix_fallocate(image_fd, 0, int(bytes_text))' infra/runner/secure-runner-image.py grep -Fq 'pass_fds=(image_fd,)' infra/runner/secure-runner-image.py @@ -58,8 +71,15 @@ grep -Fq 'find-free-subid-range.awk' "$installer" grep -Fq 'usermod "${subid_args[@]}" "$runner_user"' "$installer" grep -Fq 'subid_args+=(--add-subuids "$subid_range")' "$installer" assert_absent 'usermod.*(sudo|docker)|/var/run/docker.sock|/etc/sudoers' "$installer" -grep -Fq 'runs-on: akyldash-ci' .gitea/workflows/runner-smoke.yaml +for workflow in .gitea/workflows/*.yaml; do + grep -Fq 'runs-on: akyldash-ci' "$workflow" + if grep -Fq 'runs-on: ubuntu-latest' "$workflow"; then + echo "Workflow still targets the unavailable ubuntu-latest label: $workflow" >&2 + exit 1 + fi +done grep -Fq 'test ! -S /var/run/docker.sock' .gitea/workflows/runner-smoke.yaml +grep -Fq 'for socket in /run/user/*/docker.sock' .gitea/workflows/runner-smoke.yaml grep -Fq 'test ! -e /etc/akyldash/production.env' .gitea/workflows/runner-smoke.yaml grep -Fq 'test ! -e /etc/akyldash/review_secret' .gitea/workflows/runner-smoke.yaml assert_absent '(^|[[:space:]])(docker|docker-compose|deploy)([[:space:]]|$)|production.env=|review_secret=' .gitea/workflows/runner-smoke.yaml diff --git a/infra/runner/test-runner-log-acl.sh b/infra/runner/test-runner-log-acl.sh new file mode 100644 index 0000000..10df1e5 --- /dev/null +++ b/infra/runner/test-runner-log-acl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +installer=infra/runner/install-runner.sh + +grep -Fq 'logrotate acl docker-ce-rootless-extras' "$installer" +grep -Fq 'readonly runner_log_dir=/var/log/akyldash-runner' "$installer" +grep -Fq 'readonly runner_log="$runner_log_dir/act_runner.log"' "$installer" +grep -Fq 'install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner "$runner_log_dir"' "$installer" +grep -Fq 'install -o "$runner_user" -g "$runner_user" -m 0600 /dev/null "$runner_log"' "$installer" +grep -Fq 'chown "$runner_user:$runner_user" "$runner_log"' "$installer" +grep -Fq 'chmod 0600 "$runner_log"' "$installer" +grep -Fq 'setfacl -b "$runner_log_dir"' "$installer" +grep -Fq 'setfacl -k "$runner_log_dir"' "$installer" +grep -Fq 'setfacl -b "$runner_log"' "$installer" +grep -Fq 'setfacl -m u:codex:--x "$runner_log_dir"' "$installer" +grep -Fq 'setfacl -m u:codex:r--,g::---,m::r-- "$runner_log"' "$installer" +grep -Fq "grep -Fqx 'group::---'" "$installer" +grep -Fq "grep -Fqx 'other::---'" "$installer" +grep -Fq 'UMask=0077' "$installer" +grep -Fq 'ExecStartPost=/usr/bin/setfacl -m u:codex:r--,g::---,m::r-- /var/log/akyldash-runner/act_runner.log' "$installer" +grep -Fq ' copytruncate' "$installer" +grep -Fq ' postrotate' "$installer" +grep -Fq ' /usr/bin/setfacl -m u:codex:r--,g::---,m::r-- /var/log/akyldash-runner/act_runner.log' "$installer" +grep -Fq ' endscript' "$installer" + +if grep -Eq 'setfacl .*(-R|default:)|chmod 0644.*act_runner.log|other::r--' "$installer"; then + echo 'Runner log ACL must stay limited to codex reading the single log file' >&2 + exit 1 +fi + +echo 'runner log ACL checks passed' diff --git a/infra/test/run-as-unprivileged.sh b/infra/test/run-as-unprivileged.sh new file mode 100644 index 0000000..7e6e07a --- /dev/null +++ b/infra/test/run-as-unprivileged.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# == 0)); then + echo 'Usage: run-as-unprivileged.sh command [args...]' >&2 + exit 2 +fi + +if [[ $EUID -eq 0 ]]; then + if ! command -v setpriv >/dev/null 2>&1; then + echo 'Cannot run the guard test as non-root: setpriv is required.' >&2 + exit 1 + fi + exec setpriv --reuid=65534 --regid=65534 --clear-groups -- "$@" +fi + +exec "$@" diff --git a/infra/test/test-run-as-unprivileged.sh b/infra/test/test-run-as-unprivileged.sh new file mode 100644 index 0000000..05e9a81 --- /dev/null +++ b/infra/test/test-run-as-unprivileged.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +helper="$(dirname "$0")/run-as-unprivileged.sh" +expected_uid=$EUID +if [[ $EUID -eq 0 ]]; then + expected_uid=65534 +fi + +if ! actual_uid=$(bash "$helper" id -u); then + echo 'Failed to run the test command as an unprivileged user.' >&2 + exit 1 +fi +if [[ $actual_uid != "$expected_uid" ]]; then + echo "Expected test command UID $expected_uid, got $actual_uid." >&2 + exit 1 +fi + +echo "PASS test command runs as UID $actual_uid"