From 73aa147881985676ead7a151befb772b5c8e8433 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Wed, 14 Oct 2026 23:16:03 +0300 Subject: [PATCH] ci: validate Caddy without Docker and document BuildKit path --- .gitea/workflows/ci.yaml | 12 ++++- docs/operations/CI_RUNNER.md | 51 +++++++++++++++++--- docs/operations/CURRENT_STATE.md | 11 +++-- infra/runner/test-ci-container-operations.sh | 16 ++++++ 4 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 infra/runner/test-ci-container-operations.sh diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 40f6e70..3f3a22c 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -32,6 +32,7 @@ jobs: - name: Check runner config and isolation guards run: | bash infra/runner/test-runner-config.sh + bash infra/runner/test-ci-container-operations.sh bash infra/runner/test-install-runner-idempotency.sh bash infra/runner/test-install-runner-guard.sh - name: Check infrastructure shell syntax @@ -43,6 +44,7 @@ jobs: infra/bootstrap/test-audit-host.sh infra/bootstrap/test-verify-helper.sh \ infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \ infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \ + infra/runner/test-ci-container-operations.sh \ infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh python3 -c "import ast,pathlib; ast.parse(pathlib.Path('infra/runner/secure-runner-image.py').read_text())" - name: Run rollback helper tests @@ -50,9 +52,15 @@ jobs: - name: Validate production Compose and Caddyfile run: | secret_file=$(mktemp) - trap 'rm -f "$secret_file"' EXIT + caddy_dir=$(mktemp -d) + trap 'rm -f "$secret_file"; rm -rf "$caddy_dir"' EXIT REVIEW_SECRET_FILE="$secret_file" docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet - docker run --rm --entrypoint caddy -v "$PWD/deploy/production/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.11.6-alpine@sha256:881bbc60f9986d5ab8e7cfd6cf7e4ef3c9c0439fef2429d035d065577882f028 validate --config /etc/caddy/Caddyfile --adapter caddyfile + test "$(uname -m)" = x86_64 + caddy_archive="$caddy_dir/caddy.tar.gz" + curl --fail --location --retry 3 --output "$caddy_archive" https://github.com/caddyserver/caddy/releases/download/v2.11.6/caddy_2.11.6_linux_amd64.tar.gz + echo '422771007d505ea97efd1177a4905b2c1a471cd426668f2ace3bcda3d8e30b11f9b1610bfb02c6ad60f2a795f56124f2f5eec6409c17d5a0dd4c21a11375fb94 '"$caddy_archive" | sha512sum --check + tar -xzf "$caddy_archive" -C "$caddy_dir" caddy + "$caddy_dir/caddy" validate --config deploy/production/Caddyfile --adapter caddyfile - name: Build production images run: | docker build -f deploy/production/Dockerfile.backend -t akyldash-backend:ci . diff --git a/docs/operations/CI_RUNNER.md b/docs/operations/CI_RUNNER.md index 3fa879d..07d2817 100644 --- a/docs/operations/CI_RUNNER.md +++ b/docs/operations/CI_RUNNER.md @@ -55,12 +55,51 @@ API account cannot independently list the registered labels; the owner reports that the runner is Idle and the smoke workflow succeeds. The job container receives neither the rootless Docker socket nor the system -Docker socket (`container.docker_host: "-"`). Workflows must not call -`docker run`, `docker build`, or another command that needs a daemon under this -boundary. The existing `ci.yaml` image-build commands need a separately -isolated build path before the full CI job can pass; do not expose the runner -account's rootless socket to jobs because it would expose the runner account's -files. +Docker socket (`container.docker_host: "-"`). The CI operations have different +engine requirements: + +| Operation | Purpose | Engine requirement | +| --- | --- | --- | +| `docker compose ... config --quiet` | Interpolate and validate production Compose configuration | Compose client only; this command does not start containers or contact a daemon. | +| Caddy `validate` | Check the production Caddyfile syntax and directives | Caddy binary only; CI downloads the exact version used by the pinned image and checks its SHA-512. | +| `docker build` for backend and OpenSearch | Exercise Dockerfile instructions, base-image resolution, and the OpenSearch plugin install | A container builder is required; ordinary Python/shell tests cannot replace this check. | + +Do not run rootless BuildKit inside the current job container. The official +BuildKit rootless container recipe requires relaxing seccomp/AppArmor and +system path restrictions; its documented no-process-sandbox fallback lets +build steps signal processes in the BuildKit daemon and is explicitly +discouraged. This runner intentionally keeps its job security profile intact. + +The recommended image-build path is a separate, disposable BuildKit VM, with +the current runner acting only as a `buildctl` client. Run BuildKit as a +non-root account with its own subordinate UID/GID range and scratch disk; use +mutual TLS over a private network, no Docker socket, no production credentials, +no route to production, one build at a time, and delete the VM and cache after +the job. Keep the test job on `akyldash-ci`; place image builds in a dependent +job on a separate builder label. This keeps arbitrary PR Dockerfile commands +off the runner and production host kernel. Gitea 1.25.5 and act_runner 0.2.13 +need no special BuildKit integration: the workflow invokes pinned `buildctl` +against the remote BuildKit API. + +For a first sizing estimate, allocate the disposable builder 2 vCPU, 4 GiB RAM, +and 10 GiB temporary disk at concurrency 1. Disable persistent cache or cap it +to 6 GiB with BuildKit garbage collection; remove OCI output archives after +validation. These are starting limits, not measured requirements. Reduce them +only after recording peak memory and disk use for both Dockerfiles. The current +VPS has 7.7 GiB RAM and the runner already has 3 GiB job / 4 GiB account limits, +so colocating a builder there would contend with jobs and share the host kernel; +it is not the recommended deployment. + +Provisioning that disposable VM and private network is an owner-level bootstrap +step. It must pin BuildKit and its checksum, create the non-root account and +subuid/subgid mapping, enforce CPU/RAM/disk and single-job limits, expose only a +TLS-authenticated BuildKit endpoint to the runner, issue a short-lived +per-job client certificate, and destroy the VM after the build. Do not enable +`--privileged`, relax seccomp/AppArmor, set +`--oci-worker-no-process-sandbox`, mount either Docker socket, or provide +production secrets. Until this bootstrap is approved and available, keep both +image builds required and expect the full CI job to remain blocked at those +steps. Production deployment is not automated. ## Installation and registration diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index 8cf14f9..3ea5cf8 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -119,9 +119,14 @@ unverified. runner service's temporary `ProtectHome=read-only` workaround is also committed to the installer with an exact known-drift migration guard. - Runner jobs do not receive either Docker socket (`container.docker_host: "-"`) - or production secrets. As a result, Docker-dependent `docker run`/`docker - build` steps in `ci.yaml` must be reconciled with that boundary; changing its - label alone may not make the full image job runnable. + or production secrets. PR #46 changes Caddyfile validation to use the pinned + Caddy binary directly; `docker compose config --quiet` remains a client-only + static check. The two production image builds still require a builder. The + PR audit rejects nested rootless BuildKit in the job container because it + requires weaker sandbox settings, and recommends a disposable, separately + isolated BuildKit VM reached through a short-lived, TLS-authenticated + `buildctl` connection. No VPS changes were made; image builds remain + unverified until the owner provisions that builder. - An isolated local Compose smoke test built backend/OpenSearch images, reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP 200 for `/review` through Caddy. Its containers, networks, and volumes were diff --git a/infra/runner/test-ci-container-operations.sh b/infra/runner/test-ci-container-operations.sh new file mode 100644 index 0000000..fb9678f --- /dev/null +++ b/infra/runner/test-ci-container-operations.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +workflow=.gitea/workflows/ci.yaml + +grep -Fq 'docker compose --env-file deploy/production/.env.example -f deploy/production/compose.yaml config --quiet' "$workflow" +grep -Fq 'caddy_2.11.6_linux_amd64.tar.gz' "$workflow" +grep -Fq 'sha512sum --check' "$workflow" +grep -Fq 'caddy" validate --config deploy/production/Caddyfile --adapter caddyfile' "$workflow" +if grep -Eq '(^|[[:space:]])docker run([[:space:]]|$)' "$workflow"; then + echo 'CI must validate Caddyfile directly in the job container, without docker run' >&2 + exit 1 +fi +test "$(grep -Ec 'docker build -f deploy/production/Dockerfile\.(backend|opensearch)' "$workflow")" -eq 2 + +echo 'CI container operation checks passed'