Make runner installer safe to resume

.gitea/workflows/ci.yaml

@@ -32,6 +32,7 @@ jobs:
       - name: Check runner config and isolation guards
         run: |
           bash infra/runner/test-runner-config.sh
+          bash infra/runner/test-install-runner-idempotency.sh
           bash infra/runner/test-install-runner-guard.sh
       - name: Check infrastructure shell syntax
         run: |
@@ -40,8 +41,9 @@ jobs:
             infra/deploy/install-helper.sh infra/deploy/compose-helper.sh infra/deploy/test-compose-helper.sh \
             infra/deploy/prepare-production-config.sh infra/deploy/test-production-config-guard.sh \
             infra/bootstrap/test-audit-host.sh infra/bootstrap/test-verify-helper.sh \
-            infra/runner/install-runner.sh infra/runner/act-runner-healthcheck.sh \
-            infra/runner/test-runner-config.sh infra/runner/test-install-runner-guard.sh
+            infra/runner/install-runner.sh infra/runner/install-runner-lib.sh \
+            infra/runner/act-runner-healthcheck.sh infra/runner/test-runner-config.sh \
+            infra/runner/test-install-runner-idempotency.sh infra/runner/test-install-runner-guard.sh
       - name: Run rollback helper tests
         run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v
       - name: Validate production Compose and Caddyfile

docs/operations/CI_RUNNER.md

@@ -49,16 +49,24 @@ for runner images/workspaces; and configures systemd services. It starts only
 the separate rootless Docker daemon; the runner and health timer stay disabled
 until repository registration. It does not change
 SSH/UFW, production Docker configuration, application data, or production
-secrets. It refuses to overwrite existing runner paths and requires at least
-25 GiB free on `/` before reserving storage.
+secrets. It can be rerun after partial setup: it validates and reuses an
+existing runner account, subordinate ID mappings, image, and mount unit. An
+existing image must be exactly 15 GiB and contain ext4; a mismatch stops the
+installer without resizing or formatting it. The runner account must retain
+a non-root system UID below `UID_MIN`, its expected home, primary group, and
+sole supplementary group. The mount
+is started only when it is not already active, and an active mount must be the
+expected ext4 loop device backed by the runner image. Conflicting files or
+state stop the installer for manual inspection. A new image requires at least
+25 GiB free on `/`.
 
 If the reviewed checkout is not present at the inventoried path, first verify
-that the target is absent and clone the reviewed PR branch there. This path is
-currently absent per the read-only audit; do not overwrite it if that changes.
+that the target is absent and clone the reviewed `main` there. Do not overwrite
+an existing checkout.
 
 ```bash
 sudo test ! -e /opt/akyldash/current
-sudo git clone --branch feature/infrastructure-deployment-phase-2 \
+sudo git clone --branch main \
   --single-branch https://git.vakanaut.ru/admin/akyldash.git \
   /opt/akyldash/current
 ```
@@ -114,16 +122,22 @@ that token; do not put it in command-line arguments or environment variables.
 
 Update `runner_version` and its expected official SHA-256 together in
 `infra/runner/install-runner.sh`, review upstream release compatibility with
-the installed Gitea, and open a PR. From the reviewed checkout, re-run the
-installer only after resolving its existing-path guards. Do not replace the
-binary while a job is active. Stop the runner, install/verify the new binary,
-then start it and confirm Online plus a smoke run. Preserve `/var/lib/akyldash-runner/.runner`.
+the installed Gitea, and open a PR. Do not replace the binary while a job is
+active. Stop the runner, install/verify the new binary, then start it and
+confirm Online plus a smoke run. Preserve `/var/lib/akyldash-runner/.runner`.
 
 ## Recovery and rollback
 
 The runner is disposable: production data and secrets are not on its filesystem.
 For a runner process failure, inspect `systemctl status` and
 `journalctl -u akyldash-act-runner.service`; restart the fixed runner service.
+If installation stops after creating or enabling the mount unit, inspect the
+image and mount with `stat`, `blkid`, `findmnt`, and `losetup`, then rerun the
+reviewed installer only when the image is 15 GiB ext4 and the active mount is
+backed by it. The installer recognizes that state and does not format or
+restart the mounted filesystem. If size, filesystem, unit contents, or backing
+device differ, stop and resolve the state manually; never format the existing
+image as a retry.
 For lost/corrupt registration state, stop the service and obtain a fresh
 repository-level token from Settings → Actions → Runners. Preserve the 15 GiB
 data image and state for diagnosis; re-register only after the repository
@@ -133,6 +147,9 @@ automatic recovery step.
 
 ## Status
 
-The configuration and scripts are prepared in Git. They have not been
-installed on the VPS, a repository-level token is not available to this agent,
-and no runner Online state or workflow execution is claimed yet.
+The root installer and registration are incomplete. The owner reports that the
+15 GiB ext4 loop-backed mount is active on `/dev/loop0` after manually running
+`systemctl daemon-reload`, `enable`, and `start` when the first installer run
+stopped at mount activation. This PR has not changed the VPS and does not claim
+that remaining runner components are installed, a repository token is
+available, the runner is Online, or a workflow has run.

docs/operations/CURRENT_STATE.md

@@ -86,8 +86,11 @@ UFW rule details, and protected file metadata remain unverified.
   3 GiB RAM, 512 processes), account slice limits (2.5 CPU, 4 GiB RAM, 2048
   tasks), rotated file logs, and a systemd health timer. Jobs use the
   `akyldash-ci` label and do not receive the rootless or production Docker
-  socket. This is repository code only: no runner package, account, store,
-  service, token, or registration is installed on the VPS.
+  socket. The owner reports a partial runner installation: the 15 GiB ext4
+  loop-backed store is active on `/dev/loop0` after manually enabling and
+  starting its mount unit when the installer stopped. Remaining components,
+  repository registration, and Online state are unverified; this PR makes no
+  VPS changes.
 - Gitea repository runner inventory could not be read by the current API
   principal (403). The owner has not supplied runner state, and no runner
   Online status or runner-hosted workflow is claimed. The manual

infra/runner/install-runner-lib.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+runner_die() { printf 'ERROR: %s\n' "$*" >&2; return 1; }
+
+runner_ensure_account() {
+  local user=$1 home=$2 entry uid passwd_home shell groups uid_min
+  if ! entry=$(getent passwd "$user"); then
+    [[ ! -e $home && ! -L $home ]] || { runner_die "runner home already exists without its account: $home"; return 1; }
+    useradd --system --create-home --home-dir "$home" --shell /usr/sbin/nologin "$user" || return 1
+    return
+  fi
+  IFS=: read -r _ _ uid _ _ passwd_home shell <<<"$entry"
+  uid_min=$(awk '$1 == "UID_MIN" && $2 ~ /^[0-9]+$/ {print $2; exit}' /etc/login.defs)
+  [[ $uid =~ ^[0-9]+$ && $uid -gt 0 && $uid_min =~ ^[0-9]+$ && $uid -lt $uid_min && $passwd_home == "$home" && $shell == /usr/sbin/nologin ]] || { runner_die "existing $user account has an unexpected UID, home, or shell"; return 1; }
+  [[ $(id -gn "$user") == "$user" ]] || { runner_die "existing $user account has an unexpected primary group"; return 1; }
+  groups=$(id -nG "$user")
+  [[ $groups == "$user" ]] || { runner_die "existing $user account has unexpected supplementary groups: $groups"; return 1; }
+  [[ -d $home && ! -L $home && $(stat -c %U "$home") == "$user" ]] || { runner_die "existing $user home is missing or has unexpected ownership"; return 1; }
+}
+
+runner_ensure_image() {
+  local image=$1 size=$2 bytes=$3 actual filesystem
+  if [[ -e $image || -L $image ]]; then
+    [[ -f $image && ! -L $image ]] || { runner_die "existing runner image is not a regular file: $image"; return 1; }
+    actual=$(stat -c %s "$image") || return
+    [[ $actual == "$bytes" ]] || { runner_die "existing runner image has size $actual bytes; expected $bytes; refusing to resize or format"; return 1; }
+    filesystem=$(blkid -p -s TYPE -o value "$image" 2>/dev/null || true)
+    [[ $filesystem == ext4 ]] || { runner_die "existing runner image does not contain ext4; refusing to format"; return 1; }
+    return
+  fi
+  fallocate -l "$size" "$image" || return 1
+  mkfs.ext4 -F -q "$image" || return 1
+  actual=$(stat -c %s "$image") || return
+  filesystem=$(blkid -p -s TYPE -o value "$image" 2>/dev/null || true)
+  [[ $actual == "$bytes" && $filesystem == ext4 ]] || { runner_die 'new runner image failed size or ext4 verification'; return 1; }
+}
+
+runner_install_file() {
+  local source=$1 destination=$2 owner=$3 group=$4 mode=${5#0}
+  if [[ -e $destination || -L $destination ]]; then
+    [[ -f $destination && ! -L $destination ]] && cmp -s "$source" "$destination" || { runner_die "existing runner file differs from reviewed configuration: $destination"; return 1; }
+    [[ $(stat -c '%U:%G:%a' "$destination") == "$owner:$group:$mode" ]] || { runner_die "existing runner file has unexpected ownership or permissions: $destination"; return 1; }
+    return
+  fi
+  install -o "$owner" -g "$group" -m "0$mode" "$source" "$destination"
+}
+
+runner_install_content() {
+  local destination=$1 owner=$2 group=$3 mode=$4 temporary
+  temporary=$(mktemp)
+  cat >"$temporary"
+  if ! runner_install_file "$temporary" "$destination" "$owner" "$group" "$mode"; then
+    rm -f "$temporary"
+    return 1
+  fi
+  rm -f "$temporary"
+}
+
+runner_validate_mount() {
+  local image=$1 store=$2 target filesystem source backing mount_info
+  mount_info=$(findmnt --noheadings --output TARGET,FSTYPE,SOURCE --mountpoint "$store") || { runner_die 'runner store is not mounted'; return 1; }
+  read -r target filesystem source <<<"$mount_info" || { runner_die 'runner store mount information is empty'; return 1; }
+  [[ $target == "$store" && $filesystem == ext4 && $source == /dev/loop* ]] || { runner_die 'runner store is mounted from an unexpected target, filesystem, or device'; return 1; }
+  backing=$(losetup --noheadings --output BACK-FILE "$source") || { runner_die 'cannot identify the runner loop backing file'; return 1; }
+  [[ $(realpath -e "$backing") == "$(realpath -e "$image")" ]] || { runner_die 'runner store is backed by a different image'; return 1; }
+}
+
+runner_ensure_mount() {
+  local image=$1 store=$2 unit_name=$3 unit_file=$4 unit_tmp mounted=0
+  [[ -d $store && ! -L $store ]] || { runner_die "runner mount point is missing or is not a directory: $store"; return 1; }
+  if mountpoint -q "$store"; then
+    runner_validate_mount "$image" "$store" || return 1
+    mounted=1
+  fi
+  unit_tmp=$(mktemp)
+  if ! cat >"$unit_tmp" <<EOF
+[Unit]
+Description=Bounded storage for the AkylDash CI runner
+Before=local-fs.target
+
+[Mount]
+What=$image
+Where=$store
+Type=ext4
+Options=loop,nosuid,nodev
+
+[Install]
+WantedBy=local-fs.target
+EOF
+  then
+    rm -f "$unit_tmp"
+    runner_die 'failed to prepare runner mount unit'
+    return 1
+  fi
+  if [[ -e $unit_file || -L $unit_file ]]; then
+    if [[ ! -f $unit_file || -L $unit_file ]] || ! cmp -s "$unit_tmp" "$unit_file"; then
+      rm -f "$unit_tmp"
+      runner_die "existing runner mount unit differs from the reviewed configuration: $unit_file"
+      return 1
+    fi
+    [[ $(stat -c '%u:%g:%a' "$unit_file") == "$(id -u):$(id -g):644" ]] || { rm -f "$unit_tmp"; runner_die 'existing runner mount unit has unexpected ownership or permissions'; return 1; }
+  else
+    install -m 0644 "$unit_tmp" "$unit_file" || { rm -f "$unit_tmp"; return 1; }
+  fi
+  rm -f "$unit_tmp"
+
+  systemctl daemon-reload || return 1
+  if ((mounted)); then
+    systemctl enable "$unit_name" || return 1
+  else
+    systemctl enable "$unit_name" || return 1
+    systemctl start "$unit_name" || return 1
+    runner_validate_mount "$image" "$store" || return 1
+  fi
+}

infra/runner/install-runner.sh

@@ -5,7 +5,9 @@ readonly runner_user=akyldash-runner
 readonly runner_home=/var/lib/akyldash-runner
 readonly runner_store="$runner_home/docker-data"
 readonly runner_image="$runner_home/docker-store.img"
-readonly runner_mount_unit="/etc/systemd/system/$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)"
+readonly runner_mount_unit_name="$(systemd-escape --path --suffix=mount /var/lib/akyldash-runner/docker-data)"
+readonly runner_mount_unit="/etc/systemd/system/$runner_mount_unit_name"
+readonly runner_image_bytes=16106127360
 readonly runner_version=0.2.13
 readonly runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2
 readonly runner_url="https://dl.gitea.com/gitea-runner/$runner_version/act_runner-$runner_version-linux-amd64"
@@ -28,70 +30,92 @@ done
 unsafe=$(find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit)
 [[ -z $unsafe ]] || die 'the repository checkout contains files writable by non-root users'
 git -C "$checkout" diff --quiet HEAD -- infra/runner || die 'runner files differ from the reviewed Git revision'
+source "$checkout/infra/runner/install-runner-lib.sh"
 [[ -r /etc/subuid && -r /etc/subgid ]] || die 'subordinate ID allocation files are unavailable'
-subid_range=$(awk -f "$(dirname "$0")/find-free-subid-range.awk" /etc/subuid /etc/subgid) || die 'could not find a free 65536-ID range shared by /etc/subuid and /etc/subgid'
+if getent passwd "$runner_user" >/dev/null; then
+  runner_ensure_account "$runner_user" "$runner_home" || die 'runner account validation failed'
+else
+  [[ ! -e $runner_home && ! -L $runner_home ]] || die 'runner home exists without its account; inspect it before continuing'
+fi
+if [[ -e $runner_image || -L $runner_image ]]; then
+  runner_ensure_image "$runner_image" 15G "$runner_image_bytes" || die 'runner image validation failed'
+else
+  [[ $(df -Pk / | awk 'NR==2 {print $4}') -ge 25000000 ]] || die 'need at least 25 GiB free before reserving 15 GiB for runner data'
+fi
 
-for path in "$runner_home" "$runner_image" "$runner_store" /etc/akyldash-runner; do
-  [[ ! -e $path ]] || die "$path already exists; inspect it and follow the documented recovery procedure"
+for file in /etc/subuid /etc/subgid; do
+  awk -F: '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ && (NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ || $3 < 1) { bad=1 } END { exit bad }' "$file" || die "$file contains an invalid subordinate ID entry"
+done
+subuid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subuid) || die 'runner has multiple subuid allocations; inspect them before continuing'
+subgid_pair=$(awk -F: -v user="$runner_user" '$1 == user { count++; pair=$2 ":" $3 } END { if (count > 1) exit 2; if (count == 1) print pair }' /etc/subgid) || die 'runner has multiple subgid allocations; inspect them before continuing'
+for pair in "$subuid_pair" "$subgid_pair"; do
+  [[ -z $pair || $pair =~ ^[0-9]+:65536$ ]] || die 'existing runner subordinate ID allocation is not a single 65536-ID range'
+done
+if [[ -n $subuid_pair && -n $subgid_pair && $subuid_pair != "$subgid_pair" ]]; then
+  die 'runner subuid and subgid allocations differ; inspect them before continuing'
+fi
+existing_pair=${subuid_pair:-$subgid_pair}
+if [[ -n $existing_pair ]]; then
+  subid_start=${existing_pair%%:*}
+  [[ $subid_start =~ ^[1-9][0-9]*$ && $subid_start -ge 100000 && $subid_start -le 4294901760 ]] || die 'existing runner subordinate ID range is outside the supported range'
+  subid_end=$((subid_start + 65535))
+  subid_range="$subid_start-$subid_end"
+else
+  subid_range=$(awk -f "$(dirname "$0")/find-free-subid-range.awk" /etc/subuid /etc/subgid) || die 'could not find a free 65536-ID range shared by /etc/subuid and /etc/subgid'
+  subid_start=${subid_range%-*}
+  subid_end=${subid_range#*-}
+fi
+for file in /etc/subuid /etc/subgid; do
+  awk -F: -v user="$runner_user" -v start="$subid_start" -v end="$subid_end" '$1 != user && $2 <= end && $2 + $3 - 1 >= start { conflict=1 } END { exit conflict }' "$file" || die "runner subordinate ID range conflicts with another entry in $file"
 done
-! id "$runner_user" >/dev/null 2>&1 || die "user $runner_user already exists; inspect it before continuing"
-[[ ! -e /usr/local/bin/act_runner ]] || die '/usr/local/bin/act_runner already exists'
-[[ ! -e /etc/systemd/system/akyldash-act-runner.service ]] || die 'runner service already exists'
-[[ ! -e /etc/systemd/system/akyldash-runner-healthcheck.service ]] || die 'runner health monitor already exists'
-[[ ! -e /etc/logrotate.d/akyldash-runner ]] || die 'runner logrotate policy already exists'
-[[ ! -e $runner_mount_unit ]] || die 'runner data mount unit already exists'
-[[ $(df -Pk / | awk 'NR==2 {print $4}') -ge 25000000 ]] || die 'need at least 25 GiB free before reserving 15 GiB for runner data'
-
 apt-get update
 DEBIAN_FRONTEND=noninteractive apt-get install -y uidmap dbus-user-session curl logrotate docker-ce-rootless-extras
 
-useradd --system --create-home --home-dir "$runner_home" --shell /usr/sbin/nologin "$runner_user"
-usermod --add-subuids "$subid_range" --add-subgids "$subid_range" "$runner_user"
+runner_ensure_account "$runner_user" "$runner_home" || die 'runner account creation failed'
+subid_args=()
+[[ -n $subuid_pair ]] || subid_args+=(--add-subuids "$subid_range")
+[[ -n $subgid_pair ]] || subid_args+=(--add-subgids "$subid_range")
+if ((${#subid_args[@]})); then
+  usermod "${subid_args[@]}" "$runner_user"
+fi
 runner_uid=$(id -u "$runner_user")
 
-install -d -o "$runner_user" -g "$runner_user" -m 0700 "$runner_home" "$runner_store" "$runner_home/.config/docker" "$runner_home/.config/systemd/user/docker.service.d"
+install -d -o "$runner_user" -g "$runner_user" -m 0700 "$runner_home" "$runner_home/.config/docker" "$runner_home/.config/systemd/user/docker.service.d"
+if [[ ! -e $runner_store && ! -L $runner_store ]]; then
+  install -d -o root -g root -m 0755 "$runner_store"
+fi
+[[ -d $runner_store && ! -L $runner_store ]] || die 'runner mount path is not a directory'
 install -d -o root -g "$runner_user" -m 0750 /etc/akyldash-runner
 install -d -o "$runner_user" -g "$runner_user" -m 0750 /var/log/akyldash-runner
 
-fallocate -l 15G "$runner_image"
-mkfs.ext4 -F -q "$runner_image"
-cat >"$runner_mount_unit" <<EOF
-[Unit]
-Description=Bounded storage for the AkylDash CI runner
-Before=local-fs.target
-
-[Mount]
-What=$runner_image
-Where=$runner_store
-Type=ext4
-Options=loop,nosuid,nodev
-
-[Install]
-WantedBy=local-fs.target
-EOF
-systemctl daemon-reload
-systemctl enable --now "$(basename "$runner_mount_unit" .mount)"
+runner_ensure_image "$runner_image" 15G "$runner_image_bytes" || die 'runner image initialization failed'
+runner_ensure_mount "$runner_image" "$runner_store" "$runner_mount_unit_name" "$runner_mount_unit" || die 'runner mount activation failed'
 chown "$runner_user:$runner_user" "$runner_store"
 chmod 0700 "$runner_store"
 
-install -o root -g root -m 0755 "$(dirname "$0")/act-runner-healthcheck.sh" /usr/local/sbin/akyldash-runner-healthcheck
-install -o root -g "$runner_user" -m 0640 "$(dirname "$0")/config.yaml" /etc/akyldash-runner/config.yaml
-tmp_binary=$(mktemp)
-trap 'rm -f "$tmp_binary"' EXIT
-curl --fail --location --silent --show-error "$runner_url" -o "$tmp_binary"
-printf '%s  %s\n' "$runner_sha256" "$tmp_binary" | sha256sum --check --status || die 'official act_runner checksum mismatch'
-install -o root -g root -m 0755 "$tmp_binary" /usr/local/bin/act_runner
+runner_install_file "$(dirname "$0")/act-runner-healthcheck.sh" /usr/local/sbin/akyldash-runner-healthcheck root root 0755 || die 'healthcheck command differs from reviewed configuration'
+runner_install_file "$(dirname "$0")/config.yaml" /etc/akyldash-runner/config.yaml root "$runner_user" 0640 || die 'runner config differs from reviewed configuration'
+if [[ -e /usr/local/bin/act_runner || -L /usr/local/bin/act_runner ]]; then
+  [[ -f /usr/local/bin/act_runner && ! -L /usr/local/bin/act_runner ]] || die 'existing act_runner path is not a regular file'
+  printf '%s  %s\n' "$runner_sha256" /usr/local/bin/act_runner | sha256sum --check --status || die 'existing act_runner binary does not match the pinned release'
+else
+  tmp_binary=$(mktemp)
+  trap 'rm -f "$tmp_binary"' EXIT
+  curl --fail --location --silent --show-error "$runner_url" -o "$tmp_binary"
+  printf '%s  %s\n' "$runner_sha256" "$tmp_binary" | sha256sum --check --status || die 'official act_runner checksum mismatch'
+  install -o root -g root -m 0755 "$tmp_binary" /usr/local/bin/act_runner
+  rm -f "$tmp_binary"
+  trap - EXIT
+fi
 
-cat >"$runner_home/.config/docker/daemon.json" <<EOF
+runner_install_content "$runner_home/.config/docker/daemon.json" "$runner_user" "$runner_user" 0600 <<EOF
 {
   "data-root": "$runner_store",
   "log-driver": "local",
   "log-opts": {"max-size": "10m", "max-file": "2"}
 }
 EOF
-chown "$runner_user:$runner_user" "$runner_home/.config/docker/daemon.json"
-chmod 0600 "$runner_home/.config/docker/daemon.json"
-cat >"$runner_home/.config/systemd/user/docker.service.d/limits.conf" <<'EOF'
+runner_install_content "$runner_home/.config/systemd/user/docker.service.d/limits.conf" "$runner_user" "$runner_user" 0644 <<'EOF'
 [Service]
 Delegate=yes
 CPUQuota=200%
@@ -100,9 +124,7 @@ TasksMax=1024
 Restart=always
 RestartSec=5
 EOF
-chown -R "$runner_user:$runner_user" "$runner_home/.config"
-
-install -o root -g root -m 0644 /dev/stdin /etc/systemd/system/akyldash-act-runner.service <<EOF
+runner_install_content /etc/systemd/system/akyldash-act-runner.service root root 0644 <<EOF
 [Unit]
 Description=AkylDash repository Gitea Actions runner
 After=network-online.target local-fs.target
@@ -140,14 +162,14 @@ StandardError=append:/var/log/akyldash-runner/act_runner.log
 WantedBy=multi-user.target
 EOF
 
-mkdir -p "/etc/systemd/system/user-$runner_uid.slice.d"
-cat >"/etc/systemd/system/user-$runner_uid.slice.d/limits.conf" <<EOF
+install -d -o root -g root -m 0755 "/etc/systemd/system/user-$runner_uid.slice.d"
+runner_install_content "/etc/systemd/system/user-$runner_uid.slice.d/limits.conf" root root 0644 <<EOF
 [Slice]
 CPUQuota=250%
 MemoryMax=4G
 TasksMax=2048
 EOF
-cat >/etc/logrotate.d/akyldash-runner <<'EOF'
+runner_install_content /etc/logrotate.d/akyldash-runner root root 0644 <<'EOF'
 /var/log/akyldash-runner/act_runner.log {
     daily
     rotate 7
@@ -158,14 +180,20 @@ cat >/etc/logrotate.d/akyldash-runner <<'EOF'
     copytruncate
 }
 EOF
-install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-healthcheck.service" /etc/systemd/system/akyldash-runner-healthcheck.service
-install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-healthcheck.timer" /etc/systemd/system/akyldash-runner-healthcheck.timer
-install -o root -g root -m 0644 "$(dirname "$0")/akyldash-runner-restart.service" /etc/systemd/system/akyldash-runner-restart.service
+runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.service" /etc/systemd/system/akyldash-runner-healthcheck.service root root 0644
+runner_install_file "$(dirname "$0")/akyldash-runner-healthcheck.timer" /etc/systemd/system/akyldash-runner-healthcheck.timer root root 0644
+runner_install_file "$(dirname "$0")/akyldash-runner-restart.service" /etc/systemd/system/akyldash-runner-restart.service root root 0644
 
 loginctl enable-linger "$runner_user"
 systemctl daemon-reload
 systemctl start "user@$runner_uid.service"
-runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" dockerd-rootless-setuptool.sh install --force
+rootless_docker_unit="$runner_home/.config/systemd/user/docker.service"
+if [[ -e $rootless_docker_unit || -L $rootless_docker_unit ]]; then
+  [[ -f $rootless_docker_unit && ! -L $rootless_docker_unit && $(stat -c %U "$rootless_docker_unit") == "$runner_user" ]] || die 'existing rootless Docker unit has an unexpected type or owner'
+  grep -Fq 'dockerd-rootless.sh' "$rootless_docker_unit" || die 'existing rootless Docker unit differs from the expected service'
+else
+  runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" dockerd-rootless-setuptool.sh install
+fi
 runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" systemctl --user daemon-reload
 runuser -u "$runner_user" -- env HOME="$runner_home" XDG_RUNTIME_DIR="/run/user/$runner_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$runner_uid/bus" systemctl --user enable --now docker.service
 

infra/runner/test-install-runner-idempotency.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+runner_dir=$(cd "$(dirname "$0")" && pwd)
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/bin" "$tmp/store" "$tmp/partial-store"
+log=$tmp/commands.log
+export RUNNER_TEST_LOG=$log
+
+cat >"$tmp/bin/fallocate" <<'EOF'
+#!/usr/bin/env bash
+printf 'fallocate\n' >>"$RUNNER_TEST_LOG"
+truncate -s 1048576 "$3"
+EOF
+cat >"$tmp/bin/mkfs.ext4" <<'EOF'
+#!/usr/bin/env bash
+printf 'mkfs\n' >>"$RUNNER_TEST_LOG"
+printf 'ext4\n' >"$3.fs"
+EOF
+cat >"$tmp/bin/blkid" <<'EOF'
+#!/usr/bin/env bash
+cat "${@: -1}.fs"
+EOF
+cat >"$tmp/bin/systemctl" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >>"$RUNNER_TEST_LOG"
+if [[ $1 == start ]]; then
+  [[ ${RUNNER_TEST_FAIL_START:-0} == 0 ]] || exit 1
+  : >"$RUNNER_TEST_MOUNTED"
+fi
+EOF
+cat >"$tmp/bin/mountpoint" <<'EOF'
+#!/usr/bin/env bash
+[[ -e $RUNNER_TEST_MOUNTED ]]
+EOF
+cat >"$tmp/bin/findmnt" <<'EOF'
+#!/usr/bin/env bash
+printf '%s ext4 /dev/loop0\n' "$RUNNER_TEST_STORE"
+EOF
+cat >"$tmp/bin/losetup" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$RUNNER_TEST_BACKING"
+EOF
+chmod +x "$tmp/bin/"*
+PATH="$tmp/bin:$PATH"
+export PATH
+source "$runner_dir/install-runner-lib.sh"
+getent() {
+  [[ $1 == passwd ]] || return 2
+  if [[ -n ${RUNNER_TEST_PASSWD:-} ]]; then
+    printf '%s\n' "$RUNNER_TEST_PASSWD"
+  elif [[ -r ${RUNNER_TEST_ACCOUNT_STATE:-} ]]; then
+    cat "$RUNNER_TEST_ACCOUNT_STATE"
+  else
+    return 2
+  fi
+}
+id() {
+  case $1 in
+    -gn) printf '%s\n' "${RUNNER_TEST_PRIMARY_GROUP:-akyldash-runner}" ;;
+    -nG) [[ ${TEST_EXTRA_GROUP:-0} == 1 ]] && printf 'akyldash-runner docker\n' || printf 'akyldash-runner\n' ;;
+    *) command id "$@" ;;
+  esac
+}
+stat() {
+  if [[ $1 == -c && $2 == %U && $3 == ${RUNNER_TEST_HOME:-} ]]; then
+    printf '%s\n' "${RUNNER_TEST_HOME_OWNER:-akyldash-runner}"
+  else
+    command stat "$@"
+  fi
+}
+useradd() {
+  printf 'useradd\n' >>"$RUNNER_TEST_LOG"
+  local user=${@: -1} home=
+  while (($#)); do
+    if [[ $1 == --home-dir ]]; then home=$2; shift 2; else shift; fi
+  done
+  mkdir -p "$home"
+  printf '%s:x:990:990::%s:/usr/sbin/nologin\n' "$user" "$home" >"$RUNNER_TEST_ACCOUNT_STATE"
+}
+
+export RUNNER_TEST_HOME="$tmp/account-home"
+export RUNNER_TEST_ACCOUNT_STATE="$tmp/account.passwd"
+mkdir -p "$RUNNER_TEST_HOME"
+export RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin"
+runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
+TEST_EXTRA_GROUP=1
+if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
+  echo 'Runner account with extra group membership was accepted' >&2
+  exit 1
+fi
+unset TEST_EXTRA_GROUP
+if RUNNER_TEST_PASSWD="akyldash-runner:x:990:990::${tmp}/wrong-home:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
+  echo 'Runner account with a different home was accepted' >&2
+  exit 1
+fi
+if RUNNER_TEST_PASSWD="akyldash-runner:x:0:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
+  echo 'Runner account with UID 0 was accepted' >&2
+  exit 1
+fi
+if RUNNER_TEST_PASSWD="akyldash-runner:x:1000:990::${RUNNER_TEST_HOME}:/usr/sbin/nologin" runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
+  echo 'Runner account with a regular-user UID was accepted' >&2
+  exit 1
+fi
+RUNNER_TEST_PRIMARY_GROUP=unexpected
+if runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME" 2>/dev/null; then
+  echo 'Runner account with an unexpected primary group was accepted' >&2
+  exit 1
+fi
+unset RUNNER_TEST_PRIMARY_GROUP
+unset RUNNER_TEST_PASSWD
+export RUNNER_TEST_HOME="$tmp/new-account-home"
+runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
+runner_ensure_account akyldash-runner "$RUNNER_TEST_HOME"
+[[ $(grep -c '^useradd$' "$log") == 1 ]]
+
+image=$tmp/docker-store.img
+runner_ensure_image "$image" 1M 1048576
+runner_ensure_image "$image" 1M 1048576
+[[ $(grep -c '^fallocate$' "$log") == 1 ]]
+[[ $(grep -c '^mkfs$' "$log") == 1 ]]
+
+printf 'ext4\n' >"$image.fs"
+truncate -s 1048575 "$image"
+if runner_ensure_image "$image" 1M 1048576 2>/dev/null; then
+  echo 'Mismatched image size was accepted' >&2
+  exit 1
+fi
+[[ $(grep -c '^mkfs$' "$log") == 1 ]]
+truncate -s 1048576 "$image"
+printf 'ext3\n' >"$image.fs"
+if runner_ensure_image "$image" 1M 1048576 2>/dev/null; then
+  echo 'Existing non-ext4 image was accepted' >&2
+  exit 1
+fi
+[[ $(grep -c '^mkfs$' "$log") == 1 ]]
+printf 'ext4\n' >"$image.fs"
+
+unit_name='var-lib-akyldash\x2drunner-docker\x2ddata.mount'
+unit_file=$tmp/systemd/$unit_name
+mkdir -p "$(dirname "$unit_file")"
+export RUNNER_TEST_STORE="$tmp/store" RUNNER_TEST_BACKING="$image"
+export RUNNER_TEST_MOUNTED="$tmp/mounted"
+runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
+[[ $(grep -c '^start ' "$log") == 1 ]]
+grep -Fq "enable $unit_name" "$log"
+grep -Fq "start $unit_name" "$log"
+
+# A second run with the image mounted must reuse it and avoid another start.
+runner_ensure_image "$image" 1M 1048576
+runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file"
+[[ $(grep -c '^mkfs$' "$log") == 1 ]]
+[[ $(grep -c '^start ' "$log") == 1 ]]
+chmod 0664 "$unit_file"
+if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
+  echo 'Writable systemd mount unit was accepted' >&2
+  exit 1
+fi
+chmod 0644 "$unit_file"
+export RUNNER_TEST_BACKING="$tmp/unrelated.img"
+touch "$RUNNER_TEST_BACKING"
+if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$unit_file" 2>/dev/null; then
+  echo 'Active mount backed by a different image was accepted' >&2
+  exit 1
+fi
+[[ $(grep -c '^start ' "$log") == 1 ]]
+export RUNNER_TEST_BACKING="$image"
+
+# Simulate interruption after unit installation/enable, before mount activation.
+export RUNNER_TEST_STORE="$tmp/partial-store" RUNNER_TEST_MOUNTED="$tmp/partial-mounted"
+export RUNNER_TEST_FAIL_START=1
+if runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount" 2>/dev/null; then
+  echo 'Injected mount interruption was not observed' >&2
+  exit 1
+fi
+export RUNNER_TEST_FAIL_START=0
+runner_ensure_image "$image" 1M 1048576
+runner_ensure_mount "$image" "$RUNNER_TEST_STORE" "$unit_name" "$tmp/systemd/partial.mount"
+[[ $(grep -c '^mkfs$' "$log") == 1 ]]
+[[ $(grep -c '^start ' "$log") == 3 ]]
+echo 'runner installer idempotency checks passed'

infra/runner/test-runner-config.sh

@@ -17,10 +17,19 @@ grep -Eq '^  valid_volumes: \[\]$' "$config"
 grep -Fq -- '--cpus=2 --memory=3g --pids-limit=512' "$config"
 assert_absent 'var/run/docker.sock|metrics:|health_check:|production.env|review_secret' "$config"
 grep -Fq 'docker-ce-rootless-extras' "$installer"
-grep -Fq 'fallocate -l 15G' "$installer"
+grep -Fq 'runner_ensure_image "$runner_image" 15G "$runner_image_bytes"' "$installer"
+grep -Fq 'fallocate -l "$size" "$image"' infra/runner/install-runner-lib.sh
+grep -Fq 'mkfs.ext4 -F -q "$image"' infra/runner/install-runner-lib.sh
+grep -Fq 'systemctl enable "$unit_name"' infra/runner/install-runner-lib.sh
+grep -Fq 'systemctl start "$unit_name"' infra/runner/install-runner-lib.sh
+assert_absent 'basename.*runner_mount_unit|enable --now.*runner_mount_unit' "$installer"
+grep -Fq 'groups=$(id -nG "$user")' infra/runner/install-runner-lib.sh
+grep -Fq 'id -gn "$user"' infra/runner/install-runner-lib.sh
+assert_absent 'dockerd-rootless-setuptool.sh install --force' "$installer"
 grep -Fq 'runner_sha256=3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2' "$installer"
 grep -Fq 'find-free-subid-range.awk' "$installer"
-grep -Fq 'usermod --add-subuids "$subid_range" --add-subgids "$subid_range"' "$installer"
+grep -Fq 'usermod "${subid_args[@]}" "$runner_user"' "$installer"
+grep -Fq 'subid_args+=(--add-subuids "$subid_range")' "$installer"
 assert_absent 'usermod.*(sudo|docker)|/var/run/docker.sock|/etc/sudoers' "$installer"
 grep -Fq 'runs-on: akyldash-ci' .gitea/workflows/runner-smoke.yaml
 grep -Fq 'test ! -S /var/run/docker.sock' .gitea/workflows/runner-smoke.yaml