diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index bcc8dc6..a3cb1aa 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -21,6 +21,8 @@ jobs: run: python3 -m unittest discover -s infra/backup -p 'test_*.py' -v - name: Check recovery host guard run: bash infra/bootstrap/test-host-guard.sh + - name: Run rollback helper tests + run: python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v - name: Validate production Compose and Caddyfile run: | secret_file=$(mktemp) diff --git a/.impeccable/config.json b/.impeccable/config.json index a9cc933..496c0ff 100644 --- a/.impeccable/config.json +++ b/.impeccable/config.json @@ -10,7 +10,7 @@ "backend/search/review.html" ], "createdAt": "2026-10-11T15:40:02.635Z", - "reason": "Agent: DESIGN.md explicitly defines #F7F2E8 as the approved product background" + "reason": "Agent: DESIGN.md explicitly approves the #F7F2E8 felt-inspired bg token for this portal; current page background matches that user-approved palette" }, { "rule": "cramped-padding", @@ -19,7 +19,7 @@ "backend/search/review.html" ], "createdAt": "2026-10-11T15:41:28.434Z", - "reason": "Agent: both reported sections declare 1rem minimum padding in CSS; detector reports flush content despite that inset" + "reason": "Agent: current page uses 12–24px padding in bordered containers; the prior stop finding has no element detail and does not reproduce in the current detector" }, { "rule": "design-system-font", diff --git a/AGENTS.md b/AGENTS.md index 3a71787..07077e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,9 @@ Python, SQLite и OpenSearch. Production пока не развернут. Для Gitea используй `gitea-api`, не читай файлы с учётными данными. - Локальные проверки: `PYTHONPATH=backend python3 -m unittest discover -s backend -p 'test_*.py' -v` и `python3 -m unittest discover -s tools/telegram-bot -p 'test_*.py' -v`, - `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v`. + `python3 -m unittest discover -s infra/backup -p 'test_*.py' -v` и + `python3 -m unittest discover -s infra/deploy -p 'test_*.py' -v`, + `bash infra/bootstrap/test-host-guard.sh`. Production images собираются командами из `.gitea/workflows/ci.yaml`. - Перед изменениями production необходим проверенный backup. Формального механизма миграций нет; проверяй совместимость SQLite/OpenSearch. Не меняй diff --git a/deploy/production/.env.example b/deploy/production/.env.example index e4d4a4d..63ae916 100644 --- a/deploy/production/.env.example +++ b/deploy/production/.env.example @@ -1,6 +1,8 @@ # Absolute path on the production host containing minjust-normalized/. DATA_ROOT=/var/lib/akyldash/data REVIEW_SECRET_FILE=/etc/akyldash/review_secret +BACKEND_IMAGE=akyldash/backend:local +OPENSEARCH_IMAGE=akyldash/opensearch:local SEARCH_INDEX=akyldash-fragments-current OPENSEARCH_MEM_LIMIT=4g OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g diff --git a/deploy/production/compose.yaml b/deploy/production/compose.yaml index 232d1a9..636efb4 100644 --- a/deploy/production/compose.yaml +++ b/deploy/production/compose.yaml @@ -28,6 +28,7 @@ services: retries: 3 opensearch: + image: ${OPENSEARCH_IMAGE:-akyldash/opensearch:local} build: context: ../.. dockerfile: deploy/production/Dockerfile.opensearch @@ -63,6 +64,7 @@ services: retries: 10 backend: + image: ${BACKEND_IMAGE:-akyldash/backend:local} build: context: ../.. dockerfile: deploy/production/Dockerfile.backend diff --git a/docs/operations/ACCESS.md b/docs/operations/ACCESS.md index b3458e7..fc48a5e 100644 --- a/docs/operations/ACCESS.md +++ b/docs/operations/ACCESS.md @@ -24,8 +24,11 @@ checks. Never print a private key, token, or complete environment file. - `codex` can run unprivileged read-only diagnostics. The audited account has no non-interactive sudo and must not be added to `sudo` or the `docker` group. - Host package installation, service setup, firewall changes, secret creation, - backup restore, and production changes require the VPS owner to run a reviewed - root command. No restricted deploy-helper is installed yet. + backup creation, recovery apply, and deployment require owner approval and a + reviewed root command. After the owner installs the root-owned repository + checkout, `infra/deploy/install-helper.sh` can grant only the validated + rollback command through sudoers; that helper is not installed on the audited + VPS. - Never connect to other VPS hosts or Synology as part of this deployment. This execution environment's system SSH config include had unsafe file @@ -53,6 +56,6 @@ retrieve it from another machine. | Role | Operations | |---|---| | `codex` over SSH | Read-only host diagnostics; inspect service health and public listeners | -| Restricted deploy-helper | Intended future allowlisted backup/deploy/status actions; not installed | +| Restricted rollback helper | Can restore prior local backend/OpenSearch images from a verified backup after an explicit data-compatibility confirmation; not installed on the audited VPS | | VPS owner/root | Bootstrap, package installation, SSH/firewall, secret setup, recovery, and approval of production changes | | Gitea repository writer | Feature branch push and PR creation; protected-main merge needs an approval | diff --git a/docs/operations/BACKUP.md b/docs/operations/BACKUP.md index 0be85c6..b14eaa8 100644 --- a/docs/operations/BACKUP.md +++ b/docs/operations/BACKUP.md @@ -39,10 +39,10 @@ sudo python3 infra/backup/archive.py verify /var/backups/akyldash/akyldash-YYYYM Replace the timestamp placeholder with one real UTC timestamp before running. The create command refuses running services and archives the application data, OpenSearch volume, Caddy data/config volumes, production environment file, and -review secret. It records the checked-out Git revision and image IDs for each -stopped Compose service, writes a SHA-256 sidecar, and refuses to overwrite an -existing archive. It also runs SQLite `quick_check`. Start the stack again only -after verification succeeds: +review secret. It records the checked-out Git revision, deployment-file hashes, +and image references/IDs for each stopped Compose service, writes a SHA-256 +sidecar, and refuses to overwrite an existing archive. It also runs SQLite +`quick_check`. Start the stack again only after verification succeeds: ```bash cd /opt/akyldash/current @@ -52,14 +52,25 @@ sudo docker compose --env-file /etc/akyldash/production.env -f deploy/production Keep the local archive root-only, then copy it to separately managed encrypted storage. A local archive alone does not protect against loss of the VPS disk. Restore extracts only to a new `akyldash-restore-*` directory under `/tmp` or -`/var/tmp` and validates the checksum and SQLite databases; it never writes -production paths. Example test on an isolated host: +`/var/tmp` and validates the checksum and SQLite databases. Applying the staged +contents requires the separate recovery-target tool; it refuses the inventoried +production host, existing data/volumes, and an unmarked host. Example isolated +restore: ```bash sudo python3 infra/backup/archive.py restore /var/backups/akyldash/backup.tar.gz \ --target-dir /var/tmp/akyldash-restore-test ``` -This validates and stages the archive; it does not install data into Compose -volumes. Complete and verify the host-specific restore on a separate recovery -host before relying on it. Do not run an unreviewed restore over existing data. +This validates and stages the archive. On the replacement host only, after +running bootstrap with `--recovery-target` and checking out the matching Git +revision, apply the staged data: + +```bash +sudo python3 infra/backup/apply_restore.py \ + --staging-dir /var/tmp/akyldash-restore-test +``` + +The apply command restores data and named volumes but leaves services stopped. +Test the full procedure on a separate recovery host before relying on it. Do +not run an unreviewed restore over existing data. diff --git a/docs/operations/CURRENT_STATE.md b/docs/operations/CURRENT_STATE.md index 390c611..d387f82 100644 --- a/docs/operations/CURRENT_STATE.md +++ b/docs/operations/CURRENT_STATE.md @@ -44,12 +44,14 @@ from planned configuration. application data use persistent volumes/bind mounts. TLS, registry digest deployment for backend/OpenSearch, backup, rollback helper, and systemd deployment are not configured on the host. -- The repository has a cold-backup archive/verification tool with staging-only - restore and unit tests. It has not run against production; do not treat this - as an active backup or verified recovery. +- The repository has a cold-backup archive/verification tool and a one-shot + restore apply tool that requires a marked replacement host. Backup/restore + tests use synthetic data; neither tool has run against production. - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the inventoried production hostname/IP; its host-identity guard has shell tests. The mode has not been run on a VPS. +- The repository defines a root-owned rollback helper and owner-run installer + with a single-command sudoers allowlist. Neither is installed on the VPS. - An isolated local Compose smoke test built the backend/OpenSearch images, reached healthy status for OpenSearch, backend, and Caddy, and returned HTTP 200 for `/review` through Caddy. Its containers, networks, and volumes were diff --git a/docs/operations/DECISIONS.md b/docs/operations/DECISIONS.md index e665643..f57e17b 100644 --- a/docs/operations/DECISIONS.md +++ b/docs/operations/DECISIONS.md @@ -3,7 +3,7 @@ | Date | Decision | Reason and alternatives | Status / limits | |---|---|---|---| | 2026-10-11 | Use Docker Compose on the dedicated Ubuntu VPS | Matches the existing production Compose baseline and workload; Kubernetes adds no demonstrated need | Planned; Docker is not installed | -| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; use an owner-run bootstrap and a narrowly reviewed deploy-helper | No helper installed yet | +| 2026-10-11 | Keep `codex` out of sudo and the Docker group | Docker socket access is effectively root; a separate installer can grant only the root-owned rollback helper after checkout ownership and sudoers validation | Helper is defined in Git; not installed on VPS | | 2026-10-11 | Put Caddy and Search API on a proxy network and OpenSearch on a separate internal network | Keeps OpenSearch unreachable from the proxy and host; the API has no published host port | Repository Compose target only; not deployed | | 2026-10-11 | Use Caddy in Compose for HTTPS at `akyldash.kg` | Automatic certificate issuance/renewal keeps proxy config small; Nginx with separately managed certificates was considered | Caddy image pinned by digest; requires public ports 80/443 and owner confirmation of Cloudflare Full (strict) | | 2026-10-11 | Mount the review signing key as a Compose secret file | Avoids putting its value in command-line arguments; an environment variable or CLI value is easier to inspect in process metadata | Implemented in repository Compose; not deployed | diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md index 65e1dd1..070569d 100644 --- a/docs/operations/DEPLOYMENT.md +++ b/docs/operations/DEPLOYMENT.md @@ -18,21 +18,31 @@ configuration, and a backup and restore have been tested. versions and does not change SSH, UFW, sudoers, or user groups. Use `--recovery-target` only on a replacement host; it refuses the known production host identity. -2. Install the approved repository checkout at `/opt/akyldash/current` and - configure `/etc/akyldash/production.env` plus - `/etc/akyldash/review_secret` outside Git. The secret file must be - root-readable only. -3. Confirm DNS points through Cloudflare to this origin, then have the owner +2. Install the approved repository checkout at `/opt/akyldash/current`, + root-owned and not group/world-writable. Configure + `/etc/akyldash/production.env` plus `/etc/akyldash/review_secret` outside + Git. The secret file must be root-readable only. +3. After reviewing the root-owned checkout, run + `sudo bash infra/deploy/install-helper.sh`. It installs the backup reader and + rollback helper and adds a sudoers rule for that helper only; it does not + grant general sudo or Docker access. +4. Confirm DNS points through Cloudflare to this origin, then have the owner review firewall rules and open only TCP `80` and `443` in addition to the existing SSH access. The Compose Caddy service obtains and renews its HTTPS certificate automatically; Cloudflare Full (strict) still needs owner-side verification. -4. Create and verify the first backup before starting services. Load and +5. Create and verify the first backup before starting services. Load and validate the OpenSearch index, create the initial admin interactively, and - verify `/review` over HTTPS. + verify `/review` over HTTPS. Record the backend image ID for drift checks: -The remaining steps are intentionally manual until a restricted deploy-helper, -registry permissions, a live Gitea runner, and the backup path are tested. + ```bash + sudo sh -c 'docker compose --env-file /etc/akyldash/production.env -f /opt/akyldash/current/deploy/production/compose.yaml ps --quiet backend | xargs -r docker inspect --format "{{.Image}}" > /opt/akyldash/deployed-image.txt' + sudo chmod 0600 /opt/akyldash/deployed-image.txt + ``` + +Application deployment remains owner-run until registry permissions, a live +Gitea runner, and a controlled deploy procedure are tested. The helper only +supports a guarded application rollback. ## CI diff --git a/docs/operations/RECOVERY.md b/docs/operations/RECOVERY.md index 9f82d25..96b86ca 100644 --- a/docs/operations/RECOVERY.md +++ b/docs/operations/RECOVERY.md @@ -15,24 +15,32 @@ run recovery commands against the existing production by accident. 2. Create the SSH-authorized `codex` account without sudo or Docker group access. Preserve the host's existing root SSH and firewall policy until an owner has reviewed the change. -3. Check out the approved Git revision and run +3. Check out the approved Git revision, root-owned and not group/world-writable, + at `/opt/akyldash/current`, then run `sudo bash infra/bootstrap/bootstrap-ubuntu-24.04.sh --recovery-target`. The recovery mode refuses the inventoried production hostname/IP and writes a root-only marker at `/etc/akyldash/recovery-target`. It installs Docker Engine, Compose, Git, Docker log limits, and target directories. -4. Configure DNS to point through Cloudflare to the recovery host and allow the +4. Install the limited rollback helper with + `sudo bash infra/deploy/install-helper.sh`. It validates the root-owned + checkout and gives `codex` access only to that helper. +5. Configure DNS to point through Cloudflare to the recovery host and allow the owner to review/open TCP `80` and `443`. The production Compose Caddy service obtains its origin certificate automatically. Verify Cloudflare Full (strict) end to end before cutover. -5. Provide production configuration and secret files through the owner's - protected channel; never commit or paste secret values into chat. -6. Verify the archive and extract it to a new `/var/tmp/akyldash-restore-*` - staging directory with `infra/backup/archive.py`. The tool checks the archive - checksum and SQLite integrity but does not install staged files into Compose - volumes; have the owner review and perform that host-specific step. Validate - the corpus and OpenSearch index before serving traffic. -7. Start the approved image digest, verify container health and - `https://akyldash.kg/review`, then record actual versions and checks in +6. Transfer the protected backup to the new host through the owner's approved + channel. The archive contains the production environment and review secret; + keep it root-only and never paste secret values into chat. +7. Extract the verified archive to a new `/var/tmp/akyldash-restore-*` staging + directory with `infra/backup/archive.py`, then run + `sudo python3 infra/backup/apply_restore.py --staging-dir `. + Apply requires the recovery marker, matching clean Git revision, empty data + path, absent Compose volumes/config files, and no running containers. It + refuses the inventoried production host and leaves services stopped. +8. Build the backend and OpenSearch images from the matching checked-out Git + revision and explicitly start Compose with `--wait`. Verify HTTPS and + container health, record the backend image ID as described in + [DEPLOYMENT.md](DEPLOYMENT.md), then record actual versions and checks in [CURRENT_STATE.md](CURRENT_STATE.md). Do not repoint DNS or retire the old host until the owner verifies recovery and diff --git a/docs/operations/ROLLBACK.md b/docs/operations/ROLLBACK.md index 8d0a83f..c0d7bef 100644 --- a/docs/operations/ROLLBACK.md +++ b/docs/operations/ROLLBACK.md @@ -2,16 +2,33 @@ ## Current status -There is no production release, pinned image digest, backup, or deploy-helper -to roll back. Automatic rollback is not configured. +There is no production release, backup, or installed rollback helper. The +repository contains a root-owned rollback helper and a separate installer, but +neither has been run on the VPS. -After deployment automation is reviewed, rollback may switch the backend to a -previous immutable image digest only after health checks fail and only when the -data format remains compatible. This application currently has no formal -migration framework, but its SQLite files and OpenSearch indexes are persistent; -compatibility must be checked for each change. If data compatibility is -uncertain, stop and report rather than restoring or deleting production data. +After a verified cold backup is created before a release, an owner must check +that the data format is compatible with the prior backend/OpenSearch images. +Then run the limited helper with that archive: + +```bash +sudo -n /usr/local/sbin/akyldash-rollback \ + /var/backups/akyldash/.tar.gz --data-compatible +``` + +The helper verifies archive integrity, Compose image references, deployment +file hashes, and the availability of both prior local images. It makes and +verifies a second cold backup, restores only the prior backend/OpenSearch image +tags, waits for Compose healthchecks, and checks HTTPS locally. It refuses to +restore if the current Compose/Caddyfile differs from the saved release. This +does not restore data or automatically undo a release with an unknown schema +change. The application has no formal migration framework; compatibility must +be checked for every change. If it is uncertain, stop and ask the owner. + +The initial owner-run deployment must record its running backend image ID in +`/opt/akyldash/deployed-image.txt` with mode `0600`; the rollback helper updates +that record after a successful rollback, and `infra/bootstrap/verify.sh` checks +it against the running container. Any data restore requires the owner, a verified pre-change backup, and a -maintenance window. Preserve the failed release and its backup until recovery +maintenance window. Preserve the failed release and both backups until recovery is confirmed. diff --git a/infra/backup/apply_restore.py b/infra/backup/apply_restore.py new file mode 100644 index 0000000..d283a74 --- /dev/null +++ b/infra/backup/apply_restore.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Install a verified staging restore onto an explicitly marked new VPS.""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +from pathlib import Path + +from archive import COMPONENTS, check_sqlite, digest + +ROOT = Path("/opt/akyldash/current") +COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" +ENV_DEST = Path("/etc/akyldash/production.env") +SECRET_DEST = Path("/etc/akyldash/review_secret") +DATA_DEST = Path("/var/lib/akyldash/data") +MARKER = Path("/etc/akyldash/recovery-target") +PRODUCTION_FILES = ("deploy/production/compose.yaml", "deploy/production/Caddyfile") +VOLUME_TARGETS = { + "/usr/share/opensearch/data": "opensearch-data", + "/data": "caddy-data", + "/config": "caddy-config", +} + + +def run(command: list[str], *, capture: bool = False) -> str: + result = subprocess.run(command, check=True, text=True, capture_output=capture) + return result.stdout.strip() if capture else "" + + +def require_empty_directory(path: Path) -> None: + if not path.is_dir() or path.is_symlink() or any(path.iterdir()): + raise ValueError(f"restore destination must be an existing empty directory: {path}") + + +def tree_size(root: Path) -> int: + total = 0 + for directory, names, files in os.walk(root, followlinks=False): + names[:] = [name for name in names if not (Path(directory) / name).is_symlink()] + for name in files: + path = Path(directory) / name + if path.is_symlink(): + continue + if path.is_file(): + total += path.stat().st_size + return total + + +def copy_contents(source: Path, destination: Path) -> None: + for child in source.iterdir(): + target = destination / child.name + if child.is_dir() and not child.is_symlink(): + shutil.copytree(child, target, symlinks=True) + else: + shutil.copy2(child, target, follow_symlinks=False) + + +def empty_directory(path: Path) -> None: + for child in path.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + + +def validate_stage(staging: Path) -> dict: + staging = staging.resolve(strict=True) + if staging.name.startswith("akyldash-restore-") is False or staging.parent != Path("/var/tmp"): + raise ValueError("staging directory must be a direct child of /var/tmp") + if staging.stat().st_uid != 0 or staging.stat().st_mode & 0o777 != 0o700: + raise ValueError("staging directory must be root-owned mode 0700") + manifest_path = staging / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + not isinstance(manifest, dict) + or manifest.get("format") != 1 + or set(manifest.get("components", [])) != COMPONENTS + ): + raise ValueError("staging manifest is invalid") + for name in COMPONENTS - {"production-env", "review-secret"}: + component = staging / name + if not component.is_dir() or component.is_symlink(): + raise ValueError(f"staged component must be a directory: {name}") + for name in ("production-env", "review-secret"): + path = staging / name + if not path.is_file() or path.is_symlink(): + raise ValueError(f"staged sensitive component must be a regular file: {name}") + return manifest + + +def validate_checkout(manifest: dict) -> None: + if ROOT.is_symlink() or not (ROOT / ".git").exists(): + raise ValueError("approved Git checkout is missing or is a symlink") + if subprocess.check_output(["git", "-C", str(ROOT), "status", "--porcelain"], text=True).strip(): + raise ValueError("recovery checkout must be clean") + unsafe = subprocess.run( + ["find", str(ROOT), "-xdev", "(", "!", "-user", "root", "-o", "-perm", "/022", ")", "-print", "-quit"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if unsafe: + raise ValueError(f"recovery checkout contains a non-root-owned or writable path: {unsafe}") + revision = run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture=True) + if revision != manifest.get("git_revision"): + raise ValueError("checkout Git revision does not match the backup manifest") + for relative, expected in manifest.get("deployment_files", {}).items(): + if relative not in PRODUCTION_FILES or digest(ROOT / relative) != expected: + raise ValueError("Compose or Caddyfile does not match the backup manifest") + if set(manifest.get("deployment_files", {})) != set(PRODUCTION_FILES): + raise ValueError("backup manifest is missing deployment file hashes") + + +def validate_recovery_target() -> None: + if not MARKER.is_file() or MARKER.is_symlink() or MARKER.stat().st_uid != 0 or MARKER.stat().st_mode & 0o777 != 0o600: + raise ValueError("root-only recovery-target marker is required") + hostname = socket.getfqdn() + addresses = run(["ip", "-o", "-4", "addr", "show"], capture=True) + if not hostname or not addresses: + raise ValueError("cannot verify the recovery host identity") + if hostname == "avprogs.hlab.kz" or "206.223.249.118" in addresses: + raise ValueError("refusing to apply recovery data to the inventoried production VPS") + + +def compose_configuration(staging: Path) -> dict: + rendered = run( + [ + "docker", "compose", "--env-file", str(staging / "production-env"), + "-f", str(COMPOSE_FILE), "config", "--format", "json", + ], + capture=True, + ) + config = json.loads(rendered) + data_source = next( + mount["source"] + for mount in config["services"]["backend"]["volumes"] + if mount["target"] == "/app/data" + ) + secret_source = config["secrets"]["review_secret"]["file"] + if Path(data_source) != DATA_DEST or Path(secret_source) != SECRET_DEST: + raise ValueError("backup production paths do not match the clean recovery host layout") + return config + + +def expected_volume_names(config: dict) -> set[str]: + mounts = { + mount["target"]: mount + for service in config["services"].values() + for mount in service.get("volumes", []) + } + names: set[str] = set() + for target in VOLUME_TARGETS: + mount = mounts.get(target) + if not mount or mount["type"] != "volume": + raise ValueError(f"expected persistent volume at {target}") + names.add(config["volumes"][mount["source"]]["name"]) + return names + + +def volume_destinations(config: dict) -> dict[str, Path]: + mounts = { + mount["target"]: mount + for service in config["services"].values() + for mount in service.get("volumes", []) + } + destinations = {} + for target, component in VOLUME_TARGETS.items(): + mount = mounts.get(target) + if not mount or mount["type"] != "volume": + raise ValueError(f"expected persistent volume at {target}") + name = config["volumes"][mount["source"]]["name"] + destinations[component] = Path( + run(["docker", "volume", "inspect", "--format", "{{.Mountpoint}}", name], capture=True) + ) + return destinations + + +def apply_restore(staging: Path) -> None: + if os.geteuid() != 0: + raise ValueError("restore apply must run as root") + validate_recovery_target() + manifest = validate_stage(staging) + validate_checkout(manifest) + config = compose_configuration(staging) + require_empty_directory(DATA_DEST) + if ENV_DEST.exists() or SECRET_DEST.exists(): + raise ValueError("production environment and secret files must not already exist") + if run(["docker", "ps", "--quiet"], capture=True): + raise ValueError("recovery target must have no running Docker containers") + + volume_names = expected_volume_names(config) + for name in volume_names: + result = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, text=True) + if result.returncode == 0: + raise ValueError(f"recovery volume already exists: {name}") + + created_volumes: list[str] = [] + created_files: list[Path] = [] + data_started = False + try: + for name in volume_names: + run(["docker", "volume", "create", name]) + created_volumes.append(name) + + destinations = volume_destinations(config) + destinations["data"] = DATA_DEST + + required: dict[int, int] = {} + free: dict[int, int] = {} + for component, destination in destinations.items(): + device = destination.stat().st_dev + required[device] = required.get(device, 0) + tree_size(staging / component) + free[device] = shutil.disk_usage(destination).free + for device, size in required.items(): + if free[device] < size + 512 * 1024 * 1024: + raise ValueError("recovery host lacks space for a safe data copy") + + for component, destination in destinations.items(): + require_empty_directory(destination) + if component == "data": + data_started = True + copy_contents(staging / component, destination) + + for directory, names, files in os.walk(destinations["opensearch"], followlinks=False): + os.chown(directory, 1000, 1000, follow_symlinks=False) + for name in files: + os.chown(Path(directory) / name, 1000, 1000, follow_symlinks=False) + + for source, destination in ((staging / "production-env", ENV_DEST), (staging / "review-secret", SECRET_DEST)): + shutil.copyfile(source, destination) + created_files.append(destination) + os.chown(destination, 0, 0) + os.chmod(destination, 0o600) + check_sqlite(DATA_DEST) + except BaseException: + for path in created_files: + path.unlink(missing_ok=True) + if data_started: + empty_directory(DATA_DEST) + for name in created_volumes: + subprocess.run(["docker", "volume", "rm", name], check=False, capture_output=True, text=True) + raise + + MARKER.unlink() + print("PASS staged recovery data applied to the marked new host; services remain stopped") + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--staging-dir", required=True, type=Path) + args = parser.parse_args() + try: + apply_restore(args.staging_dir) + except Exception as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/backup/archive.py b/infra/backup/archive.py index a20dd99..e623ac4 100644 --- a/infra/backup/archive.py +++ b/infra/backup/archive.py @@ -80,6 +80,8 @@ def read_manifest(archive: tarfile.TarFile) -> dict: raise ValueError("backup manifest has an unsupported format or component set") if not isinstance(manifest.get("images"), dict): raise ValueError("backup manifest has no image records") + if not isinstance(manifest.get("deployment_files"), dict): + raise ValueError("backup manifest has no deployment file records") for member in archive.getmembers(): if member.name == "manifest.json": continue @@ -125,7 +127,9 @@ def check_sqlite(root: Path) -> None: connection.close() -def compose_sources(compose_file: Path, env_file: Path) -> tuple[dict[str, Path], str, dict[str, dict[str, str]]]: +def compose_sources( + compose_file: Path, env_file: Path +) -> tuple[dict[str, Path], str, dict[str, dict[str, str]], dict[str, str]]: command = ["docker", "compose", "--env-file", str(env_file), "-f", str(compose_file)] rendered = subprocess.run( [*command, "config", "--format", "json"], check=True, capture_output=True, text=True @@ -209,10 +213,20 @@ def compose_sources(compose_file: Path, env_file: Path) -> tuple[dict[str, Path] capture_output=True, text=True, ).stdout.strip() - return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images + deployment_files = { + relative: digest(repository / relative) + for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") + } + return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images, deployment_files -def create_backup(output: Path, sources: dict[str, Path], revision: str, images: dict | None = None) -> None: +def create_backup( + output: Path, + sources: dict[str, Path], + revision: str, + images: dict | None = None, + deployment_files: dict[str, str] | None = None, +) -> None: output = output.absolute() parent = output.parent if not parent.is_dir() or parent.stat().st_mode & 0o077: @@ -235,6 +249,7 @@ def create_backup(output: Path, sources: dict[str, Path], revision: str, images: "created_at": dt.datetime.now(dt.timezone.utc).isoformat(), "git_revision": revision, "images": images or {}, + "deployment_files": deployment_files or {}, "components": sorted(sources), } with tempfile.NamedTemporaryFile(dir=parent, prefix=".akyldash-backup-", delete=False) as temp: @@ -315,8 +330,8 @@ def main() -> int: if args.command in {"create", "restore"} and os.geteuid() != 0: raise ValueError("create and restore must run as root to protect production secrets and data") if args.command == "create": - sources, revision, images = compose_sources(args.compose_file, args.env_file) - create_backup(args.output, sources, revision, images) + sources, revision, images, deployment_files = compose_sources(args.compose_file, args.env_file) + create_backup(args.output, sources, revision, images, deployment_files) elif args.command == "verify": manifest = verify_archive(args.archive) print(f"PASS backup {args.archive}: {manifest['git_revision']} ({len(manifest['components'])} components)") diff --git a/infra/backup/test_apply_restore.py b/infra/backup/test_apply_restore.py new file mode 100644 index 0000000..54f1cdb --- /dev/null +++ b/infra/backup/test_apply_restore.py @@ -0,0 +1,90 @@ +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import apply_restore + + +class ApplyRestoreTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.staging = self.root / "akyldash-restore-test" + self.staging.mkdir() + for component in apply_restore.COMPONENTS: + path = self.staging / component + if component in {"production-env", "review-secret"}: + path.write_text("private", encoding="utf-8") + else: + path.mkdir() + (path / "payload.txt").write_text(component, encoding="utf-8") + self.data = self.root / "data" + self.data.mkdir() + self.volumes = { + name: self.root / name + for name in ("opensearch", "caddy-data", "caddy-config") + } + for path in self.volumes.values(): + path.mkdir() + self.env = self.root / "production.env" + self.secret = self.root / "review_secret" + self.marker = self.root / "recovery-target" + self.marker.touch(mode=0o600) + self.config = { + "volumes": { + "opensearch-data": {"name": "akyldash_opensearch-data"}, + "caddy-data": {"name": "akyldash_caddy-data"}, + "caddy-config": {"name": "akyldash_caddy-config"}, + } + } + + def tearDown(self): + self.temporary.cleanup() + + def run_apply(self): + def docker_run(command, **_kwargs): + if command[:3] == ["docker", "ps", "--quiet"]: + return "" + if command[1:3] == ["volume", "create"]: + return command[-1] + return "" + + with ( + patch("apply_restore.os.geteuid", return_value=0), + patch("apply_restore.validate_recovery_target"), + patch("apply_restore.validate_stage", return_value={"format": 1}), + patch("apply_restore.validate_checkout"), + patch("apply_restore.compose_configuration", return_value={"volumes": self.config["volumes"]}), + patch("apply_restore.expected_volume_names", return_value=set(self.config["volumes"][name]["name"] for name in self.config["volumes"])), + patch("apply_restore.volume_destinations", return_value=self.volumes), + patch("apply_restore.DATA_DEST", self.data), + patch("apply_restore.ENV_DEST", self.env), + patch("apply_restore.SECRET_DEST", self.secret), + patch("apply_restore.MARKER", self.marker), + patch("apply_restore.run", side_effect=docker_run), + patch("apply_restore.subprocess.run", return_value=SimpleNamespace(returncode=1)), + patch("apply_restore.os.chown"), + ): + apply_restore.apply_restore(self.staging) + + def test_applies_components_and_protects_secret_files(self): + self.run_apply() + self.assertEqual((self.data / "payload.txt").read_text(), "data") + self.assertEqual((self.volumes["opensearch"] / "payload.txt").read_text(), "opensearch") + self.assertEqual(self.env.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.secret.stat().st_mode & 0o777, 0o600) + self.assertFalse(self.marker.exists()) + + def test_refuses_nonempty_data_destination_before_writing_secrets(self): + (self.data / "keep.txt").write_text("existing", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "empty directory"): + self.run_apply() + self.assertEqual((self.data / "keep.txt").read_text(), "existing") + self.assertFalse(self.env.exists()) + self.assertTrue(self.marker.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/backup/test_archive.py b/infra/backup/test_archive.py index f00708e..77d6bcf 100644 --- a/infra/backup/test_archive.py +++ b/infra/backup/test_archive.py @@ -141,12 +141,13 @@ class BackupArchiveTests(unittest.TestCase): return SimpleNamespace(stdout="") with patch("archive.subprocess.run", side_effect=run): - sources, revision, images = compose_sources( + sources, revision, images, deployment_files = compose_sources( Path.cwd() / "deploy/production/compose.yaml", env_file ) self.assertEqual(revision, "") self.assertEqual(set(sources), COMPONENTS) self.assertEqual(images["backend"]["image_id"], "sha256:backend") + self.assertEqual(set(deployment_files), {"deploy/production/compose.yaml", "deploy/production/Caddyfile"}) if __name__ == "__main__": diff --git a/infra/bootstrap/verify.sh b/infra/bootstrap/verify.sh index dd167c2..e472eba 100755 --- a/infra/bootstrap/verify.sh +++ b/infra/bootstrap/verify.sh @@ -129,18 +129,22 @@ for endpoint in $udp_listeners; do failed=1 done -if [[ -x /usr/local/sbin/akyldash-deploy ]]; then - echo 'PASS restricted deploy helper present' +sudoers_rule='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *' +if [[ -x /usr/local/sbin/akyldash-rollback && $(stat -c %U:%a /usr/local/sbin/akyldash-rollback) == root:750 && \ + -f /usr/local/lib/akyldash/archive.py && $(stat -c %U:%a /usr/local/lib/akyldash/archive.py) == root:644 && \ + -f /etc/sudoers.d/akyldash-codex && $(stat -c %U:%a /etc/sudoers.d/akyldash-codex) == root:440 && \ + $(cat /etc/sudoers.d/akyldash-codex) == "$sudoers_rule" ]]; then + echo 'PASS limited rollback helper and sudoers rule present' else - echo 'FAIL restricted deploy helper missing' + echo 'FAIL limited rollback helper or sudoers rule missing/different' failed=1 fi if [[ -f /opt/akyldash/deployed-image.txt ]]; then expected=$(cat /opt/akyldash/deployed-image.txt) container=$(docker ps -aq --filter label=com.docker.compose.project=akyldash --filter label=com.docker.compose.service=backend | head -n1) - actual=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || true) - if [[ $expected == *@sha256:* && $actual == "$expected" ]]; then - printf 'PASS running backend image matches %s\n' "$expected" + actual=$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true) + if [[ $(stat -c %U:%a /opt/akyldash/deployed-image.txt) == root:600 && $expected =~ ^sha256:[0-9a-f]{64}$ && $actual == "$expected" ]]; then + printf 'PASS running backend image ID matches %s\n' "$expected" else echo 'FAIL running backend image does not match recorded digest' failed=1 diff --git a/infra/deploy/install-helper.sh b/infra/deploy/install-helper.sh new file mode 100644 index 0000000..ec14933 --- /dev/null +++ b/infra/deploy/install-helper.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $EUID -ne 0 ]]; then + echo "Run this installer as root." >&2 + exit 1 +fi + +checkout=/opt/akyldash/current +rollback_source=$checkout/infra/deploy/rollback.py +backup_source=$checkout/infra/backup/archive.py +if [[ ! -d $checkout/.git || ! -f $rollback_source || ! -f $backup_source ]]; then + echo "Install the approved repository checkout at $checkout first." >&2 + exit 1 +fi +if [[ -L $checkout ]]; then + echo "The checkout path must not be a symlink." >&2 + exit 1 +fi +for directory in /opt /opt/akyldash; do + mode=$(stat -c %a "$directory") + if [[ $(stat -c %U "$directory") != root ]] || (( (8#$mode & 022) != 0 )); then + echo "The checkout parent $directory must be root-owned and not group/world-writable." >&2 + exit 1 + fi +done +if ! id codex >/dev/null 2>&1 || id -nG codex | tr ' ' '\n' | grep -Eq '^(sudo|docker)$'; then + echo "Create codex without sudo or Docker-group membership before installing the helper." >&2 + exit 1 +fi +if grep -R -nE '(^|[[:space:],])codex([[:space:],]|$)' /etc/sudoers /etc/sudoers.d --exclude=akyldash-codex; then + echo "Existing codex sudoers rules need owner review before installing the helper." >&2 + exit 1 +fi +if [[ $(stat -c %U "$checkout") != root ]] || find "$checkout" -xdev \( ! -user root -o -perm /022 \) -print -quit | grep -q .; then + echo "The checkout must be root-owned and not group/world-writable." >&2 + exit 1 +fi +if ! command -v visudo >/dev/null 2>&1; then + echo "visudo is required to validate the restricted sudoers rule." >&2 + exit 1 +fi + +for path in /usr/local/sbin/akyldash-rollback /usr/local/lib/akyldash/archive.py /etc/sudoers.d/akyldash-codex; do + if [[ -L $path ]]; then + echo "Refusing symlink at $path." >&2 + exit 1 + fi +done + +install -d -o root -g root -m 0755 /usr/local/lib/akyldash +install -o root -g root -m 0644 "$backup_source" /usr/local/lib/akyldash/archive.py +install -o root -g root -m 0750 "$rollback_source" /usr/local/sbin/akyldash-rollback + +sudoers_file=/etc/sudoers.d/akyldash-codex +expected='codex ALL=(root) NOPASSWD: /usr/local/sbin/akyldash-rollback *' +created=false +if [[ -e $sudoers_file ]]; then + if [[ $(stat -c %U:%a "$sudoers_file") != root:440 || $(cat "$sudoers_file") != "$expected" ]]; then + echo "$sudoers_file exists with unexpected contents or permissions; review manually." >&2 + exit 1 + fi +else + temp=$(mktemp /etc/sudoers.d/.akyldash-codex.XXXXXX) + trap 'rm -f "$temp"' EXIT + printf '%s\n' "$expected" > "$temp" + chmod 0440 "$temp" + visudo -cf "$temp" + visudo -c -f /etc/sudoers + mv -n "$temp" "$sudoers_file" + if [[ $(cat "$sudoers_file") != "$expected" ]]; then + echo "Could not install the expected sudoers rule." >&2 + exit 1 + fi + created=true +fi + +if ! visudo -c -f /etc/sudoers; then + if [[ $created == true ]]; then + rm -f "$sudoers_file" + fi + echo "Sudoers validation failed; the newly added rule was removed." >&2 + exit 1 +fi +echo "Installed root-owned rollback helper; codex receives no general sudo or Docker access." diff --git a/infra/deploy/rollback.py b/infra/deploy/rollback.py new file mode 100644 index 0000000..a08e5ff --- /dev/null +++ b/infra/deploy/rollback.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Restore prior local application images from a verified cold backup.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path("/opt/akyldash/current") +COMPOSE_FILE = ROOT / "deploy/production/compose.yaml" +ENV_FILE = Path("/etc/akyldash/production.env") +BACKUP_DIR = Path("/var/backups/akyldash") +BACKUP_TOOL = Path("/usr/local/lib/akyldash/archive.py") +APP_SERVICES = {"backend", "opensearch"} + + +def image_restore_plan(manifest: dict, configured: dict[str, str]) -> dict[str, tuple[str, str]]: + images = manifest.get("images") + if not isinstance(images, dict) or not APP_SERVICES <= images.keys(): + raise ValueError("backup has no previous backend and OpenSearch images") + plan = {} + for service in sorted(APP_SERVICES): + image = images[service] + if not isinstance(image, dict): + raise ValueError(f"invalid image record for {service}") + reference = image.get("configured") + image_id = image.get("image_id") + if reference != configured.get(service): + raise ValueError(f"Compose image reference changed for {service}; refusing rollback") + if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/-]*", reference): + raise ValueError(f"{service} image reference is not a local tag") + if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError(f"invalid previous image ID for {service}") + plan[service] = (image_id, reference) + return plan + + +def run(command: list[str], *, capture: bool = False) -> str: + result = subprocess.run(command, check=True, text=True, capture_output=capture) + return result.stdout.strip() if capture else "" + + +def compose(*args: str, capture: bool = False) -> str: + return run( + ["docker", "compose", "--env-file", str(ENV_FILE), "-f", str(COMPOSE_FILE), *args], + capture=capture, + ) + + +def record_backend_image() -> None: + container = compose("ps", "--quiet", "backend", capture=True) + if not container: + raise ValueError("backend container is missing after rollback") + image_id = run(["docker", "inspect", "--format", "{{.Image}}", container], capture=True) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError("backend returned an invalid image ID") + target = Path("/opt/akyldash/deployed-image.txt") + with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".deployed-image-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(f"{image_id}\n".encode("ascii")) + os.chmod(temporary, 0o600) + try: + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + + +def rollback(backup: Path, data_compatible: bool) -> None: + if not data_compatible: + raise ValueError("confirm data compatibility with --data-compatible before rollback") + if not backup.is_file() or backup.is_symlink() or backup.resolve().parent != BACKUP_DIR.resolve(): + raise ValueError(f"backup must be a regular file directly under {BACKUP_DIR}") + if backup.stat().st_uid != 0 or backup.stat().st_mode & 0o777 != 0o600: + raise ValueError("rollback backup must be root-owned mode 0600") + checksum = Path(f"{backup}.sha256") + if checksum.is_symlink() or not checksum.is_file() or checksum.stat().st_uid != 0 or checksum.stat().st_mode & 0o777 != 0o600: + raise ValueError("rollback checksum sidecar must be root-owned mode 0600") + + sys.path.insert(0, str(BACKUP_TOOL.parent)) + from archive import digest, verify_archive + + manifest = verify_archive(backup) + deployment_files = { + relative: digest(ROOT / relative) + for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile") + } + if deployment_files != manifest.get("deployment_files"): + raise ValueError("Compose or Caddyfile changed since this backup; restore the matching Git revision first") + rendered = json.loads(compose("config", "--format", "json", capture=True)) + configured = {service: rendered["services"][service]["image"] for service in APP_SERVICES} + plan = image_restore_plan(manifest, configured) + for image_id, _reference in plan.values(): + actual = run(["docker", "image", "inspect", "--format", "{{.Id}}", image_id], capture=True) + if actual != image_id: + raise ValueError(f"previous image is unavailable: {image_id}") + + backup_name = f"akyldash-pre-rollback-{dt.datetime.now(dt.timezone.utc):%Y%m%dT%H%M%SZ}.tar.gz" + backup_output = BACKUP_DIR / backup_name + compose("stop") + try: + run( + [ + "python3", + str(BACKUP_TOOL), + "create", + "--compose-file", + str(COMPOSE_FILE), + "--env-file", + str(ENV_FILE), + "--output", + str(backup_output), + ] + ) + run(["python3", str(BACKUP_TOOL), "verify", str(backup_output)]) + except Exception: + compose("up", "--detach") + raise + + for image_id, reference in plan.values(): + run(["docker", "image", "tag", image_id, reference]) + compose("up", "--detach", "--no-build", "--wait", "--wait-timeout", "300") + run( + [ + "curl", + "--fail", + "--silent", + "--show-error", + "--resolve", + "akyldash.kg:443:127.0.0.1", + "https://akyldash.kg/review", + "--output", + "/dev/null", + ] + ) + record_backend_image() + print(f"PASS application images restored from {backup}; pre-rollback backup: {backup_output}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("backup", type=Path) + parser.add_argument("--data-compatible", action="store_true") + args = parser.parse_args() + try: + if os.geteuid() != 0: + raise ValueError("rollback must run through the restricted root helper") + rollback(args.backup, args.data_compatible) + except Exception as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/deploy/test_rollback.py b/infra/deploy/test_rollback.py new file mode 100644 index 0000000..7b5842b --- /dev/null +++ b/infra/deploy/test_rollback.py @@ -0,0 +1,36 @@ +import unittest + +from rollback import image_restore_plan + + +class RollbackPlanTests(unittest.TestCase): + def setUp(self): + self.manifest = { + "images": { + "backend": {"configured": "akyldash/backend:local", "image_id": "sha256:" + "a" * 64}, + "opensearch": {"configured": "akyldash/opensearch:local", "image_id": "sha256:" + "b" * 64}, + } + } + self.configured = { + "backend": "akyldash/backend:local", + "opensearch": "akyldash/opensearch:local", + } + + def test_plans_only_recorded_local_images(self): + plan = image_restore_plan(self.manifest, self.configured) + self.assertEqual(plan["backend"][1], "akyldash/backend:local") + self.assertEqual(plan["opensearch"][1], "akyldash/opensearch:local") + + def test_refuses_changed_compose_image_reference(self): + with self.assertRaisesRegex(ValueError, "reference changed"): + image_restore_plan(self.manifest, {**self.configured, "backend": "other/backend:tag"}) + + def test_refuses_digest_reference_that_cannot_be_retagged(self): + self.manifest["images"]["backend"]["configured"] = "registry/backend@sha256:" + "c" * 64 + self.configured["backend"] = self.manifest["images"]["backend"]["configured"] + with self.assertRaisesRegex(ValueError, "local tag"): + image_restore_plan(self.manifest, self.configured) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/inventory/production.yml b/infra/inventory/production.yml index a4b0ab3..2beb0cf 100644 --- a/infra/inventory/production.yml +++ b/infra/inventory/production.yml @@ -67,12 +67,16 @@ target: - REVIEW_SECRET (host file mounted as /run/secrets/review_secret) - Gitea credentials from environment-backed gitea-api helper - SSH key at ~/.ssh/akyldash_codex + restricted_helper: + path: /usr/local/sbin/akyldash-rollback + status: repository code and owner-run installer exist; not installed on VPS checks: - infra/bootstrap/verify.sh akyldash_vps - docker compose -f deploy/production/compose.yaml ps - curl -fsS https://akyldash.kg/review - python3 infra/backup/archive.py verify /var/backups/akyldash/.tar.gz backup_path: /var/backups/akyldash (target; archive script in infra/backup/archive.py is not installed or run) + deployed_backend_image_record: /opt/akyldash/deployed-image.txt (required by drift verifier; not present before first deployment) versions: application: backend: 0.11.3