Back up preloaded corpus before first deployment

deploy/production/README.md

@@ -42,8 +42,11 @@ backup is required when all persistent mounts are empty and no Compose
 containers exist. Continue without a backup in that case. If any persistent
 data exists, create and verify a backup before changing schema, images, or
 starting a new application version. This also applies when containers are
-absent. The archive tool refuses ambiguous storage state, running containers,
-and incomplete Compose service containers. See
+absent, including an initial installation where the normalized corpus was
+preloaded under `DATA_ROOT`; create the archive before the first `up`. Missing
+named volumes are recorded as empty backup components when Docker confirms
+they do not exist. The archive tool refuses ambiguous storage state, running
+containers, and incomplete Compose service containers. See
 [`docs/operations/BACKUP.md`](../../docs/operations/BACKUP.md) for the exact
 commands. Do not remove an older usable backup to make room; retain the
 pre-change archive until the new deployment and a newer verified backup are

docs/operations/BACKUP.md

@@ -25,10 +25,11 @@ and `/var/backups/akyldash` remains a planned root-only location.
 
 No backup is needed only when Compose has no containers and all four
 persistent mounts are confirmed empty. Named volumes not created yet count as
-empty on a clean first deployment. Review the rendered config and let the
-archive tool make this determination; missing mount definitions, inaccessible
-volumes, or missing volumes alongside existing data are an error and must stop
-deployment. Do not create placeholder containers to make backup pass.
+empty on a clean first deployment. If data has already been placed in a bind
+mount, create the archive before the first `up`; missing named volumes are
+recorded as empty components. Missing mount definitions, inaccessible Docker
+state, or partial service containers are errors and must stop deployment. Do
+not create placeholder containers to make backup pass.
 
 ## Cold backup procedure
 

docs/operations/CURRENT_STATE.md

@@ -50,8 +50,9 @@ from planned configuration.
   restore apply tool that requires a marked replacement host. Backup/restore
   tests use synthetic data; neither tool has run against production. The
   archive preflight now permits a clean first deployment to skip backup,
-  requires backup for existing persistent data even without containers, and
-  rejects missing/ambiguous mounts or incomplete/running service containers.
+  requires backup for existing persistent data even without containers,
+  records not-yet-created named volumes as empty, and rejects inaccessible or
+  ambiguous mounts and incomplete/running service containers.
 - Recovery bootstrap has an explicit `--recovery-target` mode that refuses the
   inventoried production hostname/IP; its host-identity guard has shell tests.
   The mode has not been run on a VPS.

infra/backup/archive.py

@@ -99,7 +99,7 @@ def verify_archive(path: Path) -> dict:
         return read_manifest(archive)
 
 
-def parse_sources(values: list[str]) -> dict[str, Path]:
+def parse_sources(values: list[str], required: set[str] = COMPONENTS) -> dict[str, Path]:
     sources: dict[str, Path] = {}
     for value in values:
         name, separator, raw_path = value.partition("=")
@@ -109,9 +109,9 @@ def parse_sources(values: list[str]) -> dict[str, Path]:
         if not source.is_absolute() or not source.exists() or source.is_symlink():
             raise ValueError(f"source must be an existing absolute path: {raw_path}")
         sources[name] = source.resolve(strict=True)
-    if set(sources) != COMPONENTS:
-        missing = ", ".join(sorted(COMPONENTS - sources.keys()))
-        extra = ", ".join(sorted(sources.keys() - COMPONENTS))
+    if set(sources) != required:
+        missing = ", ".join(sorted(required - sources.keys()))
+        extra = ", ".join(sorted(sources.keys() - required))
         raise ValueError(f"source set mismatch (missing: {missing or 'none'}; extra: {extra or 'none'})")
     return sources
 
@@ -188,7 +188,7 @@ def compose_sources(
     persistent_data = has_persistent_data(storage)
     if not containers and not persistent_data:
         raise ValueError("no backup is required for a clean first deployment; skip backup and start the initial stack")
-    if any(path is None for path in storage.values()):
+    if containers and any(path is None for path in storage.values()):
         raise ValueError("persistent Compose volumes are missing while data exists; refusing an incomplete backup")
     images: dict[str, dict[str, str]] = {}
     for container in containers:
@@ -236,12 +236,15 @@ def compose_sources(
         relative: digest(repository / relative)
         for relative in ("deploy/production/compose.yaml", "deploy/production/Caddyfile")
     }
-    return parse_sources([f"{name}={path}" for name, path in sources.items()]), revision, images, deployment_files
+    present = {name: path for name, path in sources.items() if path is not None}
+    missing = {name for name, path in sources.items() if path is None}
+    parsed = parse_sources([f"{name}={path}" for name, path in present.items()], COMPONENTS - missing)
+    return {**parsed, **{name: None for name in missing}}, revision, images, deployment_files
 
 
 def create_backup(
     output: Path,
-    sources: dict[str, Path],
+    sources: dict[str, Path | None],
     revision: str,
     images: dict | None = None,
     deployment_files: dict[str, str] | None = None,
@@ -253,14 +256,16 @@ def create_backup(
     if output.exists() or Path(f"{output}.sha256").exists():
         raise ValueError("refusing to overwrite an existing backup")
     for name, source in sources.items():
+        if source is None:
+            continue
         if source.is_symlink() or not (source.is_dir() or source.is_file()):
             raise ValueError(f"source must be a regular file or directory: {name}")
         if source.is_dir() and (output == source or source in output.parents):
             raise ValueError(f"backup output cannot be inside source {name}")
     for name in ("production-env", "review-secret"):
-        if not sources[name].is_file():
+        if sources[name] is None or not sources[name].is_file():
             raise ValueError(f"sensitive component must be a regular file: {name}")
-    if sources["data"].is_dir():
+    if sources["data"] is not None and sources["data"].is_dir():
         check_sqlite(sources["data"])
 
     manifest = {
@@ -283,6 +288,12 @@ def create_backup(
             info.mode = 0o600
             archive.addfile(info, io.BytesIO(encoded))
             for name, source in sorted(sources.items()):
+                if source is None:
+                    info = tarfile.TarInfo(f"{name}/")
+                    info.type = tarfile.DIRTYPE
+                    info.mode = 0o700
+                    archive.addfile(info)
+                    continue
                 def protect_secrets(member: tarfile.TarInfo) -> tarfile.TarInfo:
                     if name in {"production-env", "review-secret"}:
                         member.mode = 0o600

infra/backup/test_archive.py

@@ -238,7 +238,7 @@ class BackupArchiveTests(unittest.TestCase):
             with self.assertRaisesRegex(ValueError, "not all configured Compose services"):
                 compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
 
-    def test_existing_data_with_missing_named_volume_is_rejected_as_incomplete(self):
+    def test_existing_data_with_uncreated_named_volume_is_backed_up_as_empty(self):
         config, volume_paths, env_file, _secret = self.compose_fixture()
         (self.root / "data/search-users.sqlite3").unlink()
         (self.root / "data/existing.json").write_text("data", encoding="utf-8")
@@ -253,11 +253,23 @@ class BackupArchiveTests(unittest.TestCase):
                 return SimpleNamespace(stdout="\n".join(available) + "\n")
             if command[1] == "volume":
                 return SimpleNamespace(stdout=f"{volume_paths[command[-1]]}\n")
+            if "status" in command:
+                return SimpleNamespace(stdout="")
+            if "rev-parse" in command:
+                return SimpleNamespace(stdout="abc123\n")
             raise AssertionError(command)
 
         with patch("archive.subprocess.run", side_effect=run):
-            with self.assertRaisesRegex(ValueError, "persistent Compose volumes are missing"):
-                compose_sources(Path.cwd() / "deploy/production/compose.yaml", env_file)
+            sources, revision, images, deployment_files = compose_sources(
+                Path.cwd() / "deploy/production/compose.yaml", env_file
+            )
+        self.assertIsNone(sources["opensearch"])
+        archive = self.private / "bind-only-backup.tar.gz"
+        create_backup(archive, sources, revision, images, deployment_files)
+        manifest = verify_archive(archive)
+        self.assertEqual(manifest["git_revision"], "abc123")
+        with tarfile.open(archive, "r:gz") as saved:
+            self.assertTrue(saved.getmember("opensearch/").isdir())
 
     def test_compose_backup_records_sources_and_image_ids(self):
         data = self.root / "production-data"