Files
akyldash/infra/deploy/verify_corpus.py
agent 228dfdaca0
CI / tests-and-images (pull_request) Failing after 9s
Runner smoke test / runner-smoke (pull_request) Successful in 2s
Harden MinJust staging and legacy corpus verification
2026-10-20 01:59:32 +03:00

1082 lines
50 KiB
Python

#!/usr/bin/env python3
"""Build a resumable, read-only inventory and validate AkylDash corpus trees."""
from __future__ import annotations
import argparse
import fcntl
import hashlib
import json
import ntpath
import os
import sqlite3
import stat
import sys
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Iterable
INVENTORY_SCHEMA_VERSION = 2
CHECKPOINT_INTERVAL = 1000
CHUNK_SIZE = 1024 * 1024
MAX_JSON_BYTES = 64 * 1024 * 1024
REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
NORMALIZER_ROOT = Path(__file__).resolve().parents[2]
TOOL_SHA256 = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
sys.path.insert(0, str(NORMALIZER_ROOT))
from backend.normalization.minjust_cbd import NORMALIZER_VERSION, SCHEMA_VERSION as NORMALIZED_SCHEMA_VERSION # noqa: E402
SUPPORTED_NORMALIZER_VERSIONS = frozenset({"1.0.0", NORMALIZER_VERSION})
class InventoryError(RuntimeError):
pass
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def file_signature(value: os.stat_result) -> tuple[int, int, int, int, int, int]:
return (
value.st_dev,
value.st_ino,
value.st_size,
value.st_mtime_ns,
value.st_ctime_ns,
value.st_nlink,
)
def is_reparse(value: os.stat_result) -> bool:
return bool(getattr(value, "st_file_attributes", 0) & REPARSE_POINT)
def safe_relative_path(value: str) -> str:
path = Path(value)
drive, _ = ntpath.splitdrive(value)
if (
path.is_absolute()
or Path(value.replace("\\", "/")).is_absolute()
or drive
or "\\" in value
or "\x00" in value
or any(part in {"", ".", ".."} for part in value.replace("\\", "/").split("/"))
):
raise InventoryError(f"unsafe relative path: {value!r}")
return path.as_posix()
def open_regular_file(path: Path, root: Path) -> tuple[int, os.stat_result, str]:
relative = path.relative_to(root).as_posix()
safe_relative_path(relative)
before = path.lstat()
if stat.S_ISLNK(before.st_mode) or is_reparse(before):
raise InventoryError("symlink or reparse point")
if not stat.S_ISREG(before.st_mode):
raise InventoryError("not a regular file")
if before.st_nlink > 1:
raise InventoryError("unexpected hard link")
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
opened = os.fstat(descriptor)
if not stat.S_ISREG(opened.st_mode) or file_signature(opened) != file_signature(before):
os.close(descriptor)
raise InventoryError("file changed or was replaced before reading")
return descriptor, opened, relative
def hash_file(path: Path, root: Path) -> tuple[int, str, tuple[int, int, int, int, int, int], str]:
descriptor, before, relative = open_regular_file(path, root)
digest = hashlib.sha256()
try:
with os.fdopen(descriptor, "rb", closefd=False) as source:
for chunk in iter(lambda: source.read(CHUNK_SIZE), b""):
digest.update(chunk)
after = os.fstat(descriptor)
path_after = path.lstat()
finally:
os.close(descriptor)
if (
file_signature(before) != file_signature(after)
or file_signature(after) != file_signature(path_after)
or stat.S_ISLNK(path_after.st_mode)
or is_reparse(path_after)
):
raise InventoryError("file changed during inventory")
return after.st_size, digest.hexdigest(), file_signature(after), relative
def read_json_checked(path: Path, root: Path, expected_sha256: str | None) -> Any:
descriptor, before, _ = open_regular_file(path, root)
payload = bytearray()
digest = hashlib.sha256()
try:
with os.fdopen(descriptor, "rb", closefd=False) as source:
while chunk := source.read(CHUNK_SIZE):
payload.extend(chunk)
digest.update(chunk)
if len(payload) > MAX_JSON_BYTES:
raise InventoryError("JSON file exceeds the 64 MiB validation limit")
after = os.fstat(descriptor)
path_after = path.lstat()
finally:
os.close(descriptor)
if file_signature(before) != file_signature(after) or file_signature(after) != file_signature(path_after):
raise InventoryError("file changed during validation")
if expected_sha256 is not None and digest.hexdigest() != expected_sha256:
raise InventoryError("file changed after the inventory pass")
return json.loads(payload)
def atomic_json(path: Path, value: dict[str, Any]) -> None:
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
with temporary.open("x", encoding="utf-8") as target:
json.dump(value, target, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
target.write("\n")
target.flush()
os.fsync(target.fileno())
os.replace(temporary, path)
def database(path: Path) -> sqlite3.Connection:
connection = sqlite3.connect(path)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA journal_mode=DELETE")
connection.execute("PRAGMA synchronous=FULL")
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS files (
root_id TEXT NOT NULL,
relative_path TEXT NOT NULL,
size INTEGER,
sha256 TEXT,
status TEXT NOT NULL,
device TEXT,
inode TEXT,
mtime_ns TEXT,
ctime_ns TEXT,
last_seen_run TEXT NOT NULL,
PRIMARY KEY (root_id, relative_path)
);
CREATE TABLE IF NOT EXISTS directories (
root_id TEXT NOT NULL,
relative_path TEXT NOT NULL,
last_seen_run TEXT NOT NULL,
PRIMARY KEY (root_id, relative_path)
);
CREATE TABLE IF NOT EXISTS document_dirs (
root_id TEXT NOT NULL,
code TEXT NOT NULL,
last_seen_run TEXT NOT NULL,
PRIMARY KEY (root_id, code)
);
CREATE TABLE IF NOT EXISTS manifest_docs (
root_id TEXT NOT NULL,
code TEXT NOT NULL,
source_sha256 TEXT,
state TEXT,
schema_version TEXT,
normalizer_version TEXT,
PRIMARY KEY (root_id, code)
);
CREATE TABLE IF NOT EXISTS manifest_errors (
root_id TEXT NOT NULL,
code TEXT NOT NULL,
PRIMARY KEY (root_id, code)
);
CREATE TABLE IF NOT EXISTS findings (
run_id TEXT NOT NULL,
root_id TEXT NOT NULL,
severity TEXT NOT NULL,
kind TEXT NOT NULL,
relative_path TEXT NOT NULL,
detail TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS findings_run ON findings(run_id, severity, kind);
CREATE INDEX IF NOT EXISTS files_run ON files(root_id, last_seen_run);
CREATE INDEX IF NOT EXISTS directories_run ON directories(root_id, last_seen_run);
"""
)
return connection
def prefix_upper_bound(prefix: str) -> str:
for index in range(len(prefix) - 1, -1, -1):
codepoint = ord(prefix[index])
if codepoint < 0x10FFFF:
return prefix[:index] + chr(codepoint + 1)
raise InventoryError("path prefix has no upper bound")
@contextmanager
def inventory_lock(output_dir: Path):
path = output_dir / ".inventory.lock"
descriptor = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600)
try:
metadata = os.fstat(descriptor)
if (
not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink != 1
or stat.S_IMODE(metadata.st_mode) & 0o077
):
raise InventoryError("inventory lock must be a regular private file")
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as error:
raise InventoryError("another inventory run is using this artifact directory") from error
yield
finally:
os.close(descriptor)
def finding(
connection: sqlite3.Connection,
run_id: str,
root_id: str,
kind: str,
relative_path: str = "",
detail: str = "",
severity: str = "error",
) -> None:
connection.execute(
"INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?)",
(run_id, root_id, severity, kind, relative_path, detail[:500]),
)
def result_summary(
connection: sqlite3.Connection,
run_id: str,
status: str,
scan_status: str,
verification_status: str,
roots: dict[str, Path],
progress: dict[str, dict[str, Any]],
changed: dict[str, dict[str, int]],
) -> dict[str, Any]:
errors = connection.execute(
"SELECT severity,kind,count(*) AS count FROM findings WHERE run_id=? GROUP BY severity,kind ORDER BY severity,kind",
(run_id,),
).fetchall()
examples = connection.execute(
"SELECT root_id,kind,relative_path,detail FROM findings WHERE run_id=? ORDER BY rowid LIMIT 25",
(run_id,),
).fetchall()
return {
"schema_version": INVENTORY_SCHEMA_VERSION,
"verifier_sha256": TOOL_SHA256,
"run_id": run_id,
"updated_at": utc_now(),
"status": status,
"scan_status": scan_status,
"verification_status": verification_status,
"roots": {key: str(value) for key, value in roots.items()},
"progress": progress,
"input_changes": changed,
"finding_counts": [dict(row) for row in errors],
"finding_examples": [dict(row) for row in examples],
}
def save_progress(
connection: sqlite3.Connection,
result_path: Path,
run_id: str,
roots: dict[str, Path],
progress: dict[str, dict[str, Any]],
changed: dict[str, dict[str, int]],
scan_status: str = "running",
verification_status: str = "not_started",
) -> None:
connection.commit()
atomic_json(
result_path,
result_summary(
connection, run_id, "running", scan_status, verification_status, roots, progress, changed
),
)
def path_metadata(root: Path) -> dict[str, Any]:
value = root.stat()
return {"path": str(root), "device": value.st_dev, "inode": value.st_ino}
def validate_output_dir(output: Path, roots: dict[str, Path]) -> Path:
candidate = output.absolute()
parent = candidate.parent.resolve(strict=True)
candidate = parent / candidate.name
existed = os.path.lexists(candidate)
if existed:
metadata = candidate.lstat()
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
raise InventoryError("artifact output must be a real directory, not a link or file")
if stat.S_IMODE(metadata.st_mode) & 0o077:
raise InventoryError("existing artifact output must be private (mode 0700 or stricter)")
for root in roots.values():
if candidate == root or candidate.is_relative_to(root) or root.is_relative_to(candidate):
raise InventoryError("artifact output must be separate from all source roots")
if not existed:
try:
candidate.mkdir(mode=0o700)
except FileExistsError as error:
raise InventoryError("artifact output appeared during validation; choose a private new path") from error
metadata = candidate.lstat()
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
raise InventoryError("artifact output must be a real directory, not a link or file")
if stat.S_IMODE(metadata.st_mode) & 0o077:
raise InventoryError("artifact output must be private (mode 0700 or stricter)")
resolved = candidate.resolve(strict=True)
for root in roots.values():
if resolved == root or resolved.is_relative_to(root) or root.is_relative_to(resolved):
raise InventoryError("artifact output must be separate from all source roots")
return resolved
def insert_or_update_file(
connection: sqlite3.Connection,
run_id: str,
root_id: str,
relative_path: str,
size: int,
sha256: str,
signature: tuple[int, int, int, int, int, int],
changed: dict[str, dict[str, int]],
) -> None:
old = connection.execute(
"SELECT sha256 FROM files WHERE root_id=? AND relative_path=?",
(root_id, relative_path),
).fetchone()
if old is None:
changed[root_id]["added"] += 1
elif old["sha256"] is not None and old["sha256"] != sha256:
changed[root_id]["changed"] += 1
connection.execute(
"""INSERT INTO files
(root_id,relative_path,size,sha256,status,device,inode,mtime_ns,ctime_ns,last_seen_run)
VALUES (?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(root_id,relative_path) DO UPDATE SET
size=excluded.size,sha256=excluded.sha256,status=excluded.status,
device=excluded.device,inode=excluded.inode,mtime_ns=excluded.mtime_ns,
ctime_ns=excluded.ctime_ns,last_seen_run=excluded.last_seen_run""",
(
root_id, relative_path, size, sha256, "hashed",
*(str(value) for value in (signature[0], signature[1], signature[3], signature[4])), run_id,
),
)
def record_directory(
connection: sqlite3.Connection,
run_id: str,
root_id: str,
relative_path: str,
) -> None:
connection.execute(
"""INSERT INTO directories(root_id,relative_path,last_seen_run) VALUES(?,?,?)
ON CONFLICT(root_id,relative_path) DO UPDATE SET last_seen_run=excluded.last_seen_run""",
(root_id, relative_path, run_id),
)
parent, _, code = relative_path.rpartition("/")
if parent == "documents" and code:
connection.execute(
"""INSERT INTO document_dirs(root_id,code,last_seen_run) VALUES(?,?,?)
ON CONFLICT(root_id,code) DO UPDATE SET last_seen_run=excluded.last_seen_run""",
(root_id, code, run_id),
)
def scan_root(
connection: sqlite3.Connection,
run_id: str,
root_id: str,
root: Path,
output_path: Path,
roots: dict[str, Path],
progress: dict[str, dict[str, Any]],
changed: dict[str, dict[str, int]],
checkpoint_every: int,
stop_after_entries: int | None,
processed_before: int,
progress_callback: Callable[[dict[str, Any]], None] | None,
) -> int:
record_directory(connection, run_id, root_id, "")
stack: list[os.ScandirIterator[str]] = [os.scandir(root)]
since_checkpoint = 0
try:
while stack:
iterator = stack[-1]
try:
entry = next(iterator)
except StopIteration:
iterator.close()
stack.pop()
continue
relative = Path(entry.path).relative_to(root).as_posix()
try:
safe_relative_path(relative)
before = entry.stat(follow_symlinks=False)
if entry.is_symlink() or is_reparse(before):
finding(connection, run_id, root_id, "unsafe_link", relative, "symlink or reparse point")
elif stat.S_ISDIR(before.st_mode):
record_directory(connection, run_id, root_id, relative)
stack.append(os.scandir(entry.path))
progress[root_id]["directories"] += 1
elif stat.S_ISREG(before.st_mode):
if before.st_nlink > 1:
finding(connection, run_id, root_id, "unexpected_hard_link", relative)
else:
try:
size, sha256, signature, relative = hash_file(Path(entry.path), root)
insert_or_update_file(
connection, run_id, root_id, relative, size, sha256, signature, changed
)
progress[root_id]["files"] += 1
progress[root_id]["bytes"] += size
except (OSError, InventoryError) as error:
finding(connection, run_id, root_id, "file_read_error", relative, str(error))
connection.execute(
"""INSERT INTO files(root_id,relative_path,size,sha256,status,last_seen_run)
VALUES(?,?,NULL,NULL,'error',?)
ON CONFLICT(root_id,relative_path) DO UPDATE SET
size=NULL,sha256=NULL,status='error',last_seen_run=excluded.last_seen_run""",
(root_id, relative, run_id),
)
progress[root_id]["errors"] += 1
else:
finding(connection, run_id, root_id, "unsupported_file_type", relative)
except (OSError, InventoryError) as error:
finding(connection, run_id, root_id, "entry_read_error", relative, str(error))
progress[root_id]["errors"] += 1
progress[root_id]["entries"] += 1
processed_before += 1
since_checkpoint += 1
if since_checkpoint >= checkpoint_every:
save_progress(connection, output_path, run_id, roots, progress, changed)
if progress_callback:
progress_callback(result_summary(connection, run_id, "running", "running", "not_started", roots, progress, changed))
since_checkpoint = 0
if stop_after_entries is not None and processed_before >= stop_after_entries:
raise KeyboardInterrupt
finally:
for iterator in stack:
iterator.close()
if since_checkpoint:
save_progress(connection, output_path, run_id, roots, progress, changed)
stale_files = connection.execute(
"SELECT relative_path FROM files WHERE root_id=? AND last_seen_run<>?",
(root_id, run_id),
)
for row in stale_files:
finding(connection, run_id, root_id, "removed_file", row["relative_path"], "file absent from resumed input")
changed[root_id]["removed"] += 1
connection.execute("DELETE FROM files WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
stale_dirs = connection.execute(
"SELECT relative_path FROM directories WHERE root_id=? AND last_seen_run<>?",
(root_id, run_id),
)
for row in stale_dirs:
changed[root_id]["removed_directories"] += 1
connection.execute("DELETE FROM directories WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
connection.execute("DELETE FROM document_dirs WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
connection.commit()
return processed_before
def readonly_manifest(path: Path) -> sqlite3.Connection:
sidecars = [Path(f"{path}-{suffix}") for suffix in ("wal", "shm", "journal")]
present = [sidecar.name for sidecar in sidecars if sidecar.exists()]
if present:
raise InventoryError(f"SQLite sidecar files prevent an immutable read: {', '.join(present)}")
uri = path.resolve(strict=True).as_uri() + "?mode=ro&immutable=1"
connection = sqlite3.connect(uri, uri=True)
connection.row_factory = sqlite3.Row
return connection
def load_manifest(
connection: sqlite3.Connection,
run_id: str,
root_id: str,
root: Path,
) -> None:
manifest = root / "manifest.sqlite3"
expected = connection.execute(
"SELECT sha256,status FROM files WHERE root_id=? AND relative_path='manifest.sqlite3'",
(root_id,),
).fetchone()
if expected is None or expected["status"] != "hashed":
finding(connection, run_id, root_id, "manifest_missing_or_unhashed", "manifest.sqlite3")
return
try:
source = readonly_manifest(manifest)
try:
check = source.execute("PRAGMA quick_check").fetchone()[0]
if check != "ok":
finding(connection, run_id, root_id, "manifest_quick_check_failed", "manifest.sqlite3", str(check))
tables = {row["name"] for row in source.execute("SELECT name FROM sqlite_master WHERE type='table'")}
required = {"documents"} | ({"errors"} if root_id == "raw" else set())
if not required.issubset(tables):
finding(connection, run_id, root_id, "manifest_schema_missing", "manifest.sqlite3", ",".join(sorted(required - tables)))
return
columns = {row["name"] for row in source.execute("PRAGMA table_info(documents)")}
if root_id == "raw":
required_columns = {"code", "source_sha256", "fetched_at"}
selected = "code,source_sha256"
else:
required_columns = {"code", "source_sha256", "schema_version", "normalizer_version", "state"}
selected = "code,source_sha256,schema_version,normalizer_version,state"
if not required_columns.issubset(columns):
finding(connection, run_id, root_id, "manifest_columns_missing", "manifest.sqlite3", ",".join(sorted(required_columns - columns)))
return
connection.execute("DELETE FROM manifest_docs WHERE root_id=?", (root_id,))
for row in source.execute(f"SELECT {selected} FROM documents"):
fields = list(row)
code = str(fields[0])
try:
safe_relative_path(f"documents/{code}")
except InventoryError:
finding(connection, run_id, root_id, "unsafe_manifest_code", f"documents/{code}")
continue
source_sha256 = fields[1]
state = fields[4] if root_id == "normalized" else "success"
schema_version = fields[2] if root_id == "normalized" else None
normalizer_version = fields[3] if root_id == "normalized" else None
connection.execute(
"INSERT INTO manifest_docs VALUES (?,?,?,?,?,?)",
(root_id, code, source_sha256, state, schema_version, normalizer_version),
)
if root_id == "normalized" and state != "success":
finding(connection, run_id, root_id, "normalized_manifest_state", f"documents/{code}", str(state))
if root_id == "normalized" and schema_version != NORMALIZED_SCHEMA_VERSION:
finding(connection, run_id, root_id, "schema_version_mismatch", f"documents/{code}", str(schema_version))
if root_id == "normalized" and normalizer_version not in SUPPORTED_NORMALIZER_VERSIONS:
finding(connection, run_id, root_id, "normalizer_version_mismatch", f"documents/{code}", str(normalizer_version))
if root_id == "raw":
connection.execute("DELETE FROM manifest_errors WHERE root_id='raw'")
for row in source.execute("SELECT code FROM errors"):
code = str(row[0])
connection.execute("INSERT INTO manifest_errors VALUES ('raw',?)", (code,))
finding(connection, run_id, "raw", "raw_manifest_error", f"documents/{code}", "source fetch error recorded")
finally:
source.close()
except (OSError, sqlite3.Error, InventoryError) as error:
finding(connection, run_id, root_id, "manifest_read_error", "manifest.sqlite3", str(error))
try:
_, digest, _, _ = hash_file(manifest, root)
if digest != expected["sha256"]:
finding(connection, run_id, root_id, "manifest_changed_during_validation", "manifest.sqlite3")
except (OSError, InventoryError) as error:
finding(connection, run_id, root_id, "manifest_changed_during_validation", "manifest.sqlite3", str(error))
connection.commit()
def dir_code_sets(connection: sqlite3.Connection, root_id: str) -> tuple[set[str], set[str]]:
manifest_codes = {
row[0]
for row in connection.execute("SELECT code FROM manifest_docs WHERE root_id=?", (root_id,))
}
disk_codes = {
row[0]
for row in connection.execute("SELECT code FROM document_dirs WHERE root_id=?", (root_id,))
}
return manifest_codes, disk_codes
def raw_file_rows(connection: sqlite3.Connection, code: str) -> list[dict[str, str]]:
prefix = f"documents/{code}/"
return [
{"path": row[0], "sha256": row[1]}
for row in connection.execute(
"SELECT relative_path,sha256 FROM files WHERE root_id='raw' AND relative_path>=? AND relative_path<? ORDER BY relative_path",
(prefix, prefix_upper_bound(prefix)),
)
]
def combined_source_hash(files: Iterable[dict[str, str]]) -> str:
digest = hashlib.sha256()
for item in files:
digest.update(item["path"].encode("utf-8"))
digest.update(b"\0")
digest.update(item["sha256"].encode("ascii"))
digest.update(b"\0")
return digest.hexdigest()
def validate_raw(
connection: sqlite3.Connection,
run_id: str,
root: Path,
) -> dict[str, int]:
manifest_codes, disk_codes = dir_code_sets(connection, "raw")
for code in sorted(manifest_codes - disk_codes):
finding(connection, run_id, "raw", "missing_raw_document_directory", f"documents/{code}")
for code in sorted(disk_codes - manifest_codes):
finding(connection, run_id, "raw", "extra_raw_document_directory", f"documents/{code}")
zero_editions = 0
for code in sorted(manifest_codes & disk_codes):
path = root / "documents" / code / "metadata.json"
expected = connection.execute(
"SELECT sha256 FROM files WHERE root_id='raw' AND relative_path=? AND status='hashed'",
(f"documents/{code}/metadata.json",),
).fetchone()
if expected is None:
finding(connection, run_id, "raw", "missing_raw_metadata", f"documents/{code}/metadata.json")
continue
try:
metadata = read_json_checked(path, root, expected[0])
if not isinstance(metadata, dict):
raise InventoryError("raw document metadata must contain an object")
if str(metadata.get("Code")) != code:
finding(connection, run_id, "raw", "raw_document_identity_mismatch", f"documents/{code}/metadata.json")
except (OSError, ValueError, InventoryError) as error:
finding(connection, run_id, "raw", "raw_document_metadata_invalid", f"documents/{code}/metadata.json", str(error))
edition_dirs = connection.execute(
"SELECT relative_path FROM directories WHERE root_id='raw' AND relative_path>=? AND relative_path<?",
(f"documents/{code}/editions/", prefix_upper_bound(f"documents/{code}/editions/")),
)
edition_codes = [row[0].rsplit("/", 1)[-1] for row in edition_dirs if row[0].count("/") == 3]
if not edition_codes:
zero_editions += 1
for edition_code in edition_codes:
edition_path = f"documents/{code}/editions/{edition_code}/metadata.json"
row = connection.execute(
"SELECT sha256 FROM files WHERE root_id='raw' AND relative_path=? AND status='hashed'",
(edition_path,),
).fetchone()
if row is None:
finding(connection, run_id, "raw", "missing_raw_edition_metadata", edition_path)
continue
try:
edition = read_json_checked(root / edition_path, root, row[0])
if not isinstance(edition, dict):
raise InventoryError("raw edition metadata must contain an object")
if str(edition.get("Code")) != edition_code:
finding(connection, run_id, "raw", "raw_edition_identity_mismatch", edition_path)
except (OSError, ValueError, InventoryError) as error:
finding(connection, run_id, "raw", "raw_edition_metadata_invalid", edition_path, str(error))
return {"documents": len(manifest_codes), "directories": len(disk_codes), "zero_edition_documents": zero_editions}
def validate_fragment_file(
connection: sqlite3.Connection,
run_id: str,
root: Path,
path: Path,
relative: str,
expected_sha256: str,
raw_files: dict[str, str],
) -> tuple[int, str]:
parts = Path(relative).parts
code, edition_code, language = parts[1], parts[3], parts[4]
try:
fragments = read_json_checked(path, root, expected_sha256)
if not isinstance(fragments, list):
raise InventoryError("fragments.json must contain a list")
texts: list[str] = []
for position, fragment in enumerate(fragments, 1):
if not isinstance(fragment, dict):
finding(connection, run_id, "normalized", "fragment_invalid", relative, f"position {position}")
continue
expected_id = f"document:{code}:edition:{edition_code}:lang:{language}:fragment:{position}"
if (
fragment.get("id") != expected_id
or str(fragment.get("document_code")) != code
or str(fragment.get("edition_code")) != edition_code
or fragment.get("language") != language
or fragment.get("position") != position
):
finding(connection, run_id, "normalized", "fragment_identity_mismatch", relative, f"position {position}")
text = fragment.get("text")
if not isinstance(text, str):
finding(connection, run_id, "normalized", "fragment_text_invalid", relative, f"position {position}")
continue
texts.append(text)
text_sha = hashlib.sha256(text.encode("utf-8")).hexdigest()
if fragment.get("text_sha256") != text_sha:
finding(connection, run_id, "normalized", "fragment_text_checksum_mismatch", relative, f"position {position}")
source_path = fragment.get("source_path")
try:
safe_source = safe_relative_path(source_path)
except (TypeError, InventoryError):
finding(connection, run_id, "normalized", "unsafe_fragment_source_path", relative, f"position {position}")
continue
if raw_files and raw_files.get(safe_source) != fragment.get("source_sha256"):
finding(connection, run_id, "normalized", "fragment_source_checksum_mismatch", relative, f"position {position}")
text_path = path.parent / "content.txt"
text_rel = text_path.relative_to(root).as_posix()
text_row = connection.execute(
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
(text_rel,),
).fetchone()
if text_row is None:
finding(connection, run_id, "normalized", "missing_content_text", text_rel)
else:
try:
size, digest, _, _ = hash_file(text_path, root)
if digest != text_row[0]:
finding(connection, run_id, "normalized", "content_text_changed", text_rel)
expected_text = "\n\n".join(texts) + ("\n" if texts else "")
if size <= 128 * 1024 * 1024:
descriptor, before, _ = open_regular_file(text_path, root)
try:
with os.fdopen(descriptor, "rb", closefd=False) as source_file:
actual_text = source_file.read(128 * 1024 * 1024 + 1).decode("utf-8")
after = os.fstat(descriptor)
path_after = text_path.lstat()
finally:
os.close(descriptor)
if file_signature(before) != file_signature(after) or file_signature(after) != file_signature(path_after):
raise InventoryError("file changed during validation")
if actual_text != expected_text:
finding(connection, run_id, "normalized", "content_text_fragment_mismatch", text_rel)
except (OSError, UnicodeError, InventoryError) as error:
finding(connection, run_id, "normalized", "content_text_invalid", text_rel, str(error))
return len(fragments), "ok"
except (OSError, ValueError, InventoryError) as error:
finding(connection, run_id, "normalized", "fragment_file_invalid", relative, str(error))
return 0, "error"
def validate_normalized(
connection: sqlite3.Connection,
run_id: str,
root: Path,
raw_present: bool,
) -> dict[str, int]:
manifest_codes, disk_codes = dir_code_sets(connection, "normalized")
for code in sorted(manifest_codes - disk_codes):
finding(connection, run_id, "normalized", "missing_normalized_document_directory", f"documents/{code}")
for code in sorted(disk_codes - manifest_codes):
finding(connection, run_id, "normalized", "extra_normalized_document_directory", f"documents/{code}")
total_fragments = zero_fragment_files = zero_fragment_docs = 0
for code in sorted(manifest_codes & disk_codes):
document_rel = f"documents/{code}/document.json"
row = connection.execute(
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
(document_rel,),
).fetchone()
manifest = connection.execute(
"SELECT source_sha256,state,schema_version,normalizer_version FROM manifest_docs WHERE root_id='normalized' AND code=?",
(code,),
).fetchone()
if row is None or manifest is None:
finding(connection, run_id, "normalized", "missing_document_json", document_rel)
continue
try:
document = read_json_checked(root / document_rel, root, row[0])
if not isinstance(document, dict):
raise InventoryError("normalized document must contain an object")
if str(document.get("source_code")) != code:
finding(connection, run_id, "normalized", "document_identity_mismatch", document_rel)
if str(document.get("schema_version")) != str(manifest["schema_version"]):
finding(connection, run_id, "normalized", "document_schema_manifest_mismatch", document_rel)
normalizer = document.get("normalizer") or {}
if not isinstance(normalizer, dict) or normalizer.get("version") != manifest["normalizer_version"]:
finding(connection, run_id, "normalized", "document_normalizer_manifest_mismatch", document_rel)
source = document.get("source") or {}
if not isinstance(source, dict):
finding(connection, run_id, "normalized", "document_source_invalid", document_rel)
source = {}
if source.get("sha256") != manifest["source_sha256"]:
finding(connection, run_id, "normalized", "document_source_manifest_mismatch", document_rel)
if source.get("path") != f"documents/{code}":
finding(connection, run_id, "normalized", "document_source_path_mismatch", document_rel)
raw_entries = raw_file_rows(connection, code) if raw_present else []
raw_files = {item["path"]: item["sha256"] for item in raw_entries}
if raw_present:
computed_source = combined_source_hash(raw_entries)
if computed_source != manifest["source_sha256"]:
finding(connection, run_id, "normalized", "normalized_source_hash_mismatch", document_rel)
if source.get("files") != raw_entries:
finding(connection, run_id, "normalized", "document_source_file_list_mismatch", document_rel)
summaries = document.get("editions")
if not isinstance(summaries, list):
finding(connection, run_id, "normalized", "document_editions_invalid", document_rel)
continue
summary_codes = [str(item.get("source_code")) for item in summaries if isinstance(item, dict)]
edition_dir_prefix = f"documents/{code}/editions/"
edition_dirs = {
row[0][len(edition_dir_prefix):]
for row in connection.execute(
"SELECT relative_path FROM directories WHERE root_id='normalized' AND relative_path>=? AND relative_path<?",
(edition_dir_prefix, prefix_upper_bound(edition_dir_prefix)),
)
if row[0].count("/") == 3 and row[0].startswith(edition_dir_prefix)
}
if set(summary_codes) != edition_dirs or len(summary_codes) != len(set(summary_codes)):
finding(connection, run_id, "normalized", "edition_directory_list_mismatch", f"documents/{code}/editions")
doc_fragments = 0
for edition_code in sorted(edition_dirs):
edition_rel = f"documents/{code}/editions/{edition_code}/edition.json"
edition_row = connection.execute(
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
(edition_rel,),
).fetchone()
if edition_row is None:
finding(connection, run_id, "normalized", "missing_edition_json", edition_rel)
continue
try:
edition = read_json_checked(root / edition_rel, root, edition_row[0])
if not isinstance(edition, dict):
raise InventoryError("normalized edition must contain an object")
if str(edition.get("source_code")) != edition_code:
finding(connection, run_id, "normalized", "edition_identity_mismatch", edition_rel)
if edition.get("source_code") not in summary_codes:
finding(connection, run_id, "normalized", "edition_not_in_document", edition_rel)
if raw_present:
source_prefix = f"documents/{code}/editions/{edition_code}/"
expected_edition_files = [item for item in raw_entries if item["path"].startswith(source_prefix)]
edition_source = edition.get("source") or {}
if (
not isinstance(edition_source, dict)
or edition_source.get("path") != source_prefix.rstrip("/")
or edition_source.get("files") != expected_edition_files
):
finding(connection, run_id, "normalized", "edition_source_file_list_mismatch", edition_rel)
images = edition.get("images")
if not isinstance(images, list):
finding(connection, run_id, "normalized", "edition_images_invalid", edition_rel)
else:
for image in images:
if not isinstance(image, dict):
finding(connection, run_id, "normalized", "edition_image_invalid", edition_rel)
continue
image_path = image.get("source_path")
if image_path is None:
continue
try:
safe_image_path = safe_relative_path(image_path)
except (TypeError, InventoryError):
finding(connection, run_id, "normalized", "unsafe_image_source_path", edition_rel)
continue
if raw_present and raw_files.get(safe_image_path) != image.get("source_sha256"):
finding(connection, run_id, "normalized", "image_source_checksum_mismatch", edition_rel, safe_image_path)
languages = edition.get("available_languages")
if not isinstance(languages, list):
finding(connection, run_id, "normalized", "edition_languages_invalid", edition_rel)
continue
for language in languages:
if language not in {"ru", "ky"}:
finding(connection, run_id, "normalized", "unexpected_language", edition_rel, str(language))
continue
fragment_rel = f"documents/{code}/editions/{edition_code}/{language}/fragments.json"
fragment_row = connection.execute(
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
(fragment_rel,),
).fetchone()
for suffix in ("content.html", "content.txt", "fragments.json"):
required_rel = f"documents/{code}/editions/{edition_code}/{language}/{suffix}"
if connection.execute(
"SELECT 1 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
(required_rel,),
).fetchone() is None:
finding(connection, run_id, "normalized", "missing_language_output", required_rel)
if fragment_row is None:
continue
count, status = validate_fragment_file(
connection, run_id, root, root / fragment_rel, fragment_rel, fragment_row[0], raw_files
)
if status == "ok":
doc_fragments += count
total_fragments += count
if count == 0:
zero_fragment_files += 1
except (OSError, ValueError, InventoryError) as error:
finding(connection, run_id, "normalized", "edition_json_invalid", edition_rel, str(error))
if doc_fragments == 0:
zero_fragment_docs += 1
finding(connection, run_id, "normalized", "zero_fragment_document", f"documents/{code}", severity="warning")
except (OSError, ValueError, InventoryError) as error:
finding(connection, run_id, "normalized", "document_json_invalid", document_rel, str(error))
return {
"documents": len(manifest_codes),
"directories": len(disk_codes),
"fragments": total_fragments,
"zero_fragment_documents": zero_fragment_docs,
"zero_fragment_files": zero_fragment_files,
}
def setup_run(
output_dir: Path,
roots: dict[str, Path],
resume: bool,
) -> tuple[sqlite3.Connection, str, Path]:
output_dir = validate_output_dir(output_dir, roots)
database_path = output_dir / "inventory.sqlite3"
result_path = output_dir / "result.json"
database_exists = os.path.lexists(database_path)
if database_exists:
metadata = database_path.lstat()
if (
stat.S_ISLNK(metadata.st_mode)
or is_reparse(metadata)
or not stat.S_ISREG(metadata.st_mode)
or metadata.st_nlink > 1
):
raise InventoryError("inventory.sqlite3 must be a private regular file")
if database_exists and not resume:
raise InventoryError("inventory already exists; use --resume or choose a new artifact directory")
if resume and not database_exists:
raise InventoryError("--resume requested but inventory.sqlite3 is missing")
if not resume and any(path.name != ".inventory.lock" for path in output_dir.iterdir()):
raise InventoryError("artifact output must be empty for a new inventory")
connection = database(database_path)
root_metadata = {key: path_metadata(value) for key, value in roots.items()}
if resume:
stored = {row["key"]: json.loads(row["value"]) for row in connection.execute("SELECT key,value FROM metadata")}
if stored.get("schema_version") != INVENTORY_SCHEMA_VERSION:
raise InventoryError("inventory schema differs; use a new artifact directory")
if stored.get("verifier_sha256") != TOOL_SHA256:
raise InventoryError("verifier code differs; use a new artifact directory")
if stored.get("roots") != root_metadata:
raise InventoryError("--resume roots differ from the saved inventory")
else:
connection.executemany(
"INSERT INTO metadata VALUES (?,?)",
(
("schema_version", json.dumps(INVENTORY_SCHEMA_VERSION)),
("verifier_sha256", json.dumps(TOOL_SHA256)),
("roots", json.dumps(root_metadata, sort_keys=True)),
),
)
run_id = uuid.uuid4().hex
connection.execute(
"INSERT INTO metadata(key,value) VALUES('last_run_id',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(json.dumps(run_id),),
)
connection.execute("DELETE FROM manifest_docs")
connection.execute("DELETE FROM manifest_errors")
connection.commit()
return connection, run_id, result_path
def verify_corpus(
raw_root: Path | None,
normalized_root: Path | None,
output_dir: Path,
resume: bool = False,
checkpoint_every: int = CHECKPOINT_INTERVAL,
stop_after_entries: int | None = None,
progress_callback: Callable[[dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
if raw_root is None and normalized_root is None:
raise InventoryError("at least one of --raw-root or --normalized-root is required")
roots: dict[str, Path] = {}
for key, value in (("raw", raw_root), ("normalized", normalized_root)):
if value is None:
continue
metadata = value.lstat()
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
raise InventoryError(f"source root is not a real directory: {value}")
path = value.resolve(strict=True)
roots[key] = path
if checkpoint_every <= 0:
raise InventoryError("checkpoint interval must be greater than zero")
output_dir = validate_output_dir(output_dir, roots)
with inventory_lock(output_dir):
return _run_inventory(
roots, output_dir, resume, checkpoint_every, stop_after_entries, progress_callback
)
def _run_inventory(
roots: dict[str, Path],
output_dir: Path,
resume: bool,
checkpoint_every: int,
stop_after_entries: int | None,
progress_callback: Callable[[dict[str, Any]], None] | None,
) -> dict[str, Any]:
connection, run_id, result_path = setup_run(output_dir, roots, resume)
progress = {key: {"entries": 0, "files": 0, "directories": 0, "bytes": 0, "errors": 0} for key in roots}
changed = {key: {"added": 0, "changed": 0, "removed": 0, "removed_directories": 0} for key in roots}
atomic_json(result_path, result_summary(connection, run_id, "running", "running", "not_started", roots, progress, changed))
processed = 0
try:
for root_id, root in roots.items():
processed = scan_root(
connection, run_id, root_id, root, result_path, roots, progress, changed,
checkpoint_every, stop_after_entries, processed, progress_callback,
)
load_manifest(connection, run_id, root_id, root)
save_progress(connection, result_path, run_id, roots, progress, changed, "running", "validating")
raw_result = validate_raw(connection, run_id, roots["raw"]) if "raw" in roots else None
normalized_result = (
validate_normalized(connection, run_id, roots["normalized"], "raw" in roots)
if "normalized" in roots else None
)
connection.commit()
finding_counts = connection.execute(
"SELECT severity,count(*) FROM findings WHERE run_id=? GROUP BY severity",
(run_id,),
).fetchall()
fatal = any(row[0] == "error" and row[1] for row in finding_counts)
status = "failed" if fatal else "complete"
summary = result_summary(
connection, run_id, status, "complete", "failed" if fatal else "passed", roots, progress, changed
)
summary["manifest_summary"] = {"raw": raw_result, "normalized": normalized_result}
atomic_json(result_path, summary)
return summary
except KeyboardInterrupt:
connection.commit()
summary = result_summary(
connection, run_id, "incomplete", "incomplete", "not_finished", roots, progress, changed
)
atomic_json(result_path, summary)
return summary
except Exception as error:
connection.commit()
finding(connection, run_id, "inventory", "run_failed", "", str(error))
connection.commit()
summary = result_summary(
connection, run_id, "failed", "incomplete", "not_finished", roots, progress, changed
)
atomic_json(result_path, summary)
return summary
finally:
connection.close()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--raw-root", type=Path)
parser.add_argument("--normalized-root", type=Path)
parser.add_argument("--output-dir", required=True, type=Path)
parser.add_argument("--resume", action="store_true")
parser.add_argument("--checkpoint-every", type=int, default=CHECKPOINT_INTERVAL)
arguments = parser.parse_args()
def progress(value: dict[str, Any]) -> None:
print(json.dumps({
"run_id": value["run_id"],
"status": value["status"],
"progress": value["progress"],
"updated_at": value["updated_at"],
}, ensure_ascii=False), flush=True)
try:
result = verify_corpus(
arguments.raw_root,
arguments.normalized_root,
arguments.output_dir,
resume=arguments.resume,
checkpoint_every=arguments.checkpoint_every,
progress_callback=progress,
)
except (InventoryError, OSError) as error:
print(f"ERROR {error}", file=sys.stderr)
return 2
print(json.dumps(result, ensure_ascii=False, sort_keys=True), flush=True)
return 0 if result["status"] == "complete" else 1
if __name__ == "__main__":
raise SystemExit(main())