1082 lines
50 KiB
Python
1082 lines
50 KiB
Python
#!/usr/bin/env python3
|
|
"""Build a resumable, read-only inventory and validate AkylDash corpus trees."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import ntpath
|
|
import os
|
|
import sqlite3
|
|
import stat
|
|
import sys
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterable
|
|
|
|
INVENTORY_SCHEMA_VERSION = 2
|
|
CHECKPOINT_INTERVAL = 1000
|
|
CHUNK_SIZE = 1024 * 1024
|
|
MAX_JSON_BYTES = 64 * 1024 * 1024
|
|
REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
NORMALIZER_ROOT = Path(__file__).resolve().parents[2]
|
|
TOOL_SHA256 = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
|
|
sys.path.insert(0, str(NORMALIZER_ROOT))
|
|
from backend.normalization.minjust_cbd import NORMALIZER_VERSION, SCHEMA_VERSION as NORMALIZED_SCHEMA_VERSION # noqa: E402
|
|
SUPPORTED_NORMALIZER_VERSIONS = frozenset({"1.0.0", NORMALIZER_VERSION})
|
|
|
|
|
|
class InventoryError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def file_signature(value: os.stat_result) -> tuple[int, int, int, int, int, int]:
|
|
return (
|
|
value.st_dev,
|
|
value.st_ino,
|
|
value.st_size,
|
|
value.st_mtime_ns,
|
|
value.st_ctime_ns,
|
|
value.st_nlink,
|
|
)
|
|
|
|
|
|
def is_reparse(value: os.stat_result) -> bool:
|
|
return bool(getattr(value, "st_file_attributes", 0) & REPARSE_POINT)
|
|
|
|
|
|
def safe_relative_path(value: str) -> str:
|
|
path = Path(value)
|
|
drive, _ = ntpath.splitdrive(value)
|
|
if (
|
|
path.is_absolute()
|
|
or Path(value.replace("\\", "/")).is_absolute()
|
|
or drive
|
|
or "\\" in value
|
|
or "\x00" in value
|
|
or any(part in {"", ".", ".."} for part in value.replace("\\", "/").split("/"))
|
|
):
|
|
raise InventoryError(f"unsafe relative path: {value!r}")
|
|
return path.as_posix()
|
|
|
|
|
|
def open_regular_file(path: Path, root: Path) -> tuple[int, os.stat_result, str]:
|
|
relative = path.relative_to(root).as_posix()
|
|
safe_relative_path(relative)
|
|
before = path.lstat()
|
|
if stat.S_ISLNK(before.st_mode) or is_reparse(before):
|
|
raise InventoryError("symlink or reparse point")
|
|
if not stat.S_ISREG(before.st_mode):
|
|
raise InventoryError("not a regular file")
|
|
if before.st_nlink > 1:
|
|
raise InventoryError("unexpected hard link")
|
|
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
descriptor = os.open(path, flags)
|
|
opened = os.fstat(descriptor)
|
|
if not stat.S_ISREG(opened.st_mode) or file_signature(opened) != file_signature(before):
|
|
os.close(descriptor)
|
|
raise InventoryError("file changed or was replaced before reading")
|
|
return descriptor, opened, relative
|
|
|
|
|
|
def hash_file(path: Path, root: Path) -> tuple[int, str, tuple[int, int, int, int, int, int], str]:
|
|
descriptor, before, relative = open_regular_file(path, root)
|
|
digest = hashlib.sha256()
|
|
try:
|
|
with os.fdopen(descriptor, "rb", closefd=False) as source:
|
|
for chunk in iter(lambda: source.read(CHUNK_SIZE), b""):
|
|
digest.update(chunk)
|
|
after = os.fstat(descriptor)
|
|
path_after = path.lstat()
|
|
finally:
|
|
os.close(descriptor)
|
|
if (
|
|
file_signature(before) != file_signature(after)
|
|
or file_signature(after) != file_signature(path_after)
|
|
or stat.S_ISLNK(path_after.st_mode)
|
|
or is_reparse(path_after)
|
|
):
|
|
raise InventoryError("file changed during inventory")
|
|
return after.st_size, digest.hexdigest(), file_signature(after), relative
|
|
|
|
|
|
def read_json_checked(path: Path, root: Path, expected_sha256: str | None) -> Any:
|
|
descriptor, before, _ = open_regular_file(path, root)
|
|
payload = bytearray()
|
|
digest = hashlib.sha256()
|
|
try:
|
|
with os.fdopen(descriptor, "rb", closefd=False) as source:
|
|
while chunk := source.read(CHUNK_SIZE):
|
|
payload.extend(chunk)
|
|
digest.update(chunk)
|
|
if len(payload) > MAX_JSON_BYTES:
|
|
raise InventoryError("JSON file exceeds the 64 MiB validation limit")
|
|
after = os.fstat(descriptor)
|
|
path_after = path.lstat()
|
|
finally:
|
|
os.close(descriptor)
|
|
if file_signature(before) != file_signature(after) or file_signature(after) != file_signature(path_after):
|
|
raise InventoryError("file changed during validation")
|
|
if expected_sha256 is not None and digest.hexdigest() != expected_sha256:
|
|
raise InventoryError("file changed after the inventory pass")
|
|
return json.loads(payload)
|
|
|
|
|
|
def atomic_json(path: Path, value: dict[str, Any]) -> None:
|
|
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
with temporary.open("x", encoding="utf-8") as target:
|
|
json.dump(value, target, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
target.write("\n")
|
|
target.flush()
|
|
os.fsync(target.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def database(path: Path) -> sqlite3.Connection:
|
|
connection = sqlite3.connect(path)
|
|
connection.row_factory = sqlite3.Row
|
|
connection.execute("PRAGMA journal_mode=DELETE")
|
|
connection.execute("PRAGMA synchronous=FULL")
|
|
connection.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS files (
|
|
root_id TEXT NOT NULL,
|
|
relative_path TEXT NOT NULL,
|
|
size INTEGER,
|
|
sha256 TEXT,
|
|
status TEXT NOT NULL,
|
|
device TEXT,
|
|
inode TEXT,
|
|
mtime_ns TEXT,
|
|
ctime_ns TEXT,
|
|
last_seen_run TEXT NOT NULL,
|
|
PRIMARY KEY (root_id, relative_path)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS directories (
|
|
root_id TEXT NOT NULL,
|
|
relative_path TEXT NOT NULL,
|
|
last_seen_run TEXT NOT NULL,
|
|
PRIMARY KEY (root_id, relative_path)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS document_dirs (
|
|
root_id TEXT NOT NULL,
|
|
code TEXT NOT NULL,
|
|
last_seen_run TEXT NOT NULL,
|
|
PRIMARY KEY (root_id, code)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS manifest_docs (
|
|
root_id TEXT NOT NULL,
|
|
code TEXT NOT NULL,
|
|
source_sha256 TEXT,
|
|
state TEXT,
|
|
schema_version TEXT,
|
|
normalizer_version TEXT,
|
|
PRIMARY KEY (root_id, code)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS manifest_errors (
|
|
root_id TEXT NOT NULL,
|
|
code TEXT NOT NULL,
|
|
PRIMARY KEY (root_id, code)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS findings (
|
|
run_id TEXT NOT NULL,
|
|
root_id TEXT NOT NULL,
|
|
severity TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
relative_path TEXT NOT NULL,
|
|
detail TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS findings_run ON findings(run_id, severity, kind);
|
|
CREATE INDEX IF NOT EXISTS files_run ON files(root_id, last_seen_run);
|
|
CREATE INDEX IF NOT EXISTS directories_run ON directories(root_id, last_seen_run);
|
|
"""
|
|
)
|
|
return connection
|
|
|
|
|
|
def prefix_upper_bound(prefix: str) -> str:
|
|
for index in range(len(prefix) - 1, -1, -1):
|
|
codepoint = ord(prefix[index])
|
|
if codepoint < 0x10FFFF:
|
|
return prefix[:index] + chr(codepoint + 1)
|
|
raise InventoryError("path prefix has no upper bound")
|
|
|
|
|
|
@contextmanager
|
|
def inventory_lock(output_dir: Path):
|
|
path = output_dir / ".inventory.lock"
|
|
descriptor = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600)
|
|
try:
|
|
metadata = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(metadata.st_mode)
|
|
or metadata.st_nlink != 1
|
|
or stat.S_IMODE(metadata.st_mode) & 0o077
|
|
):
|
|
raise InventoryError("inventory lock must be a regular private file")
|
|
try:
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as error:
|
|
raise InventoryError("another inventory run is using this artifact directory") from error
|
|
yield
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def finding(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root_id: str,
|
|
kind: str,
|
|
relative_path: str = "",
|
|
detail: str = "",
|
|
severity: str = "error",
|
|
) -> None:
|
|
connection.execute(
|
|
"INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?)",
|
|
(run_id, root_id, severity, kind, relative_path, detail[:500]),
|
|
)
|
|
|
|
|
|
def result_summary(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
status: str,
|
|
scan_status: str,
|
|
verification_status: str,
|
|
roots: dict[str, Path],
|
|
progress: dict[str, dict[str, Any]],
|
|
changed: dict[str, dict[str, int]],
|
|
) -> dict[str, Any]:
|
|
errors = connection.execute(
|
|
"SELECT severity,kind,count(*) AS count FROM findings WHERE run_id=? GROUP BY severity,kind ORDER BY severity,kind",
|
|
(run_id,),
|
|
).fetchall()
|
|
examples = connection.execute(
|
|
"SELECT root_id,kind,relative_path,detail FROM findings WHERE run_id=? ORDER BY rowid LIMIT 25",
|
|
(run_id,),
|
|
).fetchall()
|
|
return {
|
|
"schema_version": INVENTORY_SCHEMA_VERSION,
|
|
"verifier_sha256": TOOL_SHA256,
|
|
"run_id": run_id,
|
|
"updated_at": utc_now(),
|
|
"status": status,
|
|
"scan_status": scan_status,
|
|
"verification_status": verification_status,
|
|
"roots": {key: str(value) for key, value in roots.items()},
|
|
"progress": progress,
|
|
"input_changes": changed,
|
|
"finding_counts": [dict(row) for row in errors],
|
|
"finding_examples": [dict(row) for row in examples],
|
|
}
|
|
|
|
|
|
def save_progress(
|
|
connection: sqlite3.Connection,
|
|
result_path: Path,
|
|
run_id: str,
|
|
roots: dict[str, Path],
|
|
progress: dict[str, dict[str, Any]],
|
|
changed: dict[str, dict[str, int]],
|
|
scan_status: str = "running",
|
|
verification_status: str = "not_started",
|
|
) -> None:
|
|
connection.commit()
|
|
atomic_json(
|
|
result_path,
|
|
result_summary(
|
|
connection, run_id, "running", scan_status, verification_status, roots, progress, changed
|
|
),
|
|
)
|
|
|
|
|
|
def path_metadata(root: Path) -> dict[str, Any]:
|
|
value = root.stat()
|
|
return {"path": str(root), "device": value.st_dev, "inode": value.st_ino}
|
|
|
|
|
|
def validate_output_dir(output: Path, roots: dict[str, Path]) -> Path:
|
|
candidate = output.absolute()
|
|
parent = candidate.parent.resolve(strict=True)
|
|
candidate = parent / candidate.name
|
|
existed = os.path.lexists(candidate)
|
|
if existed:
|
|
metadata = candidate.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
|
|
raise InventoryError("artifact output must be a real directory, not a link or file")
|
|
if stat.S_IMODE(metadata.st_mode) & 0o077:
|
|
raise InventoryError("existing artifact output must be private (mode 0700 or stricter)")
|
|
for root in roots.values():
|
|
if candidate == root or candidate.is_relative_to(root) or root.is_relative_to(candidate):
|
|
raise InventoryError("artifact output must be separate from all source roots")
|
|
if not existed:
|
|
try:
|
|
candidate.mkdir(mode=0o700)
|
|
except FileExistsError as error:
|
|
raise InventoryError("artifact output appeared during validation; choose a private new path") from error
|
|
metadata = candidate.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
|
|
raise InventoryError("artifact output must be a real directory, not a link or file")
|
|
if stat.S_IMODE(metadata.st_mode) & 0o077:
|
|
raise InventoryError("artifact output must be private (mode 0700 or stricter)")
|
|
resolved = candidate.resolve(strict=True)
|
|
for root in roots.values():
|
|
if resolved == root or resolved.is_relative_to(root) or root.is_relative_to(resolved):
|
|
raise InventoryError("artifact output must be separate from all source roots")
|
|
return resolved
|
|
|
|
|
|
def insert_or_update_file(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root_id: str,
|
|
relative_path: str,
|
|
size: int,
|
|
sha256: str,
|
|
signature: tuple[int, int, int, int, int, int],
|
|
changed: dict[str, dict[str, int]],
|
|
) -> None:
|
|
old = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id=? AND relative_path=?",
|
|
(root_id, relative_path),
|
|
).fetchone()
|
|
if old is None:
|
|
changed[root_id]["added"] += 1
|
|
elif old["sha256"] is not None and old["sha256"] != sha256:
|
|
changed[root_id]["changed"] += 1
|
|
connection.execute(
|
|
"""INSERT INTO files
|
|
(root_id,relative_path,size,sha256,status,device,inode,mtime_ns,ctime_ns,last_seen_run)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(root_id,relative_path) DO UPDATE SET
|
|
size=excluded.size,sha256=excluded.sha256,status=excluded.status,
|
|
device=excluded.device,inode=excluded.inode,mtime_ns=excluded.mtime_ns,
|
|
ctime_ns=excluded.ctime_ns,last_seen_run=excluded.last_seen_run""",
|
|
(
|
|
root_id, relative_path, size, sha256, "hashed",
|
|
*(str(value) for value in (signature[0], signature[1], signature[3], signature[4])), run_id,
|
|
),
|
|
)
|
|
|
|
|
|
def record_directory(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root_id: str,
|
|
relative_path: str,
|
|
) -> None:
|
|
connection.execute(
|
|
"""INSERT INTO directories(root_id,relative_path,last_seen_run) VALUES(?,?,?)
|
|
ON CONFLICT(root_id,relative_path) DO UPDATE SET last_seen_run=excluded.last_seen_run""",
|
|
(root_id, relative_path, run_id),
|
|
)
|
|
parent, _, code = relative_path.rpartition("/")
|
|
if parent == "documents" and code:
|
|
connection.execute(
|
|
"""INSERT INTO document_dirs(root_id,code,last_seen_run) VALUES(?,?,?)
|
|
ON CONFLICT(root_id,code) DO UPDATE SET last_seen_run=excluded.last_seen_run""",
|
|
(root_id, code, run_id),
|
|
)
|
|
|
|
|
|
def scan_root(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root_id: str,
|
|
root: Path,
|
|
output_path: Path,
|
|
roots: dict[str, Path],
|
|
progress: dict[str, dict[str, Any]],
|
|
changed: dict[str, dict[str, int]],
|
|
checkpoint_every: int,
|
|
stop_after_entries: int | None,
|
|
processed_before: int,
|
|
progress_callback: Callable[[dict[str, Any]], None] | None,
|
|
) -> int:
|
|
record_directory(connection, run_id, root_id, "")
|
|
stack: list[os.ScandirIterator[str]] = [os.scandir(root)]
|
|
since_checkpoint = 0
|
|
try:
|
|
while stack:
|
|
iterator = stack[-1]
|
|
try:
|
|
entry = next(iterator)
|
|
except StopIteration:
|
|
iterator.close()
|
|
stack.pop()
|
|
continue
|
|
relative = Path(entry.path).relative_to(root).as_posix()
|
|
try:
|
|
safe_relative_path(relative)
|
|
before = entry.stat(follow_symlinks=False)
|
|
if entry.is_symlink() or is_reparse(before):
|
|
finding(connection, run_id, root_id, "unsafe_link", relative, "symlink or reparse point")
|
|
elif stat.S_ISDIR(before.st_mode):
|
|
record_directory(connection, run_id, root_id, relative)
|
|
stack.append(os.scandir(entry.path))
|
|
progress[root_id]["directories"] += 1
|
|
elif stat.S_ISREG(before.st_mode):
|
|
if before.st_nlink > 1:
|
|
finding(connection, run_id, root_id, "unexpected_hard_link", relative)
|
|
else:
|
|
try:
|
|
size, sha256, signature, relative = hash_file(Path(entry.path), root)
|
|
insert_or_update_file(
|
|
connection, run_id, root_id, relative, size, sha256, signature, changed
|
|
)
|
|
progress[root_id]["files"] += 1
|
|
progress[root_id]["bytes"] += size
|
|
except (OSError, InventoryError) as error:
|
|
finding(connection, run_id, root_id, "file_read_error", relative, str(error))
|
|
connection.execute(
|
|
"""INSERT INTO files(root_id,relative_path,size,sha256,status,last_seen_run)
|
|
VALUES(?,?,NULL,NULL,'error',?)
|
|
ON CONFLICT(root_id,relative_path) DO UPDATE SET
|
|
size=NULL,sha256=NULL,status='error',last_seen_run=excluded.last_seen_run""",
|
|
(root_id, relative, run_id),
|
|
)
|
|
progress[root_id]["errors"] += 1
|
|
else:
|
|
finding(connection, run_id, root_id, "unsupported_file_type", relative)
|
|
except (OSError, InventoryError) as error:
|
|
finding(connection, run_id, root_id, "entry_read_error", relative, str(error))
|
|
progress[root_id]["errors"] += 1
|
|
progress[root_id]["entries"] += 1
|
|
processed_before += 1
|
|
since_checkpoint += 1
|
|
if since_checkpoint >= checkpoint_every:
|
|
save_progress(connection, output_path, run_id, roots, progress, changed)
|
|
if progress_callback:
|
|
progress_callback(result_summary(connection, run_id, "running", "running", "not_started", roots, progress, changed))
|
|
since_checkpoint = 0
|
|
if stop_after_entries is not None and processed_before >= stop_after_entries:
|
|
raise KeyboardInterrupt
|
|
finally:
|
|
for iterator in stack:
|
|
iterator.close()
|
|
if since_checkpoint:
|
|
save_progress(connection, output_path, run_id, roots, progress, changed)
|
|
stale_files = connection.execute(
|
|
"SELECT relative_path FROM files WHERE root_id=? AND last_seen_run<>?",
|
|
(root_id, run_id),
|
|
)
|
|
for row in stale_files:
|
|
finding(connection, run_id, root_id, "removed_file", row["relative_path"], "file absent from resumed input")
|
|
changed[root_id]["removed"] += 1
|
|
connection.execute("DELETE FROM files WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
|
|
stale_dirs = connection.execute(
|
|
"SELECT relative_path FROM directories WHERE root_id=? AND last_seen_run<>?",
|
|
(root_id, run_id),
|
|
)
|
|
for row in stale_dirs:
|
|
changed[root_id]["removed_directories"] += 1
|
|
connection.execute("DELETE FROM directories WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
|
|
connection.execute("DELETE FROM document_dirs WHERE root_id=? AND last_seen_run<>?", (root_id, run_id))
|
|
connection.commit()
|
|
return processed_before
|
|
|
|
|
|
def readonly_manifest(path: Path) -> sqlite3.Connection:
|
|
sidecars = [Path(f"{path}-{suffix}") for suffix in ("wal", "shm", "journal")]
|
|
present = [sidecar.name for sidecar in sidecars if sidecar.exists()]
|
|
if present:
|
|
raise InventoryError(f"SQLite sidecar files prevent an immutable read: {', '.join(present)}")
|
|
uri = path.resolve(strict=True).as_uri() + "?mode=ro&immutable=1"
|
|
connection = sqlite3.connect(uri, uri=True)
|
|
connection.row_factory = sqlite3.Row
|
|
return connection
|
|
|
|
|
|
def load_manifest(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root_id: str,
|
|
root: Path,
|
|
) -> None:
|
|
manifest = root / "manifest.sqlite3"
|
|
expected = connection.execute(
|
|
"SELECT sha256,status FROM files WHERE root_id=? AND relative_path='manifest.sqlite3'",
|
|
(root_id,),
|
|
).fetchone()
|
|
if expected is None or expected["status"] != "hashed":
|
|
finding(connection, run_id, root_id, "manifest_missing_or_unhashed", "manifest.sqlite3")
|
|
return
|
|
try:
|
|
source = readonly_manifest(manifest)
|
|
try:
|
|
check = source.execute("PRAGMA quick_check").fetchone()[0]
|
|
if check != "ok":
|
|
finding(connection, run_id, root_id, "manifest_quick_check_failed", "manifest.sqlite3", str(check))
|
|
tables = {row["name"] for row in source.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
required = {"documents"} | ({"errors"} if root_id == "raw" else set())
|
|
if not required.issubset(tables):
|
|
finding(connection, run_id, root_id, "manifest_schema_missing", "manifest.sqlite3", ",".join(sorted(required - tables)))
|
|
return
|
|
columns = {row["name"] for row in source.execute("PRAGMA table_info(documents)")}
|
|
if root_id == "raw":
|
|
required_columns = {"code", "source_sha256", "fetched_at"}
|
|
selected = "code,source_sha256"
|
|
else:
|
|
required_columns = {"code", "source_sha256", "schema_version", "normalizer_version", "state"}
|
|
selected = "code,source_sha256,schema_version,normalizer_version,state"
|
|
if not required_columns.issubset(columns):
|
|
finding(connection, run_id, root_id, "manifest_columns_missing", "manifest.sqlite3", ",".join(sorted(required_columns - columns)))
|
|
return
|
|
connection.execute("DELETE FROM manifest_docs WHERE root_id=?", (root_id,))
|
|
for row in source.execute(f"SELECT {selected} FROM documents"):
|
|
fields = list(row)
|
|
code = str(fields[0])
|
|
try:
|
|
safe_relative_path(f"documents/{code}")
|
|
except InventoryError:
|
|
finding(connection, run_id, root_id, "unsafe_manifest_code", f"documents/{code}")
|
|
continue
|
|
source_sha256 = fields[1]
|
|
state = fields[4] if root_id == "normalized" else "success"
|
|
schema_version = fields[2] if root_id == "normalized" else None
|
|
normalizer_version = fields[3] if root_id == "normalized" else None
|
|
connection.execute(
|
|
"INSERT INTO manifest_docs VALUES (?,?,?,?,?,?)",
|
|
(root_id, code, source_sha256, state, schema_version, normalizer_version),
|
|
)
|
|
if root_id == "normalized" and state != "success":
|
|
finding(connection, run_id, root_id, "normalized_manifest_state", f"documents/{code}", str(state))
|
|
if root_id == "normalized" and schema_version != NORMALIZED_SCHEMA_VERSION:
|
|
finding(connection, run_id, root_id, "schema_version_mismatch", f"documents/{code}", str(schema_version))
|
|
if root_id == "normalized" and normalizer_version not in SUPPORTED_NORMALIZER_VERSIONS:
|
|
finding(connection, run_id, root_id, "normalizer_version_mismatch", f"documents/{code}", str(normalizer_version))
|
|
if root_id == "raw":
|
|
connection.execute("DELETE FROM manifest_errors WHERE root_id='raw'")
|
|
for row in source.execute("SELECT code FROM errors"):
|
|
code = str(row[0])
|
|
connection.execute("INSERT INTO manifest_errors VALUES ('raw',?)", (code,))
|
|
finding(connection, run_id, "raw", "raw_manifest_error", f"documents/{code}", "source fetch error recorded")
|
|
finally:
|
|
source.close()
|
|
except (OSError, sqlite3.Error, InventoryError) as error:
|
|
finding(connection, run_id, root_id, "manifest_read_error", "manifest.sqlite3", str(error))
|
|
try:
|
|
_, digest, _, _ = hash_file(manifest, root)
|
|
if digest != expected["sha256"]:
|
|
finding(connection, run_id, root_id, "manifest_changed_during_validation", "manifest.sqlite3")
|
|
except (OSError, InventoryError) as error:
|
|
finding(connection, run_id, root_id, "manifest_changed_during_validation", "manifest.sqlite3", str(error))
|
|
connection.commit()
|
|
|
|
|
|
def dir_code_sets(connection: sqlite3.Connection, root_id: str) -> tuple[set[str], set[str]]:
|
|
manifest_codes = {
|
|
row[0]
|
|
for row in connection.execute("SELECT code FROM manifest_docs WHERE root_id=?", (root_id,))
|
|
}
|
|
disk_codes = {
|
|
row[0]
|
|
for row in connection.execute("SELECT code FROM document_dirs WHERE root_id=?", (root_id,))
|
|
}
|
|
return manifest_codes, disk_codes
|
|
|
|
|
|
def raw_file_rows(connection: sqlite3.Connection, code: str) -> list[dict[str, str]]:
|
|
prefix = f"documents/{code}/"
|
|
return [
|
|
{"path": row[0], "sha256": row[1]}
|
|
for row in connection.execute(
|
|
"SELECT relative_path,sha256 FROM files WHERE root_id='raw' AND relative_path>=? AND relative_path<? ORDER BY relative_path",
|
|
(prefix, prefix_upper_bound(prefix)),
|
|
)
|
|
]
|
|
|
|
|
|
def combined_source_hash(files: Iterable[dict[str, str]]) -> str:
|
|
digest = hashlib.sha256()
|
|
for item in files:
|
|
digest.update(item["path"].encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(item["sha256"].encode("ascii"))
|
|
digest.update(b"\0")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def validate_raw(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root: Path,
|
|
) -> dict[str, int]:
|
|
manifest_codes, disk_codes = dir_code_sets(connection, "raw")
|
|
for code in sorted(manifest_codes - disk_codes):
|
|
finding(connection, run_id, "raw", "missing_raw_document_directory", f"documents/{code}")
|
|
for code in sorted(disk_codes - manifest_codes):
|
|
finding(connection, run_id, "raw", "extra_raw_document_directory", f"documents/{code}")
|
|
zero_editions = 0
|
|
for code in sorted(manifest_codes & disk_codes):
|
|
path = root / "documents" / code / "metadata.json"
|
|
expected = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='raw' AND relative_path=? AND status='hashed'",
|
|
(f"documents/{code}/metadata.json",),
|
|
).fetchone()
|
|
if expected is None:
|
|
finding(connection, run_id, "raw", "missing_raw_metadata", f"documents/{code}/metadata.json")
|
|
continue
|
|
try:
|
|
metadata = read_json_checked(path, root, expected[0])
|
|
if not isinstance(metadata, dict):
|
|
raise InventoryError("raw document metadata must contain an object")
|
|
if str(metadata.get("Code")) != code:
|
|
finding(connection, run_id, "raw", "raw_document_identity_mismatch", f"documents/{code}/metadata.json")
|
|
except (OSError, ValueError, InventoryError) as error:
|
|
finding(connection, run_id, "raw", "raw_document_metadata_invalid", f"documents/{code}/metadata.json", str(error))
|
|
edition_dirs = connection.execute(
|
|
"SELECT relative_path FROM directories WHERE root_id='raw' AND relative_path>=? AND relative_path<?",
|
|
(f"documents/{code}/editions/", prefix_upper_bound(f"documents/{code}/editions/")),
|
|
)
|
|
edition_codes = [row[0].rsplit("/", 1)[-1] for row in edition_dirs if row[0].count("/") == 3]
|
|
if not edition_codes:
|
|
zero_editions += 1
|
|
for edition_code in edition_codes:
|
|
edition_path = f"documents/{code}/editions/{edition_code}/metadata.json"
|
|
row = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='raw' AND relative_path=? AND status='hashed'",
|
|
(edition_path,),
|
|
).fetchone()
|
|
if row is None:
|
|
finding(connection, run_id, "raw", "missing_raw_edition_metadata", edition_path)
|
|
continue
|
|
try:
|
|
edition = read_json_checked(root / edition_path, root, row[0])
|
|
if not isinstance(edition, dict):
|
|
raise InventoryError("raw edition metadata must contain an object")
|
|
if str(edition.get("Code")) != edition_code:
|
|
finding(connection, run_id, "raw", "raw_edition_identity_mismatch", edition_path)
|
|
except (OSError, ValueError, InventoryError) as error:
|
|
finding(connection, run_id, "raw", "raw_edition_metadata_invalid", edition_path, str(error))
|
|
return {"documents": len(manifest_codes), "directories": len(disk_codes), "zero_edition_documents": zero_editions}
|
|
|
|
|
|
def validate_fragment_file(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root: Path,
|
|
path: Path,
|
|
relative: str,
|
|
expected_sha256: str,
|
|
raw_files: dict[str, str],
|
|
) -> tuple[int, str]:
|
|
parts = Path(relative).parts
|
|
code, edition_code, language = parts[1], parts[3], parts[4]
|
|
try:
|
|
fragments = read_json_checked(path, root, expected_sha256)
|
|
if not isinstance(fragments, list):
|
|
raise InventoryError("fragments.json must contain a list")
|
|
texts: list[str] = []
|
|
for position, fragment in enumerate(fragments, 1):
|
|
if not isinstance(fragment, dict):
|
|
finding(connection, run_id, "normalized", "fragment_invalid", relative, f"position {position}")
|
|
continue
|
|
expected_id = f"document:{code}:edition:{edition_code}:lang:{language}:fragment:{position}"
|
|
if (
|
|
fragment.get("id") != expected_id
|
|
or str(fragment.get("document_code")) != code
|
|
or str(fragment.get("edition_code")) != edition_code
|
|
or fragment.get("language") != language
|
|
or fragment.get("position") != position
|
|
):
|
|
finding(connection, run_id, "normalized", "fragment_identity_mismatch", relative, f"position {position}")
|
|
text = fragment.get("text")
|
|
if not isinstance(text, str):
|
|
finding(connection, run_id, "normalized", "fragment_text_invalid", relative, f"position {position}")
|
|
continue
|
|
texts.append(text)
|
|
text_sha = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
if fragment.get("text_sha256") != text_sha:
|
|
finding(connection, run_id, "normalized", "fragment_text_checksum_mismatch", relative, f"position {position}")
|
|
source_path = fragment.get("source_path")
|
|
try:
|
|
safe_source = safe_relative_path(source_path)
|
|
except (TypeError, InventoryError):
|
|
finding(connection, run_id, "normalized", "unsafe_fragment_source_path", relative, f"position {position}")
|
|
continue
|
|
if raw_files and raw_files.get(safe_source) != fragment.get("source_sha256"):
|
|
finding(connection, run_id, "normalized", "fragment_source_checksum_mismatch", relative, f"position {position}")
|
|
text_path = path.parent / "content.txt"
|
|
text_rel = text_path.relative_to(root).as_posix()
|
|
text_row = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
|
|
(text_rel,),
|
|
).fetchone()
|
|
if text_row is None:
|
|
finding(connection, run_id, "normalized", "missing_content_text", text_rel)
|
|
else:
|
|
try:
|
|
size, digest, _, _ = hash_file(text_path, root)
|
|
if digest != text_row[0]:
|
|
finding(connection, run_id, "normalized", "content_text_changed", text_rel)
|
|
expected_text = "\n\n".join(texts) + ("\n" if texts else "")
|
|
if size <= 128 * 1024 * 1024:
|
|
descriptor, before, _ = open_regular_file(text_path, root)
|
|
try:
|
|
with os.fdopen(descriptor, "rb", closefd=False) as source_file:
|
|
actual_text = source_file.read(128 * 1024 * 1024 + 1).decode("utf-8")
|
|
after = os.fstat(descriptor)
|
|
path_after = text_path.lstat()
|
|
finally:
|
|
os.close(descriptor)
|
|
if file_signature(before) != file_signature(after) or file_signature(after) != file_signature(path_after):
|
|
raise InventoryError("file changed during validation")
|
|
if actual_text != expected_text:
|
|
finding(connection, run_id, "normalized", "content_text_fragment_mismatch", text_rel)
|
|
except (OSError, UnicodeError, InventoryError) as error:
|
|
finding(connection, run_id, "normalized", "content_text_invalid", text_rel, str(error))
|
|
return len(fragments), "ok"
|
|
except (OSError, ValueError, InventoryError) as error:
|
|
finding(connection, run_id, "normalized", "fragment_file_invalid", relative, str(error))
|
|
return 0, "error"
|
|
|
|
|
|
def validate_normalized(
|
|
connection: sqlite3.Connection,
|
|
run_id: str,
|
|
root: Path,
|
|
raw_present: bool,
|
|
) -> dict[str, int]:
|
|
manifest_codes, disk_codes = dir_code_sets(connection, "normalized")
|
|
for code in sorted(manifest_codes - disk_codes):
|
|
finding(connection, run_id, "normalized", "missing_normalized_document_directory", f"documents/{code}")
|
|
for code in sorted(disk_codes - manifest_codes):
|
|
finding(connection, run_id, "normalized", "extra_normalized_document_directory", f"documents/{code}")
|
|
total_fragments = zero_fragment_files = zero_fragment_docs = 0
|
|
for code in sorted(manifest_codes & disk_codes):
|
|
document_rel = f"documents/{code}/document.json"
|
|
row = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
|
|
(document_rel,),
|
|
).fetchone()
|
|
manifest = connection.execute(
|
|
"SELECT source_sha256,state,schema_version,normalizer_version FROM manifest_docs WHERE root_id='normalized' AND code=?",
|
|
(code,),
|
|
).fetchone()
|
|
if row is None or manifest is None:
|
|
finding(connection, run_id, "normalized", "missing_document_json", document_rel)
|
|
continue
|
|
try:
|
|
document = read_json_checked(root / document_rel, root, row[0])
|
|
if not isinstance(document, dict):
|
|
raise InventoryError("normalized document must contain an object")
|
|
if str(document.get("source_code")) != code:
|
|
finding(connection, run_id, "normalized", "document_identity_mismatch", document_rel)
|
|
if str(document.get("schema_version")) != str(manifest["schema_version"]):
|
|
finding(connection, run_id, "normalized", "document_schema_manifest_mismatch", document_rel)
|
|
normalizer = document.get("normalizer") or {}
|
|
if not isinstance(normalizer, dict) or normalizer.get("version") != manifest["normalizer_version"]:
|
|
finding(connection, run_id, "normalized", "document_normalizer_manifest_mismatch", document_rel)
|
|
source = document.get("source") or {}
|
|
if not isinstance(source, dict):
|
|
finding(connection, run_id, "normalized", "document_source_invalid", document_rel)
|
|
source = {}
|
|
if source.get("sha256") != manifest["source_sha256"]:
|
|
finding(connection, run_id, "normalized", "document_source_manifest_mismatch", document_rel)
|
|
if source.get("path") != f"documents/{code}":
|
|
finding(connection, run_id, "normalized", "document_source_path_mismatch", document_rel)
|
|
raw_entries = raw_file_rows(connection, code) if raw_present else []
|
|
raw_files = {item["path"]: item["sha256"] for item in raw_entries}
|
|
if raw_present:
|
|
computed_source = combined_source_hash(raw_entries)
|
|
if computed_source != manifest["source_sha256"]:
|
|
finding(connection, run_id, "normalized", "normalized_source_hash_mismatch", document_rel)
|
|
if source.get("files") != raw_entries:
|
|
finding(connection, run_id, "normalized", "document_source_file_list_mismatch", document_rel)
|
|
summaries = document.get("editions")
|
|
if not isinstance(summaries, list):
|
|
finding(connection, run_id, "normalized", "document_editions_invalid", document_rel)
|
|
continue
|
|
summary_codes = [str(item.get("source_code")) for item in summaries if isinstance(item, dict)]
|
|
edition_dir_prefix = f"documents/{code}/editions/"
|
|
edition_dirs = {
|
|
row[0][len(edition_dir_prefix):]
|
|
for row in connection.execute(
|
|
"SELECT relative_path FROM directories WHERE root_id='normalized' AND relative_path>=? AND relative_path<?",
|
|
(edition_dir_prefix, prefix_upper_bound(edition_dir_prefix)),
|
|
)
|
|
if row[0].count("/") == 3 and row[0].startswith(edition_dir_prefix)
|
|
}
|
|
if set(summary_codes) != edition_dirs or len(summary_codes) != len(set(summary_codes)):
|
|
finding(connection, run_id, "normalized", "edition_directory_list_mismatch", f"documents/{code}/editions")
|
|
doc_fragments = 0
|
|
for edition_code in sorted(edition_dirs):
|
|
edition_rel = f"documents/{code}/editions/{edition_code}/edition.json"
|
|
edition_row = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
|
|
(edition_rel,),
|
|
).fetchone()
|
|
if edition_row is None:
|
|
finding(connection, run_id, "normalized", "missing_edition_json", edition_rel)
|
|
continue
|
|
try:
|
|
edition = read_json_checked(root / edition_rel, root, edition_row[0])
|
|
if not isinstance(edition, dict):
|
|
raise InventoryError("normalized edition must contain an object")
|
|
if str(edition.get("source_code")) != edition_code:
|
|
finding(connection, run_id, "normalized", "edition_identity_mismatch", edition_rel)
|
|
if edition.get("source_code") not in summary_codes:
|
|
finding(connection, run_id, "normalized", "edition_not_in_document", edition_rel)
|
|
if raw_present:
|
|
source_prefix = f"documents/{code}/editions/{edition_code}/"
|
|
expected_edition_files = [item for item in raw_entries if item["path"].startswith(source_prefix)]
|
|
edition_source = edition.get("source") or {}
|
|
if (
|
|
not isinstance(edition_source, dict)
|
|
or edition_source.get("path") != source_prefix.rstrip("/")
|
|
or edition_source.get("files") != expected_edition_files
|
|
):
|
|
finding(connection, run_id, "normalized", "edition_source_file_list_mismatch", edition_rel)
|
|
images = edition.get("images")
|
|
if not isinstance(images, list):
|
|
finding(connection, run_id, "normalized", "edition_images_invalid", edition_rel)
|
|
else:
|
|
for image in images:
|
|
if not isinstance(image, dict):
|
|
finding(connection, run_id, "normalized", "edition_image_invalid", edition_rel)
|
|
continue
|
|
image_path = image.get("source_path")
|
|
if image_path is None:
|
|
continue
|
|
try:
|
|
safe_image_path = safe_relative_path(image_path)
|
|
except (TypeError, InventoryError):
|
|
finding(connection, run_id, "normalized", "unsafe_image_source_path", edition_rel)
|
|
continue
|
|
if raw_present and raw_files.get(safe_image_path) != image.get("source_sha256"):
|
|
finding(connection, run_id, "normalized", "image_source_checksum_mismatch", edition_rel, safe_image_path)
|
|
languages = edition.get("available_languages")
|
|
if not isinstance(languages, list):
|
|
finding(connection, run_id, "normalized", "edition_languages_invalid", edition_rel)
|
|
continue
|
|
for language in languages:
|
|
if language not in {"ru", "ky"}:
|
|
finding(connection, run_id, "normalized", "unexpected_language", edition_rel, str(language))
|
|
continue
|
|
fragment_rel = f"documents/{code}/editions/{edition_code}/{language}/fragments.json"
|
|
fragment_row = connection.execute(
|
|
"SELECT sha256 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
|
|
(fragment_rel,),
|
|
).fetchone()
|
|
for suffix in ("content.html", "content.txt", "fragments.json"):
|
|
required_rel = f"documents/{code}/editions/{edition_code}/{language}/{suffix}"
|
|
if connection.execute(
|
|
"SELECT 1 FROM files WHERE root_id='normalized' AND relative_path=? AND status='hashed'",
|
|
(required_rel,),
|
|
).fetchone() is None:
|
|
finding(connection, run_id, "normalized", "missing_language_output", required_rel)
|
|
if fragment_row is None:
|
|
continue
|
|
count, status = validate_fragment_file(
|
|
connection, run_id, root, root / fragment_rel, fragment_rel, fragment_row[0], raw_files
|
|
)
|
|
if status == "ok":
|
|
doc_fragments += count
|
|
total_fragments += count
|
|
if count == 0:
|
|
zero_fragment_files += 1
|
|
except (OSError, ValueError, InventoryError) as error:
|
|
finding(connection, run_id, "normalized", "edition_json_invalid", edition_rel, str(error))
|
|
if doc_fragments == 0:
|
|
zero_fragment_docs += 1
|
|
finding(connection, run_id, "normalized", "zero_fragment_document", f"documents/{code}", severity="warning")
|
|
except (OSError, ValueError, InventoryError) as error:
|
|
finding(connection, run_id, "normalized", "document_json_invalid", document_rel, str(error))
|
|
return {
|
|
"documents": len(manifest_codes),
|
|
"directories": len(disk_codes),
|
|
"fragments": total_fragments,
|
|
"zero_fragment_documents": zero_fragment_docs,
|
|
"zero_fragment_files": zero_fragment_files,
|
|
}
|
|
|
|
|
|
def setup_run(
|
|
output_dir: Path,
|
|
roots: dict[str, Path],
|
|
resume: bool,
|
|
) -> tuple[sqlite3.Connection, str, Path]:
|
|
output_dir = validate_output_dir(output_dir, roots)
|
|
database_path = output_dir / "inventory.sqlite3"
|
|
result_path = output_dir / "result.json"
|
|
database_exists = os.path.lexists(database_path)
|
|
if database_exists:
|
|
metadata = database_path.lstat()
|
|
if (
|
|
stat.S_ISLNK(metadata.st_mode)
|
|
or is_reparse(metadata)
|
|
or not stat.S_ISREG(metadata.st_mode)
|
|
or metadata.st_nlink > 1
|
|
):
|
|
raise InventoryError("inventory.sqlite3 must be a private regular file")
|
|
if database_exists and not resume:
|
|
raise InventoryError("inventory already exists; use --resume or choose a new artifact directory")
|
|
if resume and not database_exists:
|
|
raise InventoryError("--resume requested but inventory.sqlite3 is missing")
|
|
if not resume and any(path.name != ".inventory.lock" for path in output_dir.iterdir()):
|
|
raise InventoryError("artifact output must be empty for a new inventory")
|
|
connection = database(database_path)
|
|
root_metadata = {key: path_metadata(value) for key, value in roots.items()}
|
|
if resume:
|
|
stored = {row["key"]: json.loads(row["value"]) for row in connection.execute("SELECT key,value FROM metadata")}
|
|
if stored.get("schema_version") != INVENTORY_SCHEMA_VERSION:
|
|
raise InventoryError("inventory schema differs; use a new artifact directory")
|
|
if stored.get("verifier_sha256") != TOOL_SHA256:
|
|
raise InventoryError("verifier code differs; use a new artifact directory")
|
|
if stored.get("roots") != root_metadata:
|
|
raise InventoryError("--resume roots differ from the saved inventory")
|
|
else:
|
|
connection.executemany(
|
|
"INSERT INTO metadata VALUES (?,?)",
|
|
(
|
|
("schema_version", json.dumps(INVENTORY_SCHEMA_VERSION)),
|
|
("verifier_sha256", json.dumps(TOOL_SHA256)),
|
|
("roots", json.dumps(root_metadata, sort_keys=True)),
|
|
),
|
|
)
|
|
run_id = uuid.uuid4().hex
|
|
connection.execute(
|
|
"INSERT INTO metadata(key,value) VALUES('last_run_id',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
(json.dumps(run_id),),
|
|
)
|
|
connection.execute("DELETE FROM manifest_docs")
|
|
connection.execute("DELETE FROM manifest_errors")
|
|
connection.commit()
|
|
return connection, run_id, result_path
|
|
|
|
|
|
def verify_corpus(
|
|
raw_root: Path | None,
|
|
normalized_root: Path | None,
|
|
output_dir: Path,
|
|
resume: bool = False,
|
|
checkpoint_every: int = CHECKPOINT_INTERVAL,
|
|
stop_after_entries: int | None = None,
|
|
progress_callback: Callable[[dict[str, Any]], None] | None = None,
|
|
) -> dict[str, Any]:
|
|
if raw_root is None and normalized_root is None:
|
|
raise InventoryError("at least one of --raw-root or --normalized-root is required")
|
|
roots: dict[str, Path] = {}
|
|
for key, value in (("raw", raw_root), ("normalized", normalized_root)):
|
|
if value is None:
|
|
continue
|
|
metadata = value.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or is_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
|
|
raise InventoryError(f"source root is not a real directory: {value}")
|
|
path = value.resolve(strict=True)
|
|
roots[key] = path
|
|
if checkpoint_every <= 0:
|
|
raise InventoryError("checkpoint interval must be greater than zero")
|
|
output_dir = validate_output_dir(output_dir, roots)
|
|
with inventory_lock(output_dir):
|
|
return _run_inventory(
|
|
roots, output_dir, resume, checkpoint_every, stop_after_entries, progress_callback
|
|
)
|
|
|
|
|
|
def _run_inventory(
|
|
roots: dict[str, Path],
|
|
output_dir: Path,
|
|
resume: bool,
|
|
checkpoint_every: int,
|
|
stop_after_entries: int | None,
|
|
progress_callback: Callable[[dict[str, Any]], None] | None,
|
|
) -> dict[str, Any]:
|
|
connection, run_id, result_path = setup_run(output_dir, roots, resume)
|
|
progress = {key: {"entries": 0, "files": 0, "directories": 0, "bytes": 0, "errors": 0} for key in roots}
|
|
changed = {key: {"added": 0, "changed": 0, "removed": 0, "removed_directories": 0} for key in roots}
|
|
atomic_json(result_path, result_summary(connection, run_id, "running", "running", "not_started", roots, progress, changed))
|
|
processed = 0
|
|
try:
|
|
for root_id, root in roots.items():
|
|
processed = scan_root(
|
|
connection, run_id, root_id, root, result_path, roots, progress, changed,
|
|
checkpoint_every, stop_after_entries, processed, progress_callback,
|
|
)
|
|
load_manifest(connection, run_id, root_id, root)
|
|
save_progress(connection, result_path, run_id, roots, progress, changed, "running", "validating")
|
|
raw_result = validate_raw(connection, run_id, roots["raw"]) if "raw" in roots else None
|
|
normalized_result = (
|
|
validate_normalized(connection, run_id, roots["normalized"], "raw" in roots)
|
|
if "normalized" in roots else None
|
|
)
|
|
connection.commit()
|
|
finding_counts = connection.execute(
|
|
"SELECT severity,count(*) FROM findings WHERE run_id=? GROUP BY severity",
|
|
(run_id,),
|
|
).fetchall()
|
|
fatal = any(row[0] == "error" and row[1] for row in finding_counts)
|
|
status = "failed" if fatal else "complete"
|
|
summary = result_summary(
|
|
connection, run_id, status, "complete", "failed" if fatal else "passed", roots, progress, changed
|
|
)
|
|
summary["manifest_summary"] = {"raw": raw_result, "normalized": normalized_result}
|
|
atomic_json(result_path, summary)
|
|
return summary
|
|
except KeyboardInterrupt:
|
|
connection.commit()
|
|
summary = result_summary(
|
|
connection, run_id, "incomplete", "incomplete", "not_finished", roots, progress, changed
|
|
)
|
|
atomic_json(result_path, summary)
|
|
return summary
|
|
except Exception as error:
|
|
connection.commit()
|
|
finding(connection, run_id, "inventory", "run_failed", "", str(error))
|
|
connection.commit()
|
|
summary = result_summary(
|
|
connection, run_id, "failed", "incomplete", "not_finished", roots, progress, changed
|
|
)
|
|
atomic_json(result_path, summary)
|
|
return summary
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--raw-root", type=Path)
|
|
parser.add_argument("--normalized-root", type=Path)
|
|
parser.add_argument("--output-dir", required=True, type=Path)
|
|
parser.add_argument("--resume", action="store_true")
|
|
parser.add_argument("--checkpoint-every", type=int, default=CHECKPOINT_INTERVAL)
|
|
arguments = parser.parse_args()
|
|
|
|
def progress(value: dict[str, Any]) -> None:
|
|
print(json.dumps({
|
|
"run_id": value["run_id"],
|
|
"status": value["status"],
|
|
"progress": value["progress"],
|
|
"updated_at": value["updated_at"],
|
|
}, ensure_ascii=False), flush=True)
|
|
|
|
try:
|
|
result = verify_corpus(
|
|
arguments.raw_root,
|
|
arguments.normalized_root,
|
|
arguments.output_dir,
|
|
resume=arguments.resume,
|
|
checkpoint_every=arguments.checkpoint_every,
|
|
progress_callback=progress,
|
|
)
|
|
except (InventoryError, OSError) as error:
|
|
print(f"ERROR {error}", file=sys.stderr)
|
|
return 2
|
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True), flush=True)
|
|
return 0 if result["status"] == "complete" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|