721 lines
33 KiB
Python
721 lines
33 KiB
Python
"""Read-only staging client for the current MinJust CBD API."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import logging
|
|
import os
|
|
import sqlite3
|
|
import stat
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from ingestion.minjust_cbd import atomic_write, connect_manifest, json_bytes
|
|
|
|
API_ROOT = "https://cbd.minjust.gov.kg/api/v1"
|
|
APP_VERSION = "0.13.0"
|
|
REQUEST_CONTRACT = {
|
|
"version": 2,
|
|
"endpoint": "/GetEdition",
|
|
"edition_id_parameter": "editionId",
|
|
"exact_parameter": "true",
|
|
"language_parameter": "lang",
|
|
"response_id_must_match": True,
|
|
}
|
|
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
DEFAULT_DOCUMENT_BUDGET = 4 * 1024 * 1024 * 1024
|
|
LANGUAGES = (("ru", "ru", "textRusType"), ("kg", "ky", "textKyrType"))
|
|
LOGGER = logging.getLogger(__name__)
|
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
class SourceError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class SourceChanged(SourceError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ApiResponse:
|
|
value: dict
|
|
body: bytes
|
|
sha256: str
|
|
|
|
|
|
class CurrentCbdClient:
|
|
def __init__(
|
|
self,
|
|
requests_per_second: float = 1,
|
|
timeout: int = 60,
|
|
retries: int = 5,
|
|
max_response_bytes: int = MAX_RESPONSE_BYTES,
|
|
opener: Callable = urllib.request.urlopen,
|
|
sleep: Callable[[float], None] = time.sleep,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
) -> None:
|
|
if requests_per_second <= 0 or timeout <= 0 or retries <= 0 or max_response_bytes <= 0:
|
|
raise ValueError("rate, timeout, retries, and response limit must be positive")
|
|
self.interval = 1 / requests_per_second
|
|
self.timeout = timeout
|
|
self.retries = retries
|
|
self.max_response_bytes = max_response_bytes
|
|
self.opener = opener
|
|
self.sleep = sleep
|
|
self.clock = clock
|
|
self.last_request = 0.0
|
|
|
|
def request_json(
|
|
self,
|
|
path: str,
|
|
*,
|
|
method: str = "GET",
|
|
parameters: tuple[tuple[str, object], ...] = (),
|
|
payload: dict | None = None,
|
|
) -> ApiResponse:
|
|
if not path.startswith("/") or ".." in path.split("/"):
|
|
raise ValueError("API path must be an absolute endpoint path")
|
|
query = urllib.parse.urlencode(parameters, doseq=True)
|
|
url = API_ROOT + path + (f"?{query}" if query else "")
|
|
body = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":")).encode() if method == "POST" else None
|
|
for attempt in range(self.retries):
|
|
wait = self.interval - (self.clock() - self.last_request)
|
|
if wait > 0:
|
|
self.sleep(wait)
|
|
request = urllib.request.Request(
|
|
url,
|
|
data=body,
|
|
method=method,
|
|
headers={
|
|
"Accept": "application/json",
|
|
**({"Content-Type": "application/json"} if body is not None else {}),
|
|
"User-Agent": f"Akyldash/{APP_VERSION} (+https://cbd.minjust.gov.kg)",
|
|
},
|
|
)
|
|
try:
|
|
self.last_request = self.clock()
|
|
with self.opener(request, timeout=self.timeout) as response:
|
|
raw = response.read(self.max_response_bytes + 1)
|
|
if len(raw) > self.max_response_bytes:
|
|
raise SourceError(f"MinJust response exceeds {self.max_response_bytes} bytes")
|
|
value = json.loads(raw)
|
|
if not isinstance(value, dict):
|
|
raise SourceError("MinJust response must be a JSON object")
|
|
return ApiResponse(value, raw, hashlib.sha256(raw).hexdigest())
|
|
except urllib.error.HTTPError as error:
|
|
if error.code != 429 and error.code < 500:
|
|
raise SourceError(f"MinJust returned HTTP {error.code} for {path}") from error
|
|
delay = 2**attempt
|
|
if error.code == 429:
|
|
delay = max(delay, retry_after_seconds(error.headers.get("Retry-After")))
|
|
except (TimeoutError, urllib.error.URLError, http.client.IncompleteRead, json.JSONDecodeError) as error:
|
|
delay = 2**attempt
|
|
LOGGER.warning("MinJust request failed for %s (attempt %s/%s)", path, attempt + 1, self.retries)
|
|
last_error = error
|
|
if attempt + 1 < self.retries:
|
|
self.sleep(delay)
|
|
raise SourceError(f"MinJust request failed after {self.retries} attempts: {path}") from locals().get("last_error")
|
|
|
|
def document(self, alias: str) -> ApiResponse:
|
|
response = self.request_json("/GetDocument", parameters=(("DocumentCode", alias),))
|
|
value = response.value
|
|
canonical = value.get("documentCode")
|
|
editions = value.get("editions")
|
|
if not isinstance(canonical, int) or isinstance(canonical, bool) or canonical < 1:
|
|
raise SourceError("MinJust document has an invalid canonical documentCode")
|
|
if not isinstance(editions, list):
|
|
raise SourceError("MinJust document is missing editions")
|
|
ids = [edition_id(item) for item in editions]
|
|
if len(ids) != len(set(ids)):
|
|
raise SourceError(f"MinJust document {canonical} contains duplicate edition IDs")
|
|
return response
|
|
|
|
def edition(self, identifier: int, language: str) -> ApiResponse:
|
|
if identifier < 1 or language not in {"ru", "kg"}:
|
|
raise ValueError("edition ID or requested language is invalid")
|
|
response = self.request_json(
|
|
"/GetEdition",
|
|
parameters=(("editionId", identifier), ("lang", language), ("exact", "true")),
|
|
)
|
|
returned_id = response.value.get("id")
|
|
if not isinstance(returned_id, int) or isinstance(returned_id, bool) or returned_id < 1:
|
|
raise SourceError(f"MinJust returned an invalid edition response ID for {identifier}")
|
|
if returned_id != identifier:
|
|
raise SourceChanged(f"MinJust exact edition ID {returned_id} does not match requested ID {identifier}")
|
|
return response
|
|
|
|
|
|
def retry_after_seconds(value: str | None) -> float:
|
|
if not value:
|
|
return 0
|
|
try:
|
|
return max(0, float(value))
|
|
except ValueError:
|
|
try:
|
|
return max(0, (parsedate_to_datetime(value) - datetime.now(timezone.utc)).total_seconds())
|
|
except (TypeError, ValueError, OverflowError):
|
|
return 0
|
|
|
|
|
|
def edition_id(value: object) -> int:
|
|
if not isinstance(value, dict):
|
|
raise SourceError("MinJust document contains an invalid edition")
|
|
identifier = value.get("id")
|
|
if not isinstance(identifier, int) or isinstance(identifier, bool) or identifier < 1:
|
|
raise SourceError("MinJust edition has an invalid id")
|
|
return identifier
|
|
|
|
|
|
def open_manifest(path: Path) -> sqlite3.Connection:
|
|
connection = sqlite3.connect(path)
|
|
connection.execute("PRAGMA journal_mode=DELETE")
|
|
connection.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS documents (
|
|
alias TEXT PRIMARY KEY,
|
|
canonical_id INTEGER,
|
|
response_sha256 TEXT,
|
|
expected_editions INTEGER,
|
|
state TEXT NOT NULL,
|
|
error TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS editions (
|
|
canonical_id INTEGER NOT NULL,
|
|
edition_id INTEGER NOT NULL,
|
|
source_order INTEGER NOT NULL,
|
|
response_sha256 TEXT NOT NULL,
|
|
PRIMARY KEY (canonical_id, edition_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS languages (
|
|
canonical_id INTEGER NOT NULL,
|
|
edition_id INTEGER NOT NULL,
|
|
request_language TEXT NOT NULL,
|
|
archive_language TEXT NOT NULL,
|
|
response_sha256 TEXT,
|
|
content_sha256 TEXT,
|
|
byte_count INTEGER NOT NULL DEFAULT 0,
|
|
state TEXT NOT NULL,
|
|
error TEXT,
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY (canonical_id, edition_id, request_language)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS generation (
|
|
contract_sha256 TEXT NOT NULL
|
|
);
|
|
"""
|
|
)
|
|
return connection
|
|
|
|
|
|
def reject_symlink_components(path: Path) -> None:
|
|
current = Path(path.anchor)
|
|
for part in path.parts[1:]:
|
|
current /= part
|
|
try:
|
|
mode = current.lstat().st_mode
|
|
except FileNotFoundError:
|
|
continue
|
|
if stat.S_ISLNK(mode):
|
|
raise ValueError(f"staging path must not contain a symbolic link: {current}")
|
|
if current != path and not stat.S_ISDIR(mode):
|
|
raise ValueError(f"staging parent must be a directory: {current}")
|
|
|
|
|
|
def ensure_stage_path(root: Path, path: Path) -> Path:
|
|
root = root.absolute()
|
|
path = path.absolute()
|
|
try:
|
|
relative = path.relative_to(root)
|
|
except ValueError as error:
|
|
raise ValueError("staging file escaped its private generation") from error
|
|
current = root
|
|
if current.is_symlink():
|
|
raise ValueError("staging root must not be a symbolic link")
|
|
for part in relative.parts:
|
|
current /= part
|
|
try:
|
|
mode = current.lstat().st_mode
|
|
except FileNotFoundError:
|
|
continue
|
|
if stat.S_ISLNK(mode):
|
|
raise ValueError(f"staging path must not contain a symbolic link: {relative}")
|
|
if current != path and not stat.S_ISDIR(mode):
|
|
raise ValueError(f"staging parent must be a directory: {relative}")
|
|
return path
|
|
|
|
|
|
def stage_write(root: Path, path: Path, content: bytes) -> None:
|
|
path = ensure_stage_path(root, path)
|
|
atomic_write(path, content)
|
|
ensure_stage_path(root, path)
|
|
|
|
|
|
def stage_mkdir(root: Path, path: Path) -> None:
|
|
path = ensure_stage_path(root, path)
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
ensure_stage_path(root, path)
|
|
if not path.is_dir():
|
|
raise ValueError("staging path must be a directory")
|
|
|
|
|
|
def validate_stage_tree(output: Path) -> None:
|
|
for relative in (
|
|
Path("current-api.sqlite3"), Path("current-api.sqlite3-journal"),
|
|
Path("current-api.sqlite3-wal"), Path("current-api.sqlite3-shm"),
|
|
Path("manifest.sqlite3"), Path("manifest.sqlite3-journal"),
|
|
Path("manifest.sqlite3-wal"), Path("manifest.sqlite3-shm"),
|
|
Path("request-contract.json"),
|
|
):
|
|
path = ensure_stage_path(output, output / relative)
|
|
if path.exists() and not path.is_file():
|
|
raise ValueError(f"staging checkpoint must be a regular file: {relative}")
|
|
for relative in (Path(".work"), Path("documents")):
|
|
root = ensure_stage_path(output, output / relative)
|
|
if root.exists() and not root.is_dir():
|
|
raise ValueError(f"staging path must be a directory: {relative}")
|
|
if root.is_dir():
|
|
for directory, directories, files in os.walk(root, followlinks=False):
|
|
for name in directories + files:
|
|
ensure_stage_path(output, Path(directory) / name)
|
|
|
|
|
|
def validate_stage_root(output: Path, resume: bool) -> Path:
|
|
requested = Path(output)
|
|
if ".." in requested.parts:
|
|
raise ValueError("staging output path must not contain '..'")
|
|
destination = Path(os.path.abspath(requested))
|
|
reject_symlink_components(destination)
|
|
parent = destination.parent
|
|
if not parent.is_dir():
|
|
raise ValueError("staging output parent must already exist")
|
|
for ancestor in (parent, *parent.parents):
|
|
info = ancestor.lstat()
|
|
mode = stat.S_IMODE(info.st_mode)
|
|
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
|
raise ValueError("staging output ancestors must be real directories")
|
|
sticky_parent = bool(mode & stat.S_ISVTX)
|
|
if mode & 0o022 and not sticky_parent:
|
|
raise ValueError("staging output parent is not trusted")
|
|
for relative in (Path("data/minjust-cbd"), Path("data/minjust-normalized")):
|
|
legacy = (REPOSITORY_ROOT / relative).resolve()
|
|
if destination == legacy or destination.is_relative_to(legacy) or legacy.is_relative_to(destination):
|
|
raise ValueError("current API staging must not overlap the existing corpus")
|
|
if destination.exists():
|
|
if destination.is_symlink() or not destination.is_dir():
|
|
raise ValueError("staging output must be a real directory")
|
|
else:
|
|
destination.mkdir(mode=0o700)
|
|
root_info = destination.lstat()
|
|
root_mode = stat.S_IMODE(root_info.st_mode)
|
|
if (
|
|
stat.S_ISLNK(root_info.st_mode)
|
|
or not stat.S_ISDIR(root_info.st_mode)
|
|
or root_info.st_uid != os.geteuid()
|
|
or root_mode & 0o077
|
|
or (root_mode & 0o700) != 0o700
|
|
):
|
|
raise ValueError("staging output must be a private directory owned by the current user")
|
|
validate_stage_tree(destination)
|
|
if any(destination.iterdir()) and not resume:
|
|
raise ValueError("staging output is not empty; use --resume for this generation")
|
|
contract_path = destination / "request-contract.json"
|
|
if contract_path.exists():
|
|
try:
|
|
contract = json.loads(contract_path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
|
raise SourceChanged("staging generation request contract is invalid") from error
|
|
if contract != REQUEST_CONTRACT:
|
|
raise SourceChanged("staging generation uses a different request contract")
|
|
else:
|
|
if any(destination.iterdir()):
|
|
raise SourceChanged("staging generation has no exact-edition request contract; start a new generation")
|
|
stage_write(destination, contract_path, json_bytes(REQUEST_CONTRACT))
|
|
return destination
|
|
|
|
|
|
def stage_document(
|
|
output: Path,
|
|
alias: str,
|
|
client: CurrentCbdClient,
|
|
*,
|
|
resume: bool = False,
|
|
max_document_bytes: int = DEFAULT_DOCUMENT_BUDGET,
|
|
) -> dict:
|
|
"""Stage one document and its official editions without touching legacy roots."""
|
|
if not alias or len(alias) > 64 or "/" in alias or "\\" in alias:
|
|
raise ValueError("document code is invalid")
|
|
if max_document_bytes <= 0:
|
|
raise ValueError("document byte budget must be positive")
|
|
output = validate_stage_root(output, resume)
|
|
work_root = output / ".work" / "documents"
|
|
target_root = output / "documents"
|
|
connection = open_manifest(output / "current-api.sqlite3")
|
|
raw_manifest = connect_manifest(output / "manifest.sqlite3")
|
|
try:
|
|
contract_sha = hashlib.sha256(json.dumps(REQUEST_CONTRACT, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
saved_contract = connection.execute("SELECT contract_sha256 FROM generation").fetchone()
|
|
if saved_contract and saved_contract[0] != contract_sha:
|
|
raise SourceChanged("checkpoint request contract changed; start a new staging generation")
|
|
if not saved_contract:
|
|
if connection.execute("SELECT COUNT(*) FROM documents").fetchone()[0]:
|
|
raise SourceChanged("existing checkpoints have no request contract; start a new staging generation")
|
|
with connection:
|
|
connection.execute("INSERT INTO generation (contract_sha256) VALUES (?)", (contract_sha,))
|
|
response = client.document(alias)
|
|
source = response.value
|
|
canonical = source["documentCode"]
|
|
record_sha = response.sha256
|
|
editions = source["editions"]
|
|
previous = connection.execute(
|
|
"SELECT canonical_id, response_sha256, expected_editions FROM documents WHERE alias = ?",
|
|
(alias,),
|
|
).fetchone()
|
|
if previous and previous[1] is not None and (
|
|
previous[0] != canonical or previous[1] != record_sha or previous[2] != len(editions)
|
|
):
|
|
raise SourceChanged("document metadata changed; start a new staging generation")
|
|
working_document = work_root / str(canonical)
|
|
final_document = target_root / str(canonical)
|
|
ensure_stage_path(output, working_document)
|
|
ensure_stage_path(output, final_document)
|
|
if final_document.is_symlink():
|
|
raise ValueError("staging target must not be a symbolic link")
|
|
if final_document.exists() and not previous:
|
|
raise ValueError("staging target already exists without a matching checkpoint")
|
|
if final_document.exists():
|
|
final_metadata = ensure_stage_path(output, final_document / "metadata.json")
|
|
source_document = ensure_stage_path(output, final_document / "source-document.json")
|
|
metadata = json.loads(final_metadata.read_text(encoding="utf-8"))
|
|
if (
|
|
metadata.get("SourceApi", {}).get("response_sha256") != record_sha
|
|
or hashlib.sha256(source_document.read_bytes()).hexdigest() != record_sha
|
|
):
|
|
raise SourceChanged("published staging document does not match the current source record")
|
|
for edition in editions:
|
|
identifier = edition_id(edition)
|
|
for request_language, archive_language, type_field in LANGUAGES:
|
|
html_path = ensure_stage_path(
|
|
output,
|
|
final_document / "editions" / str(identifier) / f"{archive_language}.html",
|
|
)
|
|
checkpoint = connection.execute(
|
|
"SELECT state, content_sha256 FROM languages WHERE canonical_id = ? AND edition_id = ? AND request_language = ?",
|
|
(canonical, identifier, request_language),
|
|
).fetchone()
|
|
if not edition.get(type_field):
|
|
if not checkpoint or checkpoint[0] != "unavailable":
|
|
raise SourceError(f"published staging document has no checkpoint for edition {identifier} {request_language}")
|
|
elif (
|
|
not checkpoint
|
|
or checkpoint[0] != "complete"
|
|
or not html_path.is_file()
|
|
or sha256_file(html_path) != checkpoint[1]
|
|
):
|
|
raise SourceError(f"published staging document failed verification for edition {identifier} {request_language}")
|
|
document_sha = hash_tree(final_document)
|
|
fetched_at = utc_now()
|
|
manifest_row = raw_manifest.execute(
|
|
"SELECT source_sha256 FROM documents WHERE code = ?", (canonical,)
|
|
).fetchone()
|
|
if manifest_row and manifest_row[0] != document_sha:
|
|
raise SourceError("published staging document does not match its raw manifest")
|
|
with raw_manifest:
|
|
raw_manifest.execute(
|
|
"INSERT OR IGNORE INTO documents (code, source_sha256, fetched_at) VALUES (?, ?, ?)",
|
|
(canonical, document_sha, fetched_at),
|
|
)
|
|
raw_manifest.execute(
|
|
"INSERT INTO errors (code, message, failed_at) VALUES (?, ?, ?) "
|
|
"ON CONFLICT(code) DO UPDATE SET message=excluded.message, failed_at=excluded.failed_at",
|
|
(canonical, "attachments and embedded media are not independently verified", fetched_at),
|
|
)
|
|
with connection:
|
|
connection.execute(
|
|
"UPDATE documents SET state='text_complete_incomplete_attachments', error=NULL, updated_at=? WHERE alias=?",
|
|
(fetched_at, alias),
|
|
)
|
|
return {
|
|
"alias": alias,
|
|
"canonical_id": canonical,
|
|
"editions": len(editions),
|
|
"source_sha256": document_sha,
|
|
"state": "text_complete_incomplete_attachments",
|
|
"resumed": True,
|
|
}
|
|
stage_mkdir(output, working_document)
|
|
document_metadata = map_document_metadata(source, alias, response.sha256)
|
|
stage_write(output, working_document / "metadata.json", json_bytes(document_metadata))
|
|
stage_write(output, working_document / "source-document.json", response.body)
|
|
with connection:
|
|
connection.execute(
|
|
"INSERT INTO documents (alias, canonical_id, response_sha256, expected_editions, state, updated_at) VALUES (?, ?, ?, ?, 'fetching', ?) "
|
|
"ON CONFLICT(alias) DO UPDATE SET canonical_id=excluded.canonical_id, response_sha256=excluded.response_sha256, "
|
|
"expected_editions=excluded.expected_editions, state='fetching', error=NULL, updated_at=excluded.updated_at",
|
|
(alias, canonical, record_sha, len(editions), utc_now()),
|
|
)
|
|
for order, edition in enumerate(editions):
|
|
identifier = edition_id(edition)
|
|
existing = connection.execute(
|
|
"SELECT source_order, response_sha256 FROM editions WHERE canonical_id = ? AND edition_id = ?",
|
|
(canonical, identifier),
|
|
).fetchone()
|
|
if existing and existing != (order, hashlib.sha256(json_bytes(edition)).hexdigest()):
|
|
raise SourceChanged(f"edition metadata changed for {identifier}")
|
|
connection.execute(
|
|
"INSERT OR IGNORE INTO editions (canonical_id, edition_id, source_order, response_sha256) VALUES (?, ?, ?, ?)",
|
|
(canonical, identifier, order, hashlib.sha256(json_bytes(edition)).hexdigest()),
|
|
)
|
|
|
|
incomplete = []
|
|
for order, edition in enumerate(editions):
|
|
identifier = edition_id(edition)
|
|
directory = working_document / "editions" / str(identifier)
|
|
stage_mkdir(output, directory)
|
|
stage_write(output, directory / "metadata.json", json_bytes(map_edition_metadata(edition, order)))
|
|
for request_language, archive_language, type_field in LANGUAGES:
|
|
html_path = ensure_stage_path(output, directory / f"{archive_language}.html")
|
|
checkpoint = connection.execute(
|
|
"SELECT state, content_sha256, byte_count FROM languages WHERE canonical_id = ? AND edition_id = ? AND request_language = ?",
|
|
(canonical, identifier, request_language),
|
|
).fetchone()
|
|
if checkpoint and checkpoint[0] == "complete":
|
|
if not html_path.is_file() or sha256_file(html_path) != checkpoint[1]:
|
|
raise SourceError(f"staged content hash mismatch for edition {identifier} {request_language}")
|
|
continue
|
|
if not edition.get(type_field):
|
|
save_language_checkpoint(
|
|
connection, canonical, identifier, request_language, archive_language,
|
|
None, None, 0, "unavailable", "source metadata reports no language file",
|
|
)
|
|
continue
|
|
try:
|
|
language_response = client.edition(identifier, request_language)
|
|
response_id = language_response.value.get("id")
|
|
if isinstance(response_id, bool) or not isinstance(response_id, int) or response_id != identifier:
|
|
raise SourceChanged(f"MinJust exact edition ID mismatch for {identifier}")
|
|
for field in ("nameRus", "nameKyr"):
|
|
if edition.get(field) is not None and language_response.value.get(field) != edition[field]:
|
|
raise SourceChanged(f"MinJust exact edition metadata mismatch for {identifier}: {field}")
|
|
content = language_response.value.get("contentRu")
|
|
if not isinstance(content, str) or not content:
|
|
raise SourceError(f"MinJust returned no content for edition {identifier} language {request_language}")
|
|
encoded = content.encode("utf-8")
|
|
completed_bytes = connection.execute(
|
|
"SELECT COALESCE(SUM(byte_count), 0) FROM languages WHERE canonical_id = ? AND state = 'complete'",
|
|
(canonical,),
|
|
).fetchone()[0]
|
|
if completed_bytes + len(encoded) > max_document_bytes:
|
|
raise SourceError(f"document {canonical} exceeds its {max_document_bytes}-byte staging budget")
|
|
response_metadata = {
|
|
key: value
|
|
for key, value in language_response.value.items()
|
|
if key not in {"contentRu", "contentKg"}
|
|
}
|
|
response_metadata["requested_language"] = request_language
|
|
response_metadata["archive_language"] = archive_language
|
|
response_metadata["requested_edition_id"] = identifier
|
|
response_metadata["exact"] = True
|
|
response_metadata["response_sha256"] = language_response.sha256
|
|
stage_write(
|
|
output,
|
|
directory / "source" / f"{request_language}.json",
|
|
json_bytes(response_metadata),
|
|
)
|
|
stage_write(output, html_path, encoded)
|
|
save_language_checkpoint(
|
|
connection, canonical, identifier, request_language, archive_language,
|
|
language_response.sha256, hashlib.sha256(encoded).hexdigest(), len(encoded), "complete", None,
|
|
)
|
|
except Exception as error:
|
|
message = str(error)[:500]
|
|
save_language_checkpoint(
|
|
connection, canonical, identifier, request_language, archive_language,
|
|
None, None, 0, "error", message,
|
|
)
|
|
incomplete.append({"edition_id": identifier, "language": request_language, "error": message})
|
|
if isinstance(error, SourceChanged) or (isinstance(error, SourceError) and "budget" in message):
|
|
raise
|
|
|
|
if incomplete:
|
|
with connection:
|
|
connection.execute(
|
|
"UPDATE documents SET state='incomplete', error=?, updated_at=? WHERE alias=?",
|
|
(json.dumps(incomplete, ensure_ascii=False), utc_now(), alias),
|
|
)
|
|
return {"alias": alias, "canonical_id": canonical, "editions": len(editions), "incomplete": incomplete}
|
|
|
|
working_metadata = ensure_stage_path(output, working_document / "metadata.json")
|
|
metadata = json.loads(working_metadata.read_text(encoding="utf-8"))
|
|
metadata["SourceCompleteness"] = {
|
|
"text": "complete",
|
|
"attachments": "not independently verified",
|
|
}
|
|
stage_write(output, working_metadata, json_bytes(metadata))
|
|
stage_mkdir(output, target_root)
|
|
ensure_stage_path(output, working_document)
|
|
ensure_stage_path(output, final_document)
|
|
if final_document.exists():
|
|
raise ValueError("refusing to replace a document in this staging generation")
|
|
os.replace(working_document, final_document)
|
|
validate_stage_tree(output)
|
|
document_sha = hash_tree(final_document)
|
|
fetched_at = utc_now()
|
|
with raw_manifest:
|
|
raw_manifest.execute(
|
|
"INSERT INTO documents (code, source_sha256, fetched_at) VALUES (?, ?, ?)",
|
|
(canonical, document_sha, fetched_at),
|
|
)
|
|
raw_manifest.execute(
|
|
"INSERT INTO errors (code, message, failed_at) VALUES (?, ?, ?)",
|
|
(canonical, "attachments and embedded media are not independently verified", fetched_at),
|
|
)
|
|
with connection:
|
|
connection.execute(
|
|
"UPDATE documents SET state='text_complete_incomplete_attachments', updated_at=? WHERE alias=?",
|
|
(fetched_at, alias),
|
|
)
|
|
return {
|
|
"alias": alias,
|
|
"canonical_id": canonical,
|
|
"editions": len(editions),
|
|
"source_sha256": document_sha,
|
|
"state": "text_complete_incomplete_attachments",
|
|
}
|
|
except Exception as error:
|
|
with connection:
|
|
connection.execute(
|
|
"INSERT INTO documents (alias, state, error, updated_at) VALUES (?, 'error', ?, ?) "
|
|
"ON CONFLICT(alias) DO UPDATE SET state='error', error=excluded.error, updated_at=excluded.updated_at",
|
|
(alias, str(error)[:500], utc_now()),
|
|
)
|
|
raise
|
|
finally:
|
|
raw_manifest.close()
|
|
connection.close()
|
|
|
|
|
|
def map_document_metadata(source: dict, alias: str, response_sha256: str) -> dict:
|
|
canonical = source["documentCode"]
|
|
return {
|
|
"Code": canonical,
|
|
"Name": {"Rus": source.get("nameRus"), "Kyr": source.get("nameKyr")},
|
|
"Title": {"Rus": source.get("headerRu"), "Kyr": source.get("headerKg")},
|
|
"Number": source.get("number"),
|
|
"DateAdopted": source.get("dateAdopted"),
|
|
"DateInclusion": source.get("dateInclusion"),
|
|
"DateOfEntry": source.get("dateOfEntry"),
|
|
"DatePublication": source.get("datePublication"),
|
|
"DateRegistration": source.get("dateRegistration"),
|
|
"IsPublicInRegister": source.get("isPublicInRegister"),
|
|
"SourceStatus": source.get("status"),
|
|
"SourceClassId": source.get("classId"),
|
|
"SourceRefTypeId": source.get("refTypeId"),
|
|
"SourceAuthorityIds": source.get("authoritiesId"),
|
|
"SourcePublicationIds": source.get("sourcePublicationsId"),
|
|
"SourceMetadata": {key: value for key, value in source.items() if key != "editions"},
|
|
"SourceDocumentCode": alias,
|
|
"SourceMediaStatus": "not independently verified",
|
|
"SourceApi": {"endpoint": "/GetDocument", "response_sha256": response_sha256},
|
|
}
|
|
|
|
|
|
def map_edition_metadata(source: dict, source_order: int) -> dict:
|
|
return {
|
|
"Code": edition_id(source),
|
|
"Name": {"Rus": source.get("nameRus"), "Kyr": source.get("nameKyr")},
|
|
"SourceOrder": source_order,
|
|
"SourceTextTypes": {
|
|
"ru": source.get("textRusType"),
|
|
"ky": source.get("textKyrType"),
|
|
},
|
|
"SourceMediaStatus": "not independently verified",
|
|
"SourceMetadata": source,
|
|
}
|
|
|
|
|
|
def save_language_checkpoint(
|
|
connection: sqlite3.Connection,
|
|
canonical: int,
|
|
edition: int,
|
|
request_language: str,
|
|
archive_language: str,
|
|
response_sha256: str | None,
|
|
content_sha256: str | None,
|
|
byte_count: int,
|
|
state: str,
|
|
error: str | None,
|
|
) -> None:
|
|
with connection:
|
|
connection.execute(
|
|
"INSERT INTO languages (canonical_id, edition_id, request_language, archive_language, response_sha256, content_sha256, byte_count, state, error, updated_at) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(canonical_id, edition_id, request_language) DO UPDATE SET "
|
|
"archive_language=excluded.archive_language, response_sha256=excluded.response_sha256, content_sha256=excluded.content_sha256, "
|
|
"byte_count=excluded.byte_count, state=excluded.state, error=excluded.error, updated_at=excluded.updated_at",
|
|
(canonical, edition, request_language, archive_language, response_sha256, content_sha256, byte_count, state, error, utc_now()),
|
|
)
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def hash_tree(root: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
|
digest.update(path.relative_to(root).as_posix().encode())
|
|
digest.update(b"\0")
|
|
digest.update(sha256_file(path).encode("ascii"))
|
|
digest.update(b"\0")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", type=Path, required=True, help="new, separate staging generation")
|
|
parser.add_argument("--document-code", required=True, help="one official document alias or canonical code")
|
|
parser.add_argument("--resume", action="store_true", help="resume only this staging generation")
|
|
parser.add_argument("--requests-per-second", type=float, default=1)
|
|
parser.add_argument("--timeout", type=int, default=60)
|
|
parser.add_argument("--retries", type=int, default=5)
|
|
parser.add_argument("--max-document-bytes", type=int, default=DEFAULT_DOCUMENT_BUDGET)
|
|
parser.add_argument("--version", action="version", version=APP_VERSION)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
arguments = parse_args()
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
result = stage_document(
|
|
arguments.output,
|
|
arguments.document_code,
|
|
CurrentCbdClient(arguments.requests_per_second, arguments.timeout, arguments.retries),
|
|
resume=arguments.resume,
|
|
max_document_bytes=arguments.max_document_bytes,
|
|
)
|
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
return int(result.get("state") != "text_complete_incomplete_attachments")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|