fix: harden document normalization
Reject overlapping source and destination roots before any write so normalization cannot replace the raw Ministry archive. Recover an interrupted directory publication before resume checks and require the normalized target to exist before skipping a manifest success. Treat malformed link and image URLs as unsafe attributes, add regressions for all review findings, and bump the backend version to 0.2.2.
This commit is contained in:
@@ -23,7 +23,7 @@ from pathlib import Path
|
||||
from typing import Callable
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
APP_VERSION = "0.2.1"
|
||||
APP_VERSION = "0.2.2"
|
||||
SCHEMA_VERSION = "1"
|
||||
NORMALIZER_VERSION = "1.0.0"
|
||||
LANGUAGES = ("ru", "ky")
|
||||
@@ -234,7 +234,10 @@ class SafeHtmlParser(HTMLParser):
|
||||
return safe
|
||||
|
||||
def safe_image(self, value: str) -> bool:
|
||||
parsed = urlsplit(value)
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
except ValueError:
|
||||
return False
|
||||
if parsed.scheme or parsed.netloc or not parsed.path or parsed.path.startswith(("/", "\\")):
|
||||
return False
|
||||
candidate = (self.edition_directory / parsed.path.replace("\\", "/")).resolve()
|
||||
@@ -249,7 +252,10 @@ def safe_link(value: str) -> bool:
|
||||
value = value.strip()
|
||||
if not value or value.startswith(("//", "\\\\")):
|
||||
return False
|
||||
parsed = urlsplit(value)
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return parsed.scheme.lower() in {"", "http", "https", "mailto"} and not (
|
||||
not parsed.scheme and parsed.netloc
|
||||
)
|
||||
@@ -531,6 +537,23 @@ def publish_directory(staged: Path, target: Path) -> None:
|
||||
shutil.rmtree(backup)
|
||||
|
||||
|
||||
def recover_directory(target: Path) -> None:
|
||||
backup = target.parent / f".{target.name}.previous"
|
||||
if not backup.exists():
|
||||
return
|
||||
if target.exists():
|
||||
shutil.rmtree(backup)
|
||||
else:
|
||||
os.replace(backup, target)
|
||||
|
||||
|
||||
def validate_roots(input_root: Path, output: Path) -> None:
|
||||
source = input_root.resolve()
|
||||
destination = output.resolve()
|
||||
if source == destination or source.is_relative_to(destination) or destination.is_relative_to(source):
|
||||
raise ValueError("--input and --output must not overlap")
|
||||
|
||||
|
||||
def normalize_archive(
|
||||
input_root: Path = Path("data/minjust-cbd"),
|
||||
output: Path = Path("data/minjust-normalized"),
|
||||
@@ -538,6 +561,7 @@ def normalize_archive(
|
||||
refresh: bool = False,
|
||||
progress: Callable[[NormalizeResult, int], None] | None = None,
|
||||
) -> NormalizeResult:
|
||||
validate_roots(input_root, output)
|
||||
document_root = input_root / "documents"
|
||||
if not document_root.is_dir():
|
||||
raise FileNotFoundError(f"Document directory not found: {document_root}")
|
||||
@@ -563,10 +587,12 @@ def normalize_archive(
|
||||
for source_directory in directories:
|
||||
discovered += 1
|
||||
code = source_directory.name
|
||||
target = output / "documents" / code
|
||||
checksum = None
|
||||
try:
|
||||
recover_directory(target)
|
||||
files, checksum = source_inventory(source_directory, input_root)
|
||||
if not refresh and known.get(code) == (
|
||||
if target.is_dir() and not refresh and known.get(code) == (
|
||||
checksum, SCHEMA_VERSION, NORMALIZER_VERSION, "success"
|
||||
):
|
||||
skipped += 1
|
||||
@@ -574,7 +600,7 @@ def normalize_archive(
|
||||
with tempfile.TemporaryDirectory(dir=staging_root) as temporary:
|
||||
staged = Path(temporary) / code
|
||||
normalize_document(source_directory, input_root, staged, files, checksum)
|
||||
publish_directory(staged, output / "documents" / code)
|
||||
publish_directory(staged, target)
|
||||
with connection:
|
||||
connection.execute(
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user