fix(search): validate topic hierarchy input

This commit is contained in:
2026-09-14 10:36:08 +03:00
parent 5f44981bd3
commit f8f98dfe46
2 changed files with 42 additions and 6 deletions

View File

@@ -66,6 +66,37 @@ class SearchTopicTaxonomyTest(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
build(root)
def test_rebuild_rejects_normalized_document_without_classifier_list(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
document = root / "documents/1"
document.mkdir(parents=True)
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
db.execute("INSERT INTO documents VALUES ('1', 'success')")
db.commit()
(document / "document.json").write_text("{}", encoding="utf-8")
with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
build(root)
def test_rebuild_rejects_invalid_nested_classifier_branch(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
document = root / "documents/1"
document.mkdir(parents=True)
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
db.execute("INSERT INTO documents VALUES ('1', 'success')")
db.commit()
(document / "document.json").write_text(
json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
encoding="utf-8",
)
with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
build(root)
if __name__ == "__main__":
unittest.main()

View File

@@ -65,12 +65,14 @@ def node_id(path):
return "source-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
def add_nodes(nodes, classifiers, parent_path, seen):
if not isinstance(classifiers, list):
def add_nodes(nodes, classifiers, parent_path, seen, document_code):
if classifiers is None:
return
if not isinstance(classifiers, list):
raise RuntimeError(f"normalized document {document_code} contains a non-list GeneralClassifiers branch")
for source in classifiers:
if not isinstance(source, dict):
continue
raise RuntimeError(f"normalized document {document_code} contains a non-object GeneralClassifiers node")
name = source.get("Name") if isinstance(source.get("Name"), dict) else {}
labels = {"ru": clean(name.get("Rus")), "ky": clean(name.get("Kyr"))}
code = source.get("Code")
@@ -92,7 +94,7 @@ def add_nodes(nodes, classifiers, parent_path, seen):
if identifier not in seen:
node["document_count"] += 1
seen.add(identifier)
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen)
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen, document_code)
def emit_node(node, children):
@@ -125,13 +127,16 @@ def build(normalized_root):
raise RuntimeError(f"cannot read normalized document {code}: {error}") from error
if not isinstance(data, dict):
raise RuntimeError(f"normalized document {code} must contain a JSON object")
return code, data.get("general_classifiers", [])
classifiers = data.get("general_classifiers")
if not isinstance(classifiers, list):
raise RuntimeError(f"normalized document {code} must contain a general_classifiers list")
return code, classifiers
with ThreadPoolExecutor(max_workers=24) as pool:
for offset in range(0, len(codes), 512):
for code, classifiers in pool.map(read, codes[offset:offset + 512]):
seen = set()
add_nodes(nodes, classifiers, [], seen)
add_nodes(nodes, classifiers, [], seen, code)
for source in classifiers if isinstance(classifiers, list) else []:
if not isinstance(source, dict):
continue