diff --git a/backend/test_search_topic_taxonomy.py b/backend/test_search_topic_taxonomy.py index 5712dae..5c01810 100644 --- a/backend/test_search_topic_taxonomy.py +++ b/backend/test_search_topic_taxonomy.py @@ -66,6 +66,37 @@ class SearchTopicTaxonomyTest(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "normalized document 1"): build(root) + def test_rebuild_rejects_normalized_document_without_classifier_list(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + document = root / "documents/1" + document.mkdir(parents=True) + with closing(sqlite3.connect(root / "manifest.sqlite3")) as db: + db.execute("CREATE TABLE documents (code TEXT, state TEXT)") + db.execute("INSERT INTO documents VALUES ('1', 'success')") + db.commit() + (document / "document.json").write_text("{}", encoding="utf-8") + + with self.assertRaisesRegex(RuntimeError, "general_classifiers list"): + build(root) + + def test_rebuild_rejects_invalid_nested_classifier_branch(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + document = root / "documents/1" + document.mkdir(parents=True) + with closing(sqlite3.connect(root / "manifest.sqlite3")) as db: + db.execute("CREATE TABLE documents (code TEXT, state TEXT)") + db.execute("INSERT INTO documents VALUES ('1', 'success')") + db.commit() + (document / "document.json").write_text( + json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}), + encoding="utf-8", + ) + + with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"): + build(root) + if __name__ == "__main__": unittest.main() diff --git a/tools/build_search_topic_taxonomy.py b/tools/build_search_topic_taxonomy.py index 9aa3073..ad05b7a 100644 --- a/tools/build_search_topic_taxonomy.py +++ b/tools/build_search_topic_taxonomy.py @@ -65,12 +65,14 @@ def node_id(path): return "source-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] -def add_nodes(nodes, classifiers, parent_path, seen): - if not isinstance(classifiers, list): +def add_nodes(nodes, classifiers, parent_path, seen, document_code): + if classifiers is None: return + if not isinstance(classifiers, list): + raise RuntimeError(f"normalized document {document_code} contains a non-list GeneralClassifiers branch") for source in classifiers: if not isinstance(source, dict): - continue + raise RuntimeError(f"normalized document {document_code} contains a non-object GeneralClassifiers node") name = source.get("Name") if isinstance(source.get("Name"), dict) else {} labels = {"ru": clean(name.get("Rus")), "ky": clean(name.get("Kyr"))} code = source.get("Code") @@ -92,7 +94,7 @@ def add_nodes(nodes, classifiers, parent_path, seen): if identifier not in seen: node["document_count"] += 1 seen.add(identifier) - add_nodes(nodes, source.get("GeneralClassifiers"), path, seen) + add_nodes(nodes, source.get("GeneralClassifiers"), path, seen, document_code) def emit_node(node, children): @@ -125,13 +127,16 @@ def build(normalized_root): raise RuntimeError(f"cannot read normalized document {code}: {error}") from error if not isinstance(data, dict): raise RuntimeError(f"normalized document {code} must contain a JSON object") - return code, data.get("general_classifiers", []) + classifiers = data.get("general_classifiers") + if not isinstance(classifiers, list): + raise RuntimeError(f"normalized document {code} must contain a general_classifiers list") + return code, classifiers with ThreadPoolExecutor(max_workers=24) as pool: for offset in range(0, len(codes), 512): for code, classifiers in pool.map(read, codes[offset:offset + 512]): seen = set() - add_nodes(nodes, classifiers, [], seen) + add_nodes(nodes, classifiers, [], seen, code) for source in classifiers if isinstance(classifiers, list) else []: if not isinstance(source, dict): continue