fix(search): validate topic hierarchy input
This commit is contained in:
@@ -66,6 +66,37 @@ class SearchTopicTaxonomyTest(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
|
||||
build(root)
|
||||
|
||||
def test_rebuild_rejects_normalized_document_without_classifier_list(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
document = root / "documents/1"
|
||||
document.mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||||
db.commit()
|
||||
(document / "document.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
|
||||
build(root)
|
||||
|
||||
def test_rebuild_rejects_invalid_nested_classifier_branch(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
document = root / "documents/1"
|
||||
document.mkdir(parents=True)
|
||||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||||
db.commit()
|
||||
(document / "document.json").write_text(
|
||||
json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
|
||||
build(root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -65,12 +65,14 @@ def node_id(path):
|
||||
return "source-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def add_nodes(nodes, classifiers, parent_path, seen):
|
||||
if not isinstance(classifiers, list):
|
||||
def add_nodes(nodes, classifiers, parent_path, seen, document_code):
|
||||
if classifiers is None:
|
||||
return
|
||||
if not isinstance(classifiers, list):
|
||||
raise RuntimeError(f"normalized document {document_code} contains a non-list GeneralClassifiers branch")
|
||||
for source in classifiers:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
raise RuntimeError(f"normalized document {document_code} contains a non-object GeneralClassifiers node")
|
||||
name = source.get("Name") if isinstance(source.get("Name"), dict) else {}
|
||||
labels = {"ru": clean(name.get("Rus")), "ky": clean(name.get("Kyr"))}
|
||||
code = source.get("Code")
|
||||
@@ -92,7 +94,7 @@ def add_nodes(nodes, classifiers, parent_path, seen):
|
||||
if identifier not in seen:
|
||||
node["document_count"] += 1
|
||||
seen.add(identifier)
|
||||
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen)
|
||||
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen, document_code)
|
||||
|
||||
|
||||
def emit_node(node, children):
|
||||
@@ -125,13 +127,16 @@ def build(normalized_root):
|
||||
raise RuntimeError(f"cannot read normalized document {code}: {error}") from error
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError(f"normalized document {code} must contain a JSON object")
|
||||
return code, data.get("general_classifiers", [])
|
||||
classifiers = data.get("general_classifiers")
|
||||
if not isinstance(classifiers, list):
|
||||
raise RuntimeError(f"normalized document {code} must contain a general_classifiers list")
|
||||
return code, classifiers
|
||||
|
||||
with ThreadPoolExecutor(max_workers=24) as pool:
|
||||
for offset in range(0, len(codes), 512):
|
||||
for code, classifiers in pool.map(read, codes[offset:offset + 512]):
|
||||
seen = set()
|
||||
add_nodes(nodes, classifiers, [], seen)
|
||||
add_nodes(nodes, classifiers, [], seen, code)
|
||||
for source in classifiers if isinstance(classifiers, list) else []:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user