agent/akyldash
1
1
Fork 0
Code Issues 4 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(search): validate topic hierarchy input

Browse Source
This commit is contained in:
Codex Agent
2026-09-14 10:36:08 +03:00
parent 5f44981bd3
commit f8f98dfe46
2 changed files with 42 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
tools/build_search_topic_taxonomy.py
View File

@@ -65,12 +65,14 @@ def node_id(path):
return "source-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
def add_nodes(nodes, classifiers, parent_path, seen):
if not isinstance(classifiers, list):
def add_nodes(nodes, classifiers, parent_path, seen, document_code):
if classifiers is None:
return
if not isinstance(classifiers, list):
raise RuntimeError(f"normalized document {document_code} contains a non-list GeneralClassifiers branch")
for source in classifiers:
if not isinstance(source, dict):
continue
raise RuntimeError(f"normalized document {document_code} contains a non-object GeneralClassifiers node")
name = source.get("Name") if isinstance(source.get("Name"), dict) else {}
labels = {"ru": clean(name.get("Rus")), "ky": clean(name.get("Kyr"))}
code = source.get("Code")
@@ -92,7 +94,7 @@ def add_nodes(nodes, classifiers, parent_path, seen):
if identifier not in seen:
node["document_count"] += 1
seen.add(identifier)
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen)
add_nodes(nodes, source.get("GeneralClassifiers"), path, seen, document_code)
def emit_node(node, children):
@@ -125,13 +127,16 @@ def build(normalized_root):
raise RuntimeError(f"cannot read normalized document {code}: {error}") from error
if not isinstance(data, dict):
raise RuntimeError(f"normalized document {code} must contain a JSON object")
return code, data.get("general_classifiers", [])
classifiers = data.get("general_classifiers")
if not isinstance(classifiers, list):
raise RuntimeError(f"normalized document {code} must contain a general_classifiers list")
return code, classifiers
with ThreadPoolExecutor(max_workers=24) as pool:
for offset in range(0, len(codes), 512):
for code, classifiers in pool.map(read, codes[offset:offset + 512]):
seen = set()
add_nodes(nodes, classifiers, [], seen)
add_nodes(nodes, classifiers, [], seen, code)
for source in classifiers if isinstance(classifiers, list) else []:
if not isinstance(source, dict):
continue