103 lines
4.7 KiB
Python
103 lines
4.7 KiB
Python
import json
|
||
import sqlite3
|
||
import tempfile
|
||
import unittest
|
||
from contextlib import closing
|
||
from pathlib import Path
|
||
|
||
from tools.build_search_topic_taxonomy import build, group_for
|
||
|
||
|
||
class SearchTopicTaxonomyTest(unittest.TestCase):
|
||
def test_ambiguous_roots_are_left_for_manual_review(self):
|
||
self.assertEqual(group_for("Иностранные инвестиции"), "review_required")
|
||
|
||
def test_preserves_source_tree_counts_and_marks_unclassified_roots(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
(root / "documents/1").mkdir(parents=True)
|
||
(root / "documents/2").mkdir(parents=True)
|
||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||
db.executemany("INSERT INTO documents VALUES (?, 'success')", [("1",), ("2",)])
|
||
db.commit()
|
||
classifier = {
|
||
"Code": None,
|
||
"Name": {"Rus": "ЗАКОНОДАТЕЛЬСТВО О ТРУДЕ", "Kyr": None},
|
||
"GeneralClassifiers": [{
|
||
"Code": "child-1",
|
||
"Name": {"Rus": "Рабочее время", "Kyr": "Иш убактысы"},
|
||
"GeneralClassifiers": [],
|
||
}],
|
||
}
|
||
unknown = {
|
||
"Code": None,
|
||
"Name": {"Rus": "НЕИЗВЕСТНАЯ РУБРИКА", "Kyr": None},
|
||
"GeneralClassifiers": [],
|
||
}
|
||
(root / "documents/1/document.json").write_text(
|
||
json.dumps({"general_classifiers": [classifier]}, ensure_ascii=False),
|
||
encoding="utf-8",
|
||
)
|
||
(root / "documents/2/document.json").write_text(
|
||
json.dumps({"general_classifiers": [classifier, unknown]}, ensure_ascii=False),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
catalog = build(root)
|
||
groups = {group["code"]: group for group in catalog["groups"]}
|
||
labor_root = groups["labor_social"]["children"][0]
|
||
|
||
self.assertEqual((catalog["source_document_count"], catalog["source_node_count"], catalog["source_root_count"]), (2, 3, 2))
|
||
self.assertEqual(labor_root["document_count"], 2)
|
||
self.assertEqual(labor_root["children"][0]["document_count"], 2)
|
||
self.assertEqual(groups["review_required"]["children"][0]["grouping_status"], "manual_review")
|
||
self.assertEqual(catalog["nodes_missing_kyrgyz_label"], 2)
|
||
|
||
def test_rebuild_fails_instead_of_omitting_successful_manifest_document(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
(root / "documents/1").mkdir(parents=True)
|
||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||
db.commit()
|
||
|
||
with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
|
||
build(root)
|
||
|
||
def test_rebuild_rejects_normalized_document_without_classifier_list(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
document = root / "documents/1"
|
||
document.mkdir(parents=True)
|
||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||
db.commit()
|
||
(document / "document.json").write_text("{}", encoding="utf-8")
|
||
|
||
with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
|
||
build(root)
|
||
|
||
def test_rebuild_rejects_invalid_nested_classifier_branch(self):
|
||
with tempfile.TemporaryDirectory() as temporary:
|
||
root = Path(temporary)
|
||
document = root / "documents/1"
|
||
document.mkdir(parents=True)
|
||
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
||
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
||
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
||
db.commit()
|
||
(document / "document.json").write_text(
|
||
json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
|
||
build(root)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|