103 lines
4.7 KiB
Python
103 lines
4.7 KiB
Python
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import closing
|
|
from pathlib import Path
|
|
|
|
from tools.build_search_topic_taxonomy import build, group_for
|
|
|
|
|
|
class SearchTopicTaxonomyTest(unittest.TestCase):
|
|
def test_ambiguous_roots_are_left_for_manual_review(self):
|
|
self.assertEqual(group_for("Иностранные инвестиции"), "review_required")
|
|
|
|
def test_preserves_source_tree_counts_and_marks_unclassified_roots(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
(root / "documents/1").mkdir(parents=True)
|
|
(root / "documents/2").mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
|
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
db.executemany("INSERT INTO documents VALUES (?, 'success')", [("1",), ("2",)])
|
|
db.commit()
|
|
classifier = {
|
|
"Code": None,
|
|
"Name": {"Rus": "ЗАКОНОДАТЕЛЬСТВО О ТРУДЕ", "Kyr": None},
|
|
"GeneralClassifiers": [{
|
|
"Code": "child-1",
|
|
"Name": {"Rus": "Рабочее время", "Kyr": "Иш убактысы"},
|
|
"GeneralClassifiers": [],
|
|
}],
|
|
}
|
|
unknown = {
|
|
"Code": None,
|
|
"Name": {"Rus": "НЕИЗВЕСТНАЯ РУБРИКА", "Kyr": None},
|
|
"GeneralClassifiers": [],
|
|
}
|
|
(root / "documents/1/document.json").write_text(
|
|
json.dumps({"general_classifiers": [classifier]}, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
(root / "documents/2/document.json").write_text(
|
|
json.dumps({"general_classifiers": [classifier, unknown]}, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
catalog = build(root)
|
|
groups = {group["code"]: group for group in catalog["groups"]}
|
|
labor_root = groups["labor_social"]["children"][0]
|
|
|
|
self.assertEqual((catalog["source_document_count"], catalog["source_node_count"], catalog["source_root_count"]), (2, 3, 2))
|
|
self.assertEqual(labor_root["document_count"], 2)
|
|
self.assertEqual(labor_root["children"][0]["document_count"], 2)
|
|
self.assertEqual(groups["review_required"]["children"][0]["grouping_status"], "manual_review")
|
|
self.assertEqual(catalog["nodes_missing_kyrgyz_label"], 2)
|
|
|
|
def test_rebuild_fails_instead_of_omitting_successful_manifest_document(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
(root / "documents/1").mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
|
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
|
db.commit()
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "normalized document 1"):
|
|
build(root)
|
|
|
|
def test_rebuild_rejects_normalized_document_without_classifier_list(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
document = root / "documents/1"
|
|
document.mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
|
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
|
db.commit()
|
|
(document / "document.json").write_text("{}", encoding="utf-8")
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "general_classifiers list"):
|
|
build(root)
|
|
|
|
def test_rebuild_rejects_invalid_nested_classifier_branch(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
document = root / "documents/1"
|
|
document.mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "manifest.sqlite3")) as db:
|
|
db.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
db.execute("INSERT INTO documents VALUES ('1', 'success')")
|
|
db.commit()
|
|
(document / "document.json").write_text(
|
|
json.dumps({"general_classifiers": [{"Name": {"Rus": "Корень"}, "GeneralClassifiers": "bad"}]}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "non-list GeneralClassifiers branch"):
|
|
build(root)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|