97 lines
4.7 KiB
Python
97 lines
4.7 KiB
Python
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from search.minjust_opensearch import export_bulk
|
|
|
|
|
|
class MinjustOpenSearchTest(unittest.TestCase):
|
|
def test_exports_atomic_bulk_and_rejects_mismatched_fragment(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
document_root = root / "normalized/documents/7"
|
|
document_root.mkdir(parents=True)
|
|
with sqlite3.connect(root / "normalized/manifest.sqlite3") as connection:
|
|
connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
connection.execute("INSERT INTO documents VALUES ('7', 'success')")
|
|
(document_root / "document.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": "1",
|
|
"source_code": "7",
|
|
"name": {"ru": "Закон", "ky": "Мыйзам"},
|
|
"type": {"ru": "Закон", "ky": "Мыйзам"},
|
|
"status": {"ru": "Действует", "ky": "Күчүндө"},
|
|
"number": "1",
|
|
"dates": {"DateAdopted": "2026-01-01"},
|
|
"authority_paths": [{"ru": ["Кабинет"], "ky": ["Кабинет"]}],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
for language, text in (("ru", "Текст \"RU\"\nстрока"), ("ky", "Кыргызча текст")):
|
|
path = document_root / f"editions/10/{language}/fragments.json"
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(
|
|
json.dumps(
|
|
[{
|
|
"id": f"document:7:edition:10:lang:{language}:fragment:1",
|
|
"document_code": "7",
|
|
"edition_code": "10",
|
|
"language": language,
|
|
"position": 1,
|
|
"type": "paragraph",
|
|
"text": text,
|
|
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"source_path": f"documents/7/editions/10/{language}.html",
|
|
"source_sha256": "b" * 64,
|
|
}],
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
output = root / "bulk.ndjson"
|
|
self.assertEqual(export_bulk(root / "normalized", output, "test-index"), (1, 2))
|
|
content = output.read_bytes()
|
|
self.assertTrue(content.endswith(b"\n"))
|
|
lines = [json.loads(line) for line in content.splitlines()]
|
|
self.assertEqual(len(lines), 4)
|
|
self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1")
|
|
self.assertIn("text_ky", lines[1])
|
|
self.assertNotIn("text_ru", lines[1])
|
|
self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока")
|
|
|
|
bad = document_root / "editions/10/ru/fragments.json"
|
|
fragments = json.loads(bad.read_text(encoding="utf-8"))
|
|
fragments[0]["document_code"] = "8"
|
|
bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "identity"):
|
|
export_bulk(root / "normalized", output, "test-index")
|
|
self.assertEqual(output.read_bytes(), content)
|
|
|
|
fragments[0]["document_code"] = "7"
|
|
fragments[0]["text"] = "Повреждено"
|
|
bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "checksum"):
|
|
export_bulk(root / "normalized", output, "test-index")
|
|
with self.assertRaisesRegex(ValueError, "inside --input"):
|
|
export_bulk(root / "normalized", root / "normalized/manifest.sqlite3", "test-index")
|
|
self.assertEqual(output.read_bytes(), content)
|
|
|
|
mapping = json.loads(
|
|
(Path(__file__).parent / "search/minjust-fragments-index.json").read_text(encoding="utf-8")
|
|
)["mappings"]
|
|
self.assertEqual(mapping["dynamic"], "strict")
|
|
self.assertEqual(mapping["properties"]["position"]["type"], "integer")
|
|
self.assertEqual(mapping["properties"]["text_ru"]["analyzer"], "russian")
|
|
self.assertEqual(mapping["properties"]["text_ky"]["analyzer"], "icu_analyzer")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|