import hashlib import io import json import sqlite3 import tempfile import unittest import urllib.error from contextlib import closing from pathlib import Path from unittest.mock import patch from search.catalog import legal_force_code, labels, source_code from search.minjust_opensearch import bulk_batches, document_codes, export_bulk, load_bulk, request_json, switch_alias class MinjustOpenSearchTest(unittest.TestCase): def test_unknown_document_type_keeps_source_label_and_has_no_inferred_force(self): value = {"ru": "Решение", "ky": "Решение"} self.assertEqual(source_code("document_type", value), "other") self.assertEqual(labels("document_type", "other"), {"ru": "Прочие документы", "ky": "Башка документтер"}) self.assertIsNone(legal_force_code(value)) self.assertEqual(source_code("status", {"ru": "Не вступил в силу", "ky": "Күчүнө кире элек"}), "not_yet_effective") self.assertEqual(legal_force_code({"ru": "Конституционный закон", "ky": "Конституционный закон"}), "constitutional") self.assertEqual(legal_force_code({"ru": "Указ", "ky": "Жарлыгы"}), "subordinate") self.assertEqual(source_code("document_type", {"ru": "устав"}), "charter") def test_export_marks_last_official_edition_current_even_when_ids_are_out_of_order(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) input_root = root / "normalized" document_root = input_root / "documents/6" document_root.mkdir(parents=True) with closing(sqlite3.connect(input_root / "manifest.sqlite3")) as connection: with connection: connection.execute("CREATE TABLE documents (code TEXT, state TEXT)") connection.execute("INSERT INTO documents VALUES ('6', 'success')") (document_root / "document.json").write_text( json.dumps({"schema_version": "1", "source_code": "6", "editions": [ {"source_code": "920492", "source_order": 0}, {"source_code": "920292", "source_order": 1}, ]}), encoding="utf-8", ) for edition in ("920292", "920492"): text = f"Текст {edition}" fragment_path = document_root / f"editions/{edition}/ru/fragments.json" fragment_path.parent.mkdir(parents=True) fragment_path.write_text(json.dumps([{ "id": f"document:6:edition:{edition}:lang:ru:fragment:1", "document_code": "6", "edition_code": edition, "language": "ru", "position": 1, "type": "paragraph", "text": text, "text_sha256": hashlib.sha256(text.encode()).hexdigest(), "source_path": f"documents/6/editions/{edition}/ru.html", "source_sha256": "b" * 64, }]), encoding="utf-8") output = root / "bulk.ndjson" self.assertEqual(export_bulk(input_root, output, "test-index"), (1, 2)) lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] flags = { lines[index]["index"]["_id"].split(":")[3]: lines[index + 1]["is_current_edition"] for index in range(0, len(lines), 2) } self.assertEqual(flags, {"920292": True, "920492": False}) def test_switches_alias_atomically_after_successful_load(self): with patch("search.minjust_opensearch.request_json", side_effect=[ {"akyldash-fragments-v1": {"aliases": {"akyldash-fragments-current": {}}}}, {"acknowledged": True}, ]) as request: switch_alias( "http://127.0.0.1:9200/", "akyldash-fragments-v2", "akyldash-fragments-current", "akyldash-fragments-v1", ) self.assertEqual(request.call_args_list[1].args[:2], ("http://127.0.0.1:9200/_aliases", "POST")) body = json.loads(request.call_args_list[1].args[2]) self.assertEqual(body["actions"][0], {"remove": {"index": "akyldash-fragments-v1", "alias": "akyldash-fragments-current", "must_exist": True}}) self.assertEqual(body["actions"][1], {"add": {"index": "akyldash-fragments-v2", "alias": "akyldash-fragments-current"}}) with self.assertRaisesRegex(ValueError, "differ"): switch_alias("http://127.0.0.1:9200", "same", "same", "old") with self.assertRaisesRegex(ValueError, "differ"): switch_alias("http://127.0.0.1:9200", "index", "", "old") with patch("search.minjust_opensearch.request_json", return_value={ "akyldash-fragments-v3": {"aliases": {"alias": {}}} }) as request: with self.assertRaisesRegex(ValueError, "currently targets"): switch_alias("http://127.0.0.1:9200", "new", "alias", "akyldash-fragments-v1") request.assert_called_once() with patch("search.minjust_opensearch.request_json", side_effect=[ {"old": {"aliases": {"alias": {}}}}, {"acknowledged": False}, {"old": {"aliases": {"alias": {}}}}, ]): with self.assertRaisesRegex(RuntimeError, "did not acknowledge"): switch_alias("http://127.0.0.1:9200", "index", "alias", "old") def test_switch_alias_recovers_when_post_succeeds_but_response_is_lost(self): with patch("search.minjust_opensearch.request_json", side_effect=[ {"old": {"aliases": {"alias": {}}}}, RuntimeError("response lost after OpenSearch applied request"), {"candidate": {"aliases": {"alias": {}}}}, ]) as request: switch_alias("http://os:9200", "candidate", "alias", "old") self.assertEqual(request.call_count, 3) self.assertEqual(request.call_args_list[1].args[1], "POST") self.assertEqual(request.call_args_list[2].args[1], "GET") def test_switch_alias_is_idempotent_after_process_stops_before_checkpoint(self): with patch("search.minjust_opensearch.request_json", return_value={ "candidate": {"aliases": {"alias": {}}} }) as request: switch_alias("http://os:9200", "candidate", "alias", "old") request.assert_called_once() def test_switch_alias_refuses_to_retry_after_unexpected_target(self): with patch("search.minjust_opensearch.request_json", side_effect=[ {"old": {"aliases": {"alias": {}}}}, RuntimeError("response lost"), {"other": {"aliases": {"alias": {}}}}, ]): with self.assertRaisesRegex(ValueError, "unexpected index"): switch_alias("http://os:9200", "candidate", "alias", "old") def test_resume_completes_checkpoint_when_alias_switched_before_previous_process_stopped(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) input_root = root / "normalized" document_root = input_root / "documents/1" document_root.mkdir(parents=True) with closing(sqlite3.connect(input_root / "manifest.sqlite3")) as connection: with connection: connection.execute("CREATE TABLE documents (code TEXT, state TEXT)") connection.execute("INSERT INTO documents VALUES ('1', 'success')") (document_root / "document.json").write_text( json.dumps({"schema_version": "1", "source_code": "1", "editions": [ {"source_code": "1", "source_order": 0}, ]}), encoding="utf-8", ) text = "текст закона" fragment = document_root / "editions/1/ru/fragments.json" fragment.parent.mkdir(parents=True) fragment.write_text(json.dumps([{ "id": "document:1:edition:1:lang:ru:fragment:1", "document_code": "1", "edition_code": "1", "language": "ru", "position": 1, "type": "paragraph", "text": text, "text_sha256": hashlib.sha256(text.encode()).hexdigest(), "source_path": "documents/1/editions/1/ru.html", "source_sha256": "b" * 64, }]), encoding="utf-8") checkpoint = root / "checkpoint.json" state = { "schema_version": 2, "url": "http://os:9200", "cluster_uuid": "cluster", "index": "candidate", "index_uuid": "index-uuid", "input": str(input_root.resolve()), "manifest_sha256": hashlib.sha256((input_root / "manifest.sqlite3").read_bytes()).hexdigest(), "limit": None, "alias": "current", "last_document_code": "1", "complete": False, } checkpoint.write_text(json.dumps(state), encoding="utf-8") with ( patch("search.minjust_opensearch.request_json", side_effect=[ {"cluster_uuid": "cluster"}, {"candidate": {"settings": {"index": {"uuid": "index-uuid"}}}}, {"errors": False, "items": [{"index": {}}]}, {"candidate": {"aliases": {"current": {}}}}, ]) as request, patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, ): self.assertEqual( load_bulk( input_root, "http://os:9200", "candidate", resume=True, checkpoint=checkpoint, alias="current", expected_old_index="previous", canary_ru="закон", canary_ky="мыйзам", ), (1, 1), ) self.assertTrue(json.loads(checkpoint.read_text(encoding="utf-8"))["complete"]) self.assertEqual(request.call_args_list[-1].args[1], "GET") validate_candidate.assert_called_once_with("http://os:9200", "candidate", 1, "закон", "мыйзам") def test_zero_fragment_candidate_never_reaches_alias_switch(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / "normalized" document = root / "documents/1" document.mkdir(parents=True) with closing(sqlite3.connect(root / "manifest.sqlite3")) as connection: connection.execute("CREATE TABLE documents (code TEXT, state TEXT)") connection.execute("INSERT INTO documents VALUES ('1', 'success')") (document / "document.json").write_text( json.dumps({"source_code": "1", "editions": []}), encoding="utf-8" ) with ( patch("search.minjust_opensearch.request_json", side_effect=[ {}, {"cluster_uuid": "cluster"}, {"test-index": {"settings": {"index": {"uuid": "index"}}}}, ]) as request, patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, patch("search.minjust_opensearch.switch_alias") as switch, ): with self.assertRaisesRegex(ValueError, "empty expected fragment inventory"): load_bulk( root, "http://127.0.0.1:9200", "test-index", checkpoint=Path(temporary) / "checkpoint.json", alias="current", expected_old_index="old-index", canary_ru="закон", canary_ky="мыйзам", ) self.assertEqual(request.call_count, 3) validate_candidate.assert_not_called() switch.assert_not_called() def test_candidate_index_requires_exact_count_and_both_language_canaries(self): from search.minjust_opensearch import validate_candidate_index with patch("search.minjust_opensearch.request_json", side_effect=[ {"_shards": {"failed": 0}}, {"count": 2}, {"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}}, {"hits": {"hits": [{"_source": {"text_ky": "сыноо"}}]}}, ]) as request: validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") self.assertEqual(request.call_count, 4) with patch("search.minjust_opensearch.request_json", side_effect=[ {"_shards": {"failed": 0}}, {"count": 1} ]): with self.assertRaisesRegex(RuntimeError, "expected 2"): validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") with patch("search.minjust_opensearch.request_json", side_effect=[ {"_shards": {"failed": 0}}, {"count": 2}, {"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}}, {"hits": {"hits": []}}, ]): with self.assertRaisesRegex(RuntimeError, "ky canary"): validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо") def test_exports_atomic_bulk_and_rejects_mismatched_fragment(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) document_root = root / "normalized/documents/7" document_root.mkdir(parents=True) with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection: with connection: connection.execute("CREATE TABLE documents (code TEXT, state TEXT)") connection.execute("INSERT INTO documents VALUES ('7', 'success')") connection.execute("INSERT INTO documents VALUES ('8', 'success')") (document_root / "document.json").write_text( json.dumps( { "schema_version": "1", "source_code": "7", "name": {"ru": "Закон", "ky": "Мыйзам"}, "type": {"ru": "Закон", "ky": "Мыйзам"}, "status": {"ru": "Действует", "ky": "Күчүндө"}, "number": "1", "dates": {"DateAdopted": "2026-01-01"}, "authority_paths": [{"ru": ["Кабинет"], "ky": ["Кабинет"]}], }, ensure_ascii=False, ), encoding="utf-8", ) empty = root / "normalized/documents/8" empty.mkdir() (empty / "document.json").write_text( json.dumps({"schema_version": "1", "source_code": "8"}), encoding="utf-8" ) for language, text in (("ru", "Текст \"RU\"\nстрока"), ("ky", "Кыргызча текст")): path = document_root / f"editions/10/{language}/fragments.json" path.parent.mkdir(parents=True) path.write_text( json.dumps( [{ "id": f"document:7:edition:10:lang:{language}:fragment:1", "document_code": "7", "edition_code": "10", "language": language, "position": 1, "type": "paragraph", "text": text, "text_sha256": hashlib.sha256(text.encode()).hexdigest(), "source_path": f"documents/7/editions/10/{language}.html", "source_sha256": "b" * 64, }], ensure_ascii=False, ), encoding="utf-8", ) output = root / "bulk.ndjson" self.assertEqual(export_bulk(root / "normalized", output, "test-index"), (2, 2)) content = output.read_bytes() self.assertTrue(content.endswith(b"\n")) lines = [json.loads(line) for line in content.splitlines()] self.assertEqual(len(lines), 4) self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1") self.assertIn("text_ky", lines[1]) self.assertTrue(lines[1]["is_current_edition"]) self.assertNotIn("text_ru", lines[1]) self.assertEqual(lines[1]["legal_force_code"], "legislative") self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока") self.assertEqual( list(bulk_batches(iter((("7", b"a\nb\n"), ("8", b"c\nd\n"))), 4)), [("7", b"a\nb\n"), ("8", b"c\nd\n")], ) self.assertEqual(document_codes(root / "normalized", 2, "8"), ["8"]) with self.assertRaisesRegex(ValueError, "selected range"): document_codes(root / "normalized", 1, "8") checkpoint = root / "checkpoint.json" with ( patch("search.minjust_opensearch.request_json") as request, patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, patch("search.minjust_opensearch.switch_alias") as switch, ): request.side_effect = [ {}, {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, {"errors": False, "items": [{"index": {}}, {"index": {}}]}, ] self.assertEqual( load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ), (2, 2), ) self.assertEqual(request.call_args_list[-1].args[3], "application/x-ndjson") validate_candidate.assert_called_once_with( "http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам" ) switch.assert_called_once_with( "http://127.0.0.1:9200", "test-index", "test-current", "test-index-old" ) state = json.loads(checkpoint.read_text(encoding="utf-8")) self.assertEqual(state["last_document_code"], "7") self.assertTrue(state["complete"]) legacy = state.copy() legacy.pop("alias") legacy["schema_version"] = 1 legacy["last_document_code"] = "8" legacy["complete"] = False checkpoint.write_text(json.dumps(legacy), encoding="utf-8") with ( patch("search.minjust_opensearch.request_json") as request, patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, patch("search.minjust_opensearch.switch_alias"), ): request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] self.assertEqual( load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, ), (1, 0), ) self.assertEqual(json.loads(checkpoint.read_text(encoding="utf-8"))["schema_version"], 2) state["last_document_code"] = "8" state["complete"] = False checkpoint.write_text(json.dumps(state), encoding="utf-8") with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ {"cluster_uuid": "cluster-2"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] with self.assertRaisesRegex(ValueError, "does not match"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ) with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] with self.assertRaisesRegex(ValueError, "alias"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, ) with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] with self.assertRaisesRegex(ValueError, "limit"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, limit=1, resume=True, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ) with ( patch("search.minjust_opensearch.request_json") as request, patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate, patch("search.minjust_opensearch.switch_alias"), ): request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] self.assertEqual( load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ), (1, 0), ) self.assertTrue(all(call.args[1] == "GET" for call in request.call_args_list)) validate_candidate.assert_called_once_with( "http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам" ) state["last_document_code"] = "9" state["complete"] = False checkpoint.write_text(json.dumps(state), encoding="utf-8") with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] with self.assertRaisesRegex(ValueError, "Resume document not found"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ) failed_checkpoint = root / "failed-checkpoint.json" failure = { "errors": True, "items": [{"index": {"_id": "bad-id", "error": {"type": "mapper", "reason": "bad value"}}}], } failed_requests = [ {}, {"cluster_uuid": "cluster-1"}, {"failed-index": {"settings": {"index": {"uuid": "index-2"}}}}, failure, ] with patch("search.minjust_opensearch.request_json", side_effect=failed_requests): with self.assertRaisesRegex(RuntimeError, "bad-id: mapper: bad value"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "failed-index", maximum_bytes=4096, checkpoint=failed_checkpoint, ) failed_state = json.loads(failed_checkpoint.read_text(encoding="utf-8")) self.assertIsNone(failed_state["last_document_code"]) self.assertFalse(failed_state["complete"]) state["last_document_code"] = "8" state["complete"] = False checkpoint.write_text(json.dumps(state), encoding="utf-8") with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection: with connection: connection.execute("INSERT INTO documents VALUES ('9', 'success')") with patch("search.minjust_opensearch.request_json") as request: request.side_effect = [ {"cluster_uuid": "cluster-1"}, {"test-index": {"settings": {"index": {"uuid": "index-1"}}}}, ] with self.assertRaisesRegex(ValueError, "manifest changed"): load_bulk( root / "normalized", "http://127.0.0.1:9200", "test-index", maximum_bytes=4096, resume=True, checkpoint=checkpoint, alias="test-current", expected_old_index="test-index-old", canary_ru="закон", canary_ky="мыйзам", ) http_error = urllib.error.HTTPError( "http://127.0.0.1:9200/test", 429, "busy", {}, io.BytesIO(b"busy"), ) with ( patch("search.minjust_opensearch.urllib.request.urlopen", side_effect=[http_error, io.BytesIO(b"{}")]), patch("search.minjust_opensearch.time.sleep") as sleep, ): self.assertEqual( request_json("http://127.0.0.1:9200/test", "GET", None, "application/json", attempts=2), {}, ) sleep.assert_called_once_with(1) with ( patch( "search.minjust_opensearch.urllib.request.urlopen", side_effect=[io.BytesIO(b"{"), io.BytesIO(b"{}")], ), patch("search.minjust_opensearch.time.sleep") as sleep, ): self.assertEqual( request_json( "http://127.0.0.1:9200/_bulk", "POST", b"{}\n{}\n", "application/x-ndjson", attempts=2, retry_invalid_json=True, ), {}, ) sleep.assert_called_once_with(1) bad = document_root / "editions/10/ru/fragments.json" fragments = json.loads(bad.read_text(encoding="utf-8")) fragments[0]["document_code"] = "8" bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8") with self.assertRaisesRegex(ValueError, "identity"): export_bulk(root / "normalized", output, "test-index") self.assertEqual(output.read_bytes(), content) fragments[0]["document_code"] = "7" fragments[0]["text"] = "Повреждено" bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8") with self.assertRaisesRegex(ValueError, "checksum"): export_bulk(root / "normalized", output, "test-index") with self.assertRaisesRegex(ValueError, "inside --input"): export_bulk(root / "normalized", root / "normalized/manifest.sqlite3", "test-index") self.assertEqual(output.read_bytes(), content) bad.write_text("", encoding="utf-8") with self.assertRaisesRegex(ValueError, "fragments.json"): export_bulk(root / "normalized", output, "test-index") self.assertEqual(output.read_bytes(), content) definition = json.loads( (Path(__file__).parent / "search/minjust-fragments-index.json").read_text(encoding="utf-8") ) mapping = definition["mappings"] self.assertEqual(definition["settings"]["index"]["number_of_shards"], 1) self.assertEqual(definition["settings"]["index"]["number_of_replicas"], 0) self.assertEqual(mapping["dynamic"], "strict") self.assertEqual(mapping["properties"]["position"]["type"], "integer") self.assertEqual(mapping["properties"]["text_ru"]["analyzer"], "russian") self.assertEqual(mapping["properties"]["text_ky"]["analyzer"], "icu_analyzer") if __name__ == "__main__": unittest.main()