611 lines
30 KiB
Python
611 lines
30 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
import urllib.error
|
|
from contextlib import closing
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from search.catalog import legal_force_code, labels, source_code
|
|
from search.minjust_opensearch import bulk_batches, document_codes, export_bulk, load_bulk, request_json, switch_alias
|
|
|
|
|
|
class MinjustOpenSearchTest(unittest.TestCase):
|
|
def test_unknown_document_type_keeps_source_label_and_has_no_inferred_force(self):
|
|
value = {"ru": "Решение", "ky": "Решение"}
|
|
|
|
self.assertEqual(source_code("document_type", value), "other")
|
|
self.assertEqual(labels("document_type", "other"), {"ru": "Прочие документы", "ky": "Башка документтер"})
|
|
self.assertIsNone(legal_force_code(value))
|
|
self.assertEqual(source_code("status", {"ru": "Не вступил в силу", "ky": "Күчүнө кире элек"}), "not_yet_effective")
|
|
self.assertEqual(legal_force_code({"ru": "Конституционный закон", "ky": "Конституционный закон"}), "constitutional")
|
|
self.assertEqual(legal_force_code({"ru": "Указ", "ky": "Жарлыгы"}), "subordinate")
|
|
self.assertEqual(source_code("document_type", {"ru": "устав"}), "charter")
|
|
|
|
def test_export_marks_last_official_edition_current_even_when_ids_are_out_of_order(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
input_root = root / "normalized"
|
|
document_root = input_root / "documents/6"
|
|
document_root.mkdir(parents=True)
|
|
with closing(sqlite3.connect(input_root / "manifest.sqlite3")) as connection:
|
|
with connection:
|
|
connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
connection.execute("INSERT INTO documents VALUES ('6', 'success')")
|
|
(document_root / "document.json").write_text(
|
|
json.dumps({"schema_version": "1", "source_code": "6", "editions": [
|
|
{"source_code": "920492", "source_order": 0},
|
|
{"source_code": "920292", "source_order": 1},
|
|
]}),
|
|
encoding="utf-8",
|
|
)
|
|
for edition in ("920292", "920492"):
|
|
text = f"Текст {edition}"
|
|
fragment_path = document_root / f"editions/{edition}/ru/fragments.json"
|
|
fragment_path.parent.mkdir(parents=True)
|
|
fragment_path.write_text(json.dumps([{
|
|
"id": f"document:6:edition:{edition}:lang:ru:fragment:1",
|
|
"document_code": "6",
|
|
"edition_code": edition,
|
|
"language": "ru",
|
|
"position": 1,
|
|
"type": "paragraph",
|
|
"text": text,
|
|
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"source_path": f"documents/6/editions/{edition}/ru.html",
|
|
"source_sha256": "b" * 64,
|
|
}]), encoding="utf-8")
|
|
|
|
output = root / "bulk.ndjson"
|
|
self.assertEqual(export_bulk(input_root, output, "test-index"), (1, 2))
|
|
lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()]
|
|
flags = {
|
|
lines[index]["index"]["_id"].split(":")[3]: lines[index + 1]["is_current_edition"]
|
|
for index in range(0, len(lines), 2)
|
|
}
|
|
|
|
self.assertEqual(flags, {"920292": True, "920492": False})
|
|
|
|
def test_switches_alias_atomically_after_successful_load(self):
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"akyldash-fragments-v1": {"aliases": {"akyldash-fragments-current": {}}}},
|
|
{"acknowledged": True},
|
|
]) as request:
|
|
switch_alias(
|
|
"http://127.0.0.1:9200/", "akyldash-fragments-v2",
|
|
"akyldash-fragments-current", "akyldash-fragments-v1",
|
|
)
|
|
|
|
self.assertEqual(request.call_args_list[1].args[:2], ("http://127.0.0.1:9200/_aliases", "POST"))
|
|
body = json.loads(request.call_args_list[1].args[2])
|
|
self.assertEqual(body["actions"][0], {"remove": {"index": "akyldash-fragments-v1", "alias": "akyldash-fragments-current", "must_exist": True}})
|
|
self.assertEqual(body["actions"][1], {"add": {"index": "akyldash-fragments-v2", "alias": "akyldash-fragments-current"}})
|
|
|
|
with self.assertRaisesRegex(ValueError, "differ"):
|
|
switch_alias("http://127.0.0.1:9200", "same", "same", "old")
|
|
with self.assertRaisesRegex(ValueError, "differ"):
|
|
switch_alias("http://127.0.0.1:9200", "index", "", "old")
|
|
with patch("search.minjust_opensearch.request_json", return_value={
|
|
"akyldash-fragments-v3": {"aliases": {"alias": {}}}
|
|
}) as request:
|
|
with self.assertRaisesRegex(ValueError, "currently targets"):
|
|
switch_alias("http://127.0.0.1:9200", "new", "alias", "akyldash-fragments-v1")
|
|
request.assert_called_once()
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"old": {"aliases": {"alias": {}}}}, {"acknowledged": False},
|
|
{"old": {"aliases": {"alias": {}}}},
|
|
]):
|
|
with self.assertRaisesRegex(RuntimeError, "did not acknowledge"):
|
|
switch_alias("http://127.0.0.1:9200", "index", "alias", "old")
|
|
|
|
def test_switch_alias_recovers_when_post_succeeds_but_response_is_lost(self):
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"old": {"aliases": {"alias": {}}}},
|
|
RuntimeError("response lost after OpenSearch applied request"),
|
|
{"candidate": {"aliases": {"alias": {}}}},
|
|
]) as request:
|
|
switch_alias("http://os:9200", "candidate", "alias", "old")
|
|
self.assertEqual(request.call_count, 3)
|
|
self.assertEqual(request.call_args_list[1].args[1], "POST")
|
|
self.assertEqual(request.call_args_list[2].args[1], "GET")
|
|
|
|
def test_switch_alias_is_idempotent_after_process_stops_before_checkpoint(self):
|
|
with patch("search.minjust_opensearch.request_json", return_value={
|
|
"candidate": {"aliases": {"alias": {}}}
|
|
}) as request:
|
|
switch_alias("http://os:9200", "candidate", "alias", "old")
|
|
request.assert_called_once()
|
|
|
|
def test_switch_alias_refuses_to_retry_after_unexpected_target(self):
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"old": {"aliases": {"alias": {}}}},
|
|
RuntimeError("response lost"),
|
|
{"other": {"aliases": {"alias": {}}}},
|
|
]):
|
|
with self.assertRaisesRegex(ValueError, "unexpected index"):
|
|
switch_alias("http://os:9200", "candidate", "alias", "old")
|
|
|
|
def test_resume_completes_checkpoint_when_alias_switched_before_previous_process_stopped(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
input_root = root / "normalized"
|
|
document_root = input_root / "documents/1"
|
|
document_root.mkdir(parents=True)
|
|
with closing(sqlite3.connect(input_root / "manifest.sqlite3")) as connection:
|
|
with connection:
|
|
connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
connection.execute("INSERT INTO documents VALUES ('1', 'success')")
|
|
(document_root / "document.json").write_text(
|
|
json.dumps({"schema_version": "1", "source_code": "1", "editions": [
|
|
{"source_code": "1", "source_order": 0},
|
|
]}),
|
|
encoding="utf-8",
|
|
)
|
|
text = "текст закона"
|
|
fragment = document_root / "editions/1/ru/fragments.json"
|
|
fragment.parent.mkdir(parents=True)
|
|
fragment.write_text(json.dumps([{
|
|
"id": "document:1:edition:1:lang:ru:fragment:1",
|
|
"document_code": "1", "edition_code": "1", "language": "ru", "position": 1,
|
|
"type": "paragraph", "text": text,
|
|
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"source_path": "documents/1/editions/1/ru.html", "source_sha256": "b" * 64,
|
|
}]), encoding="utf-8")
|
|
checkpoint = root / "checkpoint.json"
|
|
state = {
|
|
"schema_version": 2,
|
|
"url": "http://os:9200",
|
|
"cluster_uuid": "cluster",
|
|
"index": "candidate",
|
|
"index_uuid": "index-uuid",
|
|
"input": str(input_root.resolve()),
|
|
"manifest_sha256": hashlib.sha256((input_root / "manifest.sqlite3").read_bytes()).hexdigest(),
|
|
"limit": None,
|
|
"alias": "current",
|
|
"last_document_code": "1",
|
|
"complete": False,
|
|
}
|
|
checkpoint.write_text(json.dumps(state), encoding="utf-8")
|
|
|
|
with (
|
|
patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"cluster_uuid": "cluster"},
|
|
{"candidate": {"settings": {"index": {"uuid": "index-uuid"}}}},
|
|
{"errors": False, "items": [{"index": {}}]},
|
|
{"candidate": {"aliases": {"current": {}}}},
|
|
]) as request,
|
|
patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate,
|
|
):
|
|
self.assertEqual(
|
|
load_bulk(
|
|
input_root, "http://os:9200", "candidate", resume=True, checkpoint=checkpoint,
|
|
alias="current", expected_old_index="previous", canary_ru="закон", canary_ky="мыйзам",
|
|
),
|
|
(1, 1),
|
|
)
|
|
self.assertTrue(json.loads(checkpoint.read_text(encoding="utf-8"))["complete"])
|
|
self.assertEqual(request.call_args_list[-1].args[1], "GET")
|
|
validate_candidate.assert_called_once_with("http://os:9200", "candidate", 1, "закон", "мыйзам")
|
|
|
|
def test_zero_fragment_candidate_never_reaches_alias_switch(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary) / "normalized"
|
|
document = root / "documents/1"
|
|
document.mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "manifest.sqlite3")) as connection:
|
|
connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
connection.execute("INSERT INTO documents VALUES ('1', 'success')")
|
|
(document / "document.json").write_text(
|
|
json.dumps({"source_code": "1", "editions": []}), encoding="utf-8"
|
|
)
|
|
with (
|
|
patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{}, {"cluster_uuid": "cluster"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index"}}}},
|
|
]) as request,
|
|
patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate,
|
|
patch("search.minjust_opensearch.switch_alias") as switch,
|
|
):
|
|
with self.assertRaisesRegex(ValueError, "empty expected fragment inventory"):
|
|
load_bulk(
|
|
root, "http://127.0.0.1:9200", "test-index",
|
|
checkpoint=Path(temporary) / "checkpoint.json", alias="current",
|
|
expected_old_index="old-index", canary_ru="закон", canary_ky="мыйзам",
|
|
)
|
|
self.assertEqual(request.call_count, 3)
|
|
validate_candidate.assert_not_called()
|
|
switch.assert_not_called()
|
|
|
|
def test_candidate_index_requires_exact_count_and_both_language_canaries(self):
|
|
from search.minjust_opensearch import validate_candidate_index
|
|
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"_shards": {"failed": 0}},
|
|
{"count": 2},
|
|
{"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}},
|
|
{"hits": {"hits": [{"_source": {"text_ky": "сыноо"}}]}},
|
|
]) as request:
|
|
validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо")
|
|
self.assertEqual(request.call_count, 4)
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"_shards": {"failed": 0}}, {"count": 1}
|
|
]):
|
|
with self.assertRaisesRegex(RuntimeError, "expected 2"):
|
|
validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо")
|
|
with patch("search.minjust_opensearch.request_json", side_effect=[
|
|
{"_shards": {"failed": 0}}, {"count": 2},
|
|
{"hits": {"hits": [{"_source": {"text_ru": "тест"}}]}},
|
|
{"hits": {"hits": []}},
|
|
]):
|
|
with self.assertRaisesRegex(RuntimeError, "ky canary"):
|
|
validate_candidate_index("http://os:9200", "candidate", 2, "тест", "сыноо")
|
|
|
|
def test_exports_atomic_bulk_and_rejects_mismatched_fragment(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
document_root = root / "normalized/documents/7"
|
|
document_root.mkdir(parents=True)
|
|
with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection:
|
|
with connection:
|
|
connection.execute("CREATE TABLE documents (code TEXT, state TEXT)")
|
|
connection.execute("INSERT INTO documents VALUES ('7', 'success')")
|
|
connection.execute("INSERT INTO documents VALUES ('8', 'success')")
|
|
(document_root / "document.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": "1",
|
|
"source_code": "7",
|
|
"name": {"ru": "Закон", "ky": "Мыйзам"},
|
|
"type": {"ru": "Закон", "ky": "Мыйзам"},
|
|
"status": {"ru": "Действует", "ky": "Күчүндө"},
|
|
"number": "1",
|
|
"dates": {"DateAdopted": "2026-01-01"},
|
|
"authority_paths": [{"ru": ["Кабинет"], "ky": ["Кабинет"]}],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
empty = root / "normalized/documents/8"
|
|
empty.mkdir()
|
|
(empty / "document.json").write_text(
|
|
json.dumps({"schema_version": "1", "source_code": "8"}), encoding="utf-8"
|
|
)
|
|
for language, text in (("ru", "Текст \"RU\"\nстрока"), ("ky", "Кыргызча текст")):
|
|
path = document_root / f"editions/10/{language}/fragments.json"
|
|
path.parent.mkdir(parents=True)
|
|
path.write_text(
|
|
json.dumps(
|
|
[{
|
|
"id": f"document:7:edition:10:lang:{language}:fragment:1",
|
|
"document_code": "7",
|
|
"edition_code": "10",
|
|
"language": language,
|
|
"position": 1,
|
|
"type": "paragraph",
|
|
"text": text,
|
|
"text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
|
"source_path": f"documents/7/editions/10/{language}.html",
|
|
"source_sha256": "b" * 64,
|
|
}],
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
output = root / "bulk.ndjson"
|
|
self.assertEqual(export_bulk(root / "normalized", output, "test-index"), (2, 2))
|
|
content = output.read_bytes()
|
|
self.assertTrue(content.endswith(b"\n"))
|
|
lines = [json.loads(line) for line in content.splitlines()]
|
|
self.assertEqual(len(lines), 4)
|
|
self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1")
|
|
self.assertIn("text_ky", lines[1])
|
|
self.assertTrue(lines[1]["is_current_edition"])
|
|
self.assertNotIn("text_ru", lines[1])
|
|
self.assertEqual(lines[1]["legal_force_code"], "legislative")
|
|
self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока")
|
|
self.assertEqual(
|
|
list(bulk_batches(iter((("7", b"a\nb\n"), ("8", b"c\nd\n"))), 4)),
|
|
[("7", b"a\nb\n"), ("8", b"c\nd\n")],
|
|
)
|
|
self.assertEqual(document_codes(root / "normalized", 2, "8"), ["8"])
|
|
with self.assertRaisesRegex(ValueError, "selected range"):
|
|
document_codes(root / "normalized", 1, "8")
|
|
|
|
checkpoint = root / "checkpoint.json"
|
|
with (
|
|
patch("search.minjust_opensearch.request_json") as request,
|
|
patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate,
|
|
patch("search.minjust_opensearch.switch_alias") as switch,
|
|
):
|
|
request.side_effect = [
|
|
{},
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
{"errors": False, "items": [{"index": {}}, {"index": {}}]},
|
|
]
|
|
self.assertEqual(
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
),
|
|
(2, 2),
|
|
)
|
|
self.assertEqual(request.call_args_list[-1].args[3], "application/x-ndjson")
|
|
validate_candidate.assert_called_once_with(
|
|
"http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам"
|
|
)
|
|
switch.assert_called_once_with(
|
|
"http://127.0.0.1:9200", "test-index", "test-current", "test-index-old"
|
|
)
|
|
state = json.loads(checkpoint.read_text(encoding="utf-8"))
|
|
self.assertEqual(state["last_document_code"], "7")
|
|
self.assertTrue(state["complete"])
|
|
|
|
legacy = state.copy()
|
|
legacy.pop("alias")
|
|
legacy["schema_version"] = 1
|
|
legacy["last_document_code"] = "8"
|
|
legacy["complete"] = False
|
|
checkpoint.write_text(json.dumps(legacy), encoding="utf-8")
|
|
with (
|
|
patch("search.minjust_opensearch.request_json") as request,
|
|
patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate,
|
|
patch("search.minjust_opensearch.switch_alias"),
|
|
):
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
self.assertEqual(
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
),
|
|
(1, 0),
|
|
)
|
|
self.assertEqual(json.loads(checkpoint.read_text(encoding="utf-8"))["schema_version"], 2)
|
|
|
|
state["last_document_code"] = "8"
|
|
state["complete"] = False
|
|
checkpoint.write_text(json.dumps(state), encoding="utf-8")
|
|
with patch("search.minjust_opensearch.request_json") as request:
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-2"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
with self.assertRaisesRegex(ValueError, "does not match"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
)
|
|
with patch("search.minjust_opensearch.request_json") as request:
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
with self.assertRaisesRegex(ValueError, "alias"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
)
|
|
with patch("search.minjust_opensearch.request_json") as request:
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
with self.assertRaisesRegex(ValueError, "limit"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
limit=1,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
)
|
|
with (
|
|
patch("search.minjust_opensearch.request_json") as request,
|
|
patch("search.minjust_opensearch.validate_candidate_index") as validate_candidate,
|
|
patch("search.minjust_opensearch.switch_alias"),
|
|
):
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
self.assertEqual(
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
),
|
|
(1, 0),
|
|
)
|
|
self.assertTrue(all(call.args[1] == "GET" for call in request.call_args_list))
|
|
validate_candidate.assert_called_once_with(
|
|
"http://127.0.0.1:9200", "test-index", 2, "закон", "мыйзам"
|
|
)
|
|
|
|
state["last_document_code"] = "9"
|
|
state["complete"] = False
|
|
checkpoint.write_text(json.dumps(state), encoding="utf-8")
|
|
with patch("search.minjust_opensearch.request_json") as request:
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
with self.assertRaisesRegex(ValueError, "Resume document not found"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
)
|
|
|
|
failed_checkpoint = root / "failed-checkpoint.json"
|
|
failure = {
|
|
"errors": True,
|
|
"items": [{"index": {"_id": "bad-id", "error": {"type": "mapper", "reason": "bad value"}}}],
|
|
}
|
|
failed_requests = [
|
|
{},
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"failed-index": {"settings": {"index": {"uuid": "index-2"}}}},
|
|
failure,
|
|
]
|
|
with patch("search.minjust_opensearch.request_json", side_effect=failed_requests):
|
|
with self.assertRaisesRegex(RuntimeError, "bad-id: mapper: bad value"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"failed-index",
|
|
maximum_bytes=4096,
|
|
checkpoint=failed_checkpoint,
|
|
)
|
|
failed_state = json.loads(failed_checkpoint.read_text(encoding="utf-8"))
|
|
self.assertIsNone(failed_state["last_document_code"])
|
|
self.assertFalse(failed_state["complete"])
|
|
|
|
state["last_document_code"] = "8"
|
|
state["complete"] = False
|
|
checkpoint.write_text(json.dumps(state), encoding="utf-8")
|
|
with closing(sqlite3.connect(root / "normalized/manifest.sqlite3")) as connection:
|
|
with connection:
|
|
connection.execute("INSERT INTO documents VALUES ('9', 'success')")
|
|
with patch("search.minjust_opensearch.request_json") as request:
|
|
request.side_effect = [
|
|
{"cluster_uuid": "cluster-1"},
|
|
{"test-index": {"settings": {"index": {"uuid": "index-1"}}}},
|
|
]
|
|
with self.assertRaisesRegex(ValueError, "manifest changed"):
|
|
load_bulk(
|
|
root / "normalized",
|
|
"http://127.0.0.1:9200",
|
|
"test-index",
|
|
maximum_bytes=4096,
|
|
resume=True,
|
|
checkpoint=checkpoint,
|
|
alias="test-current",
|
|
expected_old_index="test-index-old",
|
|
canary_ru="закон",
|
|
canary_ky="мыйзам",
|
|
)
|
|
|
|
http_error = urllib.error.HTTPError(
|
|
"http://127.0.0.1:9200/test",
|
|
429,
|
|
"busy",
|
|
{},
|
|
io.BytesIO(b"busy"),
|
|
)
|
|
with (
|
|
patch("search.minjust_opensearch.urllib.request.urlopen", side_effect=[http_error, io.BytesIO(b"{}")]),
|
|
patch("search.minjust_opensearch.time.sleep") as sleep,
|
|
):
|
|
self.assertEqual(
|
|
request_json("http://127.0.0.1:9200/test", "GET", None, "application/json", attempts=2),
|
|
{},
|
|
)
|
|
sleep.assert_called_once_with(1)
|
|
|
|
with (
|
|
patch(
|
|
"search.minjust_opensearch.urllib.request.urlopen",
|
|
side_effect=[io.BytesIO(b"{"), io.BytesIO(b"{}")],
|
|
),
|
|
patch("search.minjust_opensearch.time.sleep") as sleep,
|
|
):
|
|
self.assertEqual(
|
|
request_json(
|
|
"http://127.0.0.1:9200/_bulk",
|
|
"POST",
|
|
b"{}\n{}\n",
|
|
"application/x-ndjson",
|
|
attempts=2,
|
|
retry_invalid_json=True,
|
|
),
|
|
{},
|
|
)
|
|
sleep.assert_called_once_with(1)
|
|
|
|
bad = document_root / "editions/10/ru/fragments.json"
|
|
fragments = json.loads(bad.read_text(encoding="utf-8"))
|
|
fragments[0]["document_code"] = "8"
|
|
bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "identity"):
|
|
export_bulk(root / "normalized", output, "test-index")
|
|
self.assertEqual(output.read_bytes(), content)
|
|
|
|
fragments[0]["document_code"] = "7"
|
|
fragments[0]["text"] = "Повреждено"
|
|
bad.write_text(json.dumps(fragments, ensure_ascii=False), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "checksum"):
|
|
export_bulk(root / "normalized", output, "test-index")
|
|
with self.assertRaisesRegex(ValueError, "inside --input"):
|
|
export_bulk(root / "normalized", root / "normalized/manifest.sqlite3", "test-index")
|
|
self.assertEqual(output.read_bytes(), content)
|
|
|
|
bad.write_text("", encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "fragments.json"):
|
|
export_bulk(root / "normalized", output, "test-index")
|
|
self.assertEqual(output.read_bytes(), content)
|
|
|
|
definition = json.loads(
|
|
(Path(__file__).parent / "search/minjust-fragments-index.json").read_text(encoding="utf-8")
|
|
)
|
|
mapping = definition["mappings"]
|
|
self.assertEqual(definition["settings"]["index"]["number_of_shards"], 1)
|
|
self.assertEqual(definition["settings"]["index"]["number_of_replicas"], 0)
|
|
self.assertEqual(mapping["dynamic"], "strict")
|
|
self.assertEqual(mapping["properties"]["position"]["type"], "integer")
|
|
self.assertEqual(mapping["properties"]["text_ru"]["analyzer"], "russian")
|
|
self.assertEqual(mapping["properties"]["text_ky"]["analyzer"], "icu_analyzer")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|