diff --git a/backend/search/api.py b/backend/search/api.py index 1be21b2..650228a 100644 --- a/backend/search/api.py +++ b/backend/search/api.py @@ -55,11 +55,12 @@ def date(value: str | None, name: str) -> str | None: def openapi() -> dict: + responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}} return { "openapi": "3.0.3", "info": {"title": "Akyldash Search API", "version": API_VERSION}, "paths": { - "/search": {"get": {"parameters": [ + "/search": {"get": {"responses": responses, "parameters": [ {"name": "q", "in": "query", "required": True, "schema": {"type": "string"}}, {"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}}, {"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}}, @@ -71,10 +72,10 @@ def openapi() -> dict: {"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}}, {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, ]}}, - "/search/filters": {"get": {}}, - "/documents/{code}": {"get": {}}, - "/documents/{code}/editions": {"get": {}}, - "/documents/{code}/editions/{edition}": {"get": {}}, + "/search/filters": {"get": {"responses": responses}}, + "/documents/{code}": {"get": {"responses": responses}}, + "/documents/{code}/editions": {"get": {"responses": responses}}, + "/documents/{code}/editions/{edition}": {"get": {"responses": responses}}, }, } @@ -108,7 +109,7 @@ class Api: sort = one(query, "sort") or "relevance" if sort not in {"relevance", "date"}: raise ApiError(400, "sort must be relevance or date") - filters: list[dict] = [{"term": {"language": language}}] + filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}] fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"} for parameter, field in fields.items(): value = one(query, parameter) @@ -149,13 +150,13 @@ class Api: language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") - fields = {"document_types": f"document_type_{language}", "statuses": f"status_{language}", "authorities": f"authority_paths_{language}"} - body = {"size": 0, "aggs": {name: {"terms": {"field": field, "size": 1000}} for name, field in fields.items()}} + fields = {"document_types": ("document_type_ru", "document_type_ky"), "statuses": ("status_ru", "status_ky"), "authorities": ("authority_paths_ru", "authority_paths_ky")} + body = {"size": 0, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} response = self.query_opensearch(body) try: aggregations = response["aggregations"] values = { - name: [{"code": item["key"], "label": item["key"], "count": item["doc_count"]} for item in aggregations[name]["buckets"]] + name: [{"code": item["key"][0 if language == "ru" else 1], "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] for name in fields } except (KeyError, TypeError) as error: diff --git a/backend/search/minjust-fragments-index.json b/backend/search/minjust-fragments-index.json index daa9381..3cb2653 100644 --- a/backend/search/minjust-fragments-index.json +++ b/backend/search/minjust-fragments-index.json @@ -12,6 +12,7 @@ "schema_version": { "type": "keyword" }, "document_code": { "type": "keyword" }, "edition_code": { "type": "keyword" }, + "is_current_edition": { "type": "boolean" }, "language": { "type": "keyword" }, "position": { "type": "integer" }, "fragment_type": { "type": "keyword" }, diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index aa1f735..15fe14e 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -42,7 +42,7 @@ def paths(document: dict, field: str, language: str) -> list[str]: return [" > ".join(item[language]) for item in document.get(field, []) if item.get(language)] -def search_document(document: dict, fragment: dict, expected: tuple[str, str, str, int]) -> dict: +def search_document(document: dict, fragment: dict, expected: tuple[str, str, str, int], current_edition: str) -> dict: document_code, edition_code, language, position = expected fragment_id = f"document:{document_code}:edition:{edition_code}:lang:{language}:fragment:{position}" required = ("id", "document_code", "edition_code", "language", "position", "type", "text", "text_sha256", "source_path", "source_sha256") @@ -64,6 +64,7 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st "schema_version": document["schema_version"], "document_code": document_code, "edition_code": edition_code, + "is_current_edition": edition_code == current_edition, "language": language, "position": position, "fragment_type": fragment["type"], @@ -118,13 +119,18 @@ def bulk_pairs( document = read_json(directory / "document.json") if document.get("source_code") != directory.name: raise ValueError(f"Document identity does not match its path: {directory}") + edition_root = directory / "editions" + editions = [path.name for path in edition_root.iterdir() if path.is_dir()] if edition_root.is_dir() else [] + if not editions: + continue + current_edition = max(editions, key=lambda value: (not value.isdigit(), int(value) if value.isdigit() else value)) for fragment_path in sorted(directory.glob("editions/*/*/fragments.json")): edition_code, language = fragment_path.parts[-3:-1] values = read_json(fragment_path) if not isinstance(values, list): raise ValueError(f"Fragments must be a list: {fragment_path}") for position, fragment in enumerate(values, 1): - source = search_document(document, fragment, (directory.name, edition_code, language, position)) + source = search_document(document, fragment, (directory.name, edition_code, language, position), current_edition) action = json.dumps({"index": {"_index": index, "_id": fragment["id"]}}, ensure_ascii=False, allow_nan=False) body = json.dumps(source, ensure_ascii=False, allow_nan=False) yield directory.name, f"{action}\n{body}\n".encode() diff --git a/backend/test_minjust_opensearch.py b/backend/test_minjust_opensearch.py index cf02ac8..a9e897d 100644 --- a/backend/test_minjust_opensearch.py +++ b/backend/test_minjust_opensearch.py @@ -91,6 +91,7 @@ class MinjustOpenSearchTest(unittest.TestCase): self.assertEqual(len(lines), 4) self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1") self.assertIn("text_ky", lines[1]) + self.assertTrue(lines[1]["is_current_edition"]) self.assertNotIn("text_ru", lines[1]) self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока") self.assertEqual( diff --git a/backend/test_search_api.py b/backend/test_search_api.py index 8800bef..a4ac6eb 100644 --- a/backend/test_search_api.py +++ b/backend/test_search_api.py @@ -47,6 +47,16 @@ class SearchApiTest(unittest.TestCase): with self.assertRaisesRegex(ApiError, "document not found"): api.handle("GET", "/documents/%2E%2E") + def test_filters_count_documents_and_return_bilingual_labels(self): + with tempfile.TemporaryDirectory() as temporary: + api = self.make_api(Path(temporary)) + response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}} + with patch("search.api.request_json", return_value=response) as request: + payload = api.handle("GET", "/search/filters?language=ky")[1] + self.assertEqual(payload["document_types"][0], {"code": "Мыйзам", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) + body = json.loads(request.call_args.args[2]) + self.assertIn("multi_terms", body["aggs"]["document_types"]) + if __name__ == "__main__": unittest.main()