fix: complete search API contract

2026-08-25 07:29:06 +03:00
parent 0bef90debd
commit 9b629d93e3
8 changed files with 90 additions and 23 deletions
--- a/README.md
+++ b/README.md
@@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения
 ## Текущее состояние

 Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии
-`0.7.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную
+`0.7.1`: исправления контракта Search API v1 и его ограничений OpenSearch.
 размеченном наборе запросов.

 | Компонент | Версия | Состояние |
 |---|---:|---|
 | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
-| Backend | `0.7.0` | добавлено атомарное переключение alias поискового индекса |
+| Backend | `0.7.1` | исправлен контракт Search API v1 |
 | Frontend | — | ещё не создан |
-| Сбор и обработка правовых данных | `0.7.0` | добавлены relevance set и baseline-метрики |
+| Сбор и обработка правовых данных | `0.7.1` | добавлены relevance set и baseline-метрики |
 | RAG и база знаний | — | ещё не созданы |

 ## Структура репозитория
@@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v

 ---

-Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан
+Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан
--- a/backend/README.md
+++ b/backend/README.md
@@ -1,6 +1,6 @@
 # Backend Акылдаш

-Версия: `0.7.0`
+Версия: `0.7.1`

 Первая backend-область проекта — загрузка правовых документов из официального
 Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
@@ -171,8 +171,8 @@ PYTHONPATH=backend python3 -m search.api

 Он публикует OpenAPI в `GET /openapi.json` и поддерживает `GET /search`,
 `/search/filters`, `/documents/{code}`, `/documents/{code}/editions` и
-`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются с
-каноническим значением ЦБД и подписью выбранного языка; применяйте `code` как
+`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются со
+стабильным кодом справочника v1 и подписями RU/KY; применяйте `code` как
 параметр поиска. API не подменяет отсутствующий язык документа.

 ## Локальный OpenSearch
@@ -225,4 +225,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la

 ---

-Акылдаш · Backend v0.7.0 · Frontend — не создан
+Акылдаш · Backend v0.7.1 · Frontend — не создан
--- a/backend/ingestion/minjust_cbd.py
+++ b/backend/ingestion/minjust_cbd.py
@@ -21,7 +21,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Callable, Iterable

-APP_VERSION = "0.7.0"
+APP_VERSION = "0.7.1"
 API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/"
 LANGUAGES = {"Rus": "ru", "Kyr": "ky"}
 IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"}
--- a/backend/normalization/minjust_cbd.py
+++ b/backend/normalization/minjust_cbd.py
@@ -23,7 +23,7 @@ from pathlib import Path
 from typing import Callable
 from urllib.parse import urlsplit

-APP_VERSION = "0.7.0"
+APP_VERSION = "0.7.1"
 SCHEMA_VERSION = "1"
 NORMALIZER_VERSION = "1.0.0"
 LANGUAGES = ("ru", "ky")
--- a/backend/search/api.py
+++ b/backend/search/api.py
@@ -3,6 +3,7 @@
 from __future__ import annotations

 import argparse
+import base64
 import datetime
 import json
 import re
@@ -17,6 +18,7 @@ API_VERSION = "v1"
 LANGUAGES = {"ru", "ky"}
 CODE = re.compile(r"^[0-9]+$")
 MAX_PAGE_SIZE = 100
+MAX_RESULT_WINDOW = 10_000


 class ApiError(Exception):
@@ -54,6 +56,26 @@ def date(value: str | None, name: str) -> str | None:
    return value


+def catalog_code(name: str, labels: tuple[str | None, str | None]) -> str:
+    value = json.dumps(labels, ensure_ascii=False, separators=(",", ":")).encode()
+    return f"{name}:v1:{base64.urlsafe_b64encode(value).decode().rstrip('=')}"
+
+
+def catalog_label(name: str, code: str, language: str) -> str:
+    prefix = f"{name}:v1:"
+    if not code.startswith(prefix):
+        raise ApiError(400, f"{name} must be a catalog code")
+    try:
+        encoded = code[len(prefix):]
+        values = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)))
+        value = values[0 if language == "ru" else 1]
+    except (IndexError, TypeError, ValueError, UnicodeError, json.JSONDecodeError) as error:
+        raise ApiError(400, f"{name} must be a catalog code") from error
+    if not isinstance(values, list) or len(values) != 2 or not isinstance(value, str):
+        raise ApiError(400, f"{name} must be a catalog code")
+    return value
+
+
 def openapi() -> dict:
    responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
    return {
@@ -73,9 +95,9 @@ def openapi() -> dict:
                {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}},
            ]}},
            "/search/filters": {"get": {"responses": responses}},
-            "/documents/{code}": {"get": {"responses": responses}},
-            "/documents/{code}/editions": {"get": {"responses": responses}},
-            "/documents/{code}/editions/{edition}": {"get": {"responses": responses}},
+            "/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
+            "/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
+            "/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
        },
    }

@@ -106,6 +128,8 @@ class Api:
            raise ApiError(400, "language must be ru or ky")
        page = parse_positive(one(query, "page"), "page", 1, 1_000_000)
        page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE)
+        if page * page_size >= MAX_RESULT_WINDOW:
+            raise ApiError(400, f"page and page_size must stay within {MAX_RESULT_WINDOW} results")
        sort = one(query, "sort") or "relevance"
        if sort not in {"relevance", "date"}:
            raise ApiError(400, "sort must be relevance or date")
@@ -114,7 +138,7 @@ class Api:
        for parameter, field in fields.items():
            value = one(query, parameter)
            if value:
-                filters.append({"term": {field: value}})
+                filters.append({"term": {field: catalog_label(parameter, value, language)}})
        date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
        if date_from and date_to and date_from > date_to:
            raise ApiError(400, "date_from must not be later than date_to")
@@ -150,14 +174,14 @@ class Api:
        language = one(query, "language") or "ru"
        if language not in LANGUAGES:
            raise ApiError(400, "language must be ru or ky")
-        fields = {"document_types": ("document_type_ru", "document_type_ky"), "statuses": ("status_ru", "status_ky"), "authorities": ("authority_paths_ru", "authority_paths_ky")}
-        body = {"size": 0, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
+        fields = {"document_types": ("document_type", "document_type_ru", "document_type_ky"), "statuses": ("status", "status_ru", "status_ky"), "authorities": ("authority", "authority_paths_ru", "authority_paths_ky")}
+        body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair[1:]], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
        response = self.query_opensearch(body)
        try:
            aggregations = response["aggregations"]
            values = {
-                name: [{"code": item["key"][0 if language == "ru" else 1], "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
-                for name in fields
+                name: [{"code": catalog_code(pair[0], (item["key"][0], item["key"][1])), "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
+                for name, pair in fields.items()
            }
        except (KeyError, TypeError) as error:
            raise ApiError(502, "search backend returned incomplete filters") from error
--- a/backend/search/minjust_opensearch.py
+++ b/backend/search/minjust_opensearch.py
@@ -15,7 +15,7 @@ import urllib.request
 from pathlib import Path
 from typing import Iterator

-APP_VERSION = "0.7.0"
+APP_VERSION = "0.7.1"
 LANGUAGES = {"ru", "ky"}
 DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")

--- a/backend/test_search_api.py
+++ b/backend/test_search_api.py
@@ -4,7 +4,7 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch

-from search.api import Api, ApiError
+from search.api import Api, ApiError, catalog_code


 class SearchApiTest(unittest.TestCase):
@@ -27,17 +27,22 @@ class SearchApiTest(unittest.TestCase):
            api = self.make_api(Path(temporary))
            response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["<em>Закон</em>"]}}]}}
            with patch("search.api.request_json", return_value=response) as request:
-                status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=%D0%94%D0%B5%D0%B9%D1%81%D1%82%D0%B2%D1%83%D0%B5%D1%82")
+                status, payload = api.handle("GET", f"/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status={catalog_code('status', ('Действует', 'Күчүндө'))}")
            self.assertEqual(status, 200)
            self.assertEqual(payload["results"][0]["snippet"], "<em>Закон</em>")
            body = json.loads(request.call_args.args[2])
            self.assertEqual((body["from"], body["size"]), (5, 6))
            self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"])
+            with self.assertRaisesRegex(ApiError, "within 10000 results"):
+                api.handle("GET", "/search?q=x&page=100&page_size=100")

    def test_document_editions_openapi_and_validation(self):
        with tempfile.TemporaryDirectory() as temporary:
            api = self.make_api(Path(temporary))
-            self.assertEqual(api.handle("GET", "/openapi.json")[1]["info"]["version"], "v1")
+            specification = api.handle("GET", "/openapi.json")[1]
+            self.assertEqual(specification["info"]["version"], "v1")
+            self.assertEqual(specification["paths"]["/documents/{code}"]["get"]["parameters"][0]["required"], True)
+            self.assertEqual(specification["paths"]["/documents/{code}/editions/{edition}"]["get"]["parameters"][1]["name"], "edition")
            self.assertEqual(api.handle("GET", "/documents/7")[1]["current_edition"]["source_code"], "10")
            self.assertEqual(api.handle("GET", "/documents/7/editions/10?language=ru")[1]["content"]["ru"]["text"], "Текст\n")
            with self.assertRaisesRegex(ApiError, "q is required"):
@@ -53,9 +58,10 @@ class SearchApiTest(unittest.TestCase):
            response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}}
            with patch("search.api.request_json", return_value=response) as request:
                payload = api.handle("GET", "/search/filters?language=ky")[1]
-            self.assertEqual(payload["document_types"][0], {"code": "Мыйзам", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
+            self.assertEqual(payload["document_types"][0], {"code": catalog_code("document_type", ("Закон", "Мыйзам")), "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
            body = json.loads(request.call_args.args[2])
            self.assertIn("multi_terms", body["aggs"]["document_types"])
+            self.assertEqual(body["query"], {"term": {"is_current_edition": True}})


 if __name__ == "__main__":
--- a/backend/test_search_api_opensearch.py
+++ b/backend/test_search_api_opensearch.py
@@ -0,0 +1,37 @@
+import json
+import os
+import unittest
+import uuid
+from pathlib import Path
+
+from search.api import Api, catalog_code
+from search.minjust_opensearch import DEFAULT_MAPPING, request_json
+
+
+@unittest.skipUnless(os.getenv("AKYLDASH_OPENSEARCH_URL"), "set AKYLDASH_OPENSEARCH_URL to run against local OpenSearch")
+class SearchApiOpenSearchTest(unittest.TestCase):
+    def test_current_editions_filters_and_catalogs(self):
+        base_url = os.environ["AKYLDASH_OPENSEARCH_URL"].rstrip("/")
+        index = f"akyldash-api-test-{uuid.uuid4().hex}"
+        request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json")
+        try:
+            documents = [
+                {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]},
+                {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]},
+                {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"]},
+            ]
+            for number, document in enumerate(documents):
+                request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json")
+            request_json(f"{base_url}/{index}/_refresh", "POST", None, "application/json")
+            api = Api(base_url, index, Path("."))
+            self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], [])
+            filtered = api.handle("GET", f"/search?q=needle&document_type={catalog_code('document_type', ('Закон', 'Мыйзам'))}")[1]
+            self.assertEqual([result["code"] for result in filtered["results"]], ["1"])
+            filters = api.handle("GET", "/search/filters")[1]
+            self.assertEqual({item["count"] for item in filters["statuses"]}, {2})
+        finally:
+            request_json(f"{base_url}/{index}", "DELETE", None, "application/json")
+
+
+if __name__ == "__main__":
+    unittest.main()