fix: use stable search catalog codes

backend/search/api.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import argparse
-import base64
 import datetime
 import json
 import re
@@ -12,6 +11,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 
 from search.minjust_opensearch import APP_VERSION, request_json
+from search.catalog import CATALOGS, labels
 
 
 API_VERSION = "v1"
@@ -56,26 +56,6 @@ def date(value: str | None, name: str) -> str | None:
     return value
 
 
-def catalog_code(name: str, labels: tuple[str | None, str | None]) -> str:
-    value = json.dumps(labels, ensure_ascii=False, separators=(",", ":")).encode()
-    return f"{name}:v1:{base64.urlsafe_b64encode(value).decode().rstrip('=')}"
-
-
-def catalog_label(name: str, code: str, language: str) -> str:
-    prefix = f"{name}:v1:"
-    if not code.startswith(prefix):
-        raise ApiError(400, f"{name} must be a catalog code")
-    try:
-        encoded = code[len(prefix):]
-        values = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)))
-        value = values[0 if language == "ru" else 1]
-    except (IndexError, TypeError, ValueError, UnicodeError, json.JSONDecodeError) as error:
-        raise ApiError(400, f"{name} must be a catalog code") from error
-    if not isinstance(values, list) or len(values) != 2 or not isinstance(value, str):
-        raise ApiError(400, f"{name} must be a catalog code")
-    return value
-
-
 def openapi() -> dict:
     responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
     return {
@@ -134,11 +114,13 @@ class Api:
         if sort not in {"relevance", "date"}:
             raise ApiError(400, "sort must be relevance or date")
         filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}]
-        fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"}
+        fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes"}
         for parameter, field in fields.items():
             value = one(query, parameter)
             if value:
-                filters.append({"term": {field: catalog_label(parameter, value, language)}})
+                if value not in CATALOGS[parameter]:
+                    raise ApiError(400, f"{parameter} must be a catalog code")
+                filters.append({"term": {field: value}})
         date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
         if date_from and date_to and date_from > date_to:
             raise ApiError(400, "date_from must not be later than date_to")
@@ -174,13 +156,13 @@ class Api:
         language = one(query, "language") or "ru"
         if language not in LANGUAGES:
             raise ApiError(400, "language must be ru or ky")
-        fields = {"document_types": ("document_type", "document_type_ru", "document_type_ky"), "statuses": ("status", "status_ru", "status_ky"), "authorities": ("authority", "authority_paths_ru", "authority_paths_ky")}
+        fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes")}
-        body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair[1:]], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
+        body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
         response = self.query_opensearch(body)
         try:
             aggregations = response["aggregations"]
             values = {
-                name: [{"code": catalog_code(pair[0], (item["key"][0], item["key"][1])), "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
+                name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
                 for name, pair in fields.items()
             }
         except (KeyError, TypeError) as error:

backend/search/catalog.py

@@ -0,0 +1,51 @@
+"""Immutable v1 search catalogs."""
+
+DOCUMENT_TYPES = {
+    "constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"),
+}
+STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")}
+AUTHORITIES = {"president": ("Президент", "Президент"), "parliament": ("Органы законодательной власти", "Мыйзам чыгаруу бийлик органдары"), "cabinet": ("Правительство и Кабинет Министров", "Өкмөт жана Министрлер Кабинети"), "ministries_and_committees": ("Министерства и государственные комитеты", "Министрликтер жана мамлекеттик комитеттер"), "administrative_agencies": ("Административные ведомства", "Административдик ведомстволор"), "national_bank": ("Национальный банк", "Улуттук банк"), "other_state_bodies": ("Иные государственные органы", "Башка мамлекеттик органдар"), "local_representative_bodies": ("Представительные органы местного самоуправления", "Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары"), "other": ("Прочие органы", "Башка органдар")}
+CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES}
+
+
+def labels(category: str, code: str) -> dict[str, str]:
+    try:
+        ru, ky = CATALOGS[category][code]
+    except KeyError as error:
+        raise ValueError(f"Unknown {category} catalog code: {code}") from error
+    return {"ru": ru, "ky": ky}
+
+
+def source_code(category: str, value: dict | None) -> str:
+    pair = ((value or {}).get("ru"), (value or {}).get("ky"))
+    if pair == (None, None):
+        return "unspecified"
+    for code, expected in CATALOGS[category].items():
+        if pair == expected or category == "document_type" and code == "provision" and pair == ("Положение", "Положение"):
+            return code
+    raise ValueError(f"Unmapped {category} catalog value: {pair!r}")
+
+
+def authority_codes(paths: list[dict]) -> list[str]:
+    codes = set()
+    for path in paths:
+        text = " ".join(path.get("ru", []) + path.get("ky", [])).lower()
+        if "президент" in text:
+            codes.add("president")
+        elif "жогорку кенеш" in text or "верховный совет" in text or "мыйзам чыгаруу" in text:
+            codes.add("parliament")
+        elif "кабинет министров" in text or "правительство" in text or "өкмөт" in text:
+            codes.add("cabinet")
+        elif "министер" in text or "мамлекеттик комитет" in text:
+            codes.add("ministries_and_committees")
+        elif "административ" in text:
+            codes.add("administrative_agencies")
+        elif "национальн" in text and "банк" in text or "улуттук банк" in text:
+            codes.add("national_bank")
+        elif "кенеш" in text or "кеңеш" in text or "айыл" in text or "местного самоуправления" in text:
+            codes.add("local_representative_bodies")
+        elif "иные государственные" in text or "башка мамлекеттик" in text:
+            codes.add("other_state_bodies")
+        else:
+            codes.add("other")
+    return sorted(codes) or ["other"]

backend/search/minjust-fragments-index.json

@@ -22,12 +22,15 @@
       "document_name_ky": { "type": "text", "analyzer": "icu_analyzer", "fields": { "keyword": { "type": "keyword", "ignore_above": 1024 } } },
       "document_type_ru": { "type": "keyword" },
       "document_type_ky": { "type": "keyword" },
+      "document_type_code": { "type": "keyword" },
       "status_ru": { "type": "keyword" },
       "status_ky": { "type": "keyword" },
+      "status_code": { "type": "keyword" },
       "number": { "type": "keyword" },
       "date_adopted": { "type": "date", "format": "strict_date" },
       "authority_paths_ru": { "type": "keyword", "ignore_above": 2048 },
       "authority_paths_ky": { "type": "keyword", "ignore_above": 2048 },
+      "authority_codes": { "type": "keyword" },
       "source_path": { "type": "keyword", "index": false },
       "source_sha256": { "type": "keyword", "index": false },
       "text_sha256": { "type": "keyword", "index": false }

backend/search/minjust_opensearch.py

@@ -15,6 +15,8 @@ import urllib.request
 from pathlib import Path
 from typing import Iterator
 
+from search.catalog import authority_codes, source_code
+
 APP_VERSION = "0.7.1"
 LANGUAGES = {"ru", "ky"}
 DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
@@ -73,12 +75,15 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st
         "document_name_ky": localized(document.get("name"), "ky"),
         "document_type_ru": localized(document.get("type"), "ru"),
         "document_type_ky": localized(document.get("type"), "ky"),
+        "document_type_code": source_code("document_type", document.get("type")),
         "status_ru": localized(document.get("status"), "ru"),
         "status_ky": localized(document.get("status"), "ky"),
+        "status_code": source_code("status", document.get("status")),
         "number": document.get("number"),
         "date_adopted": dates.get("DateAdopted"),
         "authority_paths_ru": paths(document, "authority_paths", "ru"),
         "authority_paths_ky": paths(document, "authority_paths", "ky"),
+        "authority_codes": authority_codes(document.get("authority_paths", [])),
         "source_path": fragment["source_path"],
         "source_sha256": fragment["source_sha256"],
         "text_sha256": fragment["text_sha256"],

backend/test_search_api.py

@@ -4,7 +4,7 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
-from search.api import Api, ApiError, catalog_code
+from search.api import Api, ApiError
 
 
 class SearchApiTest(unittest.TestCase):
@@ -27,12 +27,12 @@ class SearchApiTest(unittest.TestCase):
             api = self.make_api(Path(temporary))
             response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["<em>Закон</em>"]}}]}}
             with patch("search.api.request_json", return_value=response) as request:
-                status, payload = api.handle("GET", f"/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status={catalog_code('status', ('Действует', 'Күчүндө'))}")
+                status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=active")
             self.assertEqual(status, 200)
             self.assertEqual(payload["results"][0]["snippet"], "<em>Закон</em>")
             body = json.loads(request.call_args.args[2])
             self.assertEqual((body["from"], body["size"]), (5, 6))
-            self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"])
+            self.assertIn({"term": {"status_code": "active"}}, body["query"]["bool"]["filter"])
             with self.assertRaisesRegex(ApiError, "within 10000 results"):
                 api.handle("GET", "/search?q=x&page=100&page_size=100")
 
@@ -55,12 +55,12 @@ class SearchApiTest(unittest.TestCase):
     def test_filters_count_documents_and_return_bilingual_labels(self):
         with tempfile.TemporaryDirectory() as temporary:
             api = self.make_api(Path(temporary))
-            response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}}
+            response = {"aggregations": {name: {"buckets": [{"key": "law" if name == "document_types" else "active" if name == "statuses" else "parliament", "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}}
             with patch("search.api.request_json", return_value=response) as request:
                 payload = api.handle("GET", "/search/filters?language=ky")[1]
-            self.assertEqual(payload["document_types"][0], {"code": catalog_code("document_type", ("Закон", "Мыйзам")), "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
+            self.assertEqual(payload["document_types"][0], {"code": "law", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3})
             body = json.loads(request.call_args.args[2])
-            self.assertIn("multi_terms", body["aggs"]["document_types"])
+            self.assertIn("terms", body["aggs"]["document_types"])
             self.assertEqual(body["query"], {"term": {"is_current_edition": True}})
 
 

backend/test_search_api_opensearch.py

@@ -4,7 +4,7 @@ import unittest
 import uuid
 from pathlib import Path
 
-from search.api import Api, catalog_code
+from search.api import Api
 from search.minjust_opensearch import DEFAULT_MAPPING, request_json
 
 
@@ -16,16 +16,16 @@ class SearchApiOpenSearchTest(unittest.TestCase):
         request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json")
         try:
             documents = [
-                {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]},
+                {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "status_code": "repealed", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
-                {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]},
+                {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]},
-                {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"]},
+                {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "document_type_code": "decree", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"], "authority_codes": ["president"]},
             ]
             for number, document in enumerate(documents):
                 request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json")
             request_json(f"{base_url}/{index}/_refresh", "POST", None, "application/json")
             api = Api(base_url, index, Path("."))
             self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], [])
-            filtered = api.handle("GET", f"/search?q=needle&document_type={catalog_code('document_type', ('Закон', 'Мыйзам'))}")[1]
+            filtered = api.handle("GET", "/search?q=needle&document_type=law")[1]
             self.assertEqual([result["code"] for result in filtered["results"]], ["1"])
             filters = api.handle("GET", "/search/filters")[1]
             self.assertEqual({item["count"] for item in filters["statuses"]}, {2})