fix: use stable search catalog codes
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
@@ -12,6 +11,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from search.minjust_opensearch import APP_VERSION, request_json
|
||||
from search.catalog import CATALOGS, labels
|
||||
|
||||
|
||||
API_VERSION = "v1"
|
||||
@@ -56,26 +56,6 @@ def date(value: str | None, name: str) -> str | None:
|
||||
return value
|
||||
|
||||
|
||||
def catalog_code(name: str, labels: tuple[str | None, str | None]) -> str:
|
||||
value = json.dumps(labels, ensure_ascii=False, separators=(",", ":")).encode()
|
||||
return f"{name}:v1:{base64.urlsafe_b64encode(value).decode().rstrip('=')}"
|
||||
|
||||
|
||||
def catalog_label(name: str, code: str, language: str) -> str:
|
||||
prefix = f"{name}:v1:"
|
||||
if not code.startswith(prefix):
|
||||
raise ApiError(400, f"{name} must be a catalog code")
|
||||
try:
|
||||
encoded = code[len(prefix):]
|
||||
values = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)))
|
||||
value = values[0 if language == "ru" else 1]
|
||||
except (IndexError, TypeError, ValueError, UnicodeError, json.JSONDecodeError) as error:
|
||||
raise ApiError(400, f"{name} must be a catalog code") from error
|
||||
if not isinstance(values, list) or len(values) != 2 or not isinstance(value, str):
|
||||
raise ApiError(400, f"{name} must be a catalog code")
|
||||
return value
|
||||
|
||||
|
||||
def openapi() -> dict:
|
||||
responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
|
||||
return {
|
||||
@@ -134,11 +114,13 @@ class Api:
|
||||
if sort not in {"relevance", "date"}:
|
||||
raise ApiError(400, "sort must be relevance or date")
|
||||
filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}]
|
||||
fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"}
|
||||
fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes"}
|
||||
for parameter, field in fields.items():
|
||||
value = one(query, parameter)
|
||||
if value:
|
||||
filters.append({"term": {field: catalog_label(parameter, value, language)}})
|
||||
if value not in CATALOGS[parameter]:
|
||||
raise ApiError(400, f"{parameter} must be a catalog code")
|
||||
filters.append({"term": {field: value}})
|
||||
date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
|
||||
if date_from and date_to and date_from > date_to:
|
||||
raise ApiError(400, "date_from must not be later than date_to")
|
||||
@@ -174,13 +156,13 @@ class Api:
|
||||
language = one(query, "language") or "ru"
|
||||
if language not in LANGUAGES:
|
||||
raise ApiError(400, "language must be ru or ky")
|
||||
fields = {"document_types": ("document_type", "document_type_ru", "document_type_ky"), "statuses": ("status", "status_ru", "status_ky"), "authorities": ("authority", "authority_paths_ru", "authority_paths_ky")}
|
||||
body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair[1:]], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
|
||||
fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes")}
|
||||
body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
|
||||
response = self.query_opensearch(body)
|
||||
try:
|
||||
aggregations = response["aggregations"]
|
||||
values = {
|
||||
name: [{"code": catalog_code(pair[0], (item["key"][0], item["key"][1])), "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
|
||||
name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
|
||||
for name, pair in fields.items()
|
||||
}
|
||||
except (KeyError, TypeError) as error:
|
||||
|
||||
51
backend/search/catalog.py
Normal file
51
backend/search/catalog.py
Normal file
@@ -0,0 +1,51 @@
|
||||
"""Immutable v1 search catalogs."""
|
||||
|
||||
DOCUMENT_TYPES = {
|
||||
"constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"),
|
||||
}
|
||||
STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")}
|
||||
AUTHORITIES = {"president": ("Президент", "Президент"), "parliament": ("Органы законодательной власти", "Мыйзам чыгаруу бийлик органдары"), "cabinet": ("Правительство и Кабинет Министров", "Өкмөт жана Министрлер Кабинети"), "ministries_and_committees": ("Министерства и государственные комитеты", "Министрликтер жана мамлекеттик комитеттер"), "administrative_agencies": ("Административные ведомства", "Административдик ведомстволор"), "national_bank": ("Национальный банк", "Улуттук банк"), "other_state_bodies": ("Иные государственные органы", "Башка мамлекеттик органдар"), "local_representative_bodies": ("Представительные органы местного самоуправления", "Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары"), "other": ("Прочие органы", "Башка органдар")}
|
||||
CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES}
|
||||
|
||||
|
||||
def labels(category: str, code: str) -> dict[str, str]:
|
||||
try:
|
||||
ru, ky = CATALOGS[category][code]
|
||||
except KeyError as error:
|
||||
raise ValueError(f"Unknown {category} catalog code: {code}") from error
|
||||
return {"ru": ru, "ky": ky}
|
||||
|
||||
|
||||
def source_code(category: str, value: dict | None) -> str:
|
||||
pair = ((value or {}).get("ru"), (value or {}).get("ky"))
|
||||
if pair == (None, None):
|
||||
return "unspecified"
|
||||
for code, expected in CATALOGS[category].items():
|
||||
if pair == expected or category == "document_type" and code == "provision" and pair == ("Положение", "Положение"):
|
||||
return code
|
||||
raise ValueError(f"Unmapped {category} catalog value: {pair!r}")
|
||||
|
||||
|
||||
def authority_codes(paths: list[dict]) -> list[str]:
|
||||
codes = set()
|
||||
for path in paths:
|
||||
text = " ".join(path.get("ru", []) + path.get("ky", [])).lower()
|
||||
if "президент" in text:
|
||||
codes.add("president")
|
||||
elif "жогорку кенеш" in text or "верховный совет" in text or "мыйзам чыгаруу" in text:
|
||||
codes.add("parliament")
|
||||
elif "кабинет министров" in text or "правительство" in text or "өкмөт" in text:
|
||||
codes.add("cabinet")
|
||||
elif "министер" in text or "мамлекеттик комитет" in text:
|
||||
codes.add("ministries_and_committees")
|
||||
elif "административ" in text:
|
||||
codes.add("administrative_agencies")
|
||||
elif "национальн" in text and "банк" in text or "улуттук банк" in text:
|
||||
codes.add("national_bank")
|
||||
elif "кенеш" in text or "кеңеш" in text or "айыл" in text or "местного самоуправления" in text:
|
||||
codes.add("local_representative_bodies")
|
||||
elif "иные государственные" in text or "башка мамлекеттик" in text:
|
||||
codes.add("other_state_bodies")
|
||||
else:
|
||||
codes.add("other")
|
||||
return sorted(codes) or ["other"]
|
||||
@@ -22,12 +22,15 @@
|
||||
"document_name_ky": { "type": "text", "analyzer": "icu_analyzer", "fields": { "keyword": { "type": "keyword", "ignore_above": 1024 } } },
|
||||
"document_type_ru": { "type": "keyword" },
|
||||
"document_type_ky": { "type": "keyword" },
|
||||
"document_type_code": { "type": "keyword" },
|
||||
"status_ru": { "type": "keyword" },
|
||||
"status_ky": { "type": "keyword" },
|
||||
"status_code": { "type": "keyword" },
|
||||
"number": { "type": "keyword" },
|
||||
"date_adopted": { "type": "date", "format": "strict_date" },
|
||||
"authority_paths_ru": { "type": "keyword", "ignore_above": 2048 },
|
||||
"authority_paths_ky": { "type": "keyword", "ignore_above": 2048 },
|
||||
"authority_codes": { "type": "keyword" },
|
||||
"source_path": { "type": "keyword", "index": false },
|
||||
"source_sha256": { "type": "keyword", "index": false },
|
||||
"text_sha256": { "type": "keyword", "index": false }
|
||||
|
||||
@@ -15,6 +15,8 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
from search.catalog import authority_codes, source_code
|
||||
|
||||
APP_VERSION = "0.7.1"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
|
||||
@@ -73,12 +75,15 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st
|
||||
"document_name_ky": localized(document.get("name"), "ky"),
|
||||
"document_type_ru": localized(document.get("type"), "ru"),
|
||||
"document_type_ky": localized(document.get("type"), "ky"),
|
||||
"document_type_code": source_code("document_type", document.get("type")),
|
||||
"status_ru": localized(document.get("status"), "ru"),
|
||||
"status_ky": localized(document.get("status"), "ky"),
|
||||
"status_code": source_code("status", document.get("status")),
|
||||
"number": document.get("number"),
|
||||
"date_adopted": dates.get("DateAdopted"),
|
||||
"authority_paths_ru": paths(document, "authority_paths", "ru"),
|
||||
"authority_paths_ky": paths(document, "authority_paths", "ky"),
|
||||
"authority_codes": authority_codes(document.get("authority_paths", [])),
|
||||
"source_path": fragment["source_path"],
|
||||
"source_sha256": fragment["source_sha256"],
|
||||
"text_sha256": fragment["text_sha256"],
|
||||
|
||||
Reference in New Issue
Block a user