From 73ff287b2041a0b130fd90deb71c0b96756adb19 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Fri, 21 Aug 2026 06:12:47 +0300 Subject: [PATCH 1/7] feat: add search API contract --- README.md | 8 +- backend/README.md | 19 +- backend/ingestion/minjust_cbd.py | 2 +- backend/normalization/minjust_cbd.py | 2 +- backend/search/api.py | 278 ++++++++++++++++++ backend/search/minjust_opensearch.py | 2 +- backend/test_search_api.py | 52 ++++ docs/README.md | 2 +- docs/decisions/001-telegram-workspace-mvp.md | 2 +- docs/operations/project-status.md | 4 +- docs/operations/telegram-workspace-plan.md | 2 +- docs/product/frontend-search-sps-plan.md | 2 +- ...njust-document-normalization-agent-task.md | 2 +- docs/product/project-overview.md | 2 +- docs/team/ai-skills-for-beginners.md | 2 +- tools/telegram-bot/README.md | 2 +- 16 files changed, 364 insertions(+), 19 deletions(-) create mode 100644 backend/search/api.py create mode 100644 backend/test_search_api.py diff --git a/README.md b/README.md index 5edadb6..845d025 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения ## Текущее состояние Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии -`0.6.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную +`0.7.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную размеченном наборе запросов. | Компонент | Версия | Состояние | |---|---:|---| | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния | -| Backend | `0.6.0` | добавлено атомарное переключение alias поискового индекса | +| Backend | `0.7.0` | добавлено атомарное переключение alias поискового индекса | | Frontend | — | ещё не создан | -| Сбор и обработка правовых данных | `0.6.0` | добавлены relevance set и baseline-метрики | +| Сбор и обработка правовых данных | `0.7.0` | добавлены relevance set и baseline-метрики | | RAG и база знаний | — | ещё не созданы | ## Структура репозитория @@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/backend/README.md b/backend/README.md index fb45519..0873639 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # Backend Акылдаш -Версия: `0.6.0` +Версия: `0.7.0` Первая backend-область проекта — загрузка правовых документов из официального Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в @@ -160,6 +160,21 @@ python3 backend/search/minjust_opensearch.py \ версионный индекс, проверить его и переключить alias. Это не оставляет удалённые trailing-фрагменты старых документов. +## HTTP API v1 + +Запустите публичный read-only API поверх текущего alias и нормализованного +корпуса: + +```bash +PYTHONPATH=backend python3 -m search.api +``` + +Он публикует OpenAPI в `GET /openapi.json` и поддерживает `GET /search`, +`/search/filters`, `/documents/{code}`, `/documents/{code}/editions` и +`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются с +каноническим значением ЦБД и подписью выбранного языка; применяйте `code` как +параметр поиска. API не подменяет отсутствующий язык документа. + ## Локальный OpenSearch Стенд использует один узел OpenSearch без Dashboards, устанавливает @@ -210,4 +225,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la --- -Акылдаш · Backend v0.6.0 · Frontend — не создан +Акылдаш · Backend v0.7.0 · Frontend — не создан diff --git a/backend/ingestion/minjust_cbd.py b/backend/ingestion/minjust_cbd.py index 1cf8d26..82589f0 100644 --- a/backend/ingestion/minjust_cbd.py +++ b/backend/ingestion/minjust_cbd.py @@ -21,7 +21,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable, Iterable -APP_VERSION = "0.6.0" +APP_VERSION = "0.7.0" API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/" LANGUAGES = {"Rus": "ru", "Kyr": "ky"} IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"} diff --git a/backend/normalization/minjust_cbd.py b/backend/normalization/minjust_cbd.py index 8ebd579..b62237c 100644 --- a/backend/normalization/minjust_cbd.py +++ b/backend/normalization/minjust_cbd.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Callable from urllib.parse import urlsplit -APP_VERSION = "0.6.0" +APP_VERSION = "0.7.0" SCHEMA_VERSION = "1" NORMALIZER_VERSION = "1.0.0" LANGUAGES = ("ru", "ky") diff --git a/backend/search/api.py b/backend/search/api.py new file mode 100644 index 0000000..1be21b2 --- /dev/null +++ b/backend/search/api.py @@ -0,0 +1,278 @@ +"""Minimal HTTP API for the normalized legal-document corpus.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import re +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from search.minjust_opensearch import APP_VERSION, request_json + + +API_VERSION = "v1" +LANGUAGES = {"ru", "ky"} +CODE = re.compile(r"^[0-9]+$") +MAX_PAGE_SIZE = 100 + + +class ApiError(Exception): + def __init__(self, status: int, message: str): + self.status = status + self.message = message + + +def parse_positive(value: str | None, name: str, default: int, maximum: int) -> int: + if value is None: + return default + try: + parsed = int(value) + except ValueError as error: + raise ApiError(400, f"{name} must be an integer") from error + if not 1 <= parsed <= maximum: + raise ApiError(400, f"{name} must be between 1 and {maximum}") + return parsed + + +def one(query: dict[str, list[str]], name: str) -> str | None: + values = query.get(name, []) + if len(values) > 1: + raise ApiError(400, f"{name} must be specified once") + return values[0] if values else None + + +def date(value: str | None, name: str) -> str | None: + if value is None: + return None + try: + datetime.date.fromisoformat(value) + except ValueError as error: + raise ApiError(400, f"{name} must be an ISO date") from error + return value + + +def openapi() -> dict: + return { + "openapi": "3.0.3", + "info": {"title": "Akyldash Search API", "version": API_VERSION}, + "paths": { + "/search": {"get": {"parameters": [ + {"name": "q", "in": "query", "required": True, "schema": {"type": "string"}}, + {"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}}, + {"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}}, + {"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": MAX_PAGE_SIZE}}, + {"name": "document_type", "in": "query", "schema": {"type": "string"}}, + {"name": "status", "in": "query", "schema": {"type": "string"}}, + {"name": "authority", "in": "query", "schema": {"type": "string"}}, + {"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}}, + {"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}}, + {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, + ]}}, + "/search/filters": {"get": {}}, + "/documents/{code}": {"get": {}}, + "/documents/{code}/editions": {"get": {}}, + "/documents/{code}/editions/{edition}": {"get": {}}, + }, + } + + +class Api: + def __init__(self, base_url: str, index: str, data_root: Path): + self.base_url = base_url.rstrip("/") + self.index = index + self.data_root = data_root + + def search_url(self, suffix: str) -> str: + return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}" + + def query_opensearch(self, body: dict) -> dict: + try: + return request_json(self.search_url("_search"), "POST", json.dumps(body, ensure_ascii=False).encode(), "application/json") + except RuntimeError as error: + raise ApiError(502, "search backend is unavailable") from error + + def search(self, query: dict[str, list[str]]) -> dict: + text = one(query, "q") + if not text or not text.strip(): + raise ApiError(400, "q is required") + if len(text) > 500: + raise ApiError(400, "q must not exceed 500 characters") + language = one(query, "language") or "ru" + if language not in LANGUAGES: + raise ApiError(400, "language must be ru or ky") + page = parse_positive(one(query, "page"), "page", 1, 1_000_000) + page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE) + sort = one(query, "sort") or "relevance" + if sort not in {"relevance", "date"}: + raise ApiError(400, "sort must be relevance or date") + filters: list[dict] = [{"term": {"language": language}}] + fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"} + for parameter, field in fields.items(): + value = one(query, parameter) + if value: + filters.append({"term": {field: value}}) + date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to") + if date_from and date_to and date_from > date_to: + raise ApiError(400, "date_from must not be later than date_to") + if date_from or date_to: + date_range = {key: value for key, value in (("gte", date_from), ("lte", date_to)) if value} + filters.append({"range": {"date_adopted": date_range}}) + body = { + "from": (page - 1) * page_size, + "size": page_size + 1, + "_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "date_adopted", "number"], + "query": {"bool": {"filter": filters, "must": {"multi_match": {"query": text, "fields": [f"document_name_{language}", f"text_{language}"], "type": "cross_fields"}}}}, + "collapse": {"field": "document_code"}, + "highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}}, + } + if sort == "date": + body["sort"] = [{"date_adopted": "desc"}, {"_score": "desc"}] + response = self.query_opensearch(body) + try: + hits = response["hits"]["hits"] + except (KeyError, TypeError) as error: + raise ApiError(502, "search backend returned an incomplete response") from error + return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": [self.search_hit(hit, language) for hit in hits[:page_size]]} + + @staticmethod + def search_hit(hit: dict, language: str) -> dict: + source = hit.get("_source") + if not isinstance(source, dict) or not source.get("document_code"): + raise ApiError(502, "search backend returned an incomplete result") + highlight = hit.get("highlight", {}).get(f"text_{language}", []) + return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": highlight[0] if highlight else None} + + def filters(self, query: dict[str, list[str]]) -> dict: + language = one(query, "language") or "ru" + if language not in LANGUAGES: + raise ApiError(400, "language must be ru or ky") + fields = {"document_types": f"document_type_{language}", "statuses": f"status_{language}", "authorities": f"authority_paths_{language}"} + body = {"size": 0, "aggs": {name: {"terms": {"field": field, "size": 1000}} for name, field in fields.items()}} + response = self.query_opensearch(body) + try: + aggregations = response["aggregations"] + values = { + name: [{"code": item["key"], "label": item["key"], "count": item["doc_count"]} for item in aggregations[name]["buckets"]] + for name in fields + } + except (KeyError, TypeError) as error: + raise ApiError(502, "search backend returned incomplete filters") from error + return {"api_version": API_VERSION, "language": language, **values} + + def directory(self, code: str) -> Path: + if not CODE.fullmatch(code): + raise ApiError(404, "document not found") + path = self.data_root / "documents" / code + if not path.is_dir(): + raise ApiError(404, "document not found") + return path + + @staticmethod + def read_json(path: Path, message: str) -> dict: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ApiError(500, message) from error + if not isinstance(value, dict): + raise ApiError(500, message) + return value + + def document(self, code: str) -> dict: + document = self.read_json(self.directory(code) / "document.json", "document data is unavailable") + editions = document.get("editions") + if not isinstance(editions, list): + raise ApiError(500, "document data is unavailable") + return {"api_version": API_VERSION, "document": document, "current_edition": editions[-1] if editions else None} + + def editions(self, code: str) -> dict: + document = self.read_json(self.directory(code) / "document.json", "document data is unavailable") + return {"api_version": API_VERSION, "code": code, "available_languages": document.get("available_languages", []), "editions": document.get("editions", [])} + + def edition(self, code: str, edition: str, query: dict[str, list[str]]) -> dict: + if not CODE.fullmatch(edition): + raise ApiError(404, "edition not found") + directory = self.directory(code) / "editions" / edition + if not directory.is_dir(): + raise ApiError(404, "edition not found") + metadata = self.read_json(directory / "edition.json", "edition data is unavailable") + language = one(query, "language") + if language is not None and language not in LANGUAGES: + raise ApiError(400, "language must be ru or ky") + languages = [language] if language else metadata.get("available_languages", []) + content = {} + for item in languages: + if item not in metadata.get("available_languages", []): + continue + try: + content[item] = {"html": (directory / item / "content.html").read_text(encoding="utf-8"), "text": (directory / item / "content.txt").read_text(encoding="utf-8")} + except (OSError, UnicodeError) as error: + raise ApiError(500, "edition content is unavailable") from error + if language and language not in content: + raise ApiError(404, "edition language not found") + return {"api_version": API_VERSION, "edition": metadata, "content": content} + + def handle(self, method: str, path: str) -> tuple[int, dict]: + if method != "GET": + raise ApiError(405, "method not allowed") + parsed = urllib.parse.urlsplit(path) + query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] + if parts == ["openapi.json"]: + return 200, openapi() + if parts == ["search"]: + return 200, self.search(query) + if parts == ["search", "filters"]: + return 200, self.filters(query) + if len(parts) == 2 and parts[0] == "documents": + return 200, self.document(parts[1]) + if len(parts) == 3 and parts[:1] == ["documents"] and parts[2] == "editions": + return 200, self.editions(parts[1]) + if len(parts) == 4 and parts[:1] == ["documents"] and parts[2] == "editions": + return 200, self.edition(parts[1], parts[3], query) + raise ApiError(404, "endpoint not found") + + +def handler(api: Api): + class RequestHandler(BaseHTTPRequestHandler): + def respond(self, method: str): + try: + status, payload = api.handle(method, self.path) + except ApiError as error: + status, payload = error.status, {"api_version": API_VERSION, "error": error.message} + body = json.dumps(payload, ensure_ascii=False).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.respond("GET") + + def do_POST(self): + self.respond("POST") + + def log_message(self, format: str, *args): + return + + return RequestHandler + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:9200") + parser.add_argument("--index", default="akyldash-fragments-current") + parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized")) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8080) + parser.add_argument("--version", action="version", version=APP_VERSION) + arguments = parser.parse_args() + ThreadingHTTPServer((arguments.host, arguments.port), handler(Api(arguments.url, arguments.index, arguments.data))).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index 466f97a..aa1f735 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -15,7 +15,7 @@ import urllib.request from pathlib import Path from typing import Iterator -APP_VERSION = "0.6.0" +APP_VERSION = "0.7.0" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") diff --git a/backend/test_search_api.py b/backend/test_search_api.py new file mode 100644 index 0000000..8800bef --- /dev/null +++ b/backend/test_search_api.py @@ -0,0 +1,52 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from search.api import Api, ApiError + + +class SearchApiTest(unittest.TestCase): + def make_api(self, root: Path) -> Api: + document = root / "documents/7" + edition = document / "editions/10" + edition.mkdir(parents=True) + (document / "document.json").write_text(json.dumps({ + "source_code": "7", "available_languages": ["ru"], + "editions": [{"source_code": "10", "available_languages": ["ru"]},], + }), encoding="utf-8") + (edition / "edition.json").write_text(json.dumps({"source_code": "10", "available_languages": ["ru"]}), encoding="utf-8") + (edition / "ru").mkdir() + (edition / "ru/content.html").write_text("

Текст

", encoding="utf-8") + (edition / "ru/content.txt").write_text("Текст\n", encoding="utf-8") + return Api("http://opensearch:9200", "current", root) + + def test_search_pagination_filters_and_highlight(self): + with tempfile.TemporaryDirectory() as temporary: + api = self.make_api(Path(temporary)) + response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["Закон"]}}]}} + with patch("search.api.request_json", return_value=response) as request: + status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=%D0%94%D0%B5%D0%B9%D1%81%D1%82%D0%B2%D1%83%D0%B5%D1%82") + self.assertEqual(status, 200) + self.assertEqual(payload["results"][0]["snippet"], "Закон") + body = json.loads(request.call_args.args[2]) + self.assertEqual((body["from"], body["size"]), (5, 6)) + self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"]) + + def test_document_editions_openapi_and_validation(self): + with tempfile.TemporaryDirectory() as temporary: + api = self.make_api(Path(temporary)) + self.assertEqual(api.handle("GET", "/openapi.json")[1]["info"]["version"], "v1") + self.assertEqual(api.handle("GET", "/documents/7")[1]["current_edition"]["source_code"], "10") + self.assertEqual(api.handle("GET", "/documents/7/editions/10?language=ru")[1]["content"]["ru"]["text"], "Текст\n") + with self.assertRaisesRegex(ApiError, "q is required"): + api.handle("GET", "/search") + with self.assertRaisesRegex(ApiError, "date_from must be an ISO date"): + api.handle("GET", "/search?q=x&date_from=tomorrow") + with self.assertRaisesRegex(ApiError, "document not found"): + api.handle("GET", "/documents/%2E%2E") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/README.md b/docs/README.md index b3fbba1..6c2c0d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,4 +35,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/decisions/001-telegram-workspace-mvp.md b/docs/decisions/001-telegram-workspace-mvp.md index 2acba6d..90154ab 100644 --- a/docs/decisions/001-telegram-workspace-mvp.md +++ b/docs/decisions/001-telegram-workspace-mvp.md @@ -78,4 +78,4 @@ Telegram позволяет запретить пользователям отп --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/operations/project-status.md b/docs/operations/project-status.md index df80cd2..ef600a4 100644 --- a/docs/operations/project-status.md +++ b/docs/operations/project-status.md @@ -5,7 +5,7 @@ - Telegram-бот: `0.2.2` - Telegram-бот на Synology: `0.2.1` -- Backend: `0.6.0` +- Backend: `0.7.0` - Frontend: не создан ## Краткий итог @@ -234,4 +234,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/operations/telegram-workspace-plan.md b/docs/operations/telegram-workspace-plan.md index 3852ddd..f20fe80 100644 --- a/docs/operations/telegram-workspace-plan.md +++ b/docs/operations/telegram-workspace-plan.md @@ -398,4 +398,4 @@ Git сохраняет актуальную версию --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/product/frontend-search-sps-plan.md b/docs/product/frontend-search-sps-plan.md index beeece2..3c37a6a 100644 --- a/docs/product/frontend-search-sps-plan.md +++ b/docs/product/frontend-search-sps-plan.md @@ -258,4 +258,4 @@ runtime-зависимостями frontend. Регистрация в стор --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/product/minjust-document-normalization-agent-task.md b/docs/product/minjust-document-normalization-agent-task.md index dfc0246..ec4edd6 100644 --- a/docs/product/minjust-document-normalization-agent-task.md +++ b/docs/product/minjust-document-normalization-agent-task.md @@ -214,4 +214,4 @@ python3 backend/normalization/minjust_cbd.py --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/product/project-overview.md b/docs/product/project-overview.md index 10c7fa8..0971a9c 100644 --- a/docs/product/project-overview.md +++ b/docs/product/project-overview.md @@ -44,4 +44,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/docs/team/ai-skills-for-beginners.md b/docs/team/ai-skills-for-beginners.md index 6770db1..b9cdc5a 100644 --- a/docs/team/ai-skills-for-beginners.md +++ b/docs/team/ai-skills-for-beginners.md @@ -180,4 +180,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан diff --git a/tools/telegram-bot/README.md b/tools/telegram-bot/README.md index 2d65772..10148f1 100644 --- a/tools/telegram-bot/README.md +++ b/tools/telegram-bot/README.md @@ -48,4 +48,4 @@ python3 -m unittest -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан From 0bef90debdca94ccd8420dfdaa95763a9e58d1e3 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Fri, 21 Aug 2026 07:24:16 +0300 Subject: [PATCH 2/7] fix: restrict search to current editions --- backend/search/api.py | 19 ++++++++++--------- backend/search/minjust-fragments-index.json | 1 + backend/search/minjust_opensearch.py | 10 ++++++++-- backend/test_minjust_opensearch.py | 1 + backend/test_search_api.py | 10 ++++++++++ 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/backend/search/api.py b/backend/search/api.py index 1be21b2..650228a 100644 --- a/backend/search/api.py +++ b/backend/search/api.py @@ -55,11 +55,12 @@ def date(value: str | None, name: str) -> str | None: def openapi() -> dict: + responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}} return { "openapi": "3.0.3", "info": {"title": "Akyldash Search API", "version": API_VERSION}, "paths": { - "/search": {"get": {"parameters": [ + "/search": {"get": {"responses": responses, "parameters": [ {"name": "q", "in": "query", "required": True, "schema": {"type": "string"}}, {"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}}, {"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}}, @@ -71,10 +72,10 @@ def openapi() -> dict: {"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}}, {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, ]}}, - "/search/filters": {"get": {}}, - "/documents/{code}": {"get": {}}, - "/documents/{code}/editions": {"get": {}}, - "/documents/{code}/editions/{edition}": {"get": {}}, + "/search/filters": {"get": {"responses": responses}}, + "/documents/{code}": {"get": {"responses": responses}}, + "/documents/{code}/editions": {"get": {"responses": responses}}, + "/documents/{code}/editions/{edition}": {"get": {"responses": responses}}, }, } @@ -108,7 +109,7 @@ class Api: sort = one(query, "sort") or "relevance" if sort not in {"relevance", "date"}: raise ApiError(400, "sort must be relevance or date") - filters: list[dict] = [{"term": {"language": language}}] + filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}] fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"} for parameter, field in fields.items(): value = one(query, parameter) @@ -149,13 +150,13 @@ class Api: language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") - fields = {"document_types": f"document_type_{language}", "statuses": f"status_{language}", "authorities": f"authority_paths_{language}"} - body = {"size": 0, "aggs": {name: {"terms": {"field": field, "size": 1000}} for name, field in fields.items()}} + fields = {"document_types": ("document_type_ru", "document_type_ky"), "statuses": ("status_ru", "status_ky"), "authorities": ("authority_paths_ru", "authority_paths_ky")} + body = {"size": 0, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} response = self.query_opensearch(body) try: aggregations = response["aggregations"] values = { - name: [{"code": item["key"], "label": item["key"], "count": item["doc_count"]} for item in aggregations[name]["buckets"]] + name: [{"code": item["key"][0 if language == "ru" else 1], "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] for name in fields } except (KeyError, TypeError) as error: diff --git a/backend/search/minjust-fragments-index.json b/backend/search/minjust-fragments-index.json index daa9381..3cb2653 100644 --- a/backend/search/minjust-fragments-index.json +++ b/backend/search/minjust-fragments-index.json @@ -12,6 +12,7 @@ "schema_version": { "type": "keyword" }, "document_code": { "type": "keyword" }, "edition_code": { "type": "keyword" }, + "is_current_edition": { "type": "boolean" }, "language": { "type": "keyword" }, "position": { "type": "integer" }, "fragment_type": { "type": "keyword" }, diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index aa1f735..15fe14e 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -42,7 +42,7 @@ def paths(document: dict, field: str, language: str) -> list[str]: return [" > ".join(item[language]) for item in document.get(field, []) if item.get(language)] -def search_document(document: dict, fragment: dict, expected: tuple[str, str, str, int]) -> dict: +def search_document(document: dict, fragment: dict, expected: tuple[str, str, str, int], current_edition: str) -> dict: document_code, edition_code, language, position = expected fragment_id = f"document:{document_code}:edition:{edition_code}:lang:{language}:fragment:{position}" required = ("id", "document_code", "edition_code", "language", "position", "type", "text", "text_sha256", "source_path", "source_sha256") @@ -64,6 +64,7 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st "schema_version": document["schema_version"], "document_code": document_code, "edition_code": edition_code, + "is_current_edition": edition_code == current_edition, "language": language, "position": position, "fragment_type": fragment["type"], @@ -118,13 +119,18 @@ def bulk_pairs( document = read_json(directory / "document.json") if document.get("source_code") != directory.name: raise ValueError(f"Document identity does not match its path: {directory}") + edition_root = directory / "editions" + editions = [path.name for path in edition_root.iterdir() if path.is_dir()] if edition_root.is_dir() else [] + if not editions: + continue + current_edition = max(editions, key=lambda value: (not value.isdigit(), int(value) if value.isdigit() else value)) for fragment_path in sorted(directory.glob("editions/*/*/fragments.json")): edition_code, language = fragment_path.parts[-3:-1] values = read_json(fragment_path) if not isinstance(values, list): raise ValueError(f"Fragments must be a list: {fragment_path}") for position, fragment in enumerate(values, 1): - source = search_document(document, fragment, (directory.name, edition_code, language, position)) + source = search_document(document, fragment, (directory.name, edition_code, language, position), current_edition) action = json.dumps({"index": {"_index": index, "_id": fragment["id"]}}, ensure_ascii=False, allow_nan=False) body = json.dumps(source, ensure_ascii=False, allow_nan=False) yield directory.name, f"{action}\n{body}\n".encode() diff --git a/backend/test_minjust_opensearch.py b/backend/test_minjust_opensearch.py index cf02ac8..a9e897d 100644 --- a/backend/test_minjust_opensearch.py +++ b/backend/test_minjust_opensearch.py @@ -91,6 +91,7 @@ class MinjustOpenSearchTest(unittest.TestCase): self.assertEqual(len(lines), 4) self.assertEqual(lines[0]["index"]["_id"], "document:7:edition:10:lang:ky:fragment:1") self.assertIn("text_ky", lines[1]) + self.assertTrue(lines[1]["is_current_edition"]) self.assertNotIn("text_ru", lines[1]) self.assertEqual(lines[3]["text_ru"], "Текст \"RU\"\nстрока") self.assertEqual( diff --git a/backend/test_search_api.py b/backend/test_search_api.py index 8800bef..a4ac6eb 100644 --- a/backend/test_search_api.py +++ b/backend/test_search_api.py @@ -47,6 +47,16 @@ class SearchApiTest(unittest.TestCase): with self.assertRaisesRegex(ApiError, "document not found"): api.handle("GET", "/documents/%2E%2E") + def test_filters_count_documents_and_return_bilingual_labels(self): + with tempfile.TemporaryDirectory() as temporary: + api = self.make_api(Path(temporary)) + response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}} + with patch("search.api.request_json", return_value=response) as request: + payload = api.handle("GET", "/search/filters?language=ky")[1] + self.assertEqual(payload["document_types"][0], {"code": "Мыйзам", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) + body = json.loads(request.call_args.args[2]) + self.assertIn("multi_terms", body["aggs"]["document_types"]) + if __name__ == "__main__": unittest.main() From 9b629d93e3c1161c69b54ba96cbc8f9358d1c96d Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Tue, 25 Aug 2026 07:29:06 +0300 Subject: [PATCH 3/7] fix: complete search API contract --- README.md | 8 +++--- backend/README.md | 8 +++--- backend/ingestion/minjust_cbd.py | 2 +- backend/normalization/minjust_cbd.py | 2 +- backend/search/api.py | 40 +++++++++++++++++++++------ backend/search/minjust_opensearch.py | 2 +- backend/test_search_api.py | 14 +++++++--- backend/test_search_api_opensearch.py | 37 +++++++++++++++++++++++++ 8 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 backend/test_search_api_opensearch.py diff --git a/README.md b/README.md index 845d025..2b94a77 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения ## Текущее состояние Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии -`0.7.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную +`0.7.1`: исправления контракта Search API v1 и его ограничений OpenSearch. размеченном наборе запросов. | Компонент | Версия | Состояние | |---|---:|---| | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния | -| Backend | `0.7.0` | добавлено атомарное переключение alias поискового индекса | +| Backend | `0.7.1` | исправлен контракт Search API v1 | | Frontend | — | ещё не создан | -| Сбор и обработка правовых данных | `0.7.0` | добавлены relevance set и baseline-метрики | +| Сбор и обработка правовых данных | `0.7.1` | добавлены relevance set и baseline-метрики | | RAG и база знаний | — | ещё не созданы | ## Структура репозитория @@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/backend/README.md b/backend/README.md index 0873639..36950aa 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # Backend Акылдаш -Версия: `0.7.0` +Версия: `0.7.1` Первая backend-область проекта — загрузка правовых документов из официального Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в @@ -171,8 +171,8 @@ PYTHONPATH=backend python3 -m search.api Он публикует OpenAPI в `GET /openapi.json` и поддерживает `GET /search`, `/search/filters`, `/documents/{code}`, `/documents/{code}/editions` и -`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются с -каноническим значением ЦБД и подписью выбранного языка; применяйте `code` как +`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются со +стабильным кодом справочника v1 и подписями RU/KY; применяйте `code` как параметр поиска. API не подменяет отсутствующий язык документа. ## Локальный OpenSearch @@ -225,4 +225,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la --- -Акылдаш · Backend v0.7.0 · Frontend — не создан +Акылдаш · Backend v0.7.1 · Frontend — не создан diff --git a/backend/ingestion/minjust_cbd.py b/backend/ingestion/minjust_cbd.py index 82589f0..684e2cf 100644 --- a/backend/ingestion/minjust_cbd.py +++ b/backend/ingestion/minjust_cbd.py @@ -21,7 +21,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable, Iterable -APP_VERSION = "0.7.0" +APP_VERSION = "0.7.1" API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/" LANGUAGES = {"Rus": "ru", "Kyr": "ky"} IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"} diff --git a/backend/normalization/minjust_cbd.py b/backend/normalization/minjust_cbd.py index b62237c..776ffe4 100644 --- a/backend/normalization/minjust_cbd.py +++ b/backend/normalization/minjust_cbd.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Callable from urllib.parse import urlsplit -APP_VERSION = "0.7.0" +APP_VERSION = "0.7.1" SCHEMA_VERSION = "1" NORMALIZER_VERSION = "1.0.0" LANGUAGES = ("ru", "ky") diff --git a/backend/search/api.py b/backend/search/api.py index 650228a..2497af3 100644 --- a/backend/search/api.py +++ b/backend/search/api.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import base64 import datetime import json import re @@ -17,6 +18,7 @@ API_VERSION = "v1" LANGUAGES = {"ru", "ky"} CODE = re.compile(r"^[0-9]+$") MAX_PAGE_SIZE = 100 +MAX_RESULT_WINDOW = 10_000 class ApiError(Exception): @@ -54,6 +56,26 @@ def date(value: str | None, name: str) -> str | None: return value +def catalog_code(name: str, labels: tuple[str | None, str | None]) -> str: + value = json.dumps(labels, ensure_ascii=False, separators=(",", ":")).encode() + return f"{name}:v1:{base64.urlsafe_b64encode(value).decode().rstrip('=')}" + + +def catalog_label(name: str, code: str, language: str) -> str: + prefix = f"{name}:v1:" + if not code.startswith(prefix): + raise ApiError(400, f"{name} must be a catalog code") + try: + encoded = code[len(prefix):] + values = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))) + value = values[0 if language == "ru" else 1] + except (IndexError, TypeError, ValueError, UnicodeError, json.JSONDecodeError) as error: + raise ApiError(400, f"{name} must be a catalog code") from error + if not isinstance(values, list) or len(values) != 2 or not isinstance(value, str): + raise ApiError(400, f"{name} must be a catalog code") + return value + + def openapi() -> dict: responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}} return { @@ -73,9 +95,9 @@ def openapi() -> dict: {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, ]}}, "/search/filters": {"get": {"responses": responses}}, - "/documents/{code}": {"get": {"responses": responses}}, - "/documents/{code}/editions": {"get": {"responses": responses}}, - "/documents/{code}/editions/{edition}": {"get": {"responses": responses}}, + "/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, + "/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, + "/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, }, } @@ -106,6 +128,8 @@ class Api: raise ApiError(400, "language must be ru or ky") page = parse_positive(one(query, "page"), "page", 1, 1_000_000) page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE) + if page * page_size >= MAX_RESULT_WINDOW: + raise ApiError(400, f"page and page_size must stay within {MAX_RESULT_WINDOW} results") sort = one(query, "sort") or "relevance" if sort not in {"relevance", "date"}: raise ApiError(400, "sort must be relevance or date") @@ -114,7 +138,7 @@ class Api: for parameter, field in fields.items(): value = one(query, parameter) if value: - filters.append({"term": {field: value}}) + filters.append({"term": {field: catalog_label(parameter, value, language)}}) date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to") if date_from and date_to and date_from > date_to: raise ApiError(400, "date_from must not be later than date_to") @@ -150,14 +174,14 @@ class Api: language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") - fields = {"document_types": ("document_type_ru", "document_type_ky"), "statuses": ("status_ru", "status_ky"), "authorities": ("authority_paths_ru", "authority_paths_ky")} - body = {"size": 0, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} + fields = {"document_types": ("document_type", "document_type_ru", "document_type_ky"), "statuses": ("status", "status_ru", "status_ky"), "authorities": ("authority", "authority_paths_ru", "authority_paths_ky")} + body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair[1:]], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} response = self.query_opensearch(body) try: aggregations = response["aggregations"] values = { - name: [{"code": item["key"][0 if language == "ru" else 1], "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] - for name in fields + name: [{"code": catalog_code(pair[0], (item["key"][0], item["key"][1])), "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] + for name, pair in fields.items() } except (KeyError, TypeError) as error: raise ApiError(502, "search backend returned incomplete filters") from error diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index 15fe14e..f5f979f 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -15,7 +15,7 @@ import urllib.request from pathlib import Path from typing import Iterator -APP_VERSION = "0.7.0" +APP_VERSION = "0.7.1" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") diff --git a/backend/test_search_api.py b/backend/test_search_api.py index a4ac6eb..bf8aeab 100644 --- a/backend/test_search_api.py +++ b/backend/test_search_api.py @@ -4,7 +4,7 @@ import unittest from pathlib import Path from unittest.mock import patch -from search.api import Api, ApiError +from search.api import Api, ApiError, catalog_code class SearchApiTest(unittest.TestCase): @@ -27,17 +27,22 @@ class SearchApiTest(unittest.TestCase): api = self.make_api(Path(temporary)) response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["Закон"]}}]}} with patch("search.api.request_json", return_value=response) as request: - status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=%D0%94%D0%B5%D0%B9%D1%81%D1%82%D0%B2%D1%83%D0%B5%D1%82") + status, payload = api.handle("GET", f"/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status={catalog_code('status', ('Действует', 'Күчүндө'))}") self.assertEqual(status, 200) self.assertEqual(payload["results"][0]["snippet"], "Закон") body = json.loads(request.call_args.args[2]) self.assertEqual((body["from"], body["size"]), (5, 6)) self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"]) + with self.assertRaisesRegex(ApiError, "within 10000 results"): + api.handle("GET", "/search?q=x&page=100&page_size=100") def test_document_editions_openapi_and_validation(self): with tempfile.TemporaryDirectory() as temporary: api = self.make_api(Path(temporary)) - self.assertEqual(api.handle("GET", "/openapi.json")[1]["info"]["version"], "v1") + specification = api.handle("GET", "/openapi.json")[1] + self.assertEqual(specification["info"]["version"], "v1") + self.assertEqual(specification["paths"]["/documents/{code}"]["get"]["parameters"][0]["required"], True) + self.assertEqual(specification["paths"]["/documents/{code}/editions/{edition}"]["get"]["parameters"][1]["name"], "edition") self.assertEqual(api.handle("GET", "/documents/7")[1]["current_edition"]["source_code"], "10") self.assertEqual(api.handle("GET", "/documents/7/editions/10?language=ru")[1]["content"]["ru"]["text"], "Текст\n") with self.assertRaisesRegex(ApiError, "q is required"): @@ -53,9 +58,10 @@ class SearchApiTest(unittest.TestCase): response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}} with patch("search.api.request_json", return_value=response) as request: payload = api.handle("GET", "/search/filters?language=ky")[1] - self.assertEqual(payload["document_types"][0], {"code": "Мыйзам", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) + self.assertEqual(payload["document_types"][0], {"code": catalog_code("document_type", ("Закон", "Мыйзам")), "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) body = json.loads(request.call_args.args[2]) self.assertIn("multi_terms", body["aggs"]["document_types"]) + self.assertEqual(body["query"], {"term": {"is_current_edition": True}}) if __name__ == "__main__": diff --git a/backend/test_search_api_opensearch.py b/backend/test_search_api_opensearch.py new file mode 100644 index 0000000..efd59b1 --- /dev/null +++ b/backend/test_search_api_opensearch.py @@ -0,0 +1,37 @@ +import json +import os +import unittest +import uuid +from pathlib import Path + +from search.api import Api, catalog_code +from search.minjust_opensearch import DEFAULT_MAPPING, request_json + + +@unittest.skipUnless(os.getenv("AKYLDASH_OPENSEARCH_URL"), "set AKYLDASH_OPENSEARCH_URL to run against local OpenSearch") +class SearchApiOpenSearchTest(unittest.TestCase): + def test_current_editions_filters_and_catalogs(self): + base_url = os.environ["AKYLDASH_OPENSEARCH_URL"].rstrip("/") + index = f"akyldash-api-test-{uuid.uuid4().hex}" + request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json") + try: + documents = [ + {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]}, + {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]}, + {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"]}, + ] + for number, document in enumerate(documents): + request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json") + request_json(f"{base_url}/{index}/_refresh", "POST", None, "application/json") + api = Api(base_url, index, Path(".")) + self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], []) + filtered = api.handle("GET", f"/search?q=needle&document_type={catalog_code('document_type', ('Закон', 'Мыйзам'))}")[1] + self.assertEqual([result["code"] for result in filtered["results"]], ["1"]) + filters = api.handle("GET", "/search/filters")[1] + self.assertEqual({item["count"] for item in filters["statuses"]}, {2}) + finally: + request_json(f"{base_url}/{index}", "DELETE", None, "application/json") + + +if __name__ == "__main__": + unittest.main() From bc3951449aa637e4002a74bdc27a5665f11b17d0 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Tue, 25 Aug 2026 07:35:13 +0300 Subject: [PATCH 4/7] docs: align backend version --- docs/README.md | 2 +- docs/decisions/001-telegram-workspace-mvp.md | 2 +- docs/operations/project-status.md | 4 ++-- docs/operations/telegram-workspace-plan.md | 2 +- docs/product/frontend-search-sps-plan.md | 2 +- docs/product/minjust-document-normalization-agent-task.md | 2 +- docs/product/project-overview.md | 2 +- docs/team/ai-skills-for-beginners.md | 2 +- tools/telegram-bot/README.md | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/README.md b/docs/README.md index 6c2c0d2..1fa1b81 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,4 +35,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/decisions/001-telegram-workspace-mvp.md b/docs/decisions/001-telegram-workspace-mvp.md index 90154ab..4c4c177 100644 --- a/docs/decisions/001-telegram-workspace-mvp.md +++ b/docs/decisions/001-telegram-workspace-mvp.md @@ -78,4 +78,4 @@ Telegram позволяет запретить пользователям отп --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/operations/project-status.md b/docs/operations/project-status.md index ef600a4..6aa411d 100644 --- a/docs/operations/project-status.md +++ b/docs/operations/project-status.md @@ -5,7 +5,7 @@ - Telegram-бот: `0.2.2` - Telegram-бот на Synology: `0.2.1` -- Backend: `0.7.0` +- Backend: `0.7.1` - Frontend: не создан ## Краткий итог @@ -234,4 +234,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/operations/telegram-workspace-plan.md b/docs/operations/telegram-workspace-plan.md index f20fe80..6972562 100644 --- a/docs/operations/telegram-workspace-plan.md +++ b/docs/operations/telegram-workspace-plan.md @@ -398,4 +398,4 @@ Git сохраняет актуальную версию --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/product/frontend-search-sps-plan.md b/docs/product/frontend-search-sps-plan.md index 3c37a6a..0196f18 100644 --- a/docs/product/frontend-search-sps-plan.md +++ b/docs/product/frontend-search-sps-plan.md @@ -258,4 +258,4 @@ runtime-зависимостями frontend. Регистрация в стор --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/product/minjust-document-normalization-agent-task.md b/docs/product/minjust-document-normalization-agent-task.md index ec4edd6..1491bd4 100644 --- a/docs/product/minjust-document-normalization-agent-task.md +++ b/docs/product/minjust-document-normalization-agent-task.md @@ -214,4 +214,4 @@ python3 backend/normalization/minjust_cbd.py --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/product/project-overview.md b/docs/product/project-overview.md index 0971a9c..8116d78 100644 --- a/docs/product/project-overview.md +++ b/docs/product/project-overview.md @@ -44,4 +44,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/docs/team/ai-skills-for-beginners.md b/docs/team/ai-skills-for-beginners.md index b9cdc5a..6318723 100644 --- a/docs/team/ai-skills-for-beginners.md +++ b/docs/team/ai-skills-for-beginners.md @@ -180,4 +180,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан diff --git a/tools/telegram-bot/README.md b/tools/telegram-bot/README.md index 10148f1..1d6acdf 100644 --- a/tools/telegram-bot/README.md +++ b/tools/telegram-bot/README.md @@ -48,4 +48,4 @@ python3 -m unittest -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан From 0ef6ad4d20a56a28df9ead4df5d60cbcab0cf5f5 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Tue, 25 Aug 2026 08:22:07 +0300 Subject: [PATCH 5/7] docs: define search catalogs v1 --- docs/product/search-catalog-v1.md | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/product/search-catalog-v1.md diff --git a/docs/product/search-catalog-v1.md b/docs/product/search-catalog-v1.md new file mode 100644 index 0000000..7d4acca --- /dev/null +++ b/docs/product/search-catalog-v1.md @@ -0,0 +1,78 @@ +# Справочники Search API v1 + +Статус: утверждённый контракт для Search API v1. + +`code` — неизменяемый идентификатор, независимый от языка. Клиент хранит и +передаёт только `code`; русское и кыргызское названия являются подписями и могут +исправляться без изменения кода. + +## Типы документов + +| Code | RU | KY | +| --- | --- | --- | +| `constitution` | Конституция | Конституция | +| `constitutional_law` | Конституционный Закон | Конституциалык Мыйзам | +| `code` | Кодекс | Кодекс | +| `law` | Закон | Мыйзам | +| `decree` | Указ | Жарлык | +| `resolution` | Постановление | Токтом | +| `order` | Распоряжение | Распоряжение | +| `instruction` | Инструкция | Инструкция | +| `rules` | Правила | Правила | +| `procedure` | Порядок | Порядок | +| `provision` | Положение | Жобо | +| `regulation` | Регламент | Регламент | +| `charter` | Устав | Жобо (Устав) | +| `program` | Программа | Программа | +| `plan` | План | План | +| `strategy` | Стратегия | Стратегия | +| `concept` | Концепция | Концепция | +| `doctrine` | Доктрина | Доктрина | +| `agreement` | Соглашение | Соглашение | +| `declaration` | Декларация | Декларация | +| `registry` | Реестр | Реестр | +| `norms` | Нормативы | Нормативы | +| `model` | Модель | Модель | +| `matrix` | Матрица | Матрица | +| `study` | Исследование | Исследование | +| `report` | Доклад | Доклад | +| `principles` | Основные принципы | Основные принципы | + +## Статусы + +| Code | RU | KY | +| --- | --- | --- | +| `active` | Действует | Күчүндө | +| `repealed` | Утратил силу | Күчүн жоготту | +| `unspecified` | Не указан | Көрсөтүлгөн эмес | + +## Органы принятия + +Орган содержит два уровня: стабильную группу и конкретный орган. В v1 группы +следующие: + +| Code | RU | KY | +| --- | --- | --- | +| `president` | Президент | Президент | +| `parliament` | Органы законодательной власти | Мыйзам чыгаруу бийлик органдары | +| `cabinet` | Правительство и Кабинет Министров | Өкмөт жана Министрлер Кабинети | +| `ministries_and_committees` | Министерства и государственные комитеты | Министрликтер жана мамлекеттик комитеттер | +| `administrative_agencies` | Административные ведомства | Административдик ведомстволор | +| `national_bank` | Национальный банк | Улуттук банк | +| `other_state_bodies` | Иные государственные органы | Башка мамлекеттик органдар | +| `local_representative_bodies` | Представительные органы местного самоуправления | Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары | +| `other` | Прочие органы | Башка органдар | + +Конкретные министерства, муниципальные и айылные кенеши не получают ID из +подписи: в выгрузке ЦБД их поле `Code` часто равно `null`. До появления +первичного неизменяемого идентификатора API v1 выдаёт и принимает только код +группы органа. Полный каталог конкретных органов — отдельная версия (`v2`), +когда источник предоставит такие идентификаторы либо будет утверждён вручную +поддерживаемый реестр. + +## Правила совместимости + +- Код из этой таблицы нельзя переиспользовать и нельзя менять его значение. +- Новое значение добавляется только новой записью; удалённое остаётся доступно + для старых документов. +- Запрос с неизвестным кодом возвращает `400`. From 2ef33c5cc3806e6f20dd6b4891efb37a3139a711 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Tue, 25 Aug 2026 08:26:48 +0300 Subject: [PATCH 6/7] fix: use stable search catalog codes --- backend/search/api.py | 34 ++++---------- backend/search/catalog.py | 51 +++++++++++++++++++++ backend/search/minjust-fragments-index.json | 3 ++ backend/search/minjust_opensearch.py | 5 ++ backend/test_search_api.py | 12 ++--- backend/test_search_api_opensearch.py | 10 ++-- 6 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 backend/search/catalog.py diff --git a/backend/search/api.py b/backend/search/api.py index 2497af3..883dd2b 100644 --- a/backend/search/api.py +++ b/backend/search/api.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse -import base64 import datetime import json import re @@ -12,6 +11,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from search.minjust_opensearch import APP_VERSION, request_json +from search.catalog import CATALOGS, labels API_VERSION = "v1" @@ -56,26 +56,6 @@ def date(value: str | None, name: str) -> str | None: return value -def catalog_code(name: str, labels: tuple[str | None, str | None]) -> str: - value = json.dumps(labels, ensure_ascii=False, separators=(",", ":")).encode() - return f"{name}:v1:{base64.urlsafe_b64encode(value).decode().rstrip('=')}" - - -def catalog_label(name: str, code: str, language: str) -> str: - prefix = f"{name}:v1:" - if not code.startswith(prefix): - raise ApiError(400, f"{name} must be a catalog code") - try: - encoded = code[len(prefix):] - values = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))) - value = values[0 if language == "ru" else 1] - except (IndexError, TypeError, ValueError, UnicodeError, json.JSONDecodeError) as error: - raise ApiError(400, f"{name} must be a catalog code") from error - if not isinstance(values, list) or len(values) != 2 or not isinstance(value, str): - raise ApiError(400, f"{name} must be a catalog code") - return value - - def openapi() -> dict: responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}} return { @@ -134,11 +114,13 @@ class Api: if sort not in {"relevance", "date"}: raise ApiError(400, "sort must be relevance or date") filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}] - fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"} + fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes"} for parameter, field in fields.items(): value = one(query, parameter) if value: - filters.append({"term": {field: catalog_label(parameter, value, language)}}) + if value not in CATALOGS[parameter]: + raise ApiError(400, f"{parameter} must be a catalog code") + filters.append({"term": {field: value}}) date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to") if date_from and date_to and date_from > date_to: raise ApiError(400, "date_from must not be later than date_to") @@ -174,13 +156,13 @@ class Api: language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") - fields = {"document_types": ("document_type", "document_type_ru", "document_type_ky"), "statuses": ("status", "status_ru", "status_ky"), "authorities": ("authority", "authority_paths_ru", "authority_paths_ky")} - body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"multi_terms": {"terms": [{"field": field} for field in pair[1:]], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} + fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes")} + body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} response = self.query_opensearch(body) try: aggregations = response["aggregations"] values = { - name: [{"code": catalog_code(pair[0], (item["key"][0], item["key"][1])), "labels": {"ru": item["key"][0], "ky": item["key"][1]}, "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] + name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] for name, pair in fields.items() } except (KeyError, TypeError) as error: diff --git a/backend/search/catalog.py b/backend/search/catalog.py new file mode 100644 index 0000000..9fcbad2 --- /dev/null +++ b/backend/search/catalog.py @@ -0,0 +1,51 @@ +"""Immutable v1 search catalogs.""" + +DOCUMENT_TYPES = { + "constitution": ("Конституция", "Конституция"), "constitutional_law": ("Конституционный Закон", "Конституциалык Мыйзам"), "code": ("Кодекс", "Кодекс"), "law": ("Закон", "Мыйзам"), "decree": ("Указ", "Жарлык"), "resolution": ("Постановление", "Токтом"), "order": ("Распоряжение", "Распоряжение"), "instruction": ("Инструкция", "Инструкция"), "rules": ("Правила", "Правила"), "procedure": ("Порядок", "Порядок"), "provision": ("Положение", "Жобо"), "regulation": ("Регламент", "Регламент"), "charter": ("Устав", "Жобо (Устав)"), "program": ("Программа", "Программа"), "plan": ("План", "План"), "strategy": ("Стратегия", "Стратегия"), "concept": ("Концепция", "Концепция"), "doctrine": ("Доктрина", "Доктрина"), "agreement": ("Соглашение", "Соглашение"), "declaration": ("Декларация", "Декларация"), "registry": ("Реестр", "Реестр"), "norms": ("Нормативы", "Нормативы"), "model": ("Модель", "Модель"), "matrix": ("Матрица", "Матрица"), "study": ("Исследование", "Исследование"), "report": ("Доклад", "Доклад"), "principles": ("Основные принципы", "Основные принципы"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес"), +} +STATUSES = {"active": ("Действует", "Күчүндө"), "repealed": ("Утратил силу", "Күчүн жоготту"), "unspecified": ("Не указан", "Көрсөтүлгөн эмес")} +AUTHORITIES = {"president": ("Президент", "Президент"), "parliament": ("Органы законодательной власти", "Мыйзам чыгаруу бийлик органдары"), "cabinet": ("Правительство и Кабинет Министров", "Өкмөт жана Министрлер Кабинети"), "ministries_and_committees": ("Министерства и государственные комитеты", "Министрликтер жана мамлекеттик комитеттер"), "administrative_agencies": ("Административные ведомства", "Административдик ведомстволор"), "national_bank": ("Национальный банк", "Улуттук банк"), "other_state_bodies": ("Иные государственные органы", "Башка мамлекеттик органдар"), "local_representative_bodies": ("Представительные органы местного самоуправления", "Жергиликтүү өз алдынча башкаруунун өкүлчүлүктүү органдары"), "other": ("Прочие органы", "Башка органдар")} +CATALOGS = {"document_type": DOCUMENT_TYPES, "status": STATUSES, "authority": AUTHORITIES} + + +def labels(category: str, code: str) -> dict[str, str]: + try: + ru, ky = CATALOGS[category][code] + except KeyError as error: + raise ValueError(f"Unknown {category} catalog code: {code}") from error + return {"ru": ru, "ky": ky} + + +def source_code(category: str, value: dict | None) -> str: + pair = ((value or {}).get("ru"), (value or {}).get("ky")) + if pair == (None, None): + return "unspecified" + for code, expected in CATALOGS[category].items(): + if pair == expected or category == "document_type" and code == "provision" and pair == ("Положение", "Положение"): + return code + raise ValueError(f"Unmapped {category} catalog value: {pair!r}") + + +def authority_codes(paths: list[dict]) -> list[str]: + codes = set() + for path in paths: + text = " ".join(path.get("ru", []) + path.get("ky", [])).lower() + if "президент" in text: + codes.add("president") + elif "жогорку кенеш" in text or "верховный совет" in text or "мыйзам чыгаруу" in text: + codes.add("parliament") + elif "кабинет министров" in text or "правительство" in text or "өкмөт" in text: + codes.add("cabinet") + elif "министер" in text or "мамлекеттик комитет" in text: + codes.add("ministries_and_committees") + elif "административ" in text: + codes.add("administrative_agencies") + elif "национальн" in text and "банк" in text or "улуттук банк" in text: + codes.add("national_bank") + elif "кенеш" in text or "кеңеш" in text or "айыл" in text or "местного самоуправления" in text: + codes.add("local_representative_bodies") + elif "иные государственные" in text or "башка мамлекеттик" in text: + codes.add("other_state_bodies") + else: + codes.add("other") + return sorted(codes) or ["other"] diff --git a/backend/search/minjust-fragments-index.json b/backend/search/minjust-fragments-index.json index 3cb2653..1cc47b8 100644 --- a/backend/search/minjust-fragments-index.json +++ b/backend/search/minjust-fragments-index.json @@ -22,12 +22,15 @@ "document_name_ky": { "type": "text", "analyzer": "icu_analyzer", "fields": { "keyword": { "type": "keyword", "ignore_above": 1024 } } }, "document_type_ru": { "type": "keyword" }, "document_type_ky": { "type": "keyword" }, + "document_type_code": { "type": "keyword" }, "status_ru": { "type": "keyword" }, "status_ky": { "type": "keyword" }, + "status_code": { "type": "keyword" }, "number": { "type": "keyword" }, "date_adopted": { "type": "date", "format": "strict_date" }, "authority_paths_ru": { "type": "keyword", "ignore_above": 2048 }, "authority_paths_ky": { "type": "keyword", "ignore_above": 2048 }, + "authority_codes": { "type": "keyword" }, "source_path": { "type": "keyword", "index": false }, "source_sha256": { "type": "keyword", "index": false }, "text_sha256": { "type": "keyword", "index": false } diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index f5f979f..d96794b 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -15,6 +15,8 @@ import urllib.request from pathlib import Path from typing import Iterator +from search.catalog import authority_codes, source_code + APP_VERSION = "0.7.1" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") @@ -73,12 +75,15 @@ def search_document(document: dict, fragment: dict, expected: tuple[str, str, st "document_name_ky": localized(document.get("name"), "ky"), "document_type_ru": localized(document.get("type"), "ru"), "document_type_ky": localized(document.get("type"), "ky"), + "document_type_code": source_code("document_type", document.get("type")), "status_ru": localized(document.get("status"), "ru"), "status_ky": localized(document.get("status"), "ky"), + "status_code": source_code("status", document.get("status")), "number": document.get("number"), "date_adopted": dates.get("DateAdopted"), "authority_paths_ru": paths(document, "authority_paths", "ru"), "authority_paths_ky": paths(document, "authority_paths", "ky"), + "authority_codes": authority_codes(document.get("authority_paths", [])), "source_path": fragment["source_path"], "source_sha256": fragment["source_sha256"], "text_sha256": fragment["text_sha256"], diff --git a/backend/test_search_api.py b/backend/test_search_api.py index bf8aeab..ecbb6d4 100644 --- a/backend/test_search_api.py +++ b/backend/test_search_api.py @@ -4,7 +4,7 @@ import unittest from pathlib import Path from unittest.mock import patch -from search.api import Api, ApiError, catalog_code +from search.api import Api, ApiError class SearchApiTest(unittest.TestCase): @@ -27,12 +27,12 @@ class SearchApiTest(unittest.TestCase): api = self.make_api(Path(temporary)) response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["Закон"]}}]}} with patch("search.api.request_json", return_value=response) as request: - status, payload = api.handle("GET", f"/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status={catalog_code('status', ('Действует', 'Күчүндө'))}") + status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=active") self.assertEqual(status, 200) self.assertEqual(payload["results"][0]["snippet"], "Закон") body = json.loads(request.call_args.args[2]) self.assertEqual((body["from"], body["size"]), (5, 6)) - self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"]) + self.assertIn({"term": {"status_code": "active"}}, body["query"]["bool"]["filter"]) with self.assertRaisesRegex(ApiError, "within 10000 results"): api.handle("GET", "/search?q=x&page=100&page_size=100") @@ -55,12 +55,12 @@ class SearchApiTest(unittest.TestCase): def test_filters_count_documents_and_return_bilingual_labels(self): with tempfile.TemporaryDirectory() as temporary: api = self.make_api(Path(temporary)) - response = {"aggregations": {name: {"buckets": [{"key": ["Закон", "Мыйзам"], "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}} + response = {"aggregations": {name: {"buckets": [{"key": "law" if name == "document_types" else "active" if name == "statuses" else "parliament", "documents": {"value": 3}}]} for name in ("document_types", "statuses", "authorities")}} with patch("search.api.request_json", return_value=response) as request: payload = api.handle("GET", "/search/filters?language=ky")[1] - self.assertEqual(payload["document_types"][0], {"code": catalog_code("document_type", ("Закон", "Мыйзам")), "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) + self.assertEqual(payload["document_types"][0], {"code": "law", "labels": {"ru": "Закон", "ky": "Мыйзам"}, "count": 3}) body = json.loads(request.call_args.args[2]) - self.assertIn("multi_terms", body["aggs"]["document_types"]) + self.assertIn("terms", body["aggs"]["document_types"]) self.assertEqual(body["query"], {"term": {"is_current_edition": True}}) diff --git a/backend/test_search_api_opensearch.py b/backend/test_search_api_opensearch.py index efd59b1..0c5b41b 100644 --- a/backend/test_search_api_opensearch.py +++ b/backend/test_search_api_opensearch.py @@ -4,7 +4,7 @@ import unittest import uuid from pathlib import Path -from search.api import Api, catalog_code +from search.api import Api from search.minjust_opensearch import DEFAULT_MAPPING, request_json @@ -16,16 +16,16 @@ class SearchApiOpenSearchTest(unittest.TestCase): request_json(f"{base_url}/{index}", "PUT", DEFAULT_MAPPING.read_bytes(), "application/json") try: documents = [ - {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]}, - {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"]}, - {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "status_ru": "Действует", "status_ky": "Күчүндө", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"]}, + {"document_code": "1", "edition_code": "1", "is_current_edition": False, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "historic", "document_name_ru": "Old law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Утратил силу", "status_ky": "Күчүн жоготту", "status_code": "repealed", "date_adopted": "2020-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]}, + {"document_code": "1", "edition_code": "2", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current law", "document_type_ru": "Закон", "document_type_ky": "Мыйзам", "document_type_code": "law", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2021-01-01", "authority_paths_ru": ["Парламент"], "authority_paths_ky": ["Парламент"], "authority_codes": ["parliament"]}, + {"document_code": "2", "edition_code": "1", "is_current_edition": True, "language": "ru", "position": 1, "fragment_type": "paragraph", "text_ru": "needle", "document_name_ru": "Current decree", "document_type_ru": "Указ", "document_type_ky": "Жарлык", "document_type_code": "decree", "status_ru": "Действует", "status_ky": "Күчүндө", "status_code": "active", "date_adopted": "2022-01-01", "authority_paths_ru": ["Президент"], "authority_paths_ky": ["Президент"], "authority_codes": ["president"]}, ] for number, document in enumerate(documents): request_json(f"{base_url}/{index}/_doc/{number}", "PUT", json.dumps(document).encode(), "application/json") request_json(f"{base_url}/{index}/_refresh", "POST", None, "application/json") api = Api(base_url, index, Path(".")) self.assertEqual(api.handle("GET", "/search?q=historic")[1]["results"], []) - filtered = api.handle("GET", f"/search?q=needle&document_type={catalog_code('document_type', ('Закон', 'Мыйзам'))}")[1] + filtered = api.handle("GET", "/search?q=needle&document_type=law")[1] self.assertEqual([result["code"] for result in filtered["results"]], ["1"]) filters = api.handle("GET", "/search/filters")[1] self.assertEqual({item["count"] for item in filters["statuses"]}, {2}) From 0f309f34faf2c3f6683da70890a6ac1e5e6fc333 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Tue, 25 Aug 2026 08:32:21 +0300 Subject: [PATCH 7/7] docs: update changelog for search API --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeb37b9..ae18282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Не выпущено +- Завершён Search API v1: стабильные справочники, валидный OpenAPI, безопасная + пагинация и проверка актуальных редакций в локальном OpenSearch. - Добавлено безопасное переключение alias на новую версию поискового индекса после полной загрузки; checkpoint защищает возобновление загрузки от смены alias.