feat: add search API contract

This commit is contained in:
2026-08-21 06:12:47 +03:00
parent 8f47dbb4f0
commit 73ff287b20
16 changed files with 364 additions and 19 deletions

View File

@@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения
## Текущее состояние
Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии
`0.6.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную
`0.7.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную
размеченном наборе запросов.
| Компонент | Версия | Состояние |
|---|---:|---|
| Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния |
| Backend | `0.6.0` | добавлено атомарное переключение alias поискового индекса |
| Backend | `0.7.0` | добавлено атомарное переключение alias поискового индекса |
| Frontend | — | ещё не создан |
| Сбор и обработка правовых данных | `0.6.0` | добавлены relevance set и baseline-метрики |
| Сбор и обработка правовых данных | `0.7.0` | добавлены relevance set и baseline-метрики |
| RAG и база знаний | — | ещё не созданы |
## Структура репозитория
@@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -1,6 +1,6 @@
# Backend Акылдаш
Версия: `0.6.0`
Версия: `0.7.0`
Первая backend-область проекта — загрузка правовых документов из официального
Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
@@ -160,6 +160,21 @@ python3 backend/search/minjust_opensearch.py \
версионный индекс, проверить его и переключить alias. Это не оставляет
удалённые trailing-фрагменты старых документов.
## HTTP API v1
Запустите публичный read-only API поверх текущего alias и нормализованного
корпуса:
```bash
PYTHONPATH=backend python3 -m search.api
```
Он публикует OpenAPI в `GET /openapi.json` и поддерживает `GET /search`,
`/search/filters`, `/documents/{code}`, `/documents/{code}/editions` и
`/documents/{code}/editions/{edition}`. Значения фильтров возвращаются с
каноническим значением ЦБД и подписью выбранного языка; применяйте `code` как
параметр поиска. API не подменяет отсутствующий язык документа.
## Локальный OpenSearch
Стенд использует один узел OpenSearch без Dashboards, устанавливает
@@ -210,4 +225,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la
---
Акылдаш · Backend v0.6.0 · Frontend — не создан
Акылдаш · Backend v0.7.0 · Frontend — не создан

View File

@@ -21,7 +21,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Callable, Iterable
APP_VERSION = "0.6.0"
APP_VERSION = "0.7.0"
API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/"
LANGUAGES = {"Rus": "ru", "Kyr": "ky"}
IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"}

View File

@@ -23,7 +23,7 @@ from pathlib import Path
from typing import Callable
from urllib.parse import urlsplit
APP_VERSION = "0.6.0"
APP_VERSION = "0.7.0"
SCHEMA_VERSION = "1"
NORMALIZER_VERSION = "1.0.0"
LANGUAGES = ("ru", "ky")

278
backend/search/api.py Normal file
View File

@@ -0,0 +1,278 @@
"""Minimal HTTP API for the normalized legal-document corpus."""
from __future__ import annotations
import argparse
import datetime
import json
import re
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from search.minjust_opensearch import APP_VERSION, request_json
API_VERSION = "v1"
LANGUAGES = {"ru", "ky"}
CODE = re.compile(r"^[0-9]+$")
MAX_PAGE_SIZE = 100
class ApiError(Exception):
def __init__(self, status: int, message: str):
self.status = status
self.message = message
def parse_positive(value: str | None, name: str, default: int, maximum: int) -> int:
if value is None:
return default
try:
parsed = int(value)
except ValueError as error:
raise ApiError(400, f"{name} must be an integer") from error
if not 1 <= parsed <= maximum:
raise ApiError(400, f"{name} must be between 1 and {maximum}")
return parsed
def one(query: dict[str, list[str]], name: str) -> str | None:
values = query.get(name, [])
if len(values) > 1:
raise ApiError(400, f"{name} must be specified once")
return values[0] if values else None
def date(value: str | None, name: str) -> str | None:
if value is None:
return None
try:
datetime.date.fromisoformat(value)
except ValueError as error:
raise ApiError(400, f"{name} must be an ISO date") from error
return value
def openapi() -> dict:
return {
"openapi": "3.0.3",
"info": {"title": "Akyldash Search API", "version": API_VERSION},
"paths": {
"/search": {"get": {"parameters": [
{"name": "q", "in": "query", "required": True, "schema": {"type": "string"}},
{"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}},
{"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}},
{"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": MAX_PAGE_SIZE}},
{"name": "document_type", "in": "query", "schema": {"type": "string"}},
{"name": "status", "in": "query", "schema": {"type": "string"}},
{"name": "authority", "in": "query", "schema": {"type": "string"}},
{"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}},
{"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}},
{"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}},
]}},
"/search/filters": {"get": {}},
"/documents/{code}": {"get": {}},
"/documents/{code}/editions": {"get": {}},
"/documents/{code}/editions/{edition}": {"get": {}},
},
}
class Api:
def __init__(self, base_url: str, index: str, data_root: Path):
self.base_url = base_url.rstrip("/")
self.index = index
self.data_root = data_root
def search_url(self, suffix: str) -> str:
return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}"
def query_opensearch(self, body: dict) -> dict:
try:
return request_json(self.search_url("_search"), "POST", json.dumps(body, ensure_ascii=False).encode(), "application/json")
except RuntimeError as error:
raise ApiError(502, "search backend is unavailable") from error
def search(self, query: dict[str, list[str]]) -> dict:
text = one(query, "q")
if not text or not text.strip():
raise ApiError(400, "q is required")
if len(text) > 500:
raise ApiError(400, "q must not exceed 500 characters")
language = one(query, "language") or "ru"
if language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
page = parse_positive(one(query, "page"), "page", 1, 1_000_000)
page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE)
sort = one(query, "sort") or "relevance"
if sort not in {"relevance", "date"}:
raise ApiError(400, "sort must be relevance or date")
filters: list[dict] = [{"term": {"language": language}}]
fields = {"document_type": f"document_type_{language}", "status": f"status_{language}", "authority": f"authority_paths_{language}"}
for parameter, field in fields.items():
value = one(query, parameter)
if value:
filters.append({"term": {field: value}})
date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
if date_from and date_to and date_from > date_to:
raise ApiError(400, "date_from must not be later than date_to")
if date_from or date_to:
date_range = {key: value for key, value in (("gte", date_from), ("lte", date_to)) if value}
filters.append({"range": {"date_adopted": date_range}})
body = {
"from": (page - 1) * page_size,
"size": page_size + 1,
"_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "date_adopted", "number"],
"query": {"bool": {"filter": filters, "must": {"multi_match": {"query": text, "fields": [f"document_name_{language}", f"text_{language}"], "type": "cross_fields"}}}},
"collapse": {"field": "document_code"},
"highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}},
}
if sort == "date":
body["sort"] = [{"date_adopted": "desc"}, {"_score": "desc"}]
response = self.query_opensearch(body)
try:
hits = response["hits"]["hits"]
except (KeyError, TypeError) as error:
raise ApiError(502, "search backend returned an incomplete response") from error
return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": [self.search_hit(hit, language) for hit in hits[:page_size]]}
@staticmethod
def search_hit(hit: dict, language: str) -> dict:
source = hit.get("_source")
if not isinstance(source, dict) or not source.get("document_code"):
raise ApiError(502, "search backend returned an incomplete result")
highlight = hit.get("highlight", {}).get(f"text_{language}", [])
return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": highlight[0] if highlight else None}
def filters(self, query: dict[str, list[str]]) -> dict:
language = one(query, "language") or "ru"
if language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
fields = {"document_types": f"document_type_{language}", "statuses": f"status_{language}", "authorities": f"authority_paths_{language}"}
body = {"size": 0, "aggs": {name: {"terms": {"field": field, "size": 1000}} for name, field in fields.items()}}
response = self.query_opensearch(body)
try:
aggregations = response["aggregations"]
values = {
name: [{"code": item["key"], "label": item["key"], "count": item["doc_count"]} for item in aggregations[name]["buckets"]]
for name in fields
}
except (KeyError, TypeError) as error:
raise ApiError(502, "search backend returned incomplete filters") from error
return {"api_version": API_VERSION, "language": language, **values}
def directory(self, code: str) -> Path:
if not CODE.fullmatch(code):
raise ApiError(404, "document not found")
path = self.data_root / "documents" / code
if not path.is_dir():
raise ApiError(404, "document not found")
return path
@staticmethod
def read_json(path: Path, message: str) -> dict:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as error:
raise ApiError(500, message) from error
if not isinstance(value, dict):
raise ApiError(500, message)
return value
def document(self, code: str) -> dict:
document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
editions = document.get("editions")
if not isinstance(editions, list):
raise ApiError(500, "document data is unavailable")
return {"api_version": API_VERSION, "document": document, "current_edition": editions[-1] if editions else None}
def editions(self, code: str) -> dict:
document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
return {"api_version": API_VERSION, "code": code, "available_languages": document.get("available_languages", []), "editions": document.get("editions", [])}
def edition(self, code: str, edition: str, query: dict[str, list[str]]) -> dict:
if not CODE.fullmatch(edition):
raise ApiError(404, "edition not found")
directory = self.directory(code) / "editions" / edition
if not directory.is_dir():
raise ApiError(404, "edition not found")
metadata = self.read_json(directory / "edition.json", "edition data is unavailable")
language = one(query, "language")
if language is not None and language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
languages = [language] if language else metadata.get("available_languages", [])
content = {}
for item in languages:
if item not in metadata.get("available_languages", []):
continue
try:
content[item] = {"html": (directory / item / "content.html").read_text(encoding="utf-8"), "text": (directory / item / "content.txt").read_text(encoding="utf-8")}
except (OSError, UnicodeError) as error:
raise ApiError(500, "edition content is unavailable") from error
if language and language not in content:
raise ApiError(404, "edition language not found")
return {"api_version": API_VERSION, "edition": metadata, "content": content}
def handle(self, method: str, path: str) -> tuple[int, dict]:
if method != "GET":
raise ApiError(405, "method not allowed")
parsed = urllib.parse.urlsplit(path)
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
if parts == ["openapi.json"]:
return 200, openapi()
if parts == ["search"]:
return 200, self.search(query)
if parts == ["search", "filters"]:
return 200, self.filters(query)
if len(parts) == 2 and parts[0] == "documents":
return 200, self.document(parts[1])
if len(parts) == 3 and parts[:1] == ["documents"] and parts[2] == "editions":
return 200, self.editions(parts[1])
if len(parts) == 4 and parts[:1] == ["documents"] and parts[2] == "editions":
return 200, self.edition(parts[1], parts[3], query)
raise ApiError(404, "endpoint not found")
def handler(api: Api):
class RequestHandler(BaseHTTPRequestHandler):
def respond(self, method: str):
try:
status, payload = api.handle(method, self.path)
except ApiError as error:
status, payload = error.status, {"api_version": API_VERSION, "error": error.message}
body = json.dumps(payload, ensure_ascii=False).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_GET(self):
self.respond("GET")
def do_POST(self):
self.respond("POST")
def log_message(self, format: str, *args):
return
return RequestHandler
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--url", default="http://127.0.0.1:9200")
parser.add_argument("--index", default="akyldash-fragments-current")
parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized"))
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--version", action="version", version=APP_VERSION)
arguments = parser.parse_args()
ThreadingHTTPServer((arguments.host, arguments.port), handler(Api(arguments.url, arguments.index, arguments.data))).serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -15,7 +15,7 @@ import urllib.request
from pathlib import Path
from typing import Iterator
APP_VERSION = "0.6.0"
APP_VERSION = "0.7.0"
LANGUAGES = {"ru", "ky"}
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")

View File

@@ -0,0 +1,52 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from search.api import Api, ApiError
class SearchApiTest(unittest.TestCase):
def make_api(self, root: Path) -> Api:
document = root / "documents/7"
edition = document / "editions/10"
edition.mkdir(parents=True)
(document / "document.json").write_text(json.dumps({
"source_code": "7", "available_languages": ["ru"],
"editions": [{"source_code": "10", "available_languages": ["ru"]},],
}), encoding="utf-8")
(edition / "edition.json").write_text(json.dumps({"source_code": "10", "available_languages": ["ru"]}), encoding="utf-8")
(edition / "ru").mkdir()
(edition / "ru/content.html").write_text("<p>Текст</p>", encoding="utf-8")
(edition / "ru/content.txt").write_text("Текст\n", encoding="utf-8")
return Api("http://opensearch:9200", "current", root)
def test_search_pagination_filters_and_highlight(self):
with tempfile.TemporaryDirectory() as temporary:
api = self.make_api(Path(temporary))
response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}, "highlight": {"text_ru": ["<em>Закон</em>"]}}]}}
with patch("search.api.request_json", return_value=response) as request:
status, payload = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru&page=2&page_size=5&status=%D0%94%D0%B5%D0%B9%D1%81%D1%82%D0%B2%D1%83%D0%B5%D1%82")
self.assertEqual(status, 200)
self.assertEqual(payload["results"][0]["snippet"], "<em>Закон</em>")
body = json.loads(request.call_args.args[2])
self.assertEqual((body["from"], body["size"]), (5, 6))
self.assertIn({"term": {"status_ru": "Действует"}}, body["query"]["bool"]["filter"])
def test_document_editions_openapi_and_validation(self):
with tempfile.TemporaryDirectory() as temporary:
api = self.make_api(Path(temporary))
self.assertEqual(api.handle("GET", "/openapi.json")[1]["info"]["version"], "v1")
self.assertEqual(api.handle("GET", "/documents/7")[1]["current_edition"]["source_code"], "10")
self.assertEqual(api.handle("GET", "/documents/7/editions/10?language=ru")[1]["content"]["ru"]["text"], "Текст\n")
with self.assertRaisesRegex(ApiError, "q is required"):
api.handle("GET", "/search")
with self.assertRaisesRegex(ApiError, "date_from must be an ISO date"):
api.handle("GET", "/search?q=x&date_from=tomorrow")
with self.assertRaisesRegex(ApiError, "document not found"):
api.handle("GET", "/documents/%2E%2E")
if __name__ == "__main__":
unittest.main()

View File

@@ -35,4 +35,4 @@
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -78,4 +78,4 @@ Telegram позволяет запретить пользователям отп
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -5,7 +5,7 @@
- Telegram-бот: `0.2.2`
- Telegram-бот на Synology: `0.2.1`
- Backend: `0.6.0`
- Backend: `0.7.0`
- Frontend: не создан
## Краткий итог
@@ -234,4 +234,4 @@
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -398,4 +398,4 @@ Git сохраняет актуальную версию
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -258,4 +258,4 @@ runtime-зависимостями frontend. Регистрация в стор
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -214,4 +214,4 @@ python3 backend/normalization/minjust_cbd.py
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -44,4 +44,4 @@
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -180,4 +180,4 @@
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан

View File

@@ -48,4 +48,4 @@ python3 -m unittest -v
---
Акылдаш · Telegram-бот v0.2.2 · Backend v0.6.0 · Frontend — не создан
Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.0 · Frontend — не создан