From 424cc164978b86558e3eaf39e68bede5535e9aa4 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Fri, 14 Aug 2026 23:37:52 +0300 Subject: [PATCH] feat: add search relevance baseline --- README.md | 10 +- backend/README.md | 26 +++- backend/ingestion/minjust_cbd.py | 2 +- backend/normalization/minjust_cbd.py | 2 +- backend/search/evaluate_relevance.py | 113 ++++++++++++++++++ backend/search/minjust_opensearch.py | 2 +- backend/search/relevance-set-v1.template.json | 52 ++++++++ backend/test_search_relevance.py | 46 +++++++ docs/README.md | 2 +- docs/decisions/001-telegram-workspace-mvp.md | 2 +- docs/operations/project-status.md | 7 +- docs/operations/telegram-workspace-plan.md | 2 +- docs/product/frontend-search-sps-plan.md | 2 +- ...njust-document-normalization-agent-task.md | 2 +- docs/product/project-overview.md | 2 +- docs/team/ai-skills-for-beginners.md | 2 +- tools/telegram-bot/README.md | 2 +- 17 files changed, 256 insertions(+), 20 deletions(-) create mode 100644 backend/search/evaluate_relevance.py create mode 100644 backend/search/relevance-set-v1.template.json create mode 100644 backend/test_search_relevance.py diff --git a/README.md b/README.md index dc37a6a..e85fd2a 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения ## Текущее состояние Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии -`0.4.1`: возобновляемая загрузка индекса документов Министерства юстиции -ЦБД Минюста КР. +`0.5.0`: подготовка поискового индекса и оценка Recall@K/MRR@K на вручную +размеченном наборе запросов. | Компонент | Версия | Состояние | |---|---:|---| | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния | -| Backend | `0.4.1` | добавлено продолжение прерванной Bulk-загрузки | +| Backend | `0.5.0` | добавлена воспроизводимая оценка качества поиска | | Frontend | — | ещё не создан | -| Сбор и обработка правовых данных | `0.4.1` | добавлено продолжение загрузки существующего индекса | +| Сбор и обработка правовых данных | `0.5.0` | добавлены relevance set и baseline-метрики | | RAG и база знаний | — | ещё не созданы | ## Структура репозитория @@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/backend/README.md b/backend/README.md index 8a13be4..9178fa9 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # Backend Акылдаш -Версия: `0.4.1` +Версия: `0.5.0` Первая backend-область проекта — загрузка правовых документов из официального Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в @@ -165,6 +165,28 @@ Security plugin отключён только для локальной разр также задаёт одну shard и ноль replicas; для production число shard следует рассчитать по размеру корпуса и настроить не менее одной replica. +## Оценка качества поиска + +`search/relevance-set-v1.template.json` содержит заготовку для 25 русских и +25 кыргызских запросов. Для каждого запроса человек должен указать реальную +формулировку и коды всех релевантных документов; пустая или неполная разметка +отклоняется. Рабочую копию следует хранить в игнорируемом каталоге `data/`, +пока набор не проверен и не разрешён к публикации. + +Baseline использует поля названия и текста соответствующего языка, оставляет в +выдаче один результат на документ и вычисляет макро-средние Recall@10 и MRR@10: + +```bash +mkdir -p data/search +cp backend/search/relevance-set-v1.template.json data/search/relevance-set-v1.json +PYTHONPATH=backend python3 -m search.evaluate_relevance \ + data/search/relevance-set-v1.json \ + > data/search/baseline-v1.json +``` + +Менять веса или анализаторы следует только после фиксации этого baseline и +разбора ошибок выдачи. + --- -Акылдаш · Backend v0.4.1 · Frontend — не создан +Акылдаш · Backend v0.5.0 · Frontend — не создан diff --git a/backend/ingestion/minjust_cbd.py b/backend/ingestion/minjust_cbd.py index d4122d4..8eed499 100644 --- a/backend/ingestion/minjust_cbd.py +++ b/backend/ingestion/minjust_cbd.py @@ -21,7 +21,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable, Iterable -APP_VERSION = "0.4.1" +APP_VERSION = "0.5.0" API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/" LANGUAGES = {"Rus": "ru", "Kyr": "ky"} IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"} diff --git a/backend/normalization/minjust_cbd.py b/backend/normalization/minjust_cbd.py index a41ca9c..0b68fc3 100644 --- a/backend/normalization/minjust_cbd.py +++ b/backend/normalization/minjust_cbd.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Callable from urllib.parse import urlsplit -APP_VERSION = "0.4.1" +APP_VERSION = "0.5.0" SCHEMA_VERSION = "1" NORMALIZER_VERSION = "1.0.0" LANGUAGES = ("ru", "ky") diff --git a/backend/search/evaluate_relevance.py b/backend/search/evaluate_relevance.py new file mode 100644 index 0000000..edba5af --- /dev/null +++ b/backend/search/evaluate_relevance.py @@ -0,0 +1,113 @@ +"""Measure document search Recall@K and MRR@K against a relevance set.""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.parse +from pathlib import Path + +from search.minjust_opensearch import APP_VERSION, request_json + + +def load_queries(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as source: + queries = json.load(source) + if not isinstance(queries, list) or not queries: + raise ValueError("Relevance set must be a non-empty JSON array") + + seen = set() + for item in queries: + if not isinstance(item, dict) or set(item) != { + "id", "language", "query", "relevant_document_codes" + }: + raise ValueError("Each query must contain id, language, query and relevant_document_codes") + codes = item["relevant_document_codes"] + if ( + not isinstance(item["id"], str) + or not item["id"].strip() + or item["id"] in seen + or not isinstance(item["language"], str) + or item["language"] not in {"ru", "ky"} + or not isinstance(item["query"], str) + or not item["query"].strip() + or not isinstance(codes, list) + or not codes + or any(not isinstance(code, str) or not code for code in codes) + or len(codes) != len(set(codes)) + ): + raise ValueError(f"Invalid relevance query: {item.get('id', '')}") + seen.add(item["id"]) + return queries + + +def search(base_url: str, index: str, item: dict, top_k: int) -> list[str]: + language = item["language"] + body = json.dumps({ + "size": top_k, + "track_total_hits": False, + "_source": ["document_code"], + "query": { + "bool": { + "filter": {"term": {"language": language}}, + "must": { + "multi_match": { + "query": item["query"], + "fields": [f"document_name_{language}", f"text_{language}"], + } + }, + } + }, + "collapse": {"field": "document_code"}, + }, ensure_ascii=False).encode() + url = f"{base_url.rstrip('/')}/{urllib.parse.quote(index, safe='')}/_search" + response = request_json(url, "POST", body, "application/json") + try: + return [hit["_source"]["document_code"] for hit in response["hits"]["hits"]] + except (KeyError, TypeError) as error: + raise RuntimeError("OpenSearch search response is incomplete") from error + + +def evaluate(queries: list[dict], base_url: str, index: str, top_k: int) -> dict: + results = [] + for item in queries: + retrieved = search(base_url, index, item, top_k) + relevant = set(item["relevant_document_codes"]) + matches = [rank for rank, code in enumerate(retrieved, 1) if code in relevant] + results.append({ + "id": item["id"], + "language": item["language"], + "query": item["query"], + "retrieved_document_codes": retrieved, + f"recall_at_{top_k}": len(relevant.intersection(retrieved)) / len(relevant), + f"reciprocal_rank_at_{top_k}": 1 / matches[0] if matches else 0.0, + }) + return { + "summary": { + "query_count": len(results), + f"recall_at_{top_k}": sum(item[f"recall_at_{top_k}"] for item in results) / len(results), + f"mrr_at_{top_k}": sum(item[f"reciprocal_rank_at_{top_k}"] for item in results) / len(results), + }, + "queries": results, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("relevance_set", type=Path) + parser.add_argument("--url", default="http://127.0.0.1:9200") + parser.add_argument("--index", default="akyldash-fragments-v1") + parser.add_argument("--top-k", type=int, default=10) + parser.add_argument("--version", action="version", version=APP_VERSION) + arguments = parser.parse_args() + if arguments.top_k <= 0: + raise SystemExit("--top-k must be greater than zero") + result = evaluate(load_queries(arguments.relevance_set), arguments.url, arguments.index, arguments.top_k) + print(json.dumps(result, ensure_ascii=False, indent=2)) + print(f"Akyldash Backend v{APP_VERSION} · Frontend — not created", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index d6d3ae0..97c9278 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -15,7 +15,7 @@ import urllib.request from pathlib import Path from typing import Iterator -APP_VERSION = "0.4.1" +APP_VERSION = "0.5.0" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") diff --git a/backend/search/relevance-set-v1.template.json b/backend/search/relevance-set-v1.template.json new file mode 100644 index 0000000..c8beee7 --- /dev/null +++ b/backend/search/relevance-set-v1.template.json @@ -0,0 +1,52 @@ +[ + {"id": "ru-01", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-02", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-03", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-04", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-05", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-06", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-07", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-08", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-09", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-10", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-11", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-12", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-13", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-14", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-15", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-16", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-17", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-18", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-19", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-20", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-21", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-22", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-23", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-24", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ru-25", "language": "ru", "query": "", "relevant_document_codes": []}, + {"id": "ky-01", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-02", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-03", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-04", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-05", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-06", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-07", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-08", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-09", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-10", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-11", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-12", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-13", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-14", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-15", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-16", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-17", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-18", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-19", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-20", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-21", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-22", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-23", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-24", "language": "ky", "query": "", "relevant_document_codes": []}, + {"id": "ky-25", "language": "ky", "query": "", "relevant_document_codes": []} +] diff --git a/backend/test_search_relevance.py b/backend/test_search_relevance.py new file mode 100644 index 0000000..42a167d --- /dev/null +++ b/backend/test_search_relevance.py @@ -0,0 +1,46 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from search.evaluate_relevance import evaluate, load_queries + + +class SearchRelevanceTest(unittest.TestCase): + def test_loads_queries_and_calculates_document_metrics(self): + queries = [ + { + "id": "ru-01", + "language": "ru", + "query": "трудовой договор", + "relevant_document_codes": ["7", "8"], + }, + { + "id": "ky-01", + "language": "ky", + "query": "эмгек келишими", + "relevant_document_codes": ["9"], + }, + ] + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "queries.json" + path.write_text(json.dumps(queries, ensure_ascii=False), encoding="utf-8") + loaded = load_queries(path) + + responses = [ + {"hits": {"hits": [{"_source": {"document_code": code}} for code in ["7", "10", "8"]]}}, + {"hits": {"hits": [{"_source": {"document_code": code}} for code in ["10", "9"]]}}, + ] + with patch("search.evaluate_relevance.request_json", side_effect=responses) as request: + result = evaluate(loaded, "http://127.0.0.1:9200", "test", 10) + + self.assertEqual(result["summary"], {"query_count": 2, "recall_at_10": 1.0, "mrr_at_10": 0.75}) + self.assertEqual(result["queries"][0]["reciprocal_rank_at_10"], 1.0) + body = json.loads(request.call_args_list[0].args[2]) + self.assertFalse(body["track_total_hits"]) + self.assertEqual(body["collapse"], {"field": "document_code"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/README.md b/docs/README.md index b054064..5d71a39 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,4 +33,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/decisions/001-telegram-workspace-mvp.md b/docs/decisions/001-telegram-workspace-mvp.md index 5d3632c..ec0dd69 100644 --- a/docs/decisions/001-telegram-workspace-mvp.md +++ b/docs/decisions/001-telegram-workspace-mvp.md @@ -78,4 +78,4 @@ Telegram позволяет запретить пользователям отп --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/operations/project-status.md b/docs/operations/project-status.md index 3711e22..ca9a871 100644 --- a/docs/operations/project-status.md +++ b/docs/operations/project-status.md @@ -5,7 +5,7 @@ - Telegram-бот: `0.2.2` - Telegram-бот на Synology: `0.2.1` -- Backend: `0.4.1` +- Backend: `0.5.0` - Frontend: не создан ## Краткий итог @@ -137,6 +137,9 @@ ### 2026-08-14 +- Добавлены шаблон relevance set v1 и воспроизводимый расчёт Recall@K/MRR@K. +- Версия backend обновлена до `0.5.0`. + - Полный локальный индекс содержит 56 295 965 фрагментов и успешно отвечает на RU/KY-запросы. - Добавлены атомарный checkpoint и безопасное продолжение прерванной загрузки @@ -223,4 +226,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/operations/telegram-workspace-plan.md b/docs/operations/telegram-workspace-plan.md index 57be505..989e68c 100644 --- a/docs/operations/telegram-workspace-plan.md +++ b/docs/operations/telegram-workspace-plan.md @@ -398,4 +398,4 @@ Git сохраняет актуальную версию --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/product/frontend-search-sps-plan.md b/docs/product/frontend-search-sps-plan.md index 392c2d1..4673010 100644 --- a/docs/product/frontend-search-sps-plan.md +++ b/docs/product/frontend-search-sps-plan.md @@ -258,4 +258,4 @@ runtime-зависимостями frontend. Регистрация в стор --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/product/minjust-document-normalization-agent-task.md b/docs/product/minjust-document-normalization-agent-task.md index 7d71fa2..00527b4 100644 --- a/docs/product/minjust-document-normalization-agent-task.md +++ b/docs/product/minjust-document-normalization-agent-task.md @@ -214,4 +214,4 @@ python3 backend/normalization/minjust_cbd.py --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/product/project-overview.md b/docs/product/project-overview.md index 2c6e54b..cfa06e5 100644 --- a/docs/product/project-overview.md +++ b/docs/product/project-overview.md @@ -44,4 +44,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/docs/team/ai-skills-for-beginners.md b/docs/team/ai-skills-for-beginners.md index a615a97..8974c6b 100644 --- a/docs/team/ai-skills-for-beginners.md +++ b/docs/team/ai-skills-for-beginners.md @@ -180,4 +180,4 @@ --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан diff --git a/tools/telegram-bot/README.md b/tools/telegram-bot/README.md index 720903c..96a9d6e 100644 --- a/tools/telegram-bot/README.md +++ b/tools/telegram-bot/README.md @@ -48,4 +48,4 @@ python3 -m unittest -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.4.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.5.0 · Frontend — не создан