diff --git a/CHANGELOG.md b/CHANGELOG.md index 03ec6e8..50ba25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Не выпущено +- Добавлена внутренняя лаборатория проверки поисковой выдачи: просмотр документов, + оценка релевантности 0–3, комментарии и SQLite-экспорт подписанных снимков. - Уточнены доступные состояния и адаптивное поведение внутреннего интерфейса оценки поисковой выдачи. - Добавлен план внутреннего интерфейса оценки поисковой выдачи юристами. diff --git a/README.md b/README.md index 2b94a77..96dc16b 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Telegram-бот — только часть рабочего окружения ## Текущее состояние Сейчас реализованы Telegram-бот-секретарь версии `0.2.2` и backend версии -`0.7.1`: исправления контракта Search API v1 и его ограничений OpenSearch. +`0.8.0`: добавлены внутреннее сохранение и оценка поисковой выдачи. размеченном наборе запросов. | Компонент | Версия | Состояние | |---|---:|---| | Telegram-бот | `0.2.2` | на Synology работает `0.2.1`; обновление после слияния | -| Backend | `0.7.1` | исправлен контракт Search API v1 | +| Backend | `0.8.0` | добавлена лаборатория оценки поисковой выдачи | | Frontend | — | ещё не создан | -| Сбор и обработка правовых данных | `0.7.1` | добавлены relevance set и baseline-метрики | +| Сбор и обработка правовых данных | `0.8.0` | добавлены relevance set и оценка выдачи | | RAG и база знаний | — | ещё не созданы | ## Структура репозитория @@ -59,4 +59,4 @@ python3 -m unittest discover -s tools/telegram-bot -v --- -Акылдаш · Telegram-бот v0.2.2 · Backend v0.7.1 · Frontend — не создан +Акылдаш · Telegram-бот v0.2.2 · Backend v0.8.0 · Frontend — не создан diff --git a/backend/README.md b/backend/README.md index 36950aa..e32f573 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # Backend Акылдаш -Версия: `0.7.1` +Версия: `0.8.0` Первая backend-область проекта — загрузка правовых документов из официального Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в @@ -216,6 +216,21 @@ PYTHONPATH=backend python3 -m search.evaluate_relevance \ Менять веса или анализаторы следует только после фиксации этого baseline и разбора ошибок выдачи. +## Внутренняя лаборатория релевантности + +Запустите Search API на localhost и откройте `http://127.0.0.1:8080/review`: + +```bash +PYTHONPATH=backend python3 -m search.api \\ + --reviews-db data/search-reviews.sqlite3 +``` + +Лаборатория показывает фактический порядок выдачи OpenSearch, позволяет открыть +текст редакции, поставить результату оценку от 0 до 3 и сохранить снимок с +комментариями. Оценки сохраняются в SQLite, экспорт доступен через +`GET /search-reviews/export`. Интерфейс предназначен только для локальной сети +или защищённого reverse proxy; не публикуйте его напрямую в интернет. + Для проверки текущей выдачи без будущего HTTP API используйте CLI: ```bash @@ -225,4 +240,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la --- -Акылдаш · Backend v0.7.1 · Frontend — не создан +Акылдаш · Backend v0.8.0 · Frontend — не создан diff --git a/backend/ingestion/minjust_cbd.py b/backend/ingestion/minjust_cbd.py index 684e2cf..cb7684f 100644 --- a/backend/ingestion/minjust_cbd.py +++ b/backend/ingestion/minjust_cbd.py @@ -21,7 +21,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable, Iterable -APP_VERSION = "0.7.1" +APP_VERSION = "0.8.0" API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/" LANGUAGES = {"Rus": "ru", "Kyr": "ky"} IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"} diff --git a/backend/normalization/minjust_cbd.py b/backend/normalization/minjust_cbd.py index 776ffe4..f3f4ee8 100644 --- a/backend/normalization/minjust_cbd.py +++ b/backend/normalization/minjust_cbd.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Callable from urllib.parse import urlsplit -APP_VERSION = "0.7.1" +APP_VERSION = "0.8.0" SCHEMA_VERSION = "1" NORMALIZER_VERSION = "1.0.0" LANGUAGES = ("ru", "ky") diff --git a/backend/search/api.py b/backend/search/api.py index 883dd2b..9a6183d 100644 --- a/backend/search/api.py +++ b/backend/search/api.py @@ -12,9 +12,11 @@ from pathlib import Path from search.minjust_opensearch import APP_VERSION, request_json from search.catalog import CATALOGS, labels +from search.reviews import ReviewSnapshots, ReviewStore API_VERSION = "v1" +SEARCH_ALGORITHM_VERSION = "search-1" LANGUAGES = {"ru", "ky"} CODE = re.compile(r"^[0-9]+$") MAX_PAGE_SIZE = 100 @@ -75,6 +77,9 @@ def openapi() -> dict: {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, ]}}, "/search/filters": {"get": {"responses": responses}}, + "/search-reviews": {"post": {"responses": {"201": {"description": "Review saved"}, "400": {"description": "Invalid review"}}}}, + "/search-reviews/export": {"get": {"responses": responses}}, + "/review": {"get": {"responses": {"200": {"description": "Review interface"}}}}, "/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, "/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, "/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, @@ -83,10 +88,12 @@ def openapi() -> dict: class Api: - def __init__(self, base_url: str, index: str, data_root: Path): + def __init__(self, base_url: str, index: str, data_root: Path, reviews_db: Path | str = ":memory:", review_secret: bytes | None = None): self.base_url = base_url.rstrip("/") self.index = index self.data_root = data_root + self.review_store = ReviewStore(reviews_db) + self.review_snapshots = ReviewSnapshots(review_secret) def search_url(self, suffix: str) -> str: return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}" @@ -142,7 +149,11 @@ class Api: hits = response["hits"]["hits"] except (KeyError, TypeError) as error: raise ApiError(502, "search backend returned an incomplete response") from error - return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": [self.search_hit(hit, language) for hit in hits[:page_size]]} + results = [self.search_hit(hit, language) for hit in hits[:page_size]] + snapshot_results = [{"rank": rank, **result} for rank, result in enumerate(results, 1)] + concrete_indexes = {hit.get("_index") for hit in hits[:page_size] if hit.get("_index")} + index_name = next(iter(concrete_indexes)) if len(concrete_indexes) == 1 else self.index + return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": results, "review_token": self.review_snapshots.create(text, language, index_name, snapshot_results, SEARCH_ALGORITHM_VERSION)} @staticmethod def search_hit(hit: dict, language: str) -> dict: @@ -221,12 +232,62 @@ class Api: raise ApiError(404, "edition language not found") return {"api_version": API_VERSION, "edition": metadata, "content": content} - def handle(self, method: str, path: str) -> tuple[int, dict]: - if method != "GET": - raise ApiError(405, "method not allowed") + def save_review(self, body: dict) -> dict: + if not isinstance(body, dict): + raise ApiError(400, "request body must be an object") + try: + snapshot = self.review_snapshots.verify(body["review_token"]) + reviewer = body["reviewer"].strip() + overall_comment = body.get("overall_comment", "").strip() + submitted = body["results"] + except (KeyError, AttributeError, TypeError, ValueError) as error: + raise ApiError(400, "review_token, reviewer and results are required") from error + if not reviewer or len(reviewer) > 120: + raise ApiError(400, "reviewer must be between 1 and 120 characters") + if len(overall_comment) > 4000: + raise ApiError(400, "overall_comment is too long") + if not isinstance(submitted, list): + raise ApiError(400, "results must be an array") + by_rank = {item["rank"]: item for item in snapshot["results"]} + if len(submitted) != len(by_rank) or {item.get("rank") for item in submitted if isinstance(item, dict)} != set(by_rank): + raise ApiError(400, "all search results must be reviewed exactly once") + results = [] + for item in submitted: + if not isinstance(item, dict) or not isinstance(item.get("rank"), int) or item["rank"] not in by_rank: + raise ApiError(400, "review result rank is invalid") + source = by_rank[item["rank"]] + if item.get("code") != source["code"]: + raise ApiError(400, "review result document does not match the search snapshot") + rating = item.get("rating") + if rating is not None and (isinstance(rating, bool) or not isinstance(rating, int) or not 0 <= rating <= 3): + raise ApiError(400, "rating must be an integer from 0 to 3") + comment = item.get("comment", "") + if not isinstance(comment, str) or len(comment) > 4000: + raise ApiError(400, "result comment is too long") + results.append({**source, "rating": rating, "comment": comment.strip()}) + if not results: + raise ApiError(400, "at least one result must be reviewed") + review = {"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "reviewer": reviewer, "query": snapshot["query"], "language": snapshot["language"], "index_name": snapshot["index_name"], "algorithm_version": snapshot["algorithm_version"], "top_result_code": snapshot["results"][0]["code"] if snapshot["results"] else None, "results": results, "overall_comment": overall_comment} + review_id = self.review_store.save(review) + return {"api_version": API_VERSION, "id": review_id, "created_at": review["created_at"]} + + @staticmethod + def review_page() -> str: + try: + return Path(__file__).with_name("review.html").read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise ApiError(500, "review interface is unavailable") from error + + def handle(self, method: str, path: str, body: dict | None = None) -> tuple[int, dict]: parsed = urllib.parse.urlsplit(path) query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] + if method == "POST" and parts == ["search-reviews"]: + return 201, self.save_review(body) + if method == "GET" and parts == ["search-reviews", "export"]: + return 200, {"api_version": API_VERSION, "reviews": self.review_store.export()} + if method != "GET": + raise ApiError(405, "method not allowed") if parts == ["openapi.json"]: return 200, openapi() if parts == ["search"]: @@ -246,7 +307,23 @@ def handler(api: Api): class RequestHandler(BaseHTTPRequestHandler): def respond(self, method: str): try: - status, payload = api.handle(method, self.path) + if method == "GET" and urllib.parse.urlsplit(self.path).path == "/review": + body = api.review_page().encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + body = None + if method == "POST": + length = int(self.headers.get("Content-Length", "0")) + if length > 1_000_000: + raise ApiError(413, "request body is too large") + body = json.loads(self.rfile.read(length) or b"{}") + status, payload = api.handle(method, self.path, body) + except json.JSONDecodeError: + status, payload = 400, {"api_version": API_VERSION, "error": "request body must be valid JSON"} except ApiError as error: status, payload = error.status, {"api_version": API_VERSION, "error": error.message} body = json.dumps(payload, ensure_ascii=False).encode() @@ -273,11 +350,14 @@ def main() -> int: parser.add_argument("--url", default="http://127.0.0.1:9200") parser.add_argument("--index", default="akyldash-fragments-current") parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized")) + parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3")) + parser.add_argument("--review-secret", default=None) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8080) parser.add_argument("--version", action="version", version=APP_VERSION) arguments = parser.parse_args() - ThreadingHTTPServer((arguments.host, arguments.port), handler(Api(arguments.url, arguments.index, arguments.data))).serve_forever() + secret = arguments.review_secret.encode() if arguments.review_secret else None + ThreadingHTTPServer((arguments.host, arguments.port), handler(Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret))).serve_forever() return 0 diff --git a/backend/search/minjust_opensearch.py b/backend/search/minjust_opensearch.py index d96794b..a051201 100644 --- a/backend/search/minjust_opensearch.py +++ b/backend/search/minjust_opensearch.py @@ -17,7 +17,7 @@ from typing import Iterator from search.catalog import authority_codes, source_code -APP_VERSION = "0.7.1" +APP_VERSION = "0.8.0" LANGUAGES = {"ru", "ky"} DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json") diff --git a/backend/search/review.html b/backend/search/review.html new file mode 100644 index 0000000..176095c --- /dev/null +++ b/backend/search/review.html @@ -0,0 +1,107 @@ + + + + + + Оценка поисковой выдачи · Акылдаш + + + + Перейти к результатам +

Оценка поисковой выдачи

Проверяйте результаты нашего OpenSearch по практическим юридическим запросам.

+
+
+ + + + +
+
+
+

Результаты

    +

    Документ

    Выберите результат, чтобы открыть текст.
    +
    +
    +
    +

    Документ

     + + + + + diff --git a/backend/search/reviews.py b/backend/search/reviews.py new file mode 100644 index 0000000..c965813 --- /dev/null +++ b/backend/search/reviews.py @@ -0,0 +1,83 @@ +"""Persistence and signed snapshots for search relevance reviews.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import secrets +import sqlite3 +import threading +import time +from pathlib import Path + + +MAX_COMMENT = 4000 +MAX_REVIEWER = 120 + + +class ReviewStore: + def __init__(self, path: Path | str = ":memory:"): + if path != ":memory:": + Path(path).parent.mkdir(parents=True, exist_ok=True) + self.connection = sqlite3.connect(path, check_same_thread=False) + self.connection.row_factory = sqlite3.Row + # ponytail: one SQLite lock; split connections only if review throughput matters. + self._lock = threading.Lock() + self.connection.execute(""" + CREATE TABLE IF NOT EXISTS search_reviews ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + reviewer TEXT NOT NULL, + query TEXT NOT NULL, + language TEXT NOT NULL, + index_name TEXT NOT NULL, + algorithm_version TEXT NOT NULL, + top_result_code TEXT, + results_json TEXT NOT NULL, + overall_comment TEXT NOT NULL + ) + """) + self.connection.commit() + + def save(self, review: dict) -> int: + with self._lock: + cursor = self.connection.execute( + "INSERT INTO search_reviews(created_at, reviewer, query, language, index_name, algorithm_version, top_result_code, results_json, overall_comment) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", + (review["created_at"], review["reviewer"], review["query"], review["language"], review["index_name"], review["algorithm_version"], review["top_result_code"], json.dumps(review["results"], ensure_ascii=False), review["overall_comment"]), + ) + self.connection.commit() + return int(cursor.lastrowid) + + def export(self) -> list[dict]: + with self._lock: + return [ + {**dict(row), "results": json.loads(row["results_json"])} + for row in self.connection.execute("SELECT * FROM search_reviews ORDER BY id") + ] + + +class ReviewSnapshots: + def __init__(self, secret: bytes | None = None, ttl: int = 3600): + self.secret = secret or secrets.token_bytes(32) + self.ttl = ttl + + def create(self, query: str, language: str, index: str, results: list[dict], algorithm_version: str) -> str: + payload = {"query": query, "language": language, "index_name": index, "algorithm_version": algorithm_version, "results": results, "expires_at": int(time.time()) + self.ttl} + encoded = base64.urlsafe_b64encode(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).decode().rstrip("=") + signature = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest() + return f"{encoded}.{signature}" + + def verify(self, token: str) -> dict: + try: + encoded, signature = token.split(".", 1) + expected = hmac.new(self.secret, encoded.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + raise ValueError + payload = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))) + if payload["expires_at"] < int(time.time()): + raise ValueError + return payload + except (ValueError, KeyError, TypeError, json.JSONDecodeError, UnicodeError) as error: + raise ValueError("invalid or expired search snapshot") from error diff --git a/backend/test_search_reviews.py b/backend/test_search_reviews.py new file mode 100644 index 0000000..b93574b --- /dev/null +++ b/backend/test_search_reviews.py @@ -0,0 +1,40 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from search.api import Api, ApiError + + +class SearchReviewTest(unittest.TestCase): + def test_review_must_cover_each_snapshot_rank_once(self): + with tempfile.TemporaryDirectory() as temporary: + api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret") + response = {"hits": {"hits": [{"_index": "search-20260827", "_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}, {"_index": "search-20260827", "_source": {"document_code": "8", "edition_code": "11", "document_name_ru": "Кодекс"}}]}} + with patch("search.api.request_json", return_value=response): + result = api.handle("GET", "/search?q=test&language=ru&page_size=2")[1] + with self.assertRaisesRegex(ApiError, "exactly once"): + api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3}]}) + + def test_saves_signed_search_snapshot_and_rejects_tampering(self): + with tempfile.TemporaryDirectory() as temporary: + api = Api("http://opensearch:9200", "current", Path(temporary), Path(temporary) / "reviews.sqlite3", b"test-secret") + response = {"hits": {"hits": [{"_source": {"document_code": "7", "edition_code": "10", "document_name_ru": "Закон"}}]}} + with patch("search.api.request_json", return_value=response): + result = api.handle("GET", "/search?q=%D0%B7%D0%B0%D0%BA%D0%BE%D0%BD&language=ru")[1] + saved = api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "7", "rating": 3, "comment": "Прямой ответ"}]}) + self.assertEqual(saved[0], 201) + exported = api.handle("GET", "/search-reviews/export")[1]["reviews"] + self.assertEqual(exported[0]["top_result_code"], "7") + self.assertEqual(exported[0]["results"][0]["rating"], 3) + with self.assertRaisesRegex(ApiError, "does not match"): + api.handle("POST", "/search-reviews", {"review_token": result["review_token"], "reviewer": "Юрист", "results": [{"rank": 1, "code": "8", "rating": 3}]}) + + def test_snapshot_and_review_page_are_available(self): + with tempfile.TemporaryDirectory() as temporary: + api = Api("http://opensearch:9200", "current", Path(temporary)) + self.assertIn("Оценка поисковой выдачи", api.review_page()) + + +if __name__ == "__main__": + unittest.main()