feat: add search relevance baseline
This commit is contained in:
113
backend/search/evaluate_relevance.py
Normal file
113
backend/search/evaluate_relevance.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Measure document search Recall@K and MRR@K against a relevance set."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
from search.minjust_opensearch import APP_VERSION, request_json
|
||||
|
||||
|
||||
def load_queries(path: Path) -> list[dict]:
|
||||
with path.open(encoding="utf-8") as source:
|
||||
queries = json.load(source)
|
||||
if not isinstance(queries, list) or not queries:
|
||||
raise ValueError("Relevance set must be a non-empty JSON array")
|
||||
|
||||
seen = set()
|
||||
for item in queries:
|
||||
if not isinstance(item, dict) or set(item) != {
|
||||
"id", "language", "query", "relevant_document_codes"
|
||||
}:
|
||||
raise ValueError("Each query must contain id, language, query and relevant_document_codes")
|
||||
codes = item["relevant_document_codes"]
|
||||
if (
|
||||
not isinstance(item["id"], str)
|
||||
or not item["id"].strip()
|
||||
or item["id"] in seen
|
||||
or not isinstance(item["language"], str)
|
||||
or item["language"] not in {"ru", "ky"}
|
||||
or not isinstance(item["query"], str)
|
||||
or not item["query"].strip()
|
||||
or not isinstance(codes, list)
|
||||
or not codes
|
||||
or any(not isinstance(code, str) or not code for code in codes)
|
||||
or len(codes) != len(set(codes))
|
||||
):
|
||||
raise ValueError(f"Invalid relevance query: {item.get('id', '<unknown>')}")
|
||||
seen.add(item["id"])
|
||||
return queries
|
||||
|
||||
|
||||
def search(base_url: str, index: str, item: dict, top_k: int) -> list[str]:
|
||||
language = item["language"]
|
||||
body = json.dumps({
|
||||
"size": top_k,
|
||||
"track_total_hits": False,
|
||||
"_source": ["document_code"],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": {"term": {"language": language}},
|
||||
"must": {
|
||||
"multi_match": {
|
||||
"query": item["query"],
|
||||
"fields": [f"document_name_{language}", f"text_{language}"],
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
"collapse": {"field": "document_code"},
|
||||
}, ensure_ascii=False).encode()
|
||||
url = f"{base_url.rstrip('/')}/{urllib.parse.quote(index, safe='')}/_search"
|
||||
response = request_json(url, "POST", body, "application/json")
|
||||
try:
|
||||
return [hit["_source"]["document_code"] for hit in response["hits"]["hits"]]
|
||||
except (KeyError, TypeError) as error:
|
||||
raise RuntimeError("OpenSearch search response is incomplete") from error
|
||||
|
||||
|
||||
def evaluate(queries: list[dict], base_url: str, index: str, top_k: int) -> dict:
|
||||
results = []
|
||||
for item in queries:
|
||||
retrieved = search(base_url, index, item, top_k)
|
||||
relevant = set(item["relevant_document_codes"])
|
||||
matches = [rank for rank, code in enumerate(retrieved, 1) if code in relevant]
|
||||
results.append({
|
||||
"id": item["id"],
|
||||
"language": item["language"],
|
||||
"query": item["query"],
|
||||
"retrieved_document_codes": retrieved,
|
||||
f"recall_at_{top_k}": len(relevant.intersection(retrieved)) / len(relevant),
|
||||
f"reciprocal_rank_at_{top_k}": 1 / matches[0] if matches else 0.0,
|
||||
})
|
||||
return {
|
||||
"summary": {
|
||||
"query_count": len(results),
|
||||
f"recall_at_{top_k}": sum(item[f"recall_at_{top_k}"] for item in results) / len(results),
|
||||
f"mrr_at_{top_k}": sum(item[f"reciprocal_rank_at_{top_k}"] for item in results) / len(results),
|
||||
},
|
||||
"queries": results,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("relevance_set", type=Path)
|
||||
parser.add_argument("--url", default="http://127.0.0.1:9200")
|
||||
parser.add_argument("--index", default="akyldash-fragments-v1")
|
||||
parser.add_argument("--top-k", type=int, default=10)
|
||||
parser.add_argument("--version", action="version", version=APP_VERSION)
|
||||
arguments = parser.parse_args()
|
||||
if arguments.top_k <= 0:
|
||||
raise SystemExit("--top-k must be greater than zero")
|
||||
result = evaluate(load_queries(arguments.relevance_set), arguments.url, arguments.index, arguments.top_k)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
print(f"Akyldash Backend v{APP_VERSION} · Frontend — not created", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -15,7 +15,7 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
APP_VERSION = "0.4.1"
|
||||
APP_VERSION = "0.5.0"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
|
||||
|
||||
|
||||
52
backend/search/relevance-set-v1.template.json
Normal file
52
backend/search/relevance-set-v1.template.json
Normal file
@@ -0,0 +1,52 @@
|
||||
[
|
||||
{"id": "ru-01", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-02", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-03", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-04", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-05", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-06", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-07", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-08", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-09", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-10", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-11", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-12", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-13", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-14", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-15", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-16", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-17", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-18", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-19", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-20", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-21", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-22", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-23", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-24", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ru-25", "language": "ru", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-01", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-02", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-03", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-04", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-05", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-06", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-07", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-08", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-09", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-10", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-11", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-12", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-13", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-14", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-15", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-16", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-17", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-18", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-19", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-20", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-21", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-22", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-23", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-24", "language": "ky", "query": "", "relevant_document_codes": []},
|
||||
{"id": "ky-25", "language": "ky", "query": "", "relevant_document_codes": []}
|
||||
]
|
||||
Reference in New Issue
Block a user