fix: relax content search for long queries
This commit is contained in:
1 parent
9832c0c795
commit
bc57cb82fe
6 files changed
+63
-7
No files matched your search
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Backend Акылдаш
|
||||
|
||||
Версия: `0.10.3`
|
||||
Версия: `0.10.4`
|
||||
|
||||
Первая backend-область проекта — загрузка правовых документов из официального
|
||||
Open Data API ЦБД Минюста Кыргызской Республики. Код расположен в
|
||||
@@ -240,4 +240,4 @@ PYTHONPATH=backend python3 -m search.query "ЖЧК ачуу тартиби" --la
|
||||
|
||||
---
|
||||
|
||||
Акылдаш · Backend v0.10.3 · Frontend — не создан
|
||||
Акылдаш · Backend v0.10.4 · Frontend — не создан
|
||||
@@ -17,7 +17,7 @@ from search.reviews import ReviewSnapshots, ReviewStore
|
||||
|
||||
|
||||
API_VERSION = "v1"
|
||||
SEARCH_ALGORITHM_VERSION = "search-5"
|
||||
SEARCH_ALGORITHM_VERSION = "search-6"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
CODE = re.compile(r"^[0-9]+$")
|
||||
MAX_PAGE_SIZE = 100
|
||||
|
||||
@@ -17,7 +17,7 @@ from typing import Iterator
|
||||
|
||||
from search.catalog import authority_codes, legal_force_code, source_code
|
||||
|
||||
APP_VERSION = "0.10.3"
|
||||
APP_VERSION = "0.10.4"
|
||||
LANGUAGES = {"ru", "ky"}
|
||||
DEFAULT_MAPPING = Path(__file__).with_name("minjust-fragments-index.json")
|
||||
|
||||
|
||||
+35
-2
@@ -24,6 +24,18 @@ LEGAL_FORM_ALIASES = {
|
||||
}
|
||||
COMPANY_REGISTRATION_DOCUMENTS = (("230044970", 3000), ("667", 2000), ("4", 1000))
|
||||
TAX_PROFIT_DOCUMENTS = (("112340", 3000),)
|
||||
BOILERPLATE_PREFIXES = {
|
||||
"ru": ("как регулируется", "правовое регулирование", "судебная практика", "сроки и ответственность"),
|
||||
"ky": (),
|
||||
}
|
||||
|
||||
|
||||
def relaxed_query_text(language: str, query: str) -> str:
|
||||
value = query.strip()
|
||||
for prefix in BOILERPLATE_PREFIXES[language]:
|
||||
if value.casefold().startswith(prefix):
|
||||
return value[len(prefix):].strip()
|
||||
return value
|
||||
|
||||
|
||||
def query_variants(language: str, query: str) -> list[str]:
|
||||
@@ -100,6 +112,7 @@ def tax_profit_clauses(language: str, query: str) -> list[dict]:
|
||||
def build_search_query(language: str, query: str) -> dict:
|
||||
registration_clauses = company_registration_clauses(language, query)
|
||||
intent_clauses = [*registration_clauses, *tax_profit_clauses(language, query)]
|
||||
variants = query_variants(language, query)
|
||||
full_text = [
|
||||
{
|
||||
"multi_match": {
|
||||
@@ -109,13 +122,33 @@ def build_search_query(language: str, query: str) -> dict:
|
||||
"operator": "OR" if registration_clauses else "AND",
|
||||
}
|
||||
}
|
||||
for variant in query_variants(language, query)
|
||||
for variant in variants
|
||||
]
|
||||
relaxed_text = []
|
||||
if not registration_clauses:
|
||||
seen_relaxed = set()
|
||||
for variant in variants:
|
||||
value = relaxed_query_text(language, variant)
|
||||
words = value.split()
|
||||
if value in seen_relaxed or len(words) < 2 or (value == variant.strip() and len(words) < 3):
|
||||
continue
|
||||
if len(words) < 3 and not any(len(word) >= 5 for word in words):
|
||||
continue
|
||||
seen_relaxed.add(value)
|
||||
relaxed_text.append({
|
||||
"multi_match": {
|
||||
"query": value,
|
||||
"fields": [f"document_name_{language}^4", f"text_{language}"],
|
||||
"type": "cross_fields",
|
||||
"operator": "OR",
|
||||
"boost": 0.1,
|
||||
}
|
||||
})
|
||||
phrase_matches = [
|
||||
{"match_phrase": {f"text_{language}": {"query": variant, "boost": 3}}}
|
||||
for variant in query_variants(language, query)
|
||||
]
|
||||
planned = {"should": [*full_text, *phrase_matches, *intent_clauses], "minimum_should_match": 1}
|
||||
planned = {"should": [*full_text, *phrase_matches, *relaxed_text, *intent_clauses], "minimum_should_match": 1}
|
||||
return {"bool": planned}
|
||||
|
||||
|
||||
|
||||
@@ -117,7 +117,7 @@
|
||||
<div class="review__actions"><div class="review__actions-inner"><label>Ваше имя или псевдоним<input id="reviewer" maxlength="120" autocomplete="name" placeholder="Например, Айжан К."></label> <label>Комментарий к выдаче <input id="overall-comment" maxlength="4000" placeholder="Что важно учесть? (необязательно)"></label> <button class="review__button--primary" id="save" type="button">Сохранить разметку</button></div></div>
|
||||
<dialog id="document-dialog"><button class="review__dialog-close" id="dialog-close" type="button">Закрыть</button><h2 id="dialog-heading">Документ</h2><div class="review__document-body" id="dialog-body"></div></dialog>
|
||||
<div class="review__tour-spotlight" id="tutorial-spotlight" hidden aria-hidden="true"></div><dialog class="review__tour" id="tutorial-dialog" aria-labelledby="tutorial-heading" aria-describedby="tutorial-text"><div class="review__tour-content"><p class="review__tour-kicker">Короткое знакомство</p><h2 id="tutorial-heading"></h2><p id="tutorial-text"></p><div class="review__tour-footer"><span class="review__tour-progress" id="tutorial-progress"></span><div class="review__tour-actions"><button id="tutorial-skip" type="button">Пропустить</button><button class="review__button--primary" id="tutorial-next" type="button">Далее</button></div></div></div></dialog>
|
||||
<footer class="review__header">Акылдаш · Backend v0.10.3 · Внутренняя лаборатория релевантности</footer>
|
||||
<footer class="review__header">Акылдаш · Backend v0.10.4 · Внутренняя лаборатория релевантности</footer>
|
||||
<script>
|
||||
const DRAFT_KEY = 'akyldash-search-review-draft';
|
||||
const TOUR_KEY = 'akyldash-search-review-tour-seen';
|
||||
|
||||
@@ -49,6 +49,29 @@ class SearchRelevanceTest(unittest.TestCase):
|
||||
self.assertIn("регистрация осоо", [variant.casefold() for variant in query_variants("ru", "регистрация общества с ограниченной ответственностью")])
|
||||
self.assertIn("жчк ачуу тартиби", [variant.casefold() for variant in query_variants("ky", "жоопкерчилиги чектелген коом ачуу тартиби")])
|
||||
|
||||
def test_relaxed_match_drops_ru_template_prefix_and_stays_low_weight(self):
|
||||
body = json.loads(build_search_body("ru", "правовое регулирование расторжение договора аренды", 10))
|
||||
clauses = body["query"]["bool"]["must"]["bool"]["should"]
|
||||
strict = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("operator") == "AND"]
|
||||
relaxed = [item["multi_match"] for item in clauses if "multi_match" in item and item["multi_match"].get("boost") == 0.1]
|
||||
self.assertEqual(strict[0]["query"], "правовое регулирование расторжение договора аренды")
|
||||
self.assertEqual(relaxed[0]["query"], "расторжение договора аренды")
|
||||
self.assertEqual(relaxed[0]["operator"], "OR")
|
||||
self.assertNotIn("boost", strict[0])
|
||||
|
||||
def test_relaxed_match_supports_kyrgyz_and_avoids_short_named_acts(self):
|
||||
ky = json.loads(build_search_body("ky", "эмгек келишимин бузуу шарттары", 10))
|
||||
ky_clauses = ky["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertTrue(any(item.get("multi_match", {}).get("operator") == "OR" and item["multi_match"].get("boost") == 0.1 for item in ky_clauses))
|
||||
|
||||
named_act = json.loads(build_search_body("ru", "Гражданский кодекс", 10))
|
||||
act_clauses = named_act["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertFalse(any(item.get("multi_match", {}).get("boost") == 0.1 for item in act_clauses))
|
||||
|
||||
short_template = json.loads(build_search_body("ru", "как регулируется взыскание алиментов", 10))
|
||||
short_clauses = short_template["query"]["bool"]["must"]["bool"]["should"]
|
||||
self.assertTrue(any(item.get("multi_match", {}).get("query") == "взыскание алиментов" and item["multi_match"].get("boost") == 0.1 for item in short_clauses))
|
||||
|
||||
def test_company_registration_intent_boosts_current_documents(self):
|
||||
for language, query, status in (
|
||||
("ru", "как открыть ОсОО", "Действует"),
|
||||
|
||||
Reference in new issue
Block a user