244 lines
11 KiB
Python
244 lines
11 KiB
Python
"""Run document searches against the local OpenSearch index."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import urllib.parse
|
||
|
||
from search.minjust_opensearch import APP_VERSION, request_json
|
||
|
||
|
||
LEGAL_FORM_ALIASES = {
|
||
"ru": {
|
||
"осоо": ("ооо", "общество с ограниченной ответственностью"),
|
||
"ооо": ("осоо", "общество с ограниченной ответственностью"),
|
||
"общество с ограниченной ответственностью": ("осоо", "ооо"),
|
||
"общества с ограниченной ответственностью": ("осоо", "ооо"),
|
||
},
|
||
"ky": {
|
||
"жчк": ("жоопкерчилиги чектелген коом",),
|
||
"жоопкерчилиги чектелген коом": ("жчк",),
|
||
},
|
||
}
|
||
# The indexed Civil Code uses the legal term "имущественный наем" for a
|
||
# договор аренды. Keep this exact legal phrase expansion narrow: it must not
|
||
# rewrite residential hiring, employment, sublease, or leasing queries.
|
||
LEASE_TERM_ALIAS = re.compile(
|
||
r"(?<!\w)(договор(?:а|у|ом|е)?)[ \t]+аренд(?:а|ы|е|у|ой|ою)(?!\w)",
|
||
re.IGNORECASE,
|
||
)
|
||
LEASE_TERM_ALIAS_EXCLUSIONS = re.compile(
|
||
r"\b(?:квартир\w*|жил\w*|дом\w*|помещен\w*|земл\w*|участк\w*|"
|
||
r"поднайм\w*|подна[её]м\w*|субаренд\w*|лизинг\w*|труд\w*|работник\w*)\b",
|
||
re.IGNORECASE,
|
||
)
|
||
LEASE_TERM_PHRASE_BOOST = 12
|
||
COMPANY_REGISTRATION_DOCUMENTS = (("230044970", 3000), ("667", 2000), ("4", 1000))
|
||
TAX_PROFIT_DOCUMENTS = (("112340", 3000),)
|
||
BOILERPLATE_PREFIXES = {
|
||
"ru": (
|
||
"как регулируется", "правовое регулирование", "судебная практика", "сроки и ответственность",
|
||
"действующая редакция", "последние изменения", "основные положения", "комментарий к акту",
|
||
),
|
||
"ky": (),
|
||
}
|
||
ACT_TITLE_PREFIXES = {
|
||
"ru": (
|
||
r"(?:гражданский\s+)?процессуальный\s+кодекс(?:\s+(?:кыргызской\s+республики|кр))?",
|
||
r"(?:гражданск\w*|уголовн\w*|налогов\w*|трудов\w*|семейн\w*)\s+кодекс\w*(?:\s+(?:кыргызской\s+республики|кр))?",
|
||
r"кодекс\s+о\s+правонарушениях(?:\s+(?:кыргызской\s+республики|кр))?",
|
||
r"конституци\w*(?:\s+(?:кыргызской\s+республики|кр))?",
|
||
r"(?:гк|ук|нк|коап|гпк|упк|уик)(?:\s+(?:кыргызской\s+республики|кр))?",
|
||
r"закон(?:\s+(?:кыргызской\s+республики|кр))?\s+о\s+(?:защит\w*\s+прав\w*\s+потребител\w*|государственн\w+\s+регистраци\w+\s+юридическ\w+\s+лиц|труд\w*)",
|
||
),
|
||
"ky": (
|
||
r"кыргыз\w*\s+республикасынын\s+конституци\w*",
|
||
r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жарандык|кылмыш\w*|салык|эмгек|үй-бүлө|административдик|жарандык\s+процесстик)\s+кодекс\w*",
|
||
r"(?:кыргыз\w*\s+республикасынын\s+)?(?:жк|үк)(?:\s+кр)?",
|
||
),
|
||
}
|
||
|
||
|
||
def relaxed_query_text(language: str, query: str) -> str:
|
||
value = query.strip()
|
||
for prefix in BOILERPLATE_PREFIXES[language]:
|
||
if value.casefold().startswith(prefix):
|
||
value = value[len(prefix):].strip()
|
||
break
|
||
for pattern in ACT_TITLE_PREFIXES[language]:
|
||
match = re.match(rf"\s*{pattern}(?=\s|$|[.,;:!?])", value, re.IGNORECASE)
|
||
if match:
|
||
value = value[match.end():].lstrip(" \t,;:—-")
|
||
break
|
||
return value
|
||
|
||
|
||
def lease_term_variant(query: str) -> str | None:
|
||
if LEASE_TERM_ALIAS.search(query) and not LEASE_TERM_ALIAS_EXCLUSIONS.search(query):
|
||
return LEASE_TERM_ALIAS.sub(r"\1 имущественного найма", query)
|
||
return None
|
||
|
||
|
||
def query_variants(language: str, query: str) -> list[str]:
|
||
variants = [query]
|
||
for alias, expansions in LEGAL_FORM_ALIASES[language].items():
|
||
pattern = re.compile(rf"(?<!\w){re.escape(alias)}(?!\w)", re.IGNORECASE)
|
||
if pattern.search(query):
|
||
variants.extend(pattern.sub(expansion, query) for expansion in expansions)
|
||
if language == "ru":
|
||
alias = lease_term_variant(query)
|
||
if alias:
|
||
variants.append(alias)
|
||
return list(dict.fromkeys(variants))
|
||
|
||
|
||
def document_priority_clauses(language: str, documents: tuple[tuple[str, int], ...]) -> list[dict]:
|
||
status = {"ru": "Действует", "ky": "Күчүндө"}[language]
|
||
return [
|
||
{
|
||
"constant_score": {
|
||
"filter": {
|
||
"bool": {
|
||
"filter": [
|
||
{"term": {"document_code": code}},
|
||
{"term": {f"status_{language}": status}},
|
||
]
|
||
}
|
||
},
|
||
"boost": boost,
|
||
}
|
||
}
|
||
for code, boost in documents
|
||
]
|
||
|
||
|
||
def company_registration_clauses(language: str, query: str) -> list[dict]:
|
||
non_registration_context = {
|
||
"ru": (r"\bсч[её]т\w*", r"\bбанк\w*", r"\bфилиал\w*", r"\bпредставительств\w*"),
|
||
"ky": (r"\bбанк\w*", r"\bэсеп\w*", r"\bфилиал\w*", r"\bөкүлчүлүк\w*"),
|
||
}[language]
|
||
normalized_query = query.casefold()
|
||
if any(re.search(pattern, normalized_query) for pattern in non_registration_context):
|
||
return []
|
||
|
||
forms, patterns = {
|
||
"ru": (
|
||
r"(?:осоо|ооо|обществ\w*\s+с\s+ограниченной\s+ответственностью)",
|
||
(
|
||
r"\bкак\s+(?:откры\w*|созда\w*|зарегистр\w*)\s+{form}\b",
|
||
r"\b(?:порядок|процедура)\s+(?:открыти\w*|создани\w*|регистраци\w*)\s+{form}\b",
|
||
r"\b(?:государственн\w+\s+)?регистраци\w*\s+{form}\b",
|
||
r"\bсоздани\w*\s+{form}\b",
|
||
),
|
||
),
|
||
"ky": (
|
||
r"(?:жчк|жоопкерчилиги\s+чектелген\s+коом)",
|
||
(
|
||
r"\bжчк\s+ач\w*\s+тартиби\b",
|
||
r"\bжчк\s+кантип\s+ач\w*\b",
|
||
r"\b{form}\s+(?:ачуу|ачуунун|каттоо|каттоонун)\b",
|
||
r"\b(?:ачуу|каттоо)\s+{form}\b",
|
||
),
|
||
),
|
||
}[language]
|
||
if not any(re.search(pattern.format(form=forms), normalized_query) for pattern in patterns):
|
||
return []
|
||
|
||
return document_priority_clauses(language, COMPANY_REGISTRATION_DOCUMENTS)
|
||
|
||
|
||
def tax_profit_clauses(language: str, query: str) -> list[dict]:
|
||
if language != "ru" or not re.search(r"\bналог\w*\s+(?:на\s+)?прибыл\w*\b", query.casefold()):
|
||
return []
|
||
# ponytail: exact topic anchor; expand only with verified tax-code sources.
|
||
return document_priority_clauses(language, TAX_PROFIT_DOCUMENTS)
|
||
|
||
|
||
def build_search_query(language: str, query: str) -> dict:
|
||
registration_clauses = company_registration_clauses(language, query)
|
||
intent_clauses = [*registration_clauses, *tax_profit_clauses(language, query)]
|
||
variants = query_variants(language, query)
|
||
full_text = [
|
||
{
|
||
"multi_match": {
|
||
"query": variant,
|
||
"fields": [f"document_name_{language}^4", f"text_{language}"],
|
||
"type": "cross_fields",
|
||
"operator": "OR" if registration_clauses else "AND",
|
||
}
|
||
}
|
||
for variant in variants
|
||
]
|
||
relaxed_text = []
|
||
if not registration_clauses:
|
||
seen_relaxed = set()
|
||
for variant in variants:
|
||
value = relaxed_query_text(language, variant)
|
||
words = value.split()
|
||
if value in seen_relaxed or len(words) < 2 or (value == variant.strip() and len(words) < 3):
|
||
continue
|
||
if len(words) < 3 and not any(len(word) >= 5 for word in words):
|
||
continue
|
||
seen_relaxed.add(value)
|
||
relaxed_text.append({
|
||
"multi_match": {
|
||
"query": value,
|
||
"fields": [f"document_name_{language}^4", f"text_{language}"],
|
||
"type": "cross_fields",
|
||
"operator": "OR",
|
||
"minimum_should_match": 2,
|
||
"boost": 0.1,
|
||
}
|
||
})
|
||
lease_variant = lease_term_variant(query) if language == "ru" else None
|
||
phrase_matches = [
|
||
{"match_phrase": {f"text_{language}": {
|
||
"query": variant,
|
||
"boost": LEASE_TERM_PHRASE_BOOST if variant == lease_variant else 3,
|
||
}}}
|
||
for variant in variants
|
||
]
|
||
planned = {"should": [*full_text, *phrase_matches, *relaxed_text, *intent_clauses], "minimum_should_match": 1}
|
||
return {"bool": planned}
|
||
|
||
|
||
def build_search_body(language: str, query: str, top_k: int) -> bytes:
|
||
return json.dumps({
|
||
"size": top_k,
|
||
"track_total_hits": False,
|
||
"_source": ["document_code"],
|
||
"query": {"bool": {"filter": [{"term": {"language": language}}, {"term": {"is_current_edition": True}}], "must": build_search_query(language, query)}},
|
||
"collapse": {"field": "document_code"},
|
||
}, ensure_ascii=False).encode()
|
||
|
||
|
||
def search_documents(base_url: str, index: str, language: str, query: str, top_k: int) -> list[str]:
|
||
url = f"{base_url.rstrip('/')}/{urllib.parse.quote(index, safe='')}/_search"
|
||
response = request_json(url, "POST", build_search_body(language, query, top_k), "application/json")
|
||
try:
|
||
return [hit["_source"]["document_code"] for hit in response["hits"]["hits"]]
|
||
except (KeyError, TypeError) as error:
|
||
raise RuntimeError("OpenSearch search response is incomplete") from error
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("query")
|
||
parser.add_argument("--language", choices=("ru", "ky"), required=True)
|
||
parser.add_argument("--url", default="http://127.0.0.1:9200")
|
||
parser.add_argument("--index", default="akyldash-fragments-v1")
|
||
parser.add_argument("--top-k", type=int, default=10)
|
||
parser.add_argument("--version", action="version", version=APP_VERSION)
|
||
arguments = parser.parse_args()
|
||
if arguments.top_k <= 0:
|
||
raise SystemExit("--top-k must be greater than zero")
|
||
print(json.dumps(search_documents(arguments.url, arguments.index, arguments.language, arguments.query, arguments.top_k), ensure_ascii=False))
|
||
print(f"Akyldash · Backend v{APP_VERSION} · Frontend v0.3.4")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|