Files
akyldash/backend/search/api.py
T

491 lines
28 KiB
Python

"""Minimal HTTP API for the normalized legal-document corpus."""
from __future__ import annotations
import argparse
import datetime
import getpass
import http.cookies
import json
import os
import re
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from search.auth import AuthError, AuthStore
from search.minjust_opensearch import APP_VERSION, request_json
from search.catalog import CATALOGS, labels
from search.query import build_search_query
from search.reviews import ReviewSnapshots, ReviewStore
API_VERSION = "v1"
SEARCH_ALGORITHM_VERSION = "search-6"
LANGUAGES = {"ru", "ky"}
CODE = re.compile(r"^[0-9]+$")
MAX_PAGE_SIZE = 100
MAX_RESULT_WINDOW = 10_000
class ApiError(Exception):
def __init__(self, status: int, message: str):
self.status = status
self.message = message
def parse_positive(value: str | None, name: str, default: int, maximum: int) -> int:
if value is None:
return default
try:
parsed = int(value)
except ValueError as error:
raise ApiError(400, f"{name} must be an integer") from error
if not 1 <= parsed <= maximum:
raise ApiError(400, f"{name} must be between 1 and {maximum}")
return parsed
def one(query: dict[str, list[str]], name: str) -> str | None:
values = query.get(name, [])
if len(values) > 1:
raise ApiError(400, f"{name} must be specified once")
return values[0] if values else None
def date(value: str | None, name: str) -> str | None:
if value is None:
return None
try:
datetime.date.fromisoformat(value)
except ValueError as error:
raise ApiError(400, f"{name} must be an ISO date") from error
return value
def openapi() -> dict:
responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "401": {"description": "Authentication required"}, "403": {"description": "Additional access required"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}}
return {
"openapi": "3.0.3",
"info": {"title": "Akyldash Search API", "version": API_VERSION},
"components": {"securitySchemes": {"SessionCookie": {"type": "apiKey", "in": "cookie", "name": "akyldash_session"}}},
"security": [{"SessionCookie": []}],
"paths": {
"/auth/login": {"post": {"security": [], "responses": {"200": {"description": "Signed in and session cookie issued"}, "401": {"description": "Invalid login or password"}}}},
"/auth/logout": {"post": {"responses": {"200": {"description": "Signed out"}}}},
"/auth/me": {"get": {"responses": {"200": {"description": "Current account"}, "401": {"description": "Authentication required"}}}},
"/auth/change-password": {"post": {"responses": {"200": {"description": "Password changed"}, "400": {"description": "Invalid password"}}}},
"/admin/users": {"get": {"responses": {"200": {"description": "User list"}}}, "post": {"responses": {"201": {"description": "User created with a temporary password"}}}},
"/admin/users/{username}": {"delete": {"responses": {"200": {"description": "User deleted"}}}},
"/admin/users/{username}/reset-password": {"post": {"responses": {"200": {"description": "Temporary password set"}}}},
"/search": {"get": {"responses": responses, "parameters": [
{"name": "q", "in": "query", "required": True, "schema": {"type": "string"}},
{"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}},
{"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}},
{"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": MAX_PAGE_SIZE}},
{"name": "document_type", "in": "query", "schema": {"type": "string"}},
{"name": "status", "in": "query", "schema": {"type": "string"}},
{"name": "authority", "in": "query", "schema": {"type": "string"}},
{"name": "legal_force", "in": "query", "schema": {"type": "string"}},
{"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}},
{"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}},
{"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}},
]}},
"/search/filters": {"get": {"responses": responses}},
"/search-reviews": {"post": {"responses": {"201": {"description": "Review saved"}, "400": {"description": "Invalid review"}}}},
"/search-reviews/export": {"get": {"responses": responses}},
"/review": {"get": {"responses": {"200": {"description": "Review interface"}}}},
"/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
"/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
"/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}},
},
}
class Api:
def __init__(self, base_url: str, index: str, data_root: Path, reviews_db: Path | str = ":memory:", review_secret: bytes | None = None, users_db: Path | str = ":memory:", cookie_secure: bool = False):
self.base_url = base_url.rstrip("/")
self.index = index
self.data_root = data_root
self.review_store = ReviewStore(reviews_db)
self.review_snapshots = ReviewSnapshots(review_secret)
self.auth_store = AuthStore(users_db)
self.cookie_secure = cookie_secure
def search_url(self, suffix: str) -> str:
return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}"
def query_opensearch(self, body: dict) -> dict:
try:
return request_json(self.search_url("_search"), "POST", json.dumps(body, ensure_ascii=False).encode(), "application/json")
except RuntimeError as error:
raise ApiError(502, "search backend is unavailable") from error
def search(self, query: dict[str, list[str]]) -> dict:
text = one(query, "q")
if not text or not text.strip():
raise ApiError(400, "q is required")
if len(text) > 500:
raise ApiError(400, "q must not exceed 500 characters")
language = one(query, "language") or "ru"
if language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
page = parse_positive(one(query, "page"), "page", 1, 1_000_000)
page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE)
if page * page_size >= MAX_RESULT_WINDOW:
raise ApiError(400, f"page and page_size must stay within {MAX_RESULT_WINDOW} results")
sort = one(query, "sort") or "relevance"
if sort not in {"relevance", "date"}:
raise ApiError(400, "sort must be relevance or date")
filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}]
fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes", "legal_force": "legal_force_code"}
for parameter, field in fields.items():
value = one(query, parameter)
if value:
if value not in CATALOGS[parameter]:
raise ApiError(400, f"{parameter} must be a catalog code")
filters.append({"term": {field: value}})
date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to")
if date_from and date_to and date_from > date_to:
raise ApiError(400, "date_from must not be later than date_to")
if date_from or date_to:
date_range = {key: value for key, value in (("gte", date_from), ("lte", date_to)) if value}
filters.append({"range": {"date_adopted": date_range}})
body = {
"from": (page - 1) * page_size,
"size": page_size + 1,
"_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "legal_force_code", "date_adopted", "number"],
"query": {"bool": {"filter": filters, "must": build_search_query(language, text)}},
"collapse": {"field": "document_code"},
"highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}},
}
body["sort"] = [{"_script": {
"type": "number",
"order": "asc",
"script": {"lang": "painless", "source": "doc['status_code'].size() == 0 ? 1 : (doc['status_code'].value == 'active' ? 0 : (doc['status_code'].value == 'repealed' ? 2 : 1))"},
}}]
if sort == "date":
body["sort"].append({"date_adopted": "desc"})
body["sort"].append({"_score": "desc"})
response = self.query_opensearch(body)
try:
hits = response["hits"]["hits"]
except (KeyError, TypeError) as error:
raise ApiError(502, "search backend returned an incomplete response") from error
results = [self.search_hit(hit, language) for hit in hits[:page_size]]
snapshot_results = [{"rank": rank, **result} for rank, result in enumerate(results, 1)]
concrete_indexes = {hit.get("_index") for hit in hits[:page_size] if hit.get("_index")}
index_name = next(iter(concrete_indexes)) if len(concrete_indexes) == 1 else self.index
return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": results, "review_token": self.review_snapshots.create(text, language, index_name, snapshot_results, SEARCH_ALGORITHM_VERSION)}
@staticmethod
def search_hit(hit: dict, language: str) -> dict:
source = hit.get("_source")
if not isinstance(source, dict) or not source.get("document_code"):
raise ApiError(502, "search backend returned an incomplete result")
highlight = hit.get("highlight", {}).get(f"text_{language}", [])
legal_force = source.get("legal_force_code")
snippet = highlight[0].replace("<em>", "").replace("</em>", "") if highlight else None
return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "legal_force": labels("legal_force", legal_force)[language] if legal_force else None, "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": snippet}
def filters(self, query: dict[str, list[str]]) -> dict:
language = one(query, "language") or "ru"
if language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes"), "legal_forces": ("legal_force", "legal_force_code")}
body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}}
response = self.query_opensearch(body)
try:
aggregations = response["aggregations"]
values = {
name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]]
for name, pair in fields.items()
}
except (KeyError, TypeError) as error:
raise ApiError(502, "search backend returned incomplete filters") from error
return {"api_version": API_VERSION, "language": language, **values}
def directory(self, code: str) -> Path:
if not CODE.fullmatch(code):
raise ApiError(404, "document not found")
path = self.data_root / "documents" / code
if not path.is_dir():
raise ApiError(404, "document not found")
return path
@staticmethod
def read_json(path: Path, message: str) -> dict:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as error:
raise ApiError(500, message) from error
if not isinstance(value, dict):
raise ApiError(500, message)
return value
def document(self, code: str) -> dict:
document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
editions = document.get("editions")
if not isinstance(editions, list):
raise ApiError(500, "document data is unavailable")
return {"api_version": API_VERSION, "document": document, "current_edition": editions[-1] if editions else None}
def editions(self, code: str) -> dict:
document = self.read_json(self.directory(code) / "document.json", "document data is unavailable")
return {"api_version": API_VERSION, "code": code, "available_languages": document.get("available_languages", []), "editions": document.get("editions", [])}
def edition(self, code: str, edition: str, query: dict[str, list[str]]) -> dict:
if not CODE.fullmatch(edition):
raise ApiError(404, "edition not found")
directory = self.directory(code) / "editions" / edition
if not directory.is_dir():
raise ApiError(404, "edition not found")
metadata = self.read_json(directory / "edition.json", "edition data is unavailable")
language = one(query, "language")
if language is not None and language not in LANGUAGES:
raise ApiError(400, "language must be ru or ky")
languages = [language] if language else metadata.get("available_languages", [])
content = {}
for item in languages:
if item not in metadata.get("available_languages", []):
continue
try:
content[item] = {"html": (directory / item / "content.html").read_text(encoding="utf-8"), "text": (directory / item / "content.txt").read_text(encoding="utf-8")}
except (OSError, UnicodeError) as error:
raise ApiError(500, "edition content is unavailable") from error
if language and language not in content:
raise ApiError(404, "edition language not found")
return {"api_version": API_VERSION, "edition": metadata, "content": content}
def save_review(self, body: dict, username: str) -> dict:
if not isinstance(body, dict):
raise ApiError(400, "request body must be an object")
try:
snapshot = self.review_snapshots.verify(body["review_token"])
reviewer = username
overall_comment = body.get("overall_comment", "").strip()
submitted = body["results"]
except (KeyError, AttributeError, TypeError, ValueError) as error:
raise ApiError(400, "review_token, reviewer and results are required") from error
if not reviewer or len(reviewer) > 120:
raise ApiError(400, "reviewer must be between 1 and 120 characters")
if len(overall_comment) > 4000:
raise ApiError(400, "overall_comment is too long")
if not isinstance(submitted, list):
raise ApiError(400, "results must be an array")
by_rank = {item["rank"]: item for item in snapshot["results"]}
if len(submitted) != len(by_rank) or {item.get("rank") for item in submitted if isinstance(item, dict)} != set(by_rank):
raise ApiError(400, "all search results must be reviewed exactly once")
results = []
for item in submitted:
if not isinstance(item, dict) or not isinstance(item.get("rank"), int) or item["rank"] not in by_rank:
raise ApiError(400, "review result rank is invalid")
source = by_rank[item["rank"]]
if item.get("code") != source["code"]:
raise ApiError(400, "review result document does not match the search snapshot")
rating = item.get("rating")
if rating is not None and (isinstance(rating, bool) or not isinstance(rating, int) or not 0 <= rating <= 3):
raise ApiError(400, "rating must be an integer from 0 to 3")
comment = item.get("comment", "")
if not isinstance(comment, str) or len(comment) > 4000:
raise ApiError(400, "result comment is too long")
results.append({**source, "rating": rating, "comment": comment.strip()})
if not results:
raise ApiError(400, "at least one result must be reviewed")
review = {"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "reviewer": reviewer, "query": snapshot["query"], "language": snapshot["language"], "index_name": snapshot["index_name"], "algorithm_version": snapshot["algorithm_version"], "top_result_code": snapshot["results"][0]["code"] if snapshot["results"] else None, "results": results, "overall_comment": overall_comment}
review_id = self.review_store.save(review)
return {"api_version": API_VERSION, "id": review_id, "created_at": review["created_at"]}
@staticmethod
def review_page() -> str:
try:
return Path(__file__).with_name("review.html").read_text(encoding="utf-8")
except (OSError, UnicodeError) as error:
raise ApiError(500, "review interface is unavailable") from error
def handle(self, method: str, path: str, body: dict | None = None, user: dict | None = None) -> tuple[int, dict]:
parsed = urllib.parse.urlsplit(path)
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part]
if method == "POST" and parts == ["auth", "login"]:
try:
token, account = self.auth_store.login(body.get("username"), body.get("password"))
return 200, {"account": account, "session_token": token}
except AuthError as error:
raise ApiError(401, str(error)) from error
if method == "POST" and parts == ["auth", "logout"]:
return 200, {"ok": True}
if method == "GET" and parts == ["auth", "me"]:
return 200, {"account": user}
if method == "POST" and parts == ["auth", "change-password"]:
try:
self.auth_store.change_password(user["username"], body.get("current_password"), body.get("new_password"))
token, account = self.auth_store.login(user["username"], body.get("new_password"))
return 200, {"account": account, "session_token": token}
except AuthError as error:
raise ApiError(400, str(error)) from error
if user and user["username"].casefold() == "admin" and method == "GET" and parts == ["admin", "users"]:
return 200, {"users": self.auth_store.list_users()}
if user and user["username"].casefold() == "admin" and method == "POST" and parts == ["admin", "users"]:
try:
self.auth_store.create_user(body.get("username"), body.get("password"))
return 201, {"ok": True}
except AuthError as error:
raise ApiError(400, str(error)) from error
if user and user["username"].casefold() == "admin" and method == "DELETE" and len(parts) == 3 and parts[:2] == ["admin", "users"]:
try:
self.auth_store.delete_user(parts[2])
return 200, {"ok": True}
except AuthError as error:
raise ApiError(400, str(error)) from error
if user and user["username"].casefold() == "admin" and method == "POST" and len(parts) == 4 and parts[:2] == ["admin", "users"] and parts[3] == "reset-password":
try:
self.auth_store.reset_password(parts[2], body.get("password"))
return 200, {"ok": True}
except AuthError as error:
raise ApiError(400, str(error)) from error
if method == "POST" and parts == ["search-reviews"]:
return 201, self.save_review(body, user["username"] if user else body.get("reviewer", ""))
if method == "GET" and parts == ["search-reviews", "export"]:
return 200, {"api_version": API_VERSION, "reviews": self.review_store.export()}
if method != "GET":
raise ApiError(405, "method not allowed")
if parts == ["openapi.json"]:
return 200, openapi()
if parts == ["search"]:
return 200, self.search(query)
if parts == ["search", "filters"]:
return 200, self.filters(query)
if len(parts) == 2 and parts[0] == "documents":
return 200, self.document(parts[1])
if len(parts) == 3 and parts[:1] == ["documents"] and parts[2] == "editions":
return 200, self.editions(parts[1])
if len(parts) == 4 and parts[:1] == ["documents"] and parts[2] == "editions":
return 200, self.edition(parts[1], parts[3], query)
raise ApiError(404, "endpoint not found")
def handler(api: Api):
class RequestHandler(BaseHTTPRequestHandler):
def respond(self, method: str):
try:
route = urllib.parse.urlsplit(self.path).path
cookies = http.cookies.SimpleCookie(self.headers.get("Cookie", ""))
session = cookies.get("akyldash_session")
token = session.value if session else None
user = api.auth_store.user_for_token(token)
if method == "GET" and route in {"/", "/review"}:
body = api.review_page().encode()
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("X-Frame-Options", "DENY")
self.send_header("Referrer-Policy", "no-referrer")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if not (method == "POST" and route in {"/auth/login", "/auth/logout"}):
if not user:
raise ApiError(401, "authentication required")
if user["must_change_password"] and not (method == "POST" and route == "/auth/change-password") and not (method == "GET" and route == "/auth/me"):
raise ApiError(403, "change password before continuing")
if route.startswith("/admin/users") and user["username"].casefold() != "admin":
raise ApiError(403, "admin account required")
body = None
if method in {"POST", "DELETE"}:
length = int(self.headers.get("Content-Length", "0"))
if length > 1_000_000:
raise ApiError(413, "request body is too large")
body = json.loads(self.rfile.read(length) or b"{}")
if not isinstance(body, dict):
raise ApiError(400, "request body must be a JSON object")
if method == "POST" and route == "/auth/logout":
api.auth_store.logout(token)
status, payload = api.handle(method, self.path, body, user)
except json.JSONDecodeError:
status, payload = 400, {"api_version": API_VERSION, "error": "request body must be valid JSON"}
except ApiError as error:
status, payload = error.status, {"api_version": API_VERSION, "error": error.message}
except (ValueError, http.cookies.CookieError):
status, payload = 400, {"api_version": API_VERSION, "error": "invalid request"}
session_token = payload.pop("session_token", None) if isinstance(payload, dict) else None
body = json.dumps(payload, ensure_ascii=False).encode()
self.send_response(status)
if session_token:
secure = "; Secure" if api.cookie_secure else ""
self.send_header("Set-Cookie", f"akyldash_session={session_token}; Path=/; HttpOnly; SameSite=Strict; Max-Age=43200{secure}")
elif method == "POST" and urllib.parse.urlsplit(self.path).path == "/auth/logout":
secure = "; Secure" if api.cookie_secure else ""
self.send_header("Set-Cookie", f"akyldash_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0{secure}")
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("X-Frame-Options", "DENY")
self.send_header("Referrer-Policy", "no-referrer")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_GET(self):
self.respond("GET")
def do_POST(self):
self.respond("POST")
def do_DELETE(self):
self.respond("DELETE")
def log_message(self, format: str, *args):
return
return RequestHandler
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--url", default="http://127.0.0.1:9200")
parser.add_argument("--index", default="akyldash-fragments-current")
parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized"))
parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3"))
parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3"))
parser.add_argument("--review-secret", default=None)
parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true")
parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit")
parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--version", action="version", version=APP_VERSION)
arguments = parser.parse_args()
secret = arguments.review_secret.encode() if arguments.review_secret else None
api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure)
if arguments.create_admin or arguments.reset_admin_password:
users = api.auth_store.list_users()
admin_exists = any(user["username"].casefold() == "admin" for user in users)
if arguments.create_admin and users:
parser.error("accounts already exist")
if arguments.reset_admin_password and not admin_exists:
parser.error("admin account does not exist")
if arguments.create_admin and arguments.reset_admin_password:
parser.error("choose either --create-admin or --reset-admin-password")
password = getpass.getpass("Password for admin (12+ characters): ")
confirm = getpass.getpass("Confirm password: ")
if password != confirm:
parser.error("passwords do not match")
try:
if arguments.create_admin:
api.auth_store.create_admin(password)
else:
api.auth_store.reset_password("admin", password)
except AuthError as error:
parser.error(str(error))
print("Initial admin account created." if arguments.create_admin else "Admin password reset. Change it at next sign-in.")
return 0
ThreadingHTTPServer((arguments.host, arguments.port), handler(api)).serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())