"""Minimal HTTP API for the normalized legal-document corpus.""" from __future__ import annotations import argparse import datetime import getpass import http.cookies import json import os import re import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from search.auth import AuthError, AuthStore from search.minjust_opensearch import APP_VERSION, request_json from search.catalog import CATALOGS, labels from search.query import build_search_query from search.reviews import ReviewSnapshots, ReviewStore API_VERSION = "v1" SEARCH_ALGORITHM_VERSION = "search-6" LANGUAGES = {"ru", "ky"} CODE = re.compile(r"^[0-9]+$") MAX_PAGE_SIZE = 100 MAX_RESULT_WINDOW = 10_000 class ApiError(Exception): def __init__(self, status: int, message: str): self.status = status self.message = message def parse_positive(value: str | None, name: str, default: int, maximum: int) -> int: if value is None: return default try: parsed = int(value) except ValueError as error: raise ApiError(400, f"{name} must be an integer") from error if not 1 <= parsed <= maximum: raise ApiError(400, f"{name} must be between 1 and {maximum}") return parsed def one(query: dict[str, list[str]], name: str) -> str | None: values = query.get(name, []) if len(values) > 1: raise ApiError(400, f"{name} must be specified once") return values[0] if values else None def date(value: str | None, name: str) -> str | None: if value is None: return None try: datetime.date.fromisoformat(value) except ValueError as error: raise ApiError(400, f"{name} must be an ISO date") from error return value def openapi() -> dict: responses = {"200": {"description": "Successful response"}, "400": {"description": "Invalid request"}, "401": {"description": "Authentication required"}, "403": {"description": "Additional access required"}, "404": {"description": "Not found"}, "502": {"description": "Search backend unavailable"}} return { "openapi": "3.0.3", "info": {"title": "Akyldash Search API", "version": API_VERSION}, "components": {"securitySchemes": {"SessionCookie": {"type": "apiKey", "in": "cookie", "name": "akyldash_session"}}}, "security": [{"SessionCookie": []}], "paths": { "/auth/login": {"post": {"security": [], "responses": {"200": {"description": "Signed in and session cookie issued"}, "401": {"description": "Invalid login or password"}}}}, "/auth/logout": {"post": {"responses": {"200": {"description": "Signed out"}}}}, "/auth/me": {"get": {"responses": {"200": {"description": "Current account"}, "401": {"description": "Authentication required"}}}}, "/auth/change-password": {"post": {"responses": {"200": {"description": "Password changed"}, "400": {"description": "Invalid password"}}}}, "/admin/users": {"get": {"responses": {"200": {"description": "User list"}}}, "post": {"responses": {"201": {"description": "User created with a temporary password"}}}}, "/admin/users/{username}": {"delete": {"responses": {"200": {"description": "User deleted"}}}}, "/admin/users/{username}/reset-password": {"post": {"responses": {"200": {"description": "Temporary password set"}}}}, "/search": {"get": {"responses": responses, "parameters": [ {"name": "q", "in": "query", "required": True, "schema": {"type": "string"}}, {"name": "language", "in": "query", "schema": {"type": "string", "enum": ["ru", "ky"]}}, {"name": "page", "in": "query", "schema": {"type": "integer", "minimum": 1}}, {"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": MAX_PAGE_SIZE}}, {"name": "document_type", "in": "query", "schema": {"type": "string"}}, {"name": "status", "in": "query", "schema": {"type": "string"}}, {"name": "authority", "in": "query", "schema": {"type": "string"}}, {"name": "legal_force", "in": "query", "schema": {"type": "string"}}, {"name": "date_from", "in": "query", "schema": {"type": "string", "format": "date"}}, {"name": "date_to", "in": "query", "schema": {"type": "string", "format": "date"}}, {"name": "sort", "in": "query", "schema": {"type": "string", "enum": ["relevance", "date"]}}, ]}}, "/search/filters": {"get": {"responses": responses}}, "/search-reviews": {"post": {"responses": {"201": {"description": "Review saved"}, "400": {"description": "Invalid review"}}}}, "/search-reviews/export": {"get": {"responses": responses}}, "/review": {"get": {"responses": {"200": {"description": "Review interface"}}}}, "/documents/{code}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, "/documents/{code}/editions": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, "/documents/{code}/editions/{edition}": {"get": {"responses": responses, "parameters": [{"name": "code", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}, {"name": "edition", "in": "path", "required": True, "schema": {"type": "string", "pattern": "^[0-9]+$"}}]}}, }, } class Api: def __init__(self, base_url: str, index: str, data_root: Path, reviews_db: Path | str = ":memory:", review_secret: bytes | None = None, users_db: Path | str = ":memory:", cookie_secure: bool = False): self.base_url = base_url.rstrip("/") self.index = index self.data_root = data_root self.review_store = ReviewStore(reviews_db) self.review_snapshots = ReviewSnapshots(review_secret) self.auth_store = AuthStore(users_db) self.cookie_secure = cookie_secure def search_url(self, suffix: str) -> str: return f"{self.base_url}/{urllib.parse.quote(self.index, safe='')}/{suffix}" def query_opensearch(self, body: dict) -> dict: try: return request_json(self.search_url("_search"), "POST", json.dumps(body, ensure_ascii=False).encode(), "application/json") except RuntimeError as error: raise ApiError(502, "search backend is unavailable") from error def search(self, query: dict[str, list[str]]) -> dict: text = one(query, "q") if not text or not text.strip(): raise ApiError(400, "q is required") if len(text) > 500: raise ApiError(400, "q must not exceed 500 characters") language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") page = parse_positive(one(query, "page"), "page", 1, 1_000_000) page_size = parse_positive(one(query, "page_size"), "page_size", 20, MAX_PAGE_SIZE) if page * page_size >= MAX_RESULT_WINDOW: raise ApiError(400, f"page and page_size must stay within {MAX_RESULT_WINDOW} results") sort = one(query, "sort") or "relevance" if sort not in {"relevance", "date"}: raise ApiError(400, "sort must be relevance or date") filters: list[dict] = [{"term": {"language": language}}, {"term": {"is_current_edition": True}}] fields = {"document_type": "document_type_code", "status": "status_code", "authority": "authority_codes", "legal_force": "legal_force_code"} for parameter, field in fields.items(): value = one(query, parameter) if value: if value not in CATALOGS[parameter]: raise ApiError(400, f"{parameter} must be a catalog code") filters.append({"term": {field: value}}) date_from, date_to = date(one(query, "date_from"), "date_from"), date(one(query, "date_to"), "date_to") if date_from and date_to and date_from > date_to: raise ApiError(400, "date_from must not be later than date_to") if date_from or date_to: date_range = {key: value for key, value in (("gte", date_from), ("lte", date_to)) if value} filters.append({"range": {"date_adopted": date_range}}) body = { "from": (page - 1) * page_size, "size": page_size + 1, "_source": ["document_code", "edition_code", "document_name_ru", "document_name_ky", "document_type_ru", "document_type_ky", "status_ru", "status_ky", "legal_force_code", "date_adopted", "number"], "query": {"bool": {"filter": filters, "must": build_search_query(language, text)}}, "collapse": {"field": "document_code"}, "highlight": {"fields": {f"text_{language}": {"number_of_fragments": 1}}}, } body["sort"] = [{"_script": { "type": "number", "order": "asc", "script": {"lang": "painless", "source": "doc['status_code'].size() == 0 ? 1 : (doc['status_code'].value == 'active' ? 0 : (doc['status_code'].value == 'repealed' ? 2 : 1))"}, }}] if sort == "date": body["sort"].append({"date_adopted": "desc"}) body["sort"].append({"_score": "desc"}) response = self.query_opensearch(body) try: hits = response["hits"]["hits"] except (KeyError, TypeError) as error: raise ApiError(502, "search backend returned an incomplete response") from error results = [self.search_hit(hit, language) for hit in hits[:page_size]] snapshot_results = [{"rank": rank, **result} for rank, result in enumerate(results, 1)] concrete_indexes = {hit.get("_index") for hit in hits[:page_size] if hit.get("_index")} index_name = next(iter(concrete_indexes)) if len(concrete_indexes) == 1 else self.index return {"api_version": API_VERSION, "query": text, "language": language, "page": page, "page_size": page_size, "has_next": len(hits) > page_size, "results": results, "review_token": self.review_snapshots.create(text, language, index_name, snapshot_results, SEARCH_ALGORITHM_VERSION)} @staticmethod def search_hit(hit: dict, language: str) -> dict: source = hit.get("_source") if not isinstance(source, dict) or not source.get("document_code"): raise ApiError(502, "search backend returned an incomplete result") highlight = hit.get("highlight", {}).get(f"text_{language}", []) legal_force = source.get("legal_force_code") snippet = highlight[0].replace("", "").replace("", "") if highlight else None return {"code": source["document_code"], "edition": source.get("edition_code"), "name": source.get(f"document_name_{language}"), "type": source.get(f"document_type_{language}"), "status": source.get(f"status_{language}"), "legal_force": labels("legal_force", legal_force)[language] if legal_force else None, "date_adopted": source.get("date_adopted"), "number": source.get("number"), "snippet": snippet} def filters(self, query: dict[str, list[str]]) -> dict: language = one(query, "language") or "ru" if language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") fields = {"document_types": ("document_type", "document_type_code"), "statuses": ("status", "status_code"), "authorities": ("authority", "authority_codes"), "legal_forces": ("legal_force", "legal_force_code")} body = {"size": 0, "query": {"term": {"is_current_edition": True}}, "aggs": {name: {"terms": {"field": pair[1], "size": 1000}, "aggs": {"documents": {"cardinality": {"field": "document_code", "precision_threshold": 40000}}}} for name, pair in fields.items()}} response = self.query_opensearch(body) try: aggregations = response["aggregations"] values = { name: [{"code": item["key"], "labels": labels(pair[0], item["key"]), "count": item["documents"]["value"]} for item in aggregations[name]["buckets"]] for name, pair in fields.items() } except (KeyError, TypeError) as error: raise ApiError(502, "search backend returned incomplete filters") from error return {"api_version": API_VERSION, "language": language, **values} def directory(self, code: str) -> Path: if not CODE.fullmatch(code): raise ApiError(404, "document not found") path = self.data_root / "documents" / code if not path.is_dir(): raise ApiError(404, "document not found") return path @staticmethod def read_json(path: Path, message: str) -> dict: try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise ApiError(500, message) from error if not isinstance(value, dict): raise ApiError(500, message) return value def document(self, code: str) -> dict: document = self.read_json(self.directory(code) / "document.json", "document data is unavailable") editions = document.get("editions") if not isinstance(editions, list): raise ApiError(500, "document data is unavailable") return {"api_version": API_VERSION, "document": document, "current_edition": editions[-1] if editions else None} def editions(self, code: str) -> dict: document = self.read_json(self.directory(code) / "document.json", "document data is unavailable") return {"api_version": API_VERSION, "code": code, "available_languages": document.get("available_languages", []), "editions": document.get("editions", [])} def edition(self, code: str, edition: str, query: dict[str, list[str]]) -> dict: if not CODE.fullmatch(edition): raise ApiError(404, "edition not found") directory = self.directory(code) / "editions" / edition if not directory.is_dir(): raise ApiError(404, "edition not found") metadata = self.read_json(directory / "edition.json", "edition data is unavailable") language = one(query, "language") if language is not None and language not in LANGUAGES: raise ApiError(400, "language must be ru or ky") languages = [language] if language else metadata.get("available_languages", []) content = {} for item in languages: if item not in metadata.get("available_languages", []): continue try: content[item] = {"html": (directory / item / "content.html").read_text(encoding="utf-8"), "text": (directory / item / "content.txt").read_text(encoding="utf-8")} except (OSError, UnicodeError) as error: raise ApiError(500, "edition content is unavailable") from error if language and language not in content: raise ApiError(404, "edition language not found") return {"api_version": API_VERSION, "edition": metadata, "content": content} def save_review(self, body: dict, username: str) -> dict: if not isinstance(body, dict): raise ApiError(400, "request body must be an object") try: snapshot = self.review_snapshots.verify(body["review_token"]) reviewer = username overall_comment = body.get("overall_comment", "").strip() submitted = body["results"] except (KeyError, AttributeError, TypeError, ValueError) as error: raise ApiError(400, "review_token, reviewer and results are required") from error if not reviewer or len(reviewer) > 120: raise ApiError(400, "reviewer must be between 1 and 120 characters") if len(overall_comment) > 4000: raise ApiError(400, "overall_comment is too long") if not isinstance(submitted, list): raise ApiError(400, "results must be an array") by_rank = {item["rank"]: item for item in snapshot["results"]} if len(submitted) != len(by_rank) or {item.get("rank") for item in submitted if isinstance(item, dict)} != set(by_rank): raise ApiError(400, "all search results must be reviewed exactly once") results = [] for item in submitted: if not isinstance(item, dict) or not isinstance(item.get("rank"), int) or item["rank"] not in by_rank: raise ApiError(400, "review result rank is invalid") source = by_rank[item["rank"]] if item.get("code") != source["code"]: raise ApiError(400, "review result document does not match the search snapshot") rating = item.get("rating") if rating is not None and (isinstance(rating, bool) or not isinstance(rating, int) or not 0 <= rating <= 3): raise ApiError(400, "rating must be an integer from 0 to 3") comment = item.get("comment", "") if not isinstance(comment, str) or len(comment) > 4000: raise ApiError(400, "result comment is too long") results.append({**source, "rating": rating, "comment": comment.strip()}) if not results: raise ApiError(400, "at least one result must be reviewed") review = {"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "reviewer": reviewer, "query": snapshot["query"], "language": snapshot["language"], "index_name": snapshot["index_name"], "algorithm_version": snapshot["algorithm_version"], "top_result_code": snapshot["results"][0]["code"] if snapshot["results"] else None, "results": results, "overall_comment": overall_comment} review_id = self.review_store.save(review) return {"api_version": API_VERSION, "id": review_id, "created_at": review["created_at"]} @staticmethod def review_page() -> str: try: return Path(__file__).with_name("review.html").read_text(encoding="utf-8") except (OSError, UnicodeError) as error: raise ApiError(500, "review interface is unavailable") from error def handle(self, method: str, path: str, body: dict | None = None, user: dict | None = None) -> tuple[int, dict]: parsed = urllib.parse.urlsplit(path) query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) parts = [urllib.parse.unquote(part) for part in parsed.path.split("/") if part] if method == "POST" and parts == ["auth", "login"]: try: token, account = self.auth_store.login(body.get("username"), body.get("password")) return 200, {"account": account, "session_token": token} except AuthError as error: raise ApiError(401, str(error)) from error if method == "POST" and parts == ["auth", "logout"]: return 200, {"ok": True} if method == "GET" and parts == ["auth", "me"]: return 200, {"account": user} if method == "POST" and parts == ["auth", "change-password"]: try: self.auth_store.change_password(user["username"], body.get("current_password"), body.get("new_password")) token, account = self.auth_store.login(user["username"], body.get("new_password")) return 200, {"account": account, "session_token": token} except AuthError as error: raise ApiError(400, str(error)) from error if user and user["username"].casefold() == "admin" and method == "GET" and parts == ["admin", "users"]: return 200, {"users": self.auth_store.list_users()} if user and user["username"].casefold() == "admin" and method == "POST" and parts == ["admin", "users"]: try: self.auth_store.create_user(body.get("username"), body.get("password")) return 201, {"ok": True} except AuthError as error: raise ApiError(400, str(error)) from error if user and user["username"].casefold() == "admin" and method == "DELETE" and len(parts) == 3 and parts[:2] == ["admin", "users"]: try: self.auth_store.delete_user(parts[2]) return 200, {"ok": True} except AuthError as error: raise ApiError(400, str(error)) from error if user and user["username"].casefold() == "admin" and method == "POST" and len(parts) == 4 and parts[:2] == ["admin", "users"] and parts[3] == "reset-password": try: self.auth_store.reset_password(parts[2], body.get("password")) return 200, {"ok": True} except AuthError as error: raise ApiError(400, str(error)) from error if method == "POST" and parts == ["search-reviews"]: return 201, self.save_review(body, user["username"] if user else body.get("reviewer", "")) if method == "GET" and parts == ["search-reviews", "export"]: return 200, {"api_version": API_VERSION, "reviews": self.review_store.export()} if method != "GET": raise ApiError(405, "method not allowed") if parts == ["openapi.json"]: return 200, openapi() if parts == ["search"]: return 200, self.search(query) if parts == ["search", "filters"]: return 200, self.filters(query) if len(parts) == 2 and parts[0] == "documents": return 200, self.document(parts[1]) if len(parts) == 3 and parts[:1] == ["documents"] and parts[2] == "editions": return 200, self.editions(parts[1]) if len(parts) == 4 and parts[:1] == ["documents"] and parts[2] == "editions": return 200, self.edition(parts[1], parts[3], query) raise ApiError(404, "endpoint not found") def handler(api: Api): class RequestHandler(BaseHTTPRequestHandler): def respond(self, method: str): try: route = urllib.parse.urlsplit(self.path).path cookies = http.cookies.SimpleCookie(self.headers.get("Cookie", "")) session = cookies.get("akyldash_session") token = session.value if session else None user = api.auth_store.user_for_token(token) if method == "GET" and route in {"/", "/review"}: body = api.review_page().encode() self.send_response(200) self.send_header("Content-Type", "text/html; charset=utf-8") self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") self.send_header("X-Frame-Options", "DENY") self.send_header("Referrer-Policy", "no-referrer") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) return if not (method == "POST" and route in {"/auth/login", "/auth/logout"}): if not user: raise ApiError(401, "authentication required") if user["must_change_password"] and not (method == "POST" and route == "/auth/change-password") and not (method == "GET" and route == "/auth/me"): raise ApiError(403, "change password before continuing") if route.startswith("/admin/users") and user["username"].casefold() != "admin": raise ApiError(403, "admin account required") body = None if method in {"POST", "DELETE"}: length = int(self.headers.get("Content-Length", "0")) if length > 1_000_000: raise ApiError(413, "request body is too large") body = json.loads(self.rfile.read(length) or b"{}") if not isinstance(body, dict): raise ApiError(400, "request body must be a JSON object") if method == "POST" and route == "/auth/logout": api.auth_store.logout(token) status, payload = api.handle(method, self.path, body, user) except json.JSONDecodeError: status, payload = 400, {"api_version": API_VERSION, "error": "request body must be valid JSON"} except ApiError as error: status, payload = error.status, {"api_version": API_VERSION, "error": error.message} except (ValueError, http.cookies.CookieError): status, payload = 400, {"api_version": API_VERSION, "error": "invalid request"} session_token = payload.pop("session_token", None) if isinstance(payload, dict) else None body = json.dumps(payload, ensure_ascii=False).encode() self.send_response(status) if session_token: secure = "; Secure" if api.cookie_secure else "" self.send_header("Set-Cookie", f"akyldash_session={session_token}; Path=/; HttpOnly; SameSite=Strict; Max-Age=43200{secure}") elif method == "POST" and urllib.parse.urlsplit(self.path).path == "/auth/logout": secure = "; Secure" if api.cookie_secure else "" self.send_header("Set-Cookie", f"akyldash_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0{secure}") self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") self.send_header("X-Frame-Options", "DENY") self.send_header("Referrer-Policy", "no-referrer") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def do_GET(self): self.respond("GET") def do_POST(self): self.respond("POST") def do_DELETE(self): self.respond("DELETE") def log_message(self, format: str, *args): return return RequestHandler def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--url", default="http://127.0.0.1:9200") parser.add_argument("--index", default="akyldash-fragments-current") parser.add_argument("--data", type=Path, default=Path("data/minjust-normalized")) parser.add_argument("--reviews-db", type=Path, default=Path("data/search-reviews.sqlite3")) parser.add_argument("--users-db", type=Path, default=Path("data/search-users.sqlite3")) parser.add_argument("--review-secret", default=None) parser.add_argument("--cookie-secure", action=argparse.BooleanOptionalAction, default=os.environ.get("COOKIE_SECURE", "false").lower() == "true") parser.add_argument("--create-admin", action="store_true", help="create the initial admin account and exit") parser.add_argument("--reset-admin-password", action="store_true", help="set a temporary password for admin and exit") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8080) parser.add_argument("--version", action="version", version=APP_VERSION) arguments = parser.parse_args() secret = arguments.review_secret.encode() if arguments.review_secret else None api = Api(arguments.url, arguments.index, arguments.data, arguments.reviews_db, secret, arguments.users_db, arguments.cookie_secure) if arguments.create_admin or arguments.reset_admin_password: users = api.auth_store.list_users() admin_exists = any(user["username"].casefold() == "admin" for user in users) if arguments.create_admin and users: parser.error("accounts already exist") if arguments.reset_admin_password and not admin_exists: parser.error("admin account does not exist") if arguments.create_admin and arguments.reset_admin_password: parser.error("choose either --create-admin or --reset-admin-password") password = getpass.getpass("Password for admin (12+ characters): ") confirm = getpass.getpass("Confirm password: ") if password != confirm: parser.error("passwords do not match") try: if arguments.create_admin: api.auth_store.create_admin(password) else: api.auth_store.reset_password("admin", password) except AuthError as error: parser.error(str(error)) print("Initial admin account created." if arguments.create_admin else "Admin password reset. Change it at next sign-in.") return 0 ThreadingHTTPServer((arguments.host, arguments.port), handler(api)).serve_forever() return 0 if __name__ == "__main__": raise SystemExit(main())